secrets: pass key/value config data to plugins as env (#26455)

Co-authored-by: Michael Schurter <mschurter@hashicorp.com>
Co-authored-by: Tim Gross <tgross@hashicorp.com>

api/tasks.go

@@ -1047,16 +1047,21 @@ func (v *Vault) Canonicalize() {
 }
 
 type Secret struct {
-	Name     string         `hcl:"name,label"`
-	Provider string         `hcl:"provider,optional"`
-	Path     string         `hcl:"path,optional"`
-	Config   map[string]any `hcl:"config,block"`
+	Name     string            `hcl:"name,label"`
+	Provider string            `hcl:"provider,optional"`
+	Path     string            `hcl:"path,optional"`
+	Config   map[string]any    `hcl:"config,block"`
+	Env      map[string]string `hcl:"env,block"`
 }
 
 func (s *Secret) Canonicalize() {
 	if len(s.Config) == 0 {
 		s.Config = nil
 	}
+
+	if len(s.Env) == 0 {
+		s.Env = nil
+	}
 }
 
 // NewTask creates and initializes a new Task.

api/tasks_test.go

@@ -514,6 +514,7 @@ func TestTask_Canonicalize_Secret(t *testing.T) {
 		Provider: "test-provider",
 		Path:     "/test/path",
 		Config:   make(map[string]any),
+		Env:      make(map[string]string),
 	}
 
 	expected := &Secret{
@@ -521,6 +522,7 @@ func TestTask_Canonicalize_Secret(t *testing.T) {
 		Provider: "test-provider",
 		Path:     "/test/path",
 		Config:   nil,
+		Env:      nil,
 	}
 
 	testSecret.Canonicalize()

client/allocrunner/taskrunner/secrets_hook.go

@@ -198,7 +198,7 @@ func (h *secretsHook) buildSecretProviders(secretDir string) ([]TemplateProvider
 				tmplProvider = append(tmplProvider, p)
 			}
 		default:
-			plug, err := commonplugins.NewExternalSecretsPlugin(h.clientConfig.CommonPluginDir, s.Provider)
+			plug, err := commonplugins.NewExternalSecretsPlugin(h.clientConfig.CommonPluginDir, s.Provider, s.Env)
 			if err != nil {
 				multierror.Append(mErr, err)
 				continue

client/commonplugins/secrets_plugin.go

@@ -41,9 +41,16 @@ type externalSecretsPlugin struct {
 
 	// pluginPath is the path on the host to the plugin executable
 	pluginPath string
+
+	// env is optional envVars passed to the plugin process
+	env map[string]string
 }
 
-func NewExternalSecretsPlugin(commonPluginDir string, name string) (*externalSecretsPlugin, error) {
+// NewExternalSecretsPlugin creates an instance of a secrets plugin by validating the plugin
+// binary exists and is executable, and parsing any string key/value pairs out of the config
+// which will be used as environment variables for Fetch.
+func NewExternalSecretsPlugin(commonPluginDir string, name string, env map[string]string) (*externalSecretsPlugin, error) {
+	// validate plugin
 	executable := filepath.Join(commonPluginDir, SecretsPluginDir, name)
 	f, err := os.Stat(executable)
 	if err != nil {
@@ -58,6 +65,7 @@ func NewExternalSecretsPlugin(commonPluginDir string, name string) (*externalSec
 
 	return &externalSecretsPlugin{
 		pluginPath: executable,
+		env:        env,
 	}, nil
 }
 
@@ -96,6 +104,10 @@ func (e *externalSecretsPlugin) Fetch(ctx context.Context, path string) (*Secret
 		"CPI_OPERATION=fetch",
 	}
 
+	for env, val := range e.env {
+		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", env, val))
+	}
+
 	stdout, stderr, err := runPlugin(cmd, SecretsKillTimeout)
 	if err != nil {
 		return nil, err

client/commonplugins/secrets_plugin_test.go

@@ -21,7 +21,7 @@ func TestExternalSecretsPlugin_Fingerprint(t *testing.T) {
 	t.Run("runs successfully", func(t *testing.T) {
 		pluginDir, pluginName := setupTestPlugin(t, fmt.Appendf([]byte{}, "#!/bin/sh\ncat <<EOF\n%s\nEOF\n", `{"type": "secrets", "version": "1.0.0"}`))
 
-		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName)
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
 		must.NoError(t, err)
 
 		res, err := plugin.Fingerprint(context.Background())
@@ -34,7 +34,7 @@ func TestExternalSecretsPlugin_Fingerprint(t *testing.T) {
 	t.Run("errors on non-zero exit code", func(t *testing.T) {
 		pluginDir, pluginName := setupTestPlugin(t, fmt.Append([]byte{}, "#!/bin/sh\nexit 1\n"))
 
-		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName)
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
 		must.NoError(t, err)
 
 		res, err := plugin.Fingerprint(context.Background())
@@ -45,7 +45,7 @@ func TestExternalSecretsPlugin_Fingerprint(t *testing.T) {
 	t.Run("errors on timeout", func(t *testing.T) {
 		pluginDir, pluginName := setupTestPlugin(t, fmt.Appendf([]byte{}, "#!/bin/sh\nleep .5\n"))
 
-		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName)
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
 		must.NoError(t, err)
 
 		ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
@@ -58,7 +58,7 @@ func TestExternalSecretsPlugin_Fingerprint(t *testing.T) {
 	t.Run("errors on invalid json", func(t *testing.T) {
 		pluginDir, pluginName := setupTestPlugin(t, fmt.Append([]byte{}, "#!/bin/sh\ncat <<EOF\ninvalid\nEOF\n"))
 
-		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName)
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
 		must.NoError(t, err)
 
 		res, err := plugin.Fingerprint(context.Background())
@@ -73,7 +73,7 @@ func TestExternalSecretsPlugin_Fetch(t *testing.T) {
 	t.Run("runs successfully", func(t *testing.T) {
 		pluginDir, pluginName := setupTestPlugin(t, fmt.Appendf([]byte{}, "#!/bin/sh\ncat <<EOF\n%s\nEOF\n", `{"result": {"key": "value"}}`))
 
-		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName)
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
 		must.NoError(t, err)
 
 		res, err := plugin.Fetch(context.Background(), "test-path")
@@ -86,7 +86,7 @@ func TestExternalSecretsPlugin_Fetch(t *testing.T) {
 	t.Run("errors on non-zero exit code", func(t *testing.T) {
 		pluginDir, pluginName := setupTestPlugin(t, fmt.Append([]byte{}, "#!/bin/sh\nexit 1\n"))
 
-		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName)
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
 		must.NoError(t, err)
 
 		_, err = plugin.Fetch(context.Background(), "test-path")
@@ -96,7 +96,7 @@ func TestExternalSecretsPlugin_Fetch(t *testing.T) {
 	t.Run("errors on timeout", func(t *testing.T) {
 		pluginDir, pluginName := setupTestPlugin(t, fmt.Append([]byte{}, "#!/bin/sh\nsleep .5\n"))
 
-		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName)
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
 		must.NoError(t, err)
 
 		ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
@@ -109,12 +109,24 @@ func TestExternalSecretsPlugin_Fetch(t *testing.T) {
 	t.Run("errors on timeout", func(t *testing.T) {
 		pluginDir, pluginName := setupTestPlugin(t, fmt.Appendf([]byte{}, "#!/bin/sh\ncat <<EOF\n%s\nEOF\n", `invalid`))
 
-		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName)
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
 		must.NoError(t, err)
 
 		_, err = plugin.Fetch(context.Background(), "dummy-path")
 		must.Error(t, err)
 	})
+
+	t.Run("can be passed environment variables via config", func(t *testing.T) {
+		// test the passed envVar is parsed and set correctly by printing it as part of the SecretResponse
+		pluginDir, pluginName := setupTestPlugin(t, fmt.Appendf([]byte{}, "#!/bin/sh\ncat <<EOF\n%s\nEOF\n", `{"result": {"foo": "$TEST_KEY"}}`))
+
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, map[string]string{"TEST_KEY": "TEST_VALUE"})
+		must.NoError(t, err)
+
+		res, err := plugin.Fetch(context.Background(), "dummy-path")
+		must.NoError(t, err)
+		must.Eq(t, res.Result, map[string]string{"foo": "TEST_VALUE"})
+	})
 }
 
 func setupTestPlugin(t *testing.T, b []byte) (string, string) {

client/fingerprint/secrets.go

@@ -53,7 +53,7 @@ func (s *SecretsPluginFingerprint) Fingerprint(request *FingerprintRequest, resp
 	// map of plugin names to fingerprinted versions
 	plugins := map[string]string{}
 	for name := range files {
-		plug, err := commonplugins.NewExternalSecretsPlugin(request.Config.CommonPluginDir, name)
+		plug, err := commonplugins.NewExternalSecretsPlugin(request.Config.CommonPluginDir, name, nil)
 		if err != nil {
 			return err
 		}

command/agent/job_endpoint.go

@@ -1474,6 +1474,7 @@ func ApiTaskToStructsTask(job *structs.Job, group *structs.TaskGroup,
 				Provider: s.Provider,
 				Path:     s.Path,
 				Config:   s.Config,
+				Env:      s.Env,
 			})
 		}
 	}

e2e/secret/input/custom_secret.hcl

@@ -0,0 +1,51 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+variable "secret_value" {
+  type        = string
+  description = "The value of the randomly generated secret for this test"
+}
+
+job "custom_secret" {
+
+  constraint {
+    attribute = "${attr.kernel.name}"
+    value     = "linux"
+  }
+
+  update {
+    min_healthy_time = "1s"
+  }
+
+  group "group" {
+
+    task "task" {
+
+      driver = "docker"
+
+      config {
+        image   = "busybox:1"
+        command = "/bin/sh"
+        args    = ["-c", "sleep 300"]
+      }
+
+      secret "testsecret" {
+        provider = "test_secret_plugin"
+        path     = "some/path"
+        env {
+          // The custom plugin will output this as part of the result field
+          TEST_ENV = "${var.secret_value}"
+        }
+      }
+
+      env {
+        TEST_SECRET = "${secret.testsecret.TEST_ENV}"
+      }
+
+      resources {
+        cpu    = 128
+        memory = 64
+      }
+    }
+  }
+}

e2e/secret/input/nomad_secret.hcl

@@ -1,6 +1,11 @@
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 
+variable "secret_path" {
+  type        = string
+  description = "The path of the vault secret"
+}
+
 job "nomad_secret" {
 
   constraint {
@@ -8,6 +13,10 @@ job "nomad_secret" {
     value     = "linux"
   }
 
+  update {
+    min_healthy_time = "1s"
+  }
+
   group "group" {
 
     task "task" {
@@ -22,7 +31,7 @@ job "nomad_secret" {
 
       secret "testsecret" {
         provider = "nomad"
-        path     = "SECRET_PATH"
+        path     = "${var.secret_path}"
         config {
           namespace = "default"
         }

e2e/secret/input/vault_secret.hcl

@@ -1,6 +1,11 @@
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 
+variable "secret_path" {
+  type        = string
+  description = "The path of the vault secret"
+}
+
 job "vault_secret" {
 
   constraint {
@@ -8,6 +13,10 @@ job "vault_secret" {
     value     = "linux"
   }
 
+  update {
+    min_healthy_time = "1s"
+  }
+
   group "group" {
 
     task "task" {
@@ -24,7 +33,7 @@ job "vault_secret" {
 
       secret "testsecret" {
         provider = "vault"
-        path     = "SECRET_PATH"
+        path     = "${var.secret_path}"
         config {
           engine = "kv_v2"
         }

e2e/secret/secret_test.go

@@ -38,26 +38,16 @@ func TestVaultSecret(t *testing.T) {
 
 	submission, cleanJob := jobs3.Submit(t,
 		"./input/vault_secret.hcl",
-		jobs3.DisableRandomJobID(),
+		jobs3.DisableRandomJobID(), // our path won't match the secret path with a random jobID
 		jobs3.Namespace(ns),
-		jobs3.Detach(),
-		jobs3.ReplaceInJobSpec("SECRET_PATH", secretFullPath),
+		jobs3.Var("secret_path", secretFullPath),
 	)
 	t.Cleanup(cleanJob)
 
-	// Ensure the placed allocation reaches the running state. If the test fails
-	// here, it's likely due to permissions or pathing of the secret errors.
-	must.NoError(
-		t,
-		e2e.WaitForAllocStatusExpected(submission.JobID(), ns, []string{"running"}),
-		must.Sprint("expected running allocation"),
-	)
-
 	// Validate the nomad variable was read and parsed into the expected
 	// environment variable
-	out, err := e2e.Command("nomad", "exec", submission.AllocID("group"), "env")
-	must.NoError(t, err)
-	must.StrContains(t, out, fmt.Sprintf("TEST_SECRET=%s", secretValue))
+	out := submission.Exec("group", "task", []string{"env"})
+	must.StrContains(t, out.Stdout, fmt.Sprintf("TEST_SECRET=%s", secretValue))
 }
 
 func TestNomadSecret(t *testing.T) {
@@ -97,22 +87,31 @@ func TestNomadSecret(t *testing.T) {
 		"./input/nomad_secret.hcl",
 		jobs3.DisableRandomJobID(),
 		jobs3.Namespace(ns),
-		jobs3.Detach(),
-		jobs3.ReplaceInJobSpec("SECRET_PATH", secretFullPath),
+		jobs3.Var("secret_path", secretFullPath),
 	)
 	t.Cleanup(cleanJob)
 
-	// Ensure the placed allocation reaches the running state. If the test fails
-	// here, it's likely due to permissions or pathing of the secret errors.
-	must.NoError(
-		t,
-		e2e.WaitForAllocStatusExpected(submission.JobID(), ns, []string{"running"}),
-		must.Sprint("expected running allocation"),
+	// Validate the nomad variable was read and parsed into the expected
+	// environment variable
+	out := submission.Exec("group", "task", []string{"env"})
+	must.StrContains(t, out.Stdout, fmt.Sprintf("TEST_SECRET=%s", secretValue))
+}
+
+func TestPluginSecret(t *testing.T) {
+	// Generate a uuid value for the secret plugins env block which it will output
+	// as a part of the result field.
+	secretValue := uuid.Generate()
+
+	submission, cleanJob := jobs3.Submit(t,
+		"./input/custom_secret.hcl",
+		jobs3.DisableRandomJobID(),
+		jobs3.Namespace(ns),
+		jobs3.Var("secret_value", secretValue),
 	)
+	t.Cleanup(cleanJob)
 
 	// Validate the nomad variable was read and parsed into the expected
 	// environment variable
-	out, err := e2e.Command("nomad", "exec", submission.AllocID("group"), "env")
-	must.NoError(t, err)
-	must.StrContains(t, out, fmt.Sprintf("TEST_SECRET=%s", secretValue))
+	out := submission.Exec("group", "task", []string{"env"})
+	must.StrContains(t, out.Stdout, fmt.Sprintf("TEST_SECRET=%s", secretValue))
 }

e2e/terraform/packer/ubuntu-jammy-amd64/common-plugins/test_secret_plugin.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+fingerprint() {
+  echo {\"type\": \"secrets\", \"version\": \"0.0.1\"}
+}
+
+fetch() {
+  # return any passed environment variables as output
+  echo '{"result":{'$(printenv | awk -F= '{printf "\"%s\":\"%s\",", $1, $2}' | sed 's/,$//')'}}'
+}
+
+case "$1" in
+  fingerprint)
+    fingerprint
+    ;;
+  fetch)
+    fetch
+    ;;
+  *)
+    exit 1
+esac
+

e2e/terraform/packer/ubuntu-jammy-amd64/setup.sh

@@ -113,6 +113,10 @@ echo "Installing additional CNI network configs"
 # copy of nomad's "bridge" for connect+cni test (e2e/connect/)
 sudo mv /tmp/linux/cni/nomad_bridge_copy.conflist /opt/cni/config/
 
+echo "Installing CPI test plugins"
+mkdir_for_root /opt/nomad/data/common_plugins/secrets
+sudo mv /tmp/linux/common-plugins/test_secret_plugin.sh /opt/nomad/data/common_plugins/secrets/test_secret_plugin
+
 # Podman
 echo "Installing Podman"
 sudo apt-get -y install podman catatonit

nomad/structs/diff_test.go

@@ -9477,6 +9477,9 @@ func TestTaskDiff(t *testing.T) {
 						Config: map[string]any{
 							"foo": "bar",
 						},
+						Env: map[string]string{
+							"foo": "bar",
+						},
 					},
 				},
 			},
@@ -9489,6 +9492,9 @@ func TestTaskDiff(t *testing.T) {
 						Config: map[string]any{
 							"foo": "bar1",
 						},
+						Env: map[string]string{
+							"foo": "bar",
+						},
 					},
 				},
 			},

nomad/structs/structs.go

@@ -10443,6 +10443,7 @@ type Secret struct {
 	Provider string
 	Path     string
 	Config   map[string]any
+	Env      map[string]string
 }
 
 func (s *Secret) Equal(o *Secret) bool {
@@ -10459,6 +10460,8 @@ func (s *Secret) Equal(o *Secret) bool {
 		return false
 	case !maps.Equal(s.Config, o.Config):
 		return false
+	case !maps.Equal(s.Env, o.Env):
+		return false
 	}
 
 	return true
@@ -10481,6 +10484,7 @@ func (s *Secret) Copy() *Secret {
 		Provider: s.Provider,
 		Path:     s.Path,
 		Config:   confCopy.(map[string]any),
+		Env:      maps.Clone(s.Env),
 	}
 }
 
@@ -10507,6 +10511,16 @@ func (s *Secret) Validate() error {
 		_ = multierror.Append(&mErr, errors.New("secret path cannot be empty"))
 	}
 
+	if s.Provider == "nomad" || s.Provider == "vault" {
+		if len(s.Env) > 0 {
+			_ = multierror.Append(&mErr, fmt.Errorf("%s provider cannot use the env block", s.Provider))
+		}
+	} else {
+		if len(s.Config) > 0 {
+			_ = multierror.Append(&mErr, fmt.Errorf("custom plugin provider %s cannot use the config block", s.Provider))
+		}
+	}
+
 	return mErr.ErrorOrNil()
 }
 

nomad/structs/structs_test.go

@@ -4,6 +4,7 @@
 package structs
 
 import (
+	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -6556,16 +6557,7 @@ func TestSecrets_Validate(t *testing.T) {
 				Path:     "test-path",
 				Provider: "test-provider",
 			},
-			expectErr: fmt.Errorf("secret name cannot be empty"),
-		},
-		{
-			name: "invalid name",
-			secret: &Secret{
-				Name:     "bad-name@",
-				Path:     "test-path",
-				Provider: "test-provider",
-			},
-			expectErr: fmt.Errorf("secret name must match regex %s", validSecretName),
+			expectErr: errors.New("secret name cannot be empty"),
 		},
 		{
 			name: "missing provider",
@@ -6573,7 +6565,7 @@ func TestSecrets_Validate(t *testing.T) {
 				Name: "testsecret",
 				Path: "test-path",
 			},
-			expectErr: fmt.Errorf("secret provider cannot be empty"),
+			expectErr: errors.New("secret provider cannot be empty"),
 		},
 		{
 			name: "missing path",
@@ -6581,7 +6573,43 @@ func TestSecrets_Validate(t *testing.T) {
 				Name:     "testsecret",
 				Provider: "test-provier",
 			},
-			expectErr: fmt.Errorf("secret path cannot be empty"),
+			expectErr: errors.New("secret path cannot be empty"),
+		},
+		{
+			name: "nomad provider fails with env",
+			secret: &Secret{
+				Name:     "test-secret",
+				Provider: "nomad",
+				Path:     "test",
+				Env: map[string]string{
+					"test": "test",
+				},
+			},
+			expectErr: errors.New("nomad provider cannot use the env block"),
+		},
+		{
+			name: "vault provider fails with env",
+			secret: &Secret{
+				Name:     "test-secret",
+				Provider: "vault",
+				Path:     "test",
+				Env: map[string]string{
+					"test": "test",
+				},
+			},
+			expectErr: errors.New("vault provider cannot use the env block"),
+		},
+		{
+			name: "custom provider fails with config",
+			secret: &Secret{
+				Name:     "test-secret",
+				Provider: "test",
+				Path:     "test",
+				Config: map[string]any{
+					"test": "test",
+				},
+			},
+			expectErr: errors.New("custom plugin provider test cannot use the config block"),
 		},
 	}