Merge pull request #26681 from hashicorp/NMD-760-nomad-secrets-block

Secrets Block: merge feature branch to main

client/allocrunner/taskrunner/artifact_hook_test.go

@@ -88,7 +88,7 @@ func TestTaskRunner_ArtifactHook_PartialDone(t *testing.T) {
 	_, destdir := getter.SetupDir(t)
 
 	req := &interfaces.TaskPrestartRequest{
-		TaskEnv: taskenv.NewTaskEnv(nil, nil, nil, nil, destdir, ""),
+		TaskEnv: taskenv.NewTaskEnv(nil, nil, nil, nil, nil, destdir, ""),
 		TaskDir: &allocdir.TaskDir{Dir: destdir},
 		Task: &structs.Task{
 			Artifacts: []*structs.TaskArtifact{
@@ -180,7 +180,7 @@ func TestTaskRunner_ArtifactHook_ConcurrentDownloadSuccess(t *testing.T) {
 	_, destdir := getter.SetupDir(t)
 
 	req := &interfaces.TaskPrestartRequest{
-		TaskEnv: taskenv.NewTaskEnv(nil, nil, nil, nil, destdir, ""),
+		TaskEnv: taskenv.NewTaskEnv(nil, nil, nil, nil, nil, destdir, ""),
 		TaskDir: &allocdir.TaskDir{Dir: destdir},
 		Task: &structs.Task{
 			Artifacts: []*structs.TaskArtifact{
@@ -271,7 +271,7 @@ func TestTaskRunner_ArtifactHook_ConcurrentDownloadFailure(t *testing.T) {
 	_, destdir := getter.SetupDir(t)
 
 	req := &interfaces.TaskPrestartRequest{
-		TaskEnv: taskenv.NewTaskEnv(nil, nil, nil, nil, destdir, ""),
+		TaskEnv: taskenv.NewTaskEnv(nil, nil, nil, nil, nil, destdir, ""),
 		TaskDir: &allocdir.TaskDir{Dir: destdir},
 		Task: &structs.Task{
 			Artifacts: []*structs.TaskArtifact{

client/allocrunner/taskrunner/envoy_version_hook_test.go

@@ -26,7 +26,7 @@ var (
 	taskEnvDefault = taskenv.NewTaskEnv(nil, nil, nil, map[string]string{
 		"meta.connect.sidecar_image": envoy.ImageFormat,
 		"meta.connect.gateway_image": envoy.ImageFormat,
-	}, "", "")
+	}, nil, "", "")
 )
 
 func TestEnvoyVersionHook_semver(t *testing.T) {
@@ -147,7 +147,7 @@ func TestEnvoyVersionHook_interpolateImage(t *testing.T) {
 			"MY_ENVOY": "my/envoy",
 		}, map[string]string{
 			"MY_ENVOY": "my/envoy",
-		}, nil, nil, "", ""))
+		}, nil, nil, nil, "", ""))
 		must.Eq(t, "my/envoy", task.Config["image"])
 	})
 

client/allocrunner/taskrunner/secrets/nomad_provider.go

@@ -0,0 +1,84 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package secrets
+
+import (
+	"errors"
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/mitchellh/mapstructure"
+)
+
+const SecretProviderNomad = "nomad"
+
+type nomadProviderConfig struct {
+	Namespace string `mapstructure:"namespace"`
+}
+
+func defaultNomadConfig(namespace string) *nomadProviderConfig {
+	return &nomadProviderConfig{
+		Namespace: namespace,
+	}
+}
+
+type NomadProvider struct {
+	secret    *structs.Secret
+	secretDir string
+	tmplFile  string
+	config    *nomadProviderConfig
+}
+
+// NewNomadProvider takes a task secret and decodes the config, overwriting the default config fields
+// with any provided fields, returning an error if the secret or secret's config is invalid.
+func NewNomadProvider(secret *structs.Secret, secretDir string, tmplFile string, namespace string) (*NomadProvider, error) {
+	conf := defaultNomadConfig(namespace)
+	if err := mapstructure.Decode(secret.Config, conf); err != nil {
+		return nil, err
+	}
+
+	if err := validateNomadInputs(conf, secret.Path); err != nil {
+		return nil, err
+	}
+
+	return &NomadProvider{
+		config:    conf,
+		secret:    secret,
+		secretDir: secretDir,
+		tmplFile:  tmplFile,
+	}, nil
+}
+
+func (n *NomadProvider) BuildTemplate() *structs.Template {
+	data := fmt.Sprintf(`
+		{{ with nomadVar "%s@%s" }}
+		{{ range $k, $v := . }}
+		secret.%s.{{ $k }}={{ $v }}
+		{{ end }}
+		{{ end }}`,
+		n.secret.Path, n.config.Namespace, n.secret.Name)
+
+	return &structs.Template{
+		EmbeddedTmpl: data,
+		DestPath:     filepath.Clean(filepath.Join(n.secretDir, n.tmplFile)),
+		ChangeMode:   structs.TemplateChangeModeNoop,
+		Once:         true,
+	}
+}
+
+// validateNomadInputs ensures none of the user provided inputs contain delimiters
+// that could be used to inject other CT functions.
+func validateNomadInputs(conf *nomadProviderConfig, path string) error {
+	if strings.ContainsAny(conf.Namespace, "(){}") {
+		return errors.New("namespace cannot contain template delimiters or parenthesis")
+	}
+
+	if strings.ContainsAny(path, "(){}") {
+		return errors.New("path cannot contain template delimiters or parenthesis")
+	}
+
+	return nil
+}

client/allocrunner/taskrunner/secrets/nomad_provider_test.go

@@ -0,0 +1,82 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package secrets
+
+import (
+	"testing"
+
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/shoenig/test/must"
+)
+
+func TestNomadProvider_BuildTemplate(t *testing.T) {
+	t.Run("variable template succeeds", func(t *testing.T) {
+		testDir := t.TempDir()
+		testSecret := &structs.Secret{
+			Name:     "foo",
+			Provider: "nomad",
+			Path:     "/test/path",
+			Config: map[string]any{
+				"namespace": "dev",
+			},
+		}
+		p, err := NewNomadProvider(testSecret, testDir, "test", "default")
+		must.NoError(t, err)
+
+		tmpl := p.BuildTemplate()
+		must.NotNil(t, tmpl)
+
+		// expected template should have correct path and name
+		expectedTmpl := `
+		{{ with nomadVar "/test/path@dev" }}
+		{{ range $k, $v := . }}
+		secret.foo.{{ $k }}={{ $v }}
+		{{ end }}
+		{{ end }}`
+		// validate template string contains expected data
+		must.Eq(t, tmpl.EmbeddedTmpl, expectedTmpl)
+	})
+
+	t.Run("invalid config options errors", func(t *testing.T) {
+		testDir := t.TempDir()
+		testSecret := &structs.Secret{
+			Name:     "foo",
+			Provider: "nomad",
+			Path:     "/test/path",
+			Config: map[string]any{
+				"namespace": 123,
+			},
+		}
+		_, err := NewNomadProvider(testSecret, testDir, "test", "default")
+		must.Error(t, err)
+	})
+
+	t.Run("path with delimiter errors", func(t *testing.T) {
+		testDir := t.TempDir()
+		testSecret := &structs.Secret{
+			Name:     "foo",
+			Provider: "nomad",
+			Path:     "/test/path}}",
+			Config: map[string]any{
+				"namespace": 123,
+			},
+		}
+		_, err := NewNomadProvider(testSecret, testDir, "test", "default")
+		must.Error(t, err)
+	})
+
+	t.Run("namespace with parenthesis errors", func(t *testing.T) {
+		testDir := t.TempDir()
+		testSecret := &structs.Secret{
+			Name:     "foo",
+			Provider: "nomad",
+			Path:     "/test/path",
+			Config: map[string]any{
+				"namespace": "( some namespace )",
+			},
+		}
+		_, err := NewNomadProvider(testSecret, testDir, "test", "default")
+		must.Error(t, err)
+	})
+}

client/allocrunner/taskrunner/secrets/plugin_provider.go

@@ -0,0 +1,52 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package secrets
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/hashicorp/nomad/client/commonplugins"
+)
+
+type ExternalPluginProvider struct {
+	// plugin is the commonplugin to be executed by this secret
+	plugin commonplugins.SecretsPlugin
+
+	// name of the plugin and also the executable
+	name string
+
+	// path is the secret location used in Fetch
+	path string
+}
+
+type Response struct {
+	Result map[string]string `json:"result"`
+	Error  *string           `json:"error,omitempty"`
+}
+
+func NewExternalPluginProvider(plugin commonplugins.SecretsPlugin, name string, path string) *ExternalPluginProvider {
+	return &ExternalPluginProvider{
+		plugin: plugin,
+		name:   name,
+		path:   path,
+	}
+}
+
+func (p *ExternalPluginProvider) Fetch(ctx context.Context) (map[string]string, error) {
+	resp, err := p.plugin.Fetch(ctx, p.path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch secret from plugin %s: %w", p.name, err)
+	}
+	if resp.Error != nil {
+		return nil, fmt.Errorf("error returned from secret plugin %s: %s", p.name, *resp.Error)
+	}
+
+	formatted := make(map[string]string, len(resp.Result))
+	for k, v := range resp.Result {
+		formatted[fmt.Sprintf("secret.%s.%s", p.name, k)] = v
+	}
+
+	return formatted, nil
+}

client/allocrunner/taskrunner/secrets/plugin_provider_test.go

@@ -0,0 +1,86 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package secrets
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/hashicorp/nomad/client/commonplugins"
+	"github.com/shoenig/test/must"
+	"github.com/stretchr/testify/mock"
+)
+
+type MockSecretPlugin struct {
+	mock.Mock
+}
+
+func (m *MockSecretPlugin) Fingerprint(ctx context.Context) (*commonplugins.PluginFingerprint, error) {
+	return nil, nil
+}
+
+func (m *MockSecretPlugin) Fetch(ctx context.Context, path string) (*commonplugins.SecretResponse, error) {
+	args := m.Called()
+
+	if args.Get(0) == nil {
+		return nil, args.Error(1)
+	}
+
+	return args.Get(0).(*commonplugins.SecretResponse), args.Error(1)
+}
+
+func (m *MockSecretPlugin) Parse() (map[string]string, error) {
+	return nil, nil
+}
+
+// SecretsPlugin is tested in commonplugins package. We can use a mock here to test how
+// the ExternalPluginProvider handles various error scenarios when calling Fetch.
+func TestExternalPluginProvider_Fetch(t *testing.T) {
+	t.Run("errors if fetch errors", func(t *testing.T) {
+		mockSecretPlugin := new(MockSecretPlugin)
+		mockSecretPlugin.On("Fetch", mock.Anything).Return(nil, errors.New("something bad"))
+
+		testProvider := NewExternalPluginProvider(mockSecretPlugin, "test", "test")
+
+		vars, err := testProvider.Fetch(t.Context())
+		must.ErrorContains(t, err, "something bad")
+		must.Nil(t, vars)
+	})
+
+	t.Run("errors if fetch response contains error", func(t *testing.T) {
+		mockSecretPlugin := new(MockSecretPlugin)
+		testError := "something bad"
+		mockSecretPlugin.On("Fetch", mock.Anything).Return(&commonplugins.SecretResponse{
+			Result: nil,
+			Error:  &testError,
+		}, nil)
+
+		testProvider := NewExternalPluginProvider(mockSecretPlugin, "test", "test")
+
+		vars, err := testProvider.Fetch(t.Context())
+		must.ErrorContains(t, err, "error returned from secret plugin")
+		must.Nil(t, vars)
+	})
+
+	t.Run("formats response correctly", func(t *testing.T) {
+		mockSecretPlugin := new(MockSecretPlugin)
+		mockSecretPlugin.On("Fetch", mock.Anything).Return(&commonplugins.SecretResponse{
+			Result: map[string]string{
+				"testkey": "testvalue",
+			},
+			Error: nil,
+		}, nil)
+
+		testProvider := NewExternalPluginProvider(mockSecretPlugin, "test", "test")
+
+		result, err := testProvider.Fetch(t.Context())
+		must.NoError(t, err)
+
+		exp := map[string]string{
+			"secret.test.testkey": "testvalue",
+		}
+		must.Eq(t, exp, result)
+	})
+}

client/allocrunner/taskrunner/secrets/vault_provider.go

@@ -0,0 +1,80 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package secrets
+
+import (
+	"errors"
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/mitchellh/mapstructure"
+)
+
+const (
+	SecretProviderVault = "vault"
+
+	VAULT_KV    = "kv"
+	VAULT_KV_V2 = "kv_v2"
+)
+
+type vaultProviderConfig struct {
+	Engine string `mapstructure:"engine"`
+}
+
+func defaultVaultConfig() *vaultProviderConfig {
+	return &vaultProviderConfig{
+		Engine: VAULT_KV,
+	}
+}
+
+type VaultProvider struct {
+	secret    *structs.Secret
+	secretDir string
+	tmplFile  string
+	conf      *vaultProviderConfig
+}
+
+// NewVaultProvider takes a task secret and decodes the config, overwriting the default config fields
+// with any provided fields, returning an error if the secret or secret's config is invalid.
+func NewVaultProvider(secret *structs.Secret, secretDir string, tmplFile string) (*VaultProvider, error) {
+	conf := defaultVaultConfig()
+	if err := mapstructure.Decode(secret.Config, conf); err != nil {
+		return nil, err
+	}
+
+	if strings.ContainsAny(secret.Path, "(){}") {
+		return nil, errors.New("secret path cannot contain template delimiters or parenthesis")
+	}
+
+	return &VaultProvider{
+		secret:    secret,
+		secretDir: secretDir,
+		tmplFile:  tmplFile,
+		conf:      conf,
+	}, nil
+}
+
+func (v *VaultProvider) BuildTemplate() *structs.Template {
+	indexKey := ".Data"
+	if v.conf.Engine == VAULT_KV_V2 {
+		indexKey = ".Data.data"
+	}
+
+	data := fmt.Sprintf(`
+		{{ with secret "%s" }}
+		{{ range $k, $v := %s }}
+		secret.%s.{{ $k }}={{ $v }}
+		{{ end }}
+		{{ end }}`,
+		v.secret.Path, indexKey, v.secret.Name)
+
+	return &structs.Template{
+		EmbeddedTmpl: data,
+		DestPath:     filepath.Clean(filepath.Join(v.secretDir, v.tmplFile)),
+		ChangeMode:   structs.TemplateChangeModeNoop,
+		Once:         true,
+	}
+}

client/allocrunner/taskrunner/secrets/vault_provider_test.go

@@ -0,0 +1,89 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package secrets
+
+import (
+	"testing"
+
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/shoenig/test/must"
+)
+
+func TestVaultProvider_BuildTemplate(t *testing.T) {
+	t.Run("kv template succeeds", func(t *testing.T) {
+		testDir := t.TempDir()
+		testSecret := &structs.Secret{
+			Name:     "foo",
+			Provider: "vault",
+			Path:     "/test/path",
+		}
+		p, err := NewVaultProvider(testSecret, testDir, "test")
+		must.NoError(t, err)
+
+		tmpl := p.BuildTemplate()
+		must.NotNil(t, tmpl)
+
+		// expected template should have correct path, index, and name
+		expectedTmpl := `
+		{{ with secret "/test/path" }}
+		{{ range $k, $v := .Data }}
+		secret.foo.{{ $k }}={{ $v }}
+		{{ end }}
+		{{ end }}`
+		// validate template string contains expected data
+		must.Eq(t, tmpl.EmbeddedTmpl, expectedTmpl)
+	})
+
+	t.Run("kv_v2 template succeeds", func(t *testing.T) {
+		testDir := t.TempDir()
+		testSecret := &structs.Secret{
+			Name:     "foo",
+			Provider: "vault",
+			Path:     "/test/path",
+			Config: map[string]any{
+				"engine": VAULT_KV_V2,
+			},
+		}
+		p, err := NewVaultProvider(testSecret, testDir, "test")
+		must.NoError(t, err)
+
+		tmpl := p.BuildTemplate()
+		must.NotNil(t, tmpl)
+
+		// expected template should have correct path, index, and name
+		expectedTmpl := `
+		{{ with secret "/test/path" }}
+		{{ range $k, $v := .Data.data }}
+		secret.foo.{{ $k }}={{ $v }}
+		{{ end }}
+		{{ end }}`
+		// validate template string contains expected data
+		must.Eq(t, tmpl.EmbeddedTmpl, expectedTmpl)
+	})
+
+	t.Run("invalid config options errors", func(t *testing.T) {
+		testDir := t.TempDir()
+		testSecret := &structs.Secret{
+			Name:     "foo",
+			Provider: "vault",
+			Path:     "/test/path",
+			Config: map[string]any{
+				"engine": 123,
+			},
+		}
+		_, err := NewVaultProvider(testSecret, testDir, "test")
+		must.Error(t, err)
+	})
+
+	t.Run("path containing delimeter errors", func(t *testing.T) {
+		testDir := t.TempDir()
+		testSecret := &structs.Secret{
+			Name:     "foo",
+			Provider: "vault",
+			Path:     "/test/path}}",
+		}
+		_, err := NewVaultProvider(testSecret, testDir, "test")
+		must.Error(t, err)
+	})
+}

client/allocrunner/taskrunner/secrets_hook.go

@@ -0,0 +1,211 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package taskrunner
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/hashicorp/consul-template/renderer"
+	"github.com/hashicorp/go-envparse"
+	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
+	ti "github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces"
+	"github.com/hashicorp/nomad/client/allocrunner/taskrunner/secrets"
+	"github.com/hashicorp/nomad/client/allocrunner/taskrunner/template"
+	"github.com/hashicorp/nomad/client/commonplugins"
+	"github.com/hashicorp/nomad/client/config"
+	"github.com/hashicorp/nomad/client/taskenv"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+type TemplateProvider interface {
+	BuildTemplate() *structs.Template
+}
+
+type PluginProvider interface {
+	Fetch(context.Context) (map[string]string, error)
+}
+
+type secretsHookConfig struct {
+	// logger is used to log
+	logger log.Logger
+
+	// lifecycle is used to interact with the task's lifecycle
+	lifecycle ti.TaskLifecycle
+
+	// events is used to emit events
+	events ti.EventEmitter
+
+	// clientConfig is the Nomad Client configuration
+	clientConfig *config.Config
+
+	// envBuilder is the environment variable builder for the task.
+	envBuilder *taskenv.Builder
+
+	// nomadNamespace is the job's Nomad namespace
+	nomadNamespace string
+}
+
+type secretsHook struct {
+	// logger is used to log
+	logger log.Logger
+
+	// lifecycle is used to interact with the task's lifecycle
+	lifecycle ti.TaskLifecycle
+
+	// events is used to emit events
+	events ti.EventEmitter
+
+	// clientConfig is the Nomad Client configuration
+	clientConfig *config.Config
+
+	// envBuilder is the environment variable builder for the task
+	envBuilder *taskenv.Builder
+
+	// nomadNamespace is the job's Nomad namespace
+	nomadNamespace string
+
+	// secrets to be fetched and populated for interpolation
+	secrets []*structs.Secret
+}
+
+func newSecretsHook(conf *secretsHookConfig, secrets []*structs.Secret) *secretsHook {
+	return &secretsHook{
+		logger:         conf.logger,
+		lifecycle:      conf.lifecycle,
+		events:         conf.events,
+		clientConfig:   conf.clientConfig,
+		envBuilder:     conf.envBuilder,
+		nomadNamespace: conf.nomadNamespace,
+		secrets:        secrets,
+	}
+}
+
+func (h *secretsHook) Name() string {
+	return "secrets"
+}
+
+func (h *secretsHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error {
+	tmplProvider, pluginProvider, err := h.buildSecretProviders(req.TaskDir.SecretsDir)
+	if err != nil {
+		return err
+	}
+
+	templates := []*structs.Template{}
+	for _, p := range tmplProvider {
+		templates = append(templates, p.BuildTemplate())
+	}
+
+	vaultCluster := req.Task.GetVaultClusterName()
+	vaultConfig := h.clientConfig.GetVaultConfigs(h.logger)[vaultCluster]
+
+	mu := &sync.Mutex{}
+	contents := []byte{}
+	unblock := make(chan struct{})
+	tm, err := template.NewTaskTemplateManager(&template.TaskTemplateManagerConfig{
+		UnblockCh:            unblock,
+		Lifecycle:            h.lifecycle,
+		Events:               h.events,
+		Templates:            templates,
+		ClientConfig:         h.clientConfig,
+		VaultToken:           req.VaultToken,
+		VaultConfig:          vaultConfig,
+		VaultNamespace:       req.Alloc.Job.VaultNamespace,
+		TaskDir:              req.TaskDir.Dir,
+		EnvBuilder:           h.envBuilder,
+		MaxTemplateEventRate: template.DefaultMaxTemplateEventRate,
+		NomadNamespace:       h.nomadNamespace,
+		NomadToken:           req.NomadToken,
+		TaskID:               req.Alloc.ID + "-" + req.Task.Name,
+		Logger:               h.logger,
+
+		// This RenderFunc is used to keep any secret data from being written to disk.
+		RenderFunc: func(ri *renderer.RenderInput) (*renderer.RenderResult, error) {
+			// This RenderFunc is called by a single goroutine synchronously, but we
+			// lock the append in the event this behavior changes without us knowing.
+			mu.Lock()
+			defer mu.Unlock()
+			contents = append(contents, ri.Contents...)
+			return &renderer.RenderResult{
+				DidRender:   true,
+				WouldRender: true,
+				Contents:    ri.Contents,
+			}, nil
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	go tm.Run()
+
+	// Safeguard against the template manager continuing to run.
+	defer tm.Stop()
+
+	select {
+	case <-ctx.Done():
+		return nil
+	case <-unblock:
+	}
+
+	// Set secrets from templates
+	m, err := envparse.Parse(bytes.NewBuffer(contents))
+	if err != nil {
+		return err
+	}
+	h.envBuilder.SetSecrets(m)
+
+	// Set secrets from plugin providers
+	for _, p := range pluginProvider {
+		vars, err := p.Fetch(ctx)
+		if err != nil {
+			return err
+		}
+		h.envBuilder.SetSecrets(vars)
+	}
+
+	resp.Done = true
+	return nil
+}
+
+func (h *secretsHook) buildSecretProviders(secretDir string) ([]TemplateProvider, []PluginProvider, error) {
+	// Any configuration errors will be found when calling the secret providers constructor,
+	// so use a multierror to collect all errors and return them to the user at the same time.
+	tmplProvider, pluginProvider, mErr := []TemplateProvider{}, []PluginProvider{}, new(multierror.Error)
+
+	for idx, s := range h.secrets {
+		if s == nil {
+			continue
+		}
+
+		tmplFile := fmt.Sprintf("temp-%d", idx)
+		switch s.Provider {
+		case secrets.SecretProviderNomad:
+			if p, err := secrets.NewNomadProvider(s, secretDir, tmplFile, h.nomadNamespace); err != nil {
+				multierror.Append(mErr, err)
+			} else {
+				tmplProvider = append(tmplProvider, p)
+			}
+		case secrets.SecretProviderVault:
+			if p, err := secrets.NewVaultProvider(s, secretDir, tmplFile); err != nil {
+				multierror.Append(mErr, err)
+			} else {
+				tmplProvider = append(tmplProvider, p)
+			}
+		default:
+			plug, err := commonplugins.NewExternalSecretsPlugin(h.clientConfig.CommonPluginDir, s.Provider, s.Env)
+			if err != nil {
+				multierror.Append(mErr, err)
+				continue
+			}
+			pluginProvider = append(pluginProvider, secrets.NewExternalPluginProvider(plug, s.Name, s.Path))
+		}
+	}
+
+	return tmplProvider, pluginProvider, mErr.ErrorOrNil()
+}

client/allocrunner/taskrunner/secrets_hook_test.go

@@ -0,0 +1,319 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package taskrunner
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/client/allocdir"
+	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
+	trtesting "github.com/hashicorp/nomad/client/allocrunner/taskrunner/testing"
+	"github.com/hashicorp/nomad/client/config"
+	"github.com/hashicorp/nomad/client/taskenv"
+	"github.com/hashicorp/nomad/helper/bufconndialer"
+	"github.com/hashicorp/nomad/helper/pointer"
+	"github.com/hashicorp/nomad/helper/testlog"
+	"github.com/hashicorp/nomad/nomad/mock"
+	"github.com/hashicorp/nomad/nomad/structs"
+	structsc "github.com/hashicorp/nomad/nomad/structs/config"
+	"github.com/shoenig/test/must"
+)
+
+func TestSecretsHook_Prestart_Nomad(t *testing.T) {
+	ci.Parallel(t)
+
+	t.Run("nomad provider successfully renders valid secrets", func(t *testing.T) {
+		secretsResp := `
+		{
+		  "CreateIndex": 812,
+		  "CreateTime": 1750782609539170600,
+		  "Items": {
+		    "key2": "value2",
+		    "key1": "value1"
+		  },
+		  "ModifyIndex": 812,
+		  "ModifyTime": 1750782609539170600,
+		  "Namespace": "default",
+		  "Path": "testnomadvar"
+		}
+		`
+		count := 0 // CT expects a nomad index header that increments, or else it continues polling
+		nomadServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Add("X-Nomad-Index", strconv.Itoa(count))
+			fmt.Fprintln(w, secretsResp)
+			count += 1
+		}))
+		t.Cleanup(nomadServer.Close)
+
+		l, d := bufconndialer.New()
+		nomadServer.Listener = l
+
+		nomadServer.Start()
+
+		clientConfig := config.DefaultConfig()
+		clientConfig.TemplateDialer = d
+		clientConfig.TemplateConfig.DisableSandbox = true
+
+		taskDir := t.TempDir()
+		alloc := mock.MinAlloc()
+		task := alloc.Job.TaskGroups[0].Tasks[0]
+
+		taskEnv := taskenv.NewBuilder(mock.Node(), alloc, task, clientConfig.Region)
+		conf := &secretsHookConfig{
+			logger:       testlog.HCLogger(t),
+			lifecycle:    trtesting.NewMockTaskHooks(),
+			events:       &trtesting.MockEmitter{},
+			clientConfig: clientConfig,
+			envBuilder:   taskEnv,
+		}
+		secretHook := newSecretsHook(conf, []*structs.Secret{
+			{
+				Name:     "test_secret",
+				Provider: "nomad",
+				Path:     "testnomadvar",
+				Config: map[string]any{
+					"namespace": "default",
+				},
+			},
+			{
+				Name:     "test_secret1",
+				Provider: "nomad",
+				Path:     "testnomadvar1",
+				Config: map[string]any{
+					"namespace": "default",
+				},
+			},
+		})
+
+		req := &interfaces.TaskPrestartRequest{
+			Alloc:   alloc,
+			Task:    task,
+			TaskDir: &allocdir.TaskDir{Dir: taskDir, SecretsDir: taskDir},
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+		t.Cleanup(cancel)
+
+		err := secretHook.Prestart(ctx, req, &interfaces.TaskPrestartResponse{})
+		must.NoError(t, err)
+
+		expected := map[string]string{
+			"secret.test_secret.key1":  "value1",
+			"secret.test_secret.key2":  "value2",
+			"secret.test_secret1.key1": "value1",
+			"secret.test_secret1.key2": "value2",
+		}
+		must.Eq(t, expected, taskEnv.Build().TaskSecrets)
+	})
+
+	t.Run("returns early if context is cancelled", func(t *testing.T) {
+
+		secretsResp := `
+		{
+		  "CreateIndex": 812,
+		  "CreateTime": 1750782609539170600,
+		  "Items": {
+		    "key2": "value2",
+		    "key1": "value1"
+		  },
+		  "ModifyIndex": 812,
+		  "ModifyTime": 1750782609539170600,
+		  "Namespace": "default",
+		  "Path": "testnomadvar"
+		}
+		`
+		count := 0 // CT expects a nomad index header that increments, or else it continues polling
+		nomadServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Add("X-Nomad-Index", strconv.Itoa(count))
+			fmt.Fprintln(w, secretsResp)
+			count += 1
+		}))
+		t.Cleanup(nomadServer.Close)
+
+		l, d := bufconndialer.New()
+		nomadServer.Listener = l
+
+		nomadServer.Start()
+
+		clientConfig := config.DefaultConfig()
+		clientConfig.TemplateDialer = d
+		clientConfig.TemplateConfig.DisableSandbox = true
+
+		taskDir := t.TempDir()
+		alloc := mock.MinAlloc()
+		task := alloc.Job.TaskGroups[0].Tasks[0]
+
+		taskEnv := taskenv.NewBuilder(mock.Node(), alloc, task, clientConfig.Region)
+		conf := &secretsHookConfig{
+			logger:       testlog.HCLogger(t),
+			lifecycle:    trtesting.NewMockTaskHooks(),
+			events:       &trtesting.MockEmitter{},
+			clientConfig: clientConfig,
+			envBuilder:   taskEnv,
+		}
+		secretHook := newSecretsHook(conf, []*structs.Secret{
+			{
+				Name:     "test_secret",
+				Provider: "nomad",
+				Path:     "testnomadvar",
+				Config: map[string]any{
+					"namespace": "default",
+				},
+			},
+		})
+
+		req := &interfaces.TaskPrestartRequest{
+			Alloc:   alloc,
+			Task:    task,
+			TaskDir: &allocdir.TaskDir{Dir: taskDir, SecretsDir: taskDir},
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+		cancel() // cancel context to simulate task being stopped
+
+		err := secretHook.Prestart(ctx, req, &interfaces.TaskPrestartResponse{})
+		must.NoError(t, err)
+
+		expected := map[string]string{}
+		must.Eq(t, expected, taskEnv.Build().TaskSecrets)
+	})
+
+	t.Run("errors when failure building secret providers", func(t *testing.T) {
+		clientConfig := config.DefaultConfig()
+
+		taskDir := t.TempDir()
+		alloc := mock.MinAlloc()
+		task := alloc.Job.TaskGroups[0].Tasks[0]
+
+		taskEnv := taskenv.NewBuilder(mock.Node(), alloc, task, clientConfig.Region)
+		conf := &secretsHookConfig{
+			logger:       testlog.HCLogger(t),
+			lifecycle:    trtesting.NewMockTaskHooks(),
+			events:       &trtesting.MockEmitter{},
+			clientConfig: clientConfig,
+			envBuilder:   taskEnv,
+		}
+
+		// give an invalid secret, in this case a nomad secret with bad namespace
+		secretHook := newSecretsHook(conf, []*structs.Secret{
+			{
+				Name:     "test_secret",
+				Provider: "nomad",
+				Path:     "testnomadvar",
+				Config: map[string]any{
+					"namespace": 123,
+				},
+			},
+		})
+
+		req := &interfaces.TaskPrestartRequest{
+			Alloc:   alloc,
+			Task:    task,
+			TaskDir: &allocdir.TaskDir{Dir: taskDir, SecretsDir: taskDir},
+		}
+
+		// Prestart should error and return after building secrets
+		err := secretHook.Prestart(context.Background(), req, nil)
+		must.Error(t, err)
+
+		expected := map[string]string{}
+		must.Eq(t, expected, taskEnv.Build().TaskSecrets)
+	})
+}
+
+func TestSecretsHook_Prestart_Vault(t *testing.T) {
+	ci.Parallel(t)
+
+	secretsResp := `
+{
+  "Data": {
+    "data": {
+      "secret": "secret"
+    },
+    "metadata": {
+      "created_time": "2023-10-18T15:58:29.65137Z",
+      "custom_metadata": null,
+      "deletion_time": "",
+      "destroyed": false,
+      "version": 1
+    }
+  }
+}`
+
+	// Start test server to simulate Vault cluster responses.
+	// reqCh := make(chan any)
+	defaultVaultServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, secretsResp)
+	}))
+	t.Cleanup(defaultVaultServer.Close)
+
+	// Setup client with Vault config.
+	clientConfig := config.DefaultConfig()
+	clientConfig.TemplateConfig.DisableSandbox = true
+	clientConfig.VaultConfigs = map[string]*structsc.VaultConfig{
+		structs.VaultDefaultCluster: {
+			Name:    structs.VaultDefaultCluster,
+			Enabled: pointer.Of(true),
+			Addr:    defaultVaultServer.URL,
+		},
+	}
+
+	taskDir := t.TempDir()
+	alloc := mock.MinAlloc()
+	task := alloc.Job.TaskGroups[0].Tasks[0]
+
+	taskEnv := taskenv.NewBuilder(mock.Node(), alloc, task, clientConfig.Region)
+	conf := &secretsHookConfig{
+		logger:       testlog.HCLogger(t),
+		lifecycle:    trtesting.NewMockTaskHooks(),
+		events:       &trtesting.MockEmitter{},
+		clientConfig: clientConfig,
+		envBuilder:   taskEnv,
+	}
+	secretHook := newSecretsHook(conf, []*structs.Secret{
+		{
+			Name:     "test_secret",
+			Provider: "vault",
+			Path:     "/test/path",
+			Config: map[string]any{
+				"engine": "kv_v2",
+			},
+		},
+		{
+			Name:     "test_secret1",
+			Provider: "vault",
+			Path:     "/test/path1",
+			Config: map[string]any{
+				"engine": "kv_v2",
+			},
+		},
+	})
+
+	// Start template hook with a timeout context to ensure it exists.
+	req := &interfaces.TaskPrestartRequest{
+		Alloc:   alloc,
+		Task:    task,
+		TaskDir: &allocdir.TaskDir{Dir: taskDir, SecretsDir: taskDir},
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	t.Cleanup(cancel)
+
+	err := secretHook.Prestart(ctx, req, &interfaces.TaskPrestartResponse{})
+	must.NoError(t, err)
+
+	exp := map[string]string{
+		"secret.test_secret.secret":  "secret",
+		"secret.test_secret1.secret": "secret",
+	}
+
+	must.Eq(t, exp, taskEnv.Build().TaskSecrets)
+}

client/allocrunner/taskrunner/task_runner_hooks.go

@@ -55,40 +55,16 @@ func (tr *TaskRunner) initHooks() {
 
 	tr.logmonHookConfig = newLogMonHookConfig(task.Name, task.LogConfig, tr.taskDir.LogDir)
 
-	// Add the hook resources
-	tr.hookResources = &hookResources{}
-
-	// Create the task directory hook. This is run first to ensure the
+	// Create the task directory hook first. This is run first to ensure the
 	// directory path exists for other hooks.
-	alloc := tr.Alloc()
+	tr.hookResources = &hookResources{}
 	tr.runnerHooks = []interfaces.TaskHook{
 		newValidateHook(tr.clientConfig, hookLogger),
 		newDynamicUsersHook(tr.killCtx, tr.driverCapabilities.DynamicWorkloadUsers, tr.logger, tr.users),
 		newTaskDirHook(tr, hookLogger),
 		newIdentityHook(tr, hookLogger),
-		newLogMonHook(tr, hookLogger),
-		newDispatchHook(alloc, hookLogger),
-		newVolumeHook(tr, hookLogger),
-		newArtifactHook(tr, tr.getter, hookLogger),
-		newStatsHook(tr, tr.clientConfig.StatsCollectionInterval, tr.clientConfig.PublishAllocationMetrics, hookLogger),
-		newDeviceHook(tr.devicemanager, hookLogger),
-		newAPIHook(tr.shutdownCtx, tr.clientConfig.APIListenerRegistrar, hookLogger),
-		newWranglerHook(tr.wranglers, task.Name, alloc.ID, task.UsesCores(), hookLogger),
+		newConsulHook(hookLogger, tr),
 	}
-
-	// If the task has a CSI block, add the hook.
-	if task.CSIPluginConfig != nil {
-		tr.runnerHooks = append(tr.runnerHooks, newCSIPluginSupervisorHook(
-			&csiPluginSupervisorHookConfig{
-				clientStateDirPath: tr.clientConfig.StateDir,
-				events:             tr,
-				runner:             tr,
-				lifecycle:          tr,
-				capabilities:       tr.driverCapabilities,
-				logger:             hookLogger,
-			}))
-	}
-
 	// If Vault is enabled, add the hook
 	if task.Vault != nil && tr.vaultClientFunc != nil {
 		tr.runnerHooks = append(tr.runnerHooks, newVaultHook(&vaultHookConfig{
@@ -105,13 +81,45 @@ func (tr *TaskRunner) initHooks() {
 		}))
 	}
 
+	if len(task.Secrets) > 0 {
+		tr.runnerHooks = append(tr.runnerHooks, newSecretsHook(&secretsHookConfig{
+			logger:         tr.logger,
+			lifecycle:      tr,
+			events:         tr,
+			clientConfig:   tr.clientConfig,
+			envBuilder:     tr.envBuilder,
+			nomadNamespace: tr.alloc.Job.Namespace,
+		}, task.Secrets))
+	}
+
+	alloc := tr.Alloc()
+	tr.runnerHooks = append(tr.runnerHooks, []interfaces.TaskHook{
+		newLogMonHook(tr, hookLogger),
+		newDispatchHook(alloc, hookLogger),
+		newVolumeHook(tr, hookLogger),
+		newArtifactHook(tr, tr.getter, hookLogger),
+		newStatsHook(tr, tr.clientConfig.StatsCollectionInterval, tr.clientConfig.PublishAllocationMetrics, hookLogger),
+		newDeviceHook(tr.devicemanager, hookLogger),
+		newAPIHook(tr.shutdownCtx, tr.clientConfig.APIListenerRegistrar, hookLogger),
+		newWranglerHook(tr.wranglers, task.Name, alloc.ID, task.UsesCores(), hookLogger),
+	}...)
+
+	// If the task has a CSI block, add the hook.
+	if task.CSIPluginConfig != nil {
+		tr.runnerHooks = append(tr.runnerHooks, newCSIPluginSupervisorHook(
+			&csiPluginSupervisorHookConfig{
+				clientStateDirPath: tr.clientConfig.StateDir,
+				events:             tr,
+				runner:             tr,
+				lifecycle:          tr,
+				capabilities:       tr.driverCapabilities,
+				logger:             hookLogger,
+			}))
+	}
+
 	// Get the consul namespace for the TG of the allocation.
 	consulNamespace := tr.alloc.ConsulNamespaceForTask(tr.taskName)
 
-	// Add the consul hook (populates task secret dirs and sets the environment if
-	// consul tokens are present for the task).
-	tr.runnerHooks = append(tr.runnerHooks, newConsulHook(hookLogger, tr))
-
 	// If there are templates is enabled, add the hook
 	if len(task.Templates) != 0 {
 		tr.runnerHooks = append(tr.runnerHooks, newTemplateHook(&templateHookConfig{

client/allocrunner/taskrunner/template/template.go

@@ -18,6 +18,7 @@ import (
 
 	ctconf "github.com/hashicorp/consul-template/config"
 	"github.com/hashicorp/consul-template/manager"
+	"github.com/hashicorp/consul-template/renderer"
 	"github.com/hashicorp/consul-template/signals"
 	envparse "github.com/hashicorp/go-envparse"
 	"github.com/hashicorp/go-hclog"
@@ -132,6 +133,11 @@ type TaskTemplateManagerConfig struct {
 	TaskID string
 
 	Logger hclog.Logger
+
+	// RenderFunc allows custom rendering of templated data, and overrides the
+	// Nomad custom RenderFunc used for sandboxing. This is currently used by
+	// the secrets block to hold all templated data in memory.
+	RenderFunc renderer.Renderer
 }
 
 // Validate validates the configuration.
@@ -980,7 +986,11 @@ func newRunnerConfig(config *TaskTemplateManagerConfig,
 	sandboxEnabled := isSandboxEnabled(config)
 	sandboxDir := filepath.Dir(config.TaskDir) // alloc working directory
 	conf.ReaderFunc = ReaderFn(config.TaskID, sandboxDir, sandboxEnabled)
-	conf.RendererFunc = RenderFn(config.TaskID, sandboxDir, sandboxEnabled)
+	if config.RenderFunc != nil {
+		conf.RendererFunc = config.RenderFunc
+	} else {
+		conf.RendererFunc = RenderFn(config.TaskID, sandboxDir, sandboxEnabled)
+	}
 	conf.Finalize()
 	return conf, nil
 }

client/allocrunner/taskrunner/template/template_test.go

@@ -1699,6 +1699,7 @@ func TestTaskTemplateManager_Env_InterpolatedDest(t *testing.T) {
 		map[string]string{"NOMAD_META_path": "exists"},
 		map[string]string{},
 		map[string]string{},
+		map[string]string{},
 		d, "")
 
 	vars, err := loadTemplateEnv(templates, taskEnv)

client/commonplugins/commonplugins.go

@@ -0,0 +1,63 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package commonplugins
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os/exec"
+	"syscall"
+	"time"
+
+	"github.com/hashicorp/go-version"
+)
+
+var (
+	ErrPluginNotExists     error = errors.New("plugin not found")
+	ErrPluginNotExecutable error = errors.New("plugin not executable")
+)
+
+type CommonPlugin interface {
+	Fingerprint(ctx context.Context) (*PluginFingerprint, error)
+}
+
+// CommonPlugins are expected to respond to 'fingerprint' calls with json that
+// unmarshals to this struct.
+type PluginFingerprint struct {
+	Version *version.Version `json:"version"`
+	Type    *string          `json:"type"`
+}
+
+// runPlugin is a helper for executing the provided Cmd and capturing stdout/stderr.
+// This helper implements both the soft and hard timeouts defined by the common
+// plugins interface.
+func runPlugin(cmd *exec.Cmd, killTimeout time.Duration) (stdout, stderr []byte, err error) {
+	var errBuf bytes.Buffer
+	cmd.Stderr = &errBuf
+
+	done := make(chan error, 1)
+	cmd.Cancel = func() error {
+		var cancelErr error
+
+		_ = cmd.Process.Signal(syscall.SIGTERM)
+		killTimer := time.NewTimer(killTimeout)
+		defer killTimer.Stop()
+
+		select {
+		case <-killTimer.C:
+			cancelErr = cmd.Process.Kill()
+		case <-done:
+		}
+
+		return cancelErr
+	}
+
+	// start the command
+	stdout, err = cmd.Output()
+	done <- err
+
+	stderr = errBuf.Bytes()
+	return
+}

client/commonplugins/secrets_plugin.go

@@ -0,0 +1,126 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package commonplugins
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+
+	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/nomad/helper"
+)
+
+const (
+	SecretsPluginDir = "secrets"
+
+	// The timeout for the plugin command before it is send SIGTERM
+	SecretsCmdTimeout = 10 * time.Second
+
+	// The timeout before the command is sent SIGKILL after being SIGTERM'd
+	SecretsKillTimeout = 2 * time.Second
+)
+
+type SecretsPlugin interface {
+	CommonPlugin
+	Fetch(ctx context.Context, path string) (*SecretResponse, error)
+}
+
+type SecretResponse struct {
+	Result map[string]string `json:"result"`
+	Error  *string           `json:"error"`
+}
+
+type externalSecretsPlugin struct {
+	logger log.Logger
+
+	// pluginPath is the path on the host to the plugin executable
+	pluginPath string
+
+	// env is optional envVars passed to the plugin process
+	env map[string]string
+}
+
+// NewExternalSecretsPlugin creates an instance of a secrets plugin by validating the plugin
+// binary exists and is executable, and parsing any string key/value pairs out of the config
+// which will be used as environment variables for Fetch.
+func NewExternalSecretsPlugin(commonPluginDir string, name string, env map[string]string) (*externalSecretsPlugin, error) {
+	// validate plugin
+	executable := filepath.Join(commonPluginDir, SecretsPluginDir, name)
+	f, err := os.Stat(executable)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, fmt.Errorf("%w: %q", ErrPluginNotExists, name)
+		}
+		return nil, err
+	}
+	if !helper.IsExecutable(f) {
+		return nil, fmt.Errorf("%w: %q", ErrPluginNotExecutable, name)
+	}
+
+	return &externalSecretsPlugin{
+		pluginPath: executable,
+		env:        env,
+	}, nil
+}
+
+func (e *externalSecretsPlugin) Fingerprint(ctx context.Context) (*PluginFingerprint, error) {
+	plugCtx, cancel := context.WithTimeout(ctx, SecretsCmdTimeout)
+	defer cancel()
+
+	cmd := exec.CommandContext(plugCtx, e.pluginPath, "fingerprint")
+	cmd.Env = []string{
+		"CPI_OPERATION=fingerprint",
+	}
+
+	stdout, stderr, err := runPlugin(cmd, SecretsKillTimeout)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(stderr) > 0 {
+		e.logger.Info("fingerprint command stderr output", "msg", string(stderr))
+	}
+
+	res := &PluginFingerprint{}
+	if err := json.Unmarshal(stdout, &res); err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}
+
+func (e *externalSecretsPlugin) Fetch(ctx context.Context, path string) (*SecretResponse, error) {
+	plugCtx, cancel := context.WithTimeout(ctx, SecretsCmdTimeout)
+	defer cancel()
+
+	cmd := exec.CommandContext(plugCtx, e.pluginPath, "fetch", path)
+	cmd.Env = []string{
+		"CPI_OPERATION=fetch",
+	}
+
+	for env, val := range e.env {
+		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", env, val))
+	}
+
+	stdout, stderr, err := runPlugin(cmd, SecretsKillTimeout)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(stderr) > 0 {
+		e.logger.Info("fetch command stderr output", "msg", string(stderr))
+	}
+
+	res := &SecretResponse{}
+	if err := json.Unmarshal(stdout, &res); err != nil {
+		return nil, err
+	}
+
+	return res, nil
+}

client/commonplugins/secrets_plugin_test.go

@@ -0,0 +1,144 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package commonplugins
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/nomad/ci"
+	"github.com/shoenig/test/must"
+)
+
+func TestExternalSecretsPlugin_Fingerprint(t *testing.T) {
+	ci.Parallel(t)
+
+	t.Run("runs successfully", func(t *testing.T) {
+		pluginDir, pluginName := setupTestPlugin(t, fmt.Appendf([]byte{}, "#!/bin/sh\ncat <<EOF\n%s\nEOF\n", `{"type": "secrets", "version": "1.0.0"}`))
+
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
+		must.NoError(t, err)
+
+		res, err := plugin.Fingerprint(context.Background())
+		must.NoError(t, err)
+
+		must.Eq(t, *res.Type, "secrets")
+		must.Eq(t, res.Version.String(), "1.0.0")
+	})
+
+	t.Run("errors on non-zero exit code", func(t *testing.T) {
+		pluginDir, pluginName := setupTestPlugin(t, fmt.Append([]byte{}, "#!/bin/sh\nexit 1\n"))
+
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
+		must.NoError(t, err)
+
+		res, err := plugin.Fingerprint(context.Background())
+		must.Error(t, err)
+		must.Nil(t, res)
+	})
+
+	t.Run("errors on timeout", func(t *testing.T) {
+		pluginDir, pluginName := setupTestPlugin(t, fmt.Appendf([]byte{}, "#!/bin/sh\nleep .5\n"))
+
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
+		must.NoError(t, err)
+
+		ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+		defer cancel()
+
+		_, err = plugin.Fingerprint(ctx)
+		must.Error(t, err)
+	})
+
+	t.Run("errors on invalid json", func(t *testing.T) {
+		pluginDir, pluginName := setupTestPlugin(t, fmt.Append([]byte{}, "#!/bin/sh\ncat <<EOF\ninvalid\nEOF\n"))
+
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
+		must.NoError(t, err)
+
+		res, err := plugin.Fingerprint(context.Background())
+		must.Error(t, err)
+		must.Nil(t, res)
+	})
+}
+
+func TestExternalSecretsPlugin_Fetch(t *testing.T) {
+	ci.Parallel(t)
+
+	t.Run("runs successfully", func(t *testing.T) {
+		pluginDir, pluginName := setupTestPlugin(t, fmt.Appendf([]byte{}, "#!/bin/sh\ncat <<EOF\n%s\nEOF\n", `{"result": {"key": "value"}}`))
+
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
+		must.NoError(t, err)
+
+		res, err := plugin.Fetch(context.Background(), "test-path")
+		must.NoError(t, err)
+
+		exp := map[string]string{"key": "value"}
+		must.Eq(t, res.Result, exp)
+	})
+
+	t.Run("errors on non-zero exit code", func(t *testing.T) {
+		pluginDir, pluginName := setupTestPlugin(t, fmt.Append([]byte{}, "#!/bin/sh\nexit 1\n"))
+
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
+		must.NoError(t, err)
+
+		_, err = plugin.Fetch(context.Background(), "test-path")
+		must.Error(t, err)
+	})
+
+	t.Run("errors on timeout", func(t *testing.T) {
+		pluginDir, pluginName := setupTestPlugin(t, fmt.Append([]byte{}, "#!/bin/sh\nsleep .5\n"))
+
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
+		must.NoError(t, err)
+
+		ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+		defer cancel()
+
+		_, err = plugin.Fetch(ctx, "dummy-path")
+		must.Error(t, err)
+	})
+
+	t.Run("errors on timeout", func(t *testing.T) {
+		pluginDir, pluginName := setupTestPlugin(t, fmt.Appendf([]byte{}, "#!/bin/sh\ncat <<EOF\n%s\nEOF\n", `invalid`))
+
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, nil)
+		must.NoError(t, err)
+
+		_, err = plugin.Fetch(context.Background(), "dummy-path")
+		must.Error(t, err)
+	})
+
+	t.Run("can be passed environment variables via config", func(t *testing.T) {
+		// test the passed envVar is parsed and set correctly by printing it as part of the SecretResponse
+		pluginDir, pluginName := setupTestPlugin(t, fmt.Appendf([]byte{}, "#!/bin/sh\ncat <<EOF\n%s\nEOF\n", `{"result": {"foo": "$TEST_KEY"}}`))
+
+		plugin, err := NewExternalSecretsPlugin(pluginDir, pluginName, map[string]string{"TEST_KEY": "TEST_VALUE"})
+		must.NoError(t, err)
+
+		res, err := plugin.Fetch(context.Background(), "dummy-path")
+		must.NoError(t, err)
+		must.Eq(t, res.Result, map[string]string{"foo": "TEST_VALUE"})
+	})
+}
+
+func setupTestPlugin(t *testing.T, b []byte) (string, string) {
+	dir := t.TempDir()
+	plugin := "test-plugin"
+
+	// NewExternalSecretsPlugin expects the subdir "secrets" to be present
+	secretDir := filepath.Join(dir, SecretsPluginDir)
+	must.NoError(t, os.Mkdir(secretDir, 0755))
+
+	path := filepath.Join(secretDir, plugin)
+	must.NoError(t, os.WriteFile(path, b, 0755))
+
+	return dir, plugin
+}

client/config/config.go

@@ -340,6 +340,10 @@ type Config struct {
 	// HostNetworks is a map of the conigured host networks by name.
 	HostNetworks map[string]*structs.ClientHostNetworkConfig
 
+	// CommonPluginDir is the root directory for plugins that implement
+	// the common plugin interface
+	CommonPluginDir string
+
 	// BindWildcardDefaultHostNetwork toggles if the default host network should accept all
 	// destinations (true) or only filter on the IP of the default host network (false) when
 	// port mapping. This allows Nomad clients with no defined host networks to accept and

client/fingerprint/fingerprint.go

@@ -43,6 +43,7 @@ var (
 		"nomad":               NewNomadFingerprint,
 		"plugins_cni":         NewPluginsCNIFingerprint,
 		"host_volume_plugins": NewPluginsHostVolumeFingerprint,
+		"secrets_plugins":     NewPluginsSecretsFingerprint,
 		"signal":              NewSignalFingerprint,
 		"storage":             NewStorageFingerprint,
 		"vault":               NewVaultFingerprint,

client/fingerprint/secrets.go

@@ -0,0 +1,100 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package fingerprint
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/nomad/client/commonplugins"
+	"github.com/hashicorp/nomad/helper"
+)
+
+type SecretsPluginFingerprint struct {
+	logger hclog.Logger
+}
+
+func NewPluginsSecretsFingerprint(logger hclog.Logger) Fingerprint {
+	return &SecretsPluginFingerprint{
+		logger: logger.Named("secrets_plugins"),
+	}
+}
+
+func (s *SecretsPluginFingerprint) Fingerprint(request *FingerprintRequest, response *FingerprintResponse) error {
+	// Add builtin secrets providers
+	defer func() {
+		response.AddAttribute("plugins.secrets.nomad.version", "1.0.0")
+		response.AddAttribute("plugins.secrets.vault.version", "1.0.0")
+	}()
+	response.Detected = true
+
+	pluginDir := request.Config.CommonPluginDir
+	if pluginDir == "" {
+		return nil
+	}
+
+	secretsDir := filepath.Join(pluginDir, commonplugins.SecretsPluginDir)
+
+	files, err := helper.FindExecutableFiles(secretsDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			s.logger.Trace("secrets plugin dir does not exist", "dir", secretsDir)
+		} else {
+			s.logger.Warn("error finding secrets plugins", "dir", secretsDir, "error", err)
+		}
+		return nil // don't halt agent start
+	}
+
+	// map of plugin names to fingerprinted versions
+	plugins := map[string]string{}
+	for name := range files {
+		plug, err := commonplugins.NewExternalSecretsPlugin(request.Config.CommonPluginDir, name, nil)
+		if err != nil {
+			return err
+		}
+
+		fprint, err := plug.Fingerprint(context.Background())
+		if err != nil {
+			s.logger.Error("secrets plugin failed fingerprint", "plugin", name, "error", err)
+			continue
+		}
+
+		if fprint.Version == nil || fprint.Type == nil {
+			continue
+		}
+
+		if *fprint.Type != "secrets" {
+			continue
+		}
+
+		plugins[name] = fprint.Version.Original()
+	}
+
+	// if this was a reload, wipe what was there before
+	for k := range request.Node.Attributes {
+		if strings.HasPrefix(k, "plugins.secrets.") {
+			response.RemoveAttribute(k)
+		}
+	}
+
+	// set the attribute(s)
+	for plugin, version := range plugins {
+		s.logger.Debug("detected plugin", "plugin_id", plugin, "version", version)
+		response.AddAttribute("plugins.secrets."+plugin+".version", version)
+	}
+
+	return nil
+}
+
+func (s *SecretsPluginFingerprint) Periodic() (bool, time.Duration) {
+	return false, 0
+}
+
+func (s *SecretsPluginFingerprint) Reload() {
+	// secrets plugins are re-detected on agent reload
+}

client/fingerprint/secrets_test.go

@@ -0,0 +1,89 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package fingerprint
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/hashicorp/nomad/client/config"
+	"github.com/hashicorp/nomad/helper/testlog"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/shoenig/test/must"
+)
+
+func TestPluginsSecretsFingerprint(t *testing.T) {
+	fp := NewPluginsSecretsFingerprint(testlog.HCLogger(t))
+
+	node := &structs.Node{Attributes: map[string]string{}}
+	cfg := &config.Config{CommonPluginDir: ""}
+	req := &FingerprintRequest{Config: cfg, Node: node}
+
+	for name, path := range map[string]string{
+		"empty":        "",
+		"non-existent": "/nowhere",
+		"impossible":   "secrets_plugins_test.go",
+	} {
+		t.Run(name, func(t *testing.T) {
+			resp := FingerprintResponse{}
+			cfg.CommonPluginDir = path
+			err := fp.Fingerprint(req, &resp)
+			must.NoError(t, err)
+			must.True(t, resp.Detected) // always true due to "mkdir" built-in
+		})
+	}
+
+	if runtime.GOOS == "windows" {
+		t.Skip("test scripts not built for windows") // db TODO(1.10.0)
+	}
+
+	// happy path: dir exists. this one will contain a single valid plugin.
+	tmp := t.TempDir()
+	secretsDir := filepath.Join(tmp, "secrets")
+	os.Mkdir(secretsDir, 0777)
+	cfg.CommonPluginDir = tmp
+
+	files := []struct {
+		name     string
+		contents string
+		perm     os.FileMode
+	}{
+		// only this first one should be detected as a valid plugin
+		{"happy-plugin", "#!/usr/bin/env sh\necho '{\"type\": \"secrets\", \"version\": \"0.0.1\"}'", 0700},
+		{"not-a-plugin", "#!/usr/bin/env sh\necho 'not a version'", 0700},
+		{"unhappy-plugin", "#!/usr/bin/env sh\necho 'sad plugin is sad'; exit 1", 0700},
+		{"not-executable", "do not execute me", 0400},
+	}
+	for _, f := range files {
+		must.NoError(t, os.WriteFile(filepath.Join(secretsDir, f.name), []byte(f.contents), f.perm))
+	}
+	// directories should be ignored
+	must.NoError(t, os.Mkdir(filepath.Join(secretsDir, "a-directory"), 0700))
+
+	// do the fingerprint
+	resp := &FingerprintResponse{}
+	err := fp.Fingerprint(req, resp)
+	must.NoError(t, err)
+	must.Eq(t, map[string]string{
+		"plugins.secrets.happy-plugin.version": "0.0.1",
+		"plugins.secrets.nomad.version":        "1.0.0",
+		"plugins.secrets.vault.version":        "1.0.0",
+	}, resp.Attributes)
+
+	// do it again after deleting our one good plugin.
+	// repeat runs should wipe attributes, so nothing should remain.
+	node.Attributes = resp.Attributes
+	must.NoError(t, os.Remove(filepath.Join(secretsDir, "happy-plugin")))
+
+	resp = &FingerprintResponse{}
+	err = fp.Fingerprint(req, resp)
+	must.NoError(t, err)
+	must.Eq(t, map[string]string{
+		"plugins.secrets.happy-plugin.version": "", // empty value means removed
+		"plugins.secrets.nomad.version":        "1.0.0",
+		"plugins.secrets.vault.version":        "1.0.0",
+	}, resp.Attributes)
+}

client/taskenv/env.go

@@ -154,8 +154,8 @@ const (
 	nodeMetaPrefix      = "meta."
 )
 
-// TaskEnv is a task's environment as well as node attribute's for
-// interpolation.
+// TaskEnv is a task's environment as well as node attribute's and
+// task secrets for interpolation.
 type TaskEnv struct {
 	// NodeAttrs is the map of node attributes for interpolation
 	NodeAttrs map[string]string
@@ -163,6 +163,9 @@ type TaskEnv struct {
 	// EnvMap is the map of environment variables
 	EnvMap map[string]string
 
+	// TaskSecrets is the map of secrets populated from the secrets hook
+	TaskSecrets map[string]string
+
 	// deviceEnv is the environment variables populated from the device hooks.
 	deviceEnv map[string]string
 
@@ -185,11 +188,12 @@ type TaskEnv struct {
 
 // NewTaskEnv creates a new task environment with the given environment, device
 // environment and node attribute maps.
-func NewTaskEnv(env, envClient, deviceEnv, node map[string]string, clientTaskDir, clientAllocDir string) *TaskEnv {
+func NewTaskEnv(env, envClient, deviceEnv, node map[string]string, secrets map[string]string, clientTaskDir, clientAllocDir string) *TaskEnv {
 	return &TaskEnv{
 		NodeAttrs:            node,
 		deviceEnv:            deviceEnv,
 		EnvMap:               env,
+		TaskSecrets:          secrets,
 		EnvMapClient:         envClient,
 		clientTaskDir:        clientTaskDir,
 		clientSharedAllocDir: clientAllocDir,
@@ -311,6 +315,13 @@ func (t *TaskEnv) AllValues() (map[string]cty.Value, map[string]error, error) {
 		}
 	}
 
+	// Prepare task-based secrets for use in interpolation
+	for k, v := range t.TaskSecrets {
+		if err := addNestedKey(allMap, k, v); err != nil {
+			errs[k] = err
+		}
+	}
+
 	// Add flat envMap as a Map to allMap so users can access any key via
 	// HCL2's indexing syntax: ${env["foo...bar"]}
 	allMap["env"] = cty.MapVal(envMap)
@@ -359,10 +370,10 @@ func (t *TaskEnv) ParseAndReplace(args []string) []string {
 }
 
 // ReplaceEnv takes an arg and replaces all occurrences of environment variables
-// and Nomad variables.  If the variable is found in the passed map it is
-// replaced, otherwise the original string is returned.
+// and Node attributes, and task secrets. If the variable is found in the passed map
+// it is replaced, otherwise the original string is returned.
 func (t *TaskEnv) ReplaceEnv(arg string) string {
-	return hargs.ReplaceEnv(arg, t.EnvMap, t.NodeAttrs)
+	return hargs.ReplaceEnv(arg, t.EnvMap, t.NodeAttrs, t.TaskSecrets)
 }
 
 // replaceEnvClient takes an arg and replaces all occurrences of client-specific
@@ -425,6 +436,9 @@ type Builder struct {
 	// nodeAttrs are Node attributes and metadata
 	nodeAttrs map[string]string
 
+	// taskSecrets are secrets populated from the secrets hook
+	taskSecrets map[string]string
+
 	// taskMeta are the meta attributes on the task
 	taskMeta map[string]string
 
@@ -516,9 +530,10 @@ func NewBuilder(node *structs.Node, alloc *structs.Allocation, task *structs.Tas
 // NewEmptyBuilder creates a new environment builder.
 func NewEmptyBuilder() *Builder {
 	return &Builder{
-		mu:       &sync.RWMutex{},
-		hookEnvs: map[string]map[string]string{},
-		envvars:  make(map[string]string),
+		mu:          &sync.RWMutex{},
+		hookEnvs:    map[string]map[string]string{},
+		envvars:     make(map[string]string),
+		taskSecrets: make(map[string]string),
 	}
 }
 
@@ -649,7 +664,7 @@ func (b *Builder) buildEnv(allocDir, localDir, secretsDir string,
 
 	// Copy interpolated task env vars second as they override host env vars
 	for k, v := range b.envvars {
-		envMap[k] = hargs.ReplaceEnv(v, nodeAttrs, envMap)
+		envMap[k] = hargs.ReplaceEnv(v, nodeAttrs, envMap, b.taskSecrets)
 	}
 
 	// Copy hook env vars in the order the hooks were run
@@ -709,7 +724,13 @@ func (b *Builder) Build() *TaskEnv {
 	envMap, deviceEnvs := b.buildEnv(b.allocDir, b.localDir, b.secretsDir, nodeAttrs)
 	envMapClient, _ := b.buildEnv(b.clientSharedAllocDir, b.clientTaskLocalDir, b.clientTaskSecretsDir, nodeAttrs)
 
-	return NewTaskEnv(envMap, envMapClient, deviceEnvs, nodeAttrs, b.clientTaskRoot, b.clientSharedAllocDir)
+	return NewTaskEnv(envMap, envMapClient, deviceEnvs, nodeAttrs, b.taskSecrets, b.clientTaskRoot, b.clientSharedAllocDir)
+}
+
+func (b *Builder) SetSecrets(secrets map[string]string) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	maps.Copy(b.taskSecrets, secrets)
 }
 
 // SetHookEnv sets environment variables from a hook. Variables are

client/taskenv/env_test.go

@@ -39,6 +39,9 @@ const (
 	envOneVal = "127.0.0.1"
 	envTwoKey = "NOMAD_PORT_WEB"
 	envTwoVal = ":80"
+
+	// Secrets populated from secrets hook
+	testSecret = "foo"
 )
 
 var (
@@ -65,7 +68,13 @@ func testEnvBuilder() *Builder {
 		envOneKey: envOneVal,
 		envTwoKey: envTwoVal,
 	}
-	return NewBuilder(n, mock.Alloc(), task, "global")
+
+	b := NewBuilder(n, mock.Alloc(), task, "global")
+	b.taskSecrets = map[string]string{
+		testSecret: testSecret,
+	}
+
+	return b
 }
 
 func TestEnvironment_ParseAndReplace_Env(t *testing.T) {
@@ -145,13 +154,13 @@ func TestEnvironment_ParseAndReplace_Mixed(t *testing.T) {
 func TestEnvironment_ReplaceEnv_Mixed(t *testing.T) {
 	ci.Parallel(t)
 
-	input := fmt.Sprintf("${%v}${%v%v}", nodeNameKey, nodeAttributePrefix, attrKey)
-	exp := fmt.Sprintf("%v%v", nodeName, attrVal)
+	input := fmt.Sprintf("${%v}${%v%v}${%v}", nodeNameKey, nodeAttributePrefix, attrKey, testSecret)
+	exp := fmt.Sprintf("%v%v%v", nodeName, attrVal, testSecret)
 	env := testEnvBuilder()
 	act := env.Build().ReplaceEnv(input)
 
 	if act != exp {
-		t.Fatalf("ParseAndReplace(%v) returned %#v; want %#v", input, act, exp)
+		t.Fatalf("ReplaceEnv(%v) returned %#v; want %#v", input, act, exp)
 	}
 }
 
@@ -364,6 +373,10 @@ func TestEnvironment_AllValues(t *testing.T) {
 	}
 	env.mu.Unlock()
 
+	env.taskSecrets = map[string]string{
+		"secret.testsecret.test": "foo",
+	}
+
 	values, errs, err := env.Build().AllValues()
 	require.NoError(t, err)
 
@@ -397,6 +410,9 @@ func TestEnvironment_AllValues(t *testing.T) {
 		"node.attr.kernel.name":        "linux",
 		"node.attr.nomad.version":      "0.5.0",
 
+		// Task Secrets for interpreting task config
+		`secret.testsecret.test`: "foo",
+
 		// Env
 		"taskEnvKey":                                "taskEnvVal",
 		"NOMAD_ADDR_http":                           "127.0.0.1:80",

client/taskenv/services_test.go

@@ -122,7 +122,7 @@ func TestInterpolateServices(t *testing.T) {
 var testEnv = NewTaskEnv(
 	map[string]string{"foo": "bar", "baz": "blah"},
 	map[string]string{"foo": "bar", "baz": "blah"},
-	nil, nil, "", "")
+	nil, nil, nil, "", "")
 
 func TestInterpolate_interpolateMapStringSliceString(t *testing.T) {
 	ci.Parallel(t)
@@ -219,7 +219,7 @@ func TestInterpolate_interpolateConnect(t *testing.T) {
 		"service1":          "_service1",
 		"host1":             "_host1",
 	}
-	env := NewTaskEnv(e, e, nil, nil, "", "")
+	env := NewTaskEnv(e, e, nil, nil, nil, "", "")
 
 	connect := &structs.ConsulConnect{
 		Native: false,