secrets: add secrets hook with nomad provider (#26143)

client/allocrunner/taskrunner/secrets/nomad_provider.go

@@ -0,0 +1,80 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package secrets
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/hashicorp/go-envparse"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/mitchellh/mapstructure"
+)
+
+type nomadProviderConfig struct {
+	Namespace string `mapstructure:"namespace"`
+}
+
+func defaultNomadConfig(namespace string) *nomadProviderConfig {
+	return &nomadProviderConfig{
+		Namespace: namespace,
+	}
+}
+
+type NomadProvider struct {
+	secret   *structs.Secret
+	tmplPath string
+	config   *nomadProviderConfig
+}
+
+// NewNomadProvider takes a task secret and decodes the config, overwriting the default config fields
+// with any provided fields, returning an error if the secret or secret's config is invalid.
+func NewNomadProvider(secret *structs.Secret, path string, namespace string) (*NomadProvider, error) {
+	if secret == nil {
+		return nil, fmt.Errorf("empty secret for nomad provider")
+	}
+
+	conf := defaultNomadConfig(namespace)
+	if err := mapstructure.Decode(secret.Config, conf); err != nil {
+		return nil, err
+	}
+
+	return &NomadProvider{
+		config:   conf,
+		secret:   secret,
+		tmplPath: path,
+	}, nil
+}
+
+func (n *NomadProvider) BuildTemplate() *structs.Template {
+	data := fmt.Sprintf(`
+		{{ with nomadVar "%s@%s" }}
+		{{ range $k, $v := . }}
+		secret.%s.{{ $k }}={{ $v }}
+		{{ end }}
+		{{ end }}`,
+		n.secret.Path, n.config.Namespace, n.secret.Name)
+
+	return &structs.Template{
+		EmbeddedTmpl: data,
+		DestPath:     n.tmplPath,
+		ChangeMode:   structs.TemplateChangeModeNoop,
+		Once:         true,
+	}
+}
+
+func (n *NomadProvider) Parse() (map[string]string, error) {
+	dest := filepath.Clean(n.tmplPath)
+	f, err := os.Open(dest)
+	if err != nil {
+		return nil, fmt.Errorf("error opening env template: %v", err)
+	}
+	defer func() {
+		f.Close()
+		os.Remove(dest)
+	}()
+
+	return envparse.Parse(f)
+}

client/allocrunner/taskrunner/secrets/nomad_provider_test.go

@@ -0,0 +1,78 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package secrets
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/shoenig/test/must"
+)
+
+func TestNomadProvider_BuildTemplate(t *testing.T) {
+	t.Run("variable template succeeds", func(t *testing.T) {
+		testDir := t.TempDir()
+		testSecret := &structs.Secret{
+			Name:     "foo",
+			Provider: "nomad",
+			Path:     "/test/path",
+			Config: map[string]any{
+				"namespace": "dev",
+			},
+		}
+		p, err := NewNomadProvider(testSecret, testDir, "default")
+		must.NoError(t, err)
+
+		tmpl := p.BuildTemplate()
+		must.NotNil(t, tmpl)
+
+		// expected template should have correct path and name
+		expectedTmpl := `
+		{{ with nomadVar "/test/path@dev" }}
+		{{ range $k, $v := . }}
+		secret.foo.{{ $k }}={{ $v }}
+		{{ end }}
+		{{ end }}`
+		// validate template string contains expected data
+		must.Eq(t, tmpl.EmbeddedTmpl, expectedTmpl)
+	})
+
+	t.Run("invalid config options errors", func(t *testing.T) {
+		testDir := t.TempDir()
+		testSecret := &structs.Secret{
+			Name:     "foo",
+			Provider: "nomad",
+			Path:     "/test/path",
+			Config: map[string]any{
+				"namespace": 123,
+			},
+		}
+		_, err := NewNomadProvider(testSecret, testDir, "default")
+		must.Error(t, err)
+
+		// tmpl := p.BuildTemplate()
+		// must.Nil(t, tmpl)
+	})
+}
+
+func TestNomadProvider_Parse(t *testing.T) {
+	testDir := t.TempDir()
+
+	tmplPath := filepath.Join(testDir, "foo")
+
+	data := "foo=bar"
+	err := os.WriteFile(tmplPath, []byte(data), 0777)
+	must.NoError(t, err)
+
+	p, err := NewNomadProvider(&structs.Secret{}, tmplPath, "default")
+	must.NoError(t, err)
+
+	vars, err := p.Parse()
+	must.Eq(t, vars, map[string]string{"foo": "bar"})
+
+	_, err = os.Stat(tmplPath)
+	must.ErrorContains(t, err, "no such file")
+}

client/allocrunner/taskrunner/secrets_hook.go

@@ -0,0 +1,183 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package taskrunner
+
+import (
+	"context"
+	"fmt"
+	"maps"
+	"path/filepath"
+
+	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-multierror"
+	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
+	ti "github.com/hashicorp/nomad/client/allocrunner/taskrunner/interfaces"
+	"github.com/hashicorp/nomad/client/allocrunner/taskrunner/secrets"
+	"github.com/hashicorp/nomad/client/allocrunner/taskrunner/template"
+	"github.com/hashicorp/nomad/client/config"
+	"github.com/hashicorp/nomad/client/taskenv"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+// SecretProvider currently only supports Vault and Nomad which use CT. Future
+// work can modify this interface to include custom providers using a plugin
+// interface.
+type SecretProvider interface {
+	// BuildTemplate should construct a template appropriate for that provider
+	// and append it to the templateManager's templates.
+	BuildTemplate() *structs.Template
+
+	// Parse allows each provider implementation to parse its "response" object.
+	Parse() (map[string]string, error)
+}
+
+type secretsHookConfig struct {
+	// logger is used to log
+	logger log.Logger
+
+	// lifecycle is used to interact with the task's lifecycle
+	lifecycle ti.TaskLifecycle
+
+	// events is used to emit events
+	events ti.EventEmitter
+
+	// clientConfig is the Nomad Client configuration
+	clientConfig *config.Config
+
+	// envBuilder is the environment variable builder for the task.
+	envBuilder *taskenv.Builder
+
+	// nomadNamespace is the job's Nomad namespace
+	nomadNamespace string
+}
+
+type secretsHook struct {
+	// logger is used to log
+	logger log.Logger
+
+	// lifecycle is used to interact with the task's lifecycle
+	lifecycle ti.TaskLifecycle
+
+	// events is used to emit events
+	events ti.EventEmitter
+
+	// clientConfig is the Nomad Client configuration
+	clientConfig *config.Config
+
+	// envBuilder is the environment variable builder for the task
+	envBuilder *taskenv.Builder
+
+	// nomadNamespace is the job's Nomad namespace
+	nomadNamespace string
+
+	// secrets to be fetched and populated for interpolation
+	secrets []*structs.Secret
+
+	// taskrunner secrets map
+	taskSecrets map[string]string
+}
+
+func newSecretsHook(conf *secretsHookConfig, secrets []*structs.Secret) *secretsHook {
+	return &secretsHook{
+		logger:         conf.logger,
+		lifecycle:      conf.lifecycle,
+		events:         conf.events,
+		clientConfig:   conf.clientConfig,
+		envBuilder:     conf.envBuilder,
+		nomadNamespace: conf.nomadNamespace,
+		secrets:        secrets,
+		// Future work will inject taskSecrets from the taskRunner, so that the taskrunner
+		// can make these secrets available to other hooks.
+		taskSecrets: make(map[string]string),
+	}
+}
+
+func (h *secretsHook) Name() string {
+	return "secrets"
+}
+
+func (h *secretsHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRequest, _ *interfaces.TaskPrestartResponse) error {
+	templates := []*structs.Template{}
+
+	providers, err := h.buildSecretProviders(req.TaskDir.SecretsDir)
+	if err != nil {
+		return err
+	}
+
+	for _, p := range providers {
+		templates = append(templates, p.BuildTemplate())
+	}
+
+	vaultCluster := req.Task.GetVaultClusterName()
+	vaultConfig := h.clientConfig.GetVaultConfigs(h.logger)[vaultCluster]
+
+	unblock := make(chan struct{})
+	tm, err := template.NewTaskTemplateManager(&template.TaskTemplateManagerConfig{
+		UnblockCh:            unblock,
+		Lifecycle:            h.lifecycle,
+		Events:               h.events,
+		Templates:            templates,
+		ClientConfig:         h.clientConfig,
+		VaultToken:           req.VaultToken,
+		VaultConfig:          vaultConfig,
+		VaultNamespace:       req.Alloc.Job.VaultNamespace,
+		TaskDir:              req.TaskDir.Dir,
+		EnvBuilder:           h.envBuilder,
+		MaxTemplateEventRate: template.DefaultMaxTemplateEventRate,
+		NomadNamespace:       h.nomadNamespace,
+		NomadToken:           req.NomadToken,
+		TaskID:               req.Alloc.ID + "-" + req.Task.Name,
+		Logger:               h.logger,
+	})
+	if err != nil {
+		return err
+	}
+
+	// Run the template manager to render templates.
+	go tm.Run()
+
+	// Safeguard against the template manager continuing to run.
+	defer tm.Stop()
+
+	select {
+	case <-ctx.Done():
+		return nil
+	case <-unblock:
+	}
+
+	// parse and copy variables to taskSecrets
+	for _, p := range providers {
+		vars, err := p.Parse()
+		if err != nil {
+			return err
+		}
+		maps.Copy(h.taskSecrets, vars)
+	}
+
+	return nil
+}
+
+func (h *secretsHook) buildSecretProviders(secretDir string) ([]SecretProvider, error) {
+	// Any configuration errors will be found when calling the secret providers constructor,
+	// so use a multierror to collect all errors and return them to the user at the same time.
+	providers, mErr := []SecretProvider{}, new(multierror.Error)
+
+	for idx, s := range h.secrets {
+		tmplPath := filepath.Join(secretDir, fmt.Sprintf("temp-%d", idx))
+		switch s.Provider {
+		case "nomad":
+			if p, err := secrets.NewNomadProvider(s, tmplPath, h.nomadNamespace); err != nil {
+				multierror.Append(mErr, err)
+			} else {
+				providers = append(providers, p)
+			}
+		case "vault":
+			// Unimplemented
+		default:
+			multierror.Append(mErr, fmt.Errorf("unknown secret provider type: %s", s.Provider))
+		}
+	}
+
+	return providers, mErr.ErrorOrNil()
+}

client/allocrunner/taskrunner/secrets_hook_test.go

@@ -0,0 +1,217 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package taskrunner
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/client/allocdir"
+	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
+	trtesting "github.com/hashicorp/nomad/client/allocrunner/taskrunner/testing"
+	"github.com/hashicorp/nomad/client/config"
+	"github.com/hashicorp/nomad/client/taskenv"
+	"github.com/hashicorp/nomad/helper/bufconndialer"
+	"github.com/hashicorp/nomad/helper/testlog"
+	"github.com/hashicorp/nomad/nomad/mock"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/shoenig/test/must"
+)
+
+func TestSecretsHook_Prestart_Nomad(t *testing.T) {
+	ci.Parallel(t)
+
+	t.Run("nomad provider successfully renders valid secret", func(t *testing.T) {
+		secretsResp := `
+		{
+		  "CreateIndex": 812,
+		  "CreateTime": 1750782609539170600,
+		  "Items": {
+		    "key2": "value2",
+		    "key1": "value1"
+		  },
+		  "ModifyIndex": 812,
+		  "ModifyTime": 1750782609539170600,
+		  "Namespace": "default",
+		  "Path": "testnomadvar"
+		}
+		`
+		count := 0 // CT expects a nomad index header that increments, or else it continues polling
+		nomadServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Add("X-Nomad-Index", strconv.Itoa(count))
+			fmt.Fprintln(w, secretsResp)
+			count += 1
+		}))
+		t.Cleanup(nomadServer.Close)
+
+		l, d := bufconndialer.New()
+		nomadServer.Listener = l
+
+		nomadServer.Start()
+
+		clientConfig := config.DefaultConfig()
+		clientConfig.TemplateDialer = d
+		clientConfig.TemplateConfig.DisableSandbox = true
+
+		taskDir := t.TempDir()
+		alloc := mock.MinAlloc()
+		task := alloc.Job.TaskGroups[0].Tasks[0]
+
+		conf := &secretsHookConfig{
+			logger:       testlog.HCLogger(t),
+			lifecycle:    trtesting.NewMockTaskHooks(),
+			events:       &trtesting.MockEmitter{},
+			clientConfig: clientConfig,
+			envBuilder:   taskenv.NewBuilder(mock.Node(), alloc, task, clientConfig.Region),
+		}
+		secretHook := newSecretsHook(conf, []*structs.Secret{
+			{
+				Name:     "test_secret",
+				Provider: "nomad",
+				Path:     "testnomadvar",
+				Config: map[string]any{
+					"namespace": "default",
+				},
+			},
+		})
+
+		req := &interfaces.TaskPrestartRequest{
+			Alloc:   alloc,
+			Task:    task,
+			TaskDir: &allocdir.TaskDir{Dir: taskDir, SecretsDir: taskDir},
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+		t.Cleanup(cancel)
+
+		err := secretHook.Prestart(ctx, req, nil)
+		must.NoError(t, err)
+
+		expected := map[string]string{
+			"secret.test_secret.key1": "value1",
+			"secret.test_secret.key2": "value2",
+		}
+		must.Eq(t, expected, secretHook.taskSecrets)
+	})
+
+	t.Run("returns early if context is cancelled", func(t *testing.T) {
+
+		secretsResp := `
+		{
+		  "CreateIndex": 812,
+		  "CreateTime": 1750782609539170600,
+		  "Items": {
+		    "key2": "value2",
+		    "key1": "value1"
+		  },
+		  "ModifyIndex": 812,
+		  "ModifyTime": 1750782609539170600,
+		  "Namespace": "default",
+		  "Path": "testnomadvar"
+		}
+		`
+		count := 0 // CT expects a nomad index header that increments, or else it continues polling
+		nomadServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Add("X-Nomad-Index", strconv.Itoa(count))
+			fmt.Fprintln(w, secretsResp)
+			count += 1
+		}))
+		t.Cleanup(nomadServer.Close)
+
+		l, d := bufconndialer.New()
+		nomadServer.Listener = l
+
+		nomadServer.Start()
+
+		clientConfig := config.DefaultConfig()
+		clientConfig.TemplateDialer = d
+		clientConfig.TemplateConfig.DisableSandbox = true
+
+		taskDir := t.TempDir()
+		alloc := mock.MinAlloc()
+		task := alloc.Job.TaskGroups[0].Tasks[0]
+
+		conf := &secretsHookConfig{
+
+			logger:       testlog.HCLogger(t),
+			lifecycle:    trtesting.NewMockTaskHooks(),
+			events:       &trtesting.MockEmitter{},
+			clientConfig: clientConfig,
+			envBuilder:   taskenv.NewBuilder(mock.Node(), alloc, task, clientConfig.Region),
+		}
+		secretHook := newSecretsHook(conf, []*structs.Secret{
+			{
+				Name:     "test_secret",
+				Provider: "nomad",
+				Path:     "testnomadvar",
+				Config: map[string]any{
+					"namespace": "default",
+				},
+			},
+		})
+
+		req := &interfaces.TaskPrestartRequest{
+			Alloc:   alloc,
+			Task:    task,
+			TaskDir: &allocdir.TaskDir{Dir: taskDir, SecretsDir: taskDir},
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+		cancel() // cancel context to simulate task being stopped
+
+		err := secretHook.Prestart(ctx, req, nil)
+		must.NoError(t, err)
+
+		expected := map[string]string{}
+		must.Eq(t, expected, secretHook.taskSecrets)
+	})
+
+	t.Run("errors when failure building secret providers", func(t *testing.T) {
+		clientConfig := config.DefaultConfig()
+
+		taskDir := t.TempDir()
+		alloc := mock.MinAlloc()
+		task := alloc.Job.TaskGroups[0].Tasks[0]
+
+		conf := &secretsHookConfig{
+
+			logger:       testlog.HCLogger(t),
+			lifecycle:    trtesting.NewMockTaskHooks(),
+			events:       &trtesting.MockEmitter{},
+			clientConfig: clientConfig,
+			envBuilder:   taskenv.NewBuilder(mock.Node(), alloc, task, clientConfig.Region),
+		}
+
+		// give an invalid secret, in this case a nomad secret with bad namespace
+		secretHook := newSecretsHook(conf, []*structs.Secret{
+			{
+				Name:     "test_secret",
+				Provider: "nomad",
+				Path:     "testnomadvar",
+				Config: map[string]any{
+					"namespace": 123,
+				},
+			},
+		})
+
+		req := &interfaces.TaskPrestartRequest{
+			Alloc:   alloc,
+			Task:    task,
+			TaskDir: &allocdir.TaskDir{Dir: taskDir, SecretsDir: taskDir},
+		}
+
+		// Prestart should error and return after building secrets
+		err := secretHook.Prestart(context.Background(), req, nil)
+		must.Error(t, err)
+
+		expected := map[string]string{}
+		must.Eq(t, expected, secretHook.taskSecrets)
+	})
+}

client/allocrunner/taskrunner/task_runner_hooks.go

@@ -55,40 +55,16 @@ func (tr *TaskRunner) initHooks() {
 
 	tr.logmonHookConfig = newLogMonHookConfig(task.Name, task.LogConfig, tr.taskDir.LogDir)
 
-	// Add the hook resources
-	tr.hookResources = &hookResources{}
-
-	// Create the task directory hook. This is run first to ensure the
+	// Create the task directory hook first. This is run first to ensure the
 	// directory path exists for other hooks.
-	alloc := tr.Alloc()
+	tr.hookResources = &hookResources{}
 	tr.runnerHooks = []interfaces.TaskHook{
 		newValidateHook(tr.clientConfig, hookLogger),
 		newDynamicUsersHook(tr.killCtx, tr.driverCapabilities.DynamicWorkloadUsers, tr.logger, tr.users),
 		newTaskDirHook(tr, hookLogger),
 		newIdentityHook(tr, hookLogger),
-		newLogMonHook(tr, hookLogger),
-		newDispatchHook(alloc, hookLogger),
-		newVolumeHook(tr, hookLogger),
-		newArtifactHook(tr, tr.getter, hookLogger),
-		newStatsHook(tr, tr.clientConfig.StatsCollectionInterval, tr.clientConfig.PublishAllocationMetrics, hookLogger),
-		newDeviceHook(tr.devicemanager, hookLogger),
-		newAPIHook(tr.shutdownCtx, tr.clientConfig.APIListenerRegistrar, hookLogger),
-		newWranglerHook(tr.wranglers, task.Name, alloc.ID, task.UsesCores(), hookLogger),
+		newConsulHook(hookLogger, tr),
 	}
-
-	// If the task has a CSI block, add the hook.
-	if task.CSIPluginConfig != nil {
-		tr.runnerHooks = append(tr.runnerHooks, newCSIPluginSupervisorHook(
-			&csiPluginSupervisorHookConfig{
-				clientStateDirPath: tr.clientConfig.StateDir,
-				events:             tr,
-				runner:             tr,
-				lifecycle:          tr,
-				capabilities:       tr.driverCapabilities,
-				logger:             hookLogger,
-			}))
-	}
-
 	// If Vault is enabled, add the hook
 	if task.Vault != nil && tr.vaultClientFunc != nil {
 		tr.runnerHooks = append(tr.runnerHooks, newVaultHook(&vaultHookConfig{
@@ -105,13 +81,45 @@ func (tr *TaskRunner) initHooks() {
 		}))
 	}
 
+	if len(task.Secrets) > 0 {
+		tr.runnerHooks = append(tr.runnerHooks, newSecretsHook(&secretsHookConfig{
+			logger:         tr.logger,
+			lifecycle:      tr,
+			events:         tr,
+			clientConfig:   tr.clientConfig,
+			envBuilder:     tr.envBuilder,
+			nomadNamespace: tr.alloc.Job.Namespace,
+		}, task.Secrets))
+	}
+
+	alloc := tr.Alloc()
+	tr.runnerHooks = append(tr.runnerHooks, []interfaces.TaskHook{
+		newLogMonHook(tr, hookLogger),
+		newDispatchHook(alloc, hookLogger),
+		newVolumeHook(tr, hookLogger),
+		newArtifactHook(tr, tr.getter, hookLogger),
+		newStatsHook(tr, tr.clientConfig.StatsCollectionInterval, tr.clientConfig.PublishAllocationMetrics, hookLogger),
+		newDeviceHook(tr.devicemanager, hookLogger),
+		newAPIHook(tr.shutdownCtx, tr.clientConfig.APIListenerRegistrar, hookLogger),
+		newWranglerHook(tr.wranglers, task.Name, alloc.ID, task.UsesCores(), hookLogger),
+	}...)
+
+	// If the task has a CSI block, add the hook.
+	if task.CSIPluginConfig != nil {
+		tr.runnerHooks = append(tr.runnerHooks, newCSIPluginSupervisorHook(
+			&csiPluginSupervisorHookConfig{
+				clientStateDirPath: tr.clientConfig.StateDir,
+				events:             tr,
+				runner:             tr,
+				lifecycle:          tr,
+				capabilities:       tr.driverCapabilities,
+				logger:             hookLogger,
+			}))
+	}
+
 	// Get the consul namespace for the TG of the allocation.
 	consulNamespace := tr.alloc.ConsulNamespaceForTask(tr.taskName)
 
-	// Add the consul hook (populates task secret dirs and sets the environment if
-	// consul tokens are present for the task).
-	tr.runnerHooks = append(tr.runnerHooks, newConsulHook(hookLogger, tr))
-
 	// If there are templates is enabled, add the hook
 	if len(task.Templates) != 0 {
 		tr.runnerHooks = append(tr.runnerHooks, newTemplateHook(&templateHookConfig{