modify rawexec TaskConfig and Config to accept envvar denylist (#25511)

* modify rawexec TaskConfig and Config to accept envvar denylist
* update rawexec driver docs to include deniedEnvars options
Co-authored-by: Daniel Bennett <dbennett@hashicorp.com>

---------

Co-authored-by: Daniel Bennett <dbennett@hashicorp.com>

.changelog/25511.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+drivers/rawexec: adds denied_envvars to driver and task config options
+```

drivers/rawexec/driver.go

@@ -9,6 +9,8 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"slices"
+	"sort"
 	"strconv"
 	"time"
 
@@ -26,6 +28,7 @@ import (
 	"github.com/hashicorp/nomad/plugins/drivers/fsisolation"
 	"github.com/hashicorp/nomad/plugins/shared/hclspec"
 	pstructs "github.com/hashicorp/nomad/plugins/shared/structs"
+	"github.com/ryanuber/go-glob"
 )
 
 const (
@@ -84,6 +87,7 @@ var (
 		),
 		"denied_host_uids": hclspec.NewAttr("denied_host_uids", "string", false),
 		"denied_host_gids": hclspec.NewAttr("denied_host_gids", "string", false),
+		"denied_envvars":   hclspec.NewAttr("denied_envvars", "list(string)", false),
 	})
 
 	// taskConfigSpec is the hcl specification for the driver config section of
@@ -95,6 +99,7 @@ var (
 		"cgroup_v1_override": hclspec.NewAttr("cgroup_v1_override", "list(map(string))", false),
 		"oom_score_adj":      hclspec.NewAttr("oom_score_adj", "number", false),
 		"work_dir":           hclspec.NewAttr("work_dir", "string", false),
+		"denied_envvars":     hclspec.NewAttr("denied_envvars", "list(string)", false),
 	})
 
 	// capabilities is returned by the Capabilities RPC and indicates what
@@ -150,8 +155,9 @@ type Config struct {
 	// Enabled is set to true to enable the raw_exec driver
 	Enabled bool `codec:"enabled"`
 
-	DeniedHostUids string `codec:"denied_host_uids"`
-	DeniedHostGids string `codec:"denied_host_gids"`
+	DeniedHostUids string   `codec:"denied_host_uids"`
+	DeniedHostGids string   `codec:"denied_host_gids"`
+	DeniedEnvvars  []string `codec:"denied_envvars"`
 }
 
 // TaskConfig is the driver configuration of a task within a job
@@ -176,6 +182,9 @@ type TaskConfig struct {
 
 	// WorkDir sets the working directory of the task
 	WorkDir string `codec:"work_dir"`
+
+	//DeniedEnvvars enables the removal of specified environment variables from a given job environment
+	DeniedEnvvars []string `codec:"denied_envvars"`
 }
 
 func (t *TaskConfig) validate() error {
@@ -358,6 +367,27 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error {
 	return nil
 }
 
+func (d *Driver) buildEnvList(tc *TaskConfig, cfg *drivers.TaskConfig) []string {
+	// combine tc and cfg denyLists
+	denyList := slices.Concat(d.config.DeniedEnvvars, tc.DeniedEnvvars)
+	envList := make([]string, 0, len(cfg.Env))
+
+	for k, v := range cfg.Env {
+		found := false
+		for _, denied := range denyList {
+			if found = glob.Glob(denied, k); found {
+				break
+			}
+		}
+
+		if !found {
+			envList = append(envList, k+"="+v)
+		}
+	}
+	sort.Strings(envList)
+	return envList
+}
+
 func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drivers.DriverNetwork, error) {
 	if !d.config.Enabled {
 		return nil, nil, errDisabledDriver
@@ -402,7 +432,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
 	execCmd := &executor.ExecCommand{
 		Cmd:              driverConfig.Command,
 		Args:             driverConfig.Args,
-		Env:              cfg.EnvList(),
+		Env:              d.buildEnvList(&driverConfig, cfg),
 		User:             cfg.User,
 		TaskDir:          cfg.TaskDir().Dir,
 		WorkDir:          driverConfig.WorkDir,

drivers/rawexec/driver_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/hashicorp/nomad/ci"
 	"github.com/hashicorp/nomad/client/lib/cgroupslib"
 	"github.com/hashicorp/nomad/client/lib/numalib"
+
 	ctestutil "github.com/hashicorp/nomad/client/testutil"
 	"github.com/hashicorp/nomad/helper/pluginutils/hclutils"
 	"github.com/hashicorp/nomad/helper/testlog"
@@ -42,6 +43,20 @@ func defaultEnv() map[string]string {
 	return m
 }
 
+// genEnv returns a populated map of environment variables
+func genEnv() map[string]string {
+	return map[string]string{
+		"NOMAD_TOKEN":    "abcd",
+		"GITHUB_TOKEN":   "efg",
+		"AWS_SECRET_KEY": "hij",
+		"NOMAD_ADDR":     "klm",
+		"TEST_TOKEN":     "nop",
+		"TEST_AWS_VAR":   "qrs",
+		"VAR_TEST_AWS":   "tuv",
+		"PORT":           "wxyz",
+	}
+}
+
 func testResources(allocID, task string) *drivers.Resources {
 	if allocID == "" || task == "" {
 		panic("must be set")
@@ -662,3 +677,257 @@ func TestRawExecDriver_validate(t *testing.T) {
 		})
 	}
 }
+
+func TestRawExecDriver_buildEnvList(t *testing.T) {
+	defaultEnvironment := genEnv()
+	testCases := []struct {
+		name             string
+		taskConfig       *TaskConfig
+		driverTaskConfig *drivers.TaskConfig
+		driverConfig     *Config
+		expectedVars     []string
+	}{
+		{name: "OK, no globs",
+			taskConfig: &TaskConfig{
+				DeniedEnvvars: []string{"AWS_SECRET_KEY"},
+			},
+			driverTaskConfig: &drivers.TaskConfig{
+				Env: defaultEnvironment,
+			},
+			driverConfig: &Config{
+				DeniedEnvvars: []string{"NOMAD_TOKEN", "GITHUB_TOKEN"},
+			},
+			expectedVars: []string{
+				"NOMAD_ADDR=klm",
+				"PORT=wxyz",
+				"TEST_AWS_VAR=qrs",
+				"TEST_TOKEN=nop",
+				"VAR_TEST_AWS=tuv",
+			},
+		},
+		{name: "OK, globs",
+			taskConfig: &TaskConfig{
+				DeniedEnvvars: []string{"AWS_SECRET_KEY"},
+			},
+			driverTaskConfig: &drivers.TaskConfig{
+				Env: defaultEnvironment,
+			},
+			driverConfig: &Config{
+				DeniedEnvvars: []string{"*_TOKEN"},
+			},
+			expectedVars: []string{
+				"NOMAD_ADDR=klm",
+				"PORT=wxyz",
+				"TEST_AWS_VAR=qrs",
+				"VAR_TEST_AWS=tuv",
+			},
+		}, {name: "OK, multiple globs",
+			taskConfig: &TaskConfig{
+				DeniedEnvvars: []string{},
+			},
+			driverTaskConfig: &drivers.TaskConfig{
+				Env: defaultEnvironment,
+			},
+			driverConfig: &Config{
+				DeniedEnvvars: []string{"*AWS*"},
+			},
+			expectedVars: []string{
+				"GITHUB_TOKEN=efg",
+				"NOMAD_ADDR=klm",
+				"NOMAD_TOKEN=abcd",
+				"PORT=wxyz",
+				"TEST_TOKEN=nop",
+			},
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			d := newEnabledRawExecDriver(t)
+			d.config = tc.driverConfig
+			envList := d.buildEnvList(tc.taskConfig, tc.driverTaskConfig)
+			must.SliceEqOp(t, envList, tc.expectedVars)
+		})
+	}
+}
+
+func TestRawExecDriver_Env(t *testing.T) {
+	ci.Parallel(t)
+	ctestutil.RequireNotWindows(t)
+
+	d := newEnabledRawExecDriver(t)
+	allocID := uuid.Generate()
+	taskName := "sleep"
+
+	task :=
+		&drivers.TaskConfig{
+			AllocID:   allocID,
+			ID:        uuid.Generate(),
+			Name:      taskName,
+			Env:       genEnv(),
+			Resources: testResources(allocID, taskName),
+		}
+	testCases := []struct {
+		name         string
+		driver       *Driver
+		driverConfig *Config
+		taskConfig   *TaskConfig
+		varsExpected bool
+		deniedVars   []string
+	}{
+		{name: "no denied vars",
+			driver:       d,
+			driverConfig: nil,
+			taskConfig: &TaskConfig{
+				Command: testtask.Path(),
+				Args:    []string{"sleep", "10ms"},
+			},
+			deniedVars:   []string{},
+			varsExpected: true,
+		},
+		{name: "both levels, named vars",
+			driver: d,
+			driverConfig: &Config{
+				Enabled:       true,
+				DeniedEnvvars: []string{"NOMAD_ADDR"},
+			},
+			taskConfig: &TaskConfig{
+				Command:       testtask.Path(),
+				Args:          []string{"sleep", "10ms"},
+				DeniedEnvvars: []string{"NOMAD_TOKEN"},
+			},
+			deniedVars: []string{
+				"NOMAD_ADDR=klm",
+				"NOMAD_TOKEN=abcd",
+			},
+			varsExpected: false,
+		}, {name: "driver level, glob suffix vars",
+			driver: d,
+			driverConfig: &Config{
+				Enabled:       true,
+				DeniedEnvvars: []string{"NOMAD_*"},
+			},
+			taskConfig: &TaskConfig{
+				Command: testtask.Path(),
+				Args:    []string{"sleep", "10ms"},
+			},
+			varsExpected: false,
+			deniedVars: []string{
+				"NOMAD_ADDR=klm",
+				"NOMAD_TOKEN=abcd",
+			},
+		}, {name: "driver level, glob prefix vars",
+			driver: d,
+			driverConfig: &Config{
+				Enabled:       true,
+				DeniedEnvvars: []string{"*TOKEN"},
+			},
+			taskConfig: &TaskConfig{
+				Command: testtask.Path(),
+				Args:    []string{"sleep", "10ms"},
+			},
+			deniedVars: []string{
+				"GITHUB_TOKEN=efg",
+				"NOMAD_TOKEN=abcd",
+				"TEST_TOKEN=nop",
+			},
+			varsExpected: false,
+		}, {name: "driver level, glob prefix & suffix",
+			driver: d,
+			driverConfig: &Config{
+				Enabled:       true,
+				DeniedEnvvars: []string{"*AWS*"},
+			},
+			taskConfig: &TaskConfig{
+				Command: testtask.Path(),
+				Args:    []string{"sleep", "10ms"},
+			},
+			deniedVars: []string{
+				"AWS_SECRET_KEY=hij",
+				"TEST_AWS_VAR=qrs",
+				"VAR_TEST_AWS=tuv",
+			},
+			varsExpected: false,
+		},
+		{name: "task level, glob suffix vars",
+			driver:       d,
+			driverConfig: nil,
+			taskConfig: &TaskConfig{
+				Command:       testtask.Path(),
+				Args:          []string{"sleep", "10ms"},
+				DeniedEnvvars: []string{"NOMAD_*"},
+			},
+			varsExpected: false,
+			deniedVars: []string{
+				"NOMAD_ADDR=klm",
+				"NOMAD_TOKEN=abcd",
+			},
+		}, {name: "task level, glob prefix vars",
+			driver:       d,
+			driverConfig: nil,
+			taskConfig: &TaskConfig{
+				Command:       testtask.Path(),
+				Args:          []string{"sleep", "10ms"},
+				DeniedEnvvars: []string{"*TOKEN"},
+			},
+			deniedVars: []string{
+				"GITHUB_TOKEN=efg",
+				"NOMAD_TOKEN=abcd",
+				"TEST_TOKEN=nop",
+			},
+			varsExpected: false,
+		},
+		{name: "task level, glob prefix & suffix",
+			driver:       d,
+			driverConfig: nil,
+			taskConfig: &TaskConfig{
+				Command:       testtask.Path(),
+				Args:          []string{"sleep", "10ms"},
+				DeniedEnvvars: []string{"*AWS*"},
+			},
+			deniedVars: []string{
+				"AWS_SECRET_KEY=hij",
+				"TEST_AWS_VAR=qrs",
+				"VAR_TEST_AWS=tuv",
+			},
+			varsExpected: false,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// if set, update driver config
+			if tc.driverConfig != nil {
+				tc.driver.config = tc.driverConfig
+			}
+
+			harness := dtestutil.NewDriverHarness(t, tc.driver)
+			defer harness.Kill()
+
+			cleanup := harness.MkAllocDir(task, false)
+			defer cleanup()
+
+			harness.MakeTaskCgroup(allocID, taskName)
+
+			// set and encode task config
+			taskConfig := tc.taskConfig
+			must.NoError(t, task.EncodeConcreteDriverConfig(&taskConfig))
+
+			// start task
+			_, _, err := harness.StartTask(task)
+			must.NoError(t, err)
+			// exec an env to standard out
+			res, err := harness.ExecTask(task.ID, []string{"env"}, 1*time.Second)
+			must.NoError(t, err)
+			must.True(t, res.ExitResult.Successful())
+
+			// confirm denied variables are not found in stdout
+			for _, v := range tc.deniedVars {
+				if tc.varsExpected {
+					must.StrNotContains(t, string(res.Stdout), v)
+				}
+			}
+
+			must.NoError(t, harness.DestroyTask(task.ID, true))
+		})
+	}
+
+}

website/content/docs/drivers/raw_exec.mdx

@@ -57,6 +57,9 @@ the Nomad client has been hardened according to the [production][hardening] guid
   must be an absolute path. This will also change the working directory when
   using `nomad alloc exec`.
 
+- `denied_envvars` - (Optional) Passes a list of environment variables that
+  the driver should scrub from the task environment. Supports globbing, with "*"
+  wildcard accepted as prefix and/or suffix.
 
 ## Examples
 
@@ -159,6 +162,15 @@ config {
 }
 ```
 
+- `denied_envvars` - (Optional) Passes a list of environment variables that
+  the driver should scrub from all task environments. Supports globbing with "*"
+  wildcard accepted as prefix and/or suffix.
+
+```hcl
+config {
+  denied_envvars = ["AWS_SECRET_KEY", "*_TOKEN"]
+}
+```
 ## Client Options
 
 ~> Note: client configuration options will soon be deprecated. Please use