diff --git a/.changelog/25511.txt b/.changelog/25511.txt new file mode 100644 index 000000000..9fde16110 --- /dev/null +++ b/.changelog/25511.txt @@ -0,0 +1,3 @@ +```release-note:improvement +drivers/rawexec: adds denied_envvars to driver and task config options +``` diff --git a/drivers/rawexec/driver.go b/drivers/rawexec/driver.go index e3cdb481f..81b9c0153 100644 --- a/drivers/rawexec/driver.go +++ b/drivers/rawexec/driver.go @@ -9,6 +9,8 @@ import ( "fmt" "os" "path/filepath" + "slices" + "sort" "strconv" "time" @@ -26,6 +28,7 @@ import ( "github.com/hashicorp/nomad/plugins/drivers/fsisolation" "github.com/hashicorp/nomad/plugins/shared/hclspec" pstructs "github.com/hashicorp/nomad/plugins/shared/structs" + "github.com/ryanuber/go-glob" ) const ( @@ -84,6 +87,7 @@ var ( ), "denied_host_uids": hclspec.NewAttr("denied_host_uids", "string", false), "denied_host_gids": hclspec.NewAttr("denied_host_gids", "string", false), + "denied_envvars": hclspec.NewAttr("denied_envvars", "list(string)", false), }) // taskConfigSpec is the hcl specification for the driver config section of @@ -95,6 +99,7 @@ var ( "cgroup_v1_override": hclspec.NewAttr("cgroup_v1_override", "list(map(string))", false), "oom_score_adj": hclspec.NewAttr("oom_score_adj", "number", false), "work_dir": hclspec.NewAttr("work_dir", "string", false), + "denied_envvars": hclspec.NewAttr("denied_envvars", "list(string)", false), }) // capabilities is returned by the Capabilities RPC and indicates what @@ -150,8 +155,9 @@ type Config struct { // Enabled is set to true to enable the raw_exec driver Enabled bool `codec:"enabled"` - DeniedHostUids string `codec:"denied_host_uids"` - DeniedHostGids string `codec:"denied_host_gids"` + DeniedHostUids string `codec:"denied_host_uids"` + DeniedHostGids string `codec:"denied_host_gids"` + DeniedEnvvars []string `codec:"denied_envvars"` } // TaskConfig is the driver configuration of a task within a job @@ -176,6 +182,9 @@ type TaskConfig struct { // WorkDir sets the working directory of the task WorkDir string `codec:"work_dir"` + + //DeniedEnvvars enables the removal of specified environment variables from a given job environment + DeniedEnvvars []string `codec:"denied_envvars"` } func (t *TaskConfig) validate() error { @@ -358,6 +367,27 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { return nil } +func (d *Driver) buildEnvList(tc *TaskConfig, cfg *drivers.TaskConfig) []string { + // combine tc and cfg denyLists + denyList := slices.Concat(d.config.DeniedEnvvars, tc.DeniedEnvvars) + envList := make([]string, 0, len(cfg.Env)) + + for k, v := range cfg.Env { + found := false + for _, denied := range denyList { + if found = glob.Glob(denied, k); found { + break + } + } + + if !found { + envList = append(envList, k+"="+v) + } + } + sort.Strings(envList) + return envList +} + func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drivers.DriverNetwork, error) { if !d.config.Enabled { return nil, nil, errDisabledDriver @@ -402,7 +432,7 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive execCmd := &executor.ExecCommand{ Cmd: driverConfig.Command, Args: driverConfig.Args, - Env: cfg.EnvList(), + Env: d.buildEnvList(&driverConfig, cfg), User: cfg.User, TaskDir: cfg.TaskDir().Dir, WorkDir: driverConfig.WorkDir, diff --git a/drivers/rawexec/driver_test.go b/drivers/rawexec/driver_test.go index 78d60e2fa..b9ca50c33 100644 --- a/drivers/rawexec/driver_test.go +++ b/drivers/rawexec/driver_test.go @@ -19,6 +19,7 @@ import ( "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/client/lib/cgroupslib" "github.com/hashicorp/nomad/client/lib/numalib" + ctestutil "github.com/hashicorp/nomad/client/testutil" "github.com/hashicorp/nomad/helper/pluginutils/hclutils" "github.com/hashicorp/nomad/helper/testlog" @@ -42,6 +43,20 @@ func defaultEnv() map[string]string { return m } +// genEnv returns a populated map of environment variables +func genEnv() map[string]string { + return map[string]string{ + "NOMAD_TOKEN": "abcd", + "GITHUB_TOKEN": "efg", + "AWS_SECRET_KEY": "hij", + "NOMAD_ADDR": "klm", + "TEST_TOKEN": "nop", + "TEST_AWS_VAR": "qrs", + "VAR_TEST_AWS": "tuv", + "PORT": "wxyz", + } +} + func testResources(allocID, task string) *drivers.Resources { if allocID == "" || task == "" { panic("must be set") @@ -662,3 +677,257 @@ func TestRawExecDriver_validate(t *testing.T) { }) } } + +func TestRawExecDriver_buildEnvList(t *testing.T) { + defaultEnvironment := genEnv() + testCases := []struct { + name string + taskConfig *TaskConfig + driverTaskConfig *drivers.TaskConfig + driverConfig *Config + expectedVars []string + }{ + {name: "OK, no globs", + taskConfig: &TaskConfig{ + DeniedEnvvars: []string{"AWS_SECRET_KEY"}, + }, + driverTaskConfig: &drivers.TaskConfig{ + Env: defaultEnvironment, + }, + driverConfig: &Config{ + DeniedEnvvars: []string{"NOMAD_TOKEN", "GITHUB_TOKEN"}, + }, + expectedVars: []string{ + "NOMAD_ADDR=klm", + "PORT=wxyz", + "TEST_AWS_VAR=qrs", + "TEST_TOKEN=nop", + "VAR_TEST_AWS=tuv", + }, + }, + {name: "OK, globs", + taskConfig: &TaskConfig{ + DeniedEnvvars: []string{"AWS_SECRET_KEY"}, + }, + driverTaskConfig: &drivers.TaskConfig{ + Env: defaultEnvironment, + }, + driverConfig: &Config{ + DeniedEnvvars: []string{"*_TOKEN"}, + }, + expectedVars: []string{ + "NOMAD_ADDR=klm", + "PORT=wxyz", + "TEST_AWS_VAR=qrs", + "VAR_TEST_AWS=tuv", + }, + }, {name: "OK, multiple globs", + taskConfig: &TaskConfig{ + DeniedEnvvars: []string{}, + }, + driverTaskConfig: &drivers.TaskConfig{ + Env: defaultEnvironment, + }, + driverConfig: &Config{ + DeniedEnvvars: []string{"*AWS*"}, + }, + expectedVars: []string{ + "GITHUB_TOKEN=efg", + "NOMAD_ADDR=klm", + "NOMAD_TOKEN=abcd", + "PORT=wxyz", + "TEST_TOKEN=nop", + }, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + d := newEnabledRawExecDriver(t) + d.config = tc.driverConfig + envList := d.buildEnvList(tc.taskConfig, tc.driverTaskConfig) + must.SliceEqOp(t, envList, tc.expectedVars) + }) + } +} + +func TestRawExecDriver_Env(t *testing.T) { + ci.Parallel(t) + ctestutil.RequireNotWindows(t) + + d := newEnabledRawExecDriver(t) + allocID := uuid.Generate() + taskName := "sleep" + + task := + &drivers.TaskConfig{ + AllocID: allocID, + ID: uuid.Generate(), + Name: taskName, + Env: genEnv(), + Resources: testResources(allocID, taskName), + } + testCases := []struct { + name string + driver *Driver + driverConfig *Config + taskConfig *TaskConfig + varsExpected bool + deniedVars []string + }{ + {name: "no denied vars", + driver: d, + driverConfig: nil, + taskConfig: &TaskConfig{ + Command: testtask.Path(), + Args: []string{"sleep", "10ms"}, + }, + deniedVars: []string{}, + varsExpected: true, + }, + {name: "both levels, named vars", + driver: d, + driverConfig: &Config{ + Enabled: true, + DeniedEnvvars: []string{"NOMAD_ADDR"}, + }, + taskConfig: &TaskConfig{ + Command: testtask.Path(), + Args: []string{"sleep", "10ms"}, + DeniedEnvvars: []string{"NOMAD_TOKEN"}, + }, + deniedVars: []string{ + "NOMAD_ADDR=klm", + "NOMAD_TOKEN=abcd", + }, + varsExpected: false, + }, {name: "driver level, glob suffix vars", + driver: d, + driverConfig: &Config{ + Enabled: true, + DeniedEnvvars: []string{"NOMAD_*"}, + }, + taskConfig: &TaskConfig{ + Command: testtask.Path(), + Args: []string{"sleep", "10ms"}, + }, + varsExpected: false, + deniedVars: []string{ + "NOMAD_ADDR=klm", + "NOMAD_TOKEN=abcd", + }, + }, {name: "driver level, glob prefix vars", + driver: d, + driverConfig: &Config{ + Enabled: true, + DeniedEnvvars: []string{"*TOKEN"}, + }, + taskConfig: &TaskConfig{ + Command: testtask.Path(), + Args: []string{"sleep", "10ms"}, + }, + deniedVars: []string{ + "GITHUB_TOKEN=efg", + "NOMAD_TOKEN=abcd", + "TEST_TOKEN=nop", + }, + varsExpected: false, + }, {name: "driver level, glob prefix & suffix", + driver: d, + driverConfig: &Config{ + Enabled: true, + DeniedEnvvars: []string{"*AWS*"}, + }, + taskConfig: &TaskConfig{ + Command: testtask.Path(), + Args: []string{"sleep", "10ms"}, + }, + deniedVars: []string{ + "AWS_SECRET_KEY=hij", + "TEST_AWS_VAR=qrs", + "VAR_TEST_AWS=tuv", + }, + varsExpected: false, + }, + {name: "task level, glob suffix vars", + driver: d, + driverConfig: nil, + taskConfig: &TaskConfig{ + Command: testtask.Path(), + Args: []string{"sleep", "10ms"}, + DeniedEnvvars: []string{"NOMAD_*"}, + }, + varsExpected: false, + deniedVars: []string{ + "NOMAD_ADDR=klm", + "NOMAD_TOKEN=abcd", + }, + }, {name: "task level, glob prefix vars", + driver: d, + driverConfig: nil, + taskConfig: &TaskConfig{ + Command: testtask.Path(), + Args: []string{"sleep", "10ms"}, + DeniedEnvvars: []string{"*TOKEN"}, + }, + deniedVars: []string{ + "GITHUB_TOKEN=efg", + "NOMAD_TOKEN=abcd", + "TEST_TOKEN=nop", + }, + varsExpected: false, + }, + {name: "task level, glob prefix & suffix", + driver: d, + driverConfig: nil, + taskConfig: &TaskConfig{ + Command: testtask.Path(), + Args: []string{"sleep", "10ms"}, + DeniedEnvvars: []string{"*AWS*"}, + }, + deniedVars: []string{ + "AWS_SECRET_KEY=hij", + "TEST_AWS_VAR=qrs", + "VAR_TEST_AWS=tuv", + }, + varsExpected: false, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // if set, update driver config + if tc.driverConfig != nil { + tc.driver.config = tc.driverConfig + } + + harness := dtestutil.NewDriverHarness(t, tc.driver) + defer harness.Kill() + + cleanup := harness.MkAllocDir(task, false) + defer cleanup() + + harness.MakeTaskCgroup(allocID, taskName) + + // set and encode task config + taskConfig := tc.taskConfig + must.NoError(t, task.EncodeConcreteDriverConfig(&taskConfig)) + + // start task + _, _, err := harness.StartTask(task) + must.NoError(t, err) + // exec an env to standard out + res, err := harness.ExecTask(task.ID, []string{"env"}, 1*time.Second) + must.NoError(t, err) + must.True(t, res.ExitResult.Successful()) + + // confirm denied variables are not found in stdout + for _, v := range tc.deniedVars { + if tc.varsExpected { + must.StrNotContains(t, string(res.Stdout), v) + } + } + + must.NoError(t, harness.DestroyTask(task.ID, true)) + }) + } + +} diff --git a/website/content/docs/drivers/raw_exec.mdx b/website/content/docs/drivers/raw_exec.mdx index e2d589e85..73d4cd76c 100644 --- a/website/content/docs/drivers/raw_exec.mdx +++ b/website/content/docs/drivers/raw_exec.mdx @@ -57,6 +57,9 @@ the Nomad client has been hardened according to the [production][hardening] guid must be an absolute path. This will also change the working directory when using `nomad alloc exec`. +- `denied_envvars` - (Optional) Passes a list of environment variables that + the driver should scrub from the task environment. Supports globbing, with "*" + wildcard accepted as prefix and/or suffix. ## Examples @@ -159,6 +162,15 @@ config { } ``` +- `denied_envvars` - (Optional) Passes a list of environment variables that + the driver should scrub from all task environments. Supports globbing with "*" + wildcard accepted as prefix and/or suffix. + +```hcl +config { + denied_envvars = ["AWS_SECRET_KEY", "*_TOKEN"] +} +``` ## Client Options ~> Note: client configuration options will soon be deprecated. Please use