diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index 6c9eda48f..935c810a5 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -698,106 +698,6 @@ func TestTaskRunner_TaskEnv_Interpolated(t *testing.T) { assert.Equal(t, "global bar somebody", mockCfg.StdoutString) } -// TestTaskRunner_TaskEnv_Chroot asserts chroot drivers use chroot paths and -// not host paths. -func TestTaskRunner_TaskEnv_Chroot(t *testing.T) { - ctestutil.ExecCompatible(t) - ci.Parallel(t) - - alloc := mock.BatchAlloc() - task := alloc.Job.TaskGroups[0].Tasks[0] - task.Driver = "exec" - task.Config = map[string]interface{}{ - "command": "bash", - "args": []string{"-c", "echo $NOMAD_ALLOC_DIR; " + - "echo $NOMAD_TASK_DIR; " + - "echo $NOMAD_SECRETS_DIR; " + - "echo $PATH; ", - }, - } - - // Expect chroot paths and host $PATH - exp := fmt.Sprintf(`/alloc -/local -/secrets -%s -`, os.Getenv("PATH")) - - conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name) - defer cleanup() - - tr, err := NewTaskRunner(conf) - require.NoError(t, err) - go tr.Run() - defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) - - // Wait for task to exit and kill the task runner to run the stop hooks. - testWaitForTaskToDie(t, tr) - tr.Kill(context.Background(), structs.NewTaskEvent("kill")) - timeout := 15 * time.Second - if testutil.IsCI() { - timeout = 120 * time.Second - } - select { - case <-tr.WaitCh(): - case <-time.After(timeout): - require.Fail(t, "timeout waiting for task to exit") - } - - // Read stdout - p := filepath.Join(conf.TaskDir.LogDir, task.Name+".stdout.0") - stdout, err := ioutil.ReadFile(p) - require.NoError(t, err) - require.Equalf(t, exp, string(stdout), "expected: %s\n\nactual: %s\n", exp, stdout) -} - -// TestTaskRunner_TaskEnv_Image asserts image drivers use chroot paths and -// not host paths. Host env vars should also be excluded. -func TestTaskRunner_TaskEnv_Image(t *testing.T) { - ctestutil.DockerCompatible(t) - ci.Parallel(t) - require := require.New(t) - - alloc := mock.BatchAlloc() - task := alloc.Job.TaskGroups[0].Tasks[0] - task.Driver = "docker" - task.Config = map[string]interface{}{ - "image": "redis:7-alpine", - "network_mode": "none", - "command": "sh", - "args": []string{"-c", "echo $NOMAD_ALLOC_DIR; " + - "echo $NOMAD_TASK_DIR; " + - "echo $NOMAD_SECRETS_DIR; " + - "echo $PATH", - }, - } - - // Expect chroot paths and image specific PATH - exp := `/alloc -/local -/secrets -/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -` - - tr, conf, cleanup := runTestTaskRunner(t, alloc, task.Name) - defer cleanup() - - // Wait for task to exit and kill task runner to run the stop hooks. - testWaitForTaskToDie(t, tr) - tr.Kill(context.Background(), structs.NewTaskEvent("kill")) - select { - case <-tr.WaitCh(): - case <-time.After(15 * time.Second): - require.Fail("timeout waiting for task to exit") - } - - // Read stdout - p := filepath.Join(conf.TaskDir.LogDir, task.Name+".stdout.0") - stdout, err := ioutil.ReadFile(p) - require.NoError(err) - require.Equalf(exp, string(stdout), "expected: %s\n\nactual: %s\n", exp, stdout) -} - // TestTaskRunner_TaskEnv_None asserts raw_exec uses host paths and env vars. func TestTaskRunner_TaskEnv_None(t *testing.T) { ci.Parallel(t) diff --git a/e2e/isolation/chroot_test.go b/e2e/isolation/chroot_test.go new file mode 100644 index 000000000..9f710adc4 --- /dev/null +++ b/e2e/isolation/chroot_test.go @@ -0,0 +1,76 @@ +package isolation + +import ( + "regexp" + "testing" + + "github.com/hashicorp/nomad/e2e/e2eutil" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/shoenig/test/must" +) + +func TestChrootFS(t *testing.T) { + nomad := e2eutil.NomadClient(t) + + e2eutil.WaitForLeader(t, nomad) + e2eutil.WaitForNodesReady(t, nomad, 1) + + t.Run("testTaskEnvChroot", testExecUsesChroot) + t.Run("testTaskImageChroot", testImageUsesChroot) +} + +func testExecUsesChroot(t *testing.T) { + nomad := e2eutil.NomadClient(t) + + jobID := "exec-chroot-" + uuid.Short() + jobIDs := []string{jobID} + t.Cleanup(e2eutil.CleanupJobsAndGC(t, &jobIDs)) + + // start job + e2eutil.RegisterAndWaitForAllocs(t, nomad, "./input/chroot_exec.nomad", jobID, "") + + // get allocation + allocations, err := e2eutil.AllocsForJob(jobID, "") + must.NoError(t, err) + must.Len(t, 1, allocations) + allocID := allocations[0]["ID"] + + // wait for allocation stopped + e2eutil.WaitForAllocsStopped(t, nomad, []string{allocID}) + + // assert log contents + logs, err := e2eutil.AllocLogs(allocID, e2eutil.LogsStdOut) + must.NoError(t, err) + must.RegexMatch(t, regexp.MustCompile(`(?m:^/alloc\b)`), logs) + must.RegexMatch(t, regexp.MustCompile(`(?m:^/local\b)`), logs) + must.RegexMatch(t, regexp.MustCompile(`(?m:^/secrets\b)`), logs) + must.StrContains(t, logs, "/bin") +} + +func testImageUsesChroot(t *testing.T) { + nomad := e2eutil.NomadClient(t) + + jobID := "docker-chroot-" + uuid.Short() + jobIDs := []string{jobID} + t.Cleanup(e2eutil.CleanupJobsAndGC(t, &jobIDs)) + + // start job + e2eutil.RegisterAndWaitForAllocs(t, nomad, "./input/chroot_docker.nomad", jobID, "") + + // get allocation + allocations, err := e2eutil.AllocsForJob(jobID, "") + must.NoError(t, err) + must.Len(t, 1, allocations) + allocID := allocations[0]["ID"] + + // wait for allocation stopped + e2eutil.WaitForAllocsStopped(t, nomad, []string{allocID}) + + // assert log contents + logs, err := e2eutil.AllocLogs(allocID, e2eutil.LogsStdOut) + must.NoError(t, err) + must.RegexMatch(t, regexp.MustCompile(`(?m:^/alloc\b)`), logs) + must.RegexMatch(t, regexp.MustCompile(`(?m:^/local\b)`), logs) + must.RegexMatch(t, regexp.MustCompile(`(?m:^/secrets\b)`), logs) + must.StrContains(t, logs, "/bin") +} diff --git a/e2e/isolation/input/chroot_docker.nomad b/e2e/isolation/input/chroot_docker.nomad new file mode 100644 index 000000000..2a76956d9 --- /dev/null +++ b/e2e/isolation/input/chroot_docker.nomad @@ -0,0 +1,27 @@ +job "chroot_docker" { + datacenters = ["dc1"] + type = "batch" + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "print" { + task "env" { + driver = "docker" + config { + image = "busybox:1" + args = [ + "/bin/sh", + "-c", + "echo $NOMAD_ALLOC_DIR; echo $NOMAD_TASK_DIR; echo $NOMAD_SECRETS_DIR; echo $PATH" + ] + } + resources { + cpu = 50 + memory = 32 + } + } + } +} diff --git a/e2e/isolation/input/chroot_exec.nomad b/e2e/isolation/input/chroot_exec.nomad new file mode 100644 index 000000000..31a471a7e --- /dev/null +++ b/e2e/isolation/input/chroot_exec.nomad @@ -0,0 +1,24 @@ +job "chroot_exec" { + datacenters = ["dc1"] + type = "batch" + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + group "print" { + task "env" { + driver = "exec" + config { + command = "/bin/bash" + args = [ + "-c", + "echo $NOMAD_ALLOC_DIR; echo $NOMAD_TASK_DIR; echo $NOMAD_SECRETS_DIR; echo $PATH" + ] + } + resources { + cpu = 10 + memory = 16 + } + } + } +} diff --git a/e2e/isolation/isolation.go b/e2e/isolation/pids.go similarity index 90% rename from e2e/isolation/isolation.go rename to e2e/isolation/pids.go index 76a71ac2c..d283cb270 100644 --- a/e2e/isolation/isolation.go +++ b/e2e/isolation/pids.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/require" ) -type IsolationTest struct { +type PIDsNamespacing struct { framework.TC jobIDs []string @@ -22,21 +22,21 @@ type IsolationTest struct { func init() { framework.AddSuites(&framework.TestSuite{ - Component: "Isolation", + Component: "PIDS", CanRunLocal: true, Cases: []framework.TestCase{ - new(IsolationTest), + new(PIDsNamespacing), }, }) } -func (tc *IsolationTest) BeforeAll(f *framework.F) { +func (tc *PIDsNamespacing) BeforeAll(f *framework.F) { t := f.T() e2eutil.WaitForLeader(t, tc.Nomad()) e2eutil.WaitForNodesReady(t, tc.Nomad(), 1) } -func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing(f *framework.F) { +func (tc *PIDsNamespacing) TestIsolation_ExecDriver_PIDNamespacing(f *framework.F) { t := f.T() clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad()) @@ -66,7 +66,7 @@ func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing(f *framework.F) require.Contains(t, out, "my pid is 1\n") } -func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing_host(f *framework.F) { +func (tc *PIDsNamespacing) TestIsolation_ExecDriver_PIDNamespacing_host(f *framework.F) { t := f.T() clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad()) @@ -96,7 +96,7 @@ func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing_host(f *framewo require.NotContains(t, out, "my pid is 1\n") } -func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing_AllocExec(f *framework.F) { +func (tc *PIDsNamespacing) TestIsolation_ExecDriver_PIDNamespacing_AllocExec(f *framework.F) { t := f.T() clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad()) @@ -151,7 +151,7 @@ func (tc *IsolationTest) TestIsolation_ExecDriver_PIDNamespacing_AllocExec(f *fr require.Len(t, lines, 3) } -func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing(f *framework.F) { +func (tc *PIDsNamespacing) TestIsolation_JavaDriver_PIDNamespacing(f *framework.F) { t := f.T() clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad()) @@ -181,7 +181,7 @@ func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing(f *framework.F) require.Contains(t, out, "my pid is 1\n") } -func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing_host(f *framework.F) { +func (tc *PIDsNamespacing) TestIsolation_JavaDriver_PIDNamespacing_host(f *framework.F) { t := f.T() clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad()) @@ -211,7 +211,7 @@ func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing_host(f *framewo require.NotContains(t, out, "my pid is 1\n") } -func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing_AllocExec(f *framework.F) { +func (tc *PIDsNamespacing) TestIsolation_JavaDriver_PIDNamespacing_AllocExec(f *framework.F) { t := f.T() clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad()) @@ -266,7 +266,7 @@ func (tc *IsolationTest) TestIsolation_JavaDriver_PIDNamespacing_AllocExec(f *fr require.Len(t, lines, 3) } -func (tc *IsolationTest) TestIsolation_RawExecDriver_NoPIDNamespacing(f *framework.F) { +func (tc *PIDsNamespacing) TestIsolation_RawExecDriver_NoPIDNamespacing(f *framework.F) { t := f.T() clientNodes, err := e2eutil.ListLinuxClientNodes(tc.Nomad())