diff --git a/drivers/shared/executor/executor.go b/drivers/shared/executor/executor.go index 255a49db8..7e625f34d 100644 --- a/drivers/shared/executor/executor.go +++ b/drivers/shared/executor/executor.go @@ -252,7 +252,7 @@ func (e *UniversalExecutor) Version() (*ExecutorVersion, error) { // Launch launches the main process and returns its state. It also // configures an applies isolation on certain platforms. func (e *UniversalExecutor) Launch(command *ExecCommand) (*ProcessState, error) { - e.logger.Debug("launching command", "command", command.Cmd, "args", strings.Join(command.Args, " ")) + e.logger.Trace("preparing to launch command", "command", command.Cmd, "args", strings.Join(command.Args, " ")) e.commandCfg = command @@ -307,6 +307,7 @@ func (e *UniversalExecutor) Launch(command *ExecCommand) (*ProcessState, error) e.childCmd.Env = e.commandCfg.Env // Start the process + e.logger.Debug("launching", "command", command.Cmd, "args", strings.Join(command.Args, " ")) if err := e.childCmd.Start(); err != nil { return nil, fmt.Errorf("failed to start command path=%q --- args=%q: %v", path, e.childCmd.Args, err) } @@ -597,7 +598,8 @@ func (e *UniversalExecutor) handleStats(ch chan *cstructs.TaskResourceUsage, ctx } // lookupBin looks for path to the binary to run by looking for the binary in -// the following locations, in-order: task/local/, task/, based on host $PATH. +// the following locations, in-order: +// task/local/, task/, on the host file system, in host $PATH // The return path is absolute. func lookupBin(taskDir string, bin string) (string, error) { // Check in the local directory diff --git a/drivers/shared/executor/executor_linux.go b/drivers/shared/executor/executor_linux.go index f1a465c16..cdaea5dd8 100644 --- a/drivers/shared/executor/executor_linux.go +++ b/drivers/shared/executor/executor_linux.go @@ -18,6 +18,7 @@ import ( "github.com/hashicorp/consul-template/signals" hclog "github.com/hashicorp/go-hclog" multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/stats" cstructs "github.com/hashicorp/nomad/client/structs" shelpers "github.com/hashicorp/nomad/helper/stats" @@ -99,7 +100,7 @@ func NewExecutorWithIsolation(logger hclog.Logger) Executor { // Launch creates a new container in libcontainer and starts a new process with it func (l *LibcontainerExecutor) Launch(command *ExecCommand) (*ProcessState, error) { - l.logger.Debug("launching command", "command", command.Cmd, "args", strings.Join(command.Args, " ")) + l.logger.Trace("preparing to launch command", "command", command.Cmd, "args", strings.Join(command.Args, " ")) if command.Resources == nil { command.Resources = &drivers.Resources{ @@ -144,7 +145,8 @@ func (l *LibcontainerExecutor) Launch(command *ExecCommand) (*ProcessState, erro l.container = container // Look up the binary path and make it executable - absPath, err := lookupBin(command.TaskDir, command.Cmd) + absPath, err := lookupTaskBin(command) + if err != nil { return nil, err } @@ -155,7 +157,7 @@ func (l *LibcontainerExecutor) Launch(command *ExecCommand) (*ProcessState, erro path := absPath - // Determine the path to run as it may have to be relative to the chroot. + // Ensure that the path is contained in the chroot, and find it relative to the container rel, err := filepath.Rel(command.TaskDir, path) if err != nil { return nil, fmt.Errorf("failed to determine relative path base=%q target=%q: %v", command.TaskDir, path, err) @@ -178,6 +180,8 @@ func (l *LibcontainerExecutor) Launch(command *ExecCommand) (*ProcessState, erro return nil, err } + l.logger.Debug("launching", "command", command.Cmd, "args", strings.Join(command.Args, " ")) + // the task process will be started by the container process := &libcontainer.Process{ Args: combined, @@ -816,3 +820,57 @@ func cmdMounts(mounts []*drivers.MountConfig) []*lconfigs.Mount { return r } + +// lookupTaskBin finds the file `bin` in taskDir/local, taskDir in that order, then performs +// a PATH search inside taskDir. It returns an absolute path. See also executor.lookupBin +func lookupTaskBin(command *ExecCommand) (string, error) { + taskDir := command.TaskDir + bin := command.Cmd + + // Check in the local directory + localDir := filepath.Join(taskDir, allocdir.TaskLocal) + local := filepath.Join(localDir, bin) + if _, err := os.Stat(local); err == nil { + return local, nil + } + + // Check at the root of the task's directory + root := filepath.Join(taskDir, bin) + if _, err := os.Stat(root); err == nil { + return root, nil + } + + if strings.Contains(bin, "/") { + return "", fmt.Errorf("file %s not found under path %s", bin, taskDir) + } + + // Find the PATH + path := "/usr/local/bin:/usr/bin:/bin" + for _, e := range command.Env { + if strings.HasPrefix("PATH=", e) { + path = e[5:] + } + } + + return lookPathIn(path, taskDir, bin) +} + +// lookPathIn looks for a file with PATH inside the directory root. Like exec.LookPath +func lookPathIn(path string, root string, bin string) (string, error) { + // exec.LookPath(file string) + for _, dir := range filepath.SplitList(path) { + if dir == "" { + // match unix shell behavior, empty path element == . + dir = "." + } + path := filepath.Join(root, dir, bin) + f, err := os.Stat(path) + if err != nil { + continue + } + if m := f.Mode(); !m.IsDir() { + return path, nil + } + } + return "", fmt.Errorf("file %s not found under path %s", bin, root) +} diff --git a/drivers/shared/executor/executor_linux_test.go b/drivers/shared/executor/executor_linux_test.go index 727d2ea60..9d2a1e41e 100644 --- a/drivers/shared/executor/executor_linux_test.go +++ b/drivers/shared/executor/executor_linux_test.go @@ -48,6 +48,7 @@ func testExecutorCommandWithChroot(t *testing.T) *testExecCmd { "/lib64": "/lib64", "/usr/lib": "/usr/lib", "/bin/ls": "/bin/ls", + "/bin/cat": "/bin/cat", "/bin/echo": "/bin/echo", "/bin/bash": "/bin/bash", "/bin/sleep": "/bin/sleep", @@ -160,6 +161,84 @@ ld.so.conf.d/` }, func(err error) { t.Error(err) }) } +func TestUniversalExecutor_LookupTaskBin(t *testing.T) { + t.Parallel() + require := require.New(t) + + // Create a temp dir + tmpDir, err := ioutil.TempDir("", "") + require.Nil(err) + defer os.Remove(tmpDir) + + // Create the command + cmd := &ExecCommand{Env: []string{"PATH=/bin"}, TaskDir: tmpDir} + + // Make a foo subdir + os.MkdirAll(filepath.Join(tmpDir, "foo"), 0700) + + // Write a file under foo + filePath := filepath.Join(tmpDir, "foo", "tmp.txt") + err = ioutil.WriteFile(filePath, []byte{1, 2}, os.ModeAppend) + require.NoError(err) + + // Lookout with an absolute path to the binary + cmd.Cmd = "/foo/tmp.txt" + _, err = lookupTaskBin(cmd) + require.NoError(err) + + // Write a file under local subdir + os.MkdirAll(filepath.Join(tmpDir, "local"), 0700) + filePath2 := filepath.Join(tmpDir, "local", "tmp.txt") + ioutil.WriteFile(filePath2, []byte{1, 2}, os.ModeAppend) + + // Lookup with file name, should find the one we wrote above + cmd.Cmd = "tmp.txt" + _, err = lookupTaskBin(cmd) + require.NoError(err) + + // Lookup a host absolute path + cmd.Cmd = "/bin/sh" + _, err = lookupTaskBin(cmd) + require.Error(err) +} + +// Exec Launch looks for the binary only inside the chroot +func TestExecutor_EscapeContainer(t *testing.T) { + t.Parallel() + require := require.New(t) + testutil.ExecCompatible(t) + + testExecCmd := testExecutorCommandWithChroot(t) + execCmd, allocDir := testExecCmd.command, testExecCmd.allocDir + execCmd.Cmd = "/bin/kill" // missing from the chroot container + defer allocDir.Destroy() + + execCmd.ResourceLimits = true + + executor := NewExecutorWithIsolation(testlog.HCLogger(t)) + defer executor.Shutdown("SIGKILL", 0) + + _, err := executor.Launch(execCmd) + require.Error(err) + require.Regexp("^file /bin/kill not found under path", err) + + // Bare files are looked up using the system path, inside the container + allocDir.Destroy() + testExecCmd = testExecutorCommandWithChroot(t) + execCmd, allocDir = testExecCmd.command, testExecCmd.allocDir + execCmd.Cmd = "kill" + _, err = executor.Launch(execCmd) + require.Error(err) + require.Regexp("^file kill not found under path", err) + + allocDir.Destroy() + testExecCmd = testExecutorCommandWithChroot(t) + execCmd, allocDir = testExecCmd.command, testExecCmd.allocDir + execCmd.Cmd = "echo" + _, err = executor.Launch(execCmd) + require.NoError(err) +} + func TestExecutor_ClientCleanup(t *testing.T) { t.Parallel() testutil.ExecCompatible(t) diff --git a/drivers/shared/executor/executor_test.go b/drivers/shared/executor/executor_test.go index 3da4c7b37..65af774a5 100644 --- a/drivers/shared/executor/executor_test.go +++ b/drivers/shared/executor/executor_test.go @@ -401,9 +401,9 @@ func TestUniversalExecutor_MakeExecutable(t *testing.T) { func TestUniversalExecutor_LookupPath(t *testing.T) { t.Parallel() + require := require.New(t) // Create a temp dir tmpDir, err := ioutil.TempDir("", "") - require := require.New(t) require.Nil(err) defer os.Remove(tmpDir) @@ -415,17 +415,13 @@ func TestUniversalExecutor_LookupPath(t *testing.T) { err = ioutil.WriteFile(filePath, []byte{1, 2}, os.ModeAppend) require.Nil(err) - // Lookup with full path to binary - _, err = lookupBin("dummy", filePath) + // Lookup with full path on host to binary + path, err := lookupBin("not_tmpDir", filePath) require.Nil(err) + require.Equal(filePath, path) - // Write a file under local subdir - os.MkdirAll(filepath.Join(tmpDir, "local"), 0700) - filePath2 := filepath.Join(tmpDir, "local", "tmp.txt") - ioutil.WriteFile(filePath2, []byte{1, 2}, os.ModeAppend) - - // Lookup with file name, should find the one we wrote above - _, err = lookupBin(tmpDir, "tmp.txt") + // Lookout with an absolute path to the binary + _, err = lookupBin(tmpDir, "/foo/tmp.txt") require.Nil(err) // Write a file under task dir @@ -433,9 +429,27 @@ func TestUniversalExecutor_LookupPath(t *testing.T) { ioutil.WriteFile(filePath3, []byte{1, 2}, os.ModeAppend) // Lookup with file name, should find the one we wrote above - _, err = lookupBin(tmpDir, "tmp.txt") + path, err = lookupBin(tmpDir, "tmp.txt") require.Nil(err) + require.Equal(filepath.Join(tmpDir, "tmp.txt"), path) + // Write a file under local subdir + os.MkdirAll(filepath.Join(tmpDir, "local"), 0700) + filePath2 := filepath.Join(tmpDir, "local", "tmp.txt") + ioutil.WriteFile(filePath2, []byte{1, 2}, os.ModeAppend) + + // Lookup with file name, should find the one we wrote above + path, err = lookupBin(tmpDir, "tmp.txt") + require.Nil(err) + require.Equal(filepath.Join(tmpDir, "local", "tmp.txt"), path) + + // Lookup a host path + _, err = lookupBin(tmpDir, "/bin/sh") + require.NoError(err) + + // Lookup a host path via $PATH + _, err = lookupBin(tmpDir, "sh") + require.NoError(err) } // setupRoootfs setups the rootfs for libcontainer executor