From 1098e562fcecc2829c7f52fce66c493e800b166a Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 7 Oct 2015 19:00:05 -0700 Subject: [PATCH] Privileged exec driver --- client/driver/driver.go | 3 +- client/driver/exec_test.go | 3 +- client/driver/pexec.go | 197 ++++++++++++++++++++++++++++++++ client/driver/pexec_test.go | 216 ++++++++++++++++++++++++++++++++++++ 4 files changed, 416 insertions(+), 3 deletions(-) create mode 100644 client/driver/pexec.go create mode 100644 client/driver/pexec_test.go diff --git a/client/driver/driver.go b/client/driver/driver.go index 46b18538c..9025b952a 100644 --- a/client/driver/driver.go +++ b/client/driver/driver.go @@ -17,6 +17,7 @@ import ( var BuiltinDrivers = map[string]Factory{ "docker": NewDockerDriver, "exec": NewExecDriver, + "pexec": NewPrivilegedExecDriver, "java": NewJavaDriver, "qemu": NewQemuDriver, "rkt": NewRktDriver, @@ -112,7 +113,7 @@ func TaskEnvironmentVariables(ctx *ExecContext, task *structs.Task) environment. env.SetMeta(task.Meta) if ctx.AllocDir != nil { - env.SetAllocDir(ctx.AllocDir.AllocDir) + env.SetAllocDir(ctx.AllocDir.SharedDir) } if task.Resources != nil { diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go index 4a05d5891..feba89ae3 100644 --- a/client/driver/exec_test.go +++ b/client/driver/exec_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/environment" "github.com/hashicorp/nomad/nomad/structs" @@ -159,7 +158,7 @@ func TestExecDriver_Start_Wait_AllocDir(t *testing.T) { } // Check that data was written to the shared alloc directory. - outputFile := filepath.Join(ctx.AllocDir.AllocDir, allocdir.SharedAllocName, file) + outputFile := filepath.Join(ctx.AllocDir.SharedDir, file) act, err := ioutil.ReadFile(outputFile) if err != nil { t.Fatalf("Couldn't read expected output: %v", err) diff --git a/client/driver/pexec.go b/client/driver/pexec.go new file mode 100644 index 000000000..948b3e538 --- /dev/null +++ b/client/driver/pexec.go @@ -0,0 +1,197 @@ +package driver + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" + + "github.com/hashicorp/nomad/client/allocdir" + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/driver/args" + "github.com/hashicorp/nomad/nomad/structs" +) + +const ( + // The option that enables this driver in the Config.Options map. + pexecConfigOption = "driver.pexec.enable" + + // Null files to use as stdin. + unixNull = "/dev/null" + windowsNull = "nul" +) + +// The PexecDriver is a privileged version of the exec driver. It provides no +// resource isolation and just fork/execs. The Exec driver should be preferred +// and this should only be used when explicitly needed. +type PexecDriver struct { + DriverContext +} + +// pexecHandle is returned from Start/Open as a handle to the PID +type pexecHandle struct { + proc *os.Process + waitCh chan error + doneCh chan struct{} +} + +// NewPrivilegedExecDriver is used to create a new privileged exec driver +func NewPrivilegedExecDriver(ctx *DriverContext) Driver { + return &PexecDriver{*ctx} +} + +func (d *PexecDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { + // Check that the user has explicitly enabled this executor. + enabled := strings.ToLower(cfg.ReadDefault(pexecConfigOption, "false")) + if enabled == "1" || enabled == "true" { + d.logger.Printf("[WARN] driver.Pexec: privileged exec is enabled. Only enable if needed") + node.Attributes["driver.pexec"] = "1" + return true, nil + } + + return false, nil +} + +func (d *PexecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { + // Get the command + command, ok := task.Config["command"] + if !ok || command == "" { + return nil, fmt.Errorf("missing command for pexec driver") + } + + // Get the tasks local directory. + taskName := d.DriverContext.taskName + taskDir, ok := ctx.AllocDir.TaskDirs[taskName] + if !ok { + return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) + } + taskLocal := filepath.Join(taskDir, allocdir.TaskLocal) + + // Get the environment variables. + envVars := TaskEnvironmentVariables(ctx, task) + + // Look for arguments + var cmdArgs []string + if argRaw, ok := task.Config["args"]; ok { + parsed, err := args.ParseAndReplace(argRaw, envVars.Map()) + if err != nil { + return nil, err + } + cmdArgs = append(cmdArgs, parsed...) + } + + // Setup the command + cmd := exec.Command(command, cmdArgs...) + cmd.Dir = taskDir + cmd.Env = envVars.List() + + // Capture the stdout/stderr and redirect stdin to /dev/null + stdoutFilename := filepath.Join(taskLocal, fmt.Sprintf("%s.stdout", taskName)) + stderrFilename := filepath.Join(taskLocal, fmt.Sprintf("%s.stderr", taskName)) + stdinFilename := unixNull + if runtime.GOOS == "windows" { + stdinFilename = windowsNull + } + + stdo, err := os.OpenFile(stdoutFilename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) + if err != nil { + return nil, fmt.Errorf("Error opening file to redirect stdout: %v", err) + } + + stde, err := os.OpenFile(stderrFilename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) + if err != nil { + return nil, fmt.Errorf("Error opening file to redirect stderr: %v", err) + } + + stdi, err := os.OpenFile(stdinFilename, os.O_CREATE|os.O_RDONLY, 0666) + if err != nil { + return nil, fmt.Errorf("Error opening file to redirect stdin: %v", err) + } + + cmd.Stdout = stdo + cmd.Stderr = stde + cmd.Stdin = stdi + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start command: %v", err) + } + + // Return a driver handle + h := &pexecHandle{ + proc: cmd.Process, + doneCh: make(chan struct{}), + waitCh: make(chan error, 1), + } + go h.run() + return h, nil +} + +func (d *PexecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) { + // Split the handle + pidStr := strings.TrimPrefix(handleID, "PID:") + pid, err := strconv.Atoi(pidStr) + if err != nil { + return nil, fmt.Errorf("failed to parse handle '%s': %v", handleID, err) + } + + // Find the process + proc, err := os.FindProcess(pid) + if proc == nil || err != nil { + return nil, fmt.Errorf("failed to find PID %d: %v", pid, err) + } + + // Return a driver handle + h := &pexecHandle{ + proc: proc, + doneCh: make(chan struct{}), + waitCh: make(chan error, 1), + } + go h.run() + return h, nil +} + +func (h *pexecHandle) ID() string { + // Return a handle to the PID + return fmt.Sprintf("PID:%d", h.proc.Pid) +} + +func (h *pexecHandle) WaitCh() chan error { + return h.waitCh +} + +func (h *pexecHandle) Update(task *structs.Task) error { + // Update is not possible + return nil +} + +// Kill is used to terminate the task. We send an Interrupt +// and then provide a 5 second grace period before doing a Kill on supported +// OS's, otherwise we kill immediately. +func (h *pexecHandle) Kill() error { + if runtime.GOOS == "windows" { + return h.proc.Kill() + } + + h.proc.Signal(os.Interrupt) + select { + case <-h.doneCh: + return nil + case <-time.After(5 * time.Second): + return h.proc.Kill() + } +} + +func (h *pexecHandle) run() { + ps, err := h.proc.Wait() + close(h.doneCh) + if err != nil { + h.waitCh <- err + } else if !ps.Success() { + h.waitCh <- fmt.Errorf("task exited with error") + } + close(h.waitCh) +} diff --git a/client/driver/pexec_test.go b/client/driver/pexec_test.go new file mode 100644 index 000000000..719ace028 --- /dev/null +++ b/client/driver/pexec_test.go @@ -0,0 +1,216 @@ +package driver + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/driver/environment" + "github.com/hashicorp/nomad/nomad/structs" +) + +func TestPexecDriver_Fingerprint(t *testing.T) { + d := NewPrivilegedExecDriver(testDriverContext("")) + node := &structs.Node{ + Attributes: make(map[string]string), + } + + // Disable privileged exec. + cfg := &config.Config{Options: map[string]string{pexecConfigOption: "false"}} + + apply, err := d.Fingerprint(cfg, node) + if err != nil { + t.Fatalf("err: %v", err) + } + if apply { + t.Fatalf("should not apply") + } + if node.Attributes["driver.pexec"] != "" { + t.Fatalf("driver incorrectly enabled") + } + + // Enable privileged exec. + cfg.Options[pexecConfigOption] = "true" + apply, err = d.Fingerprint(cfg, node) + if err != nil { + t.Fatalf("err: %v", err) + } + if !apply { + t.Fatalf("should apply") + } + if node.Attributes["driver.pexec"] != "1" { + t.Fatalf("driver not enabled") + } +} + +func TestPexecDriver_StartOpen_Wait(t *testing.T) { + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "command": "/bin/sleep", + "args": "2", + }, + } + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + + d := NewPrivilegedExecDriver(driverCtx) + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + // Attempt to open + handle2, err := d.Open(ctx, handle.ID()) + handle2.(*pexecHandle).waitCh = make(chan error, 1) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle2 == nil { + t.Fatalf("missing handle") + } + + // Task should terminate quickly + select { + case err := <-handle2.WaitCh(): + if err != nil { + t.Fatalf("err: %v", err) + } + case <-time.After(3 * time.Second): + t.Fatalf("timeout") + } +} + +func TestPexecDriver_Start_Wait(t *testing.T) { + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "command": "/bin/sleep", + "args": "1", + }, + } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + + d := NewPrivilegedExecDriver(driverCtx) + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + // Update should be a no-op + err = handle.Update(task) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Task should terminate quickly + select { + case err := <-handle.WaitCh(): + if err != nil { + t.Fatalf("err: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatalf("timeout") + } +} + +func TestPexecDriver_Start_Wait_AllocDir(t *testing.T) { + exp := []byte{'w', 'i', 'n'} + file := "output.txt" + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "command": "/bin/bash", + "args": fmt.Sprintf(`-c "sleep 1; echo -n %s > $%s/%s"`, string(exp), environment.AllocDir, file), + }, + } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + + d := NewPrivilegedExecDriver(driverCtx) + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + // Task should terminate quickly + select { + case err := <-handle.WaitCh(): + if err != nil { + t.Fatalf("err: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatalf("timeout") + } + + // Check that data was written to the shared alloc directory. + outputFile := filepath.Join(ctx.AllocDir.SharedDir, file) + act, err := ioutil.ReadFile(outputFile) + if err != nil { + t.Fatalf("Couldn't read expected output: %v", err) + } + + if !reflect.DeepEqual(act, exp) { + t.Fatalf("Command outputted %v; want %v", act, exp) + } +} + +func TestPexecDriver_Start_Kill_Wait(t *testing.T) { + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "command": "/bin/sleep", + "args": "1", + }, + } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + + d := NewPrivilegedExecDriver(driverCtx) + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + go func() { + time.Sleep(100 * time.Millisecond) + err := handle.Kill() + if err != nil { + t.Fatalf("err: %v", err) + } + }() + + // Task should terminate quickly + select { + case err := <-handle.WaitCh(): + if err == nil { + t.Fatal("should err") + } + case <-time.After(2 * time.Second): + t.Fatalf("timeout") + } +}