From 60346ae8ec91d2192d617e8ecb7f2e3b6cdf4cbd Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 8 Oct 2015 11:36:22 -0700 Subject: [PATCH] Actually add the files --- client/driver/raw_exec.go | 197 +++++++++++++++++++++ client/driver/raw_exec_test.go | 216 +++++++++++++++++++++++ website/source/docs/drivers/exec.html.md | 2 +- 3 files changed, 414 insertions(+), 1 deletion(-) create mode 100644 client/driver/raw_exec.go create mode 100644 client/driver/raw_exec_test.go diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go new file mode 100644 index 000000000..ae8e6ab63 --- /dev/null +++ b/client/driver/raw_exec.go @@ -0,0 +1,197 @@ +package driver + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "time" + + "github.com/hashicorp/nomad/client/allocdir" + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/driver/args" + "github.com/hashicorp/nomad/nomad/structs" +) + +const ( + // The option that enables this driver in the Config.Options map. + rawExecConfigOption = "driver.raw_exec.enable" + + // Null files to use as stdin. + unixNull = "/dev/null" + windowsNull = "nul" +) + +// The RawExecDriver is a privileged version of the exec driver. It provides no +// resource isolation and just fork/execs. The Exec driver should be preferred +// and this should only be used when explicitly needed. +type RawExecDriver struct { + DriverContext +} + +// rawExecHandle is returned from Start/Open as a handle to the PID +type rawExecHandle struct { + proc *os.Process + waitCh chan error + doneCh chan struct{} +} + +// NewRawExecDriver is used to create a new raw exec driver +func NewRawExecDriver(ctx *DriverContext) Driver { + return &RawExecDriver{*ctx} +} + +func (d *RawExecDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { + // Check that the user has explicitly enabled this executor. + enabled := strings.ToLower(cfg.ReadDefault(rawExecConfigOption, "false")) + if enabled == "1" || enabled == "true" { + d.logger.Printf("[WARN] driver.raw_exec: raw exec is enabled. Only enable if needed") + node.Attributes["driver.raw_exec"] = "1" + return true, nil + } + + return false, nil +} + +func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { + // Get the command + command, ok := task.Config["command"] + if !ok || command == "" { + return nil, fmt.Errorf("missing command for raw_exec driver") + } + + // Get the tasks local directory. + taskName := d.DriverContext.taskName + taskDir, ok := ctx.AllocDir.TaskDirs[taskName] + if !ok { + return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) + } + taskLocal := filepath.Join(taskDir, allocdir.TaskLocal) + + // Get the environment variables. + envVars := TaskEnvironmentVariables(ctx, task) + + // Look for arguments + var cmdArgs []string + if argRaw, ok := task.Config["args"]; ok { + parsed, err := args.ParseAndReplace(argRaw, envVars.Map()) + if err != nil { + return nil, err + } + cmdArgs = append(cmdArgs, parsed...) + } + + // Setup the command + cmd := exec.Command(command, cmdArgs...) + cmd.Dir = taskDir + cmd.Env = envVars.List() + + // Capture the stdout/stderr and redirect stdin to /dev/null + stdoutFilename := filepath.Join(taskLocal, fmt.Sprintf("%s.stdout", taskName)) + stderrFilename := filepath.Join(taskLocal, fmt.Sprintf("%s.stderr", taskName)) + stdinFilename := unixNull + if runtime.GOOS == "windows" { + stdinFilename = windowsNull + } + + stdo, err := os.OpenFile(stdoutFilename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) + if err != nil { + return nil, fmt.Errorf("Error opening file to redirect stdout: %v", err) + } + + stde, err := os.OpenFile(stderrFilename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) + if err != nil { + return nil, fmt.Errorf("Error opening file to redirect stderr: %v", err) + } + + stdi, err := os.OpenFile(stdinFilename, os.O_CREATE|os.O_RDONLY, 0666) + if err != nil { + return nil, fmt.Errorf("Error opening file to redirect stdin: %v", err) + } + + cmd.Stdout = stdo + cmd.Stderr = stde + cmd.Stdin = stdi + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start command: %v", err) + } + + // Return a driver handle + h := &rawExecHandle{ + proc: cmd.Process, + doneCh: make(chan struct{}), + waitCh: make(chan error, 1), + } + go h.run() + return h, nil +} + +func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) { + // Split the handle + pidStr := strings.TrimPrefix(handleID, "PID:") + pid, err := strconv.Atoi(pidStr) + if err != nil { + return nil, fmt.Errorf("failed to parse handle '%s': %v", handleID, err) + } + + // Find the process + proc, err := os.FindProcess(pid) + if proc == nil || err != nil { + return nil, fmt.Errorf("failed to find PID %d: %v", pid, err) + } + + // Return a driver handle + h := &rawExecHandle{ + proc: proc, + doneCh: make(chan struct{}), + waitCh: make(chan error, 1), + } + go h.run() + return h, nil +} + +func (h *rawExecHandle) ID() string { + // Return a handle to the PID + return fmt.Sprintf("PID:%d", h.proc.Pid) +} + +func (h *rawExecHandle) WaitCh() chan error { + return h.waitCh +} + +func (h *rawExecHandle) Update(task *structs.Task) error { + // Update is not possible + return nil +} + +// Kill is used to terminate the task. We send an Interrupt +// and then provide a 5 second grace period before doing a Kill on supported +// OS's, otherwise we kill immediately. +func (h *rawExecHandle) Kill() error { + if runtime.GOOS == "windows" { + return h.proc.Kill() + } + + h.proc.Signal(os.Interrupt) + select { + case <-h.doneCh: + return nil + case <-time.After(5 * time.Second): + return h.proc.Kill() + } +} + +func (h *rawExecHandle) run() { + ps, err := h.proc.Wait() + close(h.doneCh) + if err != nil { + h.waitCh <- err + } else if !ps.Success() { + h.waitCh <- fmt.Errorf("task exited with error") + } + close(h.waitCh) +} diff --git a/client/driver/raw_exec_test.go b/client/driver/raw_exec_test.go new file mode 100644 index 000000000..8d06a7de8 --- /dev/null +++ b/client/driver/raw_exec_test.go @@ -0,0 +1,216 @@ +package driver + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/driver/environment" + "github.com/hashicorp/nomad/nomad/structs" +) + +func TestRawExecDriver_Fingerprint(t *testing.T) { + d := NewRawExecDriver(testDriverContext("")) + node := &structs.Node{ + Attributes: make(map[string]string), + } + + // Disable raw exec. + cfg := &config.Config{Options: map[string]string{rawExecConfigOption: "false"}} + + apply, err := d.Fingerprint(cfg, node) + if err != nil { + t.Fatalf("err: %v", err) + } + if apply { + t.Fatalf("should not apply") + } + if node.Attributes["driver.raw_exec"] != "" { + t.Fatalf("driver incorrectly enabled") + } + + // Enable raw exec. + cfg.Options[rawExecConfigOption] = "true" + apply, err = d.Fingerprint(cfg, node) + if err != nil { + t.Fatalf("err: %v", err) + } + if !apply { + t.Fatalf("should apply") + } + if node.Attributes["driver.raw_exec"] != "1" { + t.Fatalf("driver not enabled") + } +} + +func TestRawExecDriver_StartOpen_Wait(t *testing.T) { + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "command": "/bin/sleep", + "args": "2", + }, + } + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + + d := NewRawExecDriver(driverCtx) + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + // Attempt to open + handle2, err := d.Open(ctx, handle.ID()) + handle2.(*rawExecHandle).waitCh = make(chan error, 1) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle2 == nil { + t.Fatalf("missing handle") + } + + // Task should terminate quickly + select { + case err := <-handle2.WaitCh(): + if err != nil { + t.Fatalf("err: %v", err) + } + case <-time.After(3 * time.Second): + t.Fatalf("timeout") + } +} + +func TestRawExecDriver_Start_Wait(t *testing.T) { + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "command": "/bin/sleep", + "args": "1", + }, + } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + + d := NewRawExecDriver(driverCtx) + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + // Update should be a no-op + err = handle.Update(task) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Task should terminate quickly + select { + case err := <-handle.WaitCh(): + if err != nil { + t.Fatalf("err: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatalf("timeout") + } +} + +func TestRawExecDriver_Start_Wait_AllocDir(t *testing.T) { + exp := []byte{'w', 'i', 'n'} + file := "output.txt" + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "command": "/bin/bash", + "args": fmt.Sprintf(`-c "sleep 1; echo -n %s > $%s/%s"`, string(exp), environment.AllocDir, file), + }, + } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + + d := NewRawExecDriver(driverCtx) + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + // Task should terminate quickly + select { + case err := <-handle.WaitCh(): + if err != nil { + t.Fatalf("err: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatalf("timeout") + } + + // Check that data was written to the shared alloc directory. + outputFile := filepath.Join(ctx.AllocDir.SharedDir, file) + act, err := ioutil.ReadFile(outputFile) + if err != nil { + t.Fatalf("Couldn't read expected output: %v", err) + } + + if !reflect.DeepEqual(act, exp) { + t.Fatalf("Command outputted %v; want %v", act, exp) + } +} + +func TestRawExecDriver_Start_Kill_Wait(t *testing.T) { + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "command": "/bin/sleep", + "args": "1", + }, + } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + + d := NewRawExecDriver(driverCtx) + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + go func() { + time.Sleep(100 * time.Millisecond) + err := handle.Kill() + if err != nil { + t.Fatalf("err: %v", err) + } + }() + + // Task should terminate quickly + select { + case err := <-handle.WaitCh(): + if err == nil { + t.Fatal("should err") + } + case <-time.After(2 * time.Second): + t.Fatalf("timeout") + } +} diff --git a/website/source/docs/drivers/exec.html.md b/website/source/docs/drivers/exec.html.md index e480f38bd..dd30af74a 100644 --- a/website/source/docs/drivers/exec.html.md +++ b/website/source/docs/drivers/exec.html.md @@ -6,7 +6,7 @@ description: |- The Exec task driver is used to run binaries using OS isolation primitives. --- -# Fork/Exec Driver +# Isolated Fork/Exec Driver Name: `exec`