Merge pull request #237 from hashicorp/f-raw-exec-driver

A raw fork/exec driver that provides no isolation.

client/driver/driver.go

@@ -15,11 +15,12 @@ import (
 // BuiltinDrivers contains the built in registered drivers
 // which are available for allocation handling
 var BuiltinDrivers = map[string]Factory{
-	"docker": NewDockerDriver,
-	"exec":   NewExecDriver,
-	"java":   NewJavaDriver,
-	"qemu":   NewQemuDriver,
-	"rkt":    NewRktDriver,
+	"docker":   NewDockerDriver,
+	"exec":     NewExecDriver,
+	"raw_exec": NewRawExecDriver,
+	"java":     NewJavaDriver,
+	"qemu":     NewQemuDriver,
+	"rkt":      NewRktDriver,
 }
 
 // NewDriver is used to instantiate and return a new driver
@@ -112,7 +113,7 @@ func TaskEnvironmentVariables(ctx *ExecContext, task *structs.Task) environment.
 	env.SetMeta(task.Meta)
 
 	if ctx.AllocDir != nil {
-		env.SetAllocDir(ctx.AllocDir.AllocDir)
+		env.SetAllocDir(ctx.AllocDir.SharedDir)
 	}
 
 	if task.Resources != nil {

client/driver/exec_test.go

@@ -8,7 +8,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/driver/environment"
 	"github.com/hashicorp/nomad/nomad/structs"
@@ -159,7 +158,7 @@ func TestExecDriver_Start_Wait_AllocDir(t *testing.T) {
 	}
 
 	// Check that data was written to the shared alloc directory.
-	outputFile := filepath.Join(ctx.AllocDir.AllocDir, allocdir.SharedAllocName, file)
+	outputFile := filepath.Join(ctx.AllocDir.SharedDir, file)
 	act, err := ioutil.ReadFile(outputFile)
 	if err != nil {
 		t.Fatalf("Couldn't read expected output: %v", err)

client/driver/raw_exec.go

@@ -0,0 +1,201 @@
+package driver
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/hashicorp/nomad/client/allocdir"
+	"github.com/hashicorp/nomad/client/config"
+	"github.com/hashicorp/nomad/client/driver/args"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+const (
+	// The option that enables this driver in the Config.Options map.
+	rawExecConfigOption = "driver.raw_exec.enable"
+
+	// Null files to use as stdin.
+	unixNull    = "/dev/null"
+	windowsNull = "nul"
+)
+
+// The RawExecDriver is a privileged version of the exec driver. It provides no
+// resource isolation and just fork/execs. The Exec driver should be preferred
+// and this should only be used when explicitly needed.
+type RawExecDriver struct {
+	DriverContext
+}
+
+// rawExecHandle is returned from Start/Open as a handle to the PID
+type rawExecHandle struct {
+	proc   *os.Process
+	waitCh chan error
+	doneCh chan struct{}
+}
+
+// NewRawExecDriver is used to create a new raw exec driver
+func NewRawExecDriver(ctx *DriverContext) Driver {
+	return &RawExecDriver{*ctx}
+}
+
+func (d *RawExecDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
+	// Check that the user has explicitly enabled this executor.
+	enabled, err := strconv.ParseBool(cfg.ReadDefault(rawExecConfigOption, "false"))
+	if err != nil {
+		return false, fmt.Errorf("Failed to parse %v option: %v", rawExecConfigOption, err)
+	}
+
+	if enabled {
+		d.logger.Printf("[WARN] driver.raw_exec: raw exec is enabled. Only enable if needed")
+		node.Attributes["driver.raw_exec"] = "1"
+		return true, nil
+	}
+
+	return false, nil
+}
+
+func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
+	// Get the command
+	command, ok := task.Config["command"]
+	if !ok || command == "" {
+		return nil, fmt.Errorf("missing command for raw_exec driver")
+	}
+
+	// Get the tasks local directory.
+	taskName := d.DriverContext.taskName
+	taskDir, ok := ctx.AllocDir.TaskDirs[taskName]
+	if !ok {
+		return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
+	}
+	taskLocal := filepath.Join(taskDir, allocdir.TaskLocal)
+
+	// Get the environment variables.
+	envVars := TaskEnvironmentVariables(ctx, task)
+
+	// Look for arguments
+	var cmdArgs []string
+	if argRaw, ok := task.Config["args"]; ok {
+		parsed, err := args.ParseAndReplace(argRaw, envVars.Map())
+		if err != nil {
+			return nil, err
+		}
+		cmdArgs = append(cmdArgs, parsed...)
+	}
+
+	// Setup the command
+	cmd := exec.Command(command, cmdArgs...)
+	cmd.Dir = taskDir
+	cmd.Env = envVars.List()
+
+	// Capture the stdout/stderr and redirect stdin to /dev/null
+	stdoutFilename := filepath.Join(taskLocal, fmt.Sprintf("%s.stdout", taskName))
+	stderrFilename := filepath.Join(taskLocal, fmt.Sprintf("%s.stderr", taskName))
+	stdinFilename := unixNull
+	if runtime.GOOS == "windows" {
+		stdinFilename = windowsNull
+	}
+
+	stdo, err := os.OpenFile(stdoutFilename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
+	if err != nil {
+		return nil, fmt.Errorf("Error opening file to redirect stdout: %v", err)
+	}
+
+	stde, err := os.OpenFile(stderrFilename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
+	if err != nil {
+		return nil, fmt.Errorf("Error opening file to redirect stderr: %v", err)
+	}
+
+	stdi, err := os.OpenFile(stdinFilename, os.O_CREATE|os.O_RDONLY, 0666)
+	if err != nil {
+		return nil, fmt.Errorf("Error opening file to redirect stdin: %v", err)
+	}
+
+	cmd.Stdout = stdo
+	cmd.Stderr = stde
+	cmd.Stdin = stdi
+
+	if err := cmd.Start(); err != nil {
+		return nil, fmt.Errorf("failed to start command: %v", err)
+	}
+
+	// Return a driver handle
+	h := &rawExecHandle{
+		proc:   cmd.Process,
+		doneCh: make(chan struct{}),
+		waitCh: make(chan error, 1),
+	}
+	go h.run()
+	return h, nil
+}
+
+func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
+	// Split the handle
+	pidStr := strings.TrimPrefix(handleID, "PID:")
+	pid, err := strconv.Atoi(pidStr)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse handle '%s': %v", handleID, err)
+	}
+
+	// Find the process
+	proc, err := os.FindProcess(pid)
+	if proc == nil || err != nil {
+		return nil, fmt.Errorf("failed to find PID %d: %v", pid, err)
+	}
+
+	// Return a driver handle
+	h := &rawExecHandle{
+		proc:   proc,
+		doneCh: make(chan struct{}),
+		waitCh: make(chan error, 1),
+	}
+	go h.run()
+	return h, nil
+}
+
+func (h *rawExecHandle) ID() string {
+	// Return a handle to the PID
+	return fmt.Sprintf("PID:%d", h.proc.Pid)
+}
+
+func (h *rawExecHandle) WaitCh() chan error {
+	return h.waitCh
+}
+
+func (h *rawExecHandle) Update(task *structs.Task) error {
+	// Update is not possible
+	return nil
+}
+
+// Kill is used to terminate the task. We send an Interrupt
+// and then provide a 5 second grace period before doing a Kill on supported
+// OS's, otherwise we kill immediately.
+func (h *rawExecHandle) Kill() error {
+	if runtime.GOOS == "windows" {
+		return h.proc.Kill()
+	}
+
+	h.proc.Signal(os.Interrupt)
+	select {
+	case <-h.doneCh:
+		return nil
+	case <-time.After(5 * time.Second):
+		return h.proc.Kill()
+	}
+}
+
+func (h *rawExecHandle) run() {
+	ps, err := h.proc.Wait()
+	close(h.doneCh)
+	if err != nil {
+		h.waitCh <- err
+	} else if !ps.Success() {
+		h.waitCh <- fmt.Errorf("task exited with error")
+	}
+	close(h.waitCh)
+}

client/driver/raw_exec_test.go

@@ -0,0 +1,216 @@
+package driver
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/nomad/client/config"
+	"github.com/hashicorp/nomad/client/driver/environment"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+func TestRawExecDriver_Fingerprint(t *testing.T) {
+	d := NewRawExecDriver(testDriverContext(""))
+	node := &structs.Node{
+		Attributes: make(map[string]string),
+	}
+
+	// Disable raw exec.
+	cfg := &config.Config{Options: map[string]string{rawExecConfigOption: "false"}}
+
+	apply, err := d.Fingerprint(cfg, node)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if apply {
+		t.Fatalf("should not apply")
+	}
+	if node.Attributes["driver.raw_exec"] != "" {
+		t.Fatalf("driver incorrectly enabled")
+	}
+
+	// Enable raw exec.
+	cfg.Options[rawExecConfigOption] = "true"
+	apply, err = d.Fingerprint(cfg, node)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if !apply {
+		t.Fatalf("should apply")
+	}
+	if node.Attributes["driver.raw_exec"] != "1" {
+		t.Fatalf("driver not enabled")
+	}
+}
+
+func TestRawExecDriver_StartOpen_Wait(t *testing.T) {
+	task := &structs.Task{
+		Name: "sleep",
+		Config: map[string]string{
+			"command": "/bin/sleep",
+			"args":    "2",
+		},
+	}
+	driverCtx := testDriverContext(task.Name)
+	ctx := testDriverExecContext(task, driverCtx)
+	defer ctx.AllocDir.Destroy()
+
+	d := NewRawExecDriver(driverCtx)
+	handle, err := d.Start(ctx, task)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if handle == nil {
+		t.Fatalf("missing handle")
+	}
+
+	// Attempt to open
+	handle2, err := d.Open(ctx, handle.ID())
+	handle2.(*rawExecHandle).waitCh = make(chan error, 1)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if handle2 == nil {
+		t.Fatalf("missing handle")
+	}
+
+	// Task should terminate quickly
+	select {
+	case err := <-handle2.WaitCh():
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+	case <-time.After(3 * time.Second):
+		t.Fatalf("timeout")
+	}
+}
+
+func TestRawExecDriver_Start_Wait(t *testing.T) {
+	task := &structs.Task{
+		Name: "sleep",
+		Config: map[string]string{
+			"command": "/bin/sleep",
+			"args":    "1",
+		},
+	}
+
+	driverCtx := testDriverContext(task.Name)
+	ctx := testDriverExecContext(task, driverCtx)
+	defer ctx.AllocDir.Destroy()
+
+	d := NewRawExecDriver(driverCtx)
+	handle, err := d.Start(ctx, task)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if handle == nil {
+		t.Fatalf("missing handle")
+	}
+
+	// Update should be a no-op
+	err = handle.Update(task)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Task should terminate quickly
+	select {
+	case err := <-handle.WaitCh():
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatalf("timeout")
+	}
+}
+
+func TestRawExecDriver_Start_Wait_AllocDir(t *testing.T) {
+	exp := []byte{'w', 'i', 'n'}
+	file := "output.txt"
+	task := &structs.Task{
+		Name: "sleep",
+		Config: map[string]string{
+			"command": "/bin/bash",
+			"args":    fmt.Sprintf(`-c "sleep 1; echo -n %s > $%s/%s"`, string(exp), environment.AllocDir, file),
+		},
+	}
+
+	driverCtx := testDriverContext(task.Name)
+	ctx := testDriverExecContext(task, driverCtx)
+	defer ctx.AllocDir.Destroy()
+
+	d := NewRawExecDriver(driverCtx)
+	handle, err := d.Start(ctx, task)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if handle == nil {
+		t.Fatalf("missing handle")
+	}
+
+	// Task should terminate quickly
+	select {
+	case err := <-handle.WaitCh():
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatalf("timeout")
+	}
+
+	// Check that data was written to the shared alloc directory.
+	outputFile := filepath.Join(ctx.AllocDir.SharedDir, file)
+	act, err := ioutil.ReadFile(outputFile)
+	if err != nil {
+		t.Fatalf("Couldn't read expected output: %v", err)
+	}
+
+	if !reflect.DeepEqual(act, exp) {
+		t.Fatalf("Command outputted %v; want %v", act, exp)
+	}
+}
+
+func TestRawExecDriver_Start_Kill_Wait(t *testing.T) {
+	task := &structs.Task{
+		Name: "sleep",
+		Config: map[string]string{
+			"command": "/bin/sleep",
+			"args":    "1",
+		},
+	}
+
+	driverCtx := testDriverContext(task.Name)
+	ctx := testDriverExecContext(task, driverCtx)
+	defer ctx.AllocDir.Destroy()
+
+	d := NewRawExecDriver(driverCtx)
+	handle, err := d.Start(ctx, task)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if handle == nil {
+		t.Fatalf("missing handle")
+	}
+
+	go func() {
+		time.Sleep(100 * time.Millisecond)
+		err := handle.Kill()
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+	}()
+
+	// Task should terminate quickly
+	select {
+	case err := <-handle.WaitCh():
+		if err == nil {
+			t.Fatal("should err")
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatalf("timeout")
+	}
+}

website/source/docs/agent/config.html.md

@@ -207,8 +207,8 @@ configured on server nodes.
     option is not required and has no default.
   * <a id="meta">`meta`</a>: This is a key/value mapping of metadata pairs. This
     is a free-form map and can contain any string values.
-  * `options`: This is a key/value mapping of internal configuration for clients,
-    such as for driver configuration.
+  * <a id="options">`options`</a>: This is a key/value mapping of internal
+    configuration for clients, such as for driver configuration.
   * <a id="network_interface">`network_interface`</a>: This is a string to force
     network fingerprinting to use a specific network interface
   * <a id="network_speed">`network_speed`</a>: This is an int that sets the

website/source/docs/drivers/exec.html.md

@@ -6,14 +6,15 @@ description: |-
   The Exec task driver is used to run binaries using OS isolation primitives.
 ---
 
-# Fork/Exec Driver
+# Isolated Fork/Exec Driver
 
 Name: `exec`
 
 The `exec` driver is used to simply execute a particular command for a task.
-This is the simplest driver and is extremely flexible. In particlar, because
-it can invoke any command, it can be used to call scripts or other wrappers
-which provide higher level features.
+However unlike [`raw_exec`](raw_exec.html) it uses the underlying isolation
+primitives of the operating system to limit the tasks access to resources. While
+simple, since the `exec` driver  can invoke any command, it can be used to call
+scripts or other wrappers which provide higher level features.
 
 ## Task Configuration
 

website/source/docs/drivers/raw_exec.html.md

@@ -0,0 +1,47 @@
+---
+layout: "docs"
+page_title: "Drivers: Raw Exec"
+sidebar_current: "docs-drivers-raw-exec"
+description: |-
+  The Raw Exec task driver simply fork/execs and provides no isolation.
+---
+
+# Raw Fork/Exec Driver
+
+Name: `raw_exec`
+
+The `raw_exec` driver is used to execute a command for a task without any
+isolation. Further, the task is started as the same user as the Nomad process.
+As such, it should be used with extreme care and is disabled by default.
+
+## Task Configuration
+
+The `raw_exec` driver supports the following configuration in the job spec:
+
+* `command` - The command to execute. Must be provided.
+
+* `args` - The argument list to the command, space seperated. Optional.
+
+## Client Requirements
+
+The `raw_exec` driver can run on all supported operating systems. It is however
+disabled by default. In order to be enabled, the Nomad client configuration must
+explicitly enable the `raw_exec` driver in the
+[options](../agent/config.html#options) field:
+
+```
+options = {
+    driver.raw_exec.enable = "1"
+}
+```
+
+## Client Attributes
+
+The `raw_exec` driver will set the following client attributes:
+
+* `driver.raw_exec` - This will be set to "1", indicating the
+  driver is available.
+
+## Resource Isolation
+
+The `raw_exec` driver provides no isolation.

website/source/layouts/docs.erb

@@ -49,7 +49,11 @@
                         </li>
 
 						<li<%= sidebar_current("docs-drivers-exec") %>>
-							<a href="/docs/drivers/exec.html">Fork/Exec</a>
+							<a href="/docs/drivers/exec.html">Isolated Fork/Exec</a>
+						</li>
+
+						<li<%= sidebar_current("docs-drivers-raw-exec") %>>
+							<a href="/docs/drivers/raw_exec.html">Raw Fork/Exec</a>
 						</li>
 
 						<li<%= sidebar_current("docs-drivers-java") %>>
@@ -60,6 +64,10 @@
 							<a href="/docs/drivers/qemu.html">Qemu</a>
 						</li>
 
+						<li<%= sidebar_current("docs-drivers-rkt") %>>
+							<a href="/docs/drivers/rkt.html">Rkt</a>
+						</li>
+
 						<li<%= sidebar_current("docs-drivers-custom") %>>
 							<a href="/docs/drivers/custom.html">Custom</a>
 						</li>