From d0f53c858671cc43645117788306b31b495db0eb Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Tue, 2 Feb 2016 13:38:38 -0800
Subject: [PATCH 01/68] Using a plugin to create the executor

---
 client/driver/exec.go                   | 198 +++++++++++++++---------
 client/driver/plugins/executor.go       | 104 +++++++++++++
 client/driver/plugins/executor_linux.go |  35 +++++
 command/executor_plugin.go              |  32 ++++
 4 files changed, 298 insertions(+), 71 deletions(-)
 create mode 100644 client/driver/plugins/executor.go
 create mode 100644 client/driver/plugins/executor_linux.go
 create mode 100644 command/executor_plugin.go

diff --git a/client/driver/exec.go b/client/driver/exec.go
index fb24f021f..aff405a71 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -4,17 +4,20 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
-	"path/filepath"
+	"os/exec"
+	//"path/filepath"
 	"syscall"
 	"time"
 
-	"github.com/hashicorp/nomad/client/allocdir"
+	"github.com/hashicorp/go-plugin"
+	//"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/driver/executor"
+	"github.com/hashicorp/nomad/client/driver/plugins"
 	cstructs "github.com/hashicorp/nomad/client/driver/structs"
-	"github.com/hashicorp/nomad/client/getter"
+	//"github.com/hashicorp/nomad/client/getter"
 	"github.com/hashicorp/nomad/nomad/structs"
-	"github.com/mitchellh/mapstructure"
+	//"github.com/mitchellh/mapstructure"
 )
 
 // ExecDriver fork/execs tasks using as many of the underlying OS's isolation
@@ -32,11 +35,13 @@ type ExecDriverConfig struct {
 
 // execHandle is returned from Start/Open as a handle to the PID
 type execHandle struct {
-	cmd         executor.Executor
-	killTimeout time.Duration
-	logger      *log.Logger
-	waitCh      chan *cstructs.WaitResult
-	doneCh      chan struct{}
+	pluginClient *plugin.Client
+	executor     plugins.Executor
+	cmd          executor.Executor
+	killTimeout  time.Duration
+	logger       *log.Logger
+	waitCh       chan *cstructs.WaitResult
+	doneCh       chan struct{}
 }
 
 // NewExecDriver is used to create a new exec driver
@@ -63,58 +68,82 @@ func (d *ExecDriver) Periodic() (bool, time.Duration) {
 }
 
 func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
-	var driverConfig ExecDriverConfig
-	if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
-		return nil, err
-	}
-	// Get the command to be ran
-	command := driverConfig.Command
-	if command == "" {
-		return nil, fmt.Errorf("missing command for exec driver")
+	//	var driverConfig ExecDriverConfig
+	//	if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
+	//		return nil, err
+	//	}
+	//	// Get the command to be ran
+	//	command := driverConfig.Command
+	//	if command == "" {
+	//		return nil, fmt.Errorf("missing command for exec driver")
+	//	}
+	//
+	//	// Create a location to download the artifact.
+	//	taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
+	//	if !ok {
+	//		return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
+	//	}
+	//
+	//	// Check if an artificat is specified and attempt to download it
+	//	source, ok := task.Config["artifact_source"]
+	//	if ok && source != "" {
+	//		// Proceed to download an artifact to be executed.
+	//		_, err := getter.GetArtifact(
+	//			filepath.Join(taskDir, allocdir.TaskLocal),
+	//			driverConfig.ArtifactSource,
+	//			driverConfig.Checksum,
+	//			d.logger,
+	//		)
+	//		if err != nil {
+	//			return nil, err
+	//		}
+	//	}
+	//
+	//	// Setup the command
+	//	execCtx := executor.NewExecutorContext(d.taskEnv)
+	//	cmd := executor.Command(execCtx, command, driverConfig.Args...)
+	//	if err := cmd.Limit(task.Resources); err != nil {
+	//		return nil, fmt.Errorf("failed to constrain resources: %s", err)
+	//	}
+	//
+	//	// Populate environment variables
+	//	cmd.Command().Env = d.taskEnv.EnvList()
+	//
+	//	if err := cmd.ConfigureTaskDir(d.taskName, ctx.AllocDir); err != nil {
+	//		return nil, fmt.Errorf("failed to configure task directory: %v", err)
+	//	}
+	//
+	//	if err := cmd.Start(); err != nil {
+	//		return nil, fmt.Errorf("failed to start command: %v", err)
+	//	}
+	//
+	executorClient := plugin.NewClient(&plugin.ClientConfig{
+		HandshakeConfig: plugins.HandshakeConfig,
+		Plugins:         plugins.PluginMap,
+		Cmd:             exec.Command("/home/diptanuc/Projects/gocode/bin/nomad"),
+	})
+
+	rpcClient, err := executorClient.Client()
+	if err != nil {
+		return nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
 	}
 
-	// Create a location to download the artifact.
-	taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
-	if !ok {
-		return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
+	raw, err := rpcClient.Dispense("executor")
+	if err != nil {
+		return nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
 	}
-
-	// Check if an artificat is specified and attempt to download it
-	source, ok := task.Config["artifact_source"]
-	if ok && source != "" {
-		// Proceed to download an artifact to be executed.
-		_, err := getter.GetArtifact(
-			filepath.Join(taskDir, allocdir.TaskLocal),
-			driverConfig.ArtifactSource,
-			driverConfig.Checksum,
-			d.logger,
-		)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	// Setup the command
-	execCtx := executor.NewExecutorContext(d.taskEnv)
-	cmd := executor.Command(execCtx, command, driverConfig.Args...)
-	if err := cmd.Limit(task.Resources); err != nil {
-		return nil, fmt.Errorf("failed to constrain resources: %s", err)
-	}
-
-	// Populate environment variables
-	cmd.Command().Env = d.taskEnv.EnvList()
-
-	if err := cmd.ConfigureTaskDir(d.taskName, ctx.AllocDir); err != nil {
-		return nil, fmt.Errorf("failed to configure task directory: %v", err)
-	}
-
-	if err := cmd.Start(); err != nil {
-		return nil, fmt.Errorf("failed to start command: %v", err)
+	executorPlugin := raw.(plugins.Executor)
+	ps, err := executorPlugin.LaunchCmd(exec.Command("/bin/echo", "hello"), &plugins.ExecutorContext{})
+	if err != nil {
+		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
 	}
+	d.logger.Printf("DIPTANU Started process via plugin: %#v", ps)
 
 	// Return a driver handle
 	h := &execHandle{
-		cmd:         cmd,
+		pluginClient: executorClient,
+		executor:     executorPlugin,
+		//cmd:          cmd,
 		killTimeout: d.DriverContext.KillTimeout(task),
 		logger:      d.logger,
 		doneCh:      make(chan struct{}),
@@ -125,8 +154,9 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 }
 
 type execId struct {
-	ExecutorId  string
-	KillTimeout time.Duration
+	//ExecutorId   string
+	KillTimeout  time.Duration
+	PluginConfig *plugin.ReattachConfig
 }
 
 func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
@@ -136,29 +166,54 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	}
 
 	// Find the process
-	execCtx := executor.NewExecutorContext(d.taskEnv)
-	cmd, err := executor.OpenId(execCtx, id.ExecutorId)
+	//	execCtx := executor.NewExecutorContext(d.taskEnv)
+	//	cmd, err := executor.OpenId(execCtx, id.ExecutorId)
+	//	if err != nil {
+	//		return nil, fmt.Errorf("failed to open ID %v: %v", id.ExecutorId, err)
+	//	}
+	pluginConfig := &plugin.ClientConfig{
+		HandshakeConfig: plugins.HandshakeConfig,
+		Plugins:         plugins.PluginMap,
+		Cmd:             exec.Command("/home/diptanuc/Projects/gocode/bin/nomad"),
+		Reattach:        id.PluginConfig,
+	}
+	executor, client, err := d.executor(pluginConfig)
 	if err != nil {
-		return nil, fmt.Errorf("failed to open ID %v: %v", id.ExecutorId, err)
+		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
 
 	// Return a driver handle
 	h := &execHandle{
-		cmd:         cmd,
-		logger:      d.logger,
-		killTimeout: id.KillTimeout,
-		doneCh:      make(chan struct{}),
-		waitCh:      make(chan *cstructs.WaitResult, 1),
+		pluginClient: client,
+		executor:     executor,
+		logger:       d.logger,
+		killTimeout:  id.KillTimeout,
+		doneCh:       make(chan struct{}),
+		waitCh:       make(chan *cstructs.WaitResult, 1),
 	}
 	go h.run()
 	return h, nil
 }
 
+func (d *ExecDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *plugin.Client, error) {
+	executorClient := plugin.NewClient(config)
+	rpcClient, err := executorClient.Client()
+	if err != nil {
+		return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
+	}
+
+	raw, err := rpcClient.Dispense("executor")
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
+	}
+	executorPlugin := raw.(plugins.Executor)
+	return executorPlugin, executorClient, nil
+}
+
 func (h *execHandle) ID() string {
-	executorId, _ := h.cmd.ID()
 	id := execId{
-		ExecutorId:  executorId,
-		KillTimeout: h.killTimeout,
+		KillTimeout:  h.killTimeout,
+		PluginConfig: h.pluginClient.ReattachConfig(),
 	}
 
 	data, err := json.Marshal(id)
@@ -181,18 +236,19 @@ func (h *execHandle) Update(task *structs.Task) error {
 }
 
 func (h *execHandle) Kill() error {
-	h.cmd.Shutdown()
+	h.executor.ShutDown()
 	select {
 	case <-h.doneCh:
 		return nil
 	case <-time.After(h.killTimeout):
-		return h.cmd.ForceStop()
+		_, err := h.executor.Exit()
+		return err
 	}
 }
 
 func (h *execHandle) run() {
-	res := h.cmd.Wait()
+	ps, err := h.executor.Wait()
 	close(h.doneCh)
-	h.waitCh <- res
+	h.waitCh <- &cstructs.WaitResult{ExitCode: ps.ExitCode, Signal: 0, Err: err}
 	close(h.waitCh)
 }
diff --git a/client/driver/plugins/executor.go b/client/driver/plugins/executor.go
new file mode 100644
index 000000000..53aacb938
--- /dev/null
+++ b/client/driver/plugins/executor.go
@@ -0,0 +1,104 @@
+package plugins
+
+import (
+	"net/rpc"
+	"os/exec"
+	"time"
+
+	"github.com/hashicorp/go-plugin"
+)
+
+var HandshakeConfig = plugin.HandshakeConfig{
+	ProtocolVersion: 1,
+}
+
+var PluginMap = map[string]plugin.Plugin{
+	"executor": new(ExecutorPlugin),
+}
+
+type ExecutorContext struct {
+}
+
+type ProcessState struct {
+	Pid      int
+	ExitCode int
+	Time     time.Time
+}
+
+type Executor interface {
+	LaunchCmd(cmd *exec.Cmd, ctx *ExecutorContext) (*ProcessState, error)
+	Wait() (*ProcessState, error)
+	ShutDown() (*ProcessState, error)
+	Exit() (*ProcessState, error)
+}
+
+type ExecutorRPC struct {
+	client *rpc.Client
+}
+
+type LaunchCmdArgs struct {
+	Cmd *exec.Cmd
+	Ctx *ExecutorContext
+}
+
+func (e *ExecutorRPC) LaunchCmd(cmd *exec.Cmd, ctx *ExecutorContext) (*ProcessState, error) {
+	var ps ProcessState
+	err := e.client.Call("Plugin.LaunchCmd", LaunchCmdArgs{Cmd: cmd, Ctx: ctx}, &ps)
+	return &ps, err
+}
+
+func (e *ExecutorRPC) Wait() (*ProcessState, error) {
+	var ps ProcessState
+	err := e.client.Call("Plugin.Wait", new(interface{}), &ps)
+	return &ps, err
+}
+
+func (e *ExecutorRPC) ShutDown() (*ProcessState, error) {
+	var ps ProcessState
+	err := e.client.Call("Plugin.ShutDown", new(interface{}), &ps)
+	return &ps, err
+}
+
+func (e *ExecutorRPC) Exit() (*ProcessState, error) {
+	var ps ProcessState
+	err := e.client.Call("Plugin.Exit", new(interface{}), &ps)
+	return &ps, err
+}
+
+type ExecutorRPCServer struct {
+	Impl Executor
+}
+
+func (e *ExecutorRPCServer) LaunchCmd(args LaunchCmdArgs, ps *ProcessState) error {
+	var err error
+	ps, err = e.Impl.LaunchCmd(args.Cmd, args.Ctx)
+	return err
+}
+
+func (e *ExecutorRPCServer) Wait(args interface{}, ps *ProcessState) error {
+	var err error
+	ps, err = e.Impl.Wait()
+	return err
+}
+
+func (e *ExecutorRPCServer) ShutDown(args interface{}, ps *ProcessState) error {
+	var err error
+	ps, err = e.Impl.ShutDown()
+	return err
+}
+
+func (e *ExecutorRPCServer) Exit(args interface{}, ps *ProcessState) error {
+	var err error
+	ps, err = e.Impl.Exit()
+	return err
+}
+
+type ExecutorPlugin struct{}
+
+func (p *ExecutorPlugin) Server(*plugin.MuxBroker) (interface{}, error) {
+	return &ExecutorRPCServer{Impl: NewExecutor()}, nil
+}
+
+func (p *ExecutorPlugin) Client(b *plugin.MuxBroker, c *rpc.Client) (interface{}, error) {
+	return &ExecutorRPC{client: c}, nil
+}
diff --git a/client/driver/plugins/executor_linux.go b/client/driver/plugins/executor_linux.go
new file mode 100644
index 000000000..53df9b646
--- /dev/null
+++ b/client/driver/plugins/executor_linux.go
@@ -0,0 +1,35 @@
+package plugins
+
+import (
+	"log"
+	"os/exec"
+	"time"
+)
+
+type LinuxExecutor struct {
+	cmd *exec.Cmd
+	ctx *ExecutorContext
+
+	log *log.Logger
+}
+
+func NewExecutor() Executor {
+	return &LinuxExecutor{}
+}
+
+func (e *LinuxExecutor) LaunchCmd(cmd *exec.Cmd, ctx *ExecutorContext) (*ProcessState, error) {
+	return &ProcessState{Pid: 5, ExitCode: -1, Time: time.Now()}, nil
+}
+
+func (e *LinuxExecutor) Wait() (*ProcessState, error) {
+	time.Sleep(5 * time.Second)
+	return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
+}
+
+func (e *LinuxExecutor) Exit() (*ProcessState, error) {
+	return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
+}
+
+func (e *LinuxExecutor) ShutDown() (*ProcessState, error) {
+	return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
+}
diff --git a/command/executor_plugin.go b/command/executor_plugin.go
new file mode 100644
index 000000000..839a3ede3
--- /dev/null
+++ b/command/executor_plugin.go
@@ -0,0 +1,32 @@
+package command
+
+import (
+	"strings"
+
+	"github.com/hashicorp/go-plugin"
+
+	"github.com/hashicorp/nomad/client/driver/plugins"
+)
+
+type ExecutorPlugin struct {
+	Meta
+}
+
+func (e *ExecutorPlugin) Help() string {
+	helpText := `
+	This is a command used by Nomad internally to launch an executor plugin"
+	`
+	return strings.TrimSpace(helpText)
+}
+
+func (e *ExecutorPlugin) Synopsis() string {
+	return "internal - launch an executor plugin"
+}
+
+func (e *ExecutorPlugin) Run(args []string) int {
+	plugin.Serve(&plugin.ServeConfig{
+		HandshakeConfig: plugins.HandshakeConfig,
+		Plugins:         plugins.PluginMap,
+	})
+	return 0
+}

From dbb3570d5934e66f03bfcad984d5ccdee29ffe6a Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Tue, 2 Feb 2016 14:36:11 -0800
Subject: [PATCH 02/68] Fixed the executor command

---
 client/driver/exec.go                   |  9 ++++++-
 client/driver/plugins/executor.go       |  4 +++-
 client/driver/plugins/executor_basic.go | 32 +++++++++++++++++++++++++
 command/executor_plugin.go              |  8 +++----
 commands.go                             |  5 ++++
 5 files changed, 52 insertions(+), 6 deletions(-)
 create mode 100644 client/driver/plugins/executor_basic.go

diff --git a/client/driver/exec.go b/client/driver/exec.go
index aff405a71..ca8059c4f 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -18,6 +18,8 @@ import (
 	//"github.com/hashicorp/nomad/client/getter"
 	"github.com/hashicorp/nomad/nomad/structs"
 	//"github.com/mitchellh/mapstructure"
+
+	"github.com/hashicorp/nomad/helper/discover"
 )
 
 // ExecDriver fork/execs tasks using as many of the underlying OS's isolation
@@ -117,10 +119,15 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 	//		return nil, fmt.Errorf("failed to start command: %v", err)
 	//	}
 	//
+
+	bin, err := discover.NomadExecutable()
+	if err != nil {
+		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
+	}
 	executorClient := plugin.NewClient(&plugin.ClientConfig{
 		HandshakeConfig: plugins.HandshakeConfig,
 		Plugins:         plugins.PluginMap,
-		Cmd:             exec.Command("/home/diptanuc/Projects/gocode/bin/nomad"),
+		Cmd:             exec.Command(bin, "executor"),
 	})
 
 	rpcClient, err := executorClient.Client()
diff --git a/client/driver/plugins/executor.go b/client/driver/plugins/executor.go
index 53aacb938..a8c918036 100644
--- a/client/driver/plugins/executor.go
+++ b/client/driver/plugins/executor.go
@@ -9,7 +9,9 @@ import (
 )
 
 var HandshakeConfig = plugin.HandshakeConfig{
-	ProtocolVersion: 1,
+	ProtocolVersion:  1,
+	MagicCookieKey:   "executor_plugin",
+	MagicCookieValue: "value",
 }
 
 var PluginMap = map[string]plugin.Plugin{
diff --git a/client/driver/plugins/executor_basic.go b/client/driver/plugins/executor_basic.go
new file mode 100644
index 000000000..a2e59373d
--- /dev/null
+++ b/client/driver/plugins/executor_basic.go
@@ -0,0 +1,32 @@
+// +build !linux
+
+package plugins
+
+import (
+	"os/exec"
+	"time"
+)
+
+type BasicExecutor struct {
+}
+
+func NewExecutor() Executor {
+	return &BasicExecutor{}
+}
+
+func (e *BasicExecutor) LaunchCmd(cmd *exec.Cmd, ctx *ExecutorContext) (*ProcessState, error) {
+	return &ProcessState{Pid: 5, ExitCode: -1, Time: time.Now()}, nil
+}
+
+func (e *BasicExecutor) Wait() (*ProcessState, error) {
+	time.Sleep(5 * time.Second)
+	return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
+}
+
+func (e *BasicExecutor) Exit() (*ProcessState, error) {
+	return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
+}
+
+func (e *BasicExecutor) ShutDown() (*ProcessState, error) {
+	return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
+}
diff --git a/command/executor_plugin.go b/command/executor_plugin.go
index 839a3ede3..44f86836a 100644
--- a/command/executor_plugin.go
+++ b/command/executor_plugin.go
@@ -8,22 +8,22 @@ import (
 	"github.com/hashicorp/nomad/client/driver/plugins"
 )
 
-type ExecutorPlugin struct {
+type ExecutorPluginCommand struct {
 	Meta
 }
 
-func (e *ExecutorPlugin) Help() string {
+func (e *ExecutorPluginCommand) Help() string {
 	helpText := `
 	This is a command used by Nomad internally to launch an executor plugin"
 	`
 	return strings.TrimSpace(helpText)
 }
 
-func (e *ExecutorPlugin) Synopsis() string {
+func (e *ExecutorPluginCommand) Synopsis() string {
 	return "internal - launch an executor plugin"
 }
 
-func (e *ExecutorPlugin) Run(args []string) int {
+func (e *ExecutorPluginCommand) Run(args []string) int {
 	plugin.Serve(&plugin.ServeConfig{
 		HandshakeConfig: plugins.HandshakeConfig,
 		Plugins:         plugins.PluginMap,
diff --git a/commands.go b/commands.go
index 1b69883c8..9f5fedb12 100644
--- a/commands.go
+++ b/commands.go
@@ -57,6 +57,11 @@ func Commands(metaPtr *command.Meta) map[string]cli.CommandFactory {
 				Meta: meta,
 			}, nil
 		},
+		"executor": func() (cli.Command, error) {
+			return &command.ExecutorPluginCommand{
+				Meta: meta,
+			}, nil
+		},
 		"fs ls": func() (cli.Command, error) {
 			return &command.FSListCommand{
 				Meta: meta,

From b53cf69448390857d4e71026d6a59c38952c2522 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Tue, 2 Feb 2016 18:54:04 -0800
Subject: [PATCH 03/68] Using the plugin to spawn processes from the raw_exec
 driver

---
 client/driver/env/env.go                | 132 ++++++++++++------------
 client/driver/exec.go                   | 124 ++++++++++------------
 client/driver/plugins/executor.go       |  47 ++++++---
 client/driver/plugins/executor_basic.go |  84 +++++++++++++--
 client/driver/plugins/executor_linux.go |  27 +++--
 client/driver/raw_exec.go               | 127 +++++++++++++++--------
 6 files changed, 330 insertions(+), 211 deletions(-)

diff --git a/client/driver/env/env.go b/client/driver/env/env.go
index 1846403ac..6df5fe64f 100644
--- a/client/driver/env/env.go
+++ b/client/driver/env/env.go
@@ -55,26 +55,26 @@ const (
 // TaskEnvironment is used to expose information to a task via environment
 // variables and provide interpolation of Nomad variables.
 type TaskEnvironment struct {
-	env      map[string]string
-	meta     map[string]string
-	allocDir string
-	taskDir  string
-	cpuLimit int
-	memLimit int
-	node     *structs.Node
-	networks []*structs.NetworkResource
-	portMap  map[string]int
+	Env      map[string]string
+	Meta     map[string]string
+	AllocDir string
+	TaskDir  string
+	CpuLimit int
+	MemLimit int
+	Node     *structs.Node
+	Networks []*structs.NetworkResource
+	PortMap  map[string]int
 
 	// taskEnv is the variables that will be set in the tasks environment
-	taskEnv map[string]string
+	TaskEnv map[string]string
 
 	// nodeValues is the values that are allowed for interprolation from the
 	// node.
-	nodeValues map[string]string
+	NodeValues map[string]string
 }
 
 func NewTaskEnvironment(node *structs.Node) *TaskEnvironment {
-	return &TaskEnvironment{node: node}
+	return &TaskEnvironment{Node: node}
 }
 
 // ParseAndReplace takes the user supplied args replaces any instance of an
@@ -82,7 +82,7 @@ func NewTaskEnvironment(node *structs.Node) *TaskEnvironment {
 func (t *TaskEnvironment) ParseAndReplace(args []string) []string {
 	replaced := make([]string, len(args))
 	for i, arg := range args {
-		replaced[i] = hargs.ReplaceEnv(arg, t.taskEnv, t.nodeValues)
+		replaced[i] = hargs.ReplaceEnv(arg, t.TaskEnv, t.NodeValues)
 	}
 
 	return replaced
@@ -92,75 +92,75 @@ func (t *TaskEnvironment) ParseAndReplace(args []string) []string {
 // and nomad variables.  If the variable is found in the passed map it is
 // replaced, otherwise the original string is returned.
 func (t *TaskEnvironment) ReplaceEnv(arg string) string {
-	return hargs.ReplaceEnv(arg, t.taskEnv, t.nodeValues)
+	return hargs.ReplaceEnv(arg, t.TaskEnv, t.NodeValues)
 }
 
 // Build must be called after all the tasks environment values have been set.
 func (t *TaskEnvironment) Build() *TaskEnvironment {
-	t.nodeValues = make(map[string]string)
-	t.taskEnv = make(map[string]string)
+	t.NodeValues = make(map[string]string)
+	t.TaskEnv = make(map[string]string)
 
 	// Build the task metadata
-	for k, v := range t.meta {
-		t.taskEnv[fmt.Sprintf("%s%s", MetaPrefix, strings.ToUpper(k))] = v
+	for k, v := range t.Meta {
+		t.TaskEnv[fmt.Sprintf("%s%s", MetaPrefix, strings.ToUpper(k))] = v
 	}
 
 	// Build the ports
-	for _, network := range t.networks {
-		for label, value := range network.MapLabelToValues(t.portMap) {
+	for _, network := range t.Networks {
+		for label, value := range network.MapLabelToValues(t.PortMap) {
 			IPPort := fmt.Sprintf("%s:%d", network.IP, value)
-			t.taskEnv[fmt.Sprintf("%s%s", AddrPrefix, label)] = IPPort
+			t.TaskEnv[fmt.Sprintf("%s%s", AddrPrefix, label)] = IPPort
 
 			// Pass an explicit port mapping to the environment
-			if port, ok := t.portMap[label]; ok {
-				t.taskEnv[fmt.Sprintf("%s%s", HostPortPrefix, label)] = strconv.Itoa(port)
+			if port, ok := t.PortMap[label]; ok {
+				t.TaskEnv[fmt.Sprintf("%s%s", HostPortPrefix, label)] = strconv.Itoa(port)
 			}
 		}
 	}
 
 	// Build the directories
-	if t.allocDir != "" {
-		t.taskEnv[AllocDir] = t.allocDir
+	if t.AllocDir != "" {
+		t.TaskEnv[AllocDir] = t.AllocDir
 	}
-	if t.taskDir != "" {
-		t.taskEnv[TaskLocalDir] = t.taskDir
+	if t.TaskDir != "" {
+		t.TaskEnv[TaskLocalDir] = t.TaskDir
 	}
 
 	// Build the resource limits
-	if t.memLimit != 0 {
-		t.taskEnv[MemLimit] = strconv.Itoa(t.memLimit)
+	if t.MemLimit != 0 {
+		t.TaskEnv[MemLimit] = strconv.Itoa(t.MemLimit)
 	}
-	if t.cpuLimit != 0 {
-		t.taskEnv[CpuLimit] = strconv.Itoa(t.cpuLimit)
+	if t.CpuLimit != 0 {
+		t.TaskEnv[CpuLimit] = strconv.Itoa(t.CpuLimit)
 	}
 
 	// Build the node
-	if t.node != nil {
+	if t.Node != nil {
 		// Set up the node values.
-		t.nodeValues[nodeIdKey] = t.node.ID
-		t.nodeValues[nodeDcKey] = t.node.Datacenter
-		t.nodeValues[nodeNameKey] = t.node.Name
-		t.nodeValues[nodeClassKey] = t.node.NodeClass
+		t.NodeValues[nodeIdKey] = t.Node.ID
+		t.NodeValues[nodeDcKey] = t.Node.Datacenter
+		t.NodeValues[nodeNameKey] = t.Node.Name
+		t.NodeValues[nodeClassKey] = t.Node.NodeClass
 
 		// Set up the attributes.
-		for k, v := range t.node.Attributes {
-			t.nodeValues[fmt.Sprintf("%s%s", nodeAttributePrefix, k)] = v
+		for k, v := range t.Node.Attributes {
+			t.NodeValues[fmt.Sprintf("%s%s", nodeAttributePrefix, k)] = v
 		}
 
 		// Set up the meta.
-		for k, v := range t.node.Meta {
-			t.nodeValues[fmt.Sprintf("%s%s", nodeMetaPrefix, k)] = v
+		for k, v := range t.Node.Meta {
+			t.NodeValues[fmt.Sprintf("%s%s", nodeMetaPrefix, k)] = v
 		}
 	}
 
 	// Interpret the environment variables
-	interpreted := make(map[string]string, len(t.env))
-	for k, v := range t.env {
-		interpreted[k] = hargs.ReplaceEnv(v, t.nodeValues, t.taskEnv)
+	interpreted := make(map[string]string, len(t.Env))
+	for k, v := range t.Env {
+		interpreted[k] = hargs.ReplaceEnv(v, t.NodeValues, t.TaskEnv)
 	}
 
 	for k, v := range interpreted {
-		t.taskEnv[k] = v
+		t.TaskEnv[k] = v
 	}
 
 	return t
@@ -169,7 +169,7 @@ func (t *TaskEnvironment) Build() *TaskEnvironment {
 // EnvList returns a list of strings with NAME=value pairs.
 func (t *TaskEnvironment) EnvList() []string {
 	env := []string{}
-	for k, v := range t.taskEnv {
+	for k, v := range t.TaskEnv {
 		env = append(env, fmt.Sprintf("%s=%s", k, v))
 	}
 
@@ -178,8 +178,8 @@ func (t *TaskEnvironment) EnvList() []string {
 
 // EnvMap returns a copy of the tasks environment variables.
 func (t *TaskEnvironment) EnvMap() map[string]string {
-	m := make(map[string]string, len(t.taskEnv))
-	for k, v := range t.taskEnv {
+	m := make(map[string]string, len(t.TaskEnv))
+	for k, v := range t.TaskEnv {
 		m[k] = v
 	}
 
@@ -188,95 +188,95 @@ func (t *TaskEnvironment) EnvMap() map[string]string {
 
 // Builder methods to build the TaskEnvironment
 func (t *TaskEnvironment) SetAllocDir(dir string) *TaskEnvironment {
-	t.allocDir = dir
+	t.AllocDir = dir
 	return t
 }
 
 func (t *TaskEnvironment) ClearAllocDir() *TaskEnvironment {
-	t.allocDir = ""
+	t.AllocDir = ""
 	return t
 }
 
 func (t *TaskEnvironment) SetTaskLocalDir(dir string) *TaskEnvironment {
-	t.taskDir = dir
+	t.TaskDir = dir
 	return t
 }
 
 func (t *TaskEnvironment) ClearTaskLocalDir() *TaskEnvironment {
-	t.taskDir = ""
+	t.TaskDir = ""
 	return t
 }
 
 func (t *TaskEnvironment) SetMemLimit(limit int) *TaskEnvironment {
-	t.memLimit = limit
+	t.MemLimit = limit
 	return t
 }
 
 func (t *TaskEnvironment) ClearMemLimit() *TaskEnvironment {
-	t.memLimit = 0
+	t.MemLimit = 0
 	return t
 }
 
 func (t *TaskEnvironment) SetCpuLimit(limit int) *TaskEnvironment {
-	t.cpuLimit = limit
+	t.CpuLimit = limit
 	return t
 }
 
 func (t *TaskEnvironment) ClearCpuLimit() *TaskEnvironment {
-	t.cpuLimit = 0
+	t.CpuLimit = 0
 	return t
 }
 
 func (t *TaskEnvironment) SetNetworks(networks []*structs.NetworkResource) *TaskEnvironment {
-	t.networks = networks
+	t.Networks = networks
 	return t
 }
 
 func (t *TaskEnvironment) clearNetworks() *TaskEnvironment {
-	t.networks = nil
+	t.Networks = nil
 	return t
 }
 
 func (t *TaskEnvironment) SetPortMap(portMap map[string]int) *TaskEnvironment {
-	t.portMap = portMap
+	t.PortMap = portMap
 	return t
 }
 
 func (t *TaskEnvironment) clearPortMap() *TaskEnvironment {
-	t.portMap = nil
+	t.PortMap = nil
 	return t
 }
 
 // Takes a map of meta values to be passed to the task. The keys are capatilized
 // when the environent variable is set.
 func (t *TaskEnvironment) SetMeta(m map[string]string) *TaskEnvironment {
-	t.meta = m
+	t.Meta = m
 	return t
 }
 
 func (t *TaskEnvironment) ClearMeta() *TaskEnvironment {
-	t.meta = nil
+	t.Meta = nil
 	return t
 }
 
 func (t *TaskEnvironment) SetEnvvars(m map[string]string) *TaskEnvironment {
-	t.env = m
+	t.Env = m
 	return t
 }
 
 // Appends the given environment variables.
 func (t *TaskEnvironment) AppendEnvvars(m map[string]string) *TaskEnvironment {
-	if t.env == nil {
-		t.env = make(map[string]string, len(m))
+	if t.Env == nil {
+		t.Env = make(map[string]string, len(m))
 	}
 
 	for k, v := range m {
-		t.env[k] = v
+		t.Env[k] = v
 	}
 	return t
 }
 
 func (t *TaskEnvironment) ClearEnvvars() *TaskEnvironment {
-	t.env = nil
+	t.Env = nil
 	return t
 }
diff --git a/client/driver/exec.go b/client/driver/exec.go
index ca8059c4f..e80134c3e 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -5,21 +5,20 @@ import (
 	"fmt"
 	"log"
 	"os/exec"
-	//"path/filepath"
+	"path/filepath"
 	"syscall"
 	"time"
 
 	"github.com/hashicorp/go-plugin"
-	//"github.com/hashicorp/nomad/client/allocdir"
+	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/driver/executor"
 	"github.com/hashicorp/nomad/client/driver/plugins"
 	cstructs "github.com/hashicorp/nomad/client/driver/structs"
-	//"github.com/hashicorp/nomad/client/getter"
-	"github.com/hashicorp/nomad/nomad/structs"
-	//"github.com/mitchellh/mapstructure"
-
+	"github.com/hashicorp/nomad/client/getter"
 	"github.com/hashicorp/nomad/helper/discover"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/mitchellh/mapstructure"
 )
 
 // ExecDriver fork/execs tasks using as many of the underlying OS's isolation
@@ -70,77 +69,57 @@ func (d *ExecDriver) Periodic() (bool, time.Duration) {
 }
 
 func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
-	//	var driverConfig ExecDriverConfig
-	//	if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
-	//		return nil, err
-	//	}
-	//	// Get the command to be ran
-	//	command := driverConfig.Command
-	//	if command == "" {
-	//		return nil, fmt.Errorf("missing command for exec driver")
-	//	}
-	//
-	//	// Create a location to download the artifact.
-	//	taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
-	//	if !ok {
-	//		return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
-	//	}
-	//
-	//	// Check if an artificat is specified and attempt to download it
-	//	source, ok := task.Config["artifact_source"]
-	//	if ok && source != "" {
-	//		// Proceed to download an artifact to be executed.
-	//		_, err := getter.GetArtifact(
-	//			filepath.Join(taskDir, allocdir.TaskLocal),
-	//			driverConfig.ArtifactSource,
-	//			driverConfig.Checksum,
-	//			d.logger,
-	//		)
-	//		if err != nil {
-	//			return nil, err
-	//		}
-	//	}
-	//
-	//	// Setup the command
-	//	execCtx := executor.NewExecutorContext(d.taskEnv)
-	//	cmd := executor.Command(execCtx, command, driverConfig.Args...)
-	//	if err := cmd.Limit(task.Resources); err != nil {
-	//		return nil, fmt.Errorf("failed to constrain resources: %s", err)
-	//	}
-	//
-	//	// Populate environment variables
-	//	cmd.Command().Env = d.taskEnv.EnvList()
-	//
-	//	if err := cmd.ConfigureTaskDir(d.taskName, ctx.AllocDir); err != nil {
-	//		return nil, fmt.Errorf("failed to configure task directory: %v", err)
-	//	}
-	//
-	//	if err := cmd.Start(); err != nil {
-	//		return nil, fmt.Errorf("failed to start command: %v", err)
-	//	}
-	//
+	var driverConfig ExecDriverConfig
+	if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
+		return nil, err
+	}
+	// Get the command to be ran
+	command := driverConfig.Command
+	if command == "" {
+		return nil, fmt.Errorf("missing command for exec driver")
+	}
+
+	// Create a location to download the artifact.
+	taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
+	if !ok {
+		return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
+	}
+
+	// Check if an artificat is specified and attempt to download it
+	source, ok := task.Config["artifact_source"]
+	if ok && source != "" {
+		// Proceed to download an artifact to be executed.
+		_, err := getter.GetArtifact(
+			filepath.Join(taskDir, allocdir.TaskLocal),
+			driverConfig.ArtifactSource,
+			driverConfig.Checksum,
+			d.logger,
+		)
+		if err != nil {
+			return nil, err
+		}
+	}
 
 	bin, err := discover.NomadExecutable()
 	if err != nil {
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
-	executorClient := plugin.NewClient(&plugin.ClientConfig{
+	pluginConfig := &plugin.ClientConfig{
 		HandshakeConfig: plugins.HandshakeConfig,
 		Plugins:         plugins.PluginMap,
 		Cmd:             exec.Command(bin, "executor"),
-	})
-
-	rpcClient, err := executorClient.Client()
-	if err != nil {
-		return nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
 	}
 
-	raw, err := rpcClient.Dispense("executor")
+	executor, pluginClient, err := d.executor(pluginConfig)
 	if err != nil {
-		return nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
+		return nil, err
 	}
-	executorPlugin := raw.(plugins.Executor)
-	ps, err := executorPlugin.LaunchCmd(exec.Command("/bin/echo", "hello"), &plugins.ExecutorContext{})
+	executorCtx := &plugins.ExecutorContext{
+		TaskEnv:  d.taskEnv,
+		AllocDir: ctx.AllocDir,
+		Task:     task,
+	}
+	ps, err := executor.LaunchCmd(&plugins.ExecCommand{Cmd: command, Args: driverConfig.Args}, executorCtx)
 	if err != nil {
 		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
 	}
@@ -148,8 +127,8 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 
 	// Return a driver handle
 	h := &execHandle{
-		pluginClient: executorClient,
-		executor:     executorPlugin,
+		pluginClient: pluginClient,
+		executor:     executor,
 		//cmd:          cmd,
 		killTimeout: d.DriverContext.KillTimeout(task),
 		logger:      d.logger,
@@ -178,10 +157,16 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	//	if err != nil {
 	//		return nil, fmt.Errorf("failed to open ID %v: %v", id.ExecutorId, err)
 	//	}
+
+	bin, err := discover.NomadExecutable()
+	if err != nil {
+		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
+	}
+
 	pluginConfig := &plugin.ClientConfig{
 		HandshakeConfig: plugins.HandshakeConfig,
 		Plugins:         plugins.PluginMap,
-		Cmd:             exec.Command("/home/diptanuc/Projects/gocode/bin/nomad"),
+		Cmd:             exec.Command(bin, "executor"),
 		Reattach:        id.PluginConfig,
 	}
 	executor, client, err := d.executor(pluginConfig)
@@ -208,6 +193,7 @@ func (d *ExecDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *p
 	if err != nil {
 		return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
 	}
+	rpcClient.SyncStreams(d.config.LogOutput, d.config.LogOutput)
 
 	raw, err := rpcClient.Dispense("executor")
 	if err != nil {
@@ -248,7 +234,7 @@ func (h *execHandle) Kill() error {
 	case <-h.doneCh:
 		return nil
 	case <-time.After(h.killTimeout):
-		_, err := h.executor.Exit()
+		err := h.executor.Exit()
 		return err
 	}
 }
diff --git a/client/driver/plugins/executor.go b/client/driver/plugins/executor.go
index a8c918036..10530f6e9 100644
--- a/client/driver/plugins/executor.go
+++ b/client/driver/plugins/executor.go
@@ -1,11 +1,16 @@
 package plugins
 
 import (
+	"log"
 	"net/rpc"
-	"os/exec"
+	"os"
 	"time"
 
 	"github.com/hashicorp/go-plugin"
+
+	"github.com/hashicorp/nomad/client/allocdir"
+	"github.com/hashicorp/nomad/client/driver/env"
+	"github.com/hashicorp/nomad/nomad/structs"
 )
 
 var HandshakeConfig = plugin.HandshakeConfig{
@@ -19,6 +24,16 @@ var PluginMap = map[string]plugin.Plugin{
 }
 
 type ExecutorContext struct {
+	TaskEnv  *env.TaskEnvironment
+	AllocDir *allocdir.AllocDir
+	Task     *structs.Task
+	Chroot   bool
+	Limits   bool
+}
+
+type ExecCommand struct {
+	Cmd  string
+	Args []string
 }
 
 type ProcessState struct {
@@ -28,10 +43,10 @@ type ProcessState struct {
 }
 
 type Executor interface {
-	LaunchCmd(cmd *exec.Cmd, ctx *ExecutorContext) (*ProcessState, error)
+	LaunchCmd(cmd *ExecCommand, ctx *ExecutorContext) (*ProcessState, error)
 	Wait() (*ProcessState, error)
-	ShutDown() (*ProcessState, error)
-	Exit() (*ProcessState, error)
+	ShutDown() error
+	Exit() error
 }
 
 type ExecutorRPC struct {
@@ -39,11 +54,11 @@ type ExecutorRPC struct {
 }
 
 type LaunchCmdArgs struct {
-	Cmd *exec.Cmd
+	Cmd *ExecCommand
 	Ctx *ExecutorContext
 }
 
-func (e *ExecutorRPC) LaunchCmd(cmd *exec.Cmd, ctx *ExecutorContext) (*ProcessState, error) {
+func (e *ExecutorRPC) LaunchCmd(cmd *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
 	var ps ProcessState
 	err := e.client.Call("Plugin.LaunchCmd", LaunchCmdArgs{Cmd: cmd, Ctx: ctx}, &ps)
 	return &ps, err
@@ -55,16 +70,16 @@ func (e *ExecutorRPC) Wait() (*ProcessState, error) {
 	return &ps, err
 }
 
-func (e *ExecutorRPC) ShutDown() (*ProcessState, error) {
+func (e *ExecutorRPC) ShutDown() error {
 	var ps ProcessState
 	err := e.client.Call("Plugin.ShutDown", new(interface{}), &ps)
-	return &ps, err
+	return err
 }
 
-func (e *ExecutorRPC) Exit() (*ProcessState, error) {
+func (e *ExecutorRPC) Exit() error {
 	var ps ProcessState
 	err := e.client.Call("Plugin.Exit", new(interface{}), &ps)
-	return &ps, err
+	return err
 }
 
 type ExecutorRPCServer struct {
@@ -85,22 +100,26 @@ func (e *ExecutorRPCServer) Wait(args interface{}, ps *ProcessState) error {
 
 func (e *ExecutorRPCServer) ShutDown(args interface{}, ps *ProcessState) error {
 	var err error
-	ps, err = e.Impl.ShutDown()
+	err = e.Impl.ShutDown()
 	return err
 }
 
 func (e *ExecutorRPCServer) Exit(args interface{}, ps *ProcessState) error {
 	var err error
-	ps, err = e.Impl.Exit()
+	err = e.Impl.Exit()
 	return err
 }
 
-type ExecutorPlugin struct{}
+type ExecutorPlugin struct {
+	logger *log.Logger
+}
 
 func (p *ExecutorPlugin) Server(*plugin.MuxBroker) (interface{}, error) {
-	return &ExecutorRPCServer{Impl: NewExecutor()}, nil
+	p.logger = log.New(os.Stdout, "executor-plugin-server:", log.LstdFlags)
+	return &ExecutorRPCServer{Impl: NewExecutor(p.logger)}, nil
 }
 
 func (p *ExecutorPlugin) Client(b *plugin.MuxBroker, c *rpc.Client) (interface{}, error) {
+	p.logger = log.New(os.Stdout, "executor-plugin-client:", log.LstdFlags)
 	return &ExecutorRPC{client: c}, nil
 }
diff --git a/client/driver/plugins/executor_basic.go b/client/driver/plugins/executor_basic.go
index a2e59373d..67a4b4fb8 100644
--- a/client/driver/plugins/executor_basic.go
+++ b/client/driver/plugins/executor_basic.go
@@ -3,30 +3,96 @@
 package plugins
 
 import (
+	"fmt"
+	"log"
+	"os"
 	"os/exec"
+	"path/filepath"
+	"runtime"
+	"syscall"
 	"time"
+
+	"github.com/hashicorp/nomad/client/allocdir"
 )
 
 type BasicExecutor struct {
+	logger *log.Logger
+	cmd    exec.Cmd
 }
 
-func NewExecutor() Executor {
-	return &BasicExecutor{}
+func NewExecutor(logger *log.Logger) Executor {
+	return &BasicExecutor{logger: logger}
 }
 
-func (e *BasicExecutor) LaunchCmd(cmd *exec.Cmd, ctx *ExecutorContext) (*ProcessState, error) {
+func (e *BasicExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
+	e.cmd.Path = command.Cmd
+	e.cmd.Args = append([]string{command.Cmd}, command.Args...)
+	e.cmd.Path = ctx.TaskEnv.ReplaceEnv(e.cmd.Path)
+	e.cmd.Args = ctx.TaskEnv.ParseAndReplace(e.cmd.Args)
+
+	if filepath.Base(command.Cmd) == command.Cmd {
+		if lp, err := exec.LookPath(command.Cmd); err != nil {
+		} else {
+			e.cmd.Path = lp
+		}
+	}
+	e.configureTaskDir(ctx.Task.Name, ctx.AllocDir)
+	e.cmd.Env = ctx.TaskEnv.EnvList()
+	stdoPath := filepath.Join(e.cmd.Dir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", ctx.Task.Name))
+	stdo, err := os.OpenFile(stdoPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
+	if err != nil {
+		return nil, err
+	}
+	e.cmd.Stdout = stdo
+
+	stdePath := filepath.Join(e.cmd.Dir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", ctx.Task.Name))
+	stde, err := os.OpenFile(stdePath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
+	if err != nil {
+		return nil, err
+	}
+	e.cmd.Stderr = stde
+	if err := e.cmd.Start(); err != nil {
+		return nil, err
+	}
+
 	return &ProcessState{Pid: 5, ExitCode: -1, Time: time.Now()}, nil
 }
 
 func (e *BasicExecutor) Wait() (*ProcessState, error) {
-	time.Sleep(5 * time.Second)
-	return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
+	err := e.cmd.Wait()
+	exitCode := 1
+	if exitErr, ok := err.(*exec.ExitError); ok {
+		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
+			exitCode = status.ExitStatus()
+		}
+	}
+	return &ProcessState{Pid: 0, ExitCode: exitCode, Time: time.Now()}, nil
 }
 
-func (e *BasicExecutor) Exit() (*ProcessState, error) {
-	return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
+func (e *BasicExecutor) Exit() error {
+	proc, err := os.FindProcess(e.cmd.Process.Pid)
+	if err != nil {
+		return fmt.Errorf("failied to find user process %v: %v", e.cmd.Process.Pid, err)
+	}
+	return proc.Kill()
 }
 
-func (e *BasicExecutor) ShutDown() (*ProcessState, error) {
-	return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
+func (e *BasicExecutor) ShutDown() error {
+	proc, err := os.FindProcess(e.cmd.Process.Pid)
+	if err != nil {
+		return err
+	}
+	if runtime.GOOS == "windows" {
+		return proc.Kill()
+	}
+	return proc.Signal(os.Interrupt)
+}
+
+func (e *BasicExecutor) configureTaskDir(taskName string, allocDir *allocdir.AllocDir) error {
+	taskDir, ok := allocDir.TaskDirs[taskName]
+	if !ok {
+		return fmt.Errorf("Couldn't find task directory for task %v", taskName)
+	}
+	e.cmd.Dir = taskDir
+	return nil
 }
diff --git a/client/driver/plugins/executor_linux.go b/client/driver/plugins/executor_linux.go
index 53df9b646..3ad2acc3c 100644
--- a/client/driver/plugins/executor_linux.go
+++ b/client/driver/plugins/executor_linux.go
@@ -1,23 +1,34 @@
 package plugins
 
 import (
+	"fmt"
 	"log"
 	"os/exec"
+	"path/filepath"
 	"time"
 )
 
 type LinuxExecutor struct {
-	cmd *exec.Cmd
 	ctx *ExecutorContext
 
-	log *log.Logger
+	logger *log.Logger
 }
 
-func NewExecutor() Executor {
-	return &LinuxExecutor{}
+func NewExecutor(logger *log.Logger) Executor {
+	return &LinuxExecutor{logger: logger}
 }
 
-func (e *LinuxExecutor) LaunchCmd(cmd *exec.Cmd, ctx *ExecutorContext) (*ProcessState, error) {
+func (e *LinuxExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
+	var cmd exec.Cmd
+	cmd.Path = command.Cmd
+	cmd.Args = append([]string{name}, args...)
+	if filepath.Base(command.Cmd) == command.Cmd {
+		if lp, err := exec.LookPath(command.Cmd); err != nil {
+		} else {
+			cmd.Path = lp
+		}
+	}
+	cmd.Env = ctx.TaskEnv.EnvList()
 	return &ProcessState{Pid: 5, ExitCode: -1, Time: time.Now()}, nil
 }
 
@@ -26,10 +37,10 @@ func (e *LinuxExecutor) Wait() (*ProcessState, error) {
 	return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
 }
 
-func (e *LinuxExecutor) Exit() (*ProcessState, error) {
-	return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
+func (e *LinuxExecutor) Exit() error {
+	return nil
 }
 
 func (e *LinuxExecutor) ShutDown() (*ProcessState, error) {
-	return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
+	return nil
 }
diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go
index 5c870e80f..502d2fc37 100644
--- a/client/driver/raw_exec.go
+++ b/client/driver/raw_exec.go
@@ -4,15 +4,18 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"os/exec"
 	"path/filepath"
 	"time"
 
+	"github.com/hashicorp/go-plugin"
 	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
-	"github.com/hashicorp/nomad/client/driver/executor"
+	"github.com/hashicorp/nomad/client/driver/plugins"
 	cstructs "github.com/hashicorp/nomad/client/driver/structs"
 	"github.com/hashicorp/nomad/client/fingerprint"
 	"github.com/hashicorp/nomad/client/getter"
+	"github.com/hashicorp/nomad/helper/discover"
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/mitchellh/mapstructure"
 )
@@ -32,11 +35,12 @@ type RawExecDriver struct {
 
 // rawExecHandle is returned from Start/Open as a handle to the PID
 type rawExecHandle struct {
-	cmd         executor.Executor
-	killTimeout time.Duration
-	logger      *log.Logger
-	waitCh      chan *cstructs.WaitResult
-	doneCh      chan struct{}
+	pluginClient *plugin.Client
+	executor     plugins.Executor
+	killTimeout  time.Duration
+	logger       *log.Logger
+	waitCh       chan *cstructs.WaitResult
+	doneCh       chan struct{}
 }
 
 // NewRawExecDriver is used to create a new raw exec driver
@@ -90,40 +94,62 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 		}
 	}
 
-	// Setup the command
-	execCtx := executor.NewExecutorContext(d.taskEnv)
-	cmd := executor.NewBasicExecutor(execCtx)
-	executor.SetCommand(cmd, command, driverConfig.Args)
-	if err := cmd.Limit(task.Resources); err != nil {
-		return nil, fmt.Errorf("failed to constrain resources: %s", err)
+	bin, err := discover.NomadExecutable()
+	if err != nil {
+		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
+	}
+	pluginConfig := &plugin.ClientConfig{
+		HandshakeConfig: plugins.HandshakeConfig,
+		Plugins:         plugins.PluginMap,
+		Cmd:             exec.Command(bin, "executor"),
 	}
 
-	// Populate environment variables
-	cmd.Command().Env = d.taskEnv.EnvList()
-
-	if err := cmd.ConfigureTaskDir(d.taskName, ctx.AllocDir); err != nil {
-		return nil, fmt.Errorf("failed to configure task directory: %v", err)
+	executor, pluginClient, err := d.executor(pluginConfig)
+	if err != nil {
+		return nil, err
 	}
-
-	if err := cmd.Start(); err != nil {
-		return nil, fmt.Errorf("failed to start command: %v", err)
+	executorCtx := &plugins.ExecutorContext{
+		TaskEnv:  d.taskEnv,
+		AllocDir: ctx.AllocDir,
+		Task:     task,
 	}
+	ps, err := executor.LaunchCmd(&plugins.ExecCommand{Cmd: command, Args: driverConfig.Args}, executorCtx)
+	if err != nil {
+		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
+	}
+	d.logger.Printf("DIPTANU Started process via plugin: %#v", ps)
 
 	// Return a driver handle
-	h := &execHandle{
-		cmd:         cmd,
-		killTimeout: d.DriverContext.KillTimeout(task),
-		logger:      d.logger,
-		doneCh:      make(chan struct{}),
-		waitCh:      make(chan *cstructs.WaitResult, 1),
+	h := &rawExecHandle{
+		pluginClient: pluginClient,
+		executor:     executor,
+		killTimeout:  d.DriverContext.KillTimeout(task),
+		logger:       d.logger,
+		doneCh:       make(chan struct{}),
+		waitCh:       make(chan *cstructs.WaitResult, 1),
 	}
 	go h.run()
 	return h, nil
 }
+func (d *RawExecDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *plugin.Client, error) {
+	executorClient := plugin.NewClient(config)
+	rpcClient, err := executorClient.Client()
+	if err != nil {
+		return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
+	}
+	rpcClient.SyncStreams(d.config.LogOutput, d.config.LogOutput)
+
+	raw, err := rpcClient.Dispense("executor")
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
+	}
+	executorPlugin := raw.(plugins.Executor)
+	return executorPlugin, executorClient, nil
+}
 
 type rawExecId struct {
-	ExecutorId  string
-	KillTimeout time.Duration
+	KillTimeout  time.Duration
+	PluginConfig *plugin.ReattachConfig
 }
 
 func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
@@ -132,30 +158,39 @@ func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, e
 		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
 	}
 
-	// Find the process
-	execCtx := executor.NewExecutorContext(d.taskEnv)
-	cmd := executor.NewBasicExecutor(execCtx)
-	if err := cmd.Open(id.ExecutorId); err != nil {
-		return nil, fmt.Errorf("failed to open ID %v: %v", id.ExecutorId, err)
+	bin, err := discover.NomadExecutable()
+	if err != nil {
+		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
+	}
+
+	pluginConfig := &plugin.ClientConfig{
+		HandshakeConfig: plugins.HandshakeConfig,
+		Plugins:         plugins.PluginMap,
+		Cmd:             exec.Command(bin, "executor"),
+		Reattach:        id.PluginConfig,
+	}
+	executor, client, err := d.executor(pluginConfig)
+	if err != nil {
+		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
 
 	// Return a driver handle
 	h := &execHandle{
-		cmd:         cmd,
-		logger:      d.logger,
-		killTimeout: id.KillTimeout,
-		doneCh:      make(chan struct{}),
-		waitCh:      make(chan *cstructs.WaitResult, 1),
+		pluginClient: client,
+		executor:     executor,
+		logger:       d.logger,
+		killTimeout:  id.KillTimeout,
+		doneCh:       make(chan struct{}),
+		waitCh:       make(chan *cstructs.WaitResult, 1),
 	}
 	go h.run()
 	return h, nil
 }
 
 func (h *rawExecHandle) ID() string {
-	executorId, _ := h.cmd.ID()
 	id := rawExecId{
-		ExecutorId:  executorId,
-		KillTimeout: h.killTimeout,
+		KillTimeout:  h.killTimeout,
+		PluginConfig: h.pluginClient.ReattachConfig(),
 	}
 
 	data, err := json.Marshal(id)
@@ -178,18 +213,20 @@ func (h *rawExecHandle) Update(task *structs.Task) error {
 }
 
 func (h *rawExecHandle) Kill() error {
-	h.cmd.Shutdown()
+	h.executor.ShutDown()
 	select {
 	case <-h.doneCh:
 		return nil
 	case <-time.After(h.killTimeout):
-		return h.cmd.ForceStop()
+		err := h.executor.Exit()
+		return err
 	}
 }
 
 func (h *rawExecHandle) run() {
-	res := h.cmd.Wait()
+	ps, err := h.executor.Wait()
 	close(h.doneCh)
-	h.waitCh <- res
+	h.waitCh <- &cstructs.WaitResult{ExitCode: ps.ExitCode, Signal: 0, Err: err}
 	close(h.waitCh)
+	h.pluginClient.Kill()
 }

From c8a17597db130a1c78b20150474b853c4b2c0a98 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Wed, 3 Feb 2016 10:23:00 -0800
Subject: [PATCH 04/68] Implemented the exec functioanlity for linux

---
 client/driver/plugins/executor_basic.go |   3 +
 client/driver/plugins/executor_linux.go | 244 ++++++++++++++++++++++--
 2 files changed, 236 insertions(+), 11 deletions(-)

diff --git a/client/driver/plugins/executor_basic.go b/client/driver/plugins/executor_basic.go
index 67a4b4fb8..9cd6a508b 100644
--- a/client/driver/plugins/executor_basic.go
+++ b/client/driver/plugins/executor_basic.go
@@ -60,6 +60,9 @@ func (e *BasicExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*
 
 func (e *BasicExecutor) Wait() (*ProcessState, error) {
 	err := e.cmd.Wait()
+	if err == nil {
+		return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
+	}
 	exitCode := 1
 	if exitErr, ok := err.(*exec.ExitError); ok {
 		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
diff --git a/client/driver/plugins/executor_linux.go b/client/driver/plugins/executor_linux.go
index 3ad2acc3c..9019b8504 100644
--- a/client/driver/plugins/executor_linux.go
+++ b/client/driver/plugins/executor_linux.go
@@ -3,15 +3,49 @@ package plugins
 import (
 	"fmt"
 	"log"
+	"os"
 	"os/exec"
+	"os/user"
 	"path/filepath"
+	"runtime"
+	"strconv"
+	"sync"
+	"syscall"
 	"time"
+
+	"github.com/hashicorp/go-multierror"
+	//"github.com/opencontainers/runc/libcontainer/cgroups"
+	//cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
+	//"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
+	//cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
+
+	"github.com/hashicorp/nomad/client/allocdir"
+)
+
+var (
+	// A mapping of directories on the host OS to attempt to embed inside each
+	// task's chroot.
+	chrootEnv = map[string]string{
+		"/bin":       "/bin",
+		"/etc":       "/etc",
+		"/lib":       "/lib",
+		"/lib32":     "/lib32",
+		"/lib64":     "/lib64",
+		"/usr/bin":   "/usr/bin",
+		"/usr/lib":   "/usr/lib",
+		"/usr/share": "/usr/share",
+	}
 )
 
 type LinuxExecutor struct {
+	cmd exec.Cmd
 	ctx *ExecutorContext
 
+	//groups  *cgroupConfig.Cgroup
+	taskDir string
+
 	logger *log.Logger
+	lock   sync.Mutex
 }
 
 func NewExecutor(logger *log.Logger) Executor {
@@ -19,28 +53,216 @@ func NewExecutor(logger *log.Logger) Executor {
 }
 
 func (e *LinuxExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
-	var cmd exec.Cmd
-	cmd.Path = command.Cmd
-	cmd.Args = append([]string{name}, args...)
+	e.ctx = ctx
+	e.cmd.Path = command.Cmd
+	e.cmd.Args = append([]string{command.Cmd}, command.Args...)
 	if filepath.Base(command.Cmd) == command.Cmd {
 		if lp, err := exec.LookPath(command.Cmd); err != nil {
 		} else {
-			cmd.Path = lp
+			e.cmd.Path = lp
 		}
 	}
-	cmd.Env = ctx.TaskEnv.EnvList()
-	return &ProcessState{Pid: 5, ExitCode: -1, Time: time.Now()}, nil
+	if err := e.configureTaskDir(); err != nil {
+		return nil, err
+	}
+	if err := e.runAs("nobody"); err != nil {
+		return nil, err
+	}
+	e.cmd.Env = ctx.TaskEnv.EnvList()
+
+	stdoPath := filepath.Join(e.cmd.Dir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", ctx.Task.Name))
+	stdo, err := os.OpenFile(stdoPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
+	if err != nil {
+		return nil, err
+	}
+	e.cmd.Stdout = stdo
+
+	stdePath := filepath.Join(e.cmd.Dir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", ctx.Task.Name))
+	stde, err := os.OpenFile(stdePath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
+	if err != nil {
+		return nil, err
+	}
+	e.cmd.Stderr = stde
+
+	e.configureChroot()
+
+	if err := e.cmd.Start(); err != nil {
+		return nil, fmt.Errorf("error starting command: %v", err)
+	}
+
+	return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, Time: time.Now()}, nil
+}
+
+// ConfigureTaskDir creates the necessary directory structure for a proper
+// chroot. cleanTaskDir should be called after.
+func (e *LinuxExecutor) configureTaskDir() error {
+	taskName := e.ctx.Task.Name
+	allocDir := e.ctx.AllocDir
+	taskDir, ok := allocDir.TaskDirs[taskName]
+	if !ok {
+		fmt.Errorf("Couldn't find task directory for task %v", taskName)
+	}
+	e.taskDir = taskDir
+
+	if err := allocDir.MountSharedDir(taskName); err != nil {
+		return err
+	}
+
+	if err := allocDir.Embed(taskName, chrootEnv); err != nil {
+		return err
+	}
+
+	// Mount dev
+	dev := filepath.Join(taskDir, "dev")
+	if !e.pathExists(dev) {
+		if err := os.Mkdir(dev, 0777); err != nil {
+			return fmt.Errorf("Mkdir(%v) failed: %v", dev, err)
+		}
+
+		if err := syscall.Mount("none", dev, "devtmpfs", syscall.MS_RDONLY, ""); err != nil {
+			return fmt.Errorf("Couldn't mount /dev to %v: %v", dev, err)
+		}
+	}
+
+	// Mount proc
+	proc := filepath.Join(taskDir, "proc")
+	if !e.pathExists(proc) {
+		if err := os.Mkdir(proc, 0777); err != nil {
+			return fmt.Errorf("Mkdir(%v) failed: %v", proc, err)
+		}
+
+		if err := syscall.Mount("none", proc, "proc", syscall.MS_RDONLY, ""); err != nil {
+			return fmt.Errorf("Couldn't mount /proc to %v: %v", proc, err)
+		}
+	}
+
+	// Set the tasks AllocDir environment variable.
+	e.ctx.TaskEnv.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)).SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)).Build()
+	return nil
+}
+
+// runAs takes a user id as a string and looks up the user, and sets the command
+// to execute as that user.
+func (e *LinuxExecutor) runAs(userid string) error {
+	u, err := user.Lookup(userid)
+	if err != nil {
+		return fmt.Errorf("Failed to identify user %v: %v", userid, err)
+	}
+
+	// Convert the uid and gid
+	uid, err := strconv.ParseUint(u.Uid, 10, 32)
+	if err != nil {
+		return fmt.Errorf("Unable to convert userid to uint32: %s", err)
+	}
+	gid, err := strconv.ParseUint(u.Gid, 10, 32)
+	if err != nil {
+		return fmt.Errorf("Unable to convert groupid to uint32: %s", err)
+	}
+
+	// Set the command to run as that user and group.
+	if e.cmd.SysProcAttr == nil {
+		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
+	}
+	if e.cmd.SysProcAttr.Credential == nil {
+		e.cmd.SysProcAttr.Credential = &syscall.Credential{}
+	}
+	e.cmd.SysProcAttr.Credential.Uid = uint32(uid)
+	e.cmd.SysProcAttr.Credential.Gid = uint32(gid)
+
+	return nil
+}
+
+// pathExists is a helper function to check if the path exists.
+func (e *LinuxExecutor) pathExists(path string) bool {
+	if _, err := os.Stat(path); err != nil {
+		if os.IsNotExist(err) {
+			return false
+		}
+	}
+	return true
+}
+
+// configureChroot enters the user command into a chroot if specified in the
+// config and on an OS that supports Chroots.
+func (e *LinuxExecutor) configureChroot() {
+	if !e.ctx.Chroot {
+		return
+	}
+	if e.cmd.SysProcAttr == nil {
+		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
+	}
+
+	e.cmd.SysProcAttr.Chroot = e.taskDir
+	e.cmd.Dir = "/"
+}
+
+// cleanTaskDir is an idempotent operation to clean the task directory and
+// should be called when tearing down the task.
+func (e *LinuxExecutor) cleanTaskDir() error {
+	// Prevent a race between Wait/ForceStop
+	e.lock.Lock()
+	defer e.lock.Unlock()
+
+	// Unmount dev.
+	errs := new(multierror.Error)
+	dev := filepath.Join(e.taskDir, "dev")
+	if e.pathExists(dev) {
+		if err := syscall.Unmount(dev, 0); err != nil {
+			errs = multierror.Append(errs, fmt.Errorf("Failed to unmount dev (%v): %v", dev, err))
+		}
+
+		if err := os.RemoveAll(dev); err != nil {
+			errs = multierror.Append(errs, fmt.Errorf("Failed to delete dev directory (%v): %v", dev, err))
+		}
+	}
+
+	// Unmount
+	// proc.
+	proc := filepath.Join(e.taskDir, "proc")
+	if e.pathExists(proc) {
+		if err := syscall.Unmount(proc, 0); err != nil {
+			errs = multierror.Append(errs, fmt.Errorf("Failed to unmount proc (%v): %v", proc, err))
+		}
+
+		if err := os.RemoveAll(proc); err != nil {
+			errs = multierror.Append(errs, fmt.Errorf("Failed to delete proc directory (%v): %v", dev, err))
+		}
+	}
+
+	return errs.ErrorOrNil()
 }
 
 func (e *LinuxExecutor) Wait() (*ProcessState, error) {
-	time.Sleep(5 * time.Second)
-	return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
+	err := e.cmd.Wait()
+	if err == nil {
+		return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
+	}
+	exitCode := 1
+	if exitErr, ok := err.(*exec.ExitError); ok {
+		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
+			exitCode = status.ExitStatus()
+		}
+	}
+	e.cleanTaskDir()
+	return &ProcessState{Pid: 0, ExitCode: exitCode, Time: time.Now()}, nil
 }
 
 func (e *LinuxExecutor) Exit() error {
-	return nil
+	proc, err := os.FindProcess(e.cmd.Process.Pid)
+	if err != nil {
+		return fmt.Errorf("failied to find user process %v: %v", e.cmd.Process.Pid, err)
+	}
+	e.cleanTaskDir()
+	return proc.Kill()
 }
 
-func (e *LinuxExecutor) ShutDown() (*ProcessState, error) {
-	return nil
+func (e *LinuxExecutor) ShutDown() error {
+	proc, err := os.FindProcess(e.cmd.Process.Pid)
+	if err != nil {
+		return err
+	}
+	if runtime.GOOS == "windows" {
+		return proc.Kill()
+	}
+	return proc.Signal(os.Interrupt)
 }

From 81d46cd1574683c81d085b9ec2438bc94ee574d5 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Wed, 3 Feb 2016 11:23:22 -0800
Subject: [PATCH 05/68] Fixed the path to the log files

---
 client/driver/plugins/executor_basic.go | 7 +++++--
 client/driver/plugins/executor_linux.go | 4 ++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/client/driver/plugins/executor_basic.go b/client/driver/plugins/executor_basic.go
index 9cd6a508b..a3777ea3a 100644
--- a/client/driver/plugins/executor_basic.go
+++ b/client/driver/plugins/executor_basic.go
@@ -18,6 +18,8 @@ import (
 type BasicExecutor struct {
 	logger *log.Logger
 	cmd    exec.Cmd
+
+	taskDir string
 }
 
 func NewExecutor(logger *log.Logger) Executor {
@@ -38,14 +40,14 @@ func (e *BasicExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*
 	}
 	e.configureTaskDir(ctx.Task.Name, ctx.AllocDir)
 	e.cmd.Env = ctx.TaskEnv.EnvList()
-	stdoPath := filepath.Join(e.cmd.Dir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", ctx.Task.Name))
+	stdoPath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", ctx.Task.Name))
 	stdo, err := os.OpenFile(stdoPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
 	if err != nil {
 		return nil, err
 	}
 	e.cmd.Stdout = stdo
 
-	stdePath := filepath.Join(e.cmd.Dir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", ctx.Task.Name))
+	stdePath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", ctx.Task.Name))
 	stde, err := os.OpenFile(stdePath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
 	if err != nil {
 		return nil, err
@@ -93,6 +95,7 @@ func (e *BasicExecutor) ShutDown() error {
 
 func (e *BasicExecutor) configureTaskDir(taskName string, allocDir *allocdir.AllocDir) error {
 	taskDir, ok := allocDir.TaskDirs[taskName]
+	e.taskDir = taskDir
 	if !ok {
 		return fmt.Errorf("Couldn't find task directory for task %v", taskName)
 	}
diff --git a/client/driver/plugins/executor_linux.go b/client/driver/plugins/executor_linux.go
index 9019b8504..776d0af12 100644
--- a/client/driver/plugins/executor_linux.go
+++ b/client/driver/plugins/executor_linux.go
@@ -70,14 +70,14 @@ func (e *LinuxExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*
 	}
 	e.cmd.Env = ctx.TaskEnv.EnvList()
 
-	stdoPath := filepath.Join(e.cmd.Dir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", ctx.Task.Name))
+	stdoPath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", ctx.Task.Name))
 	stdo, err := os.OpenFile(stdoPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
 	if err != nil {
 		return nil, err
 	}
 	e.cmd.Stdout = stdo
 
-	stdePath := filepath.Join(e.cmd.Dir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", ctx.Task.Name))
+	stdePath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", ctx.Task.Name))
 	stde, err := os.OpenFile(stdePath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
 	if err != nil {
 		return nil, err

From cca3f5f1bcd36f737ce5a323e08dd82ca671fa50 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Wed, 3 Feb 2016 11:41:49 -0800
Subject: [PATCH 06/68] Limiting resources on a process

---
 client/driver/plugins/executor_linux.go | 116 +++++++++++++++++++++++-
 1 file changed, 111 insertions(+), 5 deletions(-)

diff --git a/client/driver/plugins/executor_linux.go b/client/driver/plugins/executor_linux.go
index 776d0af12..152397667 100644
--- a/client/driver/plugins/executor_linux.go
+++ b/client/driver/plugins/executor_linux.go
@@ -14,12 +14,13 @@ import (
 	"time"
 
 	"github.com/hashicorp/go-multierror"
-	//"github.com/opencontainers/runc/libcontainer/cgroups"
-	//cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
-	//"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
-	//cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
+	"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
+	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
 
 	"github.com/hashicorp/nomad/client/allocdir"
+	"github.com/hashicorp/nomad/nomad/structs"
 )
 
 var (
@@ -41,7 +42,7 @@ type LinuxExecutor struct {
 	cmd exec.Cmd
 	ctx *ExecutorContext
 
-	//groups  *cgroupConfig.Cgroup
+	groups  *cgroupConfig.Cgroup
 	taskDir string
 
 	logger *log.Logger
@@ -86,10 +87,23 @@ func (e *LinuxExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*
 
 	e.configureChroot()
 
+	if err := e.configureCgroups(e.ctx.Task.Resources); err != nil {
+		return nil, fmt.Errorf("error creating cgroups: %v", err)
+	}
+
 	if err := e.cmd.Start(); err != nil {
 		return nil, fmt.Errorf("error starting command: %v", err)
 	}
 
+	manager := e.getCgroupManager(e.groups)
+	if err := manager.Apply(e.cmd.Process.Pid); err != nil {
+		e.logger.Printf("[ERROR] unable to join cgroup: %v", err)
+		if err := e.Exit(); err != nil {
+			e.logger.Printf("[ERROR] unable to kill process: %v", err)
+		}
+		return nil, err
+	}
+
 	return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, Time: time.Now()}, nil
 }
 
@@ -244,15 +258,18 @@ func (e *LinuxExecutor) Wait() (*ProcessState, error) {
 		}
 	}
 	e.cleanTaskDir()
+	e.destroyCgroup()
 	return &ProcessState{Pid: 0, ExitCode: exitCode, Time: time.Now()}, nil
 }
 
 func (e *LinuxExecutor) Exit() error {
+	e.logger.Printf("[INFO] Exiting plugin for task %q", e.ctx.Task.Name)
 	proc, err := os.FindProcess(e.cmd.Process.Pid)
 	if err != nil {
 		return fmt.Errorf("failied to find user process %v: %v", e.cmd.Process.Pid, err)
 	}
 	e.cleanTaskDir()
+	e.destroyCgroup()
 	return proc.Kill()
 }
 
@@ -266,3 +283,92 @@ func (e *LinuxExecutor) ShutDown() error {
 	}
 	return proc.Signal(os.Interrupt)
 }
+
+// configureCgroups converts a Nomad Resources specification into the equivalent
+// cgroup configuration. It returns an error if the resources are invalid.
+func (e *LinuxExecutor) configureCgroups(resources *structs.Resources) error {
+	e.groups = &cgroupConfig.Cgroup{}
+	e.groups.Resources = &cgroupConfig.Resources{}
+	e.groups.Name = structs.GenerateUUID()
+
+	// TODO: verify this is needed for things like network access
+	e.groups.Resources.AllowAllDevices = true
+
+	if resources.MemoryMB > 0 {
+		// Total amount of memory allowed to consume
+		e.groups.Resources.Memory = int64(resources.MemoryMB * 1024 * 1024)
+		// Disable swap to avoid issues on the machine
+		e.groups.Resources.MemorySwap = int64(-1)
+	}
+
+	if resources.CPU < 2 {
+		return fmt.Errorf("resources.CPU must be equal to or greater than 2: %v", resources.CPU)
+	}
+
+	// Set the relative CPU shares for this cgroup.
+	e.groups.Resources.CpuShares = int64(resources.CPU)
+
+	if resources.IOPS != 0 {
+		// Validate it is in an acceptable range.
+		if resources.IOPS < 10 || resources.IOPS > 1000 {
+			return fmt.Errorf("resources.IOPS must be between 10 and 1000: %d", resources.IOPS)
+		}
+
+		e.groups.Resources.BlkioWeight = uint16(resources.IOPS)
+	}
+
+	return nil
+}
+
+// destroyCgroup kills all processes in the cgroup and removes the cgroup
+// configuration from the host.
+func (e *LinuxExecutor) destroyCgroup() error {
+	if e.groups == nil {
+		return fmt.Errorf("Can't destroy: cgroup configuration empty")
+	}
+
+	// Prevent a race between Wait/ForceStop
+	e.lock.Lock()
+	defer e.lock.Unlock()
+
+	manager := e.getCgroupManager(e.groups)
+	pids, err := manager.GetPids()
+	if err != nil {
+		return fmt.Errorf("Failed to get pids in the cgroup %v: %v", e.groups.Name, err)
+	}
+
+	errs := new(multierror.Error)
+	for _, pid := range pids {
+		process, err := os.FindProcess(pid)
+		if err != nil {
+			multierror.Append(errs, fmt.Errorf("Failed to find Pid %v: %v", pid, err))
+			continue
+		}
+
+		if err := process.Kill(); err != nil && err.Error() != "os: process already finished" {
+			multierror.Append(errs, fmt.Errorf("Failed to kill Pid %v: %v", pid, err))
+			continue
+		}
+	}
+
+	// Remove the cgroup.
+	if err := manager.Destroy(); err != nil {
+		multierror.Append(errs, fmt.Errorf("Failed to delete the cgroup directories: %v", err))
+	}
+
+	if len(errs.Errors) != 0 {
+		return fmt.Errorf("Failed to destroy cgroup: %v", errs)
+	}
+
+	return nil
+}
+
+// getCgroupManager returns the correct libcontainer cgroup manager.
+func (e *LinuxExecutor) getCgroupManager(groups *cgroupConfig.Cgroup) cgroups.Manager {
+	var manager cgroups.Manager
+	manager = &cgroupFs.Manager{Cgroups: groups}
+	if systemd.UseSystemd() {
+		manager = &systemd.Manager{Cgroups: groups}
+	}
+	return manager
+}

From 9b820797546ff0a72af715f019957920f298a8c7 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Wed, 3 Feb 2016 11:54:54 -0800
Subject: [PATCH 07/68] Killing the plugin after wait returns

---
 client/driver/exec.go                    |   8 +-
 client/driver/plugins/executor.go        | 124 ----------------------
 client/driver/plugins/executor_plugin.go | 125 +++++++++++++++++++++++
 3 files changed, 126 insertions(+), 131 deletions(-)
 create mode 100644 client/driver/plugins/executor_plugin.go

diff --git a/client/driver/exec.go b/client/driver/exec.go
index e80134c3e..bf696f31a 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -151,13 +151,6 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
 	}
 
-	// Find the process
-	//	execCtx := executor.NewExecutorContext(d.taskEnv)
-	//	cmd, err := executor.OpenId(execCtx, id.ExecutorId)
-	//	if err != nil {
-	//		return nil, fmt.Errorf("failed to open ID %v: %v", id.ExecutorId, err)
-	//	}
-
 	bin, err := discover.NomadExecutable()
 	if err != nil {
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
@@ -244,4 +237,5 @@ func (h *execHandle) run() {
 	close(h.doneCh)
 	h.waitCh <- &cstructs.WaitResult{ExitCode: ps.ExitCode, Signal: 0, Err: err}
 	close(h.waitCh)
+	h.pluginClient.Kill()
 }
diff --git a/client/driver/plugins/executor.go b/client/driver/plugins/executor.go
index 10530f6e9..d5c343e19 100644
--- a/client/driver/plugins/executor.go
+++ b/client/driver/plugins/executor.go
@@ -1,125 +1 @@
 package plugins
-
-import (
-	"log"
-	"net/rpc"
-	"os"
-	"time"
-
-	"github.com/hashicorp/go-plugin"
-
-	"github.com/hashicorp/nomad/client/allocdir"
-	"github.com/hashicorp/nomad/client/driver/env"
-	"github.com/hashicorp/nomad/nomad/structs"
-)
-
-var HandshakeConfig = plugin.HandshakeConfig{
-	ProtocolVersion:  1,
-	MagicCookieKey:   "executor_plugin",
-	MagicCookieValue: "value",
-}
-
-var PluginMap = map[string]plugin.Plugin{
-	"executor": new(ExecutorPlugin),
-}
-
-type ExecutorContext struct {
-	TaskEnv  *env.TaskEnvironment
-	AllocDir *allocdir.AllocDir
-	Task     *structs.Task
-	Chroot   bool
-	Limits   bool
-}
-
-type ExecCommand struct {
-	Cmd  string
-	Args []string
-}
-
-type ProcessState struct {
-	Pid      int
-	ExitCode int
-	Time     time.Time
-}
-
-type Executor interface {
-	LaunchCmd(cmd *ExecCommand, ctx *ExecutorContext) (*ProcessState, error)
-	Wait() (*ProcessState, error)
-	ShutDown() error
-	Exit() error
-}
-
-type ExecutorRPC struct {
-	client *rpc.Client
-}
-
-type LaunchCmdArgs struct {
-	Cmd *ExecCommand
-	Ctx *ExecutorContext
-}
-
-func (e *ExecutorRPC) LaunchCmd(cmd *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
-	var ps ProcessState
-	err := e.client.Call("Plugin.LaunchCmd", LaunchCmdArgs{Cmd: cmd, Ctx: ctx}, &ps)
-	return &ps, err
-}
-
-func (e *ExecutorRPC) Wait() (*ProcessState, error) {
-	var ps ProcessState
-	err := e.client.Call("Plugin.Wait", new(interface{}), &ps)
-	return &ps, err
-}
-
-func (e *ExecutorRPC) ShutDown() error {
-	var ps ProcessState
-	err := e.client.Call("Plugin.ShutDown", new(interface{}), &ps)
-	return err
-}
-
-func (e *ExecutorRPC) Exit() error {
-	var ps ProcessState
-	err := e.client.Call("Plugin.Exit", new(interface{}), &ps)
-	return err
-}
-
-type ExecutorRPCServer struct {
-	Impl Executor
-}
-
-func (e *ExecutorRPCServer) LaunchCmd(args LaunchCmdArgs, ps *ProcessState) error {
-	var err error
-	ps, err = e.Impl.LaunchCmd(args.Cmd, args.Ctx)
-	return err
-}
-
-func (e *ExecutorRPCServer) Wait(args interface{}, ps *ProcessState) error {
-	var err error
-	ps, err = e.Impl.Wait()
-	return err
-}
-
-func (e *ExecutorRPCServer) ShutDown(args interface{}, ps *ProcessState) error {
-	var err error
-	err = e.Impl.ShutDown()
-	return err
-}
-
-func (e *ExecutorRPCServer) Exit(args interface{}, ps *ProcessState) error {
-	var err error
-	err = e.Impl.Exit()
-	return err
-}
-
-type ExecutorPlugin struct {
-	logger *log.Logger
-}
-
-func (p *ExecutorPlugin) Server(*plugin.MuxBroker) (interface{}, error) {
-	p.logger = log.New(os.Stdout, "executor-plugin-server:", log.LstdFlags)
-	return &ExecutorRPCServer{Impl: NewExecutor(p.logger)}, nil
-}
-
-func (p *ExecutorPlugin) Client(b *plugin.MuxBroker, c *rpc.Client) (interface{}, error) {
-	p.logger = log.New(os.Stdout, "executor-plugin-client:", log.LstdFlags)
-	return &ExecutorRPC{client: c}, nil
-}
diff --git a/client/driver/plugins/executor_plugin.go b/client/driver/plugins/executor_plugin.go
new file mode 100644
index 000000000..10530f6e9
--- /dev/null
+++ b/client/driver/plugins/executor_plugin.go
@@ -0,0 +1,125 @@
+package plugins
+
+import (
+	"log"
+	"net/rpc"
+	"os"
+	"time"
+
+	"github.com/hashicorp/go-plugin"
+
+	"github.com/hashicorp/nomad/client/allocdir"
+	"github.com/hashicorp/nomad/client/driver/env"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+var HandshakeConfig = plugin.HandshakeConfig{
+	ProtocolVersion:  1,
+	MagicCookieKey:   "executor_plugin",
+	MagicCookieValue: "value",
+}
+
+var PluginMap = map[string]plugin.Plugin{
+	"executor": new(ExecutorPlugin),
+}
+
+type ExecutorContext struct {
+	TaskEnv  *env.TaskEnvironment
+	AllocDir *allocdir.AllocDir
+	Task     *structs.Task
+	Chroot   bool
+	Limits   bool
+}
+
+type ExecCommand struct {
+	Cmd  string
+	Args []string
+}
+
+type ProcessState struct {
+	Pid      int
+	ExitCode int
+	Time     time.Time
+}
+
+type Executor interface {
+	LaunchCmd(cmd *ExecCommand, ctx *ExecutorContext) (*ProcessState, error)
+	Wait() (*ProcessState, error)
+	ShutDown() error
+	Exit() error
+}
+
+type ExecutorRPC struct {
+	client *rpc.Client
+}
+
+type LaunchCmdArgs struct {
+	Cmd *ExecCommand
+	Ctx *ExecutorContext
+}
+
+func (e *ExecutorRPC) LaunchCmd(cmd *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
+	var ps ProcessState
+	err := e.client.Call("Plugin.LaunchCmd", LaunchCmdArgs{Cmd: cmd, Ctx: ctx}, &ps)
+	return &ps, err
+}
+
+func (e *ExecutorRPC) Wait() (*ProcessState, error) {
+	var ps ProcessState
+	err := e.client.Call("Plugin.Wait", new(interface{}), &ps)
+	return &ps, err
+}
+
+func (e *ExecutorRPC) ShutDown() error {
+	var ps ProcessState
+	err := e.client.Call("Plugin.ShutDown", new(interface{}), &ps)
+	return err
+}
+
+func (e *ExecutorRPC) Exit() error {
+	var ps ProcessState
+	err := e.client.Call("Plugin.Exit", new(interface{}), &ps)
+	return err
+}
+
+type ExecutorRPCServer struct {
+	Impl Executor
+}
+
+func (e *ExecutorRPCServer) LaunchCmd(args LaunchCmdArgs, ps *ProcessState) error {
+	var err error
+	ps, err = e.Impl.LaunchCmd(args.Cmd, args.Ctx)
+	return err
+}
+
+func (e *ExecutorRPCServer) Wait(args interface{}, ps *ProcessState) error {
+	var err error
+	ps, err = e.Impl.Wait()
+	return err
+}
+
+func (e *ExecutorRPCServer) ShutDown(args interface{}, ps *ProcessState) error {
+	var err error
+	err = e.Impl.ShutDown()
+	return err
+}
+
+func (e *ExecutorRPCServer) Exit(args interface{}, ps *ProcessState) error {
+	var err error
+	err = e.Impl.Exit()
+	return err
+}
+
+type ExecutorPlugin struct {
+	logger *log.Logger
+}
+
+func (p *ExecutorPlugin) Server(*plugin.MuxBroker) (interface{}, error) {
+	p.logger = log.New(os.Stdout, "executor-plugin-server:", log.LstdFlags)
+	return &ExecutorRPCServer{Impl: NewExecutor(p.logger)}, nil
+}
+
+func (p *ExecutorPlugin) Client(b *plugin.MuxBroker, c *rpc.Client) (interface{}, error) {
+	p.logger = log.New(os.Stdout, "executor-plugin-client:", log.LstdFlags)
+	return &ExecutorRPC{client: c}, nil
+}

From ed72a67ee02eaf0af8488a36790e4c9cfb252a9c Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Wed, 3 Feb 2016 16:03:43 -0800
Subject: [PATCH 08/68] Creating the universal executor

---
 client/driver/plugins/executor.go        | 163 ++++++++++
 client/driver/plugins/executor_basic.go  | 119 ++-----
 client/driver/plugins/executor_linux.go  | 377 +++++++++--------------
 client/driver/plugins/executor_plugin.go |  31 --
 4 files changed, 324 insertions(+), 366 deletions(-)

diff --git a/client/driver/plugins/executor.go b/client/driver/plugins/executor.go
index d5c343e19..e1fbfee61 100644
--- a/client/driver/plugins/executor.go
+++ b/client/driver/plugins/executor.go
@@ -1 +1,164 @@
 package plugins
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"sync"
+	"syscall"
+	"time"
+
+	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
+
+	"github.com/hashicorp/nomad/client/allocdir"
+	"github.com/hashicorp/nomad/client/driver/env"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+type ExecutorContext struct {
+	TaskEnv  *env.TaskEnvironment
+	AllocDir *allocdir.AllocDir
+	Task     *structs.Task
+	Chroot   bool
+	Limits   bool
+}
+
+type ExecCommand struct {
+	Cmd  string
+	Args []string
+}
+
+type ProcessState struct {
+	Pid      int
+	ExitCode int
+	Time     time.Time
+}
+
+type Executor interface {
+	LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error)
+	Wait() (*ProcessState, error)
+	ShutDown() error
+	Exit() error
+}
+
+type UniversalExecutor struct {
+	cmd exec.Cmd
+	ctx *ExecutorContext
+
+	taskDir string
+	groups  *cgroupConfig.Cgroup
+
+	logger *log.Logger
+	lock   sync.Mutex
+}
+
+func NewExecutor(logger *log.Logger) Executor {
+	return &UniversalExecutor{logger: logger}
+}
+
+func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
+	e.logger.Printf("LAUNCH COMMAND")
+	e.ctx = ctx
+	e.cmd.Path = command.Cmd
+	e.cmd.Args = append([]string{command.Cmd}, command.Args...)
+	if filepath.Base(command.Cmd) == command.Cmd {
+		if lp, err := exec.LookPath(command.Cmd); err != nil {
+		} else {
+			e.cmd.Path = lp
+		}
+	}
+	if err := e.configureTaskDir(); err != nil {
+		return nil, err
+	}
+	if err := e.configureIsolation(); err != nil {
+		return nil, err
+	}
+
+	if err := e.runAs("nobody"); err != nil {
+		return nil, err
+	}
+
+	stdoPath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", ctx.Task.Name))
+	stdo, err := os.OpenFile(stdoPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
+	if err != nil {
+		return nil, err
+	}
+	e.cmd.Stdout = stdo
+
+	stdePath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", ctx.Task.Name))
+	stde, err := os.OpenFile(stdePath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
+	if err != nil {
+		return nil, err
+	}
+	e.cmd.Stderr = stde
+
+	e.cmd.Env = ctx.TaskEnv.EnvList()
+
+	if err := e.cmd.Start(); err != nil {
+		return nil, fmt.Errorf("error starting command: %v", err)
+	}
+
+	e.applyLimits()
+	return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, Time: time.Now()}, nil
+}
+
+func (e *UniversalExecutor) Wait() (*ProcessState, error) {
+	err := e.cmd.Wait()
+	if err == nil {
+		return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
+	}
+	exitCode := 1
+	if exitErr, ok := err.(*exec.ExitError); ok {
+		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
+			exitCode = status.ExitStatus()
+		}
+	}
+	if e.ctx.Chroot {
+		e.removeChrootMounts()
+	}
+	if e.ctx.Limits {
+		e.destroyCgroup()
+	}
+	return &ProcessState{Pid: 0, ExitCode: exitCode, Time: time.Now()}, nil
+}
+
+func (e *UniversalExecutor) Exit() error {
+	e.logger.Printf("[INFO] Exiting plugin for task %q", e.ctx.Task.Name)
+	proc, err := os.FindProcess(e.cmd.Process.Pid)
+	if err != nil {
+		return fmt.Errorf("failied to find user process %v: %v", e.cmd.Process.Pid, err)
+	}
+	if e.ctx.Chroot {
+		e.removeChrootMounts()
+	}
+	if e.ctx.Limits {
+		e.destroyCgroup()
+	}
+	return proc.Kill()
+}
+
+func (e *UniversalExecutor) ShutDown() error {
+	proc, err := os.FindProcess(e.cmd.Process.Pid)
+	if err != nil {
+		return err
+	}
+	if runtime.GOOS == "windows" {
+		return proc.Kill()
+	}
+	return proc.Signal(os.Interrupt)
+}
+
+func (e *UniversalExecutor) configureTaskDir() error {
+	e.logger.Printf("DIPTANNUUUU CONDIFURE")
+	taskDir, ok := e.ctx.AllocDir.TaskDirs[e.ctx.Task.Name]
+	fmt.Printf("DIPTANU TASKDIR : %v", taskDir)
+	e.taskDir = taskDir
+	if !ok {
+		return fmt.Errorf("Couldn't find task directory for task %v", e.ctx.Task.Name)
+	}
+	e.cmd.Dir = taskDir
+	return nil
+}
diff --git a/client/driver/plugins/executor_basic.go b/client/driver/plugins/executor_basic.go
index a3777ea3a..464cb3b3f 100644
--- a/client/driver/plugins/executor_basic.go
+++ b/client/driver/plugins/executor_basic.go
@@ -2,103 +2,26 @@
 
 package plugins
 
-import (
-	"fmt"
-	"log"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"runtime"
-	"syscall"
-	"time"
-
-	"github.com/hashicorp/nomad/client/allocdir"
-)
-
-type BasicExecutor struct {
-	logger *log.Logger
-	cmd    exec.Cmd
-
-	taskDir string
-}
-
-func NewExecutor(logger *log.Logger) Executor {
-	return &BasicExecutor{logger: logger}
-}
-
-func (e *BasicExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
-	e.cmd.Path = command.Cmd
-	e.cmd.Args = append([]string{command.Cmd}, command.Args...)
-	e.cmd.Path = ctx.TaskEnv.ReplaceEnv(e.cmd.Path)
-	e.cmd.Args = ctx.TaskEnv.ParseAndReplace(e.cmd.Args)
-
-	if filepath.Base(command.Cmd) == command.Cmd {
-		if lp, err := exec.LookPath(command.Cmd); err != nil {
-		} else {
-			e.cmd.Path = lp
-		}
-	}
-	e.configureTaskDir(ctx.Task.Name, ctx.AllocDir)
-	e.cmd.Env = ctx.TaskEnv.EnvList()
-	stdoPath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", ctx.Task.Name))
-	stdo, err := os.OpenFile(stdoPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
-	if err != nil {
-		return nil, err
-	}
-	e.cmd.Stdout = stdo
-
-	stdePath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", ctx.Task.Name))
-	stde, err := os.OpenFile(stdePath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
-	if err != nil {
-		return nil, err
-	}
-	e.cmd.Stderr = stde
-	if err := e.cmd.Start(); err != nil {
-		return nil, err
-	}
-
-	return &ProcessState{Pid: 5, ExitCode: -1, Time: time.Now()}, nil
-}
-
-func (e *BasicExecutor) Wait() (*ProcessState, error) {
-	err := e.cmd.Wait()
-	if err == nil {
-		return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
-	}
-	exitCode := 1
-	if exitErr, ok := err.(*exec.ExitError); ok {
-		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
-			exitCode = status.ExitStatus()
-		}
-	}
-	return &ProcessState{Pid: 0, ExitCode: exitCode, Time: time.Now()}, nil
-}
-
-func (e *BasicExecutor) Exit() error {
-	proc, err := os.FindProcess(e.cmd.Process.Pid)
-	if err != nil {
-		return fmt.Errorf("failied to find user process %v: %v", e.cmd.Process.Pid, err)
-	}
-	return proc.Kill()
-}
-
-func (e *BasicExecutor) ShutDown() error {
-	proc, err := os.FindProcess(e.cmd.Process.Pid)
-	if err != nil {
-		return err
-	}
-	if runtime.GOOS == "windows" {
-		return proc.Kill()
-	}
-	return proc.Signal(os.Interrupt)
-}
-
-func (e *BasicExecutor) configureTaskDir(taskName string, allocDir *allocdir.AllocDir) error {
-	taskDir, ok := allocDir.TaskDirs[taskName]
-	e.taskDir = taskDir
-	if !ok {
-		return fmt.Errorf("Couldn't find task directory for task %v", taskName)
-	}
-	e.cmd.Dir = taskDir
+func (e *UniversalExecutor) configureChroot() error {
+	return nil
+}
+
+func (e *UniversalExecutor) destroyCgroup() error {
+	return nil
+}
+
+func (e *UniversalExecutor) removeChrootMounts() error {
+	return nil
+}
+
+func (e *UniversalExecutor) runAs(userid string) error {
+	return nil
+}
+
+func (e *UniversalExecutor) applyLimits() error {
+	return nil
+}
+
+func (e *UniversalExecutor) configureIsolation() error {
 	return nil
 }
diff --git a/client/driver/plugins/executor_linux.go b/client/driver/plugins/executor_linux.go
index 152397667..27dc24941 100644
--- a/client/driver/plugins/executor_linux.go
+++ b/client/driver/plugins/executor_linux.go
@@ -2,16 +2,11 @@ package plugins
 
 import (
 	"fmt"
-	"log"
 	"os"
-	"os/exec"
 	"os/user"
 	"path/filepath"
-	"runtime"
 	"strconv"
-	"sync"
 	"syscall"
-	"time"
 
 	"github.com/hashicorp/go-multierror"
 	"github.com/opencontainers/runc/libcontainer/cgroups"
@@ -38,255 +33,39 @@ var (
 	}
 )
 
-type LinuxExecutor struct {
-	cmd exec.Cmd
-	ctx *ExecutorContext
-
-	groups  *cgroupConfig.Cgroup
-	taskDir string
-
-	logger *log.Logger
-	lock   sync.Mutex
-}
-
-func NewExecutor(logger *log.Logger) Executor {
-	return &LinuxExecutor{logger: logger}
-}
-
-func (e *LinuxExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
-	e.ctx = ctx
-	e.cmd.Path = command.Cmd
-	e.cmd.Args = append([]string{command.Cmd}, command.Args...)
-	if filepath.Base(command.Cmd) == command.Cmd {
-		if lp, err := exec.LookPath(command.Cmd); err != nil {
-		} else {
-			e.cmd.Path = lp
+func (e *UniversalExecutor) configureIsolation() error {
+	if e.ctx.Chroot {
+		if err := e.configureChroot(); err != nil {
+			return err
 		}
 	}
-	if err := e.configureTaskDir(); err != nil {
-		return nil, err
-	}
-	if err := e.runAs("nobody"); err != nil {
-		return nil, err
-	}
-	e.cmd.Env = ctx.TaskEnv.EnvList()
 
-	stdoPath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", ctx.Task.Name))
-	stdo, err := os.OpenFile(stdoPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
-	if err != nil {
-		return nil, err
+	if e.ctx.Limits {
+		if err := e.configureCgroups(e.ctx.Task.Resources); err != nil {
+			return fmt.Errorf("error creating cgroups: %v", err)
+		}
 	}
-	e.cmd.Stdout = stdo
+	return nil
+}
 
-	stdePath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", ctx.Task.Name))
-	stde, err := os.OpenFile(stdePath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
-	if err != nil {
-		return nil, err
+func (e *UniversalExecutor) applyLimits() error {
+	if !e.ctx.Limits {
+		return nil
 	}
-	e.cmd.Stderr = stde
-
-	e.configureChroot()
-
-	if err := e.configureCgroups(e.ctx.Task.Resources); err != nil {
-		return nil, fmt.Errorf("error creating cgroups: %v", err)
-	}
-
-	if err := e.cmd.Start(); err != nil {
-		return nil, fmt.Errorf("error starting command: %v", err)
-	}
-
 	manager := e.getCgroupManager(e.groups)
 	if err := manager.Apply(e.cmd.Process.Pid); err != nil {
 		e.logger.Printf("[ERROR] unable to join cgroup: %v", err)
 		if err := e.Exit(); err != nil {
 			e.logger.Printf("[ERROR] unable to kill process: %v", err)
 		}
-		return nil, err
-	}
-
-	return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, Time: time.Now()}, nil
-}
-
-// ConfigureTaskDir creates the necessary directory structure for a proper
-// chroot. cleanTaskDir should be called after.
-func (e *LinuxExecutor) configureTaskDir() error {
-	taskName := e.ctx.Task.Name
-	allocDir := e.ctx.AllocDir
-	taskDir, ok := allocDir.TaskDirs[taskName]
-	if !ok {
-		fmt.Errorf("Couldn't find task directory for task %v", taskName)
-	}
-	e.taskDir = taskDir
-
-	if err := allocDir.MountSharedDir(taskName); err != nil {
 		return err
 	}
-
-	if err := allocDir.Embed(taskName, chrootEnv); err != nil {
-		return err
-	}
-
-	// Mount dev
-	dev := filepath.Join(taskDir, "dev")
-	if !e.pathExists(dev) {
-		if err := os.Mkdir(dev, 0777); err != nil {
-			return fmt.Errorf("Mkdir(%v) failed: %v", dev, err)
-		}
-
-		if err := syscall.Mount("none", dev, "devtmpfs", syscall.MS_RDONLY, ""); err != nil {
-			return fmt.Errorf("Couldn't mount /dev to %v: %v", dev, err)
-		}
-	}
-
-	// Mount proc
-	proc := filepath.Join(taskDir, "proc")
-	if !e.pathExists(proc) {
-		if err := os.Mkdir(proc, 0777); err != nil {
-			return fmt.Errorf("Mkdir(%v) failed: %v", proc, err)
-		}
-
-		if err := syscall.Mount("none", proc, "proc", syscall.MS_RDONLY, ""); err != nil {
-			return fmt.Errorf("Couldn't mount /proc to %v: %v", proc, err)
-		}
-	}
-
-	// Set the tasks AllocDir environment variable.
-	e.ctx.TaskEnv.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)).SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)).Build()
 	return nil
 }
 
-// runAs takes a user id as a string and looks up the user, and sets the command
-// to execute as that user.
-func (e *LinuxExecutor) runAs(userid string) error {
-	u, err := user.Lookup(userid)
-	if err != nil {
-		return fmt.Errorf("Failed to identify user %v: %v", userid, err)
-	}
-
-	// Convert the uid and gid
-	uid, err := strconv.ParseUint(u.Uid, 10, 32)
-	if err != nil {
-		return fmt.Errorf("Unable to convert userid to uint32: %s", err)
-	}
-	gid, err := strconv.ParseUint(u.Gid, 10, 32)
-	if err != nil {
-		return fmt.Errorf("Unable to convert groupid to uint32: %s", err)
-	}
-
-	// Set the command to run as that user and group.
-	if e.cmd.SysProcAttr == nil {
-		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
-	}
-	if e.cmd.SysProcAttr.Credential == nil {
-		e.cmd.SysProcAttr.Credential = &syscall.Credential{}
-	}
-	e.cmd.SysProcAttr.Credential.Uid = uint32(uid)
-	e.cmd.SysProcAttr.Credential.Gid = uint32(gid)
-
-	return nil
-}
-
-// pathExists is a helper function to check if the path exists.
-func (e *LinuxExecutor) pathExists(path string) bool {
-	if _, err := os.Stat(path); err != nil {
-		if os.IsNotExist(err) {
-			return false
-		}
-	}
-	return true
-}
-
-// configureChroot enters the user command into a chroot if specified in the
-// config and on an OS that supports Chroots.
-func (e *LinuxExecutor) configureChroot() {
-	if !e.ctx.Chroot {
-		return
-	}
-	if e.cmd.SysProcAttr == nil {
-		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
-	}
-
-	e.cmd.SysProcAttr.Chroot = e.taskDir
-	e.cmd.Dir = "/"
-}
-
-// cleanTaskDir is an idempotent operation to clean the task directory and
-// should be called when tearing down the task.
-func (e *LinuxExecutor) cleanTaskDir() error {
-	// Prevent a race between Wait/ForceStop
-	e.lock.Lock()
-	defer e.lock.Unlock()
-
-	// Unmount dev.
-	errs := new(multierror.Error)
-	dev := filepath.Join(e.taskDir, "dev")
-	if e.pathExists(dev) {
-		if err := syscall.Unmount(dev, 0); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("Failed to unmount dev (%v): %v", dev, err))
-		}
-
-		if err := os.RemoveAll(dev); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("Failed to delete dev directory (%v): %v", dev, err))
-		}
-	}
-
-	// Unmount
-	// proc.
-	proc := filepath.Join(e.taskDir, "proc")
-	if e.pathExists(proc) {
-		if err := syscall.Unmount(proc, 0); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("Failed to unmount proc (%v): %v", proc, err))
-		}
-
-		if err := os.RemoveAll(proc); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("Failed to delete proc directory (%v): %v", dev, err))
-		}
-	}
-
-	return errs.ErrorOrNil()
-}
-
-func (e *LinuxExecutor) Wait() (*ProcessState, error) {
-	err := e.cmd.Wait()
-	if err == nil {
-		return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
-	}
-	exitCode := 1
-	if exitErr, ok := err.(*exec.ExitError); ok {
-		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
-			exitCode = status.ExitStatus()
-		}
-	}
-	e.cleanTaskDir()
-	e.destroyCgroup()
-	return &ProcessState{Pid: 0, ExitCode: exitCode, Time: time.Now()}, nil
-}
-
-func (e *LinuxExecutor) Exit() error {
-	e.logger.Printf("[INFO] Exiting plugin for task %q", e.ctx.Task.Name)
-	proc, err := os.FindProcess(e.cmd.Process.Pid)
-	if err != nil {
-		return fmt.Errorf("failied to find user process %v: %v", e.cmd.Process.Pid, err)
-	}
-	e.cleanTaskDir()
-	e.destroyCgroup()
-	return proc.Kill()
-}
-
-func (e *LinuxExecutor) ShutDown() error {
-	proc, err := os.FindProcess(e.cmd.Process.Pid)
-	if err != nil {
-		return err
-	}
-	if runtime.GOOS == "windows" {
-		return proc.Kill()
-	}
-	return proc.Signal(os.Interrupt)
-}
-
 // configureCgroups converts a Nomad Resources specification into the equivalent
 // cgroup configuration. It returns an error if the resources are invalid.
-func (e *LinuxExecutor) configureCgroups(resources *structs.Resources) error {
+func (e *UniversalExecutor) configureCgroups(resources *structs.Resources) error {
 	e.groups = &cgroupConfig.Cgroup{}
 	e.groups.Resources = &cgroupConfig.Resources{}
 	e.groups.Name = structs.GenerateUUID()
@@ -320,9 +99,133 @@ func (e *LinuxExecutor) configureCgroups(resources *structs.Resources) error {
 	return nil
 }
 
+// runAs takes a user id as a string and looks up the user, and sets the command
+// to execute as that user.
+func (e *UniversalExecutor) runAs(userid string) error {
+	u, err := user.Lookup(userid)
+	if err != nil {
+		return fmt.Errorf("Failed to identify user %v: %v", userid, err)
+	}
+
+	// Convert the uid and gid
+	uid, err := strconv.ParseUint(u.Uid, 10, 32)
+	if err != nil {
+		return fmt.Errorf("Unable to convert userid to uint32: %s", err)
+	}
+	gid, err := strconv.ParseUint(u.Gid, 10, 32)
+	if err != nil {
+		return fmt.Errorf("Unable to convert groupid to uint32: %s", err)
+	}
+
+	// Set the command to run as that user and group.
+	if e.cmd.SysProcAttr == nil {
+		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
+	}
+	if e.cmd.SysProcAttr.Credential == nil {
+		e.cmd.SysProcAttr.Credential = &syscall.Credential{}
+	}
+	e.cmd.SysProcAttr.Credential.Uid = uint32(uid)
+	e.cmd.SysProcAttr.Credential.Gid = uint32(gid)
+
+	return nil
+}
+
+// pathExists is a helper function to check if the path exists.
+func (e *UniversalExecutor) pathExists(path string) bool {
+	if _, err := os.Stat(path); err != nil {
+		if os.IsNotExist(err) {
+			return false
+		}
+	}
+	return true
+}
+
+func (e *UniversalExecutor) configureChroot() error {
+	allocDir := e.ctx.AllocDir
+	if err := allocDir.MountSharedDir(e.ctx.Task.Name); err != nil {
+		return err
+	}
+
+	if err := allocDir.Embed(e.ctx.Task.Name, chrootEnv); err != nil {
+		return err
+	}
+
+	// Mount dev
+	dev := filepath.Join(e.taskDir, "dev")
+	if !e.pathExists(dev) {
+		if err := os.Mkdir(dev, 0777); err != nil {
+			return fmt.Errorf("Mkdir(%v) failed: %v", dev, err)
+		}
+
+		if err := syscall.Mount("none", dev, "devtmpfs", syscall.MS_RDONLY, ""); err != nil {
+			return fmt.Errorf("Couldn't mount /dev to %v: %v", dev, err)
+		}
+	}
+
+	// Mount proc
+	proc := filepath.Join(e.taskDir, "proc")
+	if !e.pathExists(proc) {
+		if err := os.Mkdir(proc, 0777); err != nil {
+			return fmt.Errorf("Mkdir(%v) failed: %v", proc, err)
+		}
+
+		if err := syscall.Mount("none", proc, "proc", syscall.MS_RDONLY, ""); err != nil {
+			return fmt.Errorf("Couldn't mount /proc to %v: %v", proc, err)
+		}
+	}
+
+	// Set the tasks AllocDir environment variable.
+	e.ctx.TaskEnv.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)).SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)).Build()
+
+	if e.cmd.SysProcAttr == nil {
+		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
+	}
+
+	e.cmd.SysProcAttr.Chroot = e.taskDir
+	e.cmd.Dir = "/"
+
+	return nil
+}
+
+// cleanTaskDir is an idempotent operation to clean the task directory and
+// should be called when tearing down the task.
+func (e *UniversalExecutor) removeChrootMounts() error {
+	// Prevent a race between Wait/ForceStop
+	e.lock.Lock()
+	defer e.lock.Unlock()
+
+	// Unmount dev.
+	errs := new(multierror.Error)
+	dev := filepath.Join(e.taskDir, "dev")
+	if e.pathExists(dev) {
+		if err := syscall.Unmount(dev, 0); err != nil {
+			errs = multierror.Append(errs, fmt.Errorf("Failed to unmount dev (%v): %v", dev, err))
+		}
+
+		if err := os.RemoveAll(dev); err != nil {
+			errs = multierror.Append(errs, fmt.Errorf("Failed to delete dev directory (%v): %v", dev, err))
+		}
+	}
+
+	// Unmount
+	// proc.
+	proc := filepath.Join(e.taskDir, "proc")
+	if e.pathExists(proc) {
+		if err := syscall.Unmount(proc, 0); err != nil {
+			errs = multierror.Append(errs, fmt.Errorf("Failed to unmount proc (%v): %v", proc, err))
+		}
+
+		if err := os.RemoveAll(proc); err != nil {
+			errs = multierror.Append(errs, fmt.Errorf("Failed to delete proc directory (%v): %v", dev, err))
+		}
+	}
+
+	return errs.ErrorOrNil()
+}
+
 // destroyCgroup kills all processes in the cgroup and removes the cgroup
 // configuration from the host.
-func (e *LinuxExecutor) destroyCgroup() error {
+func (e *UniversalExecutor) destroyCgroup() error {
 	if e.groups == nil {
 		return fmt.Errorf("Can't destroy: cgroup configuration empty")
 	}
@@ -364,7 +267,7 @@ func (e *LinuxExecutor) destroyCgroup() error {
 }
 
 // getCgroupManager returns the correct libcontainer cgroup manager.
-func (e *LinuxExecutor) getCgroupManager(groups *cgroupConfig.Cgroup) cgroups.Manager {
+func (e *UniversalExecutor) getCgroupManager(groups *cgroupConfig.Cgroup) cgroups.Manager {
 	var manager cgroups.Manager
 	manager = &cgroupFs.Manager{Cgroups: groups}
 	if systemd.UseSystemd() {
diff --git a/client/driver/plugins/executor_plugin.go b/client/driver/plugins/executor_plugin.go
index 10530f6e9..cb077a7f7 100644
--- a/client/driver/plugins/executor_plugin.go
+++ b/client/driver/plugins/executor_plugin.go
@@ -4,13 +4,8 @@ import (
 	"log"
 	"net/rpc"
 	"os"
-	"time"
 
 	"github.com/hashicorp/go-plugin"
-
-	"github.com/hashicorp/nomad/client/allocdir"
-	"github.com/hashicorp/nomad/client/driver/env"
-	"github.com/hashicorp/nomad/nomad/structs"
 )
 
 var HandshakeConfig = plugin.HandshakeConfig{
@@ -23,32 +18,6 @@ var PluginMap = map[string]plugin.Plugin{
 	"executor": new(ExecutorPlugin),
 }
 
-type ExecutorContext struct {
-	TaskEnv  *env.TaskEnvironment
-	AllocDir *allocdir.AllocDir
-	Task     *structs.Task
-	Chroot   bool
-	Limits   bool
-}
-
-type ExecCommand struct {
-	Cmd  string
-	Args []string
-}
-
-type ProcessState struct {
-	Pid      int
-	ExitCode int
-	Time     time.Time
-}
-
-type Executor interface {
-	LaunchCmd(cmd *ExecCommand, ctx *ExecutorContext) (*ProcessState, error)
-	Wait() (*ProcessState, error)
-	ShutDown() error
-	Exit() error
-}
-
 type ExecutorRPC struct {
 	client *rpc.Client
 }

From f67a237361f7189e8a8e7b9aa51ccbf6d52cd5ec Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Wed, 3 Feb 2016 16:09:17 -0800
Subject: [PATCH 09/68] Introduced the flag for the user

---
 client/driver/plugins/executor.go       | 25 ++++++++++++++-----------
 client/driver/plugins/executor_linux.go |  6 +++---
 2 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/client/driver/plugins/executor.go b/client/driver/plugins/executor.go
index e1fbfee61..2f708efbe 100644
--- a/client/driver/plugins/executor.go
+++ b/client/driver/plugins/executor.go
@@ -19,11 +19,12 @@ import (
 )
 
 type ExecutorContext struct {
-	TaskEnv  *env.TaskEnvironment
-	AllocDir *allocdir.AllocDir
-	Task     *structs.Task
-	Chroot   bool
-	Limits   bool
+	TaskEnv          *env.TaskEnvironment
+	AllocDir         *allocdir.AllocDir
+	Task             *structs.Task
+	FSIsolation      bool
+	ResourceLimits   bool
+	UnprivilegedUser bool
 }
 
 type ExecCommand struct {
@@ -77,8 +78,10 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext
 		return nil, err
 	}
 
-	if err := e.runAs("nobody"); err != nil {
-		return nil, err
+	if e.ctx.UnprivilegedUser {
+		if err := e.runAs("nobody"); err != nil {
+			return nil, err
+		}
 	}
 
 	stdoPath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", ctx.Task.Name))
@@ -116,10 +119,10 @@ func (e *UniversalExecutor) Wait() (*ProcessState, error) {
 			exitCode = status.ExitStatus()
 		}
 	}
-	if e.ctx.Chroot {
+	if e.ctx.FSIsolation {
 		e.removeChrootMounts()
 	}
-	if e.ctx.Limits {
+	if e.ctx.ResourceLimits {
 		e.destroyCgroup()
 	}
 	return &ProcessState{Pid: 0, ExitCode: exitCode, Time: time.Now()}, nil
@@ -131,10 +134,10 @@ func (e *UniversalExecutor) Exit() error {
 	if err != nil {
 		return fmt.Errorf("failied to find user process %v: %v", e.cmd.Process.Pid, err)
 	}
-	if e.ctx.Chroot {
+	if e.ctx.FSIsolation {
 		e.removeChrootMounts()
 	}
-	if e.ctx.Limits {
+	if e.ctx.ResourceLimits {
 		e.destroyCgroup()
 	}
 	return proc.Kill()
diff --git a/client/driver/plugins/executor_linux.go b/client/driver/plugins/executor_linux.go
index 27dc24941..8ae41d685 100644
--- a/client/driver/plugins/executor_linux.go
+++ b/client/driver/plugins/executor_linux.go
@@ -34,13 +34,13 @@ var (
 )
 
 func (e *UniversalExecutor) configureIsolation() error {
-	if e.ctx.Chroot {
+	if e.ctx.FSIsolation {
 		if err := e.configureChroot(); err != nil {
 			return err
 		}
 	}
 
-	if e.ctx.Limits {
+	if e.ctx.ResourceLimits {
 		if err := e.configureCgroups(e.ctx.Task.Resources); err != nil {
 			return fmt.Errorf("error creating cgroups: %v", err)
 		}
@@ -49,7 +49,7 @@ func (e *UniversalExecutor) configureIsolation() error {
 }
 
 func (e *UniversalExecutor) applyLimits() error {
-	if !e.ctx.Limits {
+	if !e.ctx.ResourceLimits {
 		return nil
 	}
 	manager := e.getCgroupManager(e.groups)

From d6c17424ccdafd61cacb1746408582a3bd87ba23 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Wed, 3 Feb 2016 16:14:13 -0800
Subject: [PATCH 10/68] Turning on isolation for exec

---
 client/driver/exec.go | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index bf696f31a..5c23961d8 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -115,9 +115,12 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		return nil, err
 	}
 	executorCtx := &plugins.ExecutorContext{
-		TaskEnv:  d.taskEnv,
-		AllocDir: ctx.AllocDir,
-		Task:     task,
+		TaskEnv:          d.taskEnv,
+		AllocDir:         ctx.AllocDir,
+		Task:             task,
+		ResourceLimits:   true,
+		FSIsolation:      true,
+		UnprivilegedUser: false,
 	}
 	ps, err := executor.LaunchCmd(&plugins.ExecCommand{Cmd: command, Args: driverConfig.Args}, executorCtx)
 	if err != nil {

From 0fca2b8ee1e27825977ccc3213477491d80fb148 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Wed, 3 Feb 2016 16:26:10 -0800
Subject: [PATCH 11/68] Making the wait asynchronous

---
 client/driver/plugins/executor.go | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/client/driver/plugins/executor.go b/client/driver/plugins/executor.go
index 2f708efbe..938906531 100644
--- a/client/driver/plugins/executor.go
+++ b/client/driver/plugins/executor.go
@@ -49,15 +49,17 @@ type UniversalExecutor struct {
 	cmd exec.Cmd
 	ctx *ExecutorContext
 
-	taskDir string
-	groups  *cgroupConfig.Cgroup
+	taskDir       string
+	groups        *cgroupConfig.Cgroup
+	exitState     *ProcessState
+	processExited chan interface{}
 
 	logger *log.Logger
 	lock   sync.Mutex
 }
 
 func NewExecutor(logger *log.Logger) Executor {
-	return &UniversalExecutor{logger: logger}
+	return &UniversalExecutor{logger: logger, processExited: make(chan interface{})}
 }
 
 func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
@@ -105,13 +107,20 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext
 	}
 
 	e.applyLimits()
+	go e.wait()
 	return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, Time: time.Now()}, nil
 }
 
 func (e *UniversalExecutor) Wait() (*ProcessState, error) {
+	<-e.processExited
+	return e.exitState, nil
+}
+
+func (e *UniversalExecutor) wait() {
 	err := e.cmd.Wait()
 	if err == nil {
-		return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
+		e.exitState = &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}
+		return
 	}
 	exitCode := 1
 	if exitErr, ok := err.(*exec.ExitError); ok {
@@ -125,7 +134,8 @@ func (e *UniversalExecutor) Wait() (*ProcessState, error) {
 	if e.ctx.ResourceLimits {
 		e.destroyCgroup()
 	}
-	return &ProcessState{Pid: 0, ExitCode: exitCode, Time: time.Now()}, nil
+	e.exitState = &ProcessState{Pid: 0, ExitCode: exitCode, Time: time.Now()}
+	close(e.processExited)
 }
 
 func (e *UniversalExecutor) Exit() error {

From 560cafbdddeb89ed93a63725a28a71f814fc78d7 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Wed, 3 Feb 2016 17:22:21 -0800
Subject: [PATCH 12/68] Enabling logs and killing the plugin if we couldn't
 start the job

---
 client/driver/exec.go                    | 1 +
 client/driver/plugins/executor.go        | 3 ---
 client/driver/plugins/executor_plugin.go | 8 +++++---
 client/driver/raw_exec.go                | 9 +++++++++
 4 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index 5c23961d8..011b2c7f6 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -124,6 +124,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 	}
 	ps, err := executor.LaunchCmd(&plugins.ExecCommand{Cmd: command, Args: driverConfig.Args}, executorCtx)
 	if err != nil {
+		pluginClient.Kill()
 		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
 	}
 	d.logger.Printf("DIPTANU Started process via plugin: %#v", ps)
diff --git a/client/driver/plugins/executor.go b/client/driver/plugins/executor.go
index 938906531..4c0444f4b 100644
--- a/client/driver/plugins/executor.go
+++ b/client/driver/plugins/executor.go
@@ -63,7 +63,6 @@ func NewExecutor(logger *log.Logger) Executor {
 }
 
 func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
-	e.logger.Printf("LAUNCH COMMAND")
 	e.ctx = ctx
 	e.cmd.Path = command.Cmd
 	e.cmd.Args = append([]string{command.Cmd}, command.Args...)
@@ -165,9 +164,7 @@ func (e *UniversalExecutor) ShutDown() error {
 }
 
 func (e *UniversalExecutor) configureTaskDir() error {
-	e.logger.Printf("DIPTANNUUUU CONDIFURE")
 	taskDir, ok := e.ctx.AllocDir.TaskDirs[e.ctx.Task.Name]
-	fmt.Printf("DIPTANU TASKDIR : %v", taskDir)
 	e.taskDir = taskDir
 	if !ok {
 		return fmt.Errorf("Couldn't find task directory for task %v", e.ctx.Task.Name)
diff --git a/client/driver/plugins/executor_plugin.go b/client/driver/plugins/executor_plugin.go
index cb077a7f7..52176c575 100644
--- a/client/driver/plugins/executor_plugin.go
+++ b/client/driver/plugins/executor_plugin.go
@@ -1,6 +1,7 @@
 package plugins
 
 import (
+	"fmt"
 	"log"
 	"net/rpc"
 	"os"
@@ -28,9 +29,9 @@ type LaunchCmdArgs struct {
 }
 
 func (e *ExecutorRPC) LaunchCmd(cmd *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
-	var ps ProcessState
-	err := e.client.Call("Plugin.LaunchCmd", LaunchCmdArgs{Cmd: cmd, Ctx: ctx}, &ps)
-	return &ps, err
+	ps := new(ProcessState)
+	err := e.client.Call("Plugin.LaunchCmd", LaunchCmdArgs{Cmd: cmd, Ctx: ctx}, ps)
+	return ps, err
 }
 
 func (e *ExecutorRPC) Wait() (*ProcessState, error) {
@@ -58,6 +59,7 @@ type ExecutorRPCServer struct {
 func (e *ExecutorRPCServer) LaunchCmd(args LaunchCmdArgs, ps *ProcessState) error {
 	var err error
 	ps, err = e.Impl.LaunchCmd(args.Cmd, args.Ctx)
+	fmt.Printf("DIPTANU PS Server : %#v", ps)
 	return err
 }
 
diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go
index 502d2fc37..4a871753c 100644
--- a/client/driver/raw_exec.go
+++ b/client/driver/raw_exec.go
@@ -36,6 +36,7 @@ type RawExecDriver struct {
 // rawExecHandle is returned from Start/Open as a handle to the PID
 type rawExecHandle struct {
 	pluginClient *plugin.Client
+	userPid      int
 	executor     plugins.Executor
 	killTimeout  time.Duration
 	logger       *log.Logger
@@ -102,6 +103,8 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 		HandshakeConfig: plugins.HandshakeConfig,
 		Plugins:         plugins.PluginMap,
 		Cmd:             exec.Command(bin, "executor"),
+		SyncStdout:      d.config.LogOutput,
+		SyncStderr:      d.config.LogOutput,
 	}
 
 	executor, pluginClient, err := d.executor(pluginConfig)
@@ -115,6 +118,7 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 	}
 	ps, err := executor.LaunchCmd(&plugins.ExecCommand{Cmd: command, Args: driverConfig.Args}, executorCtx)
 	if err != nil {
+		pluginClient.Kill()
 		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
 	}
 	d.logger.Printf("DIPTANU Started process via plugin: %#v", ps)
@@ -123,6 +127,7 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 	h := &rawExecHandle{
 		pluginClient: pluginClient,
 		executor:     executor,
+		userPid:      ps.Pid,
 		killTimeout:  d.DriverContext.KillTimeout(task),
 		logger:       d.logger,
 		doneCh:       make(chan struct{}),
@@ -150,6 +155,7 @@ func (d *RawExecDriver) executor(config *plugin.ClientConfig) (plugins.Executor,
 type rawExecId struct {
 	KillTimeout  time.Duration
 	PluginConfig *plugin.ReattachConfig
+	UserPid      int
 }
 
 func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
@@ -168,6 +174,8 @@ func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, e
 		Plugins:         plugins.PluginMap,
 		Cmd:             exec.Command(bin, "executor"),
 		Reattach:        id.PluginConfig,
+		SyncStdout:      d.config.LogOutput,
+		SyncStderr:      d.config.LogOutput,
 	}
 	executor, client, err := d.executor(pluginConfig)
 	if err != nil {
@@ -191,6 +199,7 @@ func (h *rawExecHandle) ID() string {
 	id := rawExecId{
 		KillTimeout:  h.killTimeout,
 		PluginConfig: h.pluginClient.ReattachConfig(),
+		UserPid:      h.userPid,
 	}
 
 	data, err := json.Marshal(id)

From 8d24ddc296e04eb2fe38b1d26a6564fe121224c5 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Wed, 3 Feb 2016 17:46:45 -0800
Subject: [PATCH 13/68] Moving the java executor to the executor plugin

---
 client/driver/java.go     | 129 ++++++++++++++++++++++++++------------
 client/driver/raw_exec.go |   5 +-
 2 files changed, 91 insertions(+), 43 deletions(-)

diff --git a/client/driver/java.go b/client/driver/java.go
index 3cb32679c..79bf9454f 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -12,14 +12,17 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/hashicorp/go-plugin"
+	"github.com/mitchellh/mapstructure"
+
 	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
-	"github.com/hashicorp/nomad/client/driver/executor"
+	"github.com/hashicorp/nomad/client/driver/plugins"
 	cstructs "github.com/hashicorp/nomad/client/driver/structs"
 	"github.com/hashicorp/nomad/client/fingerprint"
 	"github.com/hashicorp/nomad/client/getter"
+	"github.com/hashicorp/nomad/helper/discover"
 	"github.com/hashicorp/nomad/nomad/structs"
-	"github.com/mitchellh/mapstructure"
 )
 
 // JavaDriver is a simple driver to execute applications packaged in Jars.
@@ -38,7 +41,10 @@ type JavaDriverConfig struct {
 
 // javaHandle is returned from Start/Open as a handle to the PID
 type javaHandle struct {
-	cmd         executor.Executor
+	pluginClient *plugin.Client
+	userPid      int
+	executor     plugins.Executor
+
 	killTimeout time.Duration
 	logger      *log.Logger
 	waitCh      chan *cstructs.WaitResult
@@ -138,42 +144,69 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		args = append(args, driverConfig.Args...)
 	}
 
-	// Setup the command
-	// Assumes Java is in the $PATH, but could probably be detected
-	execCtx := executor.NewExecutorContext(d.taskEnv)
-	cmd := executor.Command(execCtx, "java", args...)
-
-	// Populate environment variables
-	cmd.Command().Env = d.taskEnv.EnvList()
-
-	if err := cmd.Limit(task.Resources); err != nil {
-		return nil, fmt.Errorf("failed to constrain resources: %s", err)
+	bin, err := discover.NomadExecutable()
+	if err != nil {
+		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
+	}
+	pluginConfig := &plugin.ClientConfig{
+		HandshakeConfig: plugins.HandshakeConfig,
+		Plugins:         plugins.PluginMap,
+		Cmd:             exec.Command(bin, "executor"),
+		SyncStdout:      d.config.LogOutput,
+		SyncStderr:      d.config.LogOutput,
 	}
 
-	if err := cmd.ConfigureTaskDir(d.taskName, ctx.AllocDir); err != nil {
-		return nil, fmt.Errorf("failed to configure task directory: %v", err)
+	executor, pluginClient, err := d.executor(pluginConfig)
+	if err != nil {
+		return nil, err
 	}
-
-	if err := cmd.Start(); err != nil {
-		return nil, fmt.Errorf("failed to start source: %v", err)
+	executorCtx := &plugins.ExecutorContext{
+		TaskEnv:  d.taskEnv,
+		AllocDir: ctx.AllocDir,
+		Task:     task,
 	}
+	ps, err := executor.LaunchCmd(&plugins.ExecCommand{Cmd: "java", Args: args}, executorCtx)
+	if err != nil {
+		pluginClient.Kill()
+		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
+	}
+	d.logger.Printf("[INFO] started process with pid: %v", ps.Pid)
 
 	// Return a driver handle
 	h := &javaHandle{
-		cmd:         cmd,
-		killTimeout: d.DriverContext.KillTimeout(task),
-		logger:      d.logger,
-		doneCh:      make(chan struct{}),
-		waitCh:      make(chan *cstructs.WaitResult, 1),
+		pluginClient: pluginClient,
+		executor:     executor,
+		userPid:      ps.Pid,
+		killTimeout:  d.DriverContext.KillTimeout(task),
+		logger:       d.logger,
+		doneCh:       make(chan struct{}),
+		waitCh:       make(chan *cstructs.WaitResult, 1),
 	}
 
 	go h.run()
 	return h, nil
 }
 
+func (d *JavaDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *plugin.Client, error) {
+	executorClient := plugin.NewClient(config)
+	rpcClient, err := executorClient.Client()
+	if err != nil {
+		return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
+	}
+	rpcClient.SyncStreams(d.config.LogOutput, d.config.LogOutput)
+
+	raw, err := rpcClient.Dispense("executor")
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
+	}
+	executorPlugin := raw.(plugins.Executor)
+	return executorPlugin, executorClient, nil
+}
+
 type javaId struct {
-	ExecutorId  string
-	KillTimeout time.Duration
+	KillTimeout  time.Duration
+	PluginConfig *plugin.ReattachConfig
+	UserPid      int
 }
 
 func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
@@ -182,20 +215,33 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
 	}
 
-	// Find the process
-	execCtx := executor.NewExecutorContext(d.taskEnv)
-	cmd, err := executor.OpenId(execCtx, id.ExecutorId)
+	bin, err := discover.NomadExecutable()
 	if err != nil {
-		return nil, fmt.Errorf("failed to open ID %v: %v", id.ExecutorId, err)
+		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
+	}
+
+	pluginConfig := &plugin.ClientConfig{
+		HandshakeConfig: plugins.HandshakeConfig,
+		Plugins:         plugins.PluginMap,
+		Cmd:             exec.Command(bin, "executor"),
+		Reattach:        id.PluginConfig,
+		SyncStdout:      d.config.LogOutput,
+		SyncStderr:      d.config.LogOutput,
+	}
+	executor, client, err := d.executor(pluginConfig)
+	if err != nil {
+		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
 
 	// Return a driver handle
 	h := &javaHandle{
-		cmd:         cmd,
-		logger:      d.logger,
-		killTimeout: id.KillTimeout,
-		doneCh:      make(chan struct{}),
-		waitCh:      make(chan *cstructs.WaitResult, 1),
+		pluginClient: client,
+		executor:     executor,
+		userPid:      id.UserPid,
+		logger:       d.logger,
+		killTimeout:  id.KillTimeout,
+		doneCh:       make(chan struct{}),
+		waitCh:       make(chan *cstructs.WaitResult, 1),
 	}
 
 	go h.run()
@@ -203,10 +249,10 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 }
 
 func (h *javaHandle) ID() string {
-	executorId, _ := h.cmd.ID()
 	id := javaId{
-		ExecutorId:  executorId,
-		KillTimeout: h.killTimeout,
+		KillTimeout:  h.killTimeout,
+		PluginConfig: h.pluginClient.ReattachConfig(),
+		UserPid:      h.userPid,
 	}
 
 	data, err := json.Marshal(id)
@@ -229,18 +275,19 @@ func (h *javaHandle) Update(task *structs.Task) error {
 }
 
 func (h *javaHandle) Kill() error {
-	h.cmd.Shutdown()
+	h.executor.ShutDown()
 	select {
 	case <-h.doneCh:
 		return nil
 	case <-time.After(h.killTimeout):
-		return h.cmd.ForceStop()
+		return h.executor.Exit()
 	}
 }
 
 func (h *javaHandle) run() {
-	res := h.cmd.Wait()
+	ps, err := h.executor.Wait()
 	close(h.doneCh)
-	h.waitCh <- res
+	h.waitCh <- &cstructs.WaitResult{ExitCode: ps.ExitCode, Signal: 0, Err: err}
 	close(h.waitCh)
+	h.pluginClient.Kill()
 }
diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go
index 4a871753c..e5a945807 100644
--- a/client/driver/raw_exec.go
+++ b/client/driver/raw_exec.go
@@ -121,7 +121,7 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 		pluginClient.Kill()
 		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
 	}
-	d.logger.Printf("DIPTANU Started process via plugin: %#v", ps)
+	d.logger.Printf("[INFO] started process with pid: %v", ps.Pid)
 
 	// Return a driver handle
 	h := &rawExecHandle{
@@ -183,9 +183,10 @@ func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, e
 	}
 
 	// Return a driver handle
-	h := &execHandle{
+	h := &rawExecHandle{
 		pluginClient: client,
 		executor:     executor,
+		userPid:      id.UserPid,
 		logger:       d.logger,
 		killTimeout:  id.KillTimeout,
 		doneCh:       make(chan struct{}),

From 68d0848998b32dca1138b29a8cca370f77a246d4 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Wed, 3 Feb 2016 18:01:34 -0800
Subject: [PATCH 14/68] Moved qemu to executor plugin

---
 client/driver/qemu.go     | 135 ++++++++++++++++++++++++++------------
 client/driver/raw_exec.go |   3 +-
 2 files changed, 93 insertions(+), 45 deletions(-)

diff --git a/client/driver/qemu.go b/client/driver/qemu.go
index df36b4cd3..d9df73d36 100644
--- a/client/driver/qemu.go
+++ b/client/driver/qemu.go
@@ -11,12 +11,14 @@ import (
 	"strings"
 	"time"
 
+	"github.com/hashicorp/go-plugin"
 	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
-	"github.com/hashicorp/nomad/client/driver/executor"
+	"github.com/hashicorp/nomad/client/driver/plugins"
 	cstructs "github.com/hashicorp/nomad/client/driver/structs"
 	"github.com/hashicorp/nomad/client/fingerprint"
 	"github.com/hashicorp/nomad/client/getter"
+	"github.com/hashicorp/nomad/helper/discover"
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/mitchellh/mapstructure"
 )
@@ -42,11 +44,13 @@ type QemuDriverConfig struct {
 
 // qemuHandle is returned from Start/Open as a handle to the PID
 type qemuHandle struct {
-	cmd         executor.Executor
-	killTimeout time.Duration
-	logger      *log.Logger
-	waitCh      chan *cstructs.WaitResult
-	doneCh      chan struct{}
+	pluginClient *plugin.Client
+	userPid      int
+	executor     plugins.Executor
+	killTimeout  time.Duration
+	logger       *log.Logger
+	waitCh       chan *cstructs.WaitResult
+	doneCh       chan struct{}
 }
 
 // NewQemuDriver is used to create a new exec driver
@@ -183,39 +187,71 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		)
 	}
 
-	// Setup the command
-	execCtx := executor.NewExecutorContext(d.taskEnv)
-	cmd := executor.Command(execCtx, args[0], args[1:]...)
-	if err := cmd.Limit(task.Resources); err != nil {
-		return nil, fmt.Errorf("failed to constrain resources: %s", err)
-	}
-
-	if err := cmd.ConfigureTaskDir(d.taskName, ctx.AllocDir); err != nil {
-		return nil, fmt.Errorf("failed to configure task directory: %v", err)
-	}
-
 	d.logger.Printf("[DEBUG] Starting QemuVM command: %q", strings.Join(args, " "))
-	if err := cmd.Start(); err != nil {
-		return nil, fmt.Errorf("failed to start command: %v", err)
+	bin, err := discover.NomadExecutable()
+	if err != nil {
+		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
+	pluginConfig := &plugin.ClientConfig{
+		HandshakeConfig: plugins.HandshakeConfig,
+		Plugins:         plugins.PluginMap,
+		Cmd:             exec.Command(bin, "executor"),
+		SyncStdout:      d.config.LogOutput,
+		SyncStderr:      d.config.LogOutput,
+	}
+
+	executor, pluginClient, err := d.executor(pluginConfig)
+	if err != nil {
+		return nil, err
+	}
+	executorCtx := &plugins.ExecutorContext{
+		TaskEnv:  d.taskEnv,
+		AllocDir: ctx.AllocDir,
+		Task:     task,
+	}
+	ps, err := executor.LaunchCmd(&plugins.ExecCommand{Cmd: args[0], Args: args[1:]}, executorCtx)
+	if err != nil {
+		pluginClient.Kill()
+		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
+	}
+	d.logger.Printf("[INFO] started process with pid: %v", ps.Pid)
 	d.logger.Printf("[INFO] Started new QemuVM: %s", vmID)
 
 	// Create and Return Handle
 	h := &qemuHandle{
-		cmd:         cmd,
-		killTimeout: d.DriverContext.KillTimeout(task),
-		logger:      d.logger,
-		doneCh:      make(chan struct{}),
-		waitCh:      make(chan *cstructs.WaitResult, 1),
+		pluginClient: pluginClient,
+		executor:     executor,
+		userPid:      ps.Pid,
+		killTimeout:  d.DriverContext.KillTimeout(task),
+		logger:       d.logger,
+		doneCh:       make(chan struct{}),
+		waitCh:       make(chan *cstructs.WaitResult, 1),
 	}
 
 	go h.run()
 	return h, nil
 }
 
+func (d *QemuDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *plugin.Client, error) {
+	executorClient := plugin.NewClient(config)
+	rpcClient, err := executorClient.Client()
+	if err != nil {
+		return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
+	}
+	rpcClient.SyncStreams(d.config.LogOutput, d.config.LogOutput)
+
+	raw, err := rpcClient.Dispense("executor")
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
+	}
+	executorPlugin := raw.(plugins.Executor)
+	return executorPlugin, executorClient, nil
+}
+
 type qemuId struct {
-	ExecutorId  string
-	KillTimeout time.Duration
+	KillTimeout  time.Duration
+	PluginConfig *plugin.ReattachConfig
+	UserPid      int
 }
 
 func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
@@ -224,30 +260,42 @@ func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
 	}
 
-	// Find the process
-	execCtx := executor.NewExecutorContext(d.taskEnv)
-	cmd, err := executor.OpenId(execCtx, id.ExecutorId)
+	bin, err := discover.NomadExecutable()
 	if err != nil {
-		return nil, fmt.Errorf("failed to open ID %v: %v", id.ExecutorId, err)
+		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
+	}
+	pluginConfig := &plugin.ClientConfig{
+		HandshakeConfig: plugins.HandshakeConfig,
+		Plugins:         plugins.PluginMap,
+		Cmd:             exec.Command(bin, "executor"),
+		Reattach:        id.PluginConfig,
+		SyncStdout:      d.config.LogOutput,
+		SyncStderr:      d.config.LogOutput,
+	}
+	executor, client, err := d.executor(pluginConfig)
+	if err != nil {
+		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
 
 	// Return a driver handle
-	h := &execHandle{
-		cmd:         cmd,
-		logger:      d.logger,
-		killTimeout: id.KillTimeout,
-		doneCh:      make(chan struct{}),
-		waitCh:      make(chan *cstructs.WaitResult, 1),
+	h := &qemuHandle{
+		pluginClient: client,
+		executor:     executor,
+		userPid:      id.UserPid,
+		logger:       d.logger,
+		killTimeout:  id.KillTimeout,
+		doneCh:       make(chan struct{}),
+		waitCh:       make(chan *cstructs.WaitResult, 1),
 	}
 	go h.run()
 	return h, nil
 }
 
 func (h *qemuHandle) ID() string {
-	executorId, _ := h.cmd.ID()
 	id := qemuId{
-		ExecutorId:  executorId,
-		KillTimeout: h.killTimeout,
+		KillTimeout:  h.killTimeout,
+		PluginConfig: h.pluginClient.ReattachConfig(),
+		UserPid:      h.userPid,
 	}
 
 	data, err := json.Marshal(id)
@@ -272,18 +320,19 @@ func (h *qemuHandle) Update(task *structs.Task) error {
 // TODO: allow a 'shutdown_command' that can be executed over a ssh connection
 // to the VM
 func (h *qemuHandle) Kill() error {
-	h.cmd.Shutdown()
+	h.executor.ShutDown()
 	select {
 	case <-h.doneCh:
 		return nil
 	case <-time.After(h.killTimeout):
-		return h.cmd.ForceStop()
+		return h.executor.Exit()
 	}
 }
 
 func (h *qemuHandle) run() {
-	res := h.cmd.Wait()
+	ps, err := h.executor.Wait()
 	close(h.doneCh)
-	h.waitCh <- res
+	h.waitCh <- &cstructs.WaitResult{ExitCode: ps.ExitCode, Signal: 0, Err: err}
 	close(h.waitCh)
+	h.pluginClient.Kill()
 }
diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go
index e5a945807..e0e889b6d 100644
--- a/client/driver/raw_exec.go
+++ b/client/driver/raw_exec.go
@@ -228,8 +228,7 @@ func (h *rawExecHandle) Kill() error {
 	case <-h.doneCh:
 		return nil
 	case <-time.After(h.killTimeout):
-		err := h.executor.Exit()
-		return err
+		return h.executor.Exit()
 	}
 }
 

From f1e2644623fe654ed505861d1dae1018be29a296 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Wed, 3 Feb 2016 18:03:05 -0800
Subject: [PATCH 15/68] Removed executor and spawn daemon

---
 client/driver/exec.go                       |   2 -
 client/driver/executor/exec.go              | 123 ------
 client/driver/executor/exec_basic.go        | 131 ------
 client/driver/executor/exec_basic_test.go   |   8 -
 client/driver/executor/exec_linux.go        | 426 --------------------
 client/driver/executor/exec_linux_test.go   |  18 -
 client/driver/executor/exec_universal.go    |  14 -
 client/driver/executor/test_harness_test.go | 286 -------------
 client/driver/spawn/spawn.go                | 308 --------------
 client/driver/spawn/spawn_posix.go          |  20 -
 client/driver/spawn/spawn_test.go           | 335 ---------------
 client/driver/spawn/spawn_windows.go        |  21 -
 12 files changed, 1692 deletions(-)
 delete mode 100644 client/driver/executor/exec.go
 delete mode 100644 client/driver/executor/exec_basic.go
 delete mode 100644 client/driver/executor/exec_basic_test.go
 delete mode 100644 client/driver/executor/exec_linux.go
 delete mode 100644 client/driver/executor/exec_linux_test.go
 delete mode 100644 client/driver/executor/exec_universal.go
 delete mode 100644 client/driver/executor/test_harness_test.go
 delete mode 100644 client/driver/spawn/spawn.go
 delete mode 100644 client/driver/spawn/spawn_posix.go
 delete mode 100644 client/driver/spawn/spawn_test.go
 delete mode 100644 client/driver/spawn/spawn_windows.go

diff --git a/client/driver/exec.go b/client/driver/exec.go
index 011b2c7f6..1e54c03f4 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -12,7 +12,6 @@ import (
 	"github.com/hashicorp/go-plugin"
 	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
-	"github.com/hashicorp/nomad/client/driver/executor"
 	"github.com/hashicorp/nomad/client/driver/plugins"
 	cstructs "github.com/hashicorp/nomad/client/driver/structs"
 	"github.com/hashicorp/nomad/client/getter"
@@ -38,7 +37,6 @@ type ExecDriverConfig struct {
 type execHandle struct {
 	pluginClient *plugin.Client
 	executor     plugins.Executor
-	cmd          executor.Executor
 	killTimeout  time.Duration
 	logger       *log.Logger
 	waitCh       chan *cstructs.WaitResult
diff --git a/client/driver/executor/exec.go b/client/driver/executor/exec.go
deleted file mode 100644
index 5acf8eccc..000000000
--- a/client/driver/executor/exec.go
+++ /dev/null
@@ -1,123 +0,0 @@
-// Package executor is used to invoke child processes across various operating
-// systems in a way that provides the following features:
-//
-// - Least privilege
-// - Resource constraints
-// - Process isolation
-//
-// An operating system may be something like "windows" or "linux with systemd".
-// Executors allow drivers like `exec` and `java` to share an implementation
-// for isolation capabilities on a particular operating system.
-//
-// For example:
-//
-// - `exec` and `java` on Linux use a cgroups executor
-// - `exec` and `java` on FreeBSD use a jails executor
-//
-// However, drivers that provide their own isolation should not use executors.
-// For example, using an executor to start QEMU means that the QEMU call is
-// run inside a chroot+cgroup, even though the VM already provides isolation for
-// the task running inside it. This is an extraneous level of indirection.
-package executor
-
-import (
-	"fmt"
-	"os/exec"
-	"path/filepath"
-
-	"github.com/hashicorp/nomad/client/allocdir"
-	"github.com/hashicorp/nomad/nomad/structs"
-
-	"github.com/hashicorp/nomad/client/driver/env"
-	cstructs "github.com/hashicorp/nomad/client/driver/structs"
-)
-
-var errNoResources = fmt.Errorf("No resources are associated with this task")
-
-// Executor is an interface that any platform- or capability-specific exec
-// wrapper must implement. You should not need to implement a Java executor.
-// Rather, you would implement a cgroups executor that the Java driver will use.
-type Executor interface {
-	// Limit must be called before Start and restricts the amount of resources
-	// the process can use. Note that an error may be returned ONLY IF the
-	// executor implements resource limiting. Otherwise Limit is ignored.
-	Limit(*structs.Resources) error
-
-	// ConfigureTaskDir must be called before Start and ensures that the tasks
-	// directory is properly configured.
-	ConfigureTaskDir(taskName string, alloc *allocdir.AllocDir) error
-
-	// Start the process. This may wrap the actual process in another command,
-	// depending on the capabilities in this environment. Errors that arise from
-	// Limits or Runas may bubble through Start()
-	Start() error
-
-	// Open should be called to restore a previous execution. This might be needed if
-	// nomad is restarted.
-	Open(string) error
-
-	// Wait waits till the user's command is completed.
-	Wait() *cstructs.WaitResult
-
-	// Returns a handle that is executor specific for use in reopening.
-	ID() (string, error)
-
-	// Shutdown should use a graceful stop mechanism so the application can
-	// perform checkpointing or cleanup, if such a mechanism is available.
-	// If such a mechanism is not available, Shutdown() should call ForceStop().
-	Shutdown() error
-
-	// ForceStop will terminate the process without waiting for cleanup. Every
-	// implementations must provide this.
-	ForceStop() error
-
-	// Command provides access the underlying Cmd struct in case the Executor
-	// interface doesn't expose the functionality you need.
-	Command() *exec.Cmd
-}
-
-// ExecutorContext is a means to inject dependencies such as loggers, configs, and
-// node attributes into a Driver without having to change the Driver interface
-// each time we do it. Used in conjection with Factory, above.
-type ExecutorContext struct {
-	taskEnv *env.TaskEnvironment
-}
-
-// NewExecutorContext initializes a new DriverContext with the specified fields.
-func NewExecutorContext(taskEnv *env.TaskEnvironment) *ExecutorContext {
-	return &ExecutorContext{
-		taskEnv: taskEnv,
-	}
-}
-
-// Command returns a platform-specific Executor
-func Command(ctx *ExecutorContext, name string, args ...string) Executor {
-	executor := NewExecutor(ctx)
-	SetCommand(executor, name, args)
-	return executor
-}
-
-func SetCommand(e Executor, name string, args []string) {
-	cmd := e.Command()
-	cmd.Path = name
-	cmd.Args = append([]string{name}, args...)
-
-	if filepath.Base(name) == name {
-		if lp, err := exec.LookPath(name); err != nil {
-			// cmd.lookPathErr = err
-		} else {
-			cmd.Path = lp
-		}
-	}
-}
-
-// OpenId is similar to executor.Command but will attempt to reopen with the
-// passed ID.
-func OpenId(ctx *ExecutorContext, id string) (Executor, error) {
-	executor := NewExecutor(ctx)
-	err := executor.Open(id)
-	if err != nil {
-		return nil, err
-	}
-	return executor, nil
-}
diff --git a/client/driver/executor/exec_basic.go b/client/driver/executor/exec_basic.go
deleted file mode 100644
index b0b687245..000000000
--- a/client/driver/executor/exec_basic.go
+++ /dev/null
@@ -1,131 +0,0 @@
-package executor
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"runtime"
-	"strings"
-
-	"github.com/hashicorp/nomad/client/allocdir"
-	"github.com/hashicorp/nomad/client/driver/spawn"
-	"github.com/hashicorp/nomad/nomad/structs"
-
-	cstructs "github.com/hashicorp/nomad/client/driver/structs"
-)
-
-// BasicExecutor should work everywhere, and as a result does not include
-// any resource restrictions or runas capabilities.
-type BasicExecutor struct {
-	*ExecutorContext
-	cmd      exec.Cmd
-	spawn    *spawn.Spawner
-	taskName string
-	taskDir  string
-	allocDir string
-}
-
-func NewBasicExecutor(ctx *ExecutorContext) Executor {
-	return &BasicExecutor{ExecutorContext: ctx}
-}
-
-func (e *BasicExecutor) Limit(resources *structs.Resources) error {
-	if resources == nil {
-		return errNoResources
-	}
-	return nil
-}
-
-func (e *BasicExecutor) ConfigureTaskDir(taskName string, alloc *allocdir.AllocDir) error {
-	taskDir, ok := alloc.TaskDirs[taskName]
-	if !ok {
-		return fmt.Errorf("Couldn't find task directory for task %v", taskName)
-	}
-	e.cmd.Dir = taskDir
-
-	e.taskDir = taskDir
-	e.taskName = taskName
-	e.allocDir = alloc.AllocDir
-	return nil
-}
-
-func (e *BasicExecutor) Start() error {
-	// Parse the commands arguments and replace instances of Nomad environment
-	// variables.
-	e.cmd.Path = e.taskEnv.ReplaceEnv(e.cmd.Path)
-	e.cmd.Args = e.taskEnv.ParseAndReplace(e.cmd.Args)
-	e.cmd.Env = e.taskEnv.Build().EnvList()
-
-	spawnState := filepath.Join(e.allocDir, fmt.Sprintf("%s_%s", e.taskName, "exit_status"))
-	e.spawn = spawn.NewSpawner(spawnState)
-	e.spawn.SetCommand(&e.cmd)
-	e.spawn.SetLogs(&spawn.Logs{
-		Stdout: filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", e.taskName)),
-		Stderr: filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", e.taskName)),
-		Stdin:  os.DevNull,
-	})
-
-	return e.spawn.Spawn(nil)
-}
-
-func (e *BasicExecutor) Open(id string) error {
-	var spawn spawn.Spawner
-	dec := json.NewDecoder(strings.NewReader(id))
-	if err := dec.Decode(&spawn); err != nil {
-		return fmt.Errorf("Failed to parse id: %v", err)
-	}
-
-	// Setup the executor.
-	e.spawn = &spawn
-	return e.spawn.Valid()
-}
-
-func (e *BasicExecutor) Wait() *cstructs.WaitResult {
-	return e.spawn.Wait()
-}
-
-func (e *BasicExecutor) ID() (string, error) {
-	if e.spawn == nil {
-		return "", fmt.Errorf("Process was never started")
-	}
-
-	var buffer bytes.Buffer
-	enc := json.NewEncoder(&buffer)
-	if err := enc.Encode(e.spawn); err != nil {
-		return "", fmt.Errorf("Failed to serialize id: %v", err)
-	}
-
-	return buffer.String(), nil
-}
-
-func (e *BasicExecutor) Shutdown() error {
-	proc, err := os.FindProcess(e.spawn.UserPid)
-	if err != nil {
-		return fmt.Errorf("Failed to find user processes %v: %v", e.spawn.UserPid, err)
-	}
-
-	if runtime.GOOS == "windows" {
-		return proc.Kill()
-	}
-
-	return proc.Signal(os.Interrupt)
-}
-
-func (e *BasicExecutor) ForceStop() error {
-	proc, err := os.FindProcess(e.spawn.UserPid)
-	if err != nil {
-		return fmt.Errorf("Failed to find user processes %v: %v", e.spawn.UserPid, err)
-	}
-
-	if err := proc.Kill(); err != nil && err.Error() != "os: process already finished" {
-		return err
-	}
-	return nil
-}
-
-func (e *BasicExecutor) Command() *exec.Cmd {
-	return &e.cmd
-}
diff --git a/client/driver/executor/exec_basic_test.go b/client/driver/executor/exec_basic_test.go
deleted file mode 100644
index 1a829b95d..000000000
--- a/client/driver/executor/exec_basic_test.go
+++ /dev/null
@@ -1,8 +0,0 @@
-package executor
-
-import "testing"
-
-func TestExecutorBasic(t *testing.T) {
-	t.Parallel()
-	testExecutor(t, NewBasicExecutor, nil)
-}
diff --git a/client/driver/executor/exec_linux.go b/client/driver/executor/exec_linux.go
deleted file mode 100644
index 221751644..000000000
--- a/client/driver/executor/exec_linux.go
+++ /dev/null
@@ -1,426 +0,0 @@
-package executor
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"os"
-	"os/exec"
-	"os/user"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"sync"
-	"syscall"
-
-	"github.com/hashicorp/go-multierror"
-	"github.com/opencontainers/runc/libcontainer/cgroups"
-	cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
-	"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
-	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
-
-	"github.com/hashicorp/nomad/client/allocdir"
-	"github.com/hashicorp/nomad/client/driver/spawn"
-	cstructs "github.com/hashicorp/nomad/client/driver/structs"
-	"github.com/hashicorp/nomad/nomad/structs"
-)
-
-var (
-	// A mapping of directories on the host OS to attempt to embed inside each
-	// task's chroot.
-	chrootEnv = map[string]string{
-		"/bin":       "/bin",
-		"/etc":       "/etc",
-		"/lib":       "/lib",
-		"/lib32":     "/lib32",
-		"/lib64":     "/lib64",
-		"/usr/bin":   "/usr/bin",
-		"/usr/lib":   "/usr/lib",
-		"/usr/share": "/usr/share",
-	}
-)
-
-func NewExecutor(ctx *ExecutorContext) Executor {
-	return NewLinuxExecutor(ctx)
-}
-
-func NewLinuxExecutor(ctx *ExecutorContext) Executor {
-	return &LinuxExecutor{ExecutorContext: ctx}
-}
-
-// Linux executor is designed to run on linux kernel 2.8+.
-type LinuxExecutor struct {
-	*ExecutorContext
-	cmd  exec.Cmd
-	user *user.User
-	l    sync.Mutex
-
-	// Isolation configurations.
-	groups   *cgroupConfig.Cgroup
-	taskName string
-	taskDir  string
-	allocDir string
-
-	// Spawn process.
-	spawn *spawn.Spawner
-}
-
-func (e *LinuxExecutor) Command() *exec.Cmd {
-	return &e.cmd
-}
-
-func (e *LinuxExecutor) Limit(resources *structs.Resources) error {
-	if resources == nil {
-		return errNoResources
-	}
-
-	return e.configureCgroups(resources)
-}
-
-// execLinuxID contains the necessary information to reattach to an executed
-// process and cleanup the created cgroups.
-type ExecLinuxID struct {
-	Groups  *cgroupConfig.Cgroup
-	Spawn   *spawn.Spawner
-	TaskDir string
-}
-
-func (e *LinuxExecutor) Open(id string) error {
-	// De-serialize the ID.
-	dec := json.NewDecoder(strings.NewReader(id))
-	var execID ExecLinuxID
-	if err := dec.Decode(&execID); err != nil {
-		return fmt.Errorf("Failed to parse id: %v", err)
-	}
-
-	// Setup the executor.
-	e.groups = execID.Groups
-	e.spawn = execID.Spawn
-	e.taskDir = execID.TaskDir
-	return e.spawn.Valid()
-}
-
-func (e *LinuxExecutor) ID() (string, error) {
-	if e.groups == nil || e.spawn == nil || e.taskDir == "" {
-		return "", fmt.Errorf("LinuxExecutor not properly initialized.")
-	}
-
-	// Build the ID.
-	id := ExecLinuxID{
-		Groups:  e.groups,
-		Spawn:   e.spawn,
-		TaskDir: e.taskDir,
-	}
-
-	var buffer bytes.Buffer
-	enc := json.NewEncoder(&buffer)
-	if err := enc.Encode(id); err != nil {
-		return "", fmt.Errorf("Failed to serialize id: %v", err)
-	}
-
-	return buffer.String(), nil
-}
-
-// runAs takes a user id as a string and looks up the user, and sets the command
-// to execute as that user.
-func (e *LinuxExecutor) runAs(userid string) error {
-	u, err := user.Lookup(userid)
-	if err != nil {
-		return fmt.Errorf("Failed to identify user %v: %v", userid, err)
-	}
-
-	// Convert the uid and gid
-	uid, err := strconv.ParseUint(u.Uid, 10, 32)
-	if err != nil {
-		return fmt.Errorf("Unable to convert userid to uint32: %s", err)
-	}
-	gid, err := strconv.ParseUint(u.Gid, 10, 32)
-	if err != nil {
-		return fmt.Errorf("Unable to convert groupid to uint32: %s", err)
-	}
-
-	// Set the command to run as that user and group.
-	if e.cmd.SysProcAttr == nil {
-		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
-	}
-	if e.cmd.SysProcAttr.Credential == nil {
-		e.cmd.SysProcAttr.Credential = &syscall.Credential{}
-	}
-	e.cmd.SysProcAttr.Credential.Uid = uint32(uid)
-	e.cmd.SysProcAttr.Credential.Gid = uint32(gid)
-
-	return nil
-}
-
-func (e *LinuxExecutor) Start() error {
-	// Run as "nobody" user so we don't leak root privilege to the spawned
-	// process.
-	if err := e.runAs("nobody"); err != nil {
-		return err
-	}
-
-	// Parse the commands arguments and replace instances of Nomad environment
-	// variables.
-	e.cmd.Path = e.taskEnv.ReplaceEnv(e.cmd.Path)
-	e.cmd.Args = e.taskEnv.ParseAndReplace(e.cmd.Args)
-	e.cmd.Env = e.taskEnv.EnvList()
-
-	spawnState := filepath.Join(e.allocDir, fmt.Sprintf("%s_%s", e.taskName, "exit_status"))
-	e.spawn = spawn.NewSpawner(spawnState)
-	e.spawn.SetCommand(&e.cmd)
-	e.spawn.SetChroot(e.taskDir)
-	e.spawn.SetLogs(&spawn.Logs{
-		Stdout: filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", e.taskName)),
-		Stderr: filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", e.taskName)),
-		Stdin:  os.DevNull,
-	})
-
-	enterCgroup := func(pid int) error {
-		// Join the spawn-daemon to the cgroup.
-		manager := e.getCgroupManager(e.groups)
-
-		// Apply will place the spawn dameon into the created cgroups.
-		if err := manager.Apply(pid); err != nil {
-			return fmt.Errorf("Failed to join spawn-daemon to the cgroup (%+v): %v", e.groups, err)
-		}
-
-		return nil
-	}
-
-	return e.spawn.Spawn(enterCgroup)
-}
-
-// Wait waits til the user process exits and returns an error on non-zero exit
-// codes. Wait also cleans up the task directory and created cgroups.
-func (e *LinuxExecutor) Wait() *cstructs.WaitResult {
-	errs := new(multierror.Error)
-	res := e.spawn.Wait()
-	if res.Err != nil {
-		errs = multierror.Append(errs, res.Err)
-	}
-
-	if err := e.destroyCgroup(); err != nil {
-		errs = multierror.Append(errs, err)
-	}
-
-	if err := e.cleanTaskDir(); err != nil {
-		errs = multierror.Append(errs, err)
-	}
-
-	res.Err = errs.ErrorOrNil()
-	return res
-}
-
-// Shutdown sends the user process an interrupt signal indicating that it is
-// about to be forcefully shutdown in sometime
-func (e *LinuxExecutor) Shutdown() error {
-	proc, err := os.FindProcess(e.spawn.UserPid)
-	if err != nil {
-		return fmt.Errorf("Failed to find user processes %v: %v", e.spawn.UserPid, err)
-	}
-
-	return proc.Signal(os.Interrupt)
-}
-
-// ForceStop immediately exits the user process and cleans up both the task
-// directory and the cgroups.
-func (e *LinuxExecutor) ForceStop() error {
-	errs := new(multierror.Error)
-	if err := e.destroyCgroup(); err != nil {
-		errs = multierror.Append(errs, err)
-	}
-
-	if err := e.cleanTaskDir(); err != nil {
-		errs = multierror.Append(errs, err)
-	}
-
-	return errs.ErrorOrNil()
-}
-
-// Task Directory related functions.
-
-// ConfigureTaskDir creates the necessary directory structure for a proper
-// chroot. cleanTaskDir should be called after.
-func (e *LinuxExecutor) ConfigureTaskDir(taskName string, alloc *allocdir.AllocDir) error {
-	e.taskName = taskName
-	e.allocDir = alloc.AllocDir
-
-	taskDir, ok := alloc.TaskDirs[taskName]
-	if !ok {
-		fmt.Errorf("Couldn't find task directory for task %v", taskName)
-	}
-	e.taskDir = taskDir
-
-	if err := alloc.MountSharedDir(taskName); err != nil {
-		return err
-	}
-
-	if err := alloc.Embed(taskName, chrootEnv); err != nil {
-		return err
-	}
-
-	// Mount dev
-	dev := filepath.Join(taskDir, "dev")
-	if !e.pathExists(dev) {
-		if err := os.Mkdir(dev, 0777); err != nil {
-			return fmt.Errorf("Mkdir(%v) failed: %v", dev, err)
-		}
-
-		if err := syscall.Mount("none", dev, "devtmpfs", syscall.MS_RDONLY, ""); err != nil {
-			return fmt.Errorf("Couldn't mount /dev to %v: %v", dev, err)
-		}
-	}
-
-	// Mount proc
-	proc := filepath.Join(taskDir, "proc")
-	if !e.pathExists(proc) {
-		if err := os.Mkdir(proc, 0777); err != nil {
-			return fmt.Errorf("Mkdir(%v) failed: %v", proc, err)
-		}
-
-		if err := syscall.Mount("none", proc, "proc", syscall.MS_RDONLY, ""); err != nil {
-			return fmt.Errorf("Couldn't mount /proc to %v: %v", proc, err)
-		}
-	}
-
-	// Set the tasks AllocDir environment variable.
-	e.taskEnv.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)).SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)).Build()
-	return nil
-}
-
-// pathExists is a helper function to check if the path exists.
-func (e *LinuxExecutor) pathExists(path string) bool {
-	if _, err := os.Stat(path); err != nil {
-		if os.IsNotExist(err) {
-			return false
-		}
-	}
-	return true
-}
-
-// cleanTaskDir is an idempotent operation to clean the task directory and
-// should be called when tearing down the task.
-func (e *LinuxExecutor) cleanTaskDir() error {
-	// Prevent a race between Wait/ForceStop
-	e.l.Lock()
-	defer e.l.Unlock()
-
-	// Unmount dev.
-	errs := new(multierror.Error)
-	dev := filepath.Join(e.taskDir, "dev")
-	if e.pathExists(dev) {
-		if err := syscall.Unmount(dev, 0); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("Failed to unmount dev (%v): %v", dev, err))
-		}
-
-		if err := os.RemoveAll(dev); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("Failed to delete dev directory (%v): %v", dev, err))
-		}
-	}
-
-	// Unmount proc.
-	proc := filepath.Join(e.taskDir, "proc")
-	if e.pathExists(proc) {
-		if err := syscall.Unmount(proc, 0); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("Failed to unmount proc (%v): %v", proc, err))
-		}
-
-		if err := os.RemoveAll(proc); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("Failed to delete proc directory (%v): %v", dev, err))
-		}
-	}
-
-	return errs.ErrorOrNil()
-}
-
-// Cgroup related functions.
-
-// configureCgroups converts a Nomad Resources specification into the equivalent
-// cgroup configuration. It returns an error if the resources are invalid.
-func (e *LinuxExecutor) configureCgroups(resources *structs.Resources) error {
-	e.groups = &cgroupConfig.Cgroup{}
-	e.groups.Resources = &cgroupConfig.Resources{}
-	e.groups.Name = structs.GenerateUUID()
-
-	// TODO: verify this is needed for things like network access
-	e.groups.Resources.AllowAllDevices = true
-
-	if resources.MemoryMB > 0 {
-		// Total amount of memory allowed to consume
-		e.groups.Resources.Memory = int64(resources.MemoryMB * 1024 * 1024)
-		// Disable swap to avoid issues on the machine
-		e.groups.Resources.MemorySwap = int64(-1)
-	}
-
-	if resources.CPU < 2 {
-		return fmt.Errorf("resources.CPU must be equal to or greater than 2: %v", resources.CPU)
-	}
-
-	// Set the relative CPU shares for this cgroup.
-	e.groups.Resources.CpuShares = int64(resources.CPU)
-
-	if resources.IOPS != 0 {
-		// Validate it is in an acceptable range.
-		if resources.IOPS < 10 || resources.IOPS > 1000 {
-			return fmt.Errorf("resources.IOPS must be between 10 and 1000: %d", resources.IOPS)
-		}
-
-		e.groups.Resources.BlkioWeight = uint16(resources.IOPS)
-	}
-
-	return nil
-}
-
-// destroyCgroup kills all processes in the cgroup and removes the cgroup
-// configuration from the host.
-func (e *LinuxExecutor) destroyCgroup() error {
-	if e.groups == nil {
-		return errors.New("Can't destroy: cgroup configuration empty")
-	}
-
-	// Prevent a race between Wait/ForceStop
-	e.l.Lock()
-	defer e.l.Unlock()
-
-	manager := e.getCgroupManager(e.groups)
-	pids, err := manager.GetPids()
-	if err != nil {
-		return fmt.Errorf("Failed to get pids in the cgroup %v: %v", e.groups.Name, err)
-	}
-
-	errs := new(multierror.Error)
-	for _, pid := range pids {
-		process, err := os.FindProcess(pid)
-		if err != nil {
-			multierror.Append(errs, fmt.Errorf("Failed to find Pid %v: %v", pid, err))
-			continue
-		}
-
-		if err := process.Kill(); err != nil && err.Error() != "os: process already finished" {
-			multierror.Append(errs, fmt.Errorf("Failed to kill Pid %v: %v", pid, err))
-			continue
-		}
-	}
-
-	// Remove the cgroup.
-	if err := manager.Destroy(); err != nil {
-		multierror.Append(errs, fmt.Errorf("Failed to delete the cgroup directories: %v", err))
-	}
-
-	if len(errs.Errors) != 0 {
-		return fmt.Errorf("Failed to destroy cgroup: %v", errs)
-	}
-
-	return nil
-}
-
-// getCgroupManager returns the correct libcontainer cgroup manager.
-func (e *LinuxExecutor) getCgroupManager(groups *cgroupConfig.Cgroup) cgroups.Manager {
-	var manager cgroups.Manager
-	manager = &cgroupFs.Manager{Cgroups: groups}
-	if systemd.UseSystemd() {
-		manager = &systemd.Manager{Cgroups: groups}
-	}
-	return manager
-}
diff --git a/client/driver/executor/exec_linux_test.go b/client/driver/executor/exec_linux_test.go
deleted file mode 100644
index 6cc2d104c..000000000
--- a/client/driver/executor/exec_linux_test.go
+++ /dev/null
@@ -1,18 +0,0 @@
-package executor
-
-import (
-	"testing"
-
-	ctestutil "github.com/hashicorp/nomad/client/testutil"
-	"github.com/hashicorp/nomad/helper/testtask"
-)
-
-func init() {
-	// Add test binary to chroot during test run.
-	chrootEnv[testtask.Path()] = testtask.Path()
-}
-
-func TestExecutorLinux(t *testing.T) {
-	t.Parallel()
-	testExecutor(t, NewLinuxExecutor, ctestutil.ExecCompatible)
-}
diff --git a/client/driver/executor/exec_universal.go b/client/driver/executor/exec_universal.go
deleted file mode 100644
index 5ce25ec8e..000000000
--- a/client/driver/executor/exec_universal.go
+++ /dev/null
@@ -1,14 +0,0 @@
-// +build !linux
-
-package executor
-
-func NewExecutor(ctx *ExecutorContext) Executor {
-	return &UniversalExecutor{
-		BasicExecutor: NewBasicExecutor(ctx).(*BasicExecutor),
-	}
-}
-
-// UniversalExecutor wraps the BasicExecutor
-type UniversalExecutor struct {
-	*BasicExecutor
-}
diff --git a/client/driver/executor/test_harness_test.go b/client/driver/executor/test_harness_test.go
deleted file mode 100644
index 5574b2c7b..000000000
--- a/client/driver/executor/test_harness_test.go
+++ /dev/null
@@ -1,286 +0,0 @@
-package executor
-
-import (
-	"io/ioutil"
-	"log"
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/hashicorp/nomad/client/allocdir"
-	"github.com/hashicorp/nomad/client/driver/env"
-	"github.com/hashicorp/nomad/helper/testtask"
-	"github.com/hashicorp/nomad/nomad/mock"
-	"github.com/hashicorp/nomad/nomad/structs"
-	"github.com/hashicorp/nomad/testutil"
-)
-
-func TestMain(m *testing.M) {
-	if !testtask.Run() {
-		os.Exit(m.Run())
-	}
-}
-
-var (
-	constraint = &structs.Resources{
-		CPU:      250,
-		MemoryMB: 256,
-		Networks: []*structs.NetworkResource{
-			&structs.NetworkResource{
-				MBits:        50,
-				DynamicPorts: []structs.Port{{Label: "http"}},
-			},
-		},
-	}
-)
-
-func mockAllocDir(t *testing.T) (string, *allocdir.AllocDir) {
-	alloc := mock.Alloc()
-	task := alloc.Job.TaskGroups[0].Tasks[0]
-
-	allocDir := allocdir.NewAllocDir(filepath.Join(os.TempDir(), alloc.ID))
-	if err := allocDir.Build([]*structs.Task{task}); err != nil {
-		log.Panicf("allocDir.Build() failed: %v", err)
-	}
-
-	return task.Name, allocDir
-}
-
-func testExecutorContext() *ExecutorContext {
-	taskEnv := env.NewTaskEnvironment(mock.Node())
-	return &ExecutorContext{taskEnv: taskEnv}
-}
-
-func testExecutor(t *testing.T, buildExecutor func(*ExecutorContext) Executor, compatible func(*testing.T)) {
-	if compatible != nil {
-		compatible(t)
-	}
-
-	command := func(name string, args ...string) Executor {
-		ctx := testExecutorContext()
-		e := buildExecutor(ctx)
-		SetCommand(e, name, args)
-		testtask.SetEnv(ctx.taskEnv)
-		return e
-	}
-
-	Executor_Start_Invalid(t, command)
-	Executor_Start_Wait_Failure_Code(t, command)
-	Executor_Start_Wait(t, command)
-	Executor_Start_Kill(t, command)
-	Executor_Open(t, command, buildExecutor)
-	Executor_Open_Invalid(t, command, buildExecutor)
-}
-
-type buildExecCommand func(name string, args ...string) Executor
-
-func Executor_Start_Invalid(t *testing.T, command buildExecCommand) {
-	invalid := "/bin/foobar"
-	e := command(invalid, "1")
-
-	if err := e.Limit(constraint); err != nil {
-		log.Panicf("Limit() failed: %v", err)
-	}
-
-	task, alloc := mockAllocDir(t)
-	defer alloc.Destroy()
-	if err := e.ConfigureTaskDir(task, alloc); err != nil {
-		log.Panicf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err)
-	}
-
-	if err := e.Start(); err == nil {
-		log.Panicf("Start(%v) should have failed", invalid)
-	}
-}
-
-func Executor_Start_Wait_Failure_Code(t *testing.T, command buildExecCommand) {
-	e := command(testtask.Path(), "fail")
-
-	if err := e.Limit(constraint); err != nil {
-		log.Panicf("Limit() failed: %v", err)
-	}
-
-	task, alloc := mockAllocDir(t)
-	defer alloc.Destroy()
-	if err := e.ConfigureTaskDir(task, alloc); err != nil {
-		log.Panicf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err)
-	}
-
-	if err := e.Start(); err != nil {
-		log.Panicf("Start() failed: %v", err)
-	}
-
-	if err := e.Wait(); err == nil {
-		log.Panicf("Wait() should have failed")
-	}
-}
-
-func Executor_Start_Wait(t *testing.T, command buildExecCommand) {
-	task, alloc := mockAllocDir(t)
-	defer alloc.Destroy()
-
-	taskDir, ok := alloc.TaskDirs[task]
-	if !ok {
-		log.Panicf("No task directory found for task %v", task)
-	}
-
-	expected := "hello world"
-	file := filepath.Join(allocdir.TaskLocal, "output.txt")
-	absFilePath := filepath.Join(taskDir, file)
-	e := command(testtask.Path(), "sleep", "1s", "write", expected, file)
-
-	if err := e.Limit(constraint); err != nil {
-		log.Panicf("Limit() failed: %v", err)
-	}
-
-	if err := e.ConfigureTaskDir(task, alloc); err != nil {
-		log.Panicf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err)
-	}
-
-	if err := e.Start(); err != nil {
-		log.Panicf("Start() failed: %v", err)
-	}
-
-	if res := e.Wait(); !res.Successful() {
-		log.Panicf("Wait() failed: %v", res)
-	}
-
-	output, err := ioutil.ReadFile(absFilePath)
-	if err != nil {
-		log.Panicf("Couldn't read file %v", absFilePath)
-	}
-
-	act := string(output)
-	if act != expected {
-		log.Panicf("Command output incorrectly: want %v; got %v", expected, act)
-	}
-}
-
-func Executor_Start_Kill(t *testing.T, command buildExecCommand) {
-	task, alloc := mockAllocDir(t)
-	defer alloc.Destroy()
-
-	taskDir, ok := alloc.TaskDirs[task]
-	if !ok {
-		log.Panicf("No task directory found for task %v", task)
-	}
-
-	filePath := filepath.Join(taskDir, "output")
-	e := command(testtask.Path(), "sleep", "1s", "write", "failure", filePath)
-
-	if err := e.Limit(constraint); err != nil {
-		log.Panicf("Limit() failed: %v", err)
-	}
-
-	if err := e.ConfigureTaskDir(task, alloc); err != nil {
-		log.Panicf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err)
-	}
-
-	if err := e.Start(); err != nil {
-		log.Panicf("Start() failed: %v", err)
-	}
-
-	if err := e.Shutdown(); err != nil {
-		log.Panicf("Shutdown() failed: %v", err)
-	}
-
-	time.Sleep(time.Duration(testutil.TestMultiplier()*2) * time.Second)
-
-	// Check that the file doesn't exist.
-	if _, err := os.Stat(filePath); err == nil {
-		log.Panicf("Stat(%v) should have failed: task not killed", filePath)
-	}
-}
-
-func Executor_Open(t *testing.T, command buildExecCommand, newExecutor func(*ExecutorContext) Executor) {
-	task, alloc := mockAllocDir(t)
-	defer alloc.Destroy()
-
-	taskDir, ok := alloc.TaskDirs[task]
-	if !ok {
-		log.Panicf("No task directory found for task %v", task)
-	}
-
-	expected := "hello world"
-	file := filepath.Join(allocdir.TaskLocal, "output.txt")
-	absFilePath := filepath.Join(taskDir, file)
-	e := command(testtask.Path(), "sleep", "1s", "write", expected, file)
-
-	if err := e.Limit(constraint); err != nil {
-		log.Panicf("Limit() failed: %v", err)
-	}
-
-	if err := e.ConfigureTaskDir(task, alloc); err != nil {
-		log.Panicf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err)
-	}
-
-	if err := e.Start(); err != nil {
-		log.Panicf("Start() failed: %v", err)
-	}
-
-	id, err := e.ID()
-	if err != nil {
-		log.Panicf("ID() failed: %v", err)
-	}
-
-	e2 := newExecutor(testExecutorContext())
-	if err := e2.Open(id); err != nil {
-		log.Panicf("Open(%v) failed: %v", id, err)
-	}
-
-	if res := e2.Wait(); !res.Successful() {
-		log.Panicf("Wait() failed: %v", res)
-	}
-
-	output, err := ioutil.ReadFile(absFilePath)
-	if err != nil {
-		log.Panicf("Couldn't read file %v", absFilePath)
-	}
-
-	act := string(output)
-	if act != expected {
-		log.Panicf("Command output incorrectly: want %v; got %v", expected, act)
-	}
-}
-
-func Executor_Open_Invalid(t *testing.T, command buildExecCommand, newExecutor func(*ExecutorContext) Executor) {
-	task, alloc := mockAllocDir(t)
-	e := command(testtask.Path(), "echo", "foo")
-
-	if err := e.Limit(constraint); err != nil {
-		log.Panicf("Limit() failed: %v", err)
-	}
-
-	if err := e.ConfigureTaskDir(task, alloc); err != nil {
-		log.Panicf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err)
-	}
-
-	if err := e.Start(); err != nil {
-		log.Panicf("Start() failed: %v", err)
-	}
-
-	id, err := e.ID()
-	if err != nil {
-		log.Panicf("ID() failed: %v", err)
-	}
-
-	// Kill the task because some OSes (windows) will not let us destroy the
-	// alloc (below) if the task is still running.
-	if err := e.ForceStop(); err != nil {
-		log.Panicf("e.ForceStop() failed: %v", err)
-	}
-
-	// Wait until process is actually gone, we don't care what the result was.
-	e.Wait()
-
-	// Destroy the allocdir which removes the exit code.
-	if err := alloc.Destroy(); err != nil {
-		log.Panicf("alloc.Destroy() failed: %v", err)
-	}
-
-	e2 := newExecutor(testExecutorContext())
-	if err := e2.Open(id); err == nil {
-		log.Panicf("Open(%v) should have failed", id)
-	}
-}
diff --git a/client/driver/spawn/spawn.go b/client/driver/spawn/spawn.go
deleted file mode 100644
index 5b3200984..000000000
--- a/client/driver/spawn/spawn.go
+++ /dev/null
@@ -1,308 +0,0 @@
-package spawn
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io"
-	"os"
-	"os/exec"
-	"strconv"
-	"time"
-
-	"github.com/hashicorp/go-multierror"
-	"github.com/hashicorp/nomad/client/driver/structs"
-	"github.com/hashicorp/nomad/command"
-	"github.com/hashicorp/nomad/helper/discover"
-)
-
-// Spawner is used to start a user command in an isolated fashion that is
-// resistent to Nomad agent failure.
-type Spawner struct {
-	spawn     *os.Process
-	SpawnPid  int
-	SpawnPpid int
-	StateFile string
-	UserPid   int
-
-	// User configuration
-	UserCmd *exec.Cmd
-	Logs    *Logs
-	Chroot  string
-}
-
-// Logs is used to define the filepaths the user command's logs should be
-// redirected to. The files do not need to exist.
-type Logs struct {
-	Stdin, Stdout, Stderr string
-}
-
-// NewSpawner takes a path to a state file. This state file can be used to
-// create a new Spawner that can be used to wait on the exit status of a
-// process even through Nomad restarts.
-func NewSpawner(stateFile string) *Spawner {
-	return &Spawner{StateFile: stateFile}
-}
-
-// SetCommand sets the user command to spawn.
-func (s *Spawner) SetCommand(cmd *exec.Cmd) {
-	s.UserCmd = cmd
-}
-
-// SetLogs sets the redirection of user command log files.
-func (s *Spawner) SetLogs(l *Logs) {
-	s.Logs = l
-}
-
-// SetChroot puts the user command into a chroot.
-func (s *Spawner) SetChroot(root string) {
-	s.Chroot = root
-}
-
-// Spawn does a double-fork to start and isolate the user command. It takes a
-// call-back that is invoked with the pid of the intermediary process. If the
-// call back returns an error, the user command is not started and the spawn is
-// cancelled. This can be used to put the process into a cgroup or jail and
-// cancel starting the user process if that was not successful. An error is
-// returned if the call-back returns an error or the user-command couldn't be
-// started.
-func (s *Spawner) Spawn(cb func(pid int) error) error {
-	bin, err := discover.NomadExecutable()
-	if err != nil {
-		return fmt.Errorf("Failed to determine the nomad executable: %v", err)
-	}
-
-	exitFile, err := os.OpenFile(s.StateFile, os.O_CREATE|os.O_WRONLY, 0666)
-	if err != nil {
-		return fmt.Errorf("Error opening file to store exit status: %v", err)
-	}
-	defer exitFile.Close()
-
-	config, err := s.spawnConfig()
-	if err != nil {
-		return err
-	}
-
-	spawn := exec.Command(bin, "spawn-daemon", config)
-
-	// Capture stdout
-	spawnStdout, err := spawn.StdoutPipe()
-	if err != nil {
-		return fmt.Errorf("Failed to capture spawn-daemon stdout: %v", err)
-	}
-	defer spawnStdout.Close()
-
-	// Capture stdin.
-	spawnStdin, err := spawn.StdinPipe()
-	if err != nil {
-		return fmt.Errorf("Failed to capture spawn-daemon stdin: %v", err)
-	}
-	defer spawnStdin.Close()
-
-	if err := spawn.Start(); err != nil {
-		return fmt.Errorf("Failed to call spawn-daemon on nomad executable: %v", err)
-	}
-
-	if cb != nil {
-		cbErr := cb(spawn.Process.Pid)
-		if cbErr != nil {
-			errs := new(multierror.Error)
-			errs = multierror.Append(errs, cbErr)
-			if err := s.sendAbortCommand(spawnStdin); err != nil {
-				errs = multierror.Append(errs, err)
-			}
-
-			return errs
-		}
-	}
-
-	if err := s.sendStartCommand(spawnStdin); err != nil {
-		return err
-	}
-
-	respCh := make(chan command.SpawnStartStatus, 1)
-	errCh := make(chan error, 1)
-
-	go func() {
-		var resp command.SpawnStartStatus
-		dec := json.NewDecoder(spawnStdout)
-		if err := dec.Decode(&resp); err != nil {
-			errCh <- fmt.Errorf("Failed to parse spawn-daemon start response: %v", err)
-		}
-		respCh <- resp
-	}()
-
-	select {
-	case err := <-errCh:
-		return err
-	case resp := <-respCh:
-		if resp.ErrorMsg != "" {
-			return fmt.Errorf("Failed to execute user command: %s", resp.ErrorMsg)
-		}
-		s.UserPid = resp.UserPID
-	case <-time.After(5 * time.Second):
-		return fmt.Errorf("timed out waiting for response")
-	}
-
-	// Store the spawn process.
-	s.spawn = spawn.Process
-	s.SpawnPid = s.spawn.Pid
-	s.SpawnPpid = os.Getpid()
-	return nil
-}
-
-// spawnConfig returns a serialized config to pass to the Nomad spawn-daemon
-// command.
-func (s *Spawner) spawnConfig() (string, error) {
-	if s.UserCmd == nil {
-		return "", fmt.Errorf("Must specify user command")
-	}
-
-	config := command.DaemonConfig{
-		Cmd:            *s.UserCmd,
-		Chroot:         s.Chroot,
-		ExitStatusFile: s.StateFile,
-	}
-
-	if s.Logs != nil {
-		config.StdoutFile = s.Logs.Stdout
-		config.StdinFile = s.Logs.Stdin
-		config.StderrFile = s.Logs.Stderr
-	}
-
-	var buffer bytes.Buffer
-	enc := json.NewEncoder(&buffer)
-	if err := enc.Encode(config); err != nil {
-		return "", fmt.Errorf("Failed to serialize configuration: %v", err)
-	}
-
-	return strconv.Quote(buffer.String()), nil
-}
-
-// sendStartCommand sends the necessary command to the spawn-daemon to have it
-// start the user process.
-func (s *Spawner) sendStartCommand(w io.Writer) error {
-	enc := json.NewEncoder(w)
-	if err := enc.Encode(true); err != nil {
-		return fmt.Errorf("Failed to serialize start command: %v", err)
-	}
-
-	return nil
-}
-
-// sendAbortCommand sends the necessary command to the spawn-daemon to have it
-// abort starting the user process. This should be invoked if the spawn-daemon
-// could not be isolated into a cgroup.
-func (s *Spawner) sendAbortCommand(w io.Writer) error {
-	enc := json.NewEncoder(w)
-	if err := enc.Encode(false); err != nil {
-		return fmt.Errorf("Failed to serialize abort command: %v", err)
-	}
-
-	return nil
-}
-
-// Wait returns the exit code of the user process or an error if the wait
-// failed.
-func (s *Spawner) Wait() *structs.WaitResult {
-	if os.Getpid() == s.SpawnPpid {
-		return s.waitAsParent()
-	}
-
-	return s.pollWait()
-}
-
-// waitAsParent waits on the process if the current process was the spawner.
-func (s *Spawner) waitAsParent() *structs.WaitResult {
-	if s.SpawnPpid != os.Getpid() {
-		return structs.NewWaitResult(-1, 0, fmt.Errorf("not the parent. Spawner parent is %v; current pid is %v", s.SpawnPpid, os.Getpid()))
-	}
-
-	// Try to reattach to the spawn.
-	if s.spawn == nil {
-		// If it can't be reattached, it means the spawn process has exited so
-		// we should just read its exit file.
-		var err error
-		if s.spawn, err = os.FindProcess(s.SpawnPid); err != nil {
-			return s.pollWait()
-		}
-	}
-
-	if _, err := s.spawn.Wait(); err != nil {
-		return structs.NewWaitResult(-1, 0, err)
-	}
-
-	return s.pollWait()
-}
-
-// pollWait polls on the spawn daemon to determine when it exits. After it
-// exits, it reads the state file and returns the exit code and possibly an
-// error.
-func (s *Spawner) pollWait() *structs.WaitResult {
-	// Stat to check if it is there to avoid a race condition.
-	stat, err := os.Stat(s.StateFile)
-	if err != nil {
-		return structs.NewWaitResult(-1, 0, fmt.Errorf("Failed to Stat exit status file %v: %v", s.StateFile, err))
-	}
-
-	// If there is data it means that the file has already been written.
-	if stat.Size() > 0 {
-		return s.readExitCode()
-	}
-
-	// Read after the process exits.
-	ticker := time.NewTicker(5 * time.Second)
-	defer ticker.Stop()
-	for range ticker.C {
-		if !s.Alive() {
-			break
-		}
-	}
-
-	return s.readExitCode()
-}
-
-// readExitCode parses the state file and returns the exit code of the task. It
-// returns an error if the file can't be read.
-func (s *Spawner) readExitCode() *structs.WaitResult {
-	f, err := os.Open(s.StateFile)
-	if err != nil {
-		return structs.NewWaitResult(-1, 0, fmt.Errorf("Failed to open %v to read exit code: %v", s.StateFile, err))
-	}
-	defer f.Close()
-
-	stat, err := f.Stat()
-	if err != nil {
-		return structs.NewWaitResult(-1, 0, fmt.Errorf("Failed to stat file %v: %v", s.StateFile, err))
-	}
-
-	if stat.Size() == 0 {
-		return structs.NewWaitResult(-1, 0, fmt.Errorf("Empty state file: %v", s.StateFile))
-	}
-
-	var exitStatus command.SpawnExitStatus
-	dec := json.NewDecoder(f)
-	if err := dec.Decode(&exitStatus); err != nil {
-		return structs.NewWaitResult(-1, 0, fmt.Errorf("Failed to parse exit status from %v: %v", s.StateFile, err))
-	}
-
-	return structs.NewWaitResult(exitStatus.ExitCode, 0, nil)
-}
-
-// Valid checks that the state of the Spawner is valid and that a subsequent
-// Wait could be called. This is useful to call when reopening a Spawner
-// through client restarts. If Valid a nil error is returned.
-func (s *Spawner) Valid() error {
-	// If the spawner is still alive, then the task is running and we can wait
-	// on it.
-	if s.Alive() {
-		return nil
-	}
-
-	// The task isn't alive so check that there is a valid exit code file.
-	if res := s.readExitCode(); res.Err == nil {
-		return nil
-	}
-
-	return fmt.Errorf("Spawner not alive and exit code not written")
-}
diff --git a/client/driver/spawn/spawn_posix.go b/client/driver/spawn/spawn_posix.go
deleted file mode 100644
index 9b10caddc..000000000
--- a/client/driver/spawn/spawn_posix.go
+++ /dev/null
@@ -1,20 +0,0 @@
-// +build !windows
-
-package spawn
-
-import (
-	"os"
-	"syscall"
-)
-
-func (s *Spawner) Alive() bool {
-	if s.spawn == nil {
-		var err error
-		if s.spawn, err = os.FindProcess(s.SpawnPid); err != nil {
-			return false
-		}
-	}
-
-	err := s.spawn.Signal(syscall.Signal(0))
-	return err == nil
-}
diff --git a/client/driver/spawn/spawn_test.go b/client/driver/spawn/spawn_test.go
deleted file mode 100644
index f231a335c..000000000
--- a/client/driver/spawn/spawn_test.go
+++ /dev/null
@@ -1,335 +0,0 @@
-package spawn
-
-import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/hashicorp/nomad/helper/testtask"
-)
-
-func TestMain(m *testing.M) {
-	if !testtask.Run() {
-		os.Exit(m.Run())
-	}
-}
-
-func TestSpawn_NoCmd(t *testing.T) {
-	t.Parallel()
-	tempFile := tempFileName(t)
-	defer os.Remove(tempFile)
-
-	spawn := NewSpawner(tempFile)
-	if err := spawn.Spawn(nil); err == nil {
-		t.Fatalf("Spawn() with no user command should fail")
-	}
-}
-
-func TestSpawn_InvalidCmd(t *testing.T) {
-	t.Parallel()
-	tempFile := tempFileName(t)
-	defer os.Remove(tempFile)
-
-	spawn := NewSpawner(tempFile)
-	spawn.SetCommand(exec.Command("foo")) // non-existent command
-	if err := spawn.Spawn(nil); err == nil {
-		t.Fatalf("Spawn() with an invalid command should fail")
-	}
-}
-
-func TestSpawn_SetsLogs(t *testing.T) {
-	t.Parallel()
-	tempFile := tempFileName(t)
-	defer os.Remove(tempFile)
-
-	spawn := NewSpawner(tempFile)
-	exp := "foo"
-	spawn.SetCommand(testCommand("echo", exp))
-
-	// Create file for stdout.
-	stdout := tempFileName(t)
-	defer os.Remove(stdout)
-	spawn.SetLogs(&Logs{Stdout: stdout})
-
-	if err := spawn.Spawn(nil); err != nil {
-		t.Fatalf("Spawn() failed: %v", err)
-	}
-
-	if res := spawn.Wait(); res.ExitCode != 0 && res.Err != nil {
-		t.Fatalf("Wait() returned %v, %v; want 0, nil", res.ExitCode, res.Err)
-	}
-
-	stdout2, err := os.Open(stdout)
-	if err != nil {
-		t.Fatalf("Open() failed: %v", err)
-	}
-
-	data, err := ioutil.ReadAll(stdout2)
-	if err != nil {
-		t.Fatalf("ReadAll() failed: %v", err)
-	}
-
-	act := strings.TrimSpace(string(data))
-	if act != exp {
-		t.Fatalf("Unexpected data written to stdout; got %v; want %v", act, exp)
-	}
-}
-
-func TestSpawn_Callback(t *testing.T) {
-	t.Parallel()
-	tempFile := tempFileName(t)
-	defer os.Remove(tempFile)
-
-	spawn := NewSpawner(tempFile)
-	spawn.SetCommand(testCommand("sleep", "1s"))
-
-	called := false
-	cbErr := fmt.Errorf("ERROR CB")
-	cb := func(_ int) error {
-		called = true
-		return cbErr
-	}
-
-	if err := spawn.Spawn(cb); err == nil {
-		t.Fatalf("Spawn(%#v) should have errored; want %v", cb, cbErr)
-	}
-
-	if !called {
-		t.Fatalf("Spawn(%#v) didn't call callback", cb)
-	}
-}
-
-func TestSpawn_ParentWaitExited(t *testing.T) {
-	t.Parallel()
-	tempFile := tempFileName(t)
-	defer os.Remove(tempFile)
-
-	spawn := NewSpawner(tempFile)
-	spawn.SetCommand(testCommand("echo", "foo"))
-	if err := spawn.Spawn(nil); err != nil {
-		t.Fatalf("Spawn() failed %v", err)
-	}
-
-	time.Sleep(1 * time.Second)
-
-	if res := spawn.Wait(); res.ExitCode != 0 && res.Err != nil {
-		t.Fatalf("Wait() returned %v, %v; want 0, nil", res.ExitCode, res.Err)
-	}
-}
-
-func TestSpawn_ParentWait(t *testing.T) {
-	t.Parallel()
-	tempFile := tempFileName(t)
-	defer os.Remove(tempFile)
-
-	spawn := NewSpawner(tempFile)
-	spawn.SetCommand(testCommand("sleep", "2s"))
-	if err := spawn.Spawn(nil); err != nil {
-		t.Fatalf("Spawn() failed %v", err)
-	}
-
-	if res := spawn.Wait(); res.ExitCode != 0 && res.Err != nil {
-		t.Fatalf("Wait() returned %v, %v; want 0, nil", res.ExitCode, res.Err)
-	}
-}
-
-func TestSpawn_NonParentWaitExited(t *testing.T) {
-	t.Parallel()
-	tempFile := tempFileName(t)
-	defer os.Remove(tempFile)
-
-	spawn := NewSpawner(tempFile)
-	spawn.SetCommand(testCommand("echo", "foo"))
-	if err := spawn.Spawn(nil); err != nil {
-		t.Fatalf("Spawn() failed %v", err)
-	}
-
-	time.Sleep(1 * time.Second)
-
-	// Force the wait to assume non-parent.
-	spawn.SpawnPpid = 0
-	if res := spawn.Wait(); res.ExitCode != 0 && res.Err != nil {
-		t.Fatalf("Wait() returned %v, %v; want 0, nil", res.ExitCode, res.Err)
-	}
-}
-
-func TestSpawn_NonParentWait(t *testing.T) {
-	t.Parallel()
-	tempFile := tempFileName(t)
-	defer os.Remove(tempFile)
-
-	spawn := NewSpawner(tempFile)
-	spawn.SetCommand(testCommand("sleep", "2s"))
-	if err := spawn.Spawn(nil); err != nil {
-		t.Fatalf("Spawn() failed %v", err)
-	}
-
-	// Need to wait on the spawner, otherwise it becomes a zombie and the test
-	// only finishes after the init process cleans it. This speeds that up.
-	go func() {
-		time.Sleep(3 * time.Second)
-		if _, err := spawn.spawn.Wait(); err != nil {
-			t.FailNow()
-		}
-	}()
-
-	// Force the wait to assume non-parent.
-	spawn.SpawnPpid = 0
-	if res := spawn.Wait(); res.ExitCode != 0 && res.Err != nil {
-		t.Fatalf("Wait() returned %v, %v; want 0, nil", res.ExitCode, res.Err)
-	}
-}
-
-func TestSpawn_DeadSpawnDaemon_Parent(t *testing.T) {
-	t.Parallel()
-	tempFile := tempFileName(t)
-	defer os.Remove(tempFile)
-
-	var spawnPid int
-	cb := func(pid int) error {
-		spawnPid = pid
-		return nil
-	}
-
-	spawn := NewSpawner(tempFile)
-	spawn.SetCommand(testCommand("sleep", "5s"))
-	if err := spawn.Spawn(cb); err != nil {
-		t.Fatalf("Spawn() errored: %v", err)
-	}
-
-	proc, err := os.FindProcess(spawnPid)
-	if err != nil {
-		t.FailNow()
-	}
-
-	if err := proc.Kill(); err != nil {
-		t.FailNow()
-	}
-
-	if _, err := proc.Wait(); err != nil {
-		t.FailNow()
-	}
-
-	if res := spawn.Wait(); res.Err == nil {
-		t.Fatalf("Wait() should have failed: %v", res.Err)
-	}
-}
-
-func TestSpawn_DeadSpawnDaemon_NonParent(t *testing.T) {
-	t.Parallel()
-	tempFile := tempFileName(t)
-	defer os.Remove(tempFile)
-
-	var spawnPid int
-	cb := func(pid int) error {
-		spawnPid = pid
-		return nil
-	}
-
-	spawn := NewSpawner(tempFile)
-	spawn.SetCommand(testCommand("sleep", "2s"))
-	if err := spawn.Spawn(cb); err != nil {
-		t.Fatalf("Spawn() errored: %v", err)
-	}
-
-	proc, err := os.FindProcess(spawnPid)
-	if err != nil {
-		t.FailNow()
-	}
-
-	if err := proc.Kill(); err != nil {
-		t.FailNow()
-	}
-
-	if _, err := proc.Wait(); err != nil {
-		t.FailNow()
-	}
-
-	// Force the wait to assume non-parent.
-	spawn.SpawnPpid = 0
-	if res := spawn.Wait(); res.Err == nil {
-		t.Fatalf("Wait() should have failed: %v", res.Err)
-	}
-}
-
-func TestSpawn_Valid_TaskRunning(t *testing.T) {
-	t.Parallel()
-	tempFile := tempFileName(t)
-	defer os.Remove(tempFile)
-
-	spawn := NewSpawner(tempFile)
-	spawn.SetCommand(testCommand("sleep", "2s"))
-	if err := spawn.Spawn(nil); err != nil {
-		t.Fatalf("Spawn() failed %v", err)
-	}
-
-	if err := spawn.Valid(); err != nil {
-		t.Fatalf("Valid() failed: %v", err)
-	}
-
-	if res := spawn.Wait(); res.Err != nil {
-		t.Fatalf("Wait() failed: %v", res.Err)
-	}
-}
-
-func TestSpawn_Valid_TaskExit_ExitCode(t *testing.T) {
-	t.Parallel()
-	tempFile := tempFileName(t)
-	defer os.Remove(tempFile)
-
-	spawn := NewSpawner(tempFile)
-	spawn.SetCommand(testCommand("echo", "foo"))
-	if err := spawn.Spawn(nil); err != nil {
-		t.Fatalf("Spawn() failed %v", err)
-	}
-
-	if res := spawn.Wait(); res.Err != nil {
-		t.Fatalf("Wait() failed: %v", res.Err)
-	}
-
-	if err := spawn.Valid(); err != nil {
-		t.Fatalf("Valid() failed: %v", err)
-	}
-}
-
-func TestSpawn_Valid_TaskExit_NoExitCode(t *testing.T) {
-	t.Parallel()
-	tempFile := tempFileName(t)
-	defer os.Remove(tempFile)
-
-	spawn := NewSpawner(tempFile)
-	spawn.SetCommand(testCommand("echo", "foo"))
-	if err := spawn.Spawn(nil); err != nil {
-		t.Fatalf("Spawn() failed %v", err)
-	}
-
-	if res := spawn.Wait(); res.Err != nil {
-		t.Fatalf("Wait() failed: %v", res.Err)
-	}
-
-	// Delete the file so that it can't find the exit code.
-	os.Remove(tempFile)
-
-	if err := spawn.Valid(); err == nil {
-		t.Fatalf("Valid() should have failed")
-	}
-}
-
-func tempFileName(t *testing.T) string {
-	f, err := ioutil.TempFile("", "")
-	if err != nil {
-		t.Fatalf("TempFile() failed")
-	}
-	defer f.Close()
-	return f.Name()
-}
-
-func testCommand(args ...string) *exec.Cmd {
-	cmd := exec.Command(testtask.Path(), args...)
-	testtask.SetCmdEnv(cmd)
-	return cmd
-}
diff --git a/client/driver/spawn/spawn_windows.go b/client/driver/spawn/spawn_windows.go
deleted file mode 100644
index 9683dce97..000000000
--- a/client/driver/spawn/spawn_windows.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package spawn
-
-import "syscall"
-
-const STILL_ACTIVE = 259
-
-func (s *Spawner) Alive() bool {
-	const da = syscall.STANDARD_RIGHTS_READ | syscall.PROCESS_QUERY_INFORMATION | syscall.SYNCHRONIZE
-	h, e := syscall.OpenProcess(da, false, uint32(s.SpawnPid))
-	if e != nil {
-		return false
-	}
-
-	var ec uint32
-	e = syscall.GetExitCodeProcess(h, &ec)
-	if e != nil {
-		return false
-	}
-
-	return ec == STILL_ACTIVE
-}

From 0a92842162a10d44ed8a00fffe1e8142179205ce Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Wed, 3 Feb 2016 19:00:21 -0800
Subject: [PATCH 16/68] Fixed the rpc server

---
 client/driver/exec.go                    |  2 --
 client/driver/plugins/executor_plugin.go | 10 ++++------
 client/driver/raw_exec.go                |  1 -
 3 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index 1e54c03f4..abb3344e7 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -142,7 +142,6 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 }
 
 type execId struct {
-	//ExecutorId   string
 	KillTimeout  time.Duration
 	PluginConfig *plugin.ReattachConfig
 }
@@ -188,7 +187,6 @@ func (d *ExecDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *p
 	if err != nil {
 		return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
 	}
-	rpcClient.SyncStreams(d.config.LogOutput, d.config.LogOutput)
 
 	raw, err := rpcClient.Dispense("executor")
 	if err != nil {
diff --git a/client/driver/plugins/executor_plugin.go b/client/driver/plugins/executor_plugin.go
index 52176c575..bc683070c 100644
--- a/client/driver/plugins/executor_plugin.go
+++ b/client/driver/plugins/executor_plugin.go
@@ -1,7 +1,6 @@
 package plugins
 
 import (
-	"fmt"
 	"log"
 	"net/rpc"
 	"os"
@@ -29,8 +28,8 @@ type LaunchCmdArgs struct {
 }
 
 func (e *ExecutorRPC) LaunchCmd(cmd *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
-	ps := new(ProcessState)
-	err := e.client.Call("Plugin.LaunchCmd", LaunchCmdArgs{Cmd: cmd, Ctx: ctx}, ps)
+	var ps *ProcessState
+	err := e.client.Call("Plugin.LaunchCmd", LaunchCmdArgs{Cmd: cmd, Ctx: ctx}, &ps)
 	return ps, err
 }
 
@@ -57,9 +56,8 @@ type ExecutorRPCServer struct {
 }
 
 func (e *ExecutorRPCServer) LaunchCmd(args LaunchCmdArgs, ps *ProcessState) error {
-	var err error
-	ps, err = e.Impl.LaunchCmd(args.Cmd, args.Ctx)
-	fmt.Printf("DIPTANU PS Server : %#v", ps)
+	state, err := e.Impl.LaunchCmd(args.Cmd, args.Ctx)
+	*ps = *state
 	return err
 }
 
diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go
index e0e889b6d..8819b98e4 100644
--- a/client/driver/raw_exec.go
+++ b/client/driver/raw_exec.go
@@ -142,7 +142,6 @@ func (d *RawExecDriver) executor(config *plugin.ClientConfig) (plugins.Executor,
 	if err != nil {
 		return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
 	}
-	rpcClient.SyncStreams(d.config.LogOutput, d.config.LogOutput)
 
 	raw, err := rpcClient.Dispense("executor")
 	if err != nil {

From 837b462f5c9fb4f9f08b793bb585884deaa763c6 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Wed, 3 Feb 2016 21:18:33 -0800
Subject: [PATCH 17/68] Fixed a comment

---
 client/driver/exec.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index abb3344e7..70d93b176 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -125,7 +125,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		pluginClient.Kill()
 		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
 	}
-	d.logger.Printf("DIPTANU Started process via plugin: %#v", ps)
+	d.logger.Printf("started process via plugin with pid: %v", ps.Pid)
 
 	// Return a driver handle
 	h := &execHandle{

From fedf3c791bb8ecbe8404fc8f7925f98ed40e1a84 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Thu, 4 Feb 2016 10:09:52 -0800
Subject: [PATCH 18/68] Fixed a test

---
 client/driver/exec.go                    | 15 ++++++---------
 client/driver/exec_test.go               |  3 +++
 client/driver/java.go                    |  7 ++++---
 client/driver/plugins/executor.go        | 13 +++++++------
 client/driver/plugins/executor_linux.go  |  6 +++---
 client/driver/plugins/executor_plugin.go | 22 ++++++++++++++++++++++
 client/driver/qemu.go                    |  7 ++++---
 client/driver/raw_exec.go                |  7 ++++---
 8 files changed, 53 insertions(+), 27 deletions(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index 70d93b176..da1d96a2c 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -115,7 +115,8 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 	executorCtx := &plugins.ExecutorContext{
 		TaskEnv:          d.taskEnv,
 		AllocDir:         ctx.AllocDir,
-		Task:             task,
+		TaskName:         task.Name,
+		TaskResources:    task.Resources,
 		ResourceLimits:   true,
 		FSIsolation:      true,
 		UnprivilegedUser: false,
@@ -143,7 +144,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 
 type execId struct {
 	KillTimeout  time.Duration
-	PluginConfig *plugin.ReattachConfig
+	PluginConfig *plugins.ExecutorReattachConfig
 }
 
 func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
@@ -152,16 +153,12 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
 	}
 
-	bin, err := discover.NomadExecutable()
-	if err != nil {
-		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
-	}
+	reattachConfig := id.PluginConfig.PluginConfig()
 
 	pluginConfig := &plugin.ClientConfig{
 		HandshakeConfig: plugins.HandshakeConfig,
 		Plugins:         plugins.PluginMap,
-		Cmd:             exec.Command(bin, "executor"),
-		Reattach:        id.PluginConfig,
+		Reattach:        reattachConfig,
 	}
 	executor, client, err := d.executor(pluginConfig)
 	if err != nil {
@@ -199,7 +196,7 @@ func (d *ExecDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *p
 func (h *execHandle) ID() string {
 	id := execId{
 		KillTimeout:  h.killTimeout,
-		PluginConfig: h.pluginClient.ReattachConfig(),
+		PluginConfig: plugins.NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
 	}
 
 	data, err := json.Marshal(id)
diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go
index 5b077e9d6..a1ea5948e 100644
--- a/client/driver/exec_test.go
+++ b/client/driver/exec_test.go
@@ -70,6 +70,9 @@ func TestExecDriver_StartOpen_Wait(t *testing.T) {
 	if handle2 == nil {
 		t.Fatalf("missing handle")
 	}
+
+	handle.Kill()
+	handle2.Kill()
 }
 
 func TestExecDriver_Start_Wait(t *testing.T) {
diff --git a/client/driver/java.go b/client/driver/java.go
index 79bf9454f..c455d457f 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -161,9 +161,10 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		return nil, err
 	}
 	executorCtx := &plugins.ExecutorContext{
-		TaskEnv:  d.taskEnv,
-		AllocDir: ctx.AllocDir,
-		Task:     task,
+		TaskEnv:       d.taskEnv,
+		AllocDir:      ctx.AllocDir,
+		TaskName:      task.Name,
+		TaskResources: task.Resources,
 	}
 	ps, err := executor.LaunchCmd(&plugins.ExecCommand{Cmd: "java", Args: args}, executorCtx)
 	if err != nil {
diff --git a/client/driver/plugins/executor.go b/client/driver/plugins/executor.go
index 4c0444f4b..2106caf71 100644
--- a/client/driver/plugins/executor.go
+++ b/client/driver/plugins/executor.go
@@ -21,7 +21,8 @@ import (
 type ExecutorContext struct {
 	TaskEnv          *env.TaskEnvironment
 	AllocDir         *allocdir.AllocDir
-	Task             *structs.Task
+	TaskName         string
+	TaskResources    *structs.Resources
 	FSIsolation      bool
 	ResourceLimits   bool
 	UnprivilegedUser bool
@@ -85,14 +86,14 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext
 		}
 	}
 
-	stdoPath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", ctx.Task.Name))
+	stdoPath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", ctx.TaskName))
 	stdo, err := os.OpenFile(stdoPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
 	if err != nil {
 		return nil, err
 	}
 	e.cmd.Stdout = stdo
 
-	stdePath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", ctx.Task.Name))
+	stdePath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", ctx.TaskName))
 	stde, err := os.OpenFile(stdePath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
 	if err != nil {
 		return nil, err
@@ -138,7 +139,7 @@ func (e *UniversalExecutor) wait() {
 }
 
 func (e *UniversalExecutor) Exit() error {
-	e.logger.Printf("[INFO] Exiting plugin for task %q", e.ctx.Task.Name)
+	e.logger.Printf("[INFO] Exiting plugin for task %q", e.ctx.TaskName)
 	proc, err := os.FindProcess(e.cmd.Process.Pid)
 	if err != nil {
 		return fmt.Errorf("failied to find user process %v: %v", e.cmd.Process.Pid, err)
@@ -164,10 +165,10 @@ func (e *UniversalExecutor) ShutDown() error {
 }
 
 func (e *UniversalExecutor) configureTaskDir() error {
-	taskDir, ok := e.ctx.AllocDir.TaskDirs[e.ctx.Task.Name]
+	taskDir, ok := e.ctx.AllocDir.TaskDirs[e.ctx.TaskName]
 	e.taskDir = taskDir
 	if !ok {
-		return fmt.Errorf("Couldn't find task directory for task %v", e.ctx.Task.Name)
+		return fmt.Errorf("Couldn't find task directory for task %v", e.ctx.TaskName)
 	}
 	e.cmd.Dir = taskDir
 	return nil
diff --git a/client/driver/plugins/executor_linux.go b/client/driver/plugins/executor_linux.go
index 8ae41d685..efdd4e410 100644
--- a/client/driver/plugins/executor_linux.go
+++ b/client/driver/plugins/executor_linux.go
@@ -41,7 +41,7 @@ func (e *UniversalExecutor) configureIsolation() error {
 	}
 
 	if e.ctx.ResourceLimits {
-		if err := e.configureCgroups(e.ctx.Task.Resources); err != nil {
+		if err := e.configureCgroups(e.ctx.TaskResources); err != nil {
 			return fmt.Errorf("error creating cgroups: %v", err)
 		}
 	}
@@ -142,11 +142,11 @@ func (e *UniversalExecutor) pathExists(path string) bool {
 
 func (e *UniversalExecutor) configureChroot() error {
 	allocDir := e.ctx.AllocDir
-	if err := allocDir.MountSharedDir(e.ctx.Task.Name); err != nil {
+	if err := allocDir.MountSharedDir(e.ctx.TaskName); err != nil {
 		return err
 	}
 
-	if err := allocDir.Embed(e.ctx.Task.Name, chrootEnv); err != nil {
+	if err := allocDir.Embed(e.ctx.TaskName, chrootEnv); err != nil {
 		return err
 	}
 
diff --git a/client/driver/plugins/executor_plugin.go b/client/driver/plugins/executor_plugin.go
index bc683070c..9506dea3d 100644
--- a/client/driver/plugins/executor_plugin.go
+++ b/client/driver/plugins/executor_plugin.go
@@ -2,6 +2,7 @@ package plugins
 
 import (
 	"log"
+	"net"
 	"net/rpc"
 	"os"
 
@@ -18,6 +19,27 @@ var PluginMap = map[string]plugin.Plugin{
 	"executor": new(ExecutorPlugin),
 }
 
+type ExecutorReattachConfig struct {
+	Pid      int
+	AddrNet  string
+	AddrName string
+}
+
+func (c *ExecutorReattachConfig) PluginConfig() *plugin.ReattachConfig {
+	var addr net.Addr
+	switch c.AddrNet {
+	case "unix", "unixgram", "unixpacket":
+		addr, _ = net.ResolveUnixAddr(c.AddrNet, c.AddrName)
+	case "tcp", "tcp4", "tcp6":
+		addr, _ = net.ResolveTCPAddr(c.AddrNet, c.AddrName)
+	}
+	return &plugin.ReattachConfig{Pid: c.Pid, Addr: addr}
+}
+
+func NewExecutorReattachConfig(c *plugin.ReattachConfig) *ExecutorReattachConfig {
+	return &ExecutorReattachConfig{Pid: c.Pid, AddrNet: c.Addr.Network(), AddrName: c.Addr.String()}
+}
+
 type ExecutorRPC struct {
 	client *rpc.Client
 }
diff --git a/client/driver/qemu.go b/client/driver/qemu.go
index d9df73d36..6903d303f 100644
--- a/client/driver/qemu.go
+++ b/client/driver/qemu.go
@@ -205,9 +205,10 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		return nil, err
 	}
 	executorCtx := &plugins.ExecutorContext{
-		TaskEnv:  d.taskEnv,
-		AllocDir: ctx.AllocDir,
-		Task:     task,
+		TaskEnv:       d.taskEnv,
+		AllocDir:      ctx.AllocDir,
+		TaskName:      task.Name,
+		TaskResources: task.Resources,
 	}
 	ps, err := executor.LaunchCmd(&plugins.ExecCommand{Cmd: args[0], Args: args[1:]}, executorCtx)
 	if err != nil {
diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go
index 8819b98e4..9f42f41c8 100644
--- a/client/driver/raw_exec.go
+++ b/client/driver/raw_exec.go
@@ -112,9 +112,10 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 		return nil, err
 	}
 	executorCtx := &plugins.ExecutorContext{
-		TaskEnv:  d.taskEnv,
-		AllocDir: ctx.AllocDir,
-		Task:     task,
+		TaskEnv:       d.taskEnv,
+		AllocDir:      ctx.AllocDir,
+		TaskName:      task.Name,
+		TaskResources: task.Resources,
 	}
 	ps, err := executor.LaunchCmd(&plugins.ExecCommand{Cmd: command, Args: driverConfig.Args}, executorCtx)
 	if err != nil {

From fe662ea648d38b67b77771841969a77230dc7d88 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Thu, 4 Feb 2016 10:21:33 -0800
Subject: [PATCH 19/68] Fixed a test related to wait

---
 client/driver/plugins/executor.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/driver/plugins/executor.go b/client/driver/plugins/executor.go
index 2106caf71..209536900 100644
--- a/client/driver/plugins/executor.go
+++ b/client/driver/plugins/executor.go
@@ -117,6 +117,7 @@ func (e *UniversalExecutor) Wait() (*ProcessState, error) {
 }
 
 func (e *UniversalExecutor) wait() {
+	defer close(e.processExited)
 	err := e.cmd.Wait()
 	if err == nil {
 		e.exitState = &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}
@@ -135,7 +136,6 @@ func (e *UniversalExecutor) wait() {
 		e.destroyCgroup()
 	}
 	e.exitState = &ProcessState{Pid: 0, ExitCode: exitCode, Time: time.Now()}
-	close(e.processExited)
 }
 
 func (e *UniversalExecutor) Exit() error {

From 797d9b8c476aa3ea3f3349b053f6c52766924abd Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Thu, 4 Feb 2016 11:51:43 -0800
Subject: [PATCH 20/68] Fixed the wait rpc server

---
 client/driver/exec.go                    |  2 ++
 client/driver/plugins/executor.go        | 18 ++++++++++--------
 client/driver/plugins/executor_plugin.go | 10 +++++++---
 3 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index da1d96a2c..fc0c49314 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -179,6 +179,8 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 }
 
 func (d *ExecDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *plugin.Client, error) {
+	config.SyncStdout = d.config.LogOutput
+	config.SyncStderr = d.config.LogOutput
 	executorClient := plugin.NewClient(config)
 	rpcClient, err := executorClient.Client()
 	if err != nil {
diff --git a/client/driver/plugins/executor.go b/client/driver/plugins/executor.go
index 209536900..3a9ae2b51 100644
--- a/client/driver/plugins/executor.go
+++ b/client/driver/plugins/executor.go
@@ -65,14 +65,7 @@ func NewExecutor(logger *log.Logger) Executor {
 
 func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
 	e.ctx = ctx
-	e.cmd.Path = command.Cmd
-	e.cmd.Args = append([]string{command.Cmd}, command.Args...)
-	if filepath.Base(command.Cmd) == command.Cmd {
-		if lp, err := exec.LookPath(command.Cmd); err != nil {
-		} else {
-			e.cmd.Path = lp
-		}
-	}
+
 	if err := e.configureTaskDir(); err != nil {
 		return nil, err
 	}
@@ -102,6 +95,15 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext
 
 	e.cmd.Env = ctx.TaskEnv.EnvList()
 
+	e.cmd.Path = ctx.TaskEnv.ReplaceEnv(command.Cmd)
+	e.cmd.Args = ctx.TaskEnv.ParseAndReplace(command.Args)
+	if filepath.Base(command.Cmd) == command.Cmd {
+		if lp, err := exec.LookPath(command.Cmd); err != nil {
+		} else {
+			e.cmd.Path = lp
+		}
+	}
+
 	if err := e.cmd.Start(); err != nil {
 		return nil, fmt.Errorf("error starting command: %v", err)
 	}
diff --git a/client/driver/plugins/executor_plugin.go b/client/driver/plugins/executor_plugin.go
index 9506dea3d..8dbdbe58a 100644
--- a/client/driver/plugins/executor_plugin.go
+++ b/client/driver/plugins/executor_plugin.go
@@ -79,13 +79,17 @@ type ExecutorRPCServer struct {
 
 func (e *ExecutorRPCServer) LaunchCmd(args LaunchCmdArgs, ps *ProcessState) error {
 	state, err := e.Impl.LaunchCmd(args.Cmd, args.Ctx)
-	*ps = *state
+	if state != nil {
+		*ps = *state
+	}
 	return err
 }
 
 func (e *ExecutorRPCServer) Wait(args interface{}, ps *ProcessState) error {
-	var err error
-	ps, err = e.Impl.Wait()
+	state, err := e.Impl.Wait()
+	if state != nil {
+		*ps = *state
+	}
 	return err
 }
 

From bc3ade3be930e016d285c6e779182474e8122b44 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Thu, 4 Feb 2016 12:21:06 -0800
Subject: [PATCH 21/68] Fixed creation of the command args

---
 client/driver/plugins/executor.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/driver/plugins/executor.go b/client/driver/plugins/executor.go
index 3a9ae2b51..d4c572e32 100644
--- a/client/driver/plugins/executor.go
+++ b/client/driver/plugins/executor.go
@@ -96,7 +96,7 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext
 	e.cmd.Env = ctx.TaskEnv.EnvList()
 
 	e.cmd.Path = ctx.TaskEnv.ReplaceEnv(command.Cmd)
-	e.cmd.Args = ctx.TaskEnv.ParseAndReplace(command.Args)
+	e.cmd.Args = append([]string{e.cmd.Path}, ctx.TaskEnv.ParseAndReplace(command.Args)...)
 	if filepath.Base(command.Cmd) == command.Cmd {
 		if lp, err := exec.LookPath(command.Cmd); err != nil {
 		} else {

From dbfe40798ff39e2e2dee9dcbd0c7238732619083 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Thu, 4 Feb 2016 12:40:48 -0800
Subject: [PATCH 22/68] Fixed the kill test

---
 client/driver/exec.go             |  3 +++
 client/driver/exec_test.go        |  2 +-
 client/driver/plugins/executor.go | 12 +++++++++---
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index fc0c49314..bff40f6a9 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -226,6 +226,9 @@ func (h *execHandle) Kill() error {
 	case <-h.doneCh:
 		return nil
 	case <-time.After(h.killTimeout):
+		if h.pluginClient.Exited() {
+			return nil
+		}
 		err := h.executor.Exit()
 		return err
 	}
diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go
index a1ea5948e..16342870b 100644
--- a/client/driver/exec_test.go
+++ b/client/driver/exec_test.go
@@ -265,7 +265,7 @@ func TestExecDriver_Start_Kill_Wait(t *testing.T) {
 		Name: "sleep",
 		Config: map[string]interface{}{
 			"command": "/bin/sleep",
-			"args":    []string{"1"},
+			"args":    []string{"10"},
 		},
 		Resources: basicResources,
 	}
diff --git a/client/driver/plugins/executor.go b/client/driver/plugins/executor.go
index d4c572e32..0ad3d62aa 100644
--- a/client/driver/plugins/executor.go
+++ b/client/driver/plugins/executor.go
@@ -152,18 +152,24 @@ func (e *UniversalExecutor) Exit() error {
 	if e.ctx.ResourceLimits {
 		e.destroyCgroup()
 	}
-	return proc.Kill()
+	if err = proc.Kill(); err != nil {
+		e.logger.Printf("[DEBUG] executor.exit error: %v", err)
+	}
+	return nil
 }
 
 func (e *UniversalExecutor) ShutDown() error {
 	proc, err := os.FindProcess(e.cmd.Process.Pid)
 	if err != nil {
-		return err
+		return fmt.Errorf("executor.shutdown error: %v", err)
 	}
 	if runtime.GOOS == "windows" {
 		return proc.Kill()
 	}
-	return proc.Signal(os.Interrupt)
+	if err = proc.Signal(os.Interrupt); err != nil {
+		return fmt.Errorf("executor.shutdown error: %v", err)
+	}
+	return nil
 }
 
 func (e *UniversalExecutor) configureTaskDir() error {

From d7a03de36888a1c12e8d4d12ec8392cda33b1f51 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Thu, 4 Feb 2016 12:55:13 -0800
Subject: [PATCH 23/68] Fixed the raw_exec_driver tests

---
 client/driver/exec.go          | 19 ++++++++-----------
 client/driver/raw_exec.go      | 26 ++++++++------------------
 client/driver/raw_exec_test.go |  2 ++
 3 files changed, 18 insertions(+), 29 deletions(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index bff40f6a9..adf0829eb 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -103,9 +103,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
 	pluginConfig := &plugin.ClientConfig{
-		HandshakeConfig: plugins.HandshakeConfig,
-		Plugins:         plugins.PluginMap,
-		Cmd:             exec.Command(bin, "executor"),
+		Cmd: exec.Command(bin, "executor"),
 	}
 
 	executor, pluginClient, err := d.executor(pluginConfig)
@@ -132,11 +130,10 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 	h := &execHandle{
 		pluginClient: pluginClient,
 		executor:     executor,
-		//cmd:          cmd,
-		killTimeout: d.DriverContext.KillTimeout(task),
-		logger:      d.logger,
-		doneCh:      make(chan struct{}),
-		waitCh:      make(chan *cstructs.WaitResult, 1),
+		killTimeout:  d.DriverContext.KillTimeout(task),
+		logger:       d.logger,
+		doneCh:       make(chan struct{}),
+		waitCh:       make(chan *cstructs.WaitResult, 1),
 	}
 	go h.run()
 	return h, nil
@@ -156,9 +153,7 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	reattachConfig := id.PluginConfig.PluginConfig()
 
 	pluginConfig := &plugin.ClientConfig{
-		HandshakeConfig: plugins.HandshakeConfig,
-		Plugins:         plugins.PluginMap,
-		Reattach:        reattachConfig,
+		Reattach: reattachConfig,
 	}
 	executor, client, err := d.executor(pluginConfig)
 	if err != nil {
@@ -179,6 +174,8 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 }
 
 func (d *ExecDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *plugin.Client, error) {
+	config.HandshakeConfig = plugins.HandshakeConfig
+	config.Plugins = plugins.PluginMap
 	config.SyncStdout = d.config.LogOutput
 	config.SyncStderr = d.config.LogOutput
 	executorClient := plugin.NewClient(config)
diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go
index 9f42f41c8..ed8aa3ec0 100644
--- a/client/driver/raw_exec.go
+++ b/client/driver/raw_exec.go
@@ -100,11 +100,7 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
 	pluginConfig := &plugin.ClientConfig{
-		HandshakeConfig: plugins.HandshakeConfig,
-		Plugins:         plugins.PluginMap,
-		Cmd:             exec.Command(bin, "executor"),
-		SyncStdout:      d.config.LogOutput,
-		SyncStderr:      d.config.LogOutput,
+		Cmd: exec.Command(bin, "executor"),
 	}
 
 	executor, pluginClient, err := d.executor(pluginConfig)
@@ -138,6 +134,10 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 	return h, nil
 }
 func (d *RawExecDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *plugin.Client, error) {
+	config.HandshakeConfig = plugins.HandshakeConfig
+	config.Plugins = plugins.PluginMap
+	config.SyncStdout = d.config.LogOutput
+	config.SyncStderr = d.config.LogOutput
 	executorClient := plugin.NewClient(config)
 	rpcClient, err := executorClient.Client()
 	if err != nil {
@@ -154,8 +154,8 @@ func (d *RawExecDriver) executor(config *plugin.ClientConfig) (plugins.Executor,
 
 type rawExecId struct {
 	KillTimeout  time.Duration
-	PluginConfig *plugin.ReattachConfig
 	UserPid      int
+	PluginConfig *plugins.ExecutorReattachConfig
 }
 
 func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
@@ -164,18 +164,8 @@ func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, e
 		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
 	}
 
-	bin, err := discover.NomadExecutable()
-	if err != nil {
-		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
-	}
-
 	pluginConfig := &plugin.ClientConfig{
-		HandshakeConfig: plugins.HandshakeConfig,
-		Plugins:         plugins.PluginMap,
-		Cmd:             exec.Command(bin, "executor"),
-		Reattach:        id.PluginConfig,
-		SyncStdout:      d.config.LogOutput,
-		SyncStderr:      d.config.LogOutput,
+		Reattach: id.PluginConfig.PluginConfig(),
 	}
 	executor, client, err := d.executor(pluginConfig)
 	if err != nil {
@@ -199,7 +189,7 @@ func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, e
 func (h *rawExecHandle) ID() string {
 	id := rawExecId{
 		KillTimeout:  h.killTimeout,
-		PluginConfig: h.pluginClient.ReattachConfig(),
+		PluginConfig: plugins.NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
 		UserPid:      h.userPid,
 	}
 
diff --git a/client/driver/raw_exec_test.go b/client/driver/raw_exec_test.go
index d0af75598..fe6d8c0e6 100644
--- a/client/driver/raw_exec_test.go
+++ b/client/driver/raw_exec_test.go
@@ -91,6 +91,8 @@ func TestRawExecDriver_StartOpen_Wait(t *testing.T) {
 	case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second):
 		t.Fatalf("timeout")
 	}
+	handle.Kill()
+	handle2.Kill()
 }
 
 func TestRawExecDriver_Start_Artifact_basic(t *testing.T) {

From 443acc1520e45a3536cd729b641936caabd8b987 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Thu, 4 Feb 2016 13:22:38 -0800
Subject: [PATCH 24/68] Fixing the id generation logic for the java and qemu
 drivers

---
 client/driver/exec.go             |  1 -
 client/driver/java.go             | 29 ++++++++++-------------------
 client/driver/java_test.go        |  1 +
 client/driver/plugins/executor.go |  6 ++++++
 client/driver/qemu.go             | 27 ++++++++++-----------------
 5 files changed, 27 insertions(+), 37 deletions(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index adf0829eb..9047222a3 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -151,7 +151,6 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	}
 
 	reattachConfig := id.PluginConfig.PluginConfig()
-
 	pluginConfig := &plugin.ClientConfig{
 		Reattach: reattachConfig,
 	}
diff --git a/client/driver/java.go b/client/driver/java.go
index c455d457f..46d7f2d8e 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -149,11 +149,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
 	pluginConfig := &plugin.ClientConfig{
-		HandshakeConfig: plugins.HandshakeConfig,
-		Plugins:         plugins.PluginMap,
-		Cmd:             exec.Command(bin, "executor"),
-		SyncStdout:      d.config.LogOutput,
-		SyncStderr:      d.config.LogOutput,
+		Cmd: exec.Command(bin, "executor"),
 	}
 
 	executor, pluginClient, err := d.executor(pluginConfig)
@@ -189,12 +185,16 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 }
 
 func (d *JavaDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *plugin.Client, error) {
+	config.HandshakeConfig = plugins.HandshakeConfig
+	config.Plugins = plugins.PluginMap
+	config.SyncStdout = d.config.LogOutput
+	config.SyncStderr = d.config.LogOutput
+
 	executorClient := plugin.NewClient(config)
 	rpcClient, err := executorClient.Client()
 	if err != nil {
 		return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
 	}
-	rpcClient.SyncStreams(d.config.LogOutput, d.config.LogOutput)
 
 	raw, err := rpcClient.Dispense("executor")
 	if err != nil {
@@ -206,7 +206,7 @@ func (d *JavaDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *p
 
 type javaId struct {
 	KillTimeout  time.Duration
-	PluginConfig *plugin.ReattachConfig
+	PluginConfig *plugins.ExecutorReattachConfig
 	UserPid      int
 }
 
@@ -216,18 +216,9 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
 	}
 
-	bin, err := discover.NomadExecutable()
-	if err != nil {
-		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
-	}
-
+	reattachConfig := id.PluginConfig.PluginConfig()
 	pluginConfig := &plugin.ClientConfig{
-		HandshakeConfig: plugins.HandshakeConfig,
-		Plugins:         plugins.PluginMap,
-		Cmd:             exec.Command(bin, "executor"),
-		Reattach:        id.PluginConfig,
-		SyncStdout:      d.config.LogOutput,
-		SyncStderr:      d.config.LogOutput,
+		Reattach: reattachConfig,
 	}
 	executor, client, err := d.executor(pluginConfig)
 	if err != nil {
@@ -252,7 +243,7 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 func (h *javaHandle) ID() string {
 	id := javaId{
 		KillTimeout:  h.killTimeout,
-		PluginConfig: h.pluginClient.ReattachConfig(),
+		PluginConfig: plugins.NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
 		UserPid:      h.userPid,
 	}
 
diff --git a/client/driver/java_test.go b/client/driver/java_test.go
index 79d289a9e..a546c611f 100644
--- a/client/driver/java_test.go
+++ b/client/driver/java_test.go
@@ -87,6 +87,7 @@ func TestJavaDriver_StartOpen_Wait(t *testing.T) {
 	// There is a race condition between the handle waiting and killing. One
 	// will return an error.
 	handle.Kill()
+	handle2.Kill()
 }
 
 func TestJavaDriver_Start_Wait(t *testing.T) {
diff --git a/client/driver/plugins/executor.go b/client/driver/plugins/executor.go
index 0ad3d62aa..89ca1397e 100644
--- a/client/driver/plugins/executor.go
+++ b/client/driver/plugins/executor.go
@@ -142,6 +142,9 @@ func (e *UniversalExecutor) wait() {
 
 func (e *UniversalExecutor) Exit() error {
 	e.logger.Printf("[INFO] Exiting plugin for task %q", e.ctx.TaskName)
+	if e.cmd.Process == nil {
+		return fmt.Errorf("executor.exit error: no process found")
+	}
 	proc, err := os.FindProcess(e.cmd.Process.Pid)
 	if err != nil {
 		return fmt.Errorf("failied to find user process %v: %v", e.cmd.Process.Pid, err)
@@ -159,6 +162,9 @@ func (e *UniversalExecutor) Exit() error {
 }
 
 func (e *UniversalExecutor) ShutDown() error {
+	if e.cmd.Process == nil {
+		return fmt.Errorf("executor.shutdown error: no process found")
+	}
 	proc, err := os.FindProcess(e.cmd.Process.Pid)
 	if err != nil {
 		return fmt.Errorf("executor.shutdown error: %v", err)
diff --git a/client/driver/qemu.go b/client/driver/qemu.go
index 6903d303f..cef77ebd8 100644
--- a/client/driver/qemu.go
+++ b/client/driver/qemu.go
@@ -193,11 +193,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
 	pluginConfig := &plugin.ClientConfig{
-		HandshakeConfig: plugins.HandshakeConfig,
-		Plugins:         plugins.PluginMap,
-		Cmd:             exec.Command(bin, "executor"),
-		SyncStdout:      d.config.LogOutput,
-		SyncStderr:      d.config.LogOutput,
+		Cmd: exec.Command(bin, "executor"),
 	}
 
 	executor, pluginClient, err := d.executor(pluginConfig)
@@ -234,6 +230,10 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 }
 
 func (d *QemuDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *plugin.Client, error) {
+	config.HandshakeConfig = plugins.HandshakeConfig
+	config.Plugins = plugins.PluginMap
+	config.SyncStdout = d.config.LogOutput
+	config.SyncStderr = d.config.LogOutput
 	executorClient := plugin.NewClient(config)
 	rpcClient, err := executorClient.Client()
 	if err != nil {
@@ -251,8 +251,8 @@ func (d *QemuDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *p
 
 type qemuId struct {
 	KillTimeout  time.Duration
-	PluginConfig *plugin.ReattachConfig
 	UserPid      int
+	PluginConfig *plugins.ExecutorReattachConfig
 }
 
 func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
@@ -261,18 +261,11 @@ func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
 	}
 
-	bin, err := discover.NomadExecutable()
-	if err != nil {
-		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
-	}
+	reattachConfig := id.PluginConfig.PluginConfig()
 	pluginConfig := &plugin.ClientConfig{
-		HandshakeConfig: plugins.HandshakeConfig,
-		Plugins:         plugins.PluginMap,
-		Cmd:             exec.Command(bin, "executor"),
-		Reattach:        id.PluginConfig,
-		SyncStdout:      d.config.LogOutput,
-		SyncStderr:      d.config.LogOutput,
+		Reattach: reattachConfig,
 	}
+
 	executor, client, err := d.executor(pluginConfig)
 	if err != nil {
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
@@ -295,7 +288,7 @@ func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 func (h *qemuHandle) ID() string {
 	id := qemuId{
 		KillTimeout:  h.killTimeout,
-		PluginConfig: h.pluginClient.ReattachConfig(),
+		PluginConfig: plugins.NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
 		UserPid:      h.userPid,
 	}
 

From d24f089273d933d80400e900f9059d23f4829c3a Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Thu, 4 Feb 2016 13:34:14 -0800
Subject: [PATCH 25/68] Removed duplicate code

---
 client/driver/exec.go     | 23 ++---------------------
 client/driver/java.go     | 26 +++-----------------------
 client/driver/qemu.go     | 26 +++-----------------------
 client/driver/raw_exec.go | 24 +++---------------------
 client/driver/utils.go    | 28 ++++++++++++++++++++++++++++
 5 files changed, 39 insertions(+), 88 deletions(-)
 create mode 100644 client/driver/utils.go

diff --git a/client/driver/exec.go b/client/driver/exec.go
index 9047222a3..836cc23be 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -106,7 +106,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		Cmd: exec.Command(bin, "executor"),
 	}
 
-	executor, pluginClient, err := d.executor(pluginConfig)
+	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
 		return nil, err
 	}
@@ -154,7 +154,7 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	pluginConfig := &plugin.ClientConfig{
 		Reattach: reattachConfig,
 	}
-	executor, client, err := d.executor(pluginConfig)
+	executor, client, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
@@ -172,25 +172,6 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	return h, nil
 }
 
-func (d *ExecDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *plugin.Client, error) {
-	config.HandshakeConfig = plugins.HandshakeConfig
-	config.Plugins = plugins.PluginMap
-	config.SyncStdout = d.config.LogOutput
-	config.SyncStderr = d.config.LogOutput
-	executorClient := plugin.NewClient(config)
-	rpcClient, err := executorClient.Client()
-	if err != nil {
-		return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
-	}
-
-	raw, err := rpcClient.Dispense("executor")
-	if err != nil {
-		return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
-	}
-	executorPlugin := raw.(plugins.Executor)
-	return executorPlugin, executorClient, nil
-}
-
 func (h *execHandle) ID() string {
 	id := execId{
 		KillTimeout:  h.killTimeout,
diff --git a/client/driver/java.go b/client/driver/java.go
index 46d7f2d8e..31e30eaf1 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -152,7 +152,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		Cmd: exec.Command(bin, "executor"),
 	}
 
-	executor, pluginClient, err := d.executor(pluginConfig)
+	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
 		return nil, err
 	}
@@ -184,26 +184,6 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 	return h, nil
 }
 
-func (d *JavaDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *plugin.Client, error) {
-	config.HandshakeConfig = plugins.HandshakeConfig
-	config.Plugins = plugins.PluginMap
-	config.SyncStdout = d.config.LogOutput
-	config.SyncStderr = d.config.LogOutput
-
-	executorClient := plugin.NewClient(config)
-	rpcClient, err := executorClient.Client()
-	if err != nil {
-		return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
-	}
-
-	raw, err := rpcClient.Dispense("executor")
-	if err != nil {
-		return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
-	}
-	executorPlugin := raw.(plugins.Executor)
-	return executorPlugin, executorClient, nil
-}
-
 type javaId struct {
 	KillTimeout  time.Duration
 	PluginConfig *plugins.ExecutorReattachConfig
@@ -220,14 +200,14 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	pluginConfig := &plugin.ClientConfig{
 		Reattach: reattachConfig,
 	}
-	executor, client, err := d.executor(pluginConfig)
+	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
 
 	// Return a driver handle
 	h := &javaHandle{
-		pluginClient: client,
+		pluginClient: pluginClient,
 		executor:     executor,
 		userPid:      id.UserPid,
 		logger:       d.logger,
diff --git a/client/driver/qemu.go b/client/driver/qemu.go
index cef77ebd8..ef1cbe159 100644
--- a/client/driver/qemu.go
+++ b/client/driver/qemu.go
@@ -196,7 +196,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		Cmd: exec.Command(bin, "executor"),
 	}
 
-	executor, pluginClient, err := d.executor(pluginConfig)
+	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
 		return nil, err
 	}
@@ -229,26 +229,6 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 	return h, nil
 }
 
-func (d *QemuDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *plugin.Client, error) {
-	config.HandshakeConfig = plugins.HandshakeConfig
-	config.Plugins = plugins.PluginMap
-	config.SyncStdout = d.config.LogOutput
-	config.SyncStderr = d.config.LogOutput
-	executorClient := plugin.NewClient(config)
-	rpcClient, err := executorClient.Client()
-	if err != nil {
-		return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
-	}
-	rpcClient.SyncStreams(d.config.LogOutput, d.config.LogOutput)
-
-	raw, err := rpcClient.Dispense("executor")
-	if err != nil {
-		return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
-	}
-	executorPlugin := raw.(plugins.Executor)
-	return executorPlugin, executorClient, nil
-}
-
 type qemuId struct {
 	KillTimeout  time.Duration
 	UserPid      int
@@ -266,14 +246,14 @@ func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		Reattach: reattachConfig,
 	}
 
-	executor, client, err := d.executor(pluginConfig)
+	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
 
 	// Return a driver handle
 	h := &qemuHandle{
-		pluginClient: client,
+		pluginClient: pluginClient,
 		executor:     executor,
 		userPid:      id.UserPid,
 		logger:       d.logger,
diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go
index ed8aa3ec0..2a90cfb75 100644
--- a/client/driver/raw_exec.go
+++ b/client/driver/raw_exec.go
@@ -103,7 +103,7 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 		Cmd: exec.Command(bin, "executor"),
 	}
 
-	executor, pluginClient, err := d.executor(pluginConfig)
+	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
 		return nil, err
 	}
@@ -133,24 +133,6 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 	go h.run()
 	return h, nil
 }
-func (d *RawExecDriver) executor(config *plugin.ClientConfig) (plugins.Executor, *plugin.Client, error) {
-	config.HandshakeConfig = plugins.HandshakeConfig
-	config.Plugins = plugins.PluginMap
-	config.SyncStdout = d.config.LogOutput
-	config.SyncStderr = d.config.LogOutput
-	executorClient := plugin.NewClient(config)
-	rpcClient, err := executorClient.Client()
-	if err != nil {
-		return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
-	}
-
-	raw, err := rpcClient.Dispense("executor")
-	if err != nil {
-		return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
-	}
-	executorPlugin := raw.(plugins.Executor)
-	return executorPlugin, executorClient, nil
-}
 
 type rawExecId struct {
 	KillTimeout  time.Duration
@@ -167,14 +149,14 @@ func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, e
 	pluginConfig := &plugin.ClientConfig{
 		Reattach: id.PluginConfig.PluginConfig(),
 	}
-	executor, client, err := d.executor(pluginConfig)
+	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
 
 	// Return a driver handle
 	h := &rawExecHandle{
-		pluginClient: client,
+		pluginClient: pluginClient,
 		executor:     executor,
 		userPid:      id.UserPid,
 		logger:       d.logger,
diff --git a/client/driver/utils.go b/client/driver/utils.go
new file mode 100644
index 000000000..ded4ff01f
--- /dev/null
+++ b/client/driver/utils.go
@@ -0,0 +1,28 @@
+package driver
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/hashicorp/go-plugin"
+	"github.com/hashicorp/nomad/client/driver/plugins"
+)
+
+func createExecutor(config *plugin.ClientConfig, w io.Writer) (plugins.Executor, *plugin.Client, error) {
+	config.HandshakeConfig = plugins.HandshakeConfig
+	config.Plugins = plugins.PluginMap
+	config.SyncStdout = w
+	config.SyncStderr = w
+	executorClient := plugin.NewClient(config)
+	rpcClient, err := executorClient.Client()
+	if err != nil {
+		return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err)
+	}
+
+	raw, err := rpcClient.Dispense("executor")
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
+	}
+	executorPlugin := raw.(plugins.Executor)
+	return executorPlugin, executorClient, nil
+}

From 5ea6b85e7666ea37ea3891fec7a495ed075a1324 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Thu, 4 Feb 2016 13:53:30 -0800
Subject: [PATCH 26/68] Destroying the plugin if we can't connect to it

---
 client/driver/exec.go     |  9 +++++++++
 client/driver/java.go     |  4 ++++
 client/driver/qemu.go     |  4 ++++
 client/driver/raw_exec.go |  4 ++++
 client/driver/utils.go    | 22 ++++++++++++++++++++++
 5 files changed, 43 insertions(+)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index 836cc23be..cb12b6cd1 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -37,6 +37,7 @@ type ExecDriverConfig struct {
 type execHandle struct {
 	pluginClient *plugin.Client
 	executor     plugins.Executor
+	userPid      int
 	killTimeout  time.Duration
 	logger       *log.Logger
 	waitCh       chan *cstructs.WaitResult
@@ -129,6 +130,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 	// Return a driver handle
 	h := &execHandle{
 		pluginClient: pluginClient,
+		userPid:      ps.Pid,
 		executor:     executor,
 		killTimeout:  d.DriverContext.KillTimeout(task),
 		logger:       d.logger,
@@ -141,6 +143,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 
 type execId struct {
 	KillTimeout  time.Duration
+	UserPid      int
 	PluginConfig *plugins.ExecutorReattachConfig
 }
 
@@ -156,6 +159,10 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	}
 	executor, client, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
+		d.logger.Println("[ERROR] error connecting to plugin so destroying plugin pid and user pid")
+		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
+			d.logger.Printf("[ERROR] error destrouing plugin and userpid: %v", e)
+		}
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
 
@@ -163,6 +170,7 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	h := &execHandle{
 		pluginClient: client,
 		executor:     executor,
+		userPid:      id.UserPid,
 		logger:       d.logger,
 		killTimeout:  id.KillTimeout,
 		doneCh:       make(chan struct{}),
@@ -176,6 +184,7 @@ func (h *execHandle) ID() string {
 	id := execId{
 		KillTimeout:  h.killTimeout,
 		PluginConfig: plugins.NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
+		UserPid:      h.userPid,
 	}
 
 	data, err := json.Marshal(id)
diff --git a/client/driver/java.go b/client/driver/java.go
index 31e30eaf1..d1600a0db 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -202,6 +202,10 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	}
 	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
+		d.logger.Println("[ERROR] error connecting to plugin so destroying plugin pid and user pid")
+		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
+			d.logger.Printf("[ERROR] error destroying plugin and userpid: %v", e)
+		}
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
 
diff --git a/client/driver/qemu.go b/client/driver/qemu.go
index ef1cbe159..69a1ed178 100644
--- a/client/driver/qemu.go
+++ b/client/driver/qemu.go
@@ -248,6 +248,10 @@ func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 
 	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
+		d.logger.Println("[ERROR] error connecting to plugin so destroying plugin pid and user pid")
+		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
+			d.logger.Printf("[ERROR] error destroying plugin and userpid: %v", e)
+		}
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
 
diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go
index 2a90cfb75..fc48b5580 100644
--- a/client/driver/raw_exec.go
+++ b/client/driver/raw_exec.go
@@ -151,6 +151,10 @@ func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, e
 	}
 	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
+		d.logger.Println("[ERROR] error connecting to plugin so destroying plugin pid and user pid")
+		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
+			d.logger.Printf("[ERROR] error destrouing plugin and userpid: %v", e)
+		}
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
 
diff --git a/client/driver/utils.go b/client/driver/utils.go
index ded4ff01f..d39996718 100644
--- a/client/driver/utils.go
+++ b/client/driver/utils.go
@@ -3,7 +3,9 @@ package driver
 import (
 	"fmt"
 	"io"
+	"os"
 
+	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/go-plugin"
 	"github.com/hashicorp/nomad/client/driver/plugins"
 )
@@ -26,3 +28,23 @@ func createExecutor(config *plugin.ClientConfig, w io.Writer) (plugins.Executor,
 	executorPlugin := raw.(plugins.Executor)
 	return executorPlugin, executorClient, nil
 }
+
+func killProcess(pid int) error {
+	proc, err := os.FindProcess(pid)
+	if err != nil {
+		return err
+	}
+	return proc.Kill()
+}
+
+func destroyPlugin(pluginPid int, userPid int) error {
+	var merr error
+	if err := killProcess(pluginPid); err != nil {
+		merr = multierror.Append(merr, err)
+	}
+
+	if err := killProcess(userPid); err != nil {
+		merr = multierror.Append(merr, err)
+	}
+	return merr
+}

From b3f161729820c4bcfcea0f44e357d5d8adc05ba6 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Thu, 4 Feb 2016 15:39:29 -0800
Subject: [PATCH 27/68] Added tests for the executor

---
 client/driver/exec.go                  |   2 +-
 client/driver/plugins/executor_test.go | 158 +++++++++++++++++++++++++
 client/driver/raw_exec.go              |   2 +-
 3 files changed, 160 insertions(+), 2 deletions(-)
 create mode 100644 client/driver/plugins/executor_test.go

diff --git a/client/driver/exec.go b/client/driver/exec.go
index cb12b6cd1..8ee6b9269 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -161,7 +161,7 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	if err != nil {
 		d.logger.Println("[ERROR] error connecting to plugin so destroying plugin pid and user pid")
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
-			d.logger.Printf("[ERROR] error destrouing plugin and userpid: %v", e)
+			d.logger.Printf("[ERROR] error destroying plugin and userpid: %v", e)
 		}
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
diff --git a/client/driver/plugins/executor_test.go b/client/driver/plugins/executor_test.go
new file mode 100644
index 000000000..98fd89f11
--- /dev/null
+++ b/client/driver/plugins/executor_test.go
@@ -0,0 +1,158 @@
+package plugins
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/nomad/client/allocdir"
+	"github.com/hashicorp/nomad/client/driver/env"
+	"github.com/hashicorp/nomad/nomad/mock"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/nomad/testutil"
+)
+
+var (
+	constraint = &structs.Resources{
+		CPU:      250,
+		MemoryMB: 256,
+		Networks: []*structs.NetworkResource{
+			&structs.NetworkResource{
+				MBits:        50,
+				DynamicPorts: []structs.Port{{Label: "http"}},
+			},
+		},
+	}
+)
+
+func mockAllocDir(t *testing.T) (string, *allocdir.AllocDir) {
+	alloc := mock.Alloc()
+	task := alloc.Job.TaskGroups[0].Tasks[0]
+
+	allocDir := allocdir.NewAllocDir(filepath.Join(os.TempDir(), alloc.ID))
+	if err := allocDir.Build([]*structs.Task{task}); err != nil {
+		log.Panicf("allocDir.Build() failed: %v", err)
+	}
+
+	return task.Name, allocDir
+}
+
+func testExecutorContext(t *testing.T) *ExecutorContext {
+	taskEnv := env.NewTaskEnvironment(mock.Node())
+	taskName, allocDir := mockAllocDir(t)
+	ctx := &ExecutorContext{
+		TaskEnv:       taskEnv,
+		TaskName:      taskName,
+		AllocDir:      allocDir,
+		TaskResources: constraint,
+	}
+	return ctx
+}
+
+func TestExecutor_Start_Invalid(t *testing.T) {
+	invalid := "/bin/foobar"
+	execCmd := ExecCommand{Cmd: invalid, Args: []string{"1"}}
+	ctx := testExecutorContext(t)
+	executor := NewExecutor(log.New(os.Stdout, "", log.LstdFlags))
+	_, err := executor.LaunchCmd(&execCmd, ctx)
+	if err == nil {
+		t.Fatalf("Expected error")
+	}
+	defer ctx.AllocDir.Destroy()
+}
+
+func TestExecutor_Start_Wait_Failure_Code(t *testing.T) {
+	execCmd := ExecCommand{Cmd: "/bin/sleep", Args: []string{"fail"}}
+	ctx := testExecutorContext(t)
+	executor := NewExecutor(log.New(os.Stdout, "", log.LstdFlags))
+	ps, _ := executor.LaunchCmd(&execCmd, ctx)
+	if ps.Pid == 0 {
+		t.Fatalf("expected process to start and have non zero pid")
+	}
+	ps, _ = executor.Wait()
+	if ps.ExitCode < 1 {
+		t.Fatalf("expected exit code to be non zero, actual: %v", ps.ExitCode)
+	}
+	defer ctx.AllocDir.Destroy()
+}
+
+func TestExecutor_Start_Wait(t *testing.T) {
+	execCmd := ExecCommand{Cmd: "/bin/echo", Args: []string{"hello world"}}
+	ctx := testExecutorContext(t)
+	executor := NewExecutor(log.New(os.Stdout, "", log.LstdFlags))
+	ps, err := executor.LaunchCmd(&execCmd, ctx)
+	if err != nil {
+		t.Fatalf("error in launching command: %v", err)
+	}
+	if ps.Pid == 0 {
+		t.Fatalf("expected process to start and have non zero pid")
+	}
+	ps, err = executor.Wait()
+	if err != nil {
+		t.Fatalf("error in waiting for command: %v", err)
+	}
+	defer ctx.AllocDir.Destroy()
+
+	task := "web"
+	taskDir, ok := ctx.AllocDir.TaskDirs[task]
+	if !ok {
+		log.Panicf("No task directory found for task %v", task)
+	}
+
+	expected := "hello world"
+	file := filepath.Join(allocdir.TaskLocal, "web.stdout")
+	absFilePath := filepath.Join(taskDir, file)
+	output, err := ioutil.ReadFile(absFilePath)
+	if err != nil {
+		t.Fatalf("Couldn't read file %v", absFilePath)
+	}
+
+	act := strings.TrimSpace(string(output))
+	if act != expected {
+		t.Fatalf("Command output incorrectly: want %v; got %v", expected, act)
+	}
+}
+
+func TestExecutor_Start_Kill(t *testing.T) {
+	execCmd := ExecCommand{Cmd: "/bin/sleep", Args: []string{"10 && hello world"}}
+	ctx := testExecutorContext(t)
+	executor := NewExecutor(log.New(os.Stdout, "", log.LstdFlags))
+	ps, err := executor.LaunchCmd(&execCmd, ctx)
+	if err != nil {
+		t.Fatalf("error in launching command: %v", err)
+	}
+	if ps.Pid == 0 {
+		t.Fatalf("expected process to start and have non zero pid")
+	}
+	ps, err = executor.Wait()
+	if err != nil {
+		t.Fatalf("error in waiting for command: %v", err)
+	}
+	defer ctx.AllocDir.Destroy()
+
+	task := "web"
+	taskDir, ok := ctx.AllocDir.TaskDirs[task]
+	if !ok {
+		t.Fatalf("No task directory found for task %v", task)
+	}
+
+	file := filepath.Join(allocdir.TaskLocal, "web.stdout")
+	absFilePath := filepath.Join(taskDir, file)
+
+	time.Sleep(time.Duration(testutil.TestMultiplier()*2) * time.Second)
+
+	output, err := ioutil.ReadFile(absFilePath)
+	if err != nil {
+		t.Fatalf("Couldn't read file %v", absFilePath)
+	}
+
+	expected := ""
+	act := strings.TrimSpace(string(output))
+	if act != expected {
+		t.Fatalf("Command output incorrectly: want %v; got %v", expected, act)
+	}
+}
diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go
index fc48b5580..f3f670d25 100644
--- a/client/driver/raw_exec.go
+++ b/client/driver/raw_exec.go
@@ -153,7 +153,7 @@ func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, e
 	if err != nil {
 		d.logger.Println("[ERROR] error connecting to plugin so destroying plugin pid and user pid")
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
-			d.logger.Printf("[ERROR] error destrouing plugin and userpid: %v", e)
+			d.logger.Printf("[ERROR] error destroying plugin and userpid: %v", e)
 		}
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}

From 2b8cdc47d310219cc370c6b564538e92730fba79 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Thu, 4 Feb 2016 16:03:17 -0800
Subject: [PATCH 28/68] Moved packages around

---
 client/driver/exec.go                         |  16 +-
 .../driver/{plugins => executor}/executor.go  |   2 +-
 .../{plugins => executor}/executor_basic.go   |   2 +-
 .../{plugins => executor}/executor_linux.go   |   2 +-
 .../{plugins => executor}/executor_test.go    |   2 +-
 .../driver/{plugins => }/executor_plugin.go   |  31 +--
 client/driver/java.go                         |  16 +-
 client/driver/qemu.go                         |  16 +-
 client/driver/raw_exec.go                     |  16 +-
 client/driver/utils.go                        |  10 +-
 command/executor_plugin.go                    |   6 +-
 command/spawn_daemon.go                       | 234 ------------------
 command/spawn_daemon_darwin.go                |   4 -
 command/spawn_daemon_linux.go                 |  16 --
 command/spawn_daemon_test.go                  |  48 ----
 command/spawn_daemon_unix.go                  |  16 --
 command/spawn_daemon_windows.go               |   7 -
 commands.go                                   |   7 -
 18 files changed, 60 insertions(+), 391 deletions(-)
 rename client/driver/{plugins => executor}/executor.go (99%)
 rename client/driver/{plugins => executor}/executor_basic.go (96%)
 rename client/driver/{plugins => executor}/executor_linux.go (99%)
 rename client/driver/{plugins => executor}/executor_test.go (99%)
 rename client/driver/{plugins => }/executor_plugin.go (70%)
 delete mode 100644 command/spawn_daemon.go
 delete mode 100644 command/spawn_daemon_darwin.go
 delete mode 100644 command/spawn_daemon_linux.go
 delete mode 100644 command/spawn_daemon_test.go
 delete mode 100644 command/spawn_daemon_unix.go
 delete mode 100644 command/spawn_daemon_windows.go

diff --git a/client/driver/exec.go b/client/driver/exec.go
index 8ee6b9269..5273f1d7f 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -12,7 +12,7 @@ import (
 	"github.com/hashicorp/go-plugin"
 	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
-	"github.com/hashicorp/nomad/client/driver/plugins"
+	"github.com/hashicorp/nomad/client/driver/executor"
 	cstructs "github.com/hashicorp/nomad/client/driver/structs"
 	"github.com/hashicorp/nomad/client/getter"
 	"github.com/hashicorp/nomad/helper/discover"
@@ -36,7 +36,7 @@ type ExecDriverConfig struct {
 // execHandle is returned from Start/Open as a handle to the PID
 type execHandle struct {
 	pluginClient *plugin.Client
-	executor     plugins.Executor
+	executor     executor.Executor
 	userPid      int
 	killTimeout  time.Duration
 	logger       *log.Logger
@@ -107,11 +107,11 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		Cmd: exec.Command(bin, "executor"),
 	}
 
-	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
+	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
 		return nil, err
 	}
-	executorCtx := &plugins.ExecutorContext{
+	executorCtx := &executor.ExecutorContext{
 		TaskEnv:          d.taskEnv,
 		AllocDir:         ctx.AllocDir,
 		TaskName:         task.Name,
@@ -120,7 +120,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		FSIsolation:      true,
 		UnprivilegedUser: false,
 	}
-	ps, err := executor.LaunchCmd(&plugins.ExecCommand{Cmd: command, Args: driverConfig.Args}, executorCtx)
+	ps, err := exec.LaunchCmd(&executor.ExecCommand{Cmd: command, Args: driverConfig.Args}, executorCtx)
 	if err != nil {
 		pluginClient.Kill()
 		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
@@ -131,7 +131,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 	h := &execHandle{
 		pluginClient: pluginClient,
 		userPid:      ps.Pid,
-		executor:     executor,
+		executor:     exec,
 		killTimeout:  d.DriverContext.KillTimeout(task),
 		logger:       d.logger,
 		doneCh:       make(chan struct{}),
@@ -144,7 +144,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 type execId struct {
 	KillTimeout  time.Duration
 	UserPid      int
-	PluginConfig *plugins.ExecutorReattachConfig
+	PluginConfig *ExecutorReattachConfig
 }
 
 func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
@@ -183,7 +183,7 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 func (h *execHandle) ID() string {
 	id := execId{
 		KillTimeout:  h.killTimeout,
-		PluginConfig: plugins.NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
+		PluginConfig: NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
 		UserPid:      h.userPid,
 	}
 
diff --git a/client/driver/plugins/executor.go b/client/driver/executor/executor.go
similarity index 99%
rename from client/driver/plugins/executor.go
rename to client/driver/executor/executor.go
index 89ca1397e..60c18e17e 100644
--- a/client/driver/plugins/executor.go
+++ b/client/driver/executor/executor.go
@@ -1,4 +1,4 @@
-package plugins
+package executor
 
 import (
 	"fmt"
diff --git a/client/driver/plugins/executor_basic.go b/client/driver/executor/executor_basic.go
similarity index 96%
rename from client/driver/plugins/executor_basic.go
rename to client/driver/executor/executor_basic.go
index 464cb3b3f..88acf1105 100644
--- a/client/driver/plugins/executor_basic.go
+++ b/client/driver/executor/executor_basic.go
@@ -1,6 +1,6 @@
 // +build !linux
 
-package plugins
+package executor
 
 func (e *UniversalExecutor) configureChroot() error {
 	return nil
diff --git a/client/driver/plugins/executor_linux.go b/client/driver/executor/executor_linux.go
similarity index 99%
rename from client/driver/plugins/executor_linux.go
rename to client/driver/executor/executor_linux.go
index efdd4e410..4c946d429 100644
--- a/client/driver/plugins/executor_linux.go
+++ b/client/driver/executor/executor_linux.go
@@ -1,4 +1,4 @@
-package plugins
+package executor
 
 import (
 	"fmt"
diff --git a/client/driver/plugins/executor_test.go b/client/driver/executor/executor_test.go
similarity index 99%
rename from client/driver/plugins/executor_test.go
rename to client/driver/executor/executor_test.go
index 98fd89f11..2b7e281ac 100644
--- a/client/driver/plugins/executor_test.go
+++ b/client/driver/executor/executor_test.go
@@ -1,4 +1,4 @@
-package plugins
+package executor
 
 import (
 	"io/ioutil"
diff --git a/client/driver/plugins/executor_plugin.go b/client/driver/executor_plugin.go
similarity index 70%
rename from client/driver/plugins/executor_plugin.go
rename to client/driver/executor_plugin.go
index 8dbdbe58a..c07ad3f79 100644
--- a/client/driver/plugins/executor_plugin.go
+++ b/client/driver/executor_plugin.go
@@ -1,4 +1,4 @@
-package plugins
+package driver
 
 import (
 	"log"
@@ -7,6 +7,7 @@ import (
 	"os"
 
 	"github.com/hashicorp/go-plugin"
+	"github.com/hashicorp/nomad/client/driver/executor"
 )
 
 var HandshakeConfig = plugin.HandshakeConfig{
@@ -45,39 +46,39 @@ type ExecutorRPC struct {
 }
 
 type LaunchCmdArgs struct {
-	Cmd *ExecCommand
-	Ctx *ExecutorContext
+	Cmd *executor.ExecCommand
+	Ctx *executor.ExecutorContext
 }
 
-func (e *ExecutorRPC) LaunchCmd(cmd *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
-	var ps *ProcessState
+func (e *ExecutorRPC) LaunchCmd(cmd *executor.ExecCommand, ctx *executor.ExecutorContext) (*executor.ProcessState, error) {
+	var ps *executor.ProcessState
 	err := e.client.Call("Plugin.LaunchCmd", LaunchCmdArgs{Cmd: cmd, Ctx: ctx}, &ps)
 	return ps, err
 }
 
-func (e *ExecutorRPC) Wait() (*ProcessState, error) {
-	var ps ProcessState
+func (e *ExecutorRPC) Wait() (*executor.ProcessState, error) {
+	var ps executor.ProcessState
 	err := e.client.Call("Plugin.Wait", new(interface{}), &ps)
 	return &ps, err
 }
 
 func (e *ExecutorRPC) ShutDown() error {
-	var ps ProcessState
+	var ps executor.ProcessState
 	err := e.client.Call("Plugin.ShutDown", new(interface{}), &ps)
 	return err
 }
 
 func (e *ExecutorRPC) Exit() error {
-	var ps ProcessState
+	var ps executor.ProcessState
 	err := e.client.Call("Plugin.Exit", new(interface{}), &ps)
 	return err
 }
 
 type ExecutorRPCServer struct {
-	Impl Executor
+	Impl executor.Executor
 }
 
-func (e *ExecutorRPCServer) LaunchCmd(args LaunchCmdArgs, ps *ProcessState) error {
+func (e *ExecutorRPCServer) LaunchCmd(args LaunchCmdArgs, ps *executor.ProcessState) error {
 	state, err := e.Impl.LaunchCmd(args.Cmd, args.Ctx)
 	if state != nil {
 		*ps = *state
@@ -85,7 +86,7 @@ func (e *ExecutorRPCServer) LaunchCmd(args LaunchCmdArgs, ps *ProcessState) erro
 	return err
 }
 
-func (e *ExecutorRPCServer) Wait(args interface{}, ps *ProcessState) error {
+func (e *ExecutorRPCServer) Wait(args interface{}, ps *executor.ProcessState) error {
 	state, err := e.Impl.Wait()
 	if state != nil {
 		*ps = *state
@@ -93,13 +94,13 @@ func (e *ExecutorRPCServer) Wait(args interface{}, ps *ProcessState) error {
 	return err
 }
 
-func (e *ExecutorRPCServer) ShutDown(args interface{}, ps *ProcessState) error {
+func (e *ExecutorRPCServer) ShutDown(args interface{}, ps *executor.ProcessState) error {
 	var err error
 	err = e.Impl.ShutDown()
 	return err
 }
 
-func (e *ExecutorRPCServer) Exit(args interface{}, ps *ProcessState) error {
+func (e *ExecutorRPCServer) Exit(args interface{}, ps *executor.ProcessState) error {
 	var err error
 	err = e.Impl.Exit()
 	return err
@@ -111,7 +112,7 @@ type ExecutorPlugin struct {
 
 func (p *ExecutorPlugin) Server(*plugin.MuxBroker) (interface{}, error) {
 	p.logger = log.New(os.Stdout, "executor-plugin-server:", log.LstdFlags)
-	return &ExecutorRPCServer{Impl: NewExecutor(p.logger)}, nil
+	return &ExecutorRPCServer{Impl: executor.NewExecutor(p.logger)}, nil
 }
 
 func (p *ExecutorPlugin) Client(b *plugin.MuxBroker, c *rpc.Client) (interface{}, error) {
diff --git a/client/driver/java.go b/client/driver/java.go
index d1600a0db..74f295cf8 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -17,7 +17,7 @@ import (
 
 	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
-	"github.com/hashicorp/nomad/client/driver/plugins"
+	"github.com/hashicorp/nomad/client/driver/executor"
 	cstructs "github.com/hashicorp/nomad/client/driver/structs"
 	"github.com/hashicorp/nomad/client/fingerprint"
 	"github.com/hashicorp/nomad/client/getter"
@@ -43,7 +43,7 @@ type JavaDriverConfig struct {
 type javaHandle struct {
 	pluginClient *plugin.Client
 	userPid      int
-	executor     plugins.Executor
+	executor     executor.Executor
 
 	killTimeout time.Duration
 	logger      *log.Logger
@@ -152,17 +152,17 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		Cmd: exec.Command(bin, "executor"),
 	}
 
-	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
+	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
 		return nil, err
 	}
-	executorCtx := &plugins.ExecutorContext{
+	executorCtx := &executor.ExecutorContext{
 		TaskEnv:       d.taskEnv,
 		AllocDir:      ctx.AllocDir,
 		TaskName:      task.Name,
 		TaskResources: task.Resources,
 	}
-	ps, err := executor.LaunchCmd(&plugins.ExecCommand{Cmd: "java", Args: args}, executorCtx)
+	ps, err := exec.LaunchCmd(&executor.ExecCommand{Cmd: "java", Args: args}, executorCtx)
 	if err != nil {
 		pluginClient.Kill()
 		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
@@ -172,7 +172,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 	// Return a driver handle
 	h := &javaHandle{
 		pluginClient: pluginClient,
-		executor:     executor,
+		executor:     exec,
 		userPid:      ps.Pid,
 		killTimeout:  d.DriverContext.KillTimeout(task),
 		logger:       d.logger,
@@ -186,7 +186,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 
 type javaId struct {
 	KillTimeout  time.Duration
-	PluginConfig *plugins.ExecutorReattachConfig
+	PluginConfig *ExecutorReattachConfig
 	UserPid      int
 }
 
@@ -227,7 +227,7 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 func (h *javaHandle) ID() string {
 	id := javaId{
 		KillTimeout:  h.killTimeout,
-		PluginConfig: plugins.NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
+		PluginConfig: NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
 		UserPid:      h.userPid,
 	}
 
diff --git a/client/driver/qemu.go b/client/driver/qemu.go
index 69a1ed178..e7753fcc9 100644
--- a/client/driver/qemu.go
+++ b/client/driver/qemu.go
@@ -14,7 +14,7 @@ import (
 	"github.com/hashicorp/go-plugin"
 	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
-	"github.com/hashicorp/nomad/client/driver/plugins"
+	"github.com/hashicorp/nomad/client/driver/executor"
 	cstructs "github.com/hashicorp/nomad/client/driver/structs"
 	"github.com/hashicorp/nomad/client/fingerprint"
 	"github.com/hashicorp/nomad/client/getter"
@@ -46,7 +46,7 @@ type QemuDriverConfig struct {
 type qemuHandle struct {
 	pluginClient *plugin.Client
 	userPid      int
-	executor     plugins.Executor
+	executor     executor.Executor
 	killTimeout  time.Duration
 	logger       *log.Logger
 	waitCh       chan *cstructs.WaitResult
@@ -196,17 +196,17 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		Cmd: exec.Command(bin, "executor"),
 	}
 
-	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
+	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
 		return nil, err
 	}
-	executorCtx := &plugins.ExecutorContext{
+	executorCtx := &executor.ExecutorContext{
 		TaskEnv:       d.taskEnv,
 		AllocDir:      ctx.AllocDir,
 		TaskName:      task.Name,
 		TaskResources: task.Resources,
 	}
-	ps, err := executor.LaunchCmd(&plugins.ExecCommand{Cmd: args[0], Args: args[1:]}, executorCtx)
+	ps, err := exec.LaunchCmd(&executor.ExecCommand{Cmd: args[0], Args: args[1:]}, executorCtx)
 	if err != nil {
 		pluginClient.Kill()
 		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
@@ -217,7 +217,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 	// Create and Return Handle
 	h := &qemuHandle{
 		pluginClient: pluginClient,
-		executor:     executor,
+		executor:     exec,
 		userPid:      ps.Pid,
 		killTimeout:  d.DriverContext.KillTimeout(task),
 		logger:       d.logger,
@@ -232,7 +232,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 type qemuId struct {
 	KillTimeout  time.Duration
 	UserPid      int
-	PluginConfig *plugins.ExecutorReattachConfig
+	PluginConfig *ExecutorReattachConfig
 }
 
 func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
@@ -272,7 +272,7 @@ func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 func (h *qemuHandle) ID() string {
 	id := qemuId{
 		KillTimeout:  h.killTimeout,
-		PluginConfig: plugins.NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
+		PluginConfig: NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
 		UserPid:      h.userPid,
 	}
 
diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go
index f3f670d25..5b92e6d85 100644
--- a/client/driver/raw_exec.go
+++ b/client/driver/raw_exec.go
@@ -11,7 +11,7 @@ import (
 	"github.com/hashicorp/go-plugin"
 	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
-	"github.com/hashicorp/nomad/client/driver/plugins"
+	"github.com/hashicorp/nomad/client/driver/executor"
 	cstructs "github.com/hashicorp/nomad/client/driver/structs"
 	"github.com/hashicorp/nomad/client/fingerprint"
 	"github.com/hashicorp/nomad/client/getter"
@@ -37,7 +37,7 @@ type RawExecDriver struct {
 type rawExecHandle struct {
 	pluginClient *plugin.Client
 	userPid      int
-	executor     plugins.Executor
+	executor     executor.Executor
 	killTimeout  time.Duration
 	logger       *log.Logger
 	waitCh       chan *cstructs.WaitResult
@@ -103,17 +103,17 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 		Cmd: exec.Command(bin, "executor"),
 	}
 
-	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
+	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
 		return nil, err
 	}
-	executorCtx := &plugins.ExecutorContext{
+	executorCtx := &executor.ExecutorContext{
 		TaskEnv:       d.taskEnv,
 		AllocDir:      ctx.AllocDir,
 		TaskName:      task.Name,
 		TaskResources: task.Resources,
 	}
-	ps, err := executor.LaunchCmd(&plugins.ExecCommand{Cmd: command, Args: driverConfig.Args}, executorCtx)
+	ps, err := exec.LaunchCmd(&executor.ExecCommand{Cmd: command, Args: driverConfig.Args}, executorCtx)
 	if err != nil {
 		pluginClient.Kill()
 		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
@@ -123,7 +123,7 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 	// Return a driver handle
 	h := &rawExecHandle{
 		pluginClient: pluginClient,
-		executor:     executor,
+		executor:     exec,
 		userPid:      ps.Pid,
 		killTimeout:  d.DriverContext.KillTimeout(task),
 		logger:       d.logger,
@@ -137,7 +137,7 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 type rawExecId struct {
 	KillTimeout  time.Duration
 	UserPid      int
-	PluginConfig *plugins.ExecutorReattachConfig
+	PluginConfig *ExecutorReattachConfig
 }
 
 func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
@@ -175,7 +175,7 @@ func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, e
 func (h *rawExecHandle) ID() string {
 	id := rawExecId{
 		KillTimeout:  h.killTimeout,
-		PluginConfig: plugins.NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
+		PluginConfig: NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
 		UserPid:      h.userPid,
 	}
 
diff --git a/client/driver/utils.go b/client/driver/utils.go
index d39996718..972876eeb 100644
--- a/client/driver/utils.go
+++ b/client/driver/utils.go
@@ -7,12 +7,12 @@ import (
 
 	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/go-plugin"
-	"github.com/hashicorp/nomad/client/driver/plugins"
+	"github.com/hashicorp/nomad/client/driver/executor"
 )
 
-func createExecutor(config *plugin.ClientConfig, w io.Writer) (plugins.Executor, *plugin.Client, error) {
-	config.HandshakeConfig = plugins.HandshakeConfig
-	config.Plugins = plugins.PluginMap
+func createExecutor(config *plugin.ClientConfig, w io.Writer) (executor.Executor, *plugin.Client, error) {
+	config.HandshakeConfig = HandshakeConfig
+	config.Plugins = PluginMap
 	config.SyncStdout = w
 	config.SyncStderr = w
 	executorClient := plugin.NewClient(config)
@@ -25,7 +25,7 @@ func createExecutor(config *plugin.ClientConfig, w io.Writer) (plugins.Executor,
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err)
 	}
-	executorPlugin := raw.(plugins.Executor)
+	executorPlugin := raw.(executor.Executor)
 	return executorPlugin, executorClient, nil
 }
 
diff --git a/command/executor_plugin.go b/command/executor_plugin.go
index 44f86836a..15a84db84 100644
--- a/command/executor_plugin.go
+++ b/command/executor_plugin.go
@@ -5,7 +5,7 @@ import (
 
 	"github.com/hashicorp/go-plugin"
 
-	"github.com/hashicorp/nomad/client/driver/plugins"
+	"github.com/hashicorp/nomad/client/driver"
 )
 
 type ExecutorPluginCommand struct {
@@ -25,8 +25,8 @@ func (e *ExecutorPluginCommand) Synopsis() string {
 
 func (e *ExecutorPluginCommand) Run(args []string) int {
 	plugin.Serve(&plugin.ServeConfig{
-		HandshakeConfig: plugins.HandshakeConfig,
-		Plugins:         plugins.PluginMap,
+		HandshakeConfig: driver.HandshakeConfig,
+		Plugins:         driver.PluginMap,
 	})
 	return 0
 }
diff --git a/command/spawn_daemon.go b/command/spawn_daemon.go
deleted file mode 100644
index 52ffd8e6c..000000000
--- a/command/spawn_daemon.go
+++ /dev/null
@@ -1,234 +0,0 @@
-package command
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"os"
-	"os/exec"
-	"strconv"
-	"strings"
-	"syscall"
-)
-
-type SpawnDaemonCommand struct {
-	Meta
-	config   *DaemonConfig
-	exitFile io.WriteCloser
-}
-
-func (c *SpawnDaemonCommand) Help() string {
-	helpText := `
-Usage: nomad spawn-daemon [options] <daemon_config>
-
-  INTERNAL ONLY
-
-  Spawns a daemon process by double forking. The required daemon_config is a
-  json encoding of the DaemonConfig struct containing the isolation
-  configuration and command to run. SpawnStartStatus is json serialized to
-  stdout upon running the user command or if any error prevents its execution.
-  If there is no error, the process waits on the users command. Once the user
-  command exits, the exit code is written to a file specified in the
-  daemon_config and this process exits with the same exit status as the user
-  command.
-  `
-
-	return strings.TrimSpace(helpText)
-}
-
-func (c *SpawnDaemonCommand) Synopsis() string {
-	return "Spawn a daemon command with configurable isolation."
-}
-
-// Status of executing the user's command.
-type SpawnStartStatus struct {
-	// The PID of the user's command.
-	UserPID int
-
-	// ErrorMsg will be empty if the user command was started successfully.
-	// Otherwise it will have an error message.
-	ErrorMsg string
-}
-
-// Exit status of the user's command.
-type SpawnExitStatus struct {
-	// The exit code of the user's command.
-	ExitCode int
-}
-
-// Configuration for the command to start as a daemon.
-type DaemonConfig struct {
-	exec.Cmd
-
-	// The filepath to write the exit status to.
-	ExitStatusFile string
-
-	// The paths, if not /dev/null, must be either in the tasks root directory
-	// or in the shared alloc directory.
-	StdoutFile string
-	StdinFile  string
-	StderrFile string
-
-	// An optional path specifying the directory to chroot the process in.
-	Chroot string
-}
-
-// Whether to start the user command or abort.
-type TaskStart bool
-
-// parseConfig reads the DaemonConfig from the passed arguments. If not
-// successful, an error is returned.
-func (c *SpawnDaemonCommand) parseConfig(args []string) (*DaemonConfig, error) {
-	flags := c.Meta.FlagSet("spawn-daemon", FlagSetClient)
-	flags.Usage = func() { c.Ui.Output(c.Help()) }
-	if err := flags.Parse(args); err != nil {
-		return nil, fmt.Errorf("failed to parse args: %v", err)
-	}
-
-	// Check that we got json input.
-	args = flags.Args()
-	if len(args) != 1 {
-		return nil, fmt.Errorf("incorrect number of args; got %v; want 1", len(args))
-	}
-	jsonInput, err := strconv.Unquote(args[0])
-	if err != nil {
-		return nil, fmt.Errorf("Failed to unquote json input: %v", err)
-	}
-
-	// De-serialize the passed command.
-	var config DaemonConfig
-	dec := json.NewDecoder(strings.NewReader(jsonInput))
-	if err := dec.Decode(&config); err != nil {
-		return nil, err
-	}
-
-	return &config, nil
-}
-
-// configureLogs creates the log files and redirects the process
-// stdin/stderr/stdout to them. If unsuccessful, an error is returned.
-func (c *SpawnDaemonCommand) configureLogs() error {
-	if len(c.config.StdoutFile) != 0 {
-		stdo, err := os.OpenFile(c.config.StdoutFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
-		if err != nil {
-			return fmt.Errorf("Error opening file to redirect stdout: %v", err)
-		}
-
-		c.config.Cmd.Stdout = stdo
-	}
-
-	if len(c.config.StderrFile) != 0 {
-		stde, err := os.OpenFile(c.config.StderrFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
-		if err != nil {
-			return fmt.Errorf("Error opening file to redirect stderr: %v", err)
-		}
-		c.config.Cmd.Stderr = stde
-	}
-
-	if len(c.config.StdinFile) != 0 {
-		stdi, err := os.OpenFile(c.config.StdinFile, os.O_CREATE|os.O_RDONLY, 0666)
-		if err != nil {
-			return fmt.Errorf("Error opening file to redirect stdin: %v", err)
-		}
-		c.config.Cmd.Stdin = stdi
-	}
-
-	return nil
-}
-
-func (c *SpawnDaemonCommand) Run(args []string) int {
-	var err error
-	c.config, err = c.parseConfig(args)
-	if err != nil {
-		return c.outputStartStatus(err, 1)
-	}
-
-	// Open the file we will be using to write exit codes to. We do this early
-	// to ensure that we don't start the user process when we can't capture its
-	// exit status.
-	c.exitFile, err = os.OpenFile(c.config.ExitStatusFile, os.O_WRONLY, 0666)
-	if err != nil {
-		return c.outputStartStatus(fmt.Errorf("Error opening file to store exit status: %v", err), 1)
-	}
-
-	// Isolate the user process.
-	if err := c.isolateCmd(); err != nil {
-		return c.outputStartStatus(err, 1)
-	}
-
-	// Redirect logs.
-	if err := c.configureLogs(); err != nil {
-		return c.outputStartStatus(err, 1)
-	}
-
-	// Chroot jail the process and set its working directory.
-	c.configureChroot()
-
-	// Wait to get the start command.
-	var start TaskStart
-	dec := json.NewDecoder(os.Stdin)
-	if err := dec.Decode(&start); err != nil {
-		return c.outputStartStatus(err, 1)
-	}
-
-	// Aborted by Nomad process.
-	if !start {
-		return 0
-	}
-
-	// Spawn the user process.
-	if err := c.config.Cmd.Start(); err != nil {
-		return c.outputStartStatus(fmt.Errorf("Error starting user command: %v", err), 1)
-	}
-
-	// Indicate that the command was started successfully.
-	c.outputStartStatus(nil, 0)
-
-	// Wait and then output the exit status.
-	return c.writeExitStatus(c.config.Cmd.Wait())
-}
-
-// outputStartStatus is a helper function that outputs a SpawnStartStatus to
-// Stdout with the passed error, which may be nil to indicate no error. It
-// returns the passed status.
-func (c *SpawnDaemonCommand) outputStartStatus(err error, status int) int {
-	startStatus := &SpawnStartStatus{}
-	enc := json.NewEncoder(os.Stdout)
-
-	if err != nil {
-		startStatus.ErrorMsg = err.Error()
-	}
-
-	if c.config != nil && c.config.Cmd.Process != nil {
-		startStatus.UserPID = c.config.Process.Pid
-	}
-
-	enc.Encode(startStatus)
-	return status
-}
-
-// writeExitStatus takes in the error result from calling wait and writes out
-// the exit status to a file. It returns the same exit status as the user
-// command.
-func (c *SpawnDaemonCommand) writeExitStatus(exit error) int {
-	// Parse the exit code.
-	exitStatus := &SpawnExitStatus{}
-	if exit != nil {
-		// Default to exit code 1 if we can not get the actual exit code.
-		exitStatus.ExitCode = 1
-
-		if exiterr, ok := exit.(*exec.ExitError); ok {
-			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
-				exitStatus.ExitCode = status.ExitStatus()
-			}
-		}
-	}
-
-	if c.exitFile != nil {
-		enc := json.NewEncoder(c.exitFile)
-		enc.Encode(exitStatus)
-		c.exitFile.Close()
-	}
-
-	return exitStatus.ExitCode
-}
diff --git a/command/spawn_daemon_darwin.go b/command/spawn_daemon_darwin.go
deleted file mode 100644
index f3fe8484a..000000000
--- a/command/spawn_daemon_darwin.go
+++ /dev/null
@@ -1,4 +0,0 @@
-package command
-
-// No chroot on darwin.
-func (c *SpawnDaemonCommand) configureChroot() {}
diff --git a/command/spawn_daemon_linux.go b/command/spawn_daemon_linux.go
deleted file mode 100644
index 512ec645f..000000000
--- a/command/spawn_daemon_linux.go
+++ /dev/null
@@ -1,16 +0,0 @@
-package command
-
-import "syscall"
-
-// configureChroot enters the user command into a chroot if specified in the
-// config and on an OS that supports Chroots.
-func (c *SpawnDaemonCommand) configureChroot() {
-	if len(c.config.Chroot) != 0 {
-		if c.config.Cmd.SysProcAttr == nil {
-			c.config.Cmd.SysProcAttr = &syscall.SysProcAttr{}
-		}
-
-		c.config.Cmd.SysProcAttr.Chroot = c.config.Chroot
-		c.config.Cmd.Dir = "/"
-	}
-}
diff --git a/command/spawn_daemon_test.go b/command/spawn_daemon_test.go
deleted file mode 100644
index 5bfd6ad5a..000000000
--- a/command/spawn_daemon_test.go
+++ /dev/null
@@ -1,48 +0,0 @@
-package command
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io"
-	"os/exec"
-	"testing"
-)
-
-type nopCloser struct {
-	io.ReadWriter
-}
-
-func (n *nopCloser) Close() error {
-	return nil
-}
-
-func TestSpawnDaemon_WriteExitStatus(t *testing.T) {
-	// Check if there is python.
-	path, err := exec.LookPath("python")
-	if err != nil {
-		t.Skip("python not detected")
-	}
-
-	var b bytes.Buffer
-	daemon := &SpawnDaemonCommand{exitFile: &nopCloser{&b}}
-
-	code := 3
-	cmd := exec.Command(path, "./test-resources/exiter.py", fmt.Sprintf("%d", code))
-	err = cmd.Run()
-	actual := daemon.writeExitStatus(err)
-	if actual != code {
-		t.Fatalf("writeExitStatus(%v) returned %v; want %v", err, actual, code)
-	}
-
-	// De-serialize the passed command.
-	var exitStatus SpawnExitStatus
-	dec := json.NewDecoder(&b)
-	if err := dec.Decode(&exitStatus); err != nil {
-		t.Fatalf("failed to decode exit status: %v", err)
-	}
-
-	if exitStatus.ExitCode != code {
-		t.Fatalf("writeExitStatus(%v) wrote exit status %v; want %v", err, exitStatus.ExitCode, code)
-	}
-}
diff --git a/command/spawn_daemon_unix.go b/command/spawn_daemon_unix.go
deleted file mode 100644
index 981e52596..000000000
--- a/command/spawn_daemon_unix.go
+++ /dev/null
@@ -1,16 +0,0 @@
-// +build !windows
-
-package command
-
-import "syscall"
-
-// isolateCmd sets the session id for the process and the umask.
-func (c *SpawnDaemonCommand) isolateCmd() error {
-	if c.config.Cmd.SysProcAttr == nil {
-		c.config.Cmd.SysProcAttr = &syscall.SysProcAttr{}
-	}
-
-	c.config.Cmd.SysProcAttr.Setsid = true
-	syscall.Umask(0)
-	return nil
-}
diff --git a/command/spawn_daemon_windows.go b/command/spawn_daemon_windows.go
deleted file mode 100644
index bb2d63ed8..000000000
--- a/command/spawn_daemon_windows.go
+++ /dev/null
@@ -1,7 +0,0 @@
-// build !linux !darwin
-
-package command
-
-// No isolation on Windows.
-func (c *SpawnDaemonCommand) isolateCmd() error { return nil }
-func (c *SpawnDaemonCommand) configureChroot()  {}
diff --git a/commands.go b/commands.go
index 9f5fedb12..99294858d 100644
--- a/commands.go
+++ b/commands.go
@@ -118,13 +118,6 @@ func Commands(metaPtr *command.Meta) map[string]cli.CommandFactory {
 				Meta: meta,
 			}, nil
 		},
-
-		"spawn-daemon": func() (cli.Command, error) {
-			return &command.SpawnDaemonCommand{
-				Meta: meta,
-			}, nil
-		},
-
 		"status": func() (cli.Command, error) {
 			return &command.StatusCommand{
 				Meta: meta,

From ded50766c5de013a7e08e22aeaef14f21e6dc54f Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Thu, 4 Feb 2016 16:18:10 -0800
Subject: [PATCH 29/68] Added some docs

---
 client/driver/executor/executor.go       | 17 +++++++++++++++++
 client/driver/executor/executor_linux.go |  3 +++
 2 files changed, 20 insertions(+)

diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go
index 60c18e17e..d4bc088e7 100644
--- a/client/driver/executor/executor.go
+++ b/client/driver/executor/executor.go
@@ -18,6 +18,8 @@ import (
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
+// ExecutorContext is a wrapper to hold context to configure the command user
+// wants to run
 type ExecutorContext struct {
 	TaskEnv          *env.TaskEnvironment
 	AllocDir         *allocdir.AllocDir
@@ -28,17 +30,22 @@ type ExecutorContext struct {
 	UnprivilegedUser bool
 }
 
+// ExecCommand is a wrapper to hold the user command
 type ExecCommand struct {
 	Cmd  string
 	Args []string
 }
 
+// ProcessState holds information about the state of
+// a user process
 type ProcessState struct {
 	Pid      int
 	ExitCode int
 	Time     time.Time
 }
 
+// Executor is the interface which allows a driver to launch and supervise
+// a process user wants to run
 type Executor interface {
 	LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error)
 	Wait() (*ProcessState, error)
@@ -46,6 +53,9 @@ type Executor interface {
 	Exit() error
 }
 
+// UniversalExecutor is an implementation of the Executor which launches and
+// supervises processes. In addition to process supervision it provides resource
+// and file system isolation
 type UniversalExecutor struct {
 	cmd exec.Cmd
 	ctx *ExecutorContext
@@ -59,10 +69,13 @@ type UniversalExecutor struct {
 	lock   sync.Mutex
 }
 
+// NewExecutor returns an Executor
 func NewExecutor(logger *log.Logger) Executor {
 	return &UniversalExecutor{logger: logger, processExited: make(chan interface{})}
 }
 
+// LaunchCmd launches a process and returns it's state. It also configures an
+// applies isolation on certain platforms.
 func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
 	e.ctx = ctx
 
@@ -113,6 +126,7 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext
 	return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, Time: time.Now()}, nil
 }
 
+// Wait waits until a process has exited and returns it's exitcode and errors
 func (e *UniversalExecutor) Wait() (*ProcessState, error) {
 	<-e.processExited
 	return e.exitState, nil
@@ -140,6 +154,8 @@ func (e *UniversalExecutor) wait() {
 	e.exitState = &ProcessState{Pid: 0, ExitCode: exitCode, Time: time.Now()}
 }
 
+// Exit cleans up the alloc directory, destroys cgroups and kills the user
+// process
 func (e *UniversalExecutor) Exit() error {
 	e.logger.Printf("[INFO] Exiting plugin for task %q", e.ctx.TaskName)
 	if e.cmd.Process == nil {
@@ -161,6 +177,7 @@ func (e *UniversalExecutor) Exit() error {
 	return nil
 }
 
+// Shutdown sends an interrupt signal to the user process
 func (e *UniversalExecutor) ShutDown() error {
 	if e.cmd.Process == nil {
 		return fmt.Errorf("executor.shutdown error: no process found")
diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go
index 4c946d429..0b3406e47 100644
--- a/client/driver/executor/executor_linux.go
+++ b/client/driver/executor/executor_linux.go
@@ -33,6 +33,7 @@ var (
 	}
 )
 
+// configureIsolation configures chroot and creates cgroups
 func (e *UniversalExecutor) configureIsolation() error {
 	if e.ctx.FSIsolation {
 		if err := e.configureChroot(); err != nil {
@@ -48,6 +49,7 @@ func (e *UniversalExecutor) configureIsolation() error {
 	return nil
 }
 
+// applyLimits puts a process in a pre-configured cgroup
 func (e *UniversalExecutor) applyLimits() error {
 	if !e.ctx.ResourceLimits {
 		return nil
@@ -140,6 +142,7 @@ func (e *UniversalExecutor) pathExists(path string) bool {
 	return true
 }
 
+// configureChroot configures a chroot
 func (e *UniversalExecutor) configureChroot() error {
 	allocDir := e.ctx.AllocDir
 	if err := allocDir.MountSharedDir(e.ctx.TaskName); err != nil {

From a3282cd0f7d4434ee9db8e3ceaeb51acdd0aedc0 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Thu, 4 Feb 2016 16:28:31 -0800
Subject: [PATCH 30/68] Removed using ProcessState when we just want an empty
 interface

---
 client/driver/executor_plugin.go | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/client/driver/executor_plugin.go b/client/driver/executor_plugin.go
index c07ad3f79..5d7d644c0 100644
--- a/client/driver/executor_plugin.go
+++ b/client/driver/executor_plugin.go
@@ -63,15 +63,11 @@ func (e *ExecutorRPC) Wait() (*executor.ProcessState, error) {
 }
 
 func (e *ExecutorRPC) ShutDown() error {
-	var ps executor.ProcessState
-	err := e.client.Call("Plugin.ShutDown", new(interface{}), &ps)
-	return err
+	return e.client.Call("Plugin.ShutDown", new(interface{}), new(interface{}))
 }
 
 func (e *ExecutorRPC) Exit() error {
-	var ps executor.ProcessState
-	err := e.client.Call("Plugin.Exit", new(interface{}), &ps)
-	return err
+	return e.client.Call("Plugin.Exit", new(interface{}), new(interface{}))
 }
 
 type ExecutorRPCServer struct {
@@ -94,16 +90,12 @@ func (e *ExecutorRPCServer) Wait(args interface{}, ps *executor.ProcessState) er
 	return err
 }
 
-func (e *ExecutorRPCServer) ShutDown(args interface{}, ps *executor.ProcessState) error {
-	var err error
-	err = e.Impl.ShutDown()
-	return err
+func (e *ExecutorRPCServer) ShutDown(args interface{}, resp *interface{}) error {
+	return e.Impl.ShutDown()
 }
 
-func (e *ExecutorRPCServer) Exit(args interface{}, ps *executor.ProcessState) error {
-	var err error
-	err = e.Impl.Exit()
-	return err
+func (e *ExecutorRPCServer) Exit(args interface{}, resp *interface{}) error {
+	return e.Impl.Exit()
 }
 
 type ExecutorPlugin struct {

From 9800b7dcfef751b8915de82dbb450ba4d548b753 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Thu, 4 Feb 2016 16:38:17 -0800
Subject: [PATCH 31/68] Updated the value of the magic cookie

---
 client/driver/executor_plugin.go | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/client/driver/executor_plugin.go b/client/driver/executor_plugin.go
index 5d7d644c0..9cb5f7720 100644
--- a/client/driver/executor_plugin.go
+++ b/client/driver/executor_plugin.go
@@ -12,20 +12,23 @@ import (
 
 var HandshakeConfig = plugin.HandshakeConfig{
 	ProtocolVersion:  1,
-	MagicCookieKey:   "executor_plugin",
-	MagicCookieValue: "value",
+	MagicCookieKey:   "NOMAD_PLUGIN_MAGIC_COOKIE",
+	MagicCookieValue: "e4327c2e01eabfd75a8a67adb114fb34a757d57eee7728d857a8cec6e91a7255",
 }
 
 var PluginMap = map[string]plugin.Plugin{
 	"executor": new(ExecutorPlugin),
 }
 
+// ExecutorReattachConfig is the config that we seralize and de-serialize and
+// store in disk
 type ExecutorReattachConfig struct {
 	Pid      int
 	AddrNet  string
 	AddrName string
 }
 
+// PluginConfig returns a config from an ExecutorReattachConfig
 func (c *ExecutorReattachConfig) PluginConfig() *plugin.ReattachConfig {
 	var addr net.Addr
 	switch c.AddrNet {
@@ -45,6 +48,7 @@ type ExecutorRPC struct {
 	client *rpc.Client
 }
 
+// LaunchCmdArgs wraps a user command and the args for the purposes of RPC
 type LaunchCmdArgs struct {
 	Cmd *executor.ExecCommand
 	Ctx *executor.ExecutorContext

From d8ed2949cdb7c08157402979f7688624e3595e09 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Thu, 4 Feb 2016 17:36:31 -0800
Subject: [PATCH 32/68] Not syncing stdout and stderr of pluging with client

---
 client/driver/exec.go              |  3 ++-
 client/driver/executor/executor.go |  2 ++
 client/driver/executor_plugin.go   | 10 +++++-----
 client/driver/java.go              |  4 +++-
 client/driver/qemu.go              |  4 +++-
 client/driver/raw_exec.go          |  3 ++-
 client/driver/utils.go             |  4 +---
 command/executor_plugin.go         | 13 ++++++++++++-
 8 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index 5273f1d7f..c57c48e56 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -103,8 +103,9 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 	if err != nil {
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
+	pluginLogFile := filepath.Join(ctx.AllocDir.AllocDir, "plugin.out")
 	pluginConfig := &plugin.ClientConfig{
-		Cmd: exec.Command(bin, "executor"),
+		Cmd: exec.Command(bin, "executor", pluginLogFile),
 	}
 
 	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go
index d4bc088e7..cdf153cb3 100644
--- a/client/driver/executor/executor.go
+++ b/client/driver/executor/executor.go
@@ -77,6 +77,8 @@ func NewExecutor(logger *log.Logger) Executor {
 // LaunchCmd launches a process and returns it's state. It also configures an
 // applies isolation on certain platforms.
 func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
+	e.logger.Printf("[INFO] executor: launching command %v", command.Cmd)
+
 	e.ctx = ctx
 
 	if err := e.configureTaskDir(); err != nil {
diff --git a/client/driver/executor_plugin.go b/client/driver/executor_plugin.go
index 9cb5f7720..57a95e326 100644
--- a/client/driver/executor_plugin.go
+++ b/client/driver/executor_plugin.go
@@ -1,10 +1,10 @@
 package driver
 
 import (
+	"io"
 	"log"
 	"net"
 	"net/rpc"
-	"os"
 
 	"github.com/hashicorp/go-plugin"
 	"github.com/hashicorp/nomad/client/driver/executor"
@@ -16,8 +16,10 @@ var HandshakeConfig = plugin.HandshakeConfig{
 	MagicCookieValue: "e4327c2e01eabfd75a8a67adb114fb34a757d57eee7728d857a8cec6e91a7255",
 }
 
-var PluginMap = map[string]plugin.Plugin{
-	"executor": new(ExecutorPlugin),
+func GetPluginMap(w io.Writer) map[string]plugin.Plugin {
+	p := new(ExecutorPlugin)
+	p.logger = log.New(w, "executor-plugin-server:", log.LstdFlags)
+	return map[string]plugin.Plugin{"executor": p}
 }
 
 // ExecutorReattachConfig is the config that we seralize and de-serialize and
@@ -107,11 +109,9 @@ type ExecutorPlugin struct {
 }
 
 func (p *ExecutorPlugin) Server(*plugin.MuxBroker) (interface{}, error) {
-	p.logger = log.New(os.Stdout, "executor-plugin-server:", log.LstdFlags)
 	return &ExecutorRPCServer{Impl: executor.NewExecutor(p.logger)}, nil
 }
 
 func (p *ExecutorPlugin) Client(b *plugin.MuxBroker, c *rpc.Client) (interface{}, error) {
-	p.logger = log.New(os.Stdout, "executor-plugin-client:", log.LstdFlags)
 	return &ExecutorRPC{client: c}, nil
 }
diff --git a/client/driver/java.go b/client/driver/java.go
index 74f295cf8..60bc6c3f4 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -148,8 +148,10 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 	if err != nil {
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
+
+	pluginLogFile := filepath.Join(ctx.AllocDir.AllocDir, "plugin.out")
 	pluginConfig := &plugin.ClientConfig{
-		Cmd: exec.Command(bin, "executor"),
+		Cmd: exec.Command(bin, "executor", pluginLogFile),
 	}
 
 	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
diff --git a/client/driver/qemu.go b/client/driver/qemu.go
index e7753fcc9..cfe0f19fb 100644
--- a/client/driver/qemu.go
+++ b/client/driver/qemu.go
@@ -192,8 +192,10 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 	if err != nil {
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
+
+	pluginLogFile := filepath.Join(ctx.AllocDir.AllocDir, "plugin.out")
 	pluginConfig := &plugin.ClientConfig{
-		Cmd: exec.Command(bin, "executor"),
+		Cmd: exec.Command(bin, "executor", pluginLogFile),
 	}
 
 	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go
index 5b92e6d85..1875caab6 100644
--- a/client/driver/raw_exec.go
+++ b/client/driver/raw_exec.go
@@ -99,8 +99,9 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 	if err != nil {
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
+	pluginLogFile := filepath.Join(ctx.AllocDir.AllocDir, "plugin.out")
 	pluginConfig := &plugin.ClientConfig{
-		Cmd: exec.Command(bin, "executor"),
+		Cmd: exec.Command(bin, "executor", pluginLogFile),
 	}
 
 	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
diff --git a/client/driver/utils.go b/client/driver/utils.go
index 972876eeb..b42c4a74a 100644
--- a/client/driver/utils.go
+++ b/client/driver/utils.go
@@ -12,9 +12,7 @@ import (
 
 func createExecutor(config *plugin.ClientConfig, w io.Writer) (executor.Executor, *plugin.Client, error) {
 	config.HandshakeConfig = HandshakeConfig
-	config.Plugins = PluginMap
-	config.SyncStdout = w
-	config.SyncStderr = w
+	config.Plugins = GetPluginMap(w)
 	executorClient := plugin.NewClient(config)
 	rpcClient, err := executorClient.Client()
 	if err != nil {
diff --git a/command/executor_plugin.go b/command/executor_plugin.go
index 15a84db84..666cab961 100644
--- a/command/executor_plugin.go
+++ b/command/executor_plugin.go
@@ -1,6 +1,7 @@
 package command
 
 import (
+	"os"
 	"strings"
 
 	"github.com/hashicorp/go-plugin"
@@ -24,9 +25,19 @@ func (e *ExecutorPluginCommand) Synopsis() string {
 }
 
 func (e *ExecutorPluginCommand) Run(args []string) int {
+	if len(args) == 0 {
+		e.Ui.Error("log output file isn't provided")
+		return 1
+	}
+	logFileName := args[0]
+	stdo, err := os.OpenFile(logFileName, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
+	if err != nil {
+		e.Ui.Error(err.Error())
+		return 1
+	}
 	plugin.Serve(&plugin.ServeConfig{
 		HandshakeConfig: driver.HandshakeConfig,
-		Plugins:         driver.PluginMap,
+		Plugins:         driver.GetPluginMap(stdo),
 	})
 	return 0
 }

From dcd171a320f97f7381f13bec0146c05565af5374 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Thu, 4 Feb 2016 17:49:47 -0800
Subject: [PATCH 33/68] Running processes with exec as nobody

---
 client/driver/exec.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index c57c48e56..e0b058fad 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -119,7 +119,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		TaskResources:    task.Resources,
 		ResourceLimits:   true,
 		FSIsolation:      true,
-		UnprivilegedUser: false,
+		UnprivilegedUser: true,
 	}
 	ps, err := exec.LaunchCmd(&executor.ExecCommand{Cmd: command, Args: driverConfig.Args}, executorCtx)
 	if err != nil {

From ab493c2fb3123998fd0904802ea993420fe5bd8b Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Fri, 5 Feb 2016 00:11:09 -0800
Subject: [PATCH 34/68] Putting the plugin in the same cgroup as the user
 process

---
 client/driver/executor/executor.go       | 19 ++++++++++++++-----
 client/driver/executor/executor_basic.go |  2 +-
 client/driver/executor/executor_linux.go |  7 +++++--
 3 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go
index cdf153cb3..e568f3696 100644
--- a/client/driver/executor/executor.go
+++ b/client/driver/executor/executor.go
@@ -81,19 +81,24 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext
 
 	e.ctx = ctx
 
+	// configuring the task dir
 	if err := e.configureTaskDir(); err != nil {
 		return nil, err
 	}
+
+	// confiuguring the chroot
 	if err := e.configureIsolation(); err != nil {
 		return nil, err
 	}
 
+	// setting the user of the process
 	if e.ctx.UnprivilegedUser {
 		if err := e.runAs("nobody"); err != nil {
 			return nil, err
 		}
 	}
 
+	// configuring log rotate
 	stdoPath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", ctx.TaskName))
 	stdo, err := os.OpenFile(stdoPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
 	if err != nil {
@@ -108,8 +113,8 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext
 	}
 	e.cmd.Stderr = stde
 
+	// setting the env, path and args for the command
 	e.cmd.Env = ctx.TaskEnv.EnvList()
-
 	e.cmd.Path = ctx.TaskEnv.ReplaceEnv(command.Cmd)
 	e.cmd.Args = append([]string{e.cmd.Path}, ctx.TaskEnv.ParseAndReplace(command.Args)...)
 	if filepath.Base(command.Cmd) == command.Cmd {
@@ -119,11 +124,15 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext
 		}
 	}
 
+	// starting the process
 	if err := e.cmd.Start(); err != nil {
 		return nil, fmt.Errorf("error starting command: %v", err)
 	}
 
-	e.applyLimits()
+	// entering the user process in the cgroup
+	e.applyLimits(e.cmd.Process.Pid)
+	// entering the plugin process in cgroup
+	e.applyLimits(os.Getpid())
 	go e.wait()
 	return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, Time: time.Now()}, nil
 }
@@ -167,15 +176,15 @@ func (e *UniversalExecutor) Exit() error {
 	if err != nil {
 		return fmt.Errorf("failied to find user process %v: %v", e.cmd.Process.Pid, err)
 	}
+	if err = proc.Kill(); err != nil {
+		e.logger.Printf("[DEBUG] executor.exit error: %v", err)
+	}
 	if e.ctx.FSIsolation {
 		e.removeChrootMounts()
 	}
 	if e.ctx.ResourceLimits {
 		e.destroyCgroup()
 	}
-	if err = proc.Kill(); err != nil {
-		e.logger.Printf("[DEBUG] executor.exit error: %v", err)
-	}
 	return nil
 }
 
diff --git a/client/driver/executor/executor_basic.go b/client/driver/executor/executor_basic.go
index 88acf1105..84d080233 100644
--- a/client/driver/executor/executor_basic.go
+++ b/client/driver/executor/executor_basic.go
@@ -18,7 +18,7 @@ func (e *UniversalExecutor) runAs(userid string) error {
 	return nil
 }
 
-func (e *UniversalExecutor) applyLimits() error {
+func (e *UniversalExecutor) applyLimits(pid int) error {
 	return nil
 }
 
diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go
index 0b3406e47..2e6b282c7 100644
--- a/client/driver/executor/executor_linux.go
+++ b/client/driver/executor/executor_linux.go
@@ -50,18 +50,21 @@ func (e *UniversalExecutor) configureIsolation() error {
 }
 
 // applyLimits puts a process in a pre-configured cgroup
-func (e *UniversalExecutor) applyLimits() error {
+func (e *UniversalExecutor) applyLimits(pid int) error {
 	if !e.ctx.ResourceLimits {
 		return nil
 	}
+
+	// Entering the process in the cgroup
 	manager := e.getCgroupManager(e.groups)
-	if err := manager.Apply(e.cmd.Process.Pid); err != nil {
+	if err := manager.Apply(pid); err != nil {
 		e.logger.Printf("[ERROR] unable to join cgroup: %v", err)
 		if err := e.Exit(); err != nil {
 			e.logger.Printf("[ERROR] unable to kill process: %v", err)
 		}
 		return err
 	}
+
 	return nil
 }
 

From 2533156b9eb6c17316151dd5d35215714f221de3 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Fri, 5 Feb 2016 10:49:54 -0800
Subject: [PATCH 35/68] Corrected comments and making the plugins write to
 unique log files

---
 client/driver/exec.go                    | 11 ++--
 client/driver/executor/executor.go       | 71 +++++++++++++-----------
 client/driver/executor/executor_linux.go |  3 +-
 client/driver/executor/executor_test.go  |  8 +--
 client/driver/java.go                    | 11 ++--
 client/driver/qemu.go                    | 10 ++--
 client/driver/raw_exec.go                |  8 +--
 client/driver/utils.go                   |  5 ++
 8 files changed, 67 insertions(+), 60 deletions(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index 5cfb0fcac..429280a36 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -102,7 +102,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 	if err != nil {
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
-	pluginLogFile := filepath.Join(ctx.AllocDir.AllocDir, "plugin.out")
+	pluginLogFile := filepath.Join(ctx.AllocDir.AllocDir, fmt.Sprintf("%s-plugin.out", task.Name))
 	pluginConfig := &plugin.ClientConfig{
 		Cmd: exec.Command(bin, "executor", pluginLogFile),
 	}
@@ -125,7 +125,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		pluginClient.Kill()
 		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
 	}
-	d.logger.Printf("started process via plugin with pid: %v", ps.Pid)
+	d.logger.Printf("[INFO] driver.exec: started process via plugin with pid: %v", ps.Pid)
 
 	// Return a driver handle
 	h := &execHandle{
@@ -153,15 +153,14 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
 	}
 
-	reattachConfig := id.PluginConfig.PluginConfig()
 	pluginConfig := &plugin.ClientConfig{
-		Reattach: reattachConfig,
+		Reattach: id.PluginConfig.PluginConfig(),
 	}
 	executor, client, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
-		d.logger.Println("[ERROR] error connecting to plugin so destroying plugin pid and user pid")
+		d.logger.Println("[ERROR] driver.exec: error connecting to plugin so destroying plugin pid and user pid")
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
-			d.logger.Printf("[ERROR] error destroying plugin and userpid: %v", e)
+			d.logger.Printf("[ERROR] driver.exec: error destroying plugin and userpid: %v", e)
 		}
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go
index e568f3696..136e2208d 100644
--- a/client/driver/executor/executor.go
+++ b/client/driver/executor/executor.go
@@ -7,10 +7,12 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"sync"
 	"syscall"
 	"time"
 
+	"github.com/hashicorp/go-multierror"
 	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
 
 	"github.com/hashicorp/nomad/client/allocdir"
@@ -18,34 +20,35 @@ import (
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
-// ExecutorContext is a wrapper to hold context to configure the command user
-// wants to run
+// ExecutorContext holds context to configure the command user
+// wants to run and isolate it
 type ExecutorContext struct {
-	TaskEnv          *env.TaskEnvironment
-	AllocDir         *allocdir.AllocDir
-	TaskName         string
-	TaskResources    *structs.Resources
-	FSIsolation      bool
-	ResourceLimits   bool
-	UnprivilegedUser bool
+	TaskEnv          *env.TaskEnvironment //TaskEnv holds information about the environment of a Task
+	AllocDir         *allocdir.AllocDir   //AllocDir is the handle to do operations on the alloc dir of the Task
+	TaskName         string               // TaskName is the name of the Task
+	TaskResources    *structs.Resources   // TaskResources are the resource constraints for the Task
+	FSIsolation      bool                 // FSIsolation is a flag for drivers to impose file system isolation on certain platforms
+	ResourceLimits   bool                 // ResourceLimits is a flag for drivers to impose resource contraints on a Task on certain platforms
+	UnprivilegedUser bool                 // UnprivilegedUser is a flag for drivers to make the process run as nobody
 }
 
-// ExecCommand is a wrapper to hold the user command
+// ExecCommand holds the user command and args. It's a lightweight replacement
+// of exec.Cmd for serialization purposes.
 type ExecCommand struct {
 	Cmd  string
 	Args []string
 }
 
-// ProcessState holds information about the state of
-// a user process
+// ProcessState holds information about the state of a user process.
 type ProcessState struct {
 	Pid      int
 	ExitCode int
+	Signal   int
 	Time     time.Time
 }
 
 // Executor is the interface which allows a driver to launch and supervise
-// a process user wants to run
+// a process
 type Executor interface {
 	LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error)
 	Wait() (*ProcessState, error)
@@ -77,7 +80,7 @@ func NewExecutor(logger *log.Logger) Executor {
 // LaunchCmd launches a process and returns it's state. It also configures an
 // applies isolation on certain platforms.
 func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
-	e.logger.Printf("[INFO] executor: launching command %v", command.Cmd)
+	e.logger.Printf("[DEBUG] executor: launching command %v %v", command.Cmd, strings.Join(command.Args, ""))
 
 	e.ctx = ctx
 
@@ -86,11 +89,14 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext
 		return nil, err
 	}
 
-	// confiuguring the chroot
+	// configuring the chroot
 	if err := e.configureIsolation(); err != nil {
 		return nil, err
 	}
 
+	// entering the plugin process in cgroup
+	e.applyLimits(os.Getpid())
+
 	// setting the user of the process
 	if e.ctx.UnprivilegedUser {
 		if err := e.runAs("nobody"); err != nil {
@@ -114,6 +120,7 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext
 	e.cmd.Stderr = stde
 
 	// setting the env, path and args for the command
+	e.ctx.TaskEnv.Build()
 	e.cmd.Env = ctx.TaskEnv.EnvList()
 	e.cmd.Path = ctx.TaskEnv.ReplaceEnv(command.Cmd)
 	e.cmd.Args = append([]string{e.cmd.Path}, ctx.TaskEnv.ParseAndReplace(command.Args)...)
@@ -129,10 +136,6 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext
 		return nil, fmt.Errorf("error starting command: %v", err)
 	}
 
-	// entering the user process in the cgroup
-	e.applyLimits(e.cmd.Process.Pid)
-	// entering the plugin process in cgroup
-	e.applyLimits(os.Getpid())
 	go e.wait()
 	return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, Time: time.Now()}, nil
 }
@@ -168,24 +171,28 @@ func (e *UniversalExecutor) wait() {
 // Exit cleans up the alloc directory, destroys cgroups and kills the user
 // process
 func (e *UniversalExecutor) Exit() error {
-	e.logger.Printf("[INFO] Exiting plugin for task %q", e.ctx.TaskName)
-	if e.cmd.Process == nil {
-		return fmt.Errorf("executor.exit error: no process found")
-	}
-	proc, err := os.FindProcess(e.cmd.Process.Pid)
-	if err != nil {
-		return fmt.Errorf("failied to find user process %v: %v", e.cmd.Process.Pid, err)
-	}
-	if err = proc.Kill(); err != nil {
-		e.logger.Printf("[DEBUG] executor.exit error: %v", err)
+	var merr multierror.Error
+	if e.cmd.Process != nil {
+		proc, err := os.FindProcess(e.cmd.Process.Pid)
+		if err != nil {
+			e.logger.Printf("[ERROR] can't find process with pid: %v, err: %v", e.cmd.Process.Pid, err)
+		}
+		if err := proc.Kill(); err != nil {
+			e.logger.Printf("[ERROR] can't kill process with pid: %v, err: %v", e.cmd.Process.Pid, err)
+		}
 	}
+
 	if e.ctx.FSIsolation {
-		e.removeChrootMounts()
+		if err := e.removeChrootMounts(); err != nil {
+			merr.Errors = append(merr.Errors, err)
+		}
 	}
 	if e.ctx.ResourceLimits {
-		e.destroyCgroup()
+		if err := e.destroyCgroup(); err != nil {
+			merr.Errors = append(merr.Errors, err)
+		}
 	}
-	return nil
+	return merr.ErrorOrNil()
 }
 
 // Shutdown sends an interrupt signal to the user process
diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go
index 2e6b282c7..9c653eade 100644
--- a/client/driver/executor/executor_linux.go
+++ b/client/driver/executor/executor_linux.go
@@ -213,8 +213,7 @@ func (e *UniversalExecutor) removeChrootMounts() error {
 		}
 	}
 
-	// Unmount
-	// proc.
+	// Unmount proc.
 	proc := filepath.Join(e.taskDir, "proc")
 	if e.pathExists(proc) {
 		if err := syscall.Unmount(proc, 0); err != nil {
diff --git a/client/driver/executor/executor_test.go b/client/driver/executor/executor_test.go
index 2b7e281ac..d70d652b1 100644
--- a/client/driver/executor/executor_test.go
+++ b/client/driver/executor/executor_test.go
@@ -57,17 +57,18 @@ func TestExecutor_Start_Invalid(t *testing.T) {
 	invalid := "/bin/foobar"
 	execCmd := ExecCommand{Cmd: invalid, Args: []string{"1"}}
 	ctx := testExecutorContext(t)
+	defer ctx.AllocDir.Destroy()
 	executor := NewExecutor(log.New(os.Stdout, "", log.LstdFlags))
 	_, err := executor.LaunchCmd(&execCmd, ctx)
 	if err == nil {
 		t.Fatalf("Expected error")
 	}
-	defer ctx.AllocDir.Destroy()
 }
 
 func TestExecutor_Start_Wait_Failure_Code(t *testing.T) {
 	execCmd := ExecCommand{Cmd: "/bin/sleep", Args: []string{"fail"}}
 	ctx := testExecutorContext(t)
+	defer ctx.AllocDir.Destroy()
 	executor := NewExecutor(log.New(os.Stdout, "", log.LstdFlags))
 	ps, _ := executor.LaunchCmd(&execCmd, ctx)
 	if ps.Pid == 0 {
@@ -77,12 +78,12 @@ func TestExecutor_Start_Wait_Failure_Code(t *testing.T) {
 	if ps.ExitCode < 1 {
 		t.Fatalf("expected exit code to be non zero, actual: %v", ps.ExitCode)
 	}
-	defer ctx.AllocDir.Destroy()
 }
 
 func TestExecutor_Start_Wait(t *testing.T) {
 	execCmd := ExecCommand{Cmd: "/bin/echo", Args: []string{"hello world"}}
 	ctx := testExecutorContext(t)
+	defer ctx.AllocDir.Destroy()
 	executor := NewExecutor(log.New(os.Stdout, "", log.LstdFlags))
 	ps, err := executor.LaunchCmd(&execCmd, ctx)
 	if err != nil {
@@ -95,7 +96,6 @@ func TestExecutor_Start_Wait(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error in waiting for command: %v", err)
 	}
-	defer ctx.AllocDir.Destroy()
 
 	task := "web"
 	taskDir, ok := ctx.AllocDir.TaskDirs[task]
@@ -120,6 +120,7 @@ func TestExecutor_Start_Wait(t *testing.T) {
 func TestExecutor_Start_Kill(t *testing.T) {
 	execCmd := ExecCommand{Cmd: "/bin/sleep", Args: []string{"10 && hello world"}}
 	ctx := testExecutorContext(t)
+	defer ctx.AllocDir.Destroy()
 	executor := NewExecutor(log.New(os.Stdout, "", log.LstdFlags))
 	ps, err := executor.LaunchCmd(&execCmd, ctx)
 	if err != nil {
@@ -132,7 +133,6 @@ func TestExecutor_Start_Kill(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error in waiting for command: %v", err)
 	}
-	defer ctx.AllocDir.Destroy()
 
 	task := "web"
 	taskDir, ok := ctx.AllocDir.TaskDirs[task]
diff --git a/client/driver/java.go b/client/driver/java.go
index 494916324..1fdac01db 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -148,7 +148,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
 
-	pluginLogFile := filepath.Join(ctx.AllocDir.AllocDir, "plugin.out")
+	pluginLogFile := filepath.Join(ctx.AllocDir.AllocDir, fmt.Sprintf("%s-plugin.out", task.Name))
 	pluginConfig := &plugin.ClientConfig{
 		Cmd: exec.Command(bin, "executor", pluginLogFile),
 	}
@@ -168,7 +168,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		pluginClient.Kill()
 		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
 	}
-	d.logger.Printf("[INFO] started process with pid: %v", ps.Pid)
+	d.logger.Printf("[INFO] driver.java: started process with pid: %v", ps.Pid)
 
 	// Return a driver handle
 	h := &javaHandle{
@@ -197,15 +197,14 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
 	}
 
-	reattachConfig := id.PluginConfig.PluginConfig()
 	pluginConfig := &plugin.ClientConfig{
-		Reattach: reattachConfig,
+		Reattach: id.PluginConfig.PluginConfig(),
 	}
 	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
-		d.logger.Println("[ERROR] error connecting to plugin so destroying plugin pid and user pid")
+		d.logger.Println("[ERROR] driver.java: error connecting to plugin so destroying plugin pid and user pid")
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
-			d.logger.Printf("[ERROR] error destroying plugin and userpid: %v", e)
+			d.logger.Printf("[ERROR] driver.java: error destroying plugin and userpid: %v", e)
 		}
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
diff --git a/client/driver/qemu.go b/client/driver/qemu.go
index f1bed809a..640c416a9 100644
--- a/client/driver/qemu.go
+++ b/client/driver/qemu.go
@@ -192,7 +192,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
 
-	pluginLogFile := filepath.Join(ctx.AllocDir.AllocDir, "plugin.out")
+	pluginLogFile := filepath.Join(ctx.AllocDir.AllocDir, fmt.Sprintf("%s-plugin.out", task.Name))
 	pluginConfig := &plugin.ClientConfig{
 		Cmd: exec.Command(bin, "executor", pluginLogFile),
 	}
@@ -212,7 +212,6 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		pluginClient.Kill()
 		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
 	}
-	d.logger.Printf("[INFO] started process with pid: %v", ps.Pid)
 	d.logger.Printf("[INFO] Started new QemuVM: %s", vmID)
 
 	// Create and Return Handle
@@ -242,16 +241,15 @@ func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
 	}
 
-	reattachConfig := id.PluginConfig.PluginConfig()
 	pluginConfig := &plugin.ClientConfig{
-		Reattach: reattachConfig,
+		Reattach: id.PluginConfig.PluginConfig(),
 	}
 
 	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
-		d.logger.Println("[ERROR] error connecting to plugin so destroying plugin pid and user pid")
+		d.logger.Println("[ERROR] driver.qemu: error connecting to plugin so destroying plugin pid and user pid")
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
-			d.logger.Printf("[ERROR] error destroying plugin and userpid: %v", e)
+			d.logger.Printf("[ERROR] driver.qemu: error destroying plugin and userpid: %v", e)
 		}
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go
index 784e7292e..99fbadbab 100644
--- a/client/driver/raw_exec.go
+++ b/client/driver/raw_exec.go
@@ -98,7 +98,7 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 	if err != nil {
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
-	pluginLogFile := filepath.Join(ctx.AllocDir.AllocDir, "plugin.out")
+	pluginLogFile := filepath.Join(ctx.AllocDir.AllocDir, fmt.Sprintf("%s-plugin.out", task.Name))
 	pluginConfig := &plugin.ClientConfig{
 		Cmd: exec.Command(bin, "executor", pluginLogFile),
 	}
@@ -118,7 +118,7 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 		pluginClient.Kill()
 		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
 	}
-	d.logger.Printf("[INFO] started process with pid: %v", ps.Pid)
+	d.logger.Printf("[INFO] driver.raw_exec: started process with pid: %v", ps.Pid)
 
 	// Return a driver handle
 	h := &rawExecHandle{
@@ -151,9 +151,9 @@ func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, e
 	}
 	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
 	if err != nil {
-		d.logger.Println("[ERROR] error connecting to plugin so destroying plugin pid and user pid")
+		d.logger.Println("[ERROR] driver.raw_exec: error connecting to plugin so destroying plugin pid and user pid")
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
-			d.logger.Printf("[ERROR] error destroying plugin and userpid: %v", e)
+			d.logger.Printf("[ERROR] driver.raw_exec: error destroying plugin and userpid: %v", e)
 		}
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
diff --git a/client/driver/utils.go b/client/driver/utils.go
index b42c4a74a..e8935c80e 100644
--- a/client/driver/utils.go
+++ b/client/driver/utils.go
@@ -10,6 +10,8 @@ import (
 	"github.com/hashicorp/nomad/client/driver/executor"
 )
 
+// createExecutor launches an executor plugin and returns an instance of the
+// Executor interface
 func createExecutor(config *plugin.ClientConfig, w io.Writer) (executor.Executor, *plugin.Client, error) {
 	config.HandshakeConfig = HandshakeConfig
 	config.Plugins = GetPluginMap(w)
@@ -27,6 +29,7 @@ func createExecutor(config *plugin.ClientConfig, w io.Writer) (executor.Executor
 	return executorPlugin, executorClient, nil
 }
 
+// killProcess kills a process with the given pid
 func killProcess(pid int) error {
 	proc, err := os.FindProcess(pid)
 	if err != nil {
@@ -35,6 +38,8 @@ func killProcess(pid int) error {
 	return proc.Kill()
 }
 
+// destroyPlugin kills the plugin with the given pid and also kills the user
+// process
 func destroyPlugin(pluginPid int, userPid int) error {
 	var merr error
 	if err := killProcess(pluginPid); err != nil {

From 8a81cee4256e22dda51c6d8fb403ebca5979718b Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Fri, 5 Feb 2016 10:51:29 -0800
Subject: [PATCH 36/68] Filtered out the executor command

---
 main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 mode change 100755 => 100644 main.go

diff --git a/main.go b/main.go
old mode 100755
new mode 100644
index 36115789d..1e3558658
--- a/main.go
+++ b/main.go
@@ -32,7 +32,7 @@ func RunCustom(args []string, commands map[string]cli.CommandFactory) int {
 	commandsInclude := make([]string, 0, len(commands))
 	for k, _ := range commands {
 		switch k {
-		case "spawn-daemon":
+		case "executor":
 		default:
 			commandsInclude = append(commandsInclude, k)
 		}

From 10958c463c94289e6c8007295c64e8cb543b8db7 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Fri, 5 Feb 2016 10:55:03 -0800
Subject: [PATCH 37/68] Making the java driver apply resource constraints and
 limits

---
 client/driver/java.go | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/client/driver/java.go b/client/driver/java.go
index 1fdac01db..6ad6baabc 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -158,10 +158,13 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		return nil, err
 	}
 	executorCtx := &executor.ExecutorContext{
-		TaskEnv:       d.taskEnv,
-		AllocDir:      ctx.AllocDir,
-		TaskName:      task.Name,
-		TaskResources: task.Resources,
+		TaskEnv:          d.taskEnv,
+		AllocDir:         ctx.AllocDir,
+		TaskName:         task.Name,
+		TaskResources:    task.Resources,
+		FSIsolation:      true,
+		ResourceLimits:   true,
+		UnprivilegedUser: true,
 	}
 	ps, err := exec.LaunchCmd(&executor.ExecCommand{Cmd: "java", Args: args}, executorCtx)
 	if err != nil {

From b3d18a7a07d327852d66e173dde8db387dfe873d Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Fri, 5 Feb 2016 11:07:16 -0800
Subject: [PATCH 38/68] Added a test which isolates and constraints a process
 using the executor

---
 client/driver/executor/executor_test.go | 49 ++++++++++++++++++++++++-
 1 file changed, 47 insertions(+), 2 deletions(-)

diff --git a/client/driver/executor/executor_test.go b/client/driver/executor/executor_test.go
index d70d652b1..26876e7e9 100644
--- a/client/driver/executor/executor_test.go
+++ b/client/driver/executor/executor_test.go
@@ -11,9 +11,10 @@ import (
 
 	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/driver/env"
+	"github.com/hashicorp/nomad/client/testutil"
 	"github.com/hashicorp/nomad/nomad/mock"
 	"github.com/hashicorp/nomad/nomad/structs"
-	"github.com/hashicorp/nomad/testutil"
+	tu "github.com/hashicorp/nomad/testutil"
 )
 
 var (
@@ -117,6 +118,50 @@ func TestExecutor_Start_Wait(t *testing.T) {
 	}
 }
 
+func TestExecutor_IsolationAndConstraints(t *testing.T) {
+	testutil.ExecCompatible(t)
+
+	execCmd := ExecCommand{Cmd: "/bin/echo", Args: []string{"hello world"}}
+	ctx := testExecutorContext(t)
+	defer ctx.AllocDir.Destroy()
+
+	ctx.FSIsolation = true
+	ctx.ResourceLimits = true
+	ctx.UnprivilegedUser = true
+
+	executor := NewExecutor(log.New(os.Stdout, "", log.LstdFlags))
+	ps, err := executor.LaunchCmd(&execCmd, ctx)
+	if err != nil {
+		t.Fatalf("error in launching command: %v", err)
+	}
+	if ps.Pid == 0 {
+		t.Fatalf("expected process to start and have non zero pid")
+	}
+	ps, err = executor.Wait()
+	if err != nil {
+		t.Fatalf("error in waiting for command: %v", err)
+	}
+
+	task := "web"
+	taskDir, ok := ctx.AllocDir.TaskDirs[task]
+	if !ok {
+		log.Panicf("No task directory found for task %v", task)
+	}
+
+	expected := "hello world"
+	file := filepath.Join(allocdir.TaskLocal, "web.stdout")
+	absFilePath := filepath.Join(taskDir, file)
+	output, err := ioutil.ReadFile(absFilePath)
+	if err != nil {
+		t.Fatalf("Couldn't read file %v", absFilePath)
+	}
+
+	act := strings.TrimSpace(string(output))
+	if act != expected {
+		t.Fatalf("Command output incorrectly: want %v; got %v", expected, act)
+	}
+}
+
 func TestExecutor_Start_Kill(t *testing.T) {
 	execCmd := ExecCommand{Cmd: "/bin/sleep", Args: []string{"10 && hello world"}}
 	ctx := testExecutorContext(t)
@@ -143,7 +188,7 @@ func TestExecutor_Start_Kill(t *testing.T) {
 	file := filepath.Join(allocdir.TaskLocal, "web.stdout")
 	absFilePath := filepath.Join(taskDir, file)
 
-	time.Sleep(time.Duration(testutil.TestMultiplier()*2) * time.Second)
+	time.Sleep(time.Duration(tu.TestMultiplier()*2) * time.Second)
 
 	output, err := ioutil.ReadFile(absFilePath)
 	if err != nil {

From 1ba696abe58966fc7a00cfc1cf7ca7cfb5e854fe Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Fri, 5 Feb 2016 11:37:13 -0800
Subject: [PATCH 39/68] Added a test which shows userpid is killed when plugin
 can not be re-connected on calling Open

---
 client/driver/exec_test.go | 52 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go
index e7189f1fc..a9f941b9b 100644
--- a/client/driver/exec_test.go
+++ b/client/driver/exec_test.go
@@ -1,8 +1,10 @@
 package driver
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"reflect"
 	"testing"
@@ -75,6 +77,56 @@ func TestExecDriver_StartOpen_Wait(t *testing.T) {
 	handle2.Kill()
 }
 
+func TestExecDriver_KillUserPid_OnPluginReconnectFailure(t *testing.T) {
+	t.Parallel()
+	ctestutils.ExecCompatible(t)
+	task := &structs.Task{
+		Name: "sleep",
+		Config: map[string]interface{}{
+			"command": "/bin/sleep",
+			"args":    []string{"1000000"},
+		},
+		Resources: basicResources,
+	}
+
+	driverCtx, execCtx := testDriverContexts(task)
+	defer execCtx.AllocDir.Destroy()
+	d := NewExecDriver(driverCtx)
+
+	handle, err := d.Start(execCtx, task)
+	defer handle.Kill()
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if handle == nil {
+		t.Fatalf("missing handle")
+	}
+
+	id := &execId{}
+	if err := json.Unmarshal([]byte(handle.ID()), id); err != nil {
+		t.Fatalf("Failed to parse handle '%s': %v", handle.ID(), err)
+	}
+	pluginPid := id.PluginConfig.Pid
+	proc, err := os.FindProcess(pluginPid)
+	if err != nil {
+		t.Fatalf("can't find plugin pid: %v", pluginPid)
+	}
+	if err := proc.Kill(); err != nil {
+		t.Fatalf("can't kill plugin pid: %v", err)
+	}
+
+	// Attempt to open
+	handle2, err := d.Open(execCtx, handle.ID())
+	userProc, err := os.FindProcess(id.UserPid)
+	if err := userProc.Kill(); err == nil {
+		t.Fatalf("user process is supposed to be killed when plugin exits")
+	}
+	if handle2 != nil {
+		defer handle2.Kill()
+		t.Fatalf("expected handle2 to be nil")
+	}
+}
+
 func TestExecDriver_Start_Wait(t *testing.T) {
 	t.Parallel()
 	ctestutils.ExecCompatible(t)

From 218b9c1261935c0f70ea657732c593c449911b24 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Fri, 5 Feb 2016 13:05:49 -0800
Subject: [PATCH 40/68] Using signals to test if proces is alive

---
 client/driver/exec_test.go | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go
index a9f941b9b..a66e08a04 100644
--- a/client/driver/exec_test.go
+++ b/client/driver/exec_test.go
@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"syscall"
 	"testing"
 	"time"
 
@@ -117,14 +118,20 @@ func TestExecDriver_KillUserPid_OnPluginReconnectFailure(t *testing.T) {
 
 	// Attempt to open
 	handle2, err := d.Open(execCtx, handle.ID())
-	userProc, err := os.FindProcess(id.UserPid)
-	if err := userProc.Kill(); err == nil {
-		t.Fatalf("user process is supposed to be killed when plugin exits")
+	if err == nil {
+		t.Fatalf("expected error")
 	}
 	if handle2 != nil {
 		defer handle2.Kill()
 		t.Fatalf("expected handle2 to be nil")
 	}
+	// Test if the userpid is still present
+	userProc, err := os.FindProcess(id.UserPid)
+
+	err = userProc.Signal(syscall.Signal(0))
+	if err != nil {
+		t.Fatalf("expected user process to die")
+	}
 }
 
 func TestExecDriver_Start_Wait(t *testing.T) {

From 57a1c13cc70d62b69f2870eb79aa802a995d410c Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Fri, 5 Feb 2016 13:16:29 -0800
Subject: [PATCH 41/68] Entering the plugin pid into the cgroup after creating
 it

---
 client/driver/executor/executor.go       | 3 ---
 client/driver/executor/executor_linux.go | 9 +++++++++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go
index 136e2208d..fdafc3c05 100644
--- a/client/driver/executor/executor.go
+++ b/client/driver/executor/executor.go
@@ -94,9 +94,6 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext
 		return nil, err
 	}
 
-	// entering the plugin process in cgroup
-	e.applyLimits(os.Getpid())
-
 	// setting the user of the process
 	if e.ctx.UnprivilegedUser {
 		if err := e.runAs("nobody"); err != nil {
diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go
index 9c653eade..c95db04d0 100644
--- a/client/driver/executor/executor_linux.go
+++ b/client/driver/executor/executor_linux.go
@@ -45,6 +45,15 @@ func (e *UniversalExecutor) configureIsolation() error {
 		if err := e.configureCgroups(e.ctx.TaskResources); err != nil {
 			return fmt.Errorf("error creating cgroups: %v", err)
 		}
+		if err := e.applyLimits(os.Getpid()); err != nil {
+			if er := e.destroyCgroup(); er != nil {
+				e.logger.Printf("[ERROR] error destroying cgroup: %v", er)
+			}
+			if er := e.removeChrootMounts(); er != nil {
+				e.logger.Printf("[ERROR] error removing chroot: %v", er)
+			}
+			return fmt.Errorf("error entering the plugin process in the cgroup: %v:", err)
+		}
 	}
 	return nil
 }

From f51e377f527f11be4b1af3221b70ce78494d12e2 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Fri, 5 Feb 2016 15:17:15 -0800
Subject: [PATCH 42/68] Reserving ports on windows

---
 client/config/config.go |  6 +++++
 command/agent/agent.go  | 49 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+)

diff --git a/client/config/config.go b/client/config/config.go
index 9e07bc12e..bb595c6ad 100644
--- a/client/config/config.go
+++ b/client/config/config.go
@@ -57,6 +57,12 @@ type Config struct {
 	// Node provides the base node
 	Node *structs.Node
 
+	// PluginMaxPort defines the highest port a plugin process can use
+	PluginMaxPort int
+
+	// PluginMinPort defines the lowest port a plugin process can use
+	PluginMinPort int
+
 	// Options provides arbitrary key-value configuration for nomad internals,
 	// like fingerprinters and drivers. The format is:
 	//
diff --git a/command/agent/agent.go b/command/agent/agent.go
index 0c7f1dd48..8ae8f4823 100644
--- a/command/agent/agent.go
+++ b/command/agent/agent.go
@@ -7,6 +7,7 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"runtime"
 	"sync"
 	"time"
 
@@ -238,6 +239,32 @@ func (a *Agent) setupClient() error {
 	}
 	conf.Node.HTTPAddr = httpAddr
 
+	// Reserve some ports for the plugins
+	if runtime.GOOS == "windows" {
+		deviceName, err := a.findInterfaceNameForIP("127.0.0.1")
+		if err != nil {
+			return fmt.Errorf("error finding the device name for the ip 127.0.0.1: %v", err)
+		}
+		conf.PluginMinPort = 14000
+		conf.PluginMaxPort = 15000
+		var nr *structs.NetworkResource
+		for _, n := range conf.Node.Reserved.Networks {
+			if n.Device == deviceName {
+				nr = n
+			}
+		}
+		if nr == nil {
+			nr = &structs.NetworkResource{
+				Device:        deviceName,
+				ReservedPorts: make([]structs.Port, 0),
+			}
+		}
+		for i := conf.PluginMinPort; i <= conf.PluginMaxPort; i++ {
+			nr.ReservedPorts = append(nr.ReservedPorts, structs.Port{Label: fmt.Sprintf("plugin-%d", i), Value: i})
+		}
+
+	}
+
 	// Create the client
 	client, err := client.NewClient(conf)
 	if err != nil {
@@ -247,6 +274,28 @@ func (a *Agent) setupClient() error {
 	return nil
 }
 
+func (a *Agent) findInterfaceNameForIP(ip string) (string, error) {
+	var ifcs []net.Interface
+	var err error
+	var deviceName string
+	ifcs, err = net.Interfaces()
+	if err != nil {
+		return "", err
+	}
+	for _, ifc := range ifcs {
+		addrs, err := ifc.Addrs()
+		if err != nil {
+			return deviceName, err
+		}
+		for _, addr := range addrs {
+			if addr.String() == "127.0.0.1" {
+				return ifc.Name, nil
+			}
+		}
+	}
+	return deviceName, err
+}
+
 // Leave is used gracefully exit. Clients will inform servers
 // of their departure so that allocations can be rescheduled.
 func (a *Agent) Leave() error {

From 82c7f26bc354efc851692770c1a1d00d7f5bdd83 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Fri, 5 Feb 2016 17:07:02 -0800
Subject: [PATCH 43/68] Renamed the plugin log file

---
 client/driver/exec.go              |  2 +-
 client/driver/executor/executor.go | 32 +++++++++++++++++++++++-------
 client/driver/java.go              |  2 +-
 client/driver/qemu.go              |  2 +-
 client/driver/raw_exec.go          |  2 +-
 5 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index 429280a36..b942ea3fe 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -102,7 +102,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 	if err != nil {
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
-	pluginLogFile := filepath.Join(ctx.AllocDir.AllocDir, fmt.Sprintf("%s-plugin.out", task.Name))
+	pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name))
 	pluginConfig := &plugin.ClientConfig{
 		Cmd: exec.Command(bin, "executor", pluginLogFile),
 	}
diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go
index fdafc3c05..13b773303 100644
--- a/client/driver/executor/executor.go
+++ b/client/driver/executor/executor.go
@@ -23,13 +23,31 @@ import (
 // ExecutorContext holds context to configure the command user
 // wants to run and isolate it
 type ExecutorContext struct {
-	TaskEnv          *env.TaskEnvironment //TaskEnv holds information about the environment of a Task
-	AllocDir         *allocdir.AllocDir   //AllocDir is the handle to do operations on the alloc dir of the Task
-	TaskName         string               // TaskName is the name of the Task
-	TaskResources    *structs.Resources   // TaskResources are the resource constraints for the Task
-	FSIsolation      bool                 // FSIsolation is a flag for drivers to impose file system isolation on certain platforms
-	ResourceLimits   bool                 // ResourceLimits is a flag for drivers to impose resource contraints on a Task on certain platforms
-	UnprivilegedUser bool                 // UnprivilegedUser is a flag for drivers to make the process run as nobody
+
+	// TaskEnv holds information about the environment of a Task
+	TaskEnv *env.TaskEnvironment
+
+	// AllocDir is the handle to do operations on the alloc dir of
+	// the task
+	AllocDir *allocdir.AllocDir
+
+	// TaskName is the name of the Task
+	TaskName string
+
+	// TaskResources are the resource constraints for the Task
+	TaskResources *structs.Resources
+
+	// FSIsolation is a flag for drivers to impose file system
+	// isolation on certain platforms
+	FSIsolation bool
+
+	// ResourceLimits is a flag for drivers to impose resource
+	// contraints on a Task on certain platforms
+	ResourceLimits bool
+
+	// UnprivilegedUser is a flag for drivers to make the process
+	// run as nobody
+	UnprivilegedUser bool
 }
 
 // ExecCommand holds the user command and args. It's a lightweight replacement
diff --git a/client/driver/java.go b/client/driver/java.go
index 6ad6baabc..24b6e665f 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -148,7 +148,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
 
-	pluginLogFile := filepath.Join(ctx.AllocDir.AllocDir, fmt.Sprintf("%s-plugin.out", task.Name))
+	pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name))
 	pluginConfig := &plugin.ClientConfig{
 		Cmd: exec.Command(bin, "executor", pluginLogFile),
 	}
diff --git a/client/driver/qemu.go b/client/driver/qemu.go
index 640c416a9..849e0cd35 100644
--- a/client/driver/qemu.go
+++ b/client/driver/qemu.go
@@ -192,7 +192,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
 
-	pluginLogFile := filepath.Join(ctx.AllocDir.AllocDir, fmt.Sprintf("%s-plugin.out", task.Name))
+	pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name))
 	pluginConfig := &plugin.ClientConfig{
 		Cmd: exec.Command(bin, "executor", pluginLogFile),
 	}
diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go
index 99fbadbab..d826d85b8 100644
--- a/client/driver/raw_exec.go
+++ b/client/driver/raw_exec.go
@@ -98,7 +98,7 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 	if err != nil {
 		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
 	}
-	pluginLogFile := filepath.Join(ctx.AllocDir.AllocDir, fmt.Sprintf("%s-plugin.out", task.Name))
+	pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name))
 	pluginConfig := &plugin.ClientConfig{
 		Cmd: exec.Command(bin, "executor", pluginLogFile),
 	}

From abad46c423f8ddc740c0fe648eb2ee6b28ad6e83 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Fri, 5 Feb 2016 17:40:06 -0800
Subject: [PATCH 44/68] Updated comment

---
 client/driver/executor/executor.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go
index 13b773303..8b7e419fb 100644
--- a/client/driver/executor/executor.go
+++ b/client/driver/executor/executor.go
@@ -107,7 +107,8 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext
 		return nil, err
 	}
 
-	// configuring the chroot
+	// configuring the chroot, cgroup and enters the plugin process in the
+	// chroot
 	if err := e.configureIsolation(); err != nil {
 		return nil, err
 	}

From 84b31eff9858a91c7d544ff9276a264ed7f1441e Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Fri, 5 Feb 2016 18:07:06 -0800
Subject: [PATCH 45/68] Changed a few comments

---
 client/driver/exec.go      | 2 +-
 client/driver/exec_test.go | 2 +-
 client/driver/java.go      | 2 +-
 client/driver/raw_exec.go  | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index b942ea3fe..4aebad667 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -125,7 +125,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		pluginClient.Kill()
 		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
 	}
-	d.logger.Printf("[INFO] driver.exec: started process via plugin with pid: %v", ps.Pid)
+	d.logger.Printf("[DEBUG] driver.exec: started process via plugin with pid: %v", ps.Pid)
 
 	// Return a driver handle
 	h := &execHandle{
diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go
index a66e08a04..047584db0 100644
--- a/client/driver/exec_test.go
+++ b/client/driver/exec_test.go
@@ -122,7 +122,7 @@ func TestExecDriver_KillUserPid_OnPluginReconnectFailure(t *testing.T) {
 		t.Fatalf("expected error")
 	}
 	if handle2 != nil {
-		defer handle2.Kill()
+		handle2.Kill()
 		t.Fatalf("expected handle2 to be nil")
 	}
 	// Test if the userpid is still present
diff --git a/client/driver/java.go b/client/driver/java.go
index 24b6e665f..11daba07f 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -171,7 +171,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		pluginClient.Kill()
 		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
 	}
-	d.logger.Printf("[INFO] driver.java: started process with pid: %v", ps.Pid)
+	d.logger.Printf("[DEBUG] driver.java: started process with pid: %v", ps.Pid)
 
 	// Return a driver handle
 	h := &javaHandle{
diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go
index d826d85b8..2dd82cc6b 100644
--- a/client/driver/raw_exec.go
+++ b/client/driver/raw_exec.go
@@ -118,7 +118,7 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 		pluginClient.Kill()
 		return nil, fmt.Errorf("error starting process via the plugin: %v", err)
 	}
-	d.logger.Printf("[INFO] driver.raw_exec: started process with pid: %v", ps.Pid)
+	d.logger.Printf("[DEBUG] driver.raw_exec: started process with pid: %v", ps.Pid)
 
 	// Return a driver handle
 	h := &rawExecHandle{

From 10b7160a39cda5942e222b98a79bb05d81584fd2 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Sun, 7 Feb 2016 22:07:50 -0500
Subject: [PATCH 46/68] Added the executor to the commands map

---
 commands.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/commands.go b/commands.go
index 9bd20920b..6ad1ba3c3 100644
--- a/commands.go
+++ b/commands.go
@@ -57,6 +57,11 @@ func Commands(metaPtr *command.Meta) map[string]cli.CommandFactory {
 				Meta: meta,
 			}, nil
 		},
+		"executor": func() (cli.Command, error) {
+			return &command.ExecutorPluginCommand{
+				Meta: meta,
+			}, nil
+		},
 		"fs": func() (cli.Command, error) {
 			return &command.FSCommand{
 				Meta: meta,

From b1b57d20b0ea971f3ddb7879796086ecef482c85 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Sun, 7 Feb 2016 22:33:48 -0500
Subject: [PATCH 47/68] Isolating the plugin process from the nomad client

---
 client/driver/utils.go         |  3 +++
 client/driver/utils_posix.go   | 16 ++++++++++++++++
 client/driver/utils_windows.go |  9 +++++++++
 3 files changed, 28 insertions(+)
 create mode 100644 client/driver/utils_posix.go
 create mode 100644 client/driver/utils_windows.go

diff --git a/client/driver/utils.go b/client/driver/utils.go
index e8935c80e..f2f8fc02a 100644
--- a/client/driver/utils.go
+++ b/client/driver/utils.go
@@ -15,6 +15,9 @@ import (
 func createExecutor(config *plugin.ClientConfig, w io.Writer) (executor.Executor, *plugin.Client, error) {
 	config.HandshakeConfig = HandshakeConfig
 	config.Plugins = GetPluginMap(w)
+	if config.Cmd != nil {
+		isolateCommand(config.Cmd)
+	}
 	executorClient := plugin.NewClient(config)
 	rpcClient, err := executorClient.Client()
 	if err != nil {
diff --git a/client/driver/utils_posix.go b/client/driver/utils_posix.go
new file mode 100644
index 000000000..693bba6f6
--- /dev/null
+++ b/client/driver/utils_posix.go
@@ -0,0 +1,16 @@
+package driver
+
+import (
+	"os/exec"
+	"syscall"
+)
+
+// isolateCommand sets the setsid flag in exec.Cmd to true so that the process
+// becomes the process leader in a new session and doesn't receive signals that
+// are sent to the parent process.
+func isolateCommand(cmd *exec.Cmd) {
+	if cmd.SysProcAttr == nil {
+		cmd.SysProcAttr = &syscall.SysProcAttr{}
+	}
+	cmd.SysProcAttr.Setsid = true
+}
diff --git a/client/driver/utils_windows.go b/client/driver/utils_windows.go
new file mode 100644
index 000000000..5b2b7d842
--- /dev/null
+++ b/client/driver/utils_windows.go
@@ -0,0 +1,9 @@
+package driver
+
+import (
+	"os/exec"
+)
+
+// TODO Figure out if this is needed in Wondows
+func isolateCommand(cmd *exec.Cmd) {
+}

From 2efdd83636899b5bf244fdab9b7e325ce164e1ab Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 8 Feb 2016 07:57:31 -0800
Subject: [PATCH 48/68] Using net.IsLoopback to determine if ifc is a loopback
 device

---
 client/config/config.go |  8 ++++----
 command/agent/agent.go  | 16 ++++++++++------
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/client/config/config.go b/client/config/config.go
index bb595c6ad..0d59a1b9a 100644
--- a/client/config/config.go
+++ b/client/config/config.go
@@ -57,11 +57,11 @@ type Config struct {
 	// Node provides the base node
 	Node *structs.Node
 
-	// PluginMaxPort defines the highest port a plugin process can use
-	PluginMaxPort int
+	// ExecutorMaxPort defines the highest port a plugin process can use
+	ExecutorMaxPort int
 
-	// PluginMinPort defines the lowest port a plugin process can use
-	PluginMinPort int
+	// ExecutorMinPort defines the lowest port a plugin process can use
+	ExecutorMinPort int
 
 	// Options provides arbitrary key-value configuration for nomad internals,
 	// like fingerprinters and drivers. The format is:
diff --git a/command/agent/agent.go b/command/agent/agent.go
index 8ae8f4823..28e5e6b60 100644
--- a/command/agent/agent.go
+++ b/command/agent/agent.go
@@ -241,12 +241,16 @@ func (a *Agent) setupClient() error {
 
 	// Reserve some ports for the plugins
 	if runtime.GOOS == "windows" {
-		deviceName, err := a.findInterfaceNameForIP("127.0.0.1")
+		deviceName, err := a.findLoopbackDevice()
+		if conf.ExecutorMaxPort == 0 {
+			conf.ExecutorMaxPort = 15000
+		}
+		if conf.ExecutorMinPort == 0 {
+			conf.ExecutorMinPort = 14000
+		}
 		if err != nil {
 			return fmt.Errorf("error finding the device name for the ip 127.0.0.1: %v", err)
 		}
-		conf.PluginMinPort = 14000
-		conf.PluginMaxPort = 15000
 		var nr *structs.NetworkResource
 		for _, n := range conf.Node.Reserved.Networks {
 			if n.Device == deviceName {
@@ -259,7 +263,7 @@ func (a *Agent) setupClient() error {
 				ReservedPorts: make([]structs.Port, 0),
 			}
 		}
-		for i := conf.PluginMinPort; i <= conf.PluginMaxPort; i++ {
+		for i := conf.ExecutorMinPort; i <= conf.ExecutorMaxPort; i++ {
 			nr.ReservedPorts = append(nr.ReservedPorts, structs.Port{Label: fmt.Sprintf("plugin-%d", i), Value: i})
 		}
 
@@ -274,7 +278,7 @@ func (a *Agent) setupClient() error {
 	return nil
 }
 
-func (a *Agent) findInterfaceNameForIP(ip string) (string, error) {
+func (a *Agent) findLoopbackDevice() (string, error) {
 	var ifcs []net.Interface
 	var err error
 	var deviceName string
@@ -288,7 +292,7 @@ func (a *Agent) findInterfaceNameForIP(ip string) (string, error) {
 			return deviceName, err
 		}
 		for _, addr := range addrs {
-			if addr.String() == "127.0.0.1" {
+			if net.ParseIP(addr.String()).IsLoopback() {
 				return ifc.Name, nil
 			}
 		}

From d7a772c52ba4be2daead790a79a8580d9717c076 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 8 Feb 2016 10:05:39 -0800
Subject: [PATCH 49/68] Destroying the cgroup if we can't to executor in exec
 and java drivers

---
 client/driver/exec.go              | 10 ++++++++
 client/driver/executor/executor.go | 11 ++++----
 client/driver/java.go              | 10 ++++++++
 client/driver/utils_linux.go       | 40 ++++++++++++++++++++++++++++++
 client/driver/utils_posix.go       |  8 ++++++
 client/driver/utils_windows.go     |  6 +++++
 6 files changed, 80 insertions(+), 5 deletions(-)
 create mode 100644 client/driver/utils_linux.go

diff --git a/client/driver/exec.go b/client/driver/exec.go
index 4aebad667..6936461b6 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -17,6 +17,8 @@ import (
 	"github.com/hashicorp/nomad/helper/discover"
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/mitchellh/mapstructure"
+
+	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
 )
 
 // ExecDriver fork/execs tasks using as many of the underlying OS's isolation
@@ -36,6 +38,7 @@ type ExecDriverConfig struct {
 type execHandle struct {
 	pluginClient *plugin.Client
 	executor     executor.Executor
+	groups       *cgroupConfig.Cgroup
 	userPid      int
 	killTimeout  time.Duration
 	logger       *log.Logger
@@ -132,6 +135,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		pluginClient: pluginClient,
 		userPid:      ps.Pid,
 		executor:     exec,
+		groups:       &ps.IsolationConfig,
 		killTimeout:  d.DriverContext.KillTimeout(task),
 		logger:       d.logger,
 		doneCh:       make(chan struct{}),
@@ -144,6 +148,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 type execId struct {
 	KillTimeout  time.Duration
 	UserPid      int
+	Groups       *cgroupConfig.Cgroup
 	PluginConfig *ExecutorReattachConfig
 }
 
@@ -162,6 +167,9 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
 			d.logger.Printf("[ERROR] driver.exec: error destroying plugin and userpid: %v", e)
 		}
+		if e := destroyCgroup(id.Groups); e != nil {
+			d.logger.Printf("[ERROR] driver.exec: %v", e)
+		}
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
 
@@ -170,6 +178,7 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		pluginClient: client,
 		executor:     executor,
 		userPid:      id.UserPid,
+		groups:       id.Groups,
 		logger:       d.logger,
 		killTimeout:  id.KillTimeout,
 		doneCh:       make(chan struct{}),
@@ -184,6 +193,7 @@ func (h *execHandle) ID() string {
 		KillTimeout:  h.killTimeout,
 		PluginConfig: NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
 		UserPid:      h.userPid,
+		Groups:       h.groups,
 	}
 
 	data, err := json.Marshal(id)
diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go
index 8b7e419fb..99d1c6fe2 100644
--- a/client/driver/executor/executor.go
+++ b/client/driver/executor/executor.go
@@ -59,10 +59,11 @@ type ExecCommand struct {
 
 // ProcessState holds information about the state of a user process.
 type ProcessState struct {
-	Pid      int
-	ExitCode int
-	Signal   int
-	Time     time.Time
+	Pid             int
+	ExitCode        int
+	Signal          int
+	IsolationConfig cgroupConfig.Cgroup
+	Time            time.Time
 }
 
 // Executor is the interface which allows a driver to launch and supervise
@@ -153,7 +154,7 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext
 	}
 
 	go e.wait()
-	return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, Time: time.Now()}, nil
+	return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, IsolationConfig: *e.groups, Time: time.Now()}, nil
 }
 
 // Wait waits until a process has exited and returns it's exitcode and errors
diff --git a/client/driver/java.go b/client/driver/java.go
index 11daba07f..87e2750fe 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -14,6 +14,7 @@ import (
 
 	"github.com/hashicorp/go-plugin"
 	"github.com/mitchellh/mapstructure"
+	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
 
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/driver/executor"
@@ -43,6 +44,7 @@ type javaHandle struct {
 	pluginClient *plugin.Client
 	userPid      int
 	executor     executor.Executor
+	groups       *cgroupConfig.Cgroup
 
 	killTimeout time.Duration
 	logger      *log.Logger
@@ -178,6 +180,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		pluginClient: pluginClient,
 		executor:     exec,
 		userPid:      ps.Pid,
+		groups:       &ps.IsolationConfig,
 		killTimeout:  d.DriverContext.KillTimeout(task),
 		logger:       d.logger,
 		doneCh:       make(chan struct{}),
@@ -191,6 +194,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 type javaId struct {
 	KillTimeout  time.Duration
 	PluginConfig *ExecutorReattachConfig
+	Groups       *cgroupConfig.Cgroup
 	UserPid      int
 }
 
@@ -209,6 +213,10 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
 			d.logger.Printf("[ERROR] driver.java: error destroying plugin and userpid: %v", e)
 		}
+		if e := destroyCgroup(id.Groups); e != nil {
+			d.logger.Printf("[ERROR] driver.exec: %v", e)
+		}
+
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
 
@@ -217,6 +225,7 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		pluginClient: pluginClient,
 		executor:     executor,
 		userPid:      id.UserPid,
+		groups:       id.Groups,
 		logger:       d.logger,
 		killTimeout:  id.KillTimeout,
 		doneCh:       make(chan struct{}),
@@ -232,6 +241,7 @@ func (h *javaHandle) ID() string {
 		KillTimeout:  h.killTimeout,
 		PluginConfig: NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
 		UserPid:      h.userPid,
+		Groups:       h.groups,
 	}
 
 	data, err := json.Marshal(id)
diff --git a/client/driver/utils_linux.go b/client/driver/utils_linux.go
new file mode 100644
index 000000000..eb7022cd0
--- /dev/null
+++ b/client/driver/utils_linux.go
@@ -0,0 +1,40 @@
+package driver
+
+import (
+	"os/exec"
+	"syscall"
+
+	"fmt"
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
+	"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
+	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
+)
+
+// isolateCommand sets the setsid flag in exec.Cmd to true so that the process
+// becomes the process leader in a new session and doesn't receive signals that
+// are sent to the parent process.
+func isolateCommand(cmd *exec.Cmd) {
+	if cmd.SysProcAttr == nil {
+		cmd.SysProcAttr = &syscall.SysProcAttr{}
+	}
+	cmd.SysProcAttr.Setsid = true
+}
+
+// destroyCgroup destroys a cgroup and thereby killing all the processes in that
+// group
+func destroyCgroup(group *cgroupConfig.Cgroup) error {
+	if group == nil {
+		return nil
+	}
+	var manager cgroups.Manager
+	manager = &cgroupFs.Manager{Cgroups: group}
+	if systemd.UseSystemd() {
+		manager = &systemd.Manager{Cgroups: group}
+	}
+
+	if err := manager.Destroy(); err != nil {
+		return fmt.Errorf("failed to destroy cgroup: %v", err)
+	}
+	return nil
+}
diff --git a/client/driver/utils_posix.go b/client/driver/utils_posix.go
index 693bba6f6..cf90d109d 100644
--- a/client/driver/utils_posix.go
+++ b/client/driver/utils_posix.go
@@ -1,8 +1,12 @@
+// +build !linux
+
 package driver
 
 import (
 	"os/exec"
 	"syscall"
+
+	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
 )
 
 // isolateCommand sets the setsid flag in exec.Cmd to true so that the process
@@ -14,3 +18,7 @@ func isolateCommand(cmd *exec.Cmd) {
 	}
 	cmd.SysProcAttr.Setsid = true
 }
+
+func destroyCgroup(group *cgroupConfig.Cgroup) error {
+	return nil
+}
diff --git a/client/driver/utils_windows.go b/client/driver/utils_windows.go
index 5b2b7d842..84aff1e5f 100644
--- a/client/driver/utils_windows.go
+++ b/client/driver/utils_windows.go
@@ -2,8 +2,14 @@ package driver
 
 import (
 	"os/exec"
+
+	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
 )
 
 // TODO Figure out if this is needed in Wondows
 func isolateCommand(cmd *exec.Cmd) {
 }
+
+func destroyCgroup(group *cgroupConfig.Cgroup) error {
+	return nil
+}

From 9b2e117f684bd5620a4fb125eea6a714dced7970 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 8 Feb 2016 11:56:48 -0800
Subject: [PATCH 50/68] Making the plugin server return the same instance of
 executor rpc server

---
 client/driver/executor/executor.go | 12 +++++++-----
 client/driver/executor_plugin.go   |  6 +++++-
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go
index 99d1c6fe2..65501c0db 100644
--- a/client/driver/executor/executor.go
+++ b/client/driver/executor/executor.go
@@ -192,10 +192,11 @@ func (e *UniversalExecutor) Exit() error {
 	if e.cmd.Process != nil {
 		proc, err := os.FindProcess(e.cmd.Process.Pid)
 		if err != nil {
-			e.logger.Printf("[ERROR] can't find process with pid: %v, err: %v", e.cmd.Process.Pid, err)
-		}
-		if err := proc.Kill(); err != nil {
-			e.logger.Printf("[ERROR] can't kill process with pid: %v, err: %v", e.cmd.Process.Pid, err)
+			e.logger.Printf("[ERROR] executor: can't find process with pid: %v, err: %v",
+				e.cmd.Process.Pid, err)
+		} else if err := proc.Kill(); err != nil {
+			merr.Errors = append(merr.Errors,
+				fmt.Errorf("can't kill process with pid: %v, err: %v", e.cmd.Process.Pid, err))
 		}
 	}
 
@@ -230,11 +231,12 @@ func (e *UniversalExecutor) ShutDown() error {
 	return nil
 }
 
+// configureTaskDir sets the task dir in the executor
 func (e *UniversalExecutor) configureTaskDir() error {
 	taskDir, ok := e.ctx.AllocDir.TaskDirs[e.ctx.TaskName]
 	e.taskDir = taskDir
 	if !ok {
-		return fmt.Errorf("Couldn't find task directory for task %v", e.ctx.TaskName)
+		return fmt.Errorf("couldn't find task directory for task %v", e.ctx.TaskName)
 	}
 	e.cmd.Dir = taskDir
 	return nil
diff --git a/client/driver/executor_plugin.go b/client/driver/executor_plugin.go
index 57a95e326..1be91a3ce 100644
--- a/client/driver/executor_plugin.go
+++ b/client/driver/executor_plugin.go
@@ -106,10 +106,14 @@ func (e *ExecutorRPCServer) Exit(args interface{}, resp *interface{}) error {
 
 type ExecutorPlugin struct {
 	logger *log.Logger
+	Impl   *ExecutorRPCServer
 }
 
 func (p *ExecutorPlugin) Server(*plugin.MuxBroker) (interface{}, error) {
-	return &ExecutorRPCServer{Impl: executor.NewExecutor(p.logger)}, nil
+	if p.Impl == nil {
+		p.Impl = &ExecutorRPCServer{Impl: executor.NewExecutor(p.logger)}
+	}
+	return p.Impl, nil
 }
 
 func (p *ExecutorPlugin) Client(b *plugin.MuxBroker, c *rpc.Client) (interface{}, error) {

From 70f7b2d70f0fa5af45688a54394a05cad25e9372 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 8 Feb 2016 13:29:53 -0800
Subject: [PATCH 51/68] Setting defaults for client max and min port

---
 client/config/config.go   | 10 ++++++----
 client/driver/exec.go     |  4 ++--
 client/driver/java.go     |  4 ++--
 client/driver/qemu.go     |  4 ++--
 client/driver/raw_exec.go |  4 ++--
 client/driver/utils.go    |  5 ++++-
 command/agent/agent.go    | 12 ++++--------
 command/agent/config.go   | 16 ++++++++++++++++
 8 files changed, 38 insertions(+), 21 deletions(-)

diff --git a/client/config/config.go b/client/config/config.go
index 0d59a1b9a..dd2c2b967 100644
--- a/client/config/config.go
+++ b/client/config/config.go
@@ -57,11 +57,13 @@ type Config struct {
 	// Node provides the base node
 	Node *structs.Node
 
-	// ExecutorMaxPort defines the highest port a plugin process can use
-	ExecutorMaxPort int
+	// ClientMaxPort is the upper range of the ports that the client uses for
+	// communicating with plugin subsystems
+	ClientMaxPort uint
 
-	// ExecutorMinPort defines the lowest port a plugin process can use
-	ExecutorMinPort int
+	// ClientMinPort is the lower range of the ports that the client uses for
+	// communicating with plugin subsystems
+	ClientMinPort uint
 
 	// Options provides arbitrary key-value configuration for nomad internals,
 	// like fingerprinters and drivers. The format is:
diff --git a/client/driver/exec.go b/client/driver/exec.go
index 6936461b6..4c6d0406f 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -110,7 +110,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		Cmd: exec.Command(bin, "executor", pluginLogFile),
 	}
 
-	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
+	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
 	if err != nil {
 		return nil, err
 	}
@@ -161,7 +161,7 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	pluginConfig := &plugin.ClientConfig{
 		Reattach: id.PluginConfig.PluginConfig(),
 	}
-	executor, client, err := createExecutor(pluginConfig, d.config.LogOutput)
+	executor, client, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
 	if err != nil {
 		d.logger.Println("[ERROR] driver.exec: error connecting to plugin so destroying plugin pid and user pid")
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
diff --git a/client/driver/java.go b/client/driver/java.go
index 87e2750fe..d207e89ba 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -155,7 +155,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		Cmd: exec.Command(bin, "executor", pluginLogFile),
 	}
 
-	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
+	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
 	if err != nil {
 		return nil, err
 	}
@@ -207,7 +207,7 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	pluginConfig := &plugin.ClientConfig{
 		Reattach: id.PluginConfig.PluginConfig(),
 	}
-	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
+	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
 	if err != nil {
 		d.logger.Println("[ERROR] driver.java: error connecting to plugin so destroying plugin pid and user pid")
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
diff --git a/client/driver/qemu.go b/client/driver/qemu.go
index 849e0cd35..56f6d03f7 100644
--- a/client/driver/qemu.go
+++ b/client/driver/qemu.go
@@ -197,7 +197,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		Cmd: exec.Command(bin, "executor", pluginLogFile),
 	}
 
-	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
+	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
 	if err != nil {
 		return nil, err
 	}
@@ -245,7 +245,7 @@ func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		Reattach: id.PluginConfig.PluginConfig(),
 	}
 
-	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
+	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
 	if err != nil {
 		d.logger.Println("[ERROR] driver.qemu: error connecting to plugin so destroying plugin pid and user pid")
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go
index 2dd82cc6b..19c453a0f 100644
--- a/client/driver/raw_exec.go
+++ b/client/driver/raw_exec.go
@@ -103,7 +103,7 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 		Cmd: exec.Command(bin, "executor", pluginLogFile),
 	}
 
-	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
+	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
 	if err != nil {
 		return nil, err
 	}
@@ -149,7 +149,7 @@ func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, e
 	pluginConfig := &plugin.ClientConfig{
 		Reattach: id.PluginConfig.PluginConfig(),
 	}
-	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput)
+	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
 	if err != nil {
 		d.logger.Println("[ERROR] driver.raw_exec: error connecting to plugin so destroying plugin pid and user pid")
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
diff --git a/client/driver/utils.go b/client/driver/utils.go
index f2f8fc02a..8bcbba23a 100644
--- a/client/driver/utils.go
+++ b/client/driver/utils.go
@@ -7,14 +7,17 @@ import (
 
 	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/go-plugin"
+	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/driver/executor"
 )
 
 // createExecutor launches an executor plugin and returns an instance of the
 // Executor interface
-func createExecutor(config *plugin.ClientConfig, w io.Writer) (executor.Executor, *plugin.Client, error) {
+func createExecutor(config *plugin.ClientConfig, w io.Writer, clientConfig *config.Config) (executor.Executor, *plugin.Client, error) {
 	config.HandshakeConfig = HandshakeConfig
 	config.Plugins = GetPluginMap(w)
+	config.MaxPort = clientConfig.ClientMaxPort
+	config.MinPort = clientConfig.ClientMinPort
 	if config.Cmd != nil {
 		isolateCommand(config.Cmd)
 	}
diff --git a/command/agent/agent.go b/command/agent/agent.go
index 28e5e6b60..b9d996377 100644
--- a/command/agent/agent.go
+++ b/command/agent/agent.go
@@ -216,6 +216,8 @@ func (a *Agent) setupClient() error {
 		}
 		conf.MaxKillTimeout = dur
 	}
+	conf.ClientMaxPort = a.config.ClientConfig.ClientMaxPort
+	conf.ClientMinPort = a.config.ClientConfig.ClientMinPort
 
 	// Setup the node
 	conf.Node = new(structs.Node)
@@ -242,12 +244,6 @@ func (a *Agent) setupClient() error {
 	// Reserve some ports for the plugins
 	if runtime.GOOS == "windows" {
 		deviceName, err := a.findLoopbackDevice()
-		if conf.ExecutorMaxPort == 0 {
-			conf.ExecutorMaxPort = 15000
-		}
-		if conf.ExecutorMinPort == 0 {
-			conf.ExecutorMinPort = 14000
-		}
 		if err != nil {
 			return fmt.Errorf("error finding the device name for the ip 127.0.0.1: %v", err)
 		}
@@ -263,8 +259,8 @@ func (a *Agent) setupClient() error {
 				ReservedPorts: make([]structs.Port, 0),
 			}
 		}
-		for i := conf.ExecutorMinPort; i <= conf.ExecutorMaxPort; i++ {
-			nr.ReservedPorts = append(nr.ReservedPorts, structs.Port{Label: fmt.Sprintf("plugin-%d", i), Value: i})
+		for i := conf.ClientMinPort; i <= conf.ClientMaxPort; i++ {
+			nr.ReservedPorts = append(nr.ReservedPorts, structs.Port{Label: fmt.Sprintf("plugin-%d", i), Value: int(i)})
 		}
 
 	}
diff --git a/command/agent/config.go b/command/agent/config.go
index 6d9219f96..fc1e79d90 100644
--- a/command/agent/config.go
+++ b/command/agent/config.go
@@ -159,6 +159,14 @@ type ClientConfig struct {
 
 	// MaxKillTimeout allows capping the user-specifiable KillTimeout.
 	MaxKillTimeout string `hcl:"max_kill_timeout"`
+
+	// ClientMaxPort is the upper range of the ports that the client uses for
+	// communicating with plugin subsystems
+	ClientMaxPort uint `hcl:"client_max_port"`
+
+	// ClientMinPort is the lower range of the ports that the client uses for
+	// communicating with plugin subsystems
+	ClientMinPort uint `hcl:"client_min_port"`
 }
 
 // ServerConfig is configuration specific to the server mode
@@ -288,6 +296,8 @@ func DefaultConfig() *Config {
 			Enabled:        false,
 			NetworkSpeed:   100,
 			MaxKillTimeout: "30s",
+			ClientMinPort:  14000,
+			ClientMaxPort:  15000,
 		},
 		Server: &ServerConfig{
 			Enabled:          false,
@@ -505,6 +515,12 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig {
 	if b.MaxKillTimeout != "" {
 		result.MaxKillTimeout = b.MaxKillTimeout
 	}
+	if b.ClientMaxPort != 0 {
+		result.ClientMaxPort = b.ClientMaxPort
+	}
+	if b.ClientMinPort != 0 {
+		result.ClientMinPort = b.ClientMinPort
+	}
 
 	// Add the servers
 	result.Servers = append(result.Servers, b.Servers...)

From 41447a0d7c4b687b44f0c9347311e2f7c6f4a321 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 8 Feb 2016 13:48:26 -0800
Subject: [PATCH 52/68] Extracted the cgroup info into isolation struct

---
 client/driver/exec.go              | 72 +++++++++++++++---------------
 client/driver/executor/executor.go | 12 +++--
 client/driver/java.go              | 63 +++++++++++++-------------
 command/agent/agent.go             |  7 ++-
 4 files changed, 80 insertions(+), 74 deletions(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index 4c6d0406f..49444a65f 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -17,8 +17,6 @@ import (
 	"github.com/hashicorp/nomad/helper/discover"
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/mitchellh/mapstructure"
-
-	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
 )
 
 // ExecDriver fork/execs tasks using as many of the underlying OS's isolation
@@ -36,14 +34,14 @@ type ExecDriverConfig struct {
 
 // execHandle is returned from Start/Open as a handle to the PID
 type execHandle struct {
-	pluginClient *plugin.Client
-	executor     executor.Executor
-	groups       *cgroupConfig.Cgroup
-	userPid      int
-	killTimeout  time.Duration
-	logger       *log.Logger
-	waitCh       chan *cstructs.WaitResult
-	doneCh       chan struct{}
+	pluginClient    *plugin.Client
+	executor        executor.Executor
+	isolationConfig *executor.IsolationConfig
+	userPid         int
+	killTimeout     time.Duration
+	logger          *log.Logger
+	waitCh          chan *cstructs.WaitResult
+	doneCh          chan struct{}
 }
 
 // NewExecDriver is used to create a new exec driver
@@ -132,24 +130,24 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 
 	// Return a driver handle
 	h := &execHandle{
-		pluginClient: pluginClient,
-		userPid:      ps.Pid,
-		executor:     exec,
-		groups:       &ps.IsolationConfig,
-		killTimeout:  d.DriverContext.KillTimeout(task),
-		logger:       d.logger,
-		doneCh:       make(chan struct{}),
-		waitCh:       make(chan *cstructs.WaitResult, 1),
+		pluginClient:    pluginClient,
+		userPid:         ps.Pid,
+		executor:        exec,
+		isolationConfig: ps.IsolationConfig,
+		killTimeout:     d.DriverContext.KillTimeout(task),
+		logger:          d.logger,
+		doneCh:          make(chan struct{}),
+		waitCh:          make(chan *cstructs.WaitResult, 1),
 	}
 	go h.run()
 	return h, nil
 }
 
 type execId struct {
-	KillTimeout  time.Duration
-	UserPid      int
-	Groups       *cgroupConfig.Cgroup
-	PluginConfig *ExecutorReattachConfig
+	KillTimeout     time.Duration
+	UserPid         int
+	IsolationConfig *executor.IsolationConfig
+	PluginConfig    *ExecutorReattachConfig
 }
 
 func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
@@ -167,22 +165,24 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
 			d.logger.Printf("[ERROR] driver.exec: error destroying plugin and userpid: %v", e)
 		}
-		if e := destroyCgroup(id.Groups); e != nil {
-			d.logger.Printf("[ERROR] driver.exec: %v", e)
+		if id.IsolationConfig != nil {
+			if e := destroyCgroup(id.IsolationConfig.Cgroup); e != nil {
+				d.logger.Printf("[ERROR] driver.exec: %v", e)
+			}
 		}
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
 
 	// Return a driver handle
 	h := &execHandle{
-		pluginClient: client,
-		executor:     executor,
-		userPid:      id.UserPid,
-		groups:       id.Groups,
-		logger:       d.logger,
-		killTimeout:  id.KillTimeout,
-		doneCh:       make(chan struct{}),
-		waitCh:       make(chan *cstructs.WaitResult, 1),
+		pluginClient:    client,
+		executor:        executor,
+		userPid:         id.UserPid,
+		isolationConfig: id.IsolationConfig,
+		logger:          d.logger,
+		killTimeout:     id.KillTimeout,
+		doneCh:          make(chan struct{}),
+		waitCh:          make(chan *cstructs.WaitResult, 1),
 	}
 	go h.run()
 	return h, nil
@@ -190,10 +190,10 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 
 func (h *execHandle) ID() string {
 	id := execId{
-		KillTimeout:  h.killTimeout,
-		PluginConfig: NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
-		UserPid:      h.userPid,
-		Groups:       h.groups,
+		KillTimeout:     h.killTimeout,
+		PluginConfig:    NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
+		UserPid:         h.userPid,
+		IsolationConfig: h.isolationConfig,
 	}
 
 	data, err := json.Marshal(id)
diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go
index 65501c0db..572042d48 100644
--- a/client/driver/executor/executor.go
+++ b/client/driver/executor/executor.go
@@ -57,12 +57,18 @@ type ExecCommand struct {
 	Args []string
 }
 
+// IsolationConfig has information about the isolation mechanism the executor
+// uses to put resource constraints and isolation on the user process
+type IsolationConfig struct {
+	Cgroup *cgroupConfig.Cgroup
+}
+
 // ProcessState holds information about the state of a user process.
 type ProcessState struct {
 	Pid             int
 	ExitCode        int
 	Signal          int
-	IsolationConfig cgroupConfig.Cgroup
+	IsolationConfig *IsolationConfig
 	Time            time.Time
 }
 
@@ -152,9 +158,9 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext
 	if err := e.cmd.Start(); err != nil {
 		return nil, fmt.Errorf("error starting command: %v", err)
 	}
-
 	go e.wait()
-	return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, IsolationConfig: *e.groups, Time: time.Now()}, nil
+	ic := &IsolationConfig{Cgroup: e.groups}
+	return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, IsolationConfig: ic, Time: time.Now()}, nil
 }
 
 // Wait waits until a process has exited and returns it's exitcode and errors
diff --git a/client/driver/java.go b/client/driver/java.go
index d207e89ba..aae80fb2d 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -14,7 +14,6 @@ import (
 
 	"github.com/hashicorp/go-plugin"
 	"github.com/mitchellh/mapstructure"
-	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
 
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/driver/executor"
@@ -41,10 +40,10 @@ type JavaDriverConfig struct {
 
 // javaHandle is returned from Start/Open as a handle to the PID
 type javaHandle struct {
-	pluginClient *plugin.Client
-	userPid      int
-	executor     executor.Executor
-	groups       *cgroupConfig.Cgroup
+	pluginClient    *plugin.Client
+	userPid         int
+	executor        executor.Executor
+	isolationConfig *executor.IsolationConfig
 
 	killTimeout time.Duration
 	logger      *log.Logger
@@ -177,14 +176,14 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 
 	// Return a driver handle
 	h := &javaHandle{
-		pluginClient: pluginClient,
-		executor:     exec,
-		userPid:      ps.Pid,
-		groups:       &ps.IsolationConfig,
-		killTimeout:  d.DriverContext.KillTimeout(task),
-		logger:       d.logger,
-		doneCh:       make(chan struct{}),
-		waitCh:       make(chan *cstructs.WaitResult, 1),
+		pluginClient:    pluginClient,
+		executor:        exec,
+		userPid:         ps.Pid,
+		isolationConfig: ps.IsolationConfig,
+		killTimeout:     d.DriverContext.KillTimeout(task),
+		logger:          d.logger,
+		doneCh:          make(chan struct{}),
+		waitCh:          make(chan *cstructs.WaitResult, 1),
 	}
 
 	go h.run()
@@ -192,10 +191,10 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 }
 
 type javaId struct {
-	KillTimeout  time.Duration
-	PluginConfig *ExecutorReattachConfig
-	Groups       *cgroupConfig.Cgroup
-	UserPid      int
+	KillTimeout     time.Duration
+	PluginConfig    *ExecutorReattachConfig
+	IsolationConfig *executor.IsolationConfig
+	UserPid         int
 }
 
 func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
@@ -213,8 +212,10 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
 			d.logger.Printf("[ERROR] driver.java: error destroying plugin and userpid: %v", e)
 		}
-		if e := destroyCgroup(id.Groups); e != nil {
-			d.logger.Printf("[ERROR] driver.exec: %v", e)
+		if id.IsolationConfig != nil {
+			if e := destroyCgroup(id.IsolationConfig.Cgroup); e != nil {
+				d.logger.Printf("[ERROR] driver.exec: %v", e)
+			}
 		}
 
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
@@ -222,14 +223,14 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 
 	// Return a driver handle
 	h := &javaHandle{
-		pluginClient: pluginClient,
-		executor:     executor,
-		userPid:      id.UserPid,
-		groups:       id.Groups,
-		logger:       d.logger,
-		killTimeout:  id.KillTimeout,
-		doneCh:       make(chan struct{}),
-		waitCh:       make(chan *cstructs.WaitResult, 1),
+		pluginClient:    pluginClient,
+		executor:        executor,
+		userPid:         id.UserPid,
+		isolationConfig: id.IsolationConfig,
+		logger:          d.logger,
+		killTimeout:     id.KillTimeout,
+		doneCh:          make(chan struct{}),
+		waitCh:          make(chan *cstructs.WaitResult, 1),
 	}
 
 	go h.run()
@@ -238,10 +239,10 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 
 func (h *javaHandle) ID() string {
 	id := javaId{
-		KillTimeout:  h.killTimeout,
-		PluginConfig: NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
-		UserPid:      h.userPid,
-		Groups:       h.groups,
+		KillTimeout:     h.killTimeout,
+		PluginConfig:    NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
+		UserPid:         h.userPid,
+		IsolationConfig: h.isolationConfig,
 	}
 
 	data, err := json.Marshal(id)
diff --git a/command/agent/agent.go b/command/agent/agent.go
index b9d996377..6f0f1e8a3 100644
--- a/command/agent/agent.go
+++ b/command/agent/agent.go
@@ -216,8 +216,8 @@ func (a *Agent) setupClient() error {
 		}
 		conf.MaxKillTimeout = dur
 	}
-	conf.ClientMaxPort = a.config.ClientConfig.ClientMaxPort
-	conf.ClientMinPort = a.config.ClientConfig.ClientMinPort
+	conf.ClientMaxPort = a.config.Client.ClientMaxPort
+	conf.ClientMinPort = a.config.Client.ClientMinPort
 
 	// Setup the node
 	conf.Node = new(structs.Node)
@@ -245,7 +245,7 @@ func (a *Agent) setupClient() error {
 	if runtime.GOOS == "windows" {
 		deviceName, err := a.findLoopbackDevice()
 		if err != nil {
-			return fmt.Errorf("error finding the device name for the ip 127.0.0.1: %v", err)
+			return fmt.Errorf("error finding the device name for loopback: %v", err)
 		}
 		var nr *structs.NetworkResource
 		for _, n := range conf.Node.Reserved.Networks {
@@ -262,7 +262,6 @@ func (a *Agent) setupClient() error {
 		for i := conf.ClientMinPort; i <= conf.ClientMaxPort; i++ {
 			nr.ReservedPorts = append(nr.ReservedPorts, structs.Port{Label: fmt.Sprintf("plugin-%d", i), Value: int(i)})
 		}
-
 	}
 
 	// Create the client

From 747d3202caa6baf5e1b1290ec54db179dddfff44 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 8 Feb 2016 14:11:53 -0800
Subject: [PATCH 53/68] Moving code to mount and unmount chroot into allocdir

---
 client/allocdir/alloc_dir_darwin.go      | 11 ++++
 client/allocdir/alloc_dir_linux.go       | 73 ++++++++++++++++++++++++
 client/allocdir/alloc_dir_windows.go     | 11 ++++
 client/driver/executor/executor_linux.go | 56 ++----------------
 command/agent/config.go                  |  2 +-
 5 files changed, 101 insertions(+), 52 deletions(-)

diff --git a/client/allocdir/alloc_dir_darwin.go b/client/allocdir/alloc_dir_darwin.go
index 9c247d15d..723112857 100644
--- a/client/allocdir/alloc_dir_darwin.go
+++ b/client/allocdir/alloc_dir_darwin.go
@@ -13,3 +13,14 @@ func (d *AllocDir) mountSharedDir(dir string) error {
 func (d *AllocDir) unmountSharedDir(dir string) error {
 	return syscall.Unlink(dir)
 }
+
+// MountSpecialDirs mounts the dev and proc file system on the chroot of the
+// task. It's a no-op on darwin.
+func (d *AllocDir) MountSpecialDirs(taskDir string) error {
+	return nil
+}
+
+// UnmountSpecialDirs unmounts the dev and proc file system from the chroot
+func (d *AllocDir) UnmountSpecialDirs(taskDir string) error {
+	return nil
+}
diff --git a/client/allocdir/alloc_dir_linux.go b/client/allocdir/alloc_dir_linux.go
index 89d5df6a4..212ea61b6 100644
--- a/client/allocdir/alloc_dir_linux.go
+++ b/client/allocdir/alloc_dir_linux.go
@@ -1,8 +1,12 @@
 package allocdir
 
 import (
+	"fmt"
 	"os"
+	"path/filepath"
 	"syscall"
+
+	"github.com/hashicorp/go-multierror"
 )
 
 // Bind mounts the shared directory into the task directory. Must be root to
@@ -18,3 +22,72 @@ func (d *AllocDir) mountSharedDir(taskDir string) error {
 func (d *AllocDir) unmountSharedDir(dir string) error {
 	return syscall.Unmount(dir, 0)
 }
+
+// MountSpecialDirs mounts the dev and proc file system from the host to the
+// chroot
+func (d *AllocDir) MountSpecialDirs(taskDir string) error {
+	// Mount dev
+	dev := filepath.Join(taskDir, "dev")
+	if !d.pathExists(dev) {
+		if err := os.Mkdir(dev, 0777); err != nil {
+			return fmt.Errorf("Mkdir(%v) failed: %v", dev, err)
+		}
+
+		if err := syscall.Mount("none", dev, "devtmpfs", syscall.MS_RDONLY, ""); err != nil {
+			return fmt.Errorf("Couldn't mount /dev to %v: %v", dev, err)
+		}
+	}
+
+	// Mount proc
+	proc := filepath.Join(taskDir, "proc")
+	if !d.pathExists(proc) {
+		if err := os.Mkdir(proc, 0777); err != nil {
+			return fmt.Errorf("Mkdir(%v) failed: %v", proc, err)
+		}
+
+		if err := syscall.Mount("none", proc, "proc", syscall.MS_RDONLY, ""); err != nil {
+			return fmt.Errorf("Couldn't mount /proc to %v: %v", proc, err)
+		}
+	}
+
+	return nil
+}
+
+// UnmountSpecialDirs unmounts the dev and proc file system from the chroot
+func (d *AllocDir) UnmountSpecialDirs(taskDir string) error {
+	errs := new(multierror.Error)
+	dev := filepath.Join(taskDir, "dev")
+	if d.pathExists(dev) {
+		if err := syscall.Unmount(dev, 0); err != nil {
+			errs = multierror.Append(errs, fmt.Errorf("Failed to unmount dev (%v): %v", dev, err))
+		}
+
+		if err := os.RemoveAll(dev); err != nil {
+			errs = multierror.Append(errs, fmt.Errorf("Failed to delete dev directory (%v): %v", dev, err))
+		}
+	}
+
+	// Unmount proc.
+	proc := filepath.Join(taskDir, "proc")
+	if d.pathExists(proc) {
+		if err := syscall.Unmount(proc, 0); err != nil {
+			errs = multierror.Append(errs, fmt.Errorf("Failed to unmount proc (%v): %v", proc, err))
+		}
+
+		if err := os.RemoveAll(proc); err != nil {
+			errs = multierror.Append(errs, fmt.Errorf("Failed to delete proc directory (%v): %v", dev, err))
+		}
+	}
+
+	return errs.ErrorOrNil()
+}
+
+// pathExists is a helper function to check if the path exists.
+func (d *AllocDir) pathExists(path string) bool {
+	if _, err := os.Stat(path); err != nil {
+		if os.IsNotExist(err) {
+			return false
+		}
+	}
+	return true
+}
diff --git a/client/allocdir/alloc_dir_windows.go b/client/allocdir/alloc_dir_windows.go
index ab0692b5b..31a4ebb8e 100644
--- a/client/allocdir/alloc_dir_windows.go
+++ b/client/allocdir/alloc_dir_windows.go
@@ -23,3 +23,14 @@ func (d *AllocDir) dropDirPermissions(path string) error {
 func (d *AllocDir) unmountSharedDir(dir string) error {
 	return nil
 }
+
+// MountSpecialDirs mounts the dev and proc file system on the chroot of the
+// task. It's a no-op on windows.
+func (d *AllocDir) MountSpecialDirs(taskDir string) error {
+	return nil
+}
+
+// UnmountSpecialDirs unmounts the dev and proc file system from the chroot
+func (d *AllocDir) UnmountSpecialDirs(taskDir string) error {
+	return nil
+}
diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go
index c95db04d0..742522bbe 100644
--- a/client/driver/executor/executor_linux.go
+++ b/client/driver/executor/executor_linux.go
@@ -165,40 +165,19 @@ func (e *UniversalExecutor) configureChroot() error {
 		return err
 	}
 
-	// Mount dev
-	dev := filepath.Join(e.taskDir, "dev")
-	if !e.pathExists(dev) {
-		if err := os.Mkdir(dev, 0777); err != nil {
-			return fmt.Errorf("Mkdir(%v) failed: %v", dev, err)
-		}
-
-		if err := syscall.Mount("none", dev, "devtmpfs", syscall.MS_RDONLY, ""); err != nil {
-			return fmt.Errorf("Couldn't mount /dev to %v: %v", dev, err)
-		}
-	}
-
-	// Mount proc
-	proc := filepath.Join(e.taskDir, "proc")
-	if !e.pathExists(proc) {
-		if err := os.Mkdir(proc, 0777); err != nil {
-			return fmt.Errorf("Mkdir(%v) failed: %v", proc, err)
-		}
-
-		if err := syscall.Mount("none", proc, "proc", syscall.MS_RDONLY, ""); err != nil {
-			return fmt.Errorf("Couldn't mount /proc to %v: %v", proc, err)
-		}
-	}
-
 	// Set the tasks AllocDir environment variable.
 	e.ctx.TaskEnv.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)).SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)).Build()
 
 	if e.cmd.SysProcAttr == nil {
 		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
 	}
-
 	e.cmd.SysProcAttr.Chroot = e.taskDir
 	e.cmd.Dir = "/"
 
+	if err := allocDir.MountSpecialDirs(e.taskDir); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -209,32 +188,7 @@ func (e *UniversalExecutor) removeChrootMounts() error {
 	e.lock.Lock()
 	defer e.lock.Unlock()
 
-	// Unmount dev.
-	errs := new(multierror.Error)
-	dev := filepath.Join(e.taskDir, "dev")
-	if e.pathExists(dev) {
-		if err := syscall.Unmount(dev, 0); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("Failed to unmount dev (%v): %v", dev, err))
-		}
-
-		if err := os.RemoveAll(dev); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("Failed to delete dev directory (%v): %v", dev, err))
-		}
-	}
-
-	// Unmount proc.
-	proc := filepath.Join(e.taskDir, "proc")
-	if e.pathExists(proc) {
-		if err := syscall.Unmount(proc, 0); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("Failed to unmount proc (%v): %v", proc, err))
-		}
-
-		if err := os.RemoveAll(proc); err != nil {
-			errs = multierror.Append(errs, fmt.Errorf("Failed to delete proc directory (%v): %v", dev, err))
-		}
-	}
-
-	return errs.ErrorOrNil()
+	return e.ctx.AllocDir.UnmountSpecialDirs(e.taskDir)
 }
 
 // destroyCgroup kills all processes in the cgroup and removes the cgroup
diff --git a/command/agent/config.go b/command/agent/config.go
index fc1e79d90..89abd6300 100644
--- a/command/agent/config.go
+++ b/command/agent/config.go
@@ -297,7 +297,7 @@ func DefaultConfig() *Config {
 			NetworkSpeed:   100,
 			MaxKillTimeout: "30s",
 			ClientMinPort:  14000,
-			ClientMaxPort:  15000,
+			ClientMaxPort:  19000,
 		},
 		Server: &ServerConfig{
 			Enabled:          false,

From 25d456a8ce0d9353925662444ddbe4d7177455aa Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 8 Feb 2016 14:31:03 -0800
Subject: [PATCH 54/68] Unmounting dev and proc when open can't reconnect with
 the plugin process

---
 client/driver/exec.go | 8 ++++++++
 client/driver/java.go | 8 ++++++++
 2 files changed, 16 insertions(+)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index 49444a65f..430b387ca 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -38,6 +38,7 @@ type execHandle struct {
 	executor        executor.Executor
 	isolationConfig *executor.IsolationConfig
 	userPid         int
+	taskDir         string
 	killTimeout     time.Duration
 	logger          *log.Logger
 	waitCh          chan *cstructs.WaitResult
@@ -133,6 +134,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		pluginClient:    pluginClient,
 		userPid:         ps.Pid,
 		executor:        exec,
+		taskDir:         taskDir,
 		isolationConfig: ps.IsolationConfig,
 		killTimeout:     d.DriverContext.KillTimeout(task),
 		logger:          d.logger,
@@ -146,6 +148,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 type execId struct {
 	KillTimeout     time.Duration
 	UserPid         int
+	TaskDir         string
 	IsolationConfig *executor.IsolationConfig
 	PluginConfig    *ExecutorReattachConfig
 }
@@ -170,6 +173,9 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 				d.logger.Printf("[ERROR] driver.exec: %v", e)
 			}
 		}
+		if e := ctx.AllocDir.UnmountSpecialDirs(id.TaskDir); e != nil {
+			d.logger.Printf("[ERROR] driver.exec: error unmounting dev and proc fs: %v", e)
+		}
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
 
@@ -178,6 +184,7 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		pluginClient:    client,
 		executor:        executor,
 		userPid:         id.UserPid,
+		taskDir:         id.TaskDir,
 		isolationConfig: id.IsolationConfig,
 		logger:          d.logger,
 		killTimeout:     id.KillTimeout,
@@ -193,6 +200,7 @@ func (h *execHandle) ID() string {
 		KillTimeout:     h.killTimeout,
 		PluginConfig:    NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
 		UserPid:         h.userPid,
+		TaskDir:         h.taskDir,
 		IsolationConfig: h.isolationConfig,
 	}
 
diff --git a/client/driver/java.go b/client/driver/java.go
index aae80fb2d..3109b42ff 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -45,6 +45,7 @@ type javaHandle struct {
 	executor        executor.Executor
 	isolationConfig *executor.IsolationConfig
 
+	taskDir     string
 	killTimeout time.Duration
 	logger      *log.Logger
 	waitCh      chan *cstructs.WaitResult
@@ -180,6 +181,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		executor:        exec,
 		userPid:         ps.Pid,
 		isolationConfig: ps.IsolationConfig,
+		taskDir:         taskDir,
 		killTimeout:     d.DriverContext.KillTimeout(task),
 		logger:          d.logger,
 		doneCh:          make(chan struct{}),
@@ -194,6 +196,7 @@ type javaId struct {
 	KillTimeout     time.Duration
 	PluginConfig    *ExecutorReattachConfig
 	IsolationConfig *executor.IsolationConfig
+	TaskDir         string
 	UserPid         int
 }
 
@@ -217,6 +220,9 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 				d.logger.Printf("[ERROR] driver.exec: %v", e)
 			}
 		}
+		if e := ctx.AllocDir.UnmountSpecialDirs(id.TaskDir); e != nil {
+			d.logger.Printf("[ERROR] driver.exec: error unmounting dev and proc fs: %v", e)
+		}
 
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
@@ -227,6 +233,7 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		executor:        executor,
 		userPid:         id.UserPid,
 		isolationConfig: id.IsolationConfig,
+		taskDir:         id.TaskDir,
 		logger:          d.logger,
 		killTimeout:     id.KillTimeout,
 		doneCh:          make(chan struct{}),
@@ -242,6 +249,7 @@ func (h *javaHandle) ID() string {
 		KillTimeout:     h.killTimeout,
 		PluginConfig:    NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
 		UserPid:         h.userPid,
+		TaskDir:         h.taskDir,
 		IsolationConfig: h.isolationConfig,
 	}
 

From de4417d8d83f4818858f7ce1736b7a97ea79d7d8 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 8 Feb 2016 14:40:54 -0800
Subject: [PATCH 55/68] Removing pathExists from executor

---
 client/driver/executor/executor_linux.go | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go
index 742522bbe..0729eb90d 100644
--- a/client/driver/executor/executor_linux.go
+++ b/client/driver/executor/executor_linux.go
@@ -144,16 +144,6 @@ func (e *UniversalExecutor) runAs(userid string) error {
 	return nil
 }
 
-// pathExists is a helper function to check if the path exists.
-func (e *UniversalExecutor) pathExists(path string) bool {
-	if _, err := os.Stat(path); err != nil {
-		if os.IsNotExist(err) {
-			return false
-		}
-	}
-	return true
-}
-
 // configureChroot configures a chroot
 func (e *UniversalExecutor) configureChroot() error {
 	allocDir := e.ctx.AllocDir

From fbc3279da93f5b7b6685fda2a9a25f9391250a75 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 8 Feb 2016 14:47:10 -0800
Subject: [PATCH 56/68] Appending names of sub-system before log lines

---
 client/driver/executor/executor_linux.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go
index 0729eb90d..630ea7a18 100644
--- a/client/driver/executor/executor_linux.go
+++ b/client/driver/executor/executor_linux.go
@@ -47,10 +47,10 @@ func (e *UniversalExecutor) configureIsolation() error {
 		}
 		if err := e.applyLimits(os.Getpid()); err != nil {
 			if er := e.destroyCgroup(); er != nil {
-				e.logger.Printf("[ERROR] error destroying cgroup: %v", er)
+				e.logger.Printf("[ERROR] executor: error destroying cgroup: %v", er)
 			}
 			if er := e.removeChrootMounts(); er != nil {
-				e.logger.Printf("[ERROR] error removing chroot: %v", er)
+				e.logger.Printf("[ERROR] executor: error removing chroot: %v", er)
 			}
 			return fmt.Errorf("error entering the plugin process in the cgroup: %v:", err)
 		}
@@ -67,9 +67,9 @@ func (e *UniversalExecutor) applyLimits(pid int) error {
 	// Entering the process in the cgroup
 	manager := e.getCgroupManager(e.groups)
 	if err := manager.Apply(pid); err != nil {
-		e.logger.Printf("[ERROR] unable to join cgroup: %v", err)
+		e.logger.Printf("[ERROR] executor: unable to join cgroup: %v", err)
 		if err := e.Exit(); err != nil {
-			e.logger.Printf("[ERROR] unable to kill process: %v", err)
+			e.logger.Printf("[ERROR] executor: unable to kill process: %v", err)
 		}
 		return err
 	}

From 7dd3eec68480ed9204c67b66077dd42de0ac8735 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 8 Feb 2016 15:07:18 -0800
Subject: [PATCH 57/68] removing the prefix of the logger

---
 client/driver/executor_plugin.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/driver/executor_plugin.go b/client/driver/executor_plugin.go
index 1be91a3ce..5d1af4f5e 100644
--- a/client/driver/executor_plugin.go
+++ b/client/driver/executor_plugin.go
@@ -18,7 +18,7 @@ var HandshakeConfig = plugin.HandshakeConfig{
 
 func GetPluginMap(w io.Writer) map[string]plugin.Plugin {
 	p := new(ExecutorPlugin)
-	p.logger = log.New(w, "executor-plugin-server:", log.LstdFlags)
+	p.logger = log.New(w, "", log.LstdFlags)
 	return map[string]plugin.Plugin{"executor": p}
 }
 

From cc6e2b73f0c0ddb35fa2d07f6e6d7d711ed94452 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 8 Feb 2016 16:08:29 -0800
Subject: [PATCH 58/68] Moved the destroycgroup method into executor

---
 client/driver/exec.go                    |  8 ++--
 client/driver/executor/executor.go       |  8 +++-
 client/driver/executor/executor_basic.go |  6 ++-
 client/driver/executor/executor_linux.go | 50 ++++++++++--------------
 client/driver/java.go                    |  8 ++--
 client/driver/utils_linux.go             | 24 ------------
 client/driver/utils_posix.go             |  6 ---
 client/driver/utils_windows.go           |  6 ---
 8 files changed, 39 insertions(+), 77 deletions(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index 430b387ca..fb09e1394 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -162,15 +162,15 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	pluginConfig := &plugin.ClientConfig{
 		Reattach: id.PluginConfig.PluginConfig(),
 	}
-	executor, client, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
+	exec, client, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
 	if err != nil {
 		d.logger.Println("[ERROR] driver.exec: error connecting to plugin so destroying plugin pid and user pid")
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
 			d.logger.Printf("[ERROR] driver.exec: error destroying plugin and userpid: %v", e)
 		}
 		if id.IsolationConfig != nil {
-			if e := destroyCgroup(id.IsolationConfig.Cgroup); e != nil {
-				d.logger.Printf("[ERROR] driver.exec: %v", e)
+			if e := executor.DestroyCgroup(id.IsolationConfig.Cgroup); e != nil {
+				d.logger.Printf("[ERROR] driver.exec: destroying cgroup failed: %v", e)
 			}
 		}
 		if e := ctx.AllocDir.UnmountSpecialDirs(id.TaskDir); e != nil {
@@ -182,7 +182,7 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	// Return a driver handle
 	h := &execHandle{
 		pluginClient:    client,
-		executor:        executor,
+		executor:        exec,
 		userPid:         id.UserPid,
 		taskDir:         id.TaskDir,
 		isolationConfig: id.IsolationConfig,
diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go
index 572042d48..4384e1ce4 100644
--- a/client/driver/executor/executor.go
+++ b/client/driver/executor/executor.go
@@ -186,7 +186,9 @@ func (e *UniversalExecutor) wait() {
 		e.removeChrootMounts()
 	}
 	if e.ctx.ResourceLimits {
-		e.destroyCgroup()
+		e.lock.Lock()
+		DestroyCgroup(e.groups)
+		e.lock.Unlock()
 	}
 	e.exitState = &ProcessState{Pid: 0, ExitCode: exitCode, Time: time.Now()}
 }
@@ -212,9 +214,11 @@ func (e *UniversalExecutor) Exit() error {
 		}
 	}
 	if e.ctx.ResourceLimits {
-		if err := e.destroyCgroup(); err != nil {
+		e.lock.Lock()
+		if err := DestroyCgroup(e.groups); err != nil {
 			merr.Errors = append(merr.Errors, err)
 		}
+		e.lock.Unlock()
 	}
 	return merr.ErrorOrNil()
 }
diff --git a/client/driver/executor/executor_basic.go b/client/driver/executor/executor_basic.go
index 84d080233..9e531a547 100644
--- a/client/driver/executor/executor_basic.go
+++ b/client/driver/executor/executor_basic.go
@@ -2,11 +2,15 @@
 
 package executor
 
+import (
+	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
+)
+
 func (e *UniversalExecutor) configureChroot() error {
 	return nil
 }
 
-func (e *UniversalExecutor) destroyCgroup() error {
+func DestroyCgroup(groups *cgroupConfig.Cgroup) error {
 	return nil
 }
 
diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go
index 630ea7a18..99780e1a0 100644
--- a/client/driver/executor/executor_linux.go
+++ b/client/driver/executor/executor_linux.go
@@ -46,7 +46,7 @@ func (e *UniversalExecutor) configureIsolation() error {
 			return fmt.Errorf("error creating cgroups: %v", err)
 		}
 		if err := e.applyLimits(os.Getpid()); err != nil {
-			if er := e.destroyCgroup(); er != nil {
+			if er := DestroyCgroup(e.groups); er != nil {
 				e.logger.Printf("[ERROR] executor: error destroying cgroup: %v", er)
 			}
 			if er := e.removeChrootMounts(); er != nil {
@@ -65,7 +65,7 @@ func (e *UniversalExecutor) applyLimits(pid int) error {
 	}
 
 	// Entering the process in the cgroup
-	manager := e.getCgroupManager(e.groups)
+	manager := getCgroupManager(e.groups)
 	if err := manager.Apply(pid); err != nil {
 		e.logger.Printf("[ERROR] executor: unable to join cgroup: %v", err)
 		if err := e.Exit(); err != nil {
@@ -183,49 +183,39 @@ func (e *UniversalExecutor) removeChrootMounts() error {
 
 // destroyCgroup kills all processes in the cgroup and removes the cgroup
 // configuration from the host.
-func (e *UniversalExecutor) destroyCgroup() error {
-	if e.groups == nil {
+func DestroyCgroup(groups *cgroupConfig.Cgroup) error {
+	merrs := new(multierror.Error)
+	if groups == nil {
 		return fmt.Errorf("Can't destroy: cgroup configuration empty")
 	}
 
-	// Prevent a race between Wait/ForceStop
-	e.lock.Lock()
-	defer e.lock.Unlock()
-
-	manager := e.getCgroupManager(e.groups)
-	pids, err := manager.GetPids()
-	if err != nil {
-		return fmt.Errorf("Failed to get pids in the cgroup %v: %v", e.groups.Name, err)
-	}
-
-	errs := new(multierror.Error)
-	for _, pid := range pids {
-		process, err := os.FindProcess(pid)
-		if err != nil {
-			multierror.Append(errs, fmt.Errorf("Failed to find Pid %v: %v", pid, err))
-			continue
-		}
-
-		if err := process.Kill(); err != nil && err.Error() != "os: process already finished" {
-			multierror.Append(errs, fmt.Errorf("Failed to kill Pid %v: %v", pid, err))
-			continue
+	manager := getCgroupManager(groups)
+	if pids, perr := manager.GetPids(); perr == nil {
+		for _, pid := range pids {
+			proc, err := os.FindProcess(pid)
+			if err != nil {
+				merrs.Errors = append(merrs.Errors, fmt.Errorf("error finding process %v: %v", pid, err))
+			} else {
+				if e := proc.Kill(); e != nil {
+					merrs.Errors = append(merrs.Errors, fmt.Errorf("error killing process %v: %v", pid, e))
+				}
+			}
 		}
 	}
 
 	// Remove the cgroup.
 	if err := manager.Destroy(); err != nil {
-		multierror.Append(errs, fmt.Errorf("Failed to delete the cgroup directories: %v", err))
+		multierror.Append(merrs, fmt.Errorf("Failed to delete the cgroup directories: %v", err))
 	}
 
-	if len(errs.Errors) != 0 {
-		return fmt.Errorf("Failed to destroy cgroup: %v", errs)
+	if len(merrs.Errors) != 0 {
+		return fmt.Errorf("errors while destroying cgroup: %v", merrs)
 	}
-
 	return nil
 }
 
 // getCgroupManager returns the correct libcontainer cgroup manager.
-func (e *UniversalExecutor) getCgroupManager(groups *cgroupConfig.Cgroup) cgroups.Manager {
+func getCgroupManager(groups *cgroupConfig.Cgroup) cgroups.Manager {
 	var manager cgroups.Manager
 	manager = &cgroupFs.Manager{Cgroups: groups}
 	if systemd.UseSystemd() {
diff --git a/client/driver/java.go b/client/driver/java.go
index 3109b42ff..ee33d8b88 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -209,15 +209,15 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	pluginConfig := &plugin.ClientConfig{
 		Reattach: id.PluginConfig.PluginConfig(),
 	}
-	executor, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
+	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
 	if err != nil {
 		d.logger.Println("[ERROR] driver.java: error connecting to plugin so destroying plugin pid and user pid")
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
 			d.logger.Printf("[ERROR] driver.java: error destroying plugin and userpid: %v", e)
 		}
 		if id.IsolationConfig != nil {
-			if e := destroyCgroup(id.IsolationConfig.Cgroup); e != nil {
-				d.logger.Printf("[ERROR] driver.exec: %v", e)
+			if e := executor.DestroyCgroup(id.IsolationConfig.Cgroup); e != nil {
+				d.logger.Printf("[ERROR] driver.exec: destroying cgroup failed: %v", e)
 			}
 		}
 		if e := ctx.AllocDir.UnmountSpecialDirs(id.TaskDir); e != nil {
@@ -230,7 +230,7 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	// Return a driver handle
 	h := &javaHandle{
 		pluginClient:    pluginClient,
-		executor:        executor,
+		executor:        exec,
 		userPid:         id.UserPid,
 		isolationConfig: id.IsolationConfig,
 		taskDir:         id.TaskDir,
diff --git a/client/driver/utils_linux.go b/client/driver/utils_linux.go
index eb7022cd0..693bba6f6 100644
--- a/client/driver/utils_linux.go
+++ b/client/driver/utils_linux.go
@@ -3,12 +3,6 @@ package driver
 import (
 	"os/exec"
 	"syscall"
-
-	"fmt"
-	"github.com/opencontainers/runc/libcontainer/cgroups"
-	cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
-	"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
-	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
 )
 
 // isolateCommand sets the setsid flag in exec.Cmd to true so that the process
@@ -20,21 +14,3 @@ func isolateCommand(cmd *exec.Cmd) {
 	}
 	cmd.SysProcAttr.Setsid = true
 }
-
-// destroyCgroup destroys a cgroup and thereby killing all the processes in that
-// group
-func destroyCgroup(group *cgroupConfig.Cgroup) error {
-	if group == nil {
-		return nil
-	}
-	var manager cgroups.Manager
-	manager = &cgroupFs.Manager{Cgroups: group}
-	if systemd.UseSystemd() {
-		manager = &systemd.Manager{Cgroups: group}
-	}
-
-	if err := manager.Destroy(); err != nil {
-		return fmt.Errorf("failed to destroy cgroup: %v", err)
-	}
-	return nil
-}
diff --git a/client/driver/utils_posix.go b/client/driver/utils_posix.go
index cf90d109d..fef4a002f 100644
--- a/client/driver/utils_posix.go
+++ b/client/driver/utils_posix.go
@@ -5,8 +5,6 @@ package driver
 import (
 	"os/exec"
 	"syscall"
-
-	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
 )
 
 // isolateCommand sets the setsid flag in exec.Cmd to true so that the process
@@ -18,7 +16,3 @@ func isolateCommand(cmd *exec.Cmd) {
 	}
 	cmd.SysProcAttr.Setsid = true
 }
-
-func destroyCgroup(group *cgroupConfig.Cgroup) error {
-	return nil
-}
diff --git a/client/driver/utils_windows.go b/client/driver/utils_windows.go
index 84aff1e5f..5b2b7d842 100644
--- a/client/driver/utils_windows.go
+++ b/client/driver/utils_windows.go
@@ -2,14 +2,8 @@ package driver
 
 import (
 	"os/exec"
-
-	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
 )
 
 // TODO Figure out if this is needed in Wondows
 func isolateCommand(cmd *exec.Cmd) {
 }
-
-func destroyCgroup(group *cgroupConfig.Cgroup) error {
-	return nil
-}

From 8b2f94ff6f0ab1446ad3447879b6a1e5322085fb Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 8 Feb 2016 16:27:31 -0800
Subject: [PATCH 59/68] Added a test for merge

---
 client/driver/exec.go        | 11 +++++++----
 client/driver/utils.go       |  4 ++++
 command/agent/config_test.go |  2 ++
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index fb09e1394..24d1d0358 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -9,6 +9,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/go-plugin"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/driver/executor"
@@ -164,19 +165,21 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	}
 	exec, client, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
 	if err != nil {
+		merrs := new(multierror.Error)
+		merrs.Errors = append(merrs.Errors, err)
 		d.logger.Println("[ERROR] driver.exec: error connecting to plugin so destroying plugin pid and user pid")
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
-			d.logger.Printf("[ERROR] driver.exec: error destroying plugin and userpid: %v", e)
+			merrs.Errors = append(merrs.Errors, fmt.Errorf("error destroying plugin and userpid: %v", e))
 		}
 		if id.IsolationConfig != nil {
 			if e := executor.DestroyCgroup(id.IsolationConfig.Cgroup); e != nil {
-				d.logger.Printf("[ERROR] driver.exec: destroying cgroup failed: %v", e)
+				merrs.Errors = append(merrs.Errors, fmt.Errorf("destroying cgroup failed: %v", e))
 			}
 		}
 		if e := ctx.AllocDir.UnmountSpecialDirs(id.TaskDir); e != nil {
-			d.logger.Printf("[ERROR] driver.exec: error unmounting dev and proc fs: %v", e)
+			merrs.Errors = append(merrs.Errors, fmt.Errorf("error unmounting dev and proc fs: %v", e))
 		}
-		return nil, fmt.Errorf("error connecting to plugin: %v", err)
+		return nil, fmt.Errorf("error connecting to plugin: %v", merrs.Error())
 	}
 
 	// Return a driver handle
diff --git a/client/driver/utils.go b/client/driver/utils.go
index 8bcbba23a..188e467d2 100644
--- a/client/driver/utils.go
+++ b/client/driver/utils.go
@@ -18,9 +18,13 @@ func createExecutor(config *plugin.ClientConfig, w io.Writer, clientConfig *conf
 	config.Plugins = GetPluginMap(w)
 	config.MaxPort = clientConfig.ClientMaxPort
 	config.MinPort = clientConfig.ClientMinPort
+
+	// setting the setsid of the plugin process so that it doesn't get signals sent to
+	// the nomad client.
 	if config.Cmd != nil {
 		isolateCommand(config.Cmd)
 	}
+
 	executorClient := plugin.NewClient(config)
 	rpcClient, err := executorClient.Client()
 	if err != nil {
diff --git a/command/agent/config_test.go b/command/agent/config_test.go
index 2662e1155..aa5349a8d 100644
--- a/command/agent/config_test.go
+++ b/command/agent/config_test.go
@@ -104,6 +104,8 @@ func TestConfig_Merge(t *testing.T) {
 				"foo": "bar",
 				"baz": "zip",
 			},
+			ClientMaxPort:  20000,
+			ClientMinPort:  22000,
 			NetworkSpeed:   105,
 			MaxKillTimeout: "50s",
 		},

From 1a31340075cd0175a813b31877df1344a7f4ce5f Mon Sep 17 00:00:00 2001
From: Alex Dadgar <alex.dadgar@gmail.com>
Date: Mon, 8 Feb 2016 18:51:11 -0800
Subject: [PATCH 60/68] Unmount special directories when task finishes

---
 client/alloc_runner_test.go              |  4 +-
 client/allocdir/alloc_dir.go             | 51 +++++++++++++++++++-----
 client/allocdir/alloc_dir_darwin.go      |  4 +-
 client/allocdir/alloc_dir_linux.go       | 14 +------
 client/allocdir/alloc_dir_windows.go     |  4 +-
 client/driver/exec.go                    |  6 +--
 client/driver/executor/executor_linux.go |  3 +-
 client/driver/java.go                    | 13 +++---
 8 files changed, 62 insertions(+), 37 deletions(-)

diff --git a/client/alloc_runner_test.go b/client/alloc_runner_test.go
index f082ba08e..6b526b846 100644
--- a/client/alloc_runner_test.go
+++ b/client/alloc_runner_test.go
@@ -53,7 +53,7 @@ func TestAllocRunner_SimpleRun(t *testing.T) {
 			return false, fmt.Errorf("No updates")
 		}
 		last := upd.Allocs[upd.Count-1]
-		if last.ClientStatus == structs.AllocClientStatusDead {
+		if last.ClientStatus != structs.AllocClientStatusDead {
 			return false, fmt.Errorf("got status %v; want %v", last.ClientStatus, structs.AllocClientStatusDead)
 		}
 		return true, nil
@@ -77,7 +77,7 @@ func TestAllocRunner_TerminalUpdate_Destroy(t *testing.T) {
 			return false, fmt.Errorf("No updates")
 		}
 		last := upd.Allocs[upd.Count-1]
-		if last.ClientStatus == structs.AllocClientStatusRunning {
+		if last.ClientStatus != structs.AllocClientStatusRunning {
 			return false, fmt.Errorf("got status %v; want %v", last.ClientStatus, structs.AllocClientStatusRunning)
 		}
 		return true, nil
diff --git a/client/allocdir/alloc_dir.go b/client/allocdir/alloc_dir.go
index ab56f6982..6904cf118 100644
--- a/client/allocdir/alloc_dir.go
+++ b/client/allocdir/alloc_dir.go
@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
@@ -37,9 +38,6 @@ type AllocDir struct {
 
 	// TaskDirs is a mapping of task names to their non-shared directory.
 	TaskDirs map[string]string
-
-	// A list of locations the shared alloc has been mounted to.
-	Mounted []string
 }
 
 // AllocFileInfo holds information about a file inside the AllocDir
@@ -67,13 +65,39 @@ func NewAllocDir(allocDir string) *AllocDir {
 // Tears down previously build directory structure.
 func (d *AllocDir) Destroy() error {
 	// Unmount all mounted shared alloc dirs.
-	for _, m := range d.Mounted {
-		if err := d.unmountSharedDir(m); err != nil {
-			return fmt.Errorf("Failed to unmount shared directory: %v", err)
-		}
+	var mErr multierror.Error
+	if err := d.UnmountAll(); err != nil {
+		mErr.Errors = append(mErr.Errors, err)
 	}
 
-	return os.RemoveAll(d.AllocDir)
+	if err := os.RemoveAll(d.AllocDir); err != nil {
+		mErr.Errors = append(mErr.Errors, err)
+	}
+
+	return mErr.ErrorOrNil()
+}
+
+func (d *AllocDir) UnmountAll() error {
+	var mErr multierror.Error
+	for _, dir := range d.TaskDirs {
+		// Check if the directory has the shared alloc mounted.
+		taskAlloc := filepath.Join(dir, SharedAllocName)
+		if d.pathExists(taskAlloc) {
+			if err := d.unmountSharedDir(taskAlloc); err != nil {
+				mErr.Errors = append(mErr.Errors,
+					fmt.Errorf("failed to unmount shared alloc dir %q: %v", taskAlloc, err))
+			}
+			if err := os.RemoveAll(taskAlloc); err != nil {
+				mErr.Errors = append(mErr.Errors,
+					fmt.Errorf("failed to delete shared alloc dir %q: %v", taskAlloc, err))
+			}
+		}
+
+		// Unmount dev/ and proc/ have been mounted.
+		d.unmountSpecialDirs(dir)
+	}
+
+	return mErr.ErrorOrNil()
 }
 
 // Given a list of a task build the correct alloc structure.
@@ -248,7 +272,6 @@ func (d *AllocDir) MountSharedDir(task string) error {
 		return fmt.Errorf("Failed to mount shared directory for task %v: %v", task, err)
 	}
 
-	d.Mounted = append(d.Mounted, taskLoc)
 	return nil
 }
 
@@ -325,3 +348,13 @@ func fileCopy(src, dst string, perm os.FileMode) error {
 
 	return nil
 }
+
+// pathExists is a helper function to check if the path exists.
+func (d *AllocDir) pathExists(path string) bool {
+	if _, err := os.Stat(path); err != nil {
+		if os.IsNotExist(err) {
+			return false
+		}
+	}
+	return true
+}
diff --git a/client/allocdir/alloc_dir_darwin.go b/client/allocdir/alloc_dir_darwin.go
index 723112857..2cfdd38c3 100644
--- a/client/allocdir/alloc_dir_darwin.go
+++ b/client/allocdir/alloc_dir_darwin.go
@@ -20,7 +20,7 @@ func (d *AllocDir) MountSpecialDirs(taskDir string) error {
 	return nil
 }
 
-// UnmountSpecialDirs unmounts the dev and proc file system from the chroot
-func (d *AllocDir) UnmountSpecialDirs(taskDir string) error {
+// unmountSpecialDirs unmounts the dev and proc file system from the chroot
+func (d *AllocDir) unmountSpecialDirs(taskDir string) error {
 	return nil
 }
diff --git a/client/allocdir/alloc_dir_linux.go b/client/allocdir/alloc_dir_linux.go
index 212ea61b6..d8d44af1f 100644
--- a/client/allocdir/alloc_dir_linux.go
+++ b/client/allocdir/alloc_dir_linux.go
@@ -53,8 +53,8 @@ func (d *AllocDir) MountSpecialDirs(taskDir string) error {
 	return nil
 }
 
-// UnmountSpecialDirs unmounts the dev and proc file system from the chroot
-func (d *AllocDir) UnmountSpecialDirs(taskDir string) error {
+// unmountSpecialDirs unmounts the dev and proc file system from the chroot
+func (d *AllocDir) unmountSpecialDirs(taskDir string) error {
 	errs := new(multierror.Error)
 	dev := filepath.Join(taskDir, "dev")
 	if d.pathExists(dev) {
@@ -81,13 +81,3 @@ func (d *AllocDir) UnmountSpecialDirs(taskDir string) error {
 
 	return errs.ErrorOrNil()
 }
-
-// pathExists is a helper function to check if the path exists.
-func (d *AllocDir) pathExists(path string) bool {
-	if _, err := os.Stat(path); err != nil {
-		if os.IsNotExist(err) {
-			return false
-		}
-	}
-	return true
-}
diff --git a/client/allocdir/alloc_dir_windows.go b/client/allocdir/alloc_dir_windows.go
index 31a4ebb8e..7211125ae 100644
--- a/client/allocdir/alloc_dir_windows.go
+++ b/client/allocdir/alloc_dir_windows.go
@@ -30,7 +30,7 @@ func (d *AllocDir) MountSpecialDirs(taskDir string) error {
 	return nil
 }
 
-// UnmountSpecialDirs unmounts the dev and proc file system from the chroot
-func (d *AllocDir) UnmountSpecialDirs(taskDir string) error {
+// unmountSpecialDirs unmounts the dev and proc file system from the chroot
+func (d *AllocDir) unmountSpecialDirs(taskDir string) error {
 	return nil
 }
diff --git a/client/driver/exec.go b/client/driver/exec.go
index 24d1d0358..533ff4dd8 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -176,10 +176,10 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 				merrs.Errors = append(merrs.Errors, fmt.Errorf("destroying cgroup failed: %v", e))
 			}
 		}
-		if e := ctx.AllocDir.UnmountSpecialDirs(id.TaskDir); e != nil {
-			merrs.Errors = append(merrs.Errors, fmt.Errorf("error unmounting dev and proc fs: %v", e))
+		if e := ctx.AllocDir.UnmountAll(); e != nil {
+			merrs.Errors = append(merrs.Errors, e)
 		}
-		return nil, fmt.Errorf("error connecting to plugin: %v", merrs.Error())
+		return nil, fmt.Errorf("error connecting to plugin: %v", merrs.ErrorOrNil())
 	}
 
 	// Return a driver handle
diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go
index 99780e1a0..d156736f4 100644
--- a/client/driver/executor/executor_linux.go
+++ b/client/driver/executor/executor_linux.go
@@ -177,8 +177,7 @@ func (e *UniversalExecutor) removeChrootMounts() error {
 	// Prevent a race between Wait/ForceStop
 	e.lock.Lock()
 	defer e.lock.Unlock()
-
-	return e.ctx.AllocDir.UnmountSpecialDirs(e.taskDir)
+	return e.ctx.AllocDir.UnmountAll()
 }
 
 // destroyCgroup kills all processes in the cgroup and removes the cgroup
diff --git a/client/driver/java.go b/client/driver/java.go
index ee33d8b88..abaa0f0b8 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -12,6 +12,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/go-plugin"
 	"github.com/mitchellh/mapstructure"
 
@@ -211,20 +212,22 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 	}
 	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
 	if err != nil {
+		merrs := new(multierror.Error)
+		merrs.Errors = append(merrs.Errors, err)
 		d.logger.Println("[ERROR] driver.java: error connecting to plugin so destroying plugin pid and user pid")
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
-			d.logger.Printf("[ERROR] driver.java: error destroying plugin and userpid: %v", e)
+			merrs.Errors = append(merrs.Errors, fmt.Errorf("error destroying plugin and userpid: %v", e))
 		}
 		if id.IsolationConfig != nil {
 			if e := executor.DestroyCgroup(id.IsolationConfig.Cgroup); e != nil {
-				d.logger.Printf("[ERROR] driver.exec: destroying cgroup failed: %v", e)
+				merrs.Errors = append(merrs.Errors, fmt.Errorf("destroying cgroup failed: %v", e))
 			}
 		}
-		if e := ctx.AllocDir.UnmountSpecialDirs(id.TaskDir); e != nil {
-			d.logger.Printf("[ERROR] driver.exec: error unmounting dev and proc fs: %v", e)
+		if e := ctx.AllocDir.UnmountAll(); e != nil {
+			merrs.Errors = append(merrs.Errors, e)
 		}
 
-		return nil, fmt.Errorf("error connecting to plugin: %v", err)
+		return nil, fmt.Errorf("error connecting to plugin: %v", merrs.ErrorOrNil())
 	}
 
 	// Return a driver handle

From 0dcdaf50afe835c3e9d360189ae4f72aa50cc661 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Mon, 8 Feb 2016 19:00:26 -0800
Subject: [PATCH 61/68] Fixed a test

---
 client/driver/exec_test.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go
index 047584db0..76009316c 100644
--- a/client/driver/exec_test.go
+++ b/client/driver/exec_test.go
@@ -129,7 +129,8 @@ func TestExecDriver_KillUserPid_OnPluginReconnectFailure(t *testing.T) {
 	userProc, err := os.FindProcess(id.UserPid)
 
 	err = userProc.Signal(syscall.Signal(0))
-	if err != nil {
+
+	if err == nil {
 		t.Fatalf("expected user process to die")
 	}
 }

From e696c7eef7b79c1f501263f0dd4d74f298309e64 Mon Sep 17 00:00:00 2001
From: Alex Dadgar <alex.dadgar@gmail.com>
Date: Mon, 8 Feb 2016 19:31:57 -0800
Subject: [PATCH 62/68] Fix timeouts between signal and kill

---
 client/driver/driver.go        |  6 ++++++
 client/driver/driver_test.go   |  1 +
 client/driver/exec.go          | 12 +++++++++---
 client/driver/exec_test.go     |  2 +-
 client/driver/raw_exec_test.go |  2 +-
 5 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/client/driver/driver.go b/client/driver/driver.go
index f92b84fad..ecd0c0271 100644
--- a/client/driver/driver.go
+++ b/client/driver/driver.go
@@ -89,6 +89,12 @@ func NewDriverContext(taskName string, config *config.Config, node *structs.Node
 func (d *DriverContext) KillTimeout(task *structs.Task) time.Duration {
 	max := d.config.MaxKillTimeout.Nanoseconds()
 	desired := task.KillTimeout.Nanoseconds()
+
+	// Make the minimum time between signal and kill, 1 second.
+	if desired == 0 {
+		desired = (1 * time.Second).Nanoseconds()
+	}
+
 	if desired < max {
 		return task.KillTimeout
 	}
diff --git a/client/driver/driver_test.go b/client/driver/driver_test.go
index 0ab20d813..f76f8c68e 100644
--- a/client/driver/driver_test.go
+++ b/client/driver/driver_test.go
@@ -46,6 +46,7 @@ func testConfig() *config.Config {
 	conf := &config.Config{}
 	conf.StateDir = os.TempDir()
 	conf.AllocDir = os.TempDir()
+	conf.MaxKillTimeout = 10 * time.Second
 	return conf
 }
 
diff --git a/client/driver/exec.go b/client/driver/exec.go
index 533ff4dd8..eed58a796 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -227,7 +227,10 @@ func (h *execHandle) Update(task *structs.Task) error {
 }
 
 func (h *execHandle) Kill() error {
-	h.executor.ShutDown()
+	if err := h.executor.ShutDown(); err != nil {
+		return fmt.Errorf("executor Shutdown failed: %v", err)
+	}
+
 	select {
 	case <-h.doneCh:
 		return nil
@@ -235,8 +238,11 @@ func (h *execHandle) Kill() error {
 		if h.pluginClient.Exited() {
 			return nil
 		}
-		err := h.executor.Exit()
-		return err
+		if err := h.executor.Exit(); err != nil {
+			return fmt.Errorf("executor Exit failed: %v", err)
+		}
+
+		return nil
 	}
 }
 
diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go
index 76009316c..de02f4959 100644
--- a/client/driver/exec_test.go
+++ b/client/driver/exec_test.go
@@ -322,7 +322,7 @@ func TestExecDriver_Start_Kill_Wait(t *testing.T) {
 		Name: "sleep",
 		Config: map[string]interface{}{
 			"command": "/bin/sleep",
-			"args":    []string{"10"},
+			"args":    []string{"45"},
 		},
 		Resources: basicResources,
 	}
diff --git a/client/driver/raw_exec_test.go b/client/driver/raw_exec_test.go
index 08f578a06..a09010d3e 100644
--- a/client/driver/raw_exec_test.go
+++ b/client/driver/raw_exec_test.go
@@ -287,7 +287,7 @@ func TestRawExecDriver_Start_Kill_Wait(t *testing.T) {
 		Name: "sleep",
 		Config: map[string]interface{}{
 			"command": testtask.Path(),
-			"args":    []string{"sleep", "15s"},
+			"args":    []string{"sleep", "45s"},
 		},
 		Resources: basicResources,
 	}

From ccaf000fd6bc9187714d0de6093602e0b032f008 Mon Sep 17 00:00:00 2001
From: Alex Dadgar <alex.dadgar@gmail.com>
Date: Mon, 8 Feb 2016 19:46:46 -0800
Subject: [PATCH 63/68] Fix min duration

---
 client/driver/driver.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/client/driver/driver.go b/client/driver/driver.go
index ecd0c0271..db808b18d 100644
--- a/client/driver/driver.go
+++ b/client/driver/driver.go
@@ -96,7 +96,7 @@ func (d *DriverContext) KillTimeout(task *structs.Task) time.Duration {
 	}
 
 	if desired < max {
-		return task.KillTimeout
+		return time.Duration(desired)
 	}
 
 	return d.config.MaxKillTimeout

From f2efece8100efb2a1e0e3b457b3c428439b483bb Mon Sep 17 00:00:00 2001
From: Alex Dadgar <alex.dadgar@gmail.com>
Date: Tue, 9 Feb 2016 09:43:40 -0800
Subject: [PATCH 64/68] more time

---
 client/driver/exec_test.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go
index de02f4959..85da6bfa5 100644
--- a/client/driver/exec_test.go
+++ b/client/driver/exec_test.go
@@ -322,9 +322,10 @@ func TestExecDriver_Start_Kill_Wait(t *testing.T) {
 		Name: "sleep",
 		Config: map[string]interface{}{
 			"command": "/bin/sleep",
-			"args":    []string{"45"},
+			"args":    []string{"100"},
 		},
-		Resources: basicResources,
+		Resources:   basicResources,
+		KillTimeout: 10 * time.Second,
 	}
 
 	driverCtx, execCtx := testDriverContexts(task)

From 279f095a7d2b7811d5520ca84255da8b20b22c01 Mon Sep 17 00:00:00 2001
From: Alex Dadgar <alex.dadgar@gmail.com>
Date: Tue, 9 Feb 2016 10:00:42 -0800
Subject: [PATCH 65/68] Don't error killing exited process

---
 client/driver/executor/executor.go | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go
index 4384e1ce4..2de8e7bca 100644
--- a/client/driver/executor/executor.go
+++ b/client/driver/executor/executor.go
@@ -193,6 +193,12 @@ func (e *UniversalExecutor) wait() {
 	e.exitState = &ProcessState{Pid: 0, ExitCode: exitCode, Time: time.Now()}
 }
 
+var (
+	// finishedErr is the error message received when trying to kill and already
+	// exited process.
+	finishedErr = "os: process already finished"
+)
+
 // Exit cleans up the alloc directory, destroys cgroups and kills the user
 // process
 func (e *UniversalExecutor) Exit() error {
@@ -202,7 +208,7 @@ func (e *UniversalExecutor) Exit() error {
 		if err != nil {
 			e.logger.Printf("[ERROR] executor: can't find process with pid: %v, err: %v",
 				e.cmd.Process.Pid, err)
-		} else if err := proc.Kill(); err != nil {
+		} else if err := proc.Kill(); err != nil && err.Error() != finishedErr {
 			merr.Errors = append(merr.Errors,
 				fmt.Errorf("can't kill process with pid: %v, err: %v", e.cmd.Process.Pid, err))
 		}

From a97181157ce87ebd4f7b9ac82d79b1235776a7e5 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Tue, 9 Feb 2016 10:17:33 -0800
Subject: [PATCH 66/68] Killing the userpid if the plugin is killed while the
 client is running

---
 client/driver/exec.go          | 25 +++++++++++++++++-
 client/driver/exec_test.go     | 47 +++++++++++++++++++++++++++++++++
 client/driver/java.go          | 20 ++++++++++++++
 client/driver/qemu.go          | 14 ++++++++++
 client/driver/raw_exec.go      | 14 ++++++++++
 client/driver/raw_exec_test.go | 48 ++++++++++++++++++++++++++++++++++
 6 files changed, 167 insertions(+), 1 deletion(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index eed58a796..50ba323ac 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/go-plugin"
+	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/driver/executor"
 	cstructs "github.com/hashicorp/nomad/client/driver/structs"
@@ -40,6 +41,7 @@ type execHandle struct {
 	isolationConfig *executor.IsolationConfig
 	userPid         int
 	taskDir         string
+	allocDir        *allocdir.AllocDir
 	killTimeout     time.Duration
 	logger          *log.Logger
 	waitCh          chan *cstructs.WaitResult
@@ -136,6 +138,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		userPid:         ps.Pid,
 		executor:        exec,
 		taskDir:         taskDir,
+		allocDir:        ctx.AllocDir,
 		isolationConfig: ps.IsolationConfig,
 		killTimeout:     d.DriverContext.KillTimeout(task),
 		logger:          d.logger,
@@ -150,6 +153,7 @@ type execId struct {
 	KillTimeout     time.Duration
 	UserPid         int
 	TaskDir         string
+	AllocDir        *allocdir.AllocDir
 	IsolationConfig *executor.IsolationConfig
 	PluginConfig    *ExecutorReattachConfig
 }
@@ -188,6 +192,7 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		executor:        exec,
 		userPid:         id.UserPid,
 		taskDir:         id.TaskDir,
+		allocDir:        id.AllocDir,
 		isolationConfig: id.IsolationConfig,
 		logger:          d.logger,
 		killTimeout:     id.KillTimeout,
@@ -204,6 +209,7 @@ func (h *execHandle) ID() string {
 		PluginConfig:    NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
 		UserPid:         h.userPid,
 		TaskDir:         h.taskDir,
+		AllocDir:        h.allocDir,
 		IsolationConfig: h.isolationConfig,
 	}
 
@@ -249,7 +255,24 @@ func (h *execHandle) Kill() error {
 func (h *execHandle) run() {
 	ps, err := h.executor.Wait()
 	close(h.doneCh)
-	h.waitCh <- &cstructs.WaitResult{ExitCode: ps.ExitCode, Signal: 0, Err: err}
+
+	// If the exitcode is 0 and we had an error that means the plugin didn't
+	// connect and doesn't know the state of the user process so we are killing
+	// the user process so that when we create a new executor on restarting the
+	// new user process doesn't have collisions with resources that the older
+	// user pid might be holding onto.
+	if ps.ExitCode == 0 && err != nil {
+		if h.isolationConfig != nil {
+			if e := executor.DestroyCgroup(h.isolationConfig.Cgroup); e != nil {
+				h.logger.Printf("[ERROR] driver.exec: destroying cgroup failed while killing cgroup: %v", e)
+			}
+		}
+		if e := h.allocDir.UnmountAll(); e != nil {
+			h.logger.Printf("[ERROR] driver.exec: unmounting dev,proc and alloc dirs failed: %v", e)
+		}
+	}
+	h.waitCh <- &cstructs.WaitResult{ExitCode: ps.ExitCode, Signal: 0,
+		Err: err}
 	close(h.waitCh)
 	h.pluginClient.Kill()
 }
diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go
index de02f4959..66771ca74 100644
--- a/client/driver/exec_test.go
+++ b/client/driver/exec_test.go
@@ -135,6 +135,53 @@ func TestExecDriver_KillUserPid_OnPluginReconnectFailure(t *testing.T) {
 	}
 }
 
+func TestExecDriver_KillExecutorPid(t *testing.T) {
+	t.Parallel()
+	ctestutils.ExecCompatible(t)
+	task := &structs.Task{
+		Name: "sleep",
+		Config: map[string]interface{}{
+			"command": "/bin/sleep",
+			"args":    []string{"1000000"},
+		},
+		Resources: basicResources,
+	}
+
+	driverCtx, execCtx := testDriverContexts(task)
+	defer execCtx.AllocDir.Destroy()
+	d := NewExecDriver(driverCtx)
+
+	handle, err := d.Start(execCtx, task)
+	defer handle.Kill()
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if handle == nil {
+		t.Fatalf("missing handle")
+	}
+
+	id := &execId{}
+	if err := json.Unmarshal([]byte(handle.ID()), id); err != nil {
+		t.Fatalf("Failed to parse handle '%s': %v", handle.ID(), err)
+	}
+	pluginPid := id.PluginConfig.Pid
+	userPid := id.UserPid
+	proc, err := os.FindProcess(pluginPid)
+	if err != nil {
+		t.Fatalf("got err: %v", err)
+	}
+	if err = proc.Kill(); err != nil {
+		t.Fatalf("got err: %v", err)
+	}
+	procUser, _ := os.FindProcess(userPid)
+	if procUser != nil {
+		time.Sleep(1 * time.Second)
+		if e := procUser.Signal(syscall.Signal(0)); e == nil {
+			t.Fatalf("Expected process to be dead")
+		}
+	}
+}
+
 func TestExecDriver_Start_Wait(t *testing.T) {
 	t.Parallel()
 	ctestutils.ExecCompatible(t)
diff --git a/client/driver/java.go b/client/driver/java.go
index abaa0f0b8..0c206c174 100644
--- a/client/driver/java.go
+++ b/client/driver/java.go
@@ -16,6 +16,7 @@ import (
 	"github.com/hashicorp/go-plugin"
 	"github.com/mitchellh/mapstructure"
 
+	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/driver/executor"
 	cstructs "github.com/hashicorp/nomad/client/driver/structs"
@@ -47,6 +48,7 @@ type javaHandle struct {
 	isolationConfig *executor.IsolationConfig
 
 	taskDir     string
+	allocDir    *allocdir.AllocDir
 	killTimeout time.Duration
 	logger      *log.Logger
 	waitCh      chan *cstructs.WaitResult
@@ -183,6 +185,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		userPid:         ps.Pid,
 		isolationConfig: ps.IsolationConfig,
 		taskDir:         taskDir,
+		allocDir:        ctx.AllocDir,
 		killTimeout:     d.DriverContext.KillTimeout(task),
 		logger:          d.logger,
 		doneCh:          make(chan struct{}),
@@ -198,6 +201,7 @@ type javaId struct {
 	PluginConfig    *ExecutorReattachConfig
 	IsolationConfig *executor.IsolationConfig
 	TaskDir         string
+	AllocDir        *allocdir.AllocDir
 	UserPid         int
 }
 
@@ -237,6 +241,7 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		userPid:         id.UserPid,
 		isolationConfig: id.IsolationConfig,
 		taskDir:         id.TaskDir,
+		allocDir:        id.AllocDir,
 		logger:          d.logger,
 		killTimeout:     id.KillTimeout,
 		doneCh:          make(chan struct{}),
@@ -253,6 +258,7 @@ func (h *javaHandle) ID() string {
 		PluginConfig:    NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
 		UserPid:         h.userPid,
 		TaskDir:         h.taskDir,
+		AllocDir:        h.allocDir,
 		IsolationConfig: h.isolationConfig,
 	}
 
@@ -288,6 +294,20 @@ func (h *javaHandle) Kill() error {
 func (h *javaHandle) run() {
 	ps, err := h.executor.Wait()
 	close(h.doneCh)
+	if ps.ExitCode == 0 && err != nil {
+		if h.isolationConfig != nil {
+			if e := executor.DestroyCgroup(h.isolationConfig.Cgroup); e != nil {
+				h.logger.Printf("[ERROR] driver.java: destroying cgroup failed while killing cgroup: %v", e)
+			}
+		} else {
+			if e := killProcess(h.userPid); e != nil {
+				h.logger.Printf("[ERROR] driver.java: error killing user process: %v", e)
+			}
+		}
+		if e := h.allocDir.UnmountAll(); e != nil {
+			h.logger.Printf("[ERROR] driver.java: unmounting dev,proc and alloc dirs failed: %v", e)
+		}
+	}
 	h.waitCh <- &cstructs.WaitResult{ExitCode: ps.ExitCode, Signal: 0, Err: err}
 	close(h.waitCh)
 	h.pluginClient.Kill()
diff --git a/client/driver/qemu.go b/client/driver/qemu.go
index 56f6d03f7..deae7d273 100644
--- a/client/driver/qemu.go
+++ b/client/driver/qemu.go
@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/go-plugin"
+	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/driver/executor"
 	cstructs "github.com/hashicorp/nomad/client/driver/structs"
@@ -46,6 +47,7 @@ type qemuHandle struct {
 	pluginClient *plugin.Client
 	userPid      int
 	executor     executor.Executor
+	allocDir     *allocdir.AllocDir
 	killTimeout  time.Duration
 	logger       *log.Logger
 	waitCh       chan *cstructs.WaitResult
@@ -219,6 +221,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		pluginClient: pluginClient,
 		executor:     exec,
 		userPid:      ps.Pid,
+		allocDir:     ctx.AllocDir,
 		killTimeout:  d.DriverContext.KillTimeout(task),
 		logger:       d.logger,
 		doneCh:       make(chan struct{}),
@@ -233,6 +236,7 @@ type qemuId struct {
 	KillTimeout  time.Duration
 	UserPid      int
 	PluginConfig *ExecutorReattachConfig
+	AllocDir     *allocdir.AllocDir
 }
 
 func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
@@ -259,6 +263,7 @@ func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		pluginClient: pluginClient,
 		executor:     executor,
 		userPid:      id.UserPid,
+		allocDir:     id.AllocDir,
 		logger:       d.logger,
 		killTimeout:  id.KillTimeout,
 		doneCh:       make(chan struct{}),
@@ -273,6 +278,7 @@ func (h *qemuHandle) ID() string {
 		KillTimeout:  h.killTimeout,
 		PluginConfig: NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
 		UserPid:      h.userPid,
+		AllocDir:     h.allocDir,
 	}
 
 	data, err := json.Marshal(id)
@@ -308,6 +314,14 @@ func (h *qemuHandle) Kill() error {
 
 func (h *qemuHandle) run() {
 	ps, err := h.executor.Wait()
+	if ps.ExitCode == 0 && err != nil {
+		if e := killProcess(h.userPid); e != nil {
+			h.logger.Printf("[ERROR] driver.qemu: error killing user process: %v", e)
+		}
+		if e := h.allocDir.UnmountAll(); e != nil {
+			h.logger.Printf("[ERROR] driver.qemu: unmounting dev,proc and alloc dirs failed: %v", e)
+		}
+	}
 	close(h.doneCh)
 	h.waitCh <- &cstructs.WaitResult{ExitCode: ps.ExitCode, Signal: 0, Err: err}
 	close(h.waitCh)
diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go
index 19c453a0f..fe25ecb04 100644
--- a/client/driver/raw_exec.go
+++ b/client/driver/raw_exec.go
@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/go-plugin"
+	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/driver/executor"
 	cstructs "github.com/hashicorp/nomad/client/driver/structs"
@@ -38,6 +39,7 @@ type rawExecHandle struct {
 	userPid      int
 	executor     executor.Executor
 	killTimeout  time.Duration
+	allocDir     *allocdir.AllocDir
 	logger       *log.Logger
 	waitCh       chan *cstructs.WaitResult
 	doneCh       chan struct{}
@@ -126,6 +128,7 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
 		executor:     exec,
 		userPid:      ps.Pid,
 		killTimeout:  d.DriverContext.KillTimeout(task),
+		allocDir:     ctx.AllocDir,
 		logger:       d.logger,
 		doneCh:       make(chan struct{}),
 		waitCh:       make(chan *cstructs.WaitResult, 1),
@@ -138,6 +141,7 @@ type rawExecId struct {
 	KillTimeout  time.Duration
 	UserPid      int
 	PluginConfig *ExecutorReattachConfig
+	AllocDir     *allocdir.AllocDir
 }
 
 func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
@@ -165,6 +169,7 @@ func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, e
 		userPid:      id.UserPid,
 		logger:       d.logger,
 		killTimeout:  id.KillTimeout,
+		allocDir:     id.AllocDir,
 		doneCh:       make(chan struct{}),
 		waitCh:       make(chan *cstructs.WaitResult, 1),
 	}
@@ -177,6 +182,7 @@ func (h *rawExecHandle) ID() string {
 		KillTimeout:  h.killTimeout,
 		PluginConfig: NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
 		UserPid:      h.userPid,
+		AllocDir:     h.allocDir,
 	}
 
 	data, err := json.Marshal(id)
@@ -211,6 +217,14 @@ func (h *rawExecHandle) Kill() error {
 func (h *rawExecHandle) run() {
 	ps, err := h.executor.Wait()
 	close(h.doneCh)
+	if ps.ExitCode == 0 && err != nil {
+		if e := killProcess(h.userPid); e != nil {
+			h.logger.Printf("[ERROR] driver.raw_exec: error killing user process: %v", e)
+		}
+		if e := h.allocDir.UnmountAll(); e != nil {
+			h.logger.Printf("[ERROR] driver.raw_exec: unmounting dev,proc and alloc dirs failed: %v", e)
+		}
+	}
 	h.waitCh <- &cstructs.WaitResult{ExitCode: ps.ExitCode, Signal: 0, Err: err}
 	close(h.waitCh)
 	h.pluginClient.Kill()
diff --git a/client/driver/raw_exec_test.go b/client/driver/raw_exec_test.go
index a09010d3e..e7a3f21e8 100644
--- a/client/driver/raw_exec_test.go
+++ b/client/driver/raw_exec_test.go
@@ -1,10 +1,12 @@
 package driver
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"path/filepath"
 	"reflect"
 	"testing"
@@ -95,6 +97,52 @@ func TestRawExecDriver_StartOpen_Wait(t *testing.T) {
 	handle2.Kill()
 }
 
+func TestRawExecDriver_KillExecutorPid(t *testing.T) {
+	t.Parallel()
+	task := &structs.Task{
+		Name: "sleep",
+		Config: map[string]interface{}{
+			"command": "/bin/sleep",
+			"args":    []string{"1000000"},
+		},
+		Resources: basicResources,
+	}
+
+	driverCtx, execCtx := testDriverContexts(task)
+	defer execCtx.AllocDir.Destroy()
+	d := NewExecDriver(driverCtx)
+
+	handle, err := d.Start(execCtx, task)
+	defer handle.Kill()
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if handle == nil {
+		t.Fatalf("missing handle")
+	}
+
+	id := &rawExecId{}
+	if err := json.Unmarshal([]byte(handle.ID()), id); err != nil {
+		t.Fatalf("Failed to parse handle '%s': %v", handle.ID(), err)
+	}
+	pluginPid := id.PluginConfig.Pid
+	userPid := id.UserPid
+	proc, err := os.FindProcess(pluginPid)
+	if err != nil {
+		t.Fatalf("got err: %v", err)
+	}
+	if err = proc.Kill(); err != nil {
+		t.Fatalf("got err: %v", err)
+	}
+	time.Sleep(1 * time.Second)
+	procUser, err := os.FindProcess(userPid)
+	if err != nil {
+		if err = procUser.Kill(); err != nil {
+			t.Fatalf("expected process to be dead")
+		}
+	}
+}
+
 func TestRawExecDriver_Start_Artifact_basic(t *testing.T) {
 	t.Parallel()
 	path := testtask.Path()

From e6fcd0ec20d0a070df9a1e815f004c753f587c4a Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Tue, 9 Feb 2016 10:21:47 -0800
Subject: [PATCH 67/68] Removing taskDir from the exechandle

---
 client/driver/exec.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/client/driver/exec.go b/client/driver/exec.go
index 50ba323ac..35e73dd1b 100644
--- a/client/driver/exec.go
+++ b/client/driver/exec.go
@@ -40,7 +40,6 @@ type execHandle struct {
 	executor        executor.Executor
 	isolationConfig *executor.IsolationConfig
 	userPid         int
-	taskDir         string
 	allocDir        *allocdir.AllocDir
 	killTimeout     time.Duration
 	logger          *log.Logger
@@ -137,7 +136,6 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
 		pluginClient:    pluginClient,
 		userPid:         ps.Pid,
 		executor:        exec,
-		taskDir:         taskDir,
 		allocDir:        ctx.AllocDir,
 		isolationConfig: ps.IsolationConfig,
 		killTimeout:     d.DriverContext.KillTimeout(task),
@@ -191,7 +189,6 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 		pluginClient:    client,
 		executor:        exec,
 		userPid:         id.UserPid,
-		taskDir:         id.TaskDir,
 		allocDir:        id.AllocDir,
 		isolationConfig: id.IsolationConfig,
 		logger:          d.logger,
@@ -208,7 +205,6 @@ func (h *execHandle) ID() string {
 		KillTimeout:     h.killTimeout,
 		PluginConfig:    NewExecutorReattachConfig(h.pluginClient.ReattachConfig()),
 		UserPid:         h.userPid,
-		TaskDir:         h.taskDir,
 		AllocDir:        h.allocDir,
 		IsolationConfig: h.isolationConfig,
 	}

From c8ac49bc913b9d679cabcf4721560d355446fdd3 Mon Sep 17 00:00:00 2001
From: Diptanu Choudhury <diptanuc@gmail.com>
Date: Tue, 9 Feb 2016 11:29:02 -0800
Subject: [PATCH 68/68] Removing the killing executor pid tests

---
 client/driver/exec_test.go     | 47 ----------------------------------
 client/driver/raw_exec_test.go | 46 ---------------------------------
 2 files changed, 93 deletions(-)

diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go
index fde1cf09d..85da6bfa5 100644
--- a/client/driver/exec_test.go
+++ b/client/driver/exec_test.go
@@ -135,53 +135,6 @@ func TestExecDriver_KillUserPid_OnPluginReconnectFailure(t *testing.T) {
 	}
 }
 
-func TestExecDriver_KillExecutorPid(t *testing.T) {
-	t.Parallel()
-	ctestutils.ExecCompatible(t)
-	task := &structs.Task{
-		Name: "sleep",
-		Config: map[string]interface{}{
-			"command": "/bin/sleep",
-			"args":    []string{"1000000"},
-		},
-		Resources: basicResources,
-	}
-
-	driverCtx, execCtx := testDriverContexts(task)
-	defer execCtx.AllocDir.Destroy()
-	d := NewExecDriver(driverCtx)
-
-	handle, err := d.Start(execCtx, task)
-	defer handle.Kill()
-	if err != nil {
-		t.Fatalf("err: %v", err)
-	}
-	if handle == nil {
-		t.Fatalf("missing handle")
-	}
-
-	id := &execId{}
-	if err := json.Unmarshal([]byte(handle.ID()), id); err != nil {
-		t.Fatalf("Failed to parse handle '%s': %v", handle.ID(), err)
-	}
-	pluginPid := id.PluginConfig.Pid
-	userPid := id.UserPid
-	proc, err := os.FindProcess(pluginPid)
-	if err != nil {
-		t.Fatalf("got err: %v", err)
-	}
-	if err = proc.Kill(); err != nil {
-		t.Fatalf("got err: %v", err)
-	}
-	procUser, _ := os.FindProcess(userPid)
-	if procUser != nil {
-		time.Sleep(1 * time.Second)
-		if e := procUser.Signal(syscall.Signal(0)); e == nil {
-			t.Fatalf("Expected process to be dead")
-		}
-	}
-}
-
 func TestExecDriver_Start_Wait(t *testing.T) {
 	t.Parallel()
 	ctestutils.ExecCompatible(t)
diff --git a/client/driver/raw_exec_test.go b/client/driver/raw_exec_test.go
index e7a3f21e8..ab6a948d7 100644
--- a/client/driver/raw_exec_test.go
+++ b/client/driver/raw_exec_test.go
@@ -97,52 +97,6 @@ func TestRawExecDriver_StartOpen_Wait(t *testing.T) {
 	handle2.Kill()
 }
 
-func TestRawExecDriver_KillExecutorPid(t *testing.T) {
-	t.Parallel()
-	task := &structs.Task{
-		Name: "sleep",
-		Config: map[string]interface{}{
-			"command": "/bin/sleep",
-			"args":    []string{"1000000"},
-		},
-		Resources: basicResources,
-	}
-
-	driverCtx, execCtx := testDriverContexts(task)
-	defer execCtx.AllocDir.Destroy()
-	d := NewExecDriver(driverCtx)
-
-	handle, err := d.Start(execCtx, task)
-	defer handle.Kill()
-	if err != nil {
-		t.Fatalf("err: %v", err)
-	}
-	if handle == nil {
-		t.Fatalf("missing handle")
-	}
-
-	id := &rawExecId{}
-	if err := json.Unmarshal([]byte(handle.ID()), id); err != nil {
-		t.Fatalf("Failed to parse handle '%s': %v", handle.ID(), err)
-	}
-	pluginPid := id.PluginConfig.Pid
-	userPid := id.UserPid
-	proc, err := os.FindProcess(pluginPid)
-	if err != nil {
-		t.Fatalf("got err: %v", err)
-	}
-	if err = proc.Kill(); err != nil {
-		t.Fatalf("got err: %v", err)
-	}
-	time.Sleep(1 * time.Second)
-	procUser, err := os.FindProcess(userPid)
-	if err != nil {
-		if err = procUser.Kill(); err != nil {
-			t.Fatalf("expected process to be dead")
-		}
-	}
-}
-
 func TestRawExecDriver_Start_Artifact_basic(t *testing.T) {
 	t.Parallel()
 	path := testtask.Path()