From d7a772c52ba4be2daead790a79a8580d9717c076 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Mon, 8 Feb 2016 10:05:39 -0800 Subject: [PATCH] Destroying the cgroup if we can't to executor in exec and java drivers --- client/driver/exec.go | 10 ++++++++ client/driver/executor/executor.go | 11 ++++---- client/driver/java.go | 10 ++++++++ client/driver/utils_linux.go | 40 ++++++++++++++++++++++++++++++ client/driver/utils_posix.go | 8 ++++++ client/driver/utils_windows.go | 6 +++++ 6 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 client/driver/utils_linux.go diff --git a/client/driver/exec.go b/client/driver/exec.go index 4aebad667..6936461b6 100644 --- a/client/driver/exec.go +++ b/client/driver/exec.go @@ -17,6 +17,8 @@ import ( "github.com/hashicorp/nomad/helper/discover" "github.com/hashicorp/nomad/nomad/structs" "github.com/mitchellh/mapstructure" + + cgroupConfig "github.com/opencontainers/runc/libcontainer/configs" ) // ExecDriver fork/execs tasks using as many of the underlying OS's isolation @@ -36,6 +38,7 @@ type ExecDriverConfig struct { type execHandle struct { pluginClient *plugin.Client executor executor.Executor + groups *cgroupConfig.Cgroup userPid int killTimeout time.Duration logger *log.Logger @@ -132,6 +135,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, pluginClient: pluginClient, userPid: ps.Pid, executor: exec, + groups: &ps.IsolationConfig, killTimeout: d.DriverContext.KillTimeout(task), logger: d.logger, doneCh: make(chan struct{}), @@ -144,6 +148,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, type execId struct { KillTimeout time.Duration UserPid int + Groups *cgroupConfig.Cgroup PluginConfig *ExecutorReattachConfig } @@ -162,6 +167,9 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil { d.logger.Printf("[ERROR] driver.exec: error destroying plugin and userpid: %v", e) } + if e := destroyCgroup(id.Groups); e != nil { + d.logger.Printf("[ERROR] driver.exec: %v", e) + } return nil, fmt.Errorf("error connecting to plugin: %v", err) } @@ -170,6 +178,7 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro pluginClient: client, executor: executor, userPid: id.UserPid, + groups: id.Groups, logger: d.logger, killTimeout: id.KillTimeout, doneCh: make(chan struct{}), @@ -184,6 +193,7 @@ func (h *execHandle) ID() string { KillTimeout: h.killTimeout, PluginConfig: NewExecutorReattachConfig(h.pluginClient.ReattachConfig()), UserPid: h.userPid, + Groups: h.groups, } data, err := json.Marshal(id) diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go index 8b7e419fb..99d1c6fe2 100644 --- a/client/driver/executor/executor.go +++ b/client/driver/executor/executor.go @@ -59,10 +59,11 @@ type ExecCommand struct { // ProcessState holds information about the state of a user process. type ProcessState struct { - Pid int - ExitCode int - Signal int - Time time.Time + Pid int + ExitCode int + Signal int + IsolationConfig cgroupConfig.Cgroup + Time time.Time } // Executor is the interface which allows a driver to launch and supervise @@ -153,7 +154,7 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext } go e.wait() - return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, Time: time.Now()}, nil + return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, IsolationConfig: *e.groups, Time: time.Now()}, nil } // Wait waits until a process has exited and returns it's exitcode and errors diff --git a/client/driver/java.go b/client/driver/java.go index 11daba07f..87e2750fe 100644 --- a/client/driver/java.go +++ b/client/driver/java.go @@ -14,6 +14,7 @@ import ( "github.com/hashicorp/go-plugin" "github.com/mitchellh/mapstructure" + cgroupConfig "github.com/opencontainers/runc/libcontainer/configs" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/executor" @@ -43,6 +44,7 @@ type javaHandle struct { pluginClient *plugin.Client userPid int executor executor.Executor + groups *cgroupConfig.Cgroup killTimeout time.Duration logger *log.Logger @@ -178,6 +180,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, pluginClient: pluginClient, executor: exec, userPid: ps.Pid, + groups: &ps.IsolationConfig, killTimeout: d.DriverContext.KillTimeout(task), logger: d.logger, doneCh: make(chan struct{}), @@ -191,6 +194,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, type javaId struct { KillTimeout time.Duration PluginConfig *ExecutorReattachConfig + Groups *cgroupConfig.Cgroup UserPid int } @@ -209,6 +213,10 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil { d.logger.Printf("[ERROR] driver.java: error destroying plugin and userpid: %v", e) } + if e := destroyCgroup(id.Groups); e != nil { + d.logger.Printf("[ERROR] driver.exec: %v", e) + } + return nil, fmt.Errorf("error connecting to plugin: %v", err) } @@ -217,6 +225,7 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro pluginClient: pluginClient, executor: executor, userPid: id.UserPid, + groups: id.Groups, logger: d.logger, killTimeout: id.KillTimeout, doneCh: make(chan struct{}), @@ -232,6 +241,7 @@ func (h *javaHandle) ID() string { KillTimeout: h.killTimeout, PluginConfig: NewExecutorReattachConfig(h.pluginClient.ReattachConfig()), UserPid: h.userPid, + Groups: h.groups, } data, err := json.Marshal(id) diff --git a/client/driver/utils_linux.go b/client/driver/utils_linux.go new file mode 100644 index 000000000..eb7022cd0 --- /dev/null +++ b/client/driver/utils_linux.go @@ -0,0 +1,40 @@ +package driver + +import ( + "os/exec" + "syscall" + + "fmt" + "github.com/opencontainers/runc/libcontainer/cgroups" + cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs" + "github.com/opencontainers/runc/libcontainer/cgroups/systemd" + cgroupConfig "github.com/opencontainers/runc/libcontainer/configs" +) + +// isolateCommand sets the setsid flag in exec.Cmd to true so that the process +// becomes the process leader in a new session and doesn't receive signals that +// are sent to the parent process. +func isolateCommand(cmd *exec.Cmd) { + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + cmd.SysProcAttr.Setsid = true +} + +// destroyCgroup destroys a cgroup and thereby killing all the processes in that +// group +func destroyCgroup(group *cgroupConfig.Cgroup) error { + if group == nil { + return nil + } + var manager cgroups.Manager + manager = &cgroupFs.Manager{Cgroups: group} + if systemd.UseSystemd() { + manager = &systemd.Manager{Cgroups: group} + } + + if err := manager.Destroy(); err != nil { + return fmt.Errorf("failed to destroy cgroup: %v", err) + } + return nil +} diff --git a/client/driver/utils_posix.go b/client/driver/utils_posix.go index 693bba6f6..cf90d109d 100644 --- a/client/driver/utils_posix.go +++ b/client/driver/utils_posix.go @@ -1,8 +1,12 @@ +// +build !linux + package driver import ( "os/exec" "syscall" + + cgroupConfig "github.com/opencontainers/runc/libcontainer/configs" ) // isolateCommand sets the setsid flag in exec.Cmd to true so that the process @@ -14,3 +18,7 @@ func isolateCommand(cmd *exec.Cmd) { } cmd.SysProcAttr.Setsid = true } + +func destroyCgroup(group *cgroupConfig.Cgroup) error { + return nil +} diff --git a/client/driver/utils_windows.go b/client/driver/utils_windows.go index 5b2b7d842..84aff1e5f 100644 --- a/client/driver/utils_windows.go +++ b/client/driver/utils_windows.go @@ -2,8 +2,14 @@ package driver import ( "os/exec" + + cgroupConfig "github.com/opencontainers/runc/libcontainer/configs" ) // TODO Figure out if this is needed in Wondows func isolateCommand(cmd *exec.Cmd) { } + +func destroyCgroup(group *cgroupConfig.Cgroup) error { + return nil +}