diff --git a/client/driver/exec.go b/client/driver/exec.go index bf519e252..ae60fc974 100644 --- a/client/driver/exec.go +++ b/client/driver/exec.go @@ -196,7 +196,7 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro merrs.Errors = append(merrs.Errors, fmt.Errorf("error destroying plugin and userpid: %v", e)) } if id.IsolationConfig != nil { - if e := executor.DestroyCgroup(id.IsolationConfig.Cgroup); e != nil { + if e := executor.DestroyCgroup(id.IsolationConfig.Cgroup, id.IsolationConfig.CgroupPaths); e != nil { merrs.Errors = append(merrs.Errors, fmt.Errorf("destroying cgroup failed: %v", e)) } } @@ -294,7 +294,7 @@ func (h *execHandle) run() { // user pid might be holding onto. if ps.ExitCode == 0 && err != nil { if h.isolationConfig != nil { - if e := executor.DestroyCgroup(h.isolationConfig.Cgroup); e != nil { + if e := executor.DestroyCgroup(h.isolationConfig.Cgroup, h.isolationConfig.CgroupPaths); e != nil { h.logger.Printf("[ERR] driver.exec: destroying cgroup failed while killing cgroup: %v", e) } } diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go index 32f47ad40..282d34232 100644 --- a/client/driver/executor/executor.go +++ b/client/driver/executor/executor.go @@ -156,8 +156,9 @@ type UniversalExecutor struct { syslogServer *logging.SyslogServer syslogChan chan *logging.SyslogMessage - groups *cgroupConfig.Cgroup - cgLock sync.Mutex + groups *cgroupConfig.Cgroup + cgPaths map[string]string + cgLock sync.Mutex consulService *consul.ConsulService consulCtx *ConsulContext @@ -242,8 +243,11 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext if err := e.cmd.Start(); err != nil { return nil, err } + if err := e.applyLimits(e.cmd.Process.Pid); err != nil { + return nil, err + } go e.wait() - ic := &cstructs.IsolationConfig{Cgroup: e.groups} + ic := &cstructs.IsolationConfig{Cgroup: e.groups, CgroupPaths: e.cgPaths} return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, IsolationConfig: ic, Time: time.Now()}, nil } @@ -371,7 +375,8 @@ func (e *UniversalExecutor) Exit() error { } if e.command != nil && e.command.ResourceLimits { e.cgLock.Lock() - if err := DestroyCgroup(e.groups); err != nil { + err := DestroyCgroup(e.groups, e.cgPaths) + if err != nil { merr.Errors = append(merr.Errors, err) } e.cgLock.Unlock() diff --git a/client/driver/executor/executor_basic.go b/client/driver/executor/executor_basic.go index 7d59ee366..18aef8d54 100644 --- a/client/driver/executor/executor_basic.go +++ b/client/driver/executor/executor_basic.go @@ -8,7 +8,7 @@ func (e *UniversalExecutor) configureChroot() error { return nil } -func DestroyCgroup(groups *cgroupConfig.Cgroup) error { +func DestroyCgroup(groups *cgroupConfig.Cgroup, paths map[string]string) error { return nil } diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go index d4805c067..c3969de00 100644 --- a/client/driver/executor/executor_linux.go +++ b/client/driver/executor/executor_linux.go @@ -45,15 +45,6 @@ func (e *UniversalExecutor) configureIsolation() error { if err := e.configureCgroups(e.ctx.Task.Resources); err != nil { return fmt.Errorf("error creating cgroups: %v", err) } - if err := e.applyLimits(os.Getpid()); err != nil { - if er := DestroyCgroup(e.groups); er != nil { - e.logger.Printf("[ERR] executor: error destroying cgroup: %v", er) - } - if er := e.removeChrootMounts(); er != nil { - e.logger.Printf("[ERR] executor: error removing chroot: %v", er) - } - return fmt.Errorf("error entering the plugin process in the cgroup: %v:", err) - } } return nil } @@ -65,15 +56,14 @@ func (e *UniversalExecutor) applyLimits(pid int) error { } // Entering the process in the cgroup - manager := getCgroupManager(e.groups) + manager := getCgroupManager(e.groups, nil) if err := manager.Apply(pid); err != nil { - e.logger.Printf("[ERR] executor: unable to join cgroup: %v", err) - if err := e.Exit(); err != nil { - e.logger.Printf("[ERR] executor: unable to kill process: %v", err) + if er := e.removeChrootMounts(); er != nil { + e.logger.Printf("[ERR] executor: error removing chroot: %v", er) } return err } - + e.cgPaths = manager.GetPaths() return nil } @@ -83,11 +73,7 @@ func (e *UniversalExecutor) configureCgroups(resources *structs.Resources) error e.groups = &cgroupConfig.Cgroup{} e.groups.Resources = &cgroupConfig.Resources{} cgroupName := structs.GenerateUUID() - cgPath, err := cgroups.GetThisCgroupDir("devices") - if err != nil { - return fmt.Errorf("unable to get mount point for devices sub-system: %v", err) - } - e.groups.Path = filepath.Join(cgPath, cgroupName) + e.groups.Path = filepath.Join("/nomad", cgroupName) // TODO: verify this is needed for things like network access e.groups.Resources.AllowAllDevices = true @@ -190,21 +176,15 @@ func (e *UniversalExecutor) removeChrootMounts() error { // destroyCgroup kills all processes in the cgroup and removes the cgroup // configuration from the host. -func DestroyCgroup(groups *cgroupConfig.Cgroup) error { +func DestroyCgroup(groups *cgroupConfig.Cgroup, cgPaths map[string]string) error { merrs := new(multierror.Error) if groups == nil { return fmt.Errorf("Can't destroy: cgroup configuration empty") } - manager := getCgroupManager(groups) + manager := getCgroupManager(groups, cgPaths) if pids, perr := manager.GetPids(); perr == nil { for _, pid := range pids { - // If the pid is the pid of the executor then we don't kill it, the - // executor is going to be killed by the driver once the Wait - // returns - if pid == os.Getpid() { - continue - } proc, err := os.FindProcess(pid) if err != nil { merrs.Errors = append(merrs.Errors, fmt.Errorf("error finding process %v: %v", pid, err)) @@ -222,19 +202,15 @@ func DestroyCgroup(groups *cgroupConfig.Cgroup) error { if err := manager.Destroy(); err != nil { multierror.Append(merrs, fmt.Errorf("Failed to delete the cgroup directories: %v", err)) } - - if len(merrs.Errors) != 0 { - return fmt.Errorf("errors while destroying cgroup: %v", merrs) - } - return nil + return merrs.ErrorOrNil() } // getCgroupManager returns the correct libcontainer cgroup manager. -func getCgroupManager(groups *cgroupConfig.Cgroup) cgroups.Manager { +func getCgroupManager(groups *cgroupConfig.Cgroup, paths map[string]string) cgroups.Manager { var manager cgroups.Manager - manager = &cgroupFs.Manager{Cgroups: groups} + manager = &cgroupFs.Manager{Cgroups: groups, Paths: paths} if systemd.UseSystemd() { - manager = &systemd.Manager{Cgroups: groups} + manager = &systemd.Manager{Cgroups: groups, Paths: paths} } return manager } diff --git a/client/driver/java.go b/client/driver/java.go index 7b436cd1b..13cd36011 100644 --- a/client/driver/java.go +++ b/client/driver/java.go @@ -260,7 +260,7 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro merrs.Errors = append(merrs.Errors, fmt.Errorf("error destroying plugin and userpid: %v", e)) } if id.IsolationConfig != nil { - if e := executor.DestroyCgroup(id.IsolationConfig.Cgroup); e != nil { + if e := executor.DestroyCgroup(id.IsolationConfig.Cgroup, id.IsolationConfig.CgroupPaths); e != nil { merrs.Errors = append(merrs.Errors, fmt.Errorf("destroying cgroup failed: %v", e)) } } @@ -357,7 +357,7 @@ func (h *javaHandle) run() { close(h.doneCh) if ps.ExitCode == 0 && err != nil { if h.isolationConfig != nil { - if e := executor.DestroyCgroup(h.isolationConfig.Cgroup); e != nil { + if e := executor.DestroyCgroup(h.isolationConfig.Cgroup, h.isolationConfig.CgroupPaths); e != nil { h.logger.Printf("[ERR] driver.java: destroying cgroup failed while killing cgroup: %v", e) } } else { diff --git a/client/driver/structs/structs.go b/client/driver/structs/structs.go index 60d4860eb..ecc738e76 100644 --- a/client/driver/structs/structs.go +++ b/client/driver/structs/structs.go @@ -42,7 +42,8 @@ func (r *WaitResult) String() string { // IsolationConfig has information about the isolation mechanism the executor // uses to put resource constraints and isolation on the user process type IsolationConfig struct { - Cgroup *cgroupConfig.Cgroup + Cgroup *cgroupConfig.Cgroup + CgroupPaths map[string]string } // RecoverableError wraps an error and marks whether it is recoverable and could