Creating the universal executor

This commit is contained in:
Diptanu Choudhury
2016-02-03 16:03:43 -08:00
parent 9b82079754
commit ed72a67ee0
4 changed files with 324 additions and 366 deletions

View File

@@ -1 +1,164 @@
package plugins
import (
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"runtime"
"sync"
"syscall"
"time"
cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/driver/env"
"github.com/hashicorp/nomad/nomad/structs"
)
type ExecutorContext struct {
TaskEnv *env.TaskEnvironment
AllocDir *allocdir.AllocDir
Task *structs.Task
Chroot bool
Limits bool
}
type ExecCommand struct {
Cmd string
Args []string
}
type ProcessState struct {
Pid int
ExitCode int
Time time.Time
}
type Executor interface {
LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error)
Wait() (*ProcessState, error)
ShutDown() error
Exit() error
}
type UniversalExecutor struct {
cmd exec.Cmd
ctx *ExecutorContext
taskDir string
groups *cgroupConfig.Cgroup
logger *log.Logger
lock sync.Mutex
}
func NewExecutor(logger *log.Logger) Executor {
return &UniversalExecutor{logger: logger}
}
func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
e.logger.Printf("LAUNCH COMMAND")
e.ctx = ctx
e.cmd.Path = command.Cmd
e.cmd.Args = append([]string{command.Cmd}, command.Args...)
if filepath.Base(command.Cmd) == command.Cmd {
if lp, err := exec.LookPath(command.Cmd); err != nil {
} else {
e.cmd.Path = lp
}
}
if err := e.configureTaskDir(); err != nil {
return nil, err
}
if err := e.configureIsolation(); err != nil {
return nil, err
}
if err := e.runAs("nobody"); err != nil {
return nil, err
}
stdoPath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", ctx.Task.Name))
stdo, err := os.OpenFile(stdoPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
if err != nil {
return nil, err
}
e.cmd.Stdout = stdo
stdePath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", ctx.Task.Name))
stde, err := os.OpenFile(stdePath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
if err != nil {
return nil, err
}
e.cmd.Stderr = stde
e.cmd.Env = ctx.TaskEnv.EnvList()
if err := e.cmd.Start(); err != nil {
return nil, fmt.Errorf("error starting command: %v", err)
}
e.applyLimits()
return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, Time: time.Now()}, nil
}
func (e *UniversalExecutor) Wait() (*ProcessState, error) {
err := e.cmd.Wait()
if err == nil {
return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
}
exitCode := 1
if exitErr, ok := err.(*exec.ExitError); ok {
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
exitCode = status.ExitStatus()
}
}
if e.ctx.Chroot {
e.removeChrootMounts()
}
if e.ctx.Limits {
e.destroyCgroup()
}
return &ProcessState{Pid: 0, ExitCode: exitCode, Time: time.Now()}, nil
}
func (e *UniversalExecutor) Exit() error {
e.logger.Printf("[INFO] Exiting plugin for task %q", e.ctx.Task.Name)
proc, err := os.FindProcess(e.cmd.Process.Pid)
if err != nil {
return fmt.Errorf("failied to find user process %v: %v", e.cmd.Process.Pid, err)
}
if e.ctx.Chroot {
e.removeChrootMounts()
}
if e.ctx.Limits {
e.destroyCgroup()
}
return proc.Kill()
}
func (e *UniversalExecutor) ShutDown() error {
proc, err := os.FindProcess(e.cmd.Process.Pid)
if err != nil {
return err
}
if runtime.GOOS == "windows" {
return proc.Kill()
}
return proc.Signal(os.Interrupt)
}
func (e *UniversalExecutor) configureTaskDir() error {
e.logger.Printf("DIPTANNUUUU CONDIFURE")
taskDir, ok := e.ctx.AllocDir.TaskDirs[e.ctx.Task.Name]
fmt.Printf("DIPTANU TASKDIR : %v", taskDir)
e.taskDir = taskDir
if !ok {
return fmt.Errorf("Couldn't find task directory for task %v", e.ctx.Task.Name)
}
e.cmd.Dir = taskDir
return nil
}

View File

@@ -2,103 +2,26 @@
package plugins
import (
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"runtime"
"syscall"
"time"
"github.com/hashicorp/nomad/client/allocdir"
)
type BasicExecutor struct {
logger *log.Logger
cmd exec.Cmd
taskDir string
}
func NewExecutor(logger *log.Logger) Executor {
return &BasicExecutor{logger: logger}
}
func (e *BasicExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
e.cmd.Path = command.Cmd
e.cmd.Args = append([]string{command.Cmd}, command.Args...)
e.cmd.Path = ctx.TaskEnv.ReplaceEnv(e.cmd.Path)
e.cmd.Args = ctx.TaskEnv.ParseAndReplace(e.cmd.Args)
if filepath.Base(command.Cmd) == command.Cmd {
if lp, err := exec.LookPath(command.Cmd); err != nil {
} else {
e.cmd.Path = lp
}
}
e.configureTaskDir(ctx.Task.Name, ctx.AllocDir)
e.cmd.Env = ctx.TaskEnv.EnvList()
stdoPath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", ctx.Task.Name))
stdo, err := os.OpenFile(stdoPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
if err != nil {
return nil, err
}
e.cmd.Stdout = stdo
stdePath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", ctx.Task.Name))
stde, err := os.OpenFile(stdePath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
if err != nil {
return nil, err
}
e.cmd.Stderr = stde
if err := e.cmd.Start(); err != nil {
return nil, err
}
return &ProcessState{Pid: 5, ExitCode: -1, Time: time.Now()}, nil
}
func (e *BasicExecutor) Wait() (*ProcessState, error) {
err := e.cmd.Wait()
if err == nil {
return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
}
exitCode := 1
if exitErr, ok := err.(*exec.ExitError); ok {
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
exitCode = status.ExitStatus()
}
}
return &ProcessState{Pid: 0, ExitCode: exitCode, Time: time.Now()}, nil
}
func (e *BasicExecutor) Exit() error {
proc, err := os.FindProcess(e.cmd.Process.Pid)
if err != nil {
return fmt.Errorf("failied to find user process %v: %v", e.cmd.Process.Pid, err)
}
return proc.Kill()
}
func (e *BasicExecutor) ShutDown() error {
proc, err := os.FindProcess(e.cmd.Process.Pid)
if err != nil {
return err
}
if runtime.GOOS == "windows" {
return proc.Kill()
}
return proc.Signal(os.Interrupt)
}
func (e *BasicExecutor) configureTaskDir(taskName string, allocDir *allocdir.AllocDir) error {
taskDir, ok := allocDir.TaskDirs[taskName]
e.taskDir = taskDir
if !ok {
return fmt.Errorf("Couldn't find task directory for task %v", taskName)
}
e.cmd.Dir = taskDir
func (e *UniversalExecutor) configureChroot() error {
return nil
}
func (e *UniversalExecutor) destroyCgroup() error {
return nil
}
func (e *UniversalExecutor) removeChrootMounts() error {
return nil
}
func (e *UniversalExecutor) runAs(userid string) error {
return nil
}
func (e *UniversalExecutor) applyLimits() error {
return nil
}
func (e *UniversalExecutor) configureIsolation() error {
return nil
}

View File

@@ -2,16 +2,11 @@ package plugins
import (
"fmt"
"log"
"os"
"os/exec"
"os/user"
"path/filepath"
"runtime"
"strconv"
"sync"
"syscall"
"time"
"github.com/hashicorp/go-multierror"
"github.com/opencontainers/runc/libcontainer/cgroups"
@@ -38,255 +33,39 @@ var (
}
)
type LinuxExecutor struct {
cmd exec.Cmd
ctx *ExecutorContext
groups *cgroupConfig.Cgroup
taskDir string
logger *log.Logger
lock sync.Mutex
}
func NewExecutor(logger *log.Logger) Executor {
return &LinuxExecutor{logger: logger}
}
func (e *LinuxExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext) (*ProcessState, error) {
e.ctx = ctx
e.cmd.Path = command.Cmd
e.cmd.Args = append([]string{command.Cmd}, command.Args...)
if filepath.Base(command.Cmd) == command.Cmd {
if lp, err := exec.LookPath(command.Cmd); err != nil {
} else {
e.cmd.Path = lp
func (e *UniversalExecutor) configureIsolation() error {
if e.ctx.Chroot {
if err := e.configureChroot(); err != nil {
return err
}
}
if err := e.configureTaskDir(); err != nil {
return nil, err
}
if err := e.runAs("nobody"); err != nil {
return nil, err
}
e.cmd.Env = ctx.TaskEnv.EnvList()
stdoPath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", ctx.Task.Name))
stdo, err := os.OpenFile(stdoPath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
if err != nil {
return nil, err
if e.ctx.Limits {
if err := e.configureCgroups(e.ctx.Task.Resources); err != nil {
return fmt.Errorf("error creating cgroups: %v", err)
}
}
e.cmd.Stdout = stdo
return nil
}
stdePath := filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", ctx.Task.Name))
stde, err := os.OpenFile(stdePath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
if err != nil {
return nil, err
func (e *UniversalExecutor) applyLimits() error {
if !e.ctx.Limits {
return nil
}
e.cmd.Stderr = stde
e.configureChroot()
if err := e.configureCgroups(e.ctx.Task.Resources); err != nil {
return nil, fmt.Errorf("error creating cgroups: %v", err)
}
if err := e.cmd.Start(); err != nil {
return nil, fmt.Errorf("error starting command: %v", err)
}
manager := e.getCgroupManager(e.groups)
if err := manager.Apply(e.cmd.Process.Pid); err != nil {
e.logger.Printf("[ERROR] unable to join cgroup: %v", err)
if err := e.Exit(); err != nil {
e.logger.Printf("[ERROR] unable to kill process: %v", err)
}
return nil, err
}
return &ProcessState{Pid: e.cmd.Process.Pid, ExitCode: -1, Time: time.Now()}, nil
}
// ConfigureTaskDir creates the necessary directory structure for a proper
// chroot. cleanTaskDir should be called after.
func (e *LinuxExecutor) configureTaskDir() error {
taskName := e.ctx.Task.Name
allocDir := e.ctx.AllocDir
taskDir, ok := allocDir.TaskDirs[taskName]
if !ok {
fmt.Errorf("Couldn't find task directory for task %v", taskName)
}
e.taskDir = taskDir
if err := allocDir.MountSharedDir(taskName); err != nil {
return err
}
if err := allocDir.Embed(taskName, chrootEnv); err != nil {
return err
}
// Mount dev
dev := filepath.Join(taskDir, "dev")
if !e.pathExists(dev) {
if err := os.Mkdir(dev, 0777); err != nil {
return fmt.Errorf("Mkdir(%v) failed: %v", dev, err)
}
if err := syscall.Mount("none", dev, "devtmpfs", syscall.MS_RDONLY, ""); err != nil {
return fmt.Errorf("Couldn't mount /dev to %v: %v", dev, err)
}
}
// Mount proc
proc := filepath.Join(taskDir, "proc")
if !e.pathExists(proc) {
if err := os.Mkdir(proc, 0777); err != nil {
return fmt.Errorf("Mkdir(%v) failed: %v", proc, err)
}
if err := syscall.Mount("none", proc, "proc", syscall.MS_RDONLY, ""); err != nil {
return fmt.Errorf("Couldn't mount /proc to %v: %v", proc, err)
}
}
// Set the tasks AllocDir environment variable.
e.ctx.TaskEnv.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)).SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)).Build()
return nil
}
// runAs takes a user id as a string and looks up the user, and sets the command
// to execute as that user.
func (e *LinuxExecutor) runAs(userid string) error {
u, err := user.Lookup(userid)
if err != nil {
return fmt.Errorf("Failed to identify user %v: %v", userid, err)
}
// Convert the uid and gid
uid, err := strconv.ParseUint(u.Uid, 10, 32)
if err != nil {
return fmt.Errorf("Unable to convert userid to uint32: %s", err)
}
gid, err := strconv.ParseUint(u.Gid, 10, 32)
if err != nil {
return fmt.Errorf("Unable to convert groupid to uint32: %s", err)
}
// Set the command to run as that user and group.
if e.cmd.SysProcAttr == nil {
e.cmd.SysProcAttr = &syscall.SysProcAttr{}
}
if e.cmd.SysProcAttr.Credential == nil {
e.cmd.SysProcAttr.Credential = &syscall.Credential{}
}
e.cmd.SysProcAttr.Credential.Uid = uint32(uid)
e.cmd.SysProcAttr.Credential.Gid = uint32(gid)
return nil
}
// pathExists is a helper function to check if the path exists.
func (e *LinuxExecutor) pathExists(path string) bool {
if _, err := os.Stat(path); err != nil {
if os.IsNotExist(err) {
return false
}
}
return true
}
// configureChroot enters the user command into a chroot if specified in the
// config and on an OS that supports Chroots.
func (e *LinuxExecutor) configureChroot() {
if !e.ctx.Chroot {
return
}
if e.cmd.SysProcAttr == nil {
e.cmd.SysProcAttr = &syscall.SysProcAttr{}
}
e.cmd.SysProcAttr.Chroot = e.taskDir
e.cmd.Dir = "/"
}
// cleanTaskDir is an idempotent operation to clean the task directory and
// should be called when tearing down the task.
func (e *LinuxExecutor) cleanTaskDir() error {
// Prevent a race between Wait/ForceStop
e.lock.Lock()
defer e.lock.Unlock()
// Unmount dev.
errs := new(multierror.Error)
dev := filepath.Join(e.taskDir, "dev")
if e.pathExists(dev) {
if err := syscall.Unmount(dev, 0); err != nil {
errs = multierror.Append(errs, fmt.Errorf("Failed to unmount dev (%v): %v", dev, err))
}
if err := os.RemoveAll(dev); err != nil {
errs = multierror.Append(errs, fmt.Errorf("Failed to delete dev directory (%v): %v", dev, err))
}
}
// Unmount
// proc.
proc := filepath.Join(e.taskDir, "proc")
if e.pathExists(proc) {
if err := syscall.Unmount(proc, 0); err != nil {
errs = multierror.Append(errs, fmt.Errorf("Failed to unmount proc (%v): %v", proc, err))
}
if err := os.RemoveAll(proc); err != nil {
errs = multierror.Append(errs, fmt.Errorf("Failed to delete proc directory (%v): %v", dev, err))
}
}
return errs.ErrorOrNil()
}
func (e *LinuxExecutor) Wait() (*ProcessState, error) {
err := e.cmd.Wait()
if err == nil {
return &ProcessState{Pid: 0, ExitCode: 0, Time: time.Now()}, nil
}
exitCode := 1
if exitErr, ok := err.(*exec.ExitError); ok {
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
exitCode = status.ExitStatus()
}
}
e.cleanTaskDir()
e.destroyCgroup()
return &ProcessState{Pid: 0, ExitCode: exitCode, Time: time.Now()}, nil
}
func (e *LinuxExecutor) Exit() error {
e.logger.Printf("[INFO] Exiting plugin for task %q", e.ctx.Task.Name)
proc, err := os.FindProcess(e.cmd.Process.Pid)
if err != nil {
return fmt.Errorf("failied to find user process %v: %v", e.cmd.Process.Pid, err)
}
e.cleanTaskDir()
e.destroyCgroup()
return proc.Kill()
}
func (e *LinuxExecutor) ShutDown() error {
proc, err := os.FindProcess(e.cmd.Process.Pid)
if err != nil {
return err
}
if runtime.GOOS == "windows" {
return proc.Kill()
}
return proc.Signal(os.Interrupt)
}
// configureCgroups converts a Nomad Resources specification into the equivalent
// cgroup configuration. It returns an error if the resources are invalid.
func (e *LinuxExecutor) configureCgroups(resources *structs.Resources) error {
func (e *UniversalExecutor) configureCgroups(resources *structs.Resources) error {
e.groups = &cgroupConfig.Cgroup{}
e.groups.Resources = &cgroupConfig.Resources{}
e.groups.Name = structs.GenerateUUID()
@@ -320,9 +99,133 @@ func (e *LinuxExecutor) configureCgroups(resources *structs.Resources) error {
return nil
}
// runAs takes a user id as a string and looks up the user, and sets the command
// to execute as that user.
func (e *UniversalExecutor) runAs(userid string) error {
u, err := user.Lookup(userid)
if err != nil {
return fmt.Errorf("Failed to identify user %v: %v", userid, err)
}
// Convert the uid and gid
uid, err := strconv.ParseUint(u.Uid, 10, 32)
if err != nil {
return fmt.Errorf("Unable to convert userid to uint32: %s", err)
}
gid, err := strconv.ParseUint(u.Gid, 10, 32)
if err != nil {
return fmt.Errorf("Unable to convert groupid to uint32: %s", err)
}
// Set the command to run as that user and group.
if e.cmd.SysProcAttr == nil {
e.cmd.SysProcAttr = &syscall.SysProcAttr{}
}
if e.cmd.SysProcAttr.Credential == nil {
e.cmd.SysProcAttr.Credential = &syscall.Credential{}
}
e.cmd.SysProcAttr.Credential.Uid = uint32(uid)
e.cmd.SysProcAttr.Credential.Gid = uint32(gid)
return nil
}
// pathExists is a helper function to check if the path exists.
func (e *UniversalExecutor) pathExists(path string) bool {
if _, err := os.Stat(path); err != nil {
if os.IsNotExist(err) {
return false
}
}
return true
}
func (e *UniversalExecutor) configureChroot() error {
allocDir := e.ctx.AllocDir
if err := allocDir.MountSharedDir(e.ctx.Task.Name); err != nil {
return err
}
if err := allocDir.Embed(e.ctx.Task.Name, chrootEnv); err != nil {
return err
}
// Mount dev
dev := filepath.Join(e.taskDir, "dev")
if !e.pathExists(dev) {
if err := os.Mkdir(dev, 0777); err != nil {
return fmt.Errorf("Mkdir(%v) failed: %v", dev, err)
}
if err := syscall.Mount("none", dev, "devtmpfs", syscall.MS_RDONLY, ""); err != nil {
return fmt.Errorf("Couldn't mount /dev to %v: %v", dev, err)
}
}
// Mount proc
proc := filepath.Join(e.taskDir, "proc")
if !e.pathExists(proc) {
if err := os.Mkdir(proc, 0777); err != nil {
return fmt.Errorf("Mkdir(%v) failed: %v", proc, err)
}
if err := syscall.Mount("none", proc, "proc", syscall.MS_RDONLY, ""); err != nil {
return fmt.Errorf("Couldn't mount /proc to %v: %v", proc, err)
}
}
// Set the tasks AllocDir environment variable.
e.ctx.TaskEnv.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)).SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)).Build()
if e.cmd.SysProcAttr == nil {
e.cmd.SysProcAttr = &syscall.SysProcAttr{}
}
e.cmd.SysProcAttr.Chroot = e.taskDir
e.cmd.Dir = "/"
return nil
}
// cleanTaskDir is an idempotent operation to clean the task directory and
// should be called when tearing down the task.
func (e *UniversalExecutor) removeChrootMounts() error {
// Prevent a race between Wait/ForceStop
e.lock.Lock()
defer e.lock.Unlock()
// Unmount dev.
errs := new(multierror.Error)
dev := filepath.Join(e.taskDir, "dev")
if e.pathExists(dev) {
if err := syscall.Unmount(dev, 0); err != nil {
errs = multierror.Append(errs, fmt.Errorf("Failed to unmount dev (%v): %v", dev, err))
}
if err := os.RemoveAll(dev); err != nil {
errs = multierror.Append(errs, fmt.Errorf("Failed to delete dev directory (%v): %v", dev, err))
}
}
// Unmount
// proc.
proc := filepath.Join(e.taskDir, "proc")
if e.pathExists(proc) {
if err := syscall.Unmount(proc, 0); err != nil {
errs = multierror.Append(errs, fmt.Errorf("Failed to unmount proc (%v): %v", proc, err))
}
if err := os.RemoveAll(proc); err != nil {
errs = multierror.Append(errs, fmt.Errorf("Failed to delete proc directory (%v): %v", dev, err))
}
}
return errs.ErrorOrNil()
}
// destroyCgroup kills all processes in the cgroup and removes the cgroup
// configuration from the host.
func (e *LinuxExecutor) destroyCgroup() error {
func (e *UniversalExecutor) destroyCgroup() error {
if e.groups == nil {
return fmt.Errorf("Can't destroy: cgroup configuration empty")
}
@@ -364,7 +267,7 @@ func (e *LinuxExecutor) destroyCgroup() error {
}
// getCgroupManager returns the correct libcontainer cgroup manager.
func (e *LinuxExecutor) getCgroupManager(groups *cgroupConfig.Cgroup) cgroups.Manager {
func (e *UniversalExecutor) getCgroupManager(groups *cgroupConfig.Cgroup) cgroups.Manager {
var manager cgroups.Manager
manager = &cgroupFs.Manager{Cgroups: groups}
if systemd.UseSystemd() {

View File

@@ -4,13 +4,8 @@ import (
"log"
"net/rpc"
"os"
"time"
"github.com/hashicorp/go-plugin"
"github.com/hashicorp/nomad/client/allocdir"
"github.com/hashicorp/nomad/client/driver/env"
"github.com/hashicorp/nomad/nomad/structs"
)
var HandshakeConfig = plugin.HandshakeConfig{
@@ -23,32 +18,6 @@ var PluginMap = map[string]plugin.Plugin{
"executor": new(ExecutorPlugin),
}
type ExecutorContext struct {
TaskEnv *env.TaskEnvironment
AllocDir *allocdir.AllocDir
Task *structs.Task
Chroot bool
Limits bool
}
type ExecCommand struct {
Cmd string
Args []string
}
type ProcessState struct {
Pid int
ExitCode int
Time time.Time
}
type Executor interface {
LaunchCmd(cmd *ExecCommand, ctx *ExecutorContext) (*ProcessState, error)
Wait() (*ProcessState, error)
ShutDown() error
Exit() error
}
type ExecutorRPC struct {
client *rpc.Client
}