From 099e9de211a013c07ebe31abc31c5782a81ffdcd Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 24 Sep 2015 14:02:10 -0700 Subject: [PATCH] Fix cgroup isolation --- client/executor/exec_linux.go | 56 +++++++++++++++++++++++++++++++++-- command/spawn_daemon_linux.go | 34 +++++++++------------ 2 files changed, 67 insertions(+), 23 deletions(-) diff --git a/client/executor/exec_linux.go b/client/executor/exec_linux.go index 0adbdc4a1..eade51725 100644 --- a/client/executor/exec_linux.go +++ b/client/executor/exec_linux.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "os/exec" "os/user" @@ -103,7 +104,6 @@ func (e *LinuxExecutor) ConfigureTaskDir(taskName string, alloc *allocdir.AllocD // Mount dev dev := filepath.Join(taskDir, "dev") - fmt.Println("MOUNTED DEV: ", dev) if err := os.Mkdir(dev, 0777); err != nil { return fmt.Errorf("Mkdir(%v) failed: %v", dev) } @@ -245,7 +245,6 @@ func (e *LinuxExecutor) spawnDaemon() error { c := command.DaemonConfig{ Cmd: e.cmd.Cmd, - Groups: e.groups, Chroot: e.taskDir, StdoutFile: filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", e.taskName)), StderrFile: filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", e.taskName)), @@ -270,10 +269,44 @@ func (e *LinuxExecutor) spawnDaemon() error { spawn := exec.Command(bin, "spawn-daemon", escaped) spawn.Stdout = e.spawnOutputWriter + // Capture its Stdin. + spawnStdIn, err := spawn.StdinPipe() + if err != nil { + return err + } + if err := spawn.Start(); err != nil { fmt.Errorf("Failed to call spawn-daemon on nomad executable: %v", err) } + // Join the spawn-daemon to the cgroup. + if e.groups != nil { + manager := cgroupFs.Manager{} + manager.Cgroups = e.groups + + // Apply will place the current pid into the tasks file for each of the + // created cgroups: + // /sys/fs/cgroup/memory/user/1000.user/4.session//tasks + // + // Apply requires superuser permissions, and may fail if Nomad is not run with + // the required permissions + if err := manager.Apply(spawn.Process.Pid); err != nil { + errs := new(multierror.Error) + errs = multierror.Append(errs, fmt.Errorf("Failed to join spawn-daemon to the cgroup (config => %+v): %v", manager.Cgroups, err)) + + if err := sendAbortCommand(spawnStdIn); err != nil { + errs = multierror.Append(errs, err) + } + + return errs + } + } + + // Tell it to start. + if err := sendStartCommand(spawnStdIn); err != nil { + return err + } + // Parse the response. dec := json.NewDecoder(e.spawnOutputReader) var resp command.SpawnStartStatus @@ -289,6 +322,24 @@ func (e *LinuxExecutor) spawnDaemon() error { return nil } +func sendStartCommand(w io.Writer) error { + enc := json.NewEncoder(w) + if err := enc.Encode(true); err != nil { + return fmt.Errorf("Failed to serialize start command: %v", err) + } + + return nil +} + +func sendAbortCommand(w io.Writer) error { + enc := json.NewEncoder(w) + if err := enc.Encode(false); err != nil { + return fmt.Errorf("Failed to serialize abort command: %v", err) + } + + return nil +} + // Open's behavior is to kill all processes associated with the id and return an // error. This is done because it is not possible to re-attach to the // spawn-daemon's stdout to retrieve status messages. @@ -440,7 +491,6 @@ func (e *LinuxExecutor) destroyCgroup() error { errs := new(multierror.Error) for _, pid := range pids { - fmt.Println("PID: ", pid) process, err := os.FindProcess(pid) if err != nil { multierror.Append(errs, fmt.Errorf("Failed to find Pid %v: %v", pid, err)) diff --git a/command/spawn_daemon_linux.go b/command/spawn_daemon_linux.go index ded1fddb5..3e9ceaa3e 100644 --- a/command/spawn_daemon_linux.go +++ b/command/spawn_daemon_linux.go @@ -8,9 +8,6 @@ import ( "strconv" "strings" "syscall" - - cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs" - cgroupConfig "github.com/opencontainers/runc/libcontainer/configs" ) // Configuration for the command to start as a daemon. @@ -23,10 +20,12 @@ type DaemonConfig struct { StdinFile string StderrFile string - Groups *cgroupConfig.Cgroup Chroot string } +// Whether to start the user command or abort. +type TaskStart bool + func (c *SpawnDaemonCommand) Run(args []string) int { flags := c.Meta.FlagSet("spawn-daemon", FlagSetClient) flags.Usage = func() { c.Ui.Output(c.Help()) } @@ -53,22 +52,6 @@ func (c *SpawnDaemonCommand) Run(args []string) int { return c.outputStartStatus(err, 1) } - // Join this process to the cgroup. - if cmd.Groups != nil { - manager := cgroupFs.Manager{} - manager.Cgroups = cmd.Groups - - // Apply will place the current pid into the tasks file for each of the - // created cgroups: - // /sys/fs/cgroup/memory/user/1000.user/4.session//tasks - // - // Apply requires superuser permissions, and may fail if Nomad is not run with - // the required permissions - if err := manager.Apply(os.Getpid()); err != nil { - return c.outputStartStatus(fmt.Errorf("Failed to join cgroup (config => %v): %v", manager.Cgroups, err), 1) - } - } - // Isolate the user process. if _, err := syscall.Setsid(); err != nil { return c.outputStartStatus(fmt.Errorf("Failed setting sid: %v", err), 1) @@ -104,6 +87,17 @@ func (c *SpawnDaemonCommand) Run(args []string) int { cmd.Cmd.SysProcAttr.Chroot = cmd.Chroot cmd.Cmd.Dir = "/" + // Wait to get the start command. + var start TaskStart + dec = json.NewDecoder(os.Stdin) + if err := dec.Decode(&start); err != nil { + return c.outputStartStatus(err, 1) + } + + if !start { + return 0 + } + // Spawn the user process. if err := cmd.Cmd.Start(); err != nil { return c.outputStartStatus(fmt.Errorf("Error starting user command: %v", err), 1)