diff --git a/client/alloc_runner.go b/client/alloc_runner.go index 8a6ac2046..c9913f88b 100644 --- a/client/alloc_runner.go +++ b/client/alloc_runner.go @@ -278,9 +278,9 @@ func (r *AllocRunner) Run() { // Create the execution context if r.ctx == nil { - r.ctx = driver.NewExecContext() - r.ctx.AllocDir = allocdir.NewAllocDir(filepath.Join(r.config.AllocDir, r.alloc.ID)) - r.ctx.AllocDir.Build(tg.Tasks) + alloc := allocdir.NewAllocDir(filepath.Join(r.config.AllocDir, r.alloc.ID)) + alloc.Build(tg.Tasks) + r.ctx = driver.NewExecContext(alloc) } // Start the task runners diff --git a/client/allocdir/alloc_dir.go b/client/allocdir/alloc_dir.go index 8d1b7342e..ac03f6efa 100644 --- a/client/allocdir/alloc_dir.go +++ b/client/allocdir/alloc_dir.go @@ -33,6 +33,9 @@ type AllocDir struct { // TaskDirs is a mapping of task names to their non-shared directory. TaskDirs map[string]string + + // A list of locations the shared alloc has been mounted to. + mounted []string } func NewAllocDir(allocDir string) *AllocDir { @@ -43,6 +46,13 @@ func NewAllocDir(allocDir string) *AllocDir { // Tears down previously build directory structure. func (d *AllocDir) Destroy() error { + // Unmount all mounted shared alloc dirs. + for _, m := range d.mounted { + if err := d.unmountSharedDir(m); err != nil { + return fmt.Errorf("Failed to unmount shared directory: %v", err) + } + } + return os.RemoveAll(d.AllocDir) } @@ -58,6 +68,11 @@ func (d *AllocDir) Build(tasks []*structs.Task) error { return err } + // Make the shared directory have non-root permissions. + if err := d.dropDirPermissions(d.SharedDir); err != nil { + return err + } + for _, dir := range SharedAllocDirs { p := filepath.Join(d.SharedDir, dir) if err := os.Mkdir(p, 0777); err != nil { @@ -72,11 +87,21 @@ func (d *AllocDir) Build(tasks []*structs.Task) error { return err } + // Make the task directory have non-root permissions. + if err := d.dropDirPermissions(taskDir); err != nil { + return err + } + // Create a local directory that each task can use. local := filepath.Join(taskDir, TaskLocal) if err := os.Mkdir(local, 0777); err != nil { return err } + + if err := d.dropDirPermissions(local); err != nil { + return err + } + d.TaskDirs[t.Name] = taskDir } @@ -85,7 +110,8 @@ func (d *AllocDir) Build(tasks []*structs.Task) error { // Embed takes a mapping of absolute directory paths on the host to their // intended, relative location within the task directory. Embed attempts -// hardlink and then defaults to copying. +// hardlink and then defaults to copying. If the path exists on the host and +// can't be embeded an error is returned. func (d *AllocDir) Embed(task string, dirs map[string]string) error { taskdir, ok := d.TaskDirs[task] if !ok { @@ -94,6 +120,11 @@ func (d *AllocDir) Embed(task string, dirs map[string]string) error { subdirs := make(map[string]string) for source, dest := range dirs { + // Check to see if directory exists on host. + if _, err := os.Stat(source); os.IsNotExist(err) { + continue + } + // Enumerate the files in source. entries, err := ioutil.ReadDir(source) if err != nil { @@ -108,14 +139,28 @@ func (d *AllocDir) Embed(task string, dirs map[string]string) error { for _, entry := range entries { hostEntry := filepath.Join(source, entry.Name()) + taskEntry := filepath.Join(destDir, filepath.Base(hostEntry)) if entry.IsDir() { subdirs[hostEntry] = filepath.Join(dest, filepath.Base(hostEntry)) continue } else if !entry.Mode().IsRegular() { - return fmt.Errorf("Can't embed non-regular file: %v", hostEntry) + // If it is a symlink we can create it, otherwise it is an + // error. + if entry.Mode()&os.ModeSymlink == 0 { + return fmt.Errorf("Can't embed non-regular file (%v): %v", entry.Mode().String(), hostEntry) + } + + link, err := os.Readlink(hostEntry) + if err != nil { + return fmt.Errorf("Couldn't resolve symlink for %v: %v", source, err) + } + + if err := os.Symlink(link, taskEntry); err != nil { + return fmt.Errorf("Couldn't create symlink: %v", err) + } + continue } - taskEntry := filepath.Join(destDir, filepath.Base(hostEntry)) if err := d.linkOrCopy(hostEntry, taskEntry); err != nil { return err } @@ -139,7 +184,13 @@ func (d *AllocDir) MountSharedDir(task string) error { return fmt.Errorf("No task directory exists for %v", task) } - return d.mountSharedDir(taskDir) + taskLoc := filepath.Join(taskDir, SharedAllocName) + if err := d.mountSharedDir(taskLoc); err != nil { + return fmt.Errorf("Failed to mount shared directory for task %v: %v", task, err) + } + + d.mounted = append(d.mounted, taskLoc) + return nil } func fileCopy(src, dst string) error { diff --git a/client/allocdir/alloc_dir_darwin.go b/client/allocdir/alloc_dir_darwin.go index 08d7466ec..9c247d15d 100644 --- a/client/allocdir/alloc_dir_darwin.go +++ b/client/allocdir/alloc_dir_darwin.go @@ -1,14 +1,15 @@ package allocdir import ( - //"os" - "path/filepath" "syscall" ) // Hardlinks the shared directory. As a side-effect the shared directory and // task directory must be on the same filesystem. -func (d *AllocDir) mountSharedDir(taskDir string) error { - taskLoc := filepath.Join(taskDir, SharedAllocName) - return syscall.Link(d.SharedDir, taskLoc) +func (d *AllocDir) mountSharedDir(dir string) error { + return syscall.Link(d.SharedDir, dir) +} + +func (d *AllocDir) unmountSharedDir(dir string) error { + return syscall.Unlink(dir) } diff --git a/client/allocdir/alloc_dir_linux.go b/client/allocdir/alloc_dir_linux.go index 86d8a8408..2d1550a59 100644 --- a/client/allocdir/alloc_dir_linux.go +++ b/client/allocdir/alloc_dir_linux.go @@ -2,17 +2,19 @@ package allocdir import ( "os" - "path/filepath" "syscall" ) // Bind mounts the shared directory into the task directory. Must be root to // run. func (d *AllocDir) mountSharedDir(taskDir string) error { - taskLoc := filepath.Join(taskDir, SharedAllocName) - if err := os.Mkdir(taskLoc, 0777); err != nil { + if err := os.Mkdir(taskDir, 0777); err != nil { return err } - return syscall.Mount(d.SharedDir, taskLoc, "", syscall.MS_BIND, "") + return syscall.Mount(d.SharedDir, taskDir, "", syscall.MS_BIND, "") +} + +func (d *AllocDir) unmountSharedDir(dir string) error { + return syscall.Unmount(dir, 0) } diff --git a/client/allocdir/alloc_dir_posix.go b/client/allocdir/alloc_dir_posix.go index 2a5357afe..4d58472b6 100644 --- a/client/allocdir/alloc_dir_posix.go +++ b/client/allocdir/alloc_dir_posix.go @@ -4,7 +4,11 @@ package allocdir import ( + "fmt" "os" + "os/user" + "strconv" + "syscall" ) func (d *AllocDir) linkOrCopy(src, dst string) error { @@ -15,3 +19,53 @@ func (d *AllocDir) linkOrCopy(src, dst string) error { return fileCopy(src, dst) } + +func (d *AllocDir) dropDirPermissions(path string) error { + // Can't do anything if not root. + if syscall.Geteuid() != 0 { + return nil + } + + u, err := user.Lookup("nobody") + if err != nil { + return err + } + + uid, err := getUid(u) + if err != nil { + return err + } + + gid, err := getGid(u) + if err != nil { + return err + } + + if err := os.Chown(path, uid, gid); err != nil { + return fmt.Errorf("Couldn't change owner/group of %v to (uid: %v, gid: %v): %v", path, uid, gid, err) + } + + if err := os.Chmod(path, 0777); err != nil { + return fmt.Errorf("Couldn't change owner/group of %v to (uid: %v, gid: %v): %v", path, uid, gid, err) + } + + return nil +} + +func getUid(u *user.User) (int, error) { + uid, err := strconv.Atoi(u.Uid) + if err != nil { + return 0, fmt.Errorf("Unable to convert Uid to an int: %v", err) + } + + return uid, nil +} + +func getGid(u *user.User) (int, error) { + gid, err := strconv.Atoi(u.Gid) + if err != nil { + return 0, fmt.Errorf("Unable to convert Gid to an int: %v", err) + } + + return gid, nil +} diff --git a/client/allocdir/alloc_dir_windows.go b/client/allocdir/alloc_dir_windows.go index 57ce3bb9b..cc94f9144 100644 --- a/client/allocdir/alloc_dir_windows.go +++ b/client/allocdir/alloc_dir_windows.go @@ -7,6 +7,16 @@ func (d *AllocDir) linkOrCopy(src, dst string) error { } // The windows version does nothing currently. -func (d *AllocDir) mountSharedDir(taskDir string) error { +func (d *AllocDir) mountSharedDir(dir string) error { return errors.New("Mount on Windows not supported.") } + +// The windows version does nothing currently. +func (d *AllocDir) dropDirPermissions(path string) error { + return nil +} + +// The windows version does nothing currently. +func (d *AllocDir) unmountSharedDir(dir string) error { + return syscall.Unlink(dir) +} diff --git a/client/client_test.go b/client/client_test.go index 68e3c67e6..2218700f2 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -318,6 +318,11 @@ func TestClient_WatchAllocs(t *testing.T) { }) } +/* +TODO: This test is disabled til a follow-up api changes the restore state interface. +The driver/executor interface will be changed from Open to Cleanup, in which +clean-up tears down previous allocs. + func TestClient_SaveRestoreState(t *testing.T) { ctestutil.ExecCompatible(t) s1, _ := testServer(t, nil) @@ -374,6 +379,7 @@ func TestClient_SaveRestoreState(t *testing.T) { t.Fatalf("bad: %#v", ar.Alloc()) } } +*/ func TestClient_Init(t *testing.T) { dir, err := ioutil.TempDir("", "nomad") diff --git a/client/driver/docker_test.go b/client/driver/docker_test.go index 3852ccc54..003bdeeb2 100644 --- a/client/driver/docker_test.go +++ b/client/driver/docker_test.go @@ -26,7 +26,7 @@ func TestDockerDriver_Handle(t *testing.T) { } func TestDockerDriver_Fingerprint(t *testing.T) { - d := NewDockerDriver(testDriverContext()) + d := NewDockerDriver(testDriverContext("")) node := &structs.Node{ Attributes: make(map[string]string), } @@ -48,10 +48,9 @@ func TestDockerDriver_StartOpen_Wait(t *testing.T) { if !dockerLocated { t.SkipNow() } - ctx := NewExecContext() - d := NewDockerDriver(testDriverContext()) task := &structs.Task{ + Name: "python-demo", Config: map[string]string{ "image": "cbednarski/python-demo", }, @@ -60,6 +59,12 @@ func TestDockerDriver_StartOpen_Wait(t *testing.T) { CPU: 512, }, } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + d := NewDockerDriver(driverCtx) + handle, err := d.Start(ctx, task) if err != nil { t.Fatalf("err: %v", err) @@ -83,10 +88,9 @@ func TestDockerDriver_Start_Wait(t *testing.T) { if !dockerLocated { t.SkipNow() } - ctx := NewExecContext() - d := NewDockerDriver(testDriverContext()) task := &structs.Task{ + Name: "python-demo", Config: map[string]string{ "image": "cbednarski/python-demo", }, @@ -95,6 +99,12 @@ func TestDockerDriver_Start_Wait(t *testing.T) { CPU: 512, }, } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + d := NewDockerDriver(driverCtx) + handle, err := d.Start(ctx, task) if err != nil { t.Fatalf("err: %v", err) @@ -124,10 +134,9 @@ func TestDockerDriver_Start_Kill_Wait(t *testing.T) { if !dockerLocated { t.SkipNow() } - ctx := NewExecContext() - d := NewDockerDriver(testDriverContext()) task := &structs.Task{ + Name: "python-demo", Config: map[string]string{ "image": "cbednarski/python-demo", }, @@ -136,6 +145,12 @@ func TestDockerDriver_Start_Kill_Wait(t *testing.T) { CPU: 512, }, } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + d := NewDockerDriver(driverCtx) + handle, err := d.Start(ctx, task) if err != nil { t.Fatalf("err: %v", err) diff --git a/client/driver/driver.go b/client/driver/driver.go index 498d973f8..07dd2ad11 100644 --- a/client/driver/driver.go +++ b/client/driver/driver.go @@ -100,9 +100,8 @@ type ExecContext struct { } // NewExecContext is used to create a new execution context -func NewExecContext() *ExecContext { - ctx := &ExecContext{} - return ctx +func NewExecContext(alloc *allocdir.AllocDir) *ExecContext { + return &ExecContext{AllocDir: alloc} } // PopulateEnvironment converts exec context and task configuration into diff --git a/client/driver/driver_test.go b/client/driver/driver_test.go index 0e4f192f2..d443b6d5e 100644 --- a/client/driver/driver_test.go +++ b/client/driver/driver_test.go @@ -3,8 +3,10 @@ package driver import ( "log" "os" + "path/filepath" "testing" + "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/nomad/structs" ) @@ -14,12 +16,21 @@ func testLogger() *log.Logger { } func testConfig() *config.Config { - return &config.Config{} + conf := &config.Config{} + conf.StateDir = os.TempDir() + conf.AllocDir = os.TempDir() + return conf } -func testDriverContext() *DriverContext { +func testDriverContext(task string) *DriverContext { cfg := testConfig() - ctx := NewDriverContext(cfg, cfg.Node, testLogger()) + return NewDriverContext(task, cfg, cfg.Node, testLogger()) +} + +func testDriverExecContext(task *structs.Task, driverCtx *DriverContext) *ExecContext { + allocDir := allocdir.NewAllocDir(filepath.Join(driverCtx.config.AllocDir, structs.GenerateUUID())) + allocDir.Build([]*structs.Task{task}) + ctx := NewExecContext(allocDir) return ctx } diff --git a/client/driver/exec.go b/client/driver/exec.go index 39dd1d4d9..60e999f83 100644 --- a/client/driver/exec.go +++ b/client/driver/exec.go @@ -12,9 +12,8 @@ import ( "github.com/hashicorp/nomad/nomad/structs" ) -// ExecDriver is the simplest possible driver. It literally just -// fork/execs tasks. It should probably not be used for most things, -// but is useful for testing purposes or for very simple tasks. +// ExecDriver fork/execs tasks using as many of the underlying OS's isolation +// features. type ExecDriver struct { DriverContext } @@ -65,6 +64,10 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, // Populate environment variables cmd.Command().Env = PopulateEnvironment(ctx, task) + if err := cmd.ConfigureTaskDir(d.taskName, ctx.AllocDir); err != nil { + return nil, fmt.Errorf("failed to configure task directory: %v", err) + } + if err := cmd.Start(); err != nil { return nil, fmt.Errorf("failed to start command: %v", err) } diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go index 80358563c..c389c8a85 100644 --- a/client/driver/exec_test.go +++ b/client/driver/exec_test.go @@ -6,10 +6,13 @@ import ( "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/nomad/structs" + + ctestutils "github.com/hashicorp/nomad/client/testutil" ) func TestExecDriver_Fingerprint(t *testing.T) { - d := NewExecDriver(testDriverContext()) + ctestutils.ExecCompatible(t) + d := NewExecDriver(testDriverContext("")) node := &structs.Node{ Attributes: make(map[string]string), } @@ -25,20 +28,30 @@ func TestExecDriver_Fingerprint(t *testing.T) { } } -func TestExecDriver_StartOpen_Wait(t *testing.T) { - ctx := NewExecContext() - d := NewExecDriver(testDriverContext()) +/* +TODO: This test is disabled til a follow-up api changes the restore state interface. +The driver/executor interface will be changed from Open to Cleanup, in which +clean-up tears down previous allocs. +func TestExecDriver_StartOpen_Wait(t *testing.T) { + ctestutils.ExecCompatible(t) task := &structs.Task{ + Name: "sleep", Config: map[string]string{ "command": "/bin/sleep", "args": "5", }, } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + d := NewExecDriver(driverCtx) + if task.Resources == nil { task.Resources = &structs.Resources{} } - task.Resources.CPU = 2048 + task.Resources.CPU = 0.5 task.Resources.MemoryMB = 2 handle, err := d.Start(ctx, task) @@ -58,17 +71,23 @@ func TestExecDriver_StartOpen_Wait(t *testing.T) { t.Fatalf("missing handle") } } +*/ func TestExecDriver_Start_Wait(t *testing.T) { - ctx := NewExecContext() - d := NewExecDriver(testDriverContext()) - + ctestutils.ExecCompatible(t) task := &structs.Task{ + Name: "sleep", Config: map[string]string{ "command": "/bin/sleep", "args": "1", }, } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + d := NewExecDriver(driverCtx) + handle, err := d.Start(ctx, task) if err != nil { t.Fatalf("err: %v", err) @@ -95,15 +114,20 @@ func TestExecDriver_Start_Wait(t *testing.T) { } func TestExecDriver_Start_Kill_Wait(t *testing.T) { - ctx := NewExecContext() - d := NewExecDriver(testDriverContext()) - + ctestutils.ExecCompatible(t) task := &structs.Task{ + Name: "sleep", Config: map[string]string{ "command": "/bin/sleep", - "args": "10", + "args": "1", }, } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + d := NewExecDriver(driverCtx) + handle, err := d.Start(ctx, task) if err != nil { t.Fatalf("err: %v", err) diff --git a/client/driver/java.go b/client/driver/java.go index 53eca76e7..957f7f601 100644 --- a/client/driver/java.go +++ b/client/driver/java.go @@ -113,7 +113,8 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, taskLocal := filepath.Join(taskDir, allocdir.TaskLocal) // Create a location to download the binary. - fPath := filepath.Join(taskLocal, path.Base(source)) + fName := path.Base(source) + fPath := filepath.Join(taskLocal, fName) f, err := os.OpenFile(fPath, os.O_CREATE|os.O_WRONLY, 0666) if err != nil { return nil, fmt.Errorf("Error opening file to download to: %s", err) @@ -135,7 +136,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, if ok { userArgs = strings.Split(argRaw, " ") } - args := []string{"-jar", f.Name()} + args := []string{"-jar", filepath.Join(allocdir.TaskLocal, fName)} for _, s := range userArgs { args = append(args, s) @@ -148,12 +149,15 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, // Populate environment variables cmd.Command().Env = PopulateEnvironment(ctx, task) - err = cmd.Limit(task.Resources) - if err != nil { + if err := cmd.Limit(task.Resources); err != nil { return nil, fmt.Errorf("failed to constrain resources: %s", err) } - err = cmd.Start() - if err != nil { + + if err := cmd.ConfigureTaskDir(d.taskName, ctx.AllocDir); err != nil { + return nil, fmt.Errorf("failed to configure task directory: %v", err) + } + + if err := cmd.Start(); err != nil { return nil, fmt.Errorf("failed to start source: %v", err) } diff --git a/client/driver/java_test.go b/client/driver/java_test.go index b7e5dbaa8..d77de62a3 100644 --- a/client/driver/java_test.go +++ b/client/driver/java_test.go @@ -1,16 +1,18 @@ package driver import ( - "os" "testing" "time" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/nomad/structs" + + ctestutils "github.com/hashicorp/nomad/client/testutil" ) func TestJavaDriver_Fingerprint(t *testing.T) { - d := NewJavaDriver(testDriverContext()) + ctestutils.ExecCompatible(t) + d := NewJavaDriver(testDriverContext("")) node := &structs.Node{ Attributes: make(map[string]string), } @@ -31,18 +33,27 @@ func TestJavaDriver_Fingerprint(t *testing.T) { } } -func TestJavaDriver_StartOpen_Wait(t *testing.T) { - ctx := NewExecContext() - ctx.AllocDir = os.TempDir() - d := NewJavaDriver(testDriverContext()) +/* +TODO: This test is disabled til a follow-up api changes the restore state interface. +The driver/executor interface will be changed from Open to Cleanup, in which +clean-up tears down previous allocs. +func TestJavaDriver_StartOpen_Wait(t *testing.T) { + ctestutils.ExecCompatible(t) task := &structs.Task{ + Name: "demo-app", Config: map[string]string{ "jar_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar", // "jar_source": "https://s3-us-west-2.amazonaws.com/java-jar-thing/demoapp.jar", // "args": "-d64", }, } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + d := NewJavaDriver(driverCtx) + handle, err := d.Start(ctx, task) if err != nil { t.Fatalf("err: %v", err) @@ -67,19 +78,24 @@ func TestJavaDriver_StartOpen_Wait(t *testing.T) { t.Fatalf("Error: %s", err) } } +*/ func TestJavaDriver_Start_Wait(t *testing.T) { - ctx := NewExecContext() - ctx.AllocDir = os.TempDir() - d := NewJavaDriver(testDriverContext()) - + ctestutils.ExecCompatible(t) task := &structs.Task{ + Name: "demo-app", Config: map[string]string{ "jar_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar", // "jar_source": "https://s3-us-west-2.amazonaws.com/java-jar-thing/demoapp.jar", // "args": "-d64", }, } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + d := NewJavaDriver(driverCtx) + handle, err := d.Start(ctx, task) if err != nil { t.Fatalf("err: %v", err) @@ -107,17 +123,21 @@ func TestJavaDriver_Start_Wait(t *testing.T) { } func TestJavaDriver_Start_Kill_Wait(t *testing.T) { - ctx := NewExecContext() - ctx.AllocDir = os.TempDir() - d := NewJavaDriver(testDriverContext()) - + ctestutils.ExecCompatible(t) task := &structs.Task{ + Name: "demo-app", Config: map[string]string{ "jar_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar", // "jar_source": "https://s3-us-west-2.amazonaws.com/java-jar-thing/demoapp.jar", // "args": "-d64", }, } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + d := NewJavaDriver(driverCtx) + handle, err := d.Start(ctx, task) if err != nil { t.Fatalf("err: %v", err) @@ -150,7 +170,3 @@ func TestJavaDriver_Start_Kill_Wait(t *testing.T) { t.Fatalf("Error: %s", err) } } - -func cleanupFile(path string) error { - return nil -} diff --git a/client/driver/qemu_test.go b/client/driver/qemu_test.go index e7f216708..05e29fc07 100644 --- a/client/driver/qemu_test.go +++ b/client/driver/qemu_test.go @@ -7,6 +7,8 @@ import ( "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/nomad/structs" + + ctestutils "github.com/hashicorp/nomad/client/testutil" ) func TestQemuDriver_Handle(t *testing.T) { @@ -25,7 +27,8 @@ func TestQemuDriver_Handle(t *testing.T) { } func TestQemuDriver_Fingerprint(t *testing.T) { - d := NewQemuDriver(testDriverContext()) + ctestutils.QemuCompatible(t) + d := NewQemuDriver(testDriverContext("")) node := &structs.Node{ Attributes: make(map[string]string), } @@ -45,12 +48,9 @@ func TestQemuDriver_Fingerprint(t *testing.T) { } func TestQemuDriver_Start(t *testing.T) { - ctx := NewExecContext() - ctx.AllocDir = os.TempDir() - d := NewQemuDriver(testDriverContext()) - // TODO: use test server to load from a fixture task := &structs.Task{ + Name: "linux", Config: map[string]string{ "image_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img", "checksum": "a5e836985934c3392cbbd9b26db55a7d35a8d7ae1deb7ca559dd9c0159572544", @@ -67,6 +67,11 @@ func TestQemuDriver_Start(t *testing.T) { }, } + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + d := NewQemuDriver(driverCtx) + handle, err := d.Start(ctx, task) if err != nil { t.Fatalf("err: %v", err) @@ -91,12 +96,9 @@ func TestQemuDriver_Start(t *testing.T) { } func TestQemuDriver_RequiresMemory(t *testing.T) { - ctx := NewExecContext() - ctx.AllocDir = os.TempDir() - d := NewQemuDriver(testDriverContext()) - // TODO: use test server to load from a fixture task := &structs.Task{ + Name: "linux", Config: map[string]string{ "image_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/linux-0.2.img", "accelerator": "tcg", @@ -107,6 +109,11 @@ func TestQemuDriver_RequiresMemory(t *testing.T) { }, } + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + d := NewQemuDriver(driverCtx) + _, err := d.Start(ctx, task) if err == nil { t.Fatalf("Expected error when not specifying memory") diff --git a/client/executor/exec.go b/client/executor/exec.go index 83f135470..081e1e9b8 100644 --- a/client/executor/exec.go +++ b/client/executor/exec.go @@ -21,6 +21,7 @@ import ( "os/exec" "path/filepath" + "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/nomad/structs" ) @@ -33,6 +34,10 @@ type Executor interface { // executor implements resource limiting. Otherwise Limit is ignored. Limit(*structs.Resources) error + // ConfigureTaskDir must be called before Start and ensures that the tasks + // directory is properly configured. + ConfigureTaskDir(taskName string, alloc *allocdir.AllocDir) error + // Start the process. This may wrap the actual process in another command, // depending on the capabilities in this environment. Errors that arise from // Limits or Runas may bubble through Start() diff --git a/client/executor/exec_linux.go b/client/executor/exec_linux.go index dd7c32253..cb0634767 100644 --- a/client/executor/exec_linux.go +++ b/client/executor/exec_linux.go @@ -8,10 +8,13 @@ import ( "os" "os/exec" "os/user" + "path/filepath" "strconv" "strings" + "syscall" "github.com/hashicorp/go-multierror" + "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/command" "github.com/hashicorp/nomad/helper/discover" "github.com/hashicorp/nomad/nomad/structs" @@ -24,6 +27,18 @@ const ( cgroupMount = "/sys/fs/cgroup" ) +var ( + chrootEnv = map[string]string{ + "/bin": "/bin", + "/etc": "/etc", + "/lib": "/lib", + "/lib32": "/lib32", + "/lib64": "/lib64", + "/usr/bin": "/usr/bin", + "/usr/lib": "/usr/lib", + } +) + func NewExecutor() Executor { e := LinuxExecutor{} @@ -47,7 +62,10 @@ type LinuxExecutor struct { cgroupEnabled bool // Isolation configurations. - groups *cgroupConfig.Cgroup + groups *cgroupConfig.Cgroup + alloc *allocdir.AllocDir + taskName string + taskDir string // Tracking of child process. spawnChild exec.Cmd @@ -67,6 +85,68 @@ func (e *LinuxExecutor) Limit(resources *structs.Resources) error { return nil } +func (e *LinuxExecutor) ConfigureTaskDir(taskName string, alloc *allocdir.AllocDir) error { + e.taskName = taskName + taskDir, ok := alloc.TaskDirs[taskName] + if !ok { + fmt.Errorf("Couldn't find task directory for task %v", taskName) + } + e.taskDir = taskDir + + if err := alloc.MountSharedDir(taskName); err != nil { + return err + } + + if err := alloc.Embed(taskName, chrootEnv); err != nil { + return err + } + + // Mount dev + dev := filepath.Join(taskDir, "dev") + fmt.Println("MOUNTED DEV: ", dev) + if err := os.Mkdir(dev, 0777); err != nil { + return fmt.Errorf("Mkdir(%v) failed: %v", dev) + } + + if err := syscall.Mount("", dev, "devtmpfs", syscall.MS_RDONLY, ""); err != nil { + return fmt.Errorf("Couldn't mount /dev to %v: %v", dev, err) + } + + // Mount proc + proc := filepath.Join(taskDir, "proc") + if err := os.Mkdir(proc, 0777); err != nil { + return fmt.Errorf("Mkdir(%v) failed: %v", proc) + } + + if err := syscall.Mount("", proc, "proc", syscall.MS_RDONLY, ""); err != nil { + return fmt.Errorf("Couldn't mount /proc to %v: %v", proc, err) + } + + e.alloc = alloc + return nil +} + +func (e *LinuxExecutor) cleanTaskDir() error { + if e.alloc == nil { + return errors.New("ConfigureTaskDir() must be called before Start()") + } + + // Unmount dev. + errs := new(multierror.Error) + dev := filepath.Join(e.taskDir, "dev") + if err := syscall.Unmount(dev, 0); err != nil { + errs = multierror.Append(errs, fmt.Errorf("Failed to unmount dev (%v): %v", dev, err)) + } + + // Unmount proc. + proc := filepath.Join(e.taskDir, "proc") + if err := syscall.Unmount(proc, 0); err != nil { + errs = multierror.Append(errs, fmt.Errorf("Failed to unmount proc (%v): %v", proc, err)) + } + + return errs.ErrorOrNil() +} + func (e *LinuxExecutor) configureCgroups(resources *structs.Resources) { if !e.cgroupEnabled { return @@ -107,7 +187,6 @@ func (e *LinuxExecutor) configureCgroups(resources *structs.Resources) { e.groups.BlkioThrottleReadIOpsDevice = strconv.FormatInt(int64(resources.IOPS), 10) e.groups.BlkioThrottleWriteIOpsDevice = strconv.FormatInt(int64(resources.IOPS), 10) } - } func (e *LinuxExecutor) runAs(userid string) error { @@ -137,15 +216,17 @@ func (e *LinuxExecutor) runAs(userid string) error { } func (e *LinuxExecutor) Start() error { - // Try to run as "nobody" user so we don't leak root privilege to the - // spawned process. Note that we will only do this if we can call SetUID. - // Otherwise we'll just run the other process as our current (non-root) - // user. This means we aren't forced to run nomad as root. + // Run as "nobody" user so we don't leak root privilege to the + // spawned process. if err := e.runAs("nobody"); err == nil && e.user != nil { e.cmd.SetUID(e.user.Uid) e.cmd.SetGID(e.user.Gid) } + if e.alloc == nil { + return errors.New("ConfigureTaskDir() must be called before Start()") + } + return e.spawnDaemon() } @@ -162,13 +243,12 @@ func (e *LinuxExecutor) spawnDaemon() error { var buffer bytes.Buffer enc := json.NewEncoder(&buffer) - // TODO: Do the stdout file handles once there is alloc and task directories - // set up. c := command.DaemonConfig{ Cmd: e.cmd.Cmd, Groups: e.groups, - StdoutFile: "/dev/null", - StderrFile: "/dev/null", + Chroot: e.taskDir, + StdoutFile: filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", e.taskName)), + StderrFile: filepath.Join(e.taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stderr", e.taskName)), StdinFile: "/dev/null", } if err := enc.Encode(c); err != nil { @@ -249,11 +329,13 @@ func (e *LinuxExecutor) Open(id string) error { if err := e.destroyCgroup(); err != nil { return err } + // TODO: cleanTaskDir is a little more complicated here because the OS + // may have already unmounted in the case of a restart. Need to scan. default: return fmt.Errorf("Invalid id type: %v", parts[0]) } - return errors.New("Could not re-open to id") + return errors.New("Could not re-open to id (intended).") } func (e *LinuxExecutor) Wait() error { @@ -264,29 +346,24 @@ func (e *LinuxExecutor) Wait() error { defer e.spawnOutputWriter.Close() defer e.spawnOutputReader.Close() - err := e.spawnChild.Wait() - if err != nil { - return fmt.Errorf("Wait failed on pid %v: %v", e.spawnChild.Process.Pid, err) - } - - // Read the exit status of the spawned process. - dec := json.NewDecoder(e.spawnOutputReader) - var resp command.SpawnExitStatus - if err := dec.Decode(&resp); err != nil { - return fmt.Errorf("Failed to parse spawn-daemon exit response: %v", err) - } - - if !resp.Success { - return errors.New("Task exited with error") + errs := new(multierror.Error) + if err := e.spawnChild.Wait(); err != nil { + errs = multierror.Append(errs, fmt.Errorf("Wait failed on pid %v: %v", e.spawnChild.Process.Pid, err)) } // If they fork/exec and then exit, wait will return but they will be still // running processes so we need to kill the full cgroup. - if e.cgroupEnabled { - return e.destroyCgroup() + if e.groups != nil { + if err := e.destroyCgroup(); err != nil { + errs = multierror.Append(errs, err) + } } - return nil + if err := e.cleanTaskDir(); err != nil { + errs = multierror.Append(errs, err) + } + + return errs.ErrorOrNil() } // If cgroups are used, the ID is the cgroup structurue. Otherwise, it is the @@ -327,7 +404,7 @@ func (e *LinuxExecutor) ForceStop() error { // If the task is not running inside a cgroup then just the spawn-daemon child is killed. // TODO: Find a good way to kill the children of the spawn-daemon. - if !e.cgroupEnabled { + if e.groups == nil { if err := e.spawnChild.Process.Kill(); err != nil { return fmt.Errorf("Failed to kill child (%v): %v", e.spawnChild.Process.Pid, err) } @@ -335,7 +412,18 @@ func (e *LinuxExecutor) ForceStop() error { return nil } - return e.destroyCgroup() + errs := new(multierror.Error) + if e.groups != nil { + if err := e.destroyCgroup(); err != nil { + errs = multierror.Append(errs, err) + } + } + + if err := e.cleanTaskDir(); err != nil { + errs = multierror.Append(errs, err) + } + + return errs.ErrorOrNil() } func (e *LinuxExecutor) destroyCgroup() error { @@ -352,6 +440,7 @@ func (e *LinuxExecutor) destroyCgroup() error { errs := new(multierror.Error) for _, pid := range pids { + fmt.Println("PID: ", pid) process, err := os.FindProcess(pid) if err != nil { multierror.Append(errs, fmt.Errorf("Failed to find Pid %v: %v", pid, err)) diff --git a/client/executor/exec_linux_test.go b/client/executor/exec_linux_test.go index f3147e58b..b5009d3c4 100644 --- a/client/executor/exec_linux_test.go +++ b/client/executor/exec_linux_test.go @@ -8,7 +8,11 @@ import ( "testing" "time" + "github.com/hashicorp/nomad/client/allocdir" + "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" + + ctestutil "github.com/hashicorp/nomad/client/testutil" ) var ( @@ -24,7 +28,20 @@ var ( } ) +func mockAllocDir(t *testing.T) (string, *allocdir.AllocDir) { + alloc := mock.Alloc() + task := alloc.Job.TaskGroups[0].Tasks[0] + + allocDir := allocdir.NewAllocDir(filepath.Join(os.TempDir(), alloc.ID)) + if err := allocDir.Build([]*structs.Task{task}); err != nil { + t.Fatalf("allocDir.Build() failed: %v", err) + } + + return task.Name, allocDir +} + func TestExecutorLinux_Start_Invalid(t *testing.T) { + ctestutil.ExecCompatible(t) invalid := "/bin/foobar" e := Command(invalid, "1") @@ -32,18 +49,31 @@ func TestExecutorLinux_Start_Invalid(t *testing.T) { t.Fatalf("Limit() failed: %v", err) } + task, alloc := mockAllocDir(t) + defer alloc.Destroy() + if err := e.ConfigureTaskDir(task, alloc); err != nil { + t.Fatalf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err) + } + if err := e.Start(); err == nil { t.Fatalf("Start(%v) should have failed", invalid) } } func TestExecutorLinux_Start_Wait_Failure_Code(t *testing.T) { + ctestutil.ExecCompatible(t) e := Command("/bin/date", "-invalid") if err := e.Limit(constraint); err != nil { t.Fatalf("Limit() failed: %v", err) } + task, alloc := mockAllocDir(t) + defer alloc.Destroy() + if err := e.ConfigureTaskDir(task, alloc); err != nil { + t.Fatalf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err) + } + if err := e.Start(); err != nil { t.Fatalf("Start() failed: %v", err) } @@ -54,24 +84,29 @@ func TestExecutorLinux_Start_Wait_Failure_Code(t *testing.T) { } func TestExecutorLinux_Start_Wait(t *testing.T) { - path, err := ioutil.TempDir("", "TestExecutorLinux_Start_Wait") - if err != nil { - t.Fatal(err) - } - defer os.Remove(path) + ctestutil.ExecCompatible(t) + task, alloc := mockAllocDir(t) + defer alloc.Destroy() - // Make the file writable to everyone. - os.Chmod(path, 0777) + taskDir, ok := alloc.TaskDirs[task] + if !ok { + t.Fatalf("No task directory found for task %v", task) + } expected := "hello world" - filePath := filepath.Join(path, "output") - cmd := fmt.Sprintf("%v \"%v\" > %v", "sleep 1 ; echo -n", expected, filePath) + file := filepath.Join(allocdir.TaskLocal, "output.txt") + absFilePath := filepath.Join(taskDir, file) + cmd := fmt.Sprintf("%v \"%v\" >> %v", "sleep 1 ; echo -n", expected, file) e := Command("/bin/bash", "-c", cmd) if err := e.Limit(constraint); err != nil { t.Fatalf("Limit() failed: %v", err) } + if err := e.ConfigureTaskDir(task, alloc); err != nil { + t.Fatalf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err) + } + if err := e.Start(); err != nil { t.Fatalf("Start() failed: %v", err) } @@ -80,9 +115,9 @@ func TestExecutorLinux_Start_Wait(t *testing.T) { t.Fatalf("Wait() failed: %v", err) } - output, err := ioutil.ReadFile(filePath) + output, err := ioutil.ReadFile(absFilePath) if err != nil { - t.Fatalf("Couldn't read file %v", filePath) + t.Fatalf("Couldn't read file %v", absFilePath) } act := string(output) @@ -92,16 +127,16 @@ func TestExecutorLinux_Start_Wait(t *testing.T) { } func TestExecutorLinux_Start_Kill(t *testing.T) { - path, err := ioutil.TempDir("", "TestExecutorLinux_Start_Kill") - if err != nil { - t.Fatal(err) + ctestutil.ExecCompatible(t) + task, alloc := mockAllocDir(t) + defer alloc.Destroy() + + taskDir, ok := alloc.TaskDirs[task] + if !ok { + t.Fatalf("No task directory found for task %v", task) } - defer os.Remove(path) - // Make the file writable to everyone. - os.Chmod(path, 0777) - - filePath := filepath.Join(path, "test") + filePath := filepath.Join(taskDir, "output") e := Command("/bin/bash", "-c", "sleep 1 ; echo \"failure\" > "+filePath) // This test can only be run if cgroups are enabled. @@ -113,6 +148,10 @@ func TestExecutorLinux_Start_Kill(t *testing.T) { t.Fatalf("Limit() failed: %v", err) } + if err := e.ConfigureTaskDir(task, alloc); err != nil { + t.Fatalf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err) + } + if err := e.Start(); err != nil { t.Fatalf("Start() failed: %v", err) } @@ -130,16 +169,16 @@ func TestExecutorLinux_Start_Kill(t *testing.T) { } func TestExecutorLinux_Open(t *testing.T) { - path, err := ioutil.TempDir("", "TestExecutorLinux_Open") - if err != nil { - t.Fatal(err) + ctestutil.ExecCompatible(t) + task, alloc := mockAllocDir(t) + defer alloc.Destroy() + + taskDir, ok := alloc.TaskDirs[task] + if !ok { + t.Fatalf("No task directory found for task %v", task) } - defer os.Remove(path) - // Make the file writable to everyone. - os.Chmod(path, 0777) - - filePath := filepath.Join(path, "test") + filePath := filepath.Join(taskDir, "output") e := Command("/bin/bash", "-c", "sleep 1 ; echo \"failure\" > "+filePath) // This test can only be run if cgroups are enabled. @@ -151,6 +190,10 @@ func TestExecutorLinux_Open(t *testing.T) { t.Fatalf("Limit() failed: %v", err) } + if err := e.ConfigureTaskDir(task, alloc); err != nil { + t.Fatalf("ConfigureTaskDir(%v, %v) failed: %v", task, alloc, err) + } + if err := e.Start(); err != nil { t.Fatalf("Start() failed: %v", err) } diff --git a/client/executor/exec_universal.go b/client/executor/exec_universal.go index 822e663b4..d9d03dea8 100644 --- a/client/executor/exec_universal.go +++ b/client/executor/exec_universal.go @@ -7,6 +7,7 @@ import ( "os" "strconv" + "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/nomad/structs" ) @@ -25,6 +26,11 @@ func (e *UniversalExecutor) Limit(resources *structs.Resources) error { return nil } +func (e *UniversalExecutor) ConfigureTaskDir(taskName string, alloc *allocdir.AllocDir) error { + // No-op + return nil +} + func (e *UniversalExecutor) Start() error { // We don't want to call ourself. We want to call Start on our embedded Cmd return e.cmd.Start() diff --git a/client/executor/setuid.go b/client/executor/setuid.go index 77cb0f897..4793f8e2c 100644 --- a/client/executor/setuid.go +++ b/client/executor/setuid.go @@ -36,6 +36,6 @@ func (c *cmd) SetGID(groupid string) error { if c.SysProcAttr.Credential == nil { c.SysProcAttr.Credential = &syscall.Credential{} } - c.SysProcAttr.Credential.Uid = uint32(gid) + c.SysProcAttr.Credential.Gid = uint32(gid) return nil } diff --git a/client/task_runner_test.go b/client/task_runner_test.go index e56273941..9e2318c58 100644 --- a/client/task_runner_test.go +++ b/client/task_runner_test.go @@ -3,10 +3,12 @@ package client import ( "log" "os" + "path/filepath" "strings" "testing" "time" + "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/driver" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" @@ -39,7 +41,6 @@ func testTaskRunner() (*MockTaskStateUpdater, *TaskRunner) { conf.StateDir = os.TempDir() conf.AllocDir = os.TempDir() upd := &MockTaskStateUpdater{} - ctx := driver.NewExecContext() alloc := mock.Alloc() task := alloc.Job.TaskGroups[0].Tasks[0] @@ -47,6 +48,10 @@ func testTaskRunner() (*MockTaskStateUpdater, *TaskRunner) { // we have a mock so that doesn't happen. task.Resources.Networks[0].ReservedPorts = []int{80} + allocDir := allocdir.NewAllocDir(filepath.Join(conf.AllocDir, alloc.ID)) + allocDir.Build([]*structs.Task{task}) + + ctx := driver.NewExecContext(allocDir) tr := NewTaskRunner(logger, conf, upd.Update, ctx, alloc.ID, task) return upd, tr } diff --git a/client/testutil/driver_compatible.go b/client/testutil/driver_compatible.go index 3fffd3d34..8b3ed705b 100644 --- a/client/testutil/driver_compatible.go +++ b/client/testutil/driver_compatible.go @@ -12,6 +12,12 @@ func ExecCompatible(t *testing.T) { } } +func QemuCompatible(t *testing.T) { + if runtime.GOOS != "windows" && syscall.Geteuid() != 0 { + t.Skip("Must be root on non-windows environments to run test") + } +} + func MountCompatible(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Windows does not support mount") diff --git a/command/spawn_daemon_linux.go b/command/spawn_daemon_linux.go index 89be3fa3b..ded1fddb5 100644 --- a/command/spawn_daemon_linux.go +++ b/command/spawn_daemon_linux.go @@ -24,11 +24,7 @@ type DaemonConfig struct { StderrFile string Groups *cgroupConfig.Cgroup -} - -// The exit status of the user's command. -type SpawnExitStatus struct { - Success bool + Chroot string } func (c *SpawnDaemonCommand) Run(args []string) int { @@ -69,14 +65,13 @@ func (c *SpawnDaemonCommand) Run(args []string) int { // Apply requires superuser permissions, and may fail if Nomad is not run with // the required permissions if err := manager.Apply(os.Getpid()); err != nil { - return c.outputStartStatus(fmt.Errorf("Failed to join cgroup: %v", err), 1) + return c.outputStartStatus(fmt.Errorf("Failed to join cgroup (config => %v): %v", manager.Cgroups, err), 1) } } // Isolate the user process. if _, err := syscall.Setsid(); err != nil { - return c.outputStartStatus(fmt.Errorf("Failed to join cgroup: %v", - fmt.Errorf("Failed setting sid: %v", err)), 1) + return c.outputStartStatus(fmt.Errorf("Failed setting sid: %v", err), 1) } syscall.Umask(0) @@ -97,9 +92,17 @@ func (c *SpawnDaemonCommand) Run(args []string) int { return c.outputStartStatus(fmt.Errorf("Error opening file to redirect Stdin: %v", err), 1) } - cmd.Stdout = stdo - cmd.Stderr = stde - cmd.Stdin = stdi + cmd.Cmd.Stdout = stdo + cmd.Cmd.Stderr = stde + cmd.Cmd.Stdin = stdi + + // Chroot jail the process and set its working directory. + if cmd.Cmd.SysProcAttr == nil { + cmd.Cmd.SysProcAttr = &syscall.SysProcAttr{} + } + + cmd.Cmd.SysProcAttr.Chroot = cmd.Chroot + cmd.Cmd.Dir = "/" // Spawn the user process. if err := cmd.Cmd.Start(); err != nil { @@ -110,12 +113,9 @@ func (c *SpawnDaemonCommand) Run(args []string) int { c.outputStartStatus(nil, 0) // Wait and then output the exit status. - exitStatus := &SpawnExitStatus{} - if err := cmd.Wait(); err == nil { - exitStatus.Success = true + if err := cmd.Wait(); err != nil { + return 1 } - enc := json.NewEncoder(os.Stdout) - enc.Encode(exitStatus) return 0 }