From 9caa7ac077e2cf75b585ebf0d98da6da416fc80e Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Tue, 22 Sep 2015 21:56:29 -0700 Subject: [PATCH] Mount shared alloc dir, modified API and tests --- client/allocdir/alloc_dir.go | 136 +++++++++++++++++---- client/allocdir/alloc_dir_darwin.go | 14 +++ client/allocdir/alloc_dir_linux.go | 18 +++ client/allocdir/alloc_dir_posix.go | 17 +++ client/allocdir/alloc_dir_test.go | 72 +++++++---- client/allocdir/alloc_dir_windows.go | 12 ++ client/allocdir/builder.go | 171 --------------------------- client/allocdir/builder_windows.go | 18 --- client/driver/driver.go | 2 +- client/driver/java.go | 6 +- client/driver/qemu.go | 6 +- client/testutil/driver_compatible.go | 10 ++ 12 files changed, 244 insertions(+), 238 deletions(-) create mode 100644 client/allocdir/alloc_dir_darwin.go create mode 100644 client/allocdir/alloc_dir_linux.go create mode 100644 client/allocdir/alloc_dir_posix.go create mode 100644 client/allocdir/alloc_dir_windows.go delete mode 100644 client/allocdir/builder.go delete mode 100644 client/allocdir/builder_windows.go diff --git a/client/allocdir/alloc_dir.go b/client/allocdir/alloc_dir.go index 82f469f26..8d1b7342e 100644 --- a/client/allocdir/alloc_dir.go +++ b/client/allocdir/alloc_dir.go @@ -2,6 +2,8 @@ package allocdir import ( "fmt" + "io" + "io/ioutil" "os" "path/filepath" @@ -20,24 +22,6 @@ var ( TaskLocal = "local" ) -// Builds the necessary directory structure for running an alloc. -type AllocDirBuilder interface { - // Given a list of a task build the correct alloc structure. - Build([]*structs.Task) error - - // Tears down previously build directory structure. - Destroy() error - - // Returns the directory of a task if it was created, otherwise an error is - // returned. - TaskDir(task string) (string, error) - - // Embed takes a mapping of absolute directory paths on the host to their - // intended, relative location within the task directory. Embed attempts - // hardlink and then defaults to copying. - Embed(task string, dirs map[string]string) error -} - type AllocDir struct { // AllocDir is the directory used for storing any state // of this allocation. It will be purged on alloc destroy. @@ -57,14 +41,122 @@ func NewAllocDir(allocDir string) *AllocDir { return d } +// Tears down previously build directory structure. func (d *AllocDir) Destroy() error { return os.RemoveAll(d.AllocDir) } -func (d *AllocDir) TaskDir(task string) (string, error) { - if dir, ok := d.TaskDirs[task]; ok { - return dir, nil +// Given a list of a task build the correct alloc structure. +func (d *AllocDir) Build(tasks []*structs.Task) error { + // Make the alloc directory, owned by the nomad process. + if err := os.MkdirAll(d.AllocDir, 0700); err != nil { + return fmt.Errorf("Failed to make the alloc directory %v: %v", d.AllocDir, err) } - return "", fmt.Errorf("Task directory doesn't exist for task %v", task) + // Make the shared directory and make it availabe to all user/groups. + if err := os.Mkdir(d.SharedDir, 0777); err != nil { + return err + } + + for _, dir := range SharedAllocDirs { + p := filepath.Join(d.SharedDir, dir) + if err := os.Mkdir(p, 0777); err != nil { + return err + } + } + + // Make the task directories. + for _, t := range tasks { + taskDir := filepath.Join(d.AllocDir, t.Name) + if err := os.Mkdir(taskDir, 0777); err != nil { + return err + } + + // Create a local directory that each task can use. + local := filepath.Join(taskDir, TaskLocal) + if err := os.Mkdir(local, 0777); err != nil { + return err + } + d.TaskDirs[t.Name] = taskDir + } + + return nil +} + +// Embed takes a mapping of absolute directory paths on the host to their +// intended, relative location within the task directory. Embed attempts +// hardlink and then defaults to copying. +func (d *AllocDir) Embed(task string, dirs map[string]string) error { + taskdir, ok := d.TaskDirs[task] + if !ok { + return fmt.Errorf("Task directory doesn't exist for task %v", task) + } + + subdirs := make(map[string]string) + for source, dest := range dirs { + // Enumerate the files in source. + entries, err := ioutil.ReadDir(source) + if err != nil { + return fmt.Errorf("Couldn't read directory %v: %v", source, err) + } + + // Create destination directory. + destDir := filepath.Join(taskdir, dest) + if err := os.MkdirAll(destDir, 0777); err != nil { + return fmt.Errorf("Couldn't create destination directory %v: %v", destDir, err) + } + + for _, entry := range entries { + hostEntry := filepath.Join(source, entry.Name()) + if entry.IsDir() { + subdirs[hostEntry] = filepath.Join(dest, filepath.Base(hostEntry)) + continue + } else if !entry.Mode().IsRegular() { + return fmt.Errorf("Can't embed non-regular file: %v", hostEntry) + } + + taskEntry := filepath.Join(destDir, filepath.Base(hostEntry)) + if err := d.linkOrCopy(hostEntry, taskEntry); err != nil { + return err + } + } + } + + // Recurse on self to copy subdirectories. + if len(subdirs) != 0 { + return d.Embed(task, subdirs) + } + + return nil +} + +// MountSharedDir mounts the shared directory into the specified task's +// directory. Mount is documented at an OS level in their respective +// implementation files. +func (d *AllocDir) MountSharedDir(task string) error { + taskDir, ok := d.TaskDirs[task] + if !ok { + return fmt.Errorf("No task directory exists for %v", task) + } + + return d.mountSharedDir(taskDir) +} + +func fileCopy(src, dst string) error { + // Do a simple copy. + srcFile, err := os.Open(src) + if err != nil { + return fmt.Errorf("Couldn't open src file %v: %v", src, err) + } + + dstFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE, 0777) + if err != nil { + return fmt.Errorf("Couldn't create destination file %v: %v", dst, err) + } + + if _, err := io.Copy(dstFile, srcFile); err != nil { + return fmt.Errorf("Couldn't copy %v to %v: %v", src, dst, err) + } + + return nil } diff --git a/client/allocdir/alloc_dir_darwin.go b/client/allocdir/alloc_dir_darwin.go new file mode 100644 index 000000000..08d7466ec --- /dev/null +++ b/client/allocdir/alloc_dir_darwin.go @@ -0,0 +1,14 @@ +package allocdir + +import ( + //"os" + "path/filepath" + "syscall" +) + +// Hardlinks the shared directory. As a side-effect the shared directory and +// task directory must be on the same filesystem. +func (d *AllocDir) mountSharedDir(taskDir string) error { + taskLoc := filepath.Join(taskDir, SharedAllocName) + return syscall.Link(d.SharedDir, taskLoc) +} diff --git a/client/allocdir/alloc_dir_linux.go b/client/allocdir/alloc_dir_linux.go new file mode 100644 index 000000000..86d8a8408 --- /dev/null +++ b/client/allocdir/alloc_dir_linux.go @@ -0,0 +1,18 @@ +package allocdir + +import ( + "os" + "path/filepath" + "syscall" +) + +// Bind mounts the shared directory into the task directory. Must be root to +// run. +func (d *AllocDir) mountSharedDir(taskDir string) error { + taskLoc := filepath.Join(taskDir, SharedAllocName) + if err := os.Mkdir(taskLoc, 0777); err != nil { + return err + } + + return syscall.Mount(d.SharedDir, taskLoc, "", syscall.MS_BIND, "") +} diff --git a/client/allocdir/alloc_dir_posix.go b/client/allocdir/alloc_dir_posix.go new file mode 100644 index 000000000..2a5357afe --- /dev/null +++ b/client/allocdir/alloc_dir_posix.go @@ -0,0 +1,17 @@ +// +build !windows + +// Functions shared between linux/darwin. +package allocdir + +import ( + "os" +) + +func (d *AllocDir) linkOrCopy(src, dst string) error { + // Attempt to hardlink. + if err := os.Link(src, dst); err == nil { + return nil + } + + return fileCopy(src, dst) +} diff --git a/client/allocdir/alloc_dir_test.go b/client/allocdir/alloc_dir_test.go index 2f8d4787e..fdbed8c8b 100644 --- a/client/allocdir/alloc_dir_test.go +++ b/client/allocdir/alloc_dir_test.go @@ -4,8 +4,10 @@ import ( "io/ioutil" "os" "path/filepath" + "reflect" "testing" + "github.com/hashicorp/nomad/client/testutil" "github.com/hashicorp/nomad/nomad/structs" ) @@ -55,29 +57,15 @@ func TestAllocDir_BuildAlloc(t *testing.T) { t.Fatalf("Build(%v) didn't create AllocDir %v", tasks, d.AllocDir) } - // Create a file in the alloc dir and then check it exists in each of the - // task dirs. - allocFile := "foo" - allocFileData := []byte{'b', 'a', 'r'} - if err := ioutil.WriteFile(filepath.Join(d.AllocDir, allocFile), allocFileData, 0777); err != nil { - t.Fatalf("Couldn't create file in alloc dir: %v", err) - } - for _, task := range tasks { - tDir, err := d.TaskDir(task.Name) - if err != nil { - t.Fatalf("TaskDir(%v) failed: %v", task.Name, err) + tDir, ok := d.TaskDirs[task.Name] + if !ok { + t.Fatalf("Task directory not found for %v", task.Name) } if _, err := os.Stat(tDir); os.IsNotExist(err) { t.Fatalf("Build(%v) didn't create TaskDir %v", tasks, tDir) } - - // TODO: Enable once mount is done. - //allocExpected := filepath.Join(tDir, SharedAllocName, allocFile) - //if _, err := os.Stat(allocExpected); os.IsNotExist(err) { - //t.Fatalf("File in shared alloc dir not accessible from task dir %v: %v", tDir, err) - //} } } @@ -148,9 +136,9 @@ func TestAllocDir_EmbedDirs(t *testing.T) { } // Check that the embedding was done properly. - taskDir, err := d.TaskDir(task) - if err != nil { - t.Fatalf("TaskDir(%v) failed: %v", task, err) + taskDir, ok := d.TaskDirs[task] + if !ok { + t.Fatalf("Task directory not found for %v", task) } exp := []string{filepath.Join(taskDir, taskDest, file), filepath.Join(taskDir, taskDest, subDirName, subFile)} @@ -160,3 +148,47 @@ func TestAllocDir_EmbedDirs(t *testing.T) { } } } + +func TestAllocDir_MountSharedAlloc(t *testing.T) { + testutil.MountCompatible(t) + tmp, err := ioutil.TempDir("", "AllocDir") + if err != nil { + t.Fatalf("Couldn't create temp dir: %v", err) + } + defer os.RemoveAll(tmp) + + d := NewAllocDir(tmp) + tasks := []*structs.Task{t1, t2} + if err := d.Build(tasks); err != nil { + t.Fatalf("Build(%v) failed: %v", tasks, err) + } + + // Write a file to the shared dir. + exp := []byte{'f', 'o', 'o'} + file := "bar" + if err := ioutil.WriteFile(filepath.Join(d.SharedDir, file), exp, 0777); err != nil { + t.Fatalf("Couldn't write file to shared directory: %v", err) + } + + for _, task := range tasks { + // Mount and then check that the file exists in the task directory. + if err := d.MountSharedDir(task.Name); err != nil { + t.Fatalf("MountSharedDir(%v) failed: %v", task.Name, err) + } + + taskDir, ok := d.TaskDirs[task.Name] + if !ok { + t.Fatalf("Task directory not found for %v", task.Name) + } + + taskFile := filepath.Join(taskDir, SharedAllocName, file) + act, err := ioutil.ReadFile(taskFile) + if err != nil { + t.Fatalf("Failed to read shared alloc file from task dir: %v", err) + } + + if !reflect.DeepEqual(act, exp) { + t.Fatalf("Incorrect data read from task dir: want %v; got %v", exp, act) + } + } +} diff --git a/client/allocdir/alloc_dir_windows.go b/client/allocdir/alloc_dir_windows.go new file mode 100644 index 000000000..57ce3bb9b --- /dev/null +++ b/client/allocdir/alloc_dir_windows.go @@ -0,0 +1,12 @@ +package allocdir + +import "errors" + +func (d *AllocDir) linkOrCopy(src, dst string) error { + return fileCopy(src, dst) +} + +// The windows version does nothing currently. +func (d *AllocDir) mountSharedDir(taskDir string) error { + return errors.New("Mount on Windows not supported.") +} diff --git a/client/allocdir/builder.go b/client/allocdir/builder.go deleted file mode 100644 index dac9be645..000000000 --- a/client/allocdir/builder.go +++ /dev/null @@ -1,171 +0,0 @@ -package allocdir - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "os/user" - "path/filepath" - "strconv" - "syscall" - - "github.com/hashicorp/nomad/nomad/structs" -) - -func (d *AllocDir) Build(tasks []*structs.Task) error { - // Make the alloc directory, owned by the nomad process. - if err := os.MkdirAll(d.AllocDir, 0700); err != nil { - return fmt.Errorf("Failed to make the alloc directory %v: %v", d.AllocDir, err) - } - - // Check if the process is has root capabilities and if so set the - // user/group to nobody. - var u *user.User - if syscall.Geteuid() == 0 { - nobody, err := user.Lookup("nobody") - if err != nil { - return fmt.Errorf("Could not set owner/group on shared alloc directory: %v", err) - } - u = nobody - } - - // Make the shared directory and make it availabe to all user/groups. - if err := mkOwnedDir(d.SharedDir, u, 0777); err != nil { - return err - } - - for _, dir := range SharedAllocDirs { - p := filepath.Join(d.SharedDir, dir) - if err := mkOwnedDir(p, u, 0777); err != nil { - return err - } - } - - // Make the task directories. - for _, t := range tasks { - p := filepath.Join(d.AllocDir, t.Name) - if err := mkOwnedDir(p, u, 0777); err != nil { - return err - } - - // Create a local directory that each task can use. - local := filepath.Join(p, TaskLocal) - if err := mkOwnedDir(local, u, 0777); err != nil { - return err - } - d.TaskDirs[t.Name] = p - - // TODO: Mount the shared alloc dir into each task dir. - } - - return nil -} - -func (d *AllocDir) Embed(task string, dirs map[string]string) error { - taskdir, ok := d.TaskDirs[task] - if !ok { - return fmt.Errorf("Task directory doesn't exist for task %v", task) - } - - subdirs := make(map[string]string) - for source, dest := range dirs { - // Enumerate the files in source. - entries, err := ioutil.ReadDir(source) - if err != nil { - return fmt.Errorf("Couldn't read directory %v: %v", source, err) - } - - // Create destination directory. - destDir := filepath.Join(taskdir, dest) - if err := os.MkdirAll(destDir, 0777); err != nil { - return fmt.Errorf("Couldn't create destination directory %v: %v", destDir, err) - } - - for _, entry := range entries { - hostEntry := filepath.Join(source, entry.Name()) - if entry.IsDir() { - subdirs[hostEntry] = filepath.Join(dest, filepath.Base(hostEntry)) - continue - } else if !entry.Mode().IsRegular() { - return fmt.Errorf("Can't embed non-regular file: %v", hostEntry) - } - - taskEntry := filepath.Join(destDir, filepath.Base(hostEntry)) - - // Attempt to hardlink. - if err := os.Link(hostEntry, taskEntry); err == nil { - continue - } - - // Do a simple copy. - src, err := os.Open(hostEntry) - if err != nil { - return fmt.Errorf("Couldn't open host file %v: %v", hostEntry, err) - } - - dst, err := os.OpenFile(taskEntry, os.O_WRONLY|os.O_CREATE, 0777) - if err != nil { - return fmt.Errorf("Couldn't create task file %v: %v", taskEntry, err) - } - - if _, err := io.Copy(dst, src); err != nil { - return fmt.Errorf("Couldn't copy %v to %v: %v", hostEntry, taskEntry, err) - } - } - } - - // Recurse on self to copy subdirectories. - if len(subdirs) != 0 { - return d.Embed(task, subdirs) - } - - return nil -} - -// mkOwnedDir creates the directory specified by the path with the passed -// permissions. It also sets the user/group based on the passed user if it is -// non-nil. It returns an error if any of these operations fail. -func mkOwnedDir(path string, user *user.User, perm os.FileMode) error { - if err := os.Mkdir(path, perm); err != nil { - return fmt.Errorf("Failed to make directory %v: %v", path, err) - } - - if user == nil { - return nil - } - - uid, err := getUid(user) - if err != nil { - return err - } - - gid, err := getGid(user) - if err != nil { - return err - } - - if err := os.Chown(path, uid, gid); err != nil { - return fmt.Errorf("Couldn't change owner/group of %v to (uid: %v, gid: %v): %v", path, uid, gid, err) - } - - return nil -} - -func getUid(u *user.User) (int, error) { - uid, err := strconv.Atoi(u.Uid) - if err != nil { - return 0, fmt.Errorf("Unable to convert Uid to an int: %v", err) - } - - return uid, nil -} - -func getGid(u *user.User) (int, error) { - gid, err := strconv.Atoi(u.Gid) - if err != nil { - return 0, fmt.Errorf("Unable to convert Gid to an int: %v", err) - } - - return gid, nil -} diff --git a/client/allocdir/builder_windows.go b/client/allocdir/builder_windows.go deleted file mode 100644 index 6333909c9..000000000 --- a/client/allocdir/builder_windows.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build !linux - -package allocdir - -import ( - "errors" - - "github.com/hashicorp/nomad/nomad/structs" -) - -func (d *AllocDir) Build(tasks []*structs.Task) error { - // TODO: Need to figure out how to do mounts on windows. - return errors.New("Not implemented") -} - -func (d *AllocDir) Embed(task string, dirs map[string]string) error { - return errors.New("Not implemented") -} diff --git a/client/driver/driver.go b/client/driver/driver.go index 561222ea8..498d973f8 100644 --- a/client/driver/driver.go +++ b/client/driver/driver.go @@ -96,7 +96,7 @@ type ExecContext struct { sync.Mutex // AllocDir contains information about the alloc directory structure. - AllocDir allocdir.AllocDirBuilder + AllocDir *allocdir.AllocDir } // NewExecContext is used to create a new execution context diff --git a/client/driver/java.go b/client/driver/java.go index c94175f75..53eca76e7 100644 --- a/client/driver/java.go +++ b/client/driver/java.go @@ -106,9 +106,9 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, } // Get the tasks local directory. - taskDir, err := ctx.AllocDir.TaskDir(d.DriverContext.taskName) - if err != nil { - return nil, err + taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName] + if !ok { + return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) } taskLocal := filepath.Join(taskDir, allocdir.TaskLocal) diff --git a/client/driver/qemu.go b/client/driver/qemu.go index 6a0d5c984..540dc9068 100644 --- a/client/driver/qemu.go +++ b/client/driver/qemu.go @@ -103,9 +103,9 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, } // Get the tasks local directory. - taskDir, err := ctx.AllocDir.TaskDir(d.DriverContext.taskName) - if err != nil { - return nil, err + taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName] + if !ok { + return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) } taskLocal := filepath.Join(taskDir, allocdir.TaskLocal) diff --git a/client/testutil/driver_compatible.go b/client/testutil/driver_compatible.go index 2f34508a7..3fffd3d34 100644 --- a/client/testutil/driver_compatible.go +++ b/client/testutil/driver_compatible.go @@ -11,3 +11,13 @@ func ExecCompatible(t *testing.T) { t.Skip("Must be root on non-windows environments to run test") } } + +func MountCompatible(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows does not support mount") + } + + if syscall.Geteuid() != 0 { + t.Skip("Must be root to run test") + } +}