From 38460ca6532c5dc298d6873ea2cd96823646f154 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Mon, 21 Sep 2015 14:13:17 -0700 Subject: [PATCH] AllocDirBuilder that creates the alloc directory structure --- client/alloc_runner.go | 6 +- client/allocdir/alloc_dir.go | 65 ++++++++++++++++++ client/allocdir/alloc_dir_linux.go | 98 ++++++++++++++++++++++++++++ client/allocdir/alloc_dir_windows.go | 8 +++ client/client.go | 3 +- client/driver/driver.go | 22 ++++--- client/driver/java.go | 13 +++- client/driver/qemu.go | 12 +++- client/task_runner.go | 2 +- 9 files changed, 211 insertions(+), 18 deletions(-) create mode 100644 client/allocdir/alloc_dir.go create mode 100644 client/allocdir/alloc_dir_linux.go create mode 100644 client/allocdir/alloc_dir_windows.go diff --git a/client/alloc_runner.go b/client/alloc_runner.go index 79b2579c3..8a6ac2046 100644 --- a/client/alloc_runner.go +++ b/client/alloc_runner.go @@ -10,6 +10,7 @@ import ( "time" "github.com/hashicorp/go-multierror" + "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver" "github.com/hashicorp/nomad/nomad/structs" @@ -146,7 +147,7 @@ func (r *AllocRunner) DestroyState() error { // DestroyContext is used to destroy the context func (r *AllocRunner) DestroyContext() error { - return os.RemoveAll(r.ctx.AllocDir) + return r.ctx.AllocDir.Destroy() } // Alloc returns the associated allocation @@ -278,7 +279,8 @@ func (r *AllocRunner) Run() { // Create the execution context if r.ctx == nil { r.ctx = driver.NewExecContext() - r.ctx.AllocDir = filepath.Join(r.config.AllocDir, r.alloc.ID) + r.ctx.AllocDir = allocdir.NewAllocDir(filepath.Join(r.config.AllocDir, r.alloc.ID)) + r.ctx.AllocDir.Build(tg.Tasks) } // Start the task runners diff --git a/client/allocdir/alloc_dir.go b/client/allocdir/alloc_dir.go new file mode 100644 index 000000000..391dd1d93 --- /dev/null +++ b/client/allocdir/alloc_dir.go @@ -0,0 +1,65 @@ +package allocdir + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/hashicorp/nomad/nomad/structs" +) + +var ( + // The name of the directory that is shared across tasks in a task group. + SharedAllocName = "alloc" + + // The set of directories that exist inside eache shared alloc directory. + SharedAllocDirs = []string{"logs", "tmp", "data"} + + // The name of the directory that exists inside each task directory + // regardless of driver. + TaskLocal = "local" +) + +// Builds the necessary directory structure for running an alloc. +type AllocDirBuilder interface { + // Given a list of a task build the correct alloc structure. + Build([]*structs.Task) error + + // Tears down previously build directory structure. + Destroy() error + + // Returns the directory of a task if it was created, otherwise an error is + // returned. + TaskDir(task string) (string, error) +} + +type AllocDir struct { + // AllocDir is the directory used for storing any state + // of this allocation. It will be purged on alloc destroy. + AllocDir string + + // The shared directory is available to all tasks within the same task + // group. + SharedDir string + + // TaskDirs is a mapping of task names to their non-shared directory. + TaskDirs map[string]string +} + +func NewAllocDir(allocDir string) *AllocDir { + d := &AllocDir{AllocDir: allocDir, TaskDirs: make(map[string]string)} + d.SharedDir = filepath.Join(d.AllocDir, SharedAllocName) + return d +} + +func (d *AllocDir) Destroy() error { + return os.RemoveAll(d.AllocDir) +} + +func (d *AllocDir) TaskDir(task string) (string, error) { + if dir, ok := d.TaskDirs[task]; ok { + return dir, nil + } + + return "", fmt.Errorf("Task directory doesn't exist for task %v", task) +} diff --git a/client/allocdir/alloc_dir_linux.go b/client/allocdir/alloc_dir_linux.go new file mode 100644 index 000000000..67da9813f --- /dev/null +++ b/client/allocdir/alloc_dir_linux.go @@ -0,0 +1,98 @@ +// +build !windows + +package allocdir + +import ( + "fmt" + "github.com/hashicorp/nomad/nomad/structs" + "os" + "os/user" + "path/filepath" + "strconv" +) + +func (d *AllocDir) Build(tasks []*structs.Task) error { + // Make the alloc directory, owned by the nomad process. + if err := os.Mkdir(d.AllocDir, 0700); err != nil { + return fmt.Errorf("Failed to make the alloc directory %v: %v", d.AllocDir, err) + } + + nobody, err := user.Lookup("nobody") + if err != nil { + return fmt.Errorf("Could not set owner/group on shared alloc directory: %v", err) + } + + uid, err := getUid(nobody) + if err != nil { + return err + } + + gid, err := getGid(nobody) + if err != nil { + return err + } + + // Make the shared directory and make it availabe to all user/groups. + if err := mkOwnedDir(d.SharedDir, uid, gid, 0777); err != nil { + return err + } + + for _, dir := range SharedAllocDirs { + p := filepath.Join(d.SharedDir, dir) + if err := mkOwnedDir(p, uid, gid, 0777); err != nil { + return err + } + } + + // Make the task directories. + for _, t := range tasks { + p := filepath.Join(d.AllocDir, t.Name) + if err := mkOwnedDir(p, uid, gid, 0777); err != nil { + return err + } + + // Create a local directory that each task can use. + local := filepath.Join(p, TaskLocal) + if err := mkOwnedDir(local, uid, gid, 0777); err != nil { + return err + } + d.TaskDirs[t.Name] = local + + // TODO: Mount the shared alloc dir into each task dir. + } + + return nil +} + +// mkOwnedDir creates the directory specified by the path with the passed +// permissions. It also sets the passed uid and gid. It returns an error if any +// of these operations fail. +func mkOwnedDir(path string, uid, gid int, perm os.FileMode) error { + if err := os.Mkdir(path, perm); err != nil { + return fmt.Errorf("Failed to make directory %v: %v", path, err) + } + + if err := os.Chown(path, uid, gid); err != nil { + return fmt.Errorf("Couldn't change owner/group of %v to (uid: %v, gid: %v): %v", path, uid, gid, err) + } + + return nil +} + +func getUid(u *user.User) (int, error) { + uid, err := strconv.Atoi(u.Uid) + if err != nil { + return 0, fmt.Errorf("Unable to convert Uid to an int: %v", err) + } + + return uid, nil +} + +func getGid(u *user.User) (int, error) { + gid, err := strconv.Atoi(u.Gid) + if err != nil { + return 0, fmt.Errorf("Unable to convert Gid to an int: %v", err) + } + + return gid, nil +} diff --git a/client/allocdir/alloc_dir_windows.go b/client/allocdir/alloc_dir_windows.go new file mode 100644 index 000000000..524d40014 --- /dev/null +++ b/client/allocdir/alloc_dir_windows.go @@ -0,0 +1,8 @@ +package allocdir + +import "github.com/hashicorp/nomad/nomad/structs" + +func (r *AllocRunner) Build(tasks []*structs.Task) error { + // TODO: Need to figure out how to do mounts on windows. + return nil +} diff --git a/client/client.go b/client/client.go index d8f68c275..4f8ff2e4f 100644 --- a/client/client.go +++ b/client/client.go @@ -143,6 +143,7 @@ func NewClient(cfg *config.Config) (*Client, error) { // needed before we begin starting its various components. func (c *Client) init() error { // Ensure the alloc dir exists if we have one + // TODO(alex): Make a tmp directory if it doesn't? if c.config.AllocDir != "" { if err := os.MkdirAll(c.config.AllocDir, 0700); err != nil { return fmt.Errorf("failed creating alloc dir: %s", err) @@ -431,7 +432,7 @@ func (c *Client) fingerprint() error { // setupDrivers is used to find the available drivers func (c *Client) setupDrivers() error { var avail []string - driverCtx := driver.NewDriverContext(c.config, c.config.Node, c.logger) + driverCtx := driver.NewDriverContext("", c.config, c.config.Node, c.logger) for name := range driver.BuiltinDrivers { d, err := driver.NewDriver(name, driverCtx) if err != nil { diff --git a/client/driver/driver.go b/client/driver/driver.go index 6b0bc91a9..561222ea8 100644 --- a/client/driver/driver.go +++ b/client/driver/driver.go @@ -6,6 +6,7 @@ import ( "strings" "sync" + "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/fingerprint" "github.com/hashicorp/nomad/nomad/structs" @@ -55,20 +56,22 @@ type Driver interface { // node attributes into a Driver without having to change the Driver interface // each time we do it. Used in conjection with Factory, above. type DriverContext struct { - config *config.Config - logger *log.Logger - node *structs.Node + taskName string + config *config.Config + logger *log.Logger + node *structs.Node } // NewDriverContext initializes a new DriverContext with the specified fields. // This enables other packages to create DriverContexts but keeps the fields // private to the driver. If we want to change this later we can gorename all of // the fields in DriverContext. -func NewDriverContext(config *config.Config, node *structs.Node, logger *log.Logger) *DriverContext { +func NewDriverContext(taskName string, config *config.Config, node *structs.Node, logger *log.Logger) *DriverContext { return &DriverContext{ - config: config, - node: node, - logger: logger, + taskName: taskName, + config: config, + node: node, + logger: logger, } } @@ -92,9 +95,8 @@ type DriverHandle interface { type ExecContext struct { sync.Mutex - // AllocDir is the directory used for storing any state - // of this allocation. It will be purged on alloc destroy. - AllocDir string + // AllocDir contains information about the alloc directory structure. + AllocDir allocdir.AllocDirBuilder } // NewExecContext is used to create a new execution context diff --git a/client/driver/java.go b/client/driver/java.go index caabf535b..c94175f75 100644 --- a/client/driver/java.go +++ b/client/driver/java.go @@ -14,6 +14,7 @@ import ( "syscall" "time" + "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/executor" "github.com/hashicorp/nomad/nomad/structs" @@ -104,7 +105,15 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, return nil, fmt.Errorf("Error downloading source for Java driver: %s", err) } - fPath := filepath.Join(ctx.AllocDir, path.Base(source)) + // Get the tasks local directory. + taskDir, err := ctx.AllocDir.TaskDir(d.DriverContext.taskName) + if err != nil { + return nil, err + } + taskLocal := filepath.Join(taskDir, allocdir.TaskLocal) + + // Create a location to download the binary. + fPath := filepath.Join(taskLocal, path.Base(source)) f, err := os.OpenFile(fPath, os.O_CREATE|os.O_WRONLY, 0666) if err != nil { return nil, fmt.Errorf("Error opening file to download to: %s", err) @@ -113,7 +122,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, defer f.Close() defer resp.Body.Close() - // Copy remote file to local AllocDir for execution + // Copy remote file to local directory for execution // TODO: a retry of sort if io.Copy fails, for large binaries _, ioErr := io.Copy(f, resp.Body) if ioErr != nil { diff --git a/client/driver/qemu.go b/client/driver/qemu.go index 31d8fbea8..6a0d5c984 100644 --- a/client/driver/qemu.go +++ b/client/driver/qemu.go @@ -19,6 +19,7 @@ import ( "syscall" "time" + "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/nomad/structs" ) @@ -101,10 +102,17 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, return nil, fmt.Errorf("Error downloading source for Qemu driver: %s", err) } - // Create a location in the AllocDir to download and store the image. + // Get the tasks local directory. + taskDir, err := ctx.AllocDir.TaskDir(d.DriverContext.taskName) + if err != nil { + return nil, err + } + taskLocal := filepath.Join(taskDir, allocdir.TaskLocal) + + // Create a location in the local directory to download and store the image. // TODO: Caching vmID := fmt.Sprintf("qemu-vm-%s-%s", structs.GenerateUUID(), filepath.Base(source)) - fPath := filepath.Join(ctx.AllocDir, vmID) + fPath := filepath.Join(taskLocal, vmID) vmPath, err := os.OpenFile(fPath, os.O_CREATE|os.O_WRONLY, 0666) if err != nil { return nil, fmt.Errorf("Error opening file to download to: %s", err) diff --git a/client/task_runner.go b/client/task_runner.go index 6bdb6e5e1..14a45ffc3 100644 --- a/client/task_runner.go +++ b/client/task_runner.go @@ -129,7 +129,7 @@ func (r *TaskRunner) setStatus(status, desc string) { // createDriver makes a driver for the task func (r *TaskRunner) createDriver() (driver.Driver, error) { - driverCtx := driver.NewDriverContext(r.config, r.config.Node, r.logger) + driverCtx := driver.NewDriverContext(r.task.Name, r.config, r.config.Node, r.logger) driver, err := driver.NewDriver(r.task.Driver, driverCtx) if err != nil { err = fmt.Errorf("failed to create driver '%s' for alloc %s: %v",