From db88c38bc7239b42c51961fcf4ccf0cfbe9753ce Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 15 Oct 2015 16:40:07 -0700 Subject: [PATCH] Bind alloc dir and task local dir to docker containers and parse args correctly --- client/alloc_runner.go | 6 ++- client/driver/docker.go | 76 ++++++++++++++++++++++++------- client/driver/environment/vars.go | 8 ++++ client/fingerprint/host.go | 2 +- scripts/build.sh | 1 + 5 files changed, 74 insertions(+), 19 deletions(-) diff --git a/client/alloc_runner.go b/client/alloc_runner.go index f4613ad67..f41be4558 100644 --- a/client/alloc_runner.go +++ b/client/alloc_runner.go @@ -282,7 +282,11 @@ func (r *AllocRunner) Run() { // Create the execution context if r.ctx == nil { allocDir := allocdir.NewAllocDir(filepath.Join(r.config.AllocDir, r.alloc.ID)) - allocDir.Build(tg.Tasks) + if err := allocDir.Build(tg.Tasks); err != nil { + r.logger.Printf("[WARN] client: failed to build task directories: %v", err) + r.setStatus(structs.AllocClientStatusFailed, fmt.Sprintf("failed to build task dirs for '%s'", alloc.TaskGroup)) + return + } r.ctx = driver.NewExecContext(allocDir) } diff --git a/client/driver/docker.go b/client/driver/docker.go index 8ff6b93f8..1100061d4 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -4,12 +4,15 @@ import ( "encoding/json" "fmt" "log" + "path/filepath" "strconv" "strings" docker "github.com/fsouza/go-dockerclient" + "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/client/driver/args" "github.com/hashicorp/nomad/nomad/structs" ) @@ -97,12 +100,38 @@ func (d *DockerDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool return true, nil } -// We have to call this when we create the container AND when we start it so -// we'll make a function. -func createHostConfig(task *structs.Task) *docker.HostConfig { - // hostConfig holds options for the docker container that are unique to this - // machine, such as resource limits and port mappings - return &docker.HostConfig{ +func containerBinds(alloc *allocdir.AllocDir, task *structs.Task) ([]string, error) { + shared := alloc.SharedDir + local, ok := alloc.TaskDirs[task.Name] + if !ok { + fmt.Println("ALLOCDIR: ", alloc) + fmt.Println("TASK DIRS: ", alloc.TaskDirs) + for task, dir := range alloc.TaskDirs { + fmt.Printf("%v -> %v\n", task, dir) + } + return nil, fmt.Errorf("Failed to find task local directory: %v", task.Name) + } + + return []string{ + fmt.Sprintf("%s:%s", shared, allocdir.SharedAllocName), + fmt.Sprintf("%s:%s", local, allocdir.TaskLocal), + }, nil +} + +// createContainer initializes a struct needed to call docker.client.CreateContainer() +func createContainer(ctx *ExecContext, task *structs.Task, logger *log.Logger) (docker.CreateContainerOptions, error) { + var c docker.CreateContainerOptions + if task.Resources == nil { + logger.Printf("[ERR] driver.docker: task.Resources is empty") + return c, fmt.Errorf("task.Resources is nil and we can't constrain resource usage. We shouldn't have been able to schedule this in the first place.") + } + + binds, err := containerBinds(ctx.AllocDir, task) + if err != nil { + return c, err + } + + hostConfig := &docker.HostConfig{ // Convert MB to bytes. This is an absolute value. // // This value represents the total amount of memory a process can use. @@ -131,20 +160,16 @@ func createHostConfig(task *structs.Task) *docker.HostConfig { // - https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt // - https://www.kernel.org/doc/Documentation/scheduler/sched-design-CFS.txt CPUShares: int64(task.Resources.CPU), - } -} -// createContainer initializes a struct needed to call docker.client.CreateContainer() -func createContainer(ctx *ExecContext, task *structs.Task, logger *log.Logger) (docker.CreateContainerOptions, error) { - var c docker.CreateContainerOptions - if task.Resources == nil { - logger.Printf("[ERR] driver.docker: task.Resources is empty") - return c, fmt.Errorf("task.Resources is nil and we can't constrain resource usage. We shouldn't have been able to schedule this in the first place.") + // Binds are used to mount a host volume into the container. We mount a + // local directory for storage and a shared alloc directory that can be + // used to share data between different tasks in the same task group. + Binds: binds, } - hostConfig := createHostConfig(task) logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Config["image"]) logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Config["image"]) + logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Config["image"]) mode, ok := task.Config["network_mode"] if !ok || mode == "" { @@ -198,14 +223,31 @@ func createContainer(ctx *ExecContext, task *structs.Task, logger *log.Logger) ( hostConfig.PortBindings = dockerPorts } + // Create environment variables. + env := TaskEnvironmentVariables(ctx, task) + env.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)) + env.SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)) + config := &docker.Config{ - Env: TaskEnvironmentVariables(ctx, task).List(), + Env: env.List(), Image: task.Config["image"], } + rawArgs, hasArgs := task.Config["args"] + parsedArgs, err := args.ParseAndReplace(rawArgs, env.Map()) + if err != nil { + return c, err + } + // If the user specified a custom command to run, we'll inject it here. if command, ok := task.Config["command"]; ok { - config.Cmd = strings.Split(command, " ") + cmd := []string{command} + if hasArgs { + cmd = append(cmd, parsedArgs...) + } + config.Cmd = cmd + } else if hasArgs { + logger.Println("[DEBUG] driver.docker: ignoring args because command not specified") } return docker.CreateContainerOptions{ diff --git a/client/driver/environment/vars.go b/client/driver/environment/vars.go index 8e2d698ed..587252c87 100644 --- a/client/driver/environment/vars.go +++ b/client/driver/environment/vars.go @@ -12,6 +12,10 @@ const ( // group. AllocDir = "NOMAD_ALLOC_DIR" + // The path to the tasks local directory where it can store data that is + // persisted to the alloc is removed. + TaskLocalDir = "NOMAD_TASK_DIR" + // The tasks memory limit in MBs. MemLimit = "NOMAD_MEMORY_LIMIT" @@ -70,6 +74,10 @@ func (t TaskEnvironment) SetAllocDir(dir string) { t[AllocDir] = dir } +func (t TaskEnvironment) SetTaskLocalDir(dir string) { + t[TaskLocalDir] = dir +} + func (t TaskEnvironment) SetMemLimit(limit int) { t[MemLimit] = strconv.Itoa(limit) } diff --git a/client/fingerprint/host.go b/client/fingerprint/host.go index 6e5244bf9..ac7a347f2 100644 --- a/client/fingerprint/host.go +++ b/client/fingerprint/host.go @@ -5,7 +5,7 @@ import ( "log" "os/exec" "runtime" - "strings" + "strings" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/nomad/structs" diff --git a/scripts/build.sh b/scripts/build.sh index 36566413a..4a79e8351 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -44,6 +44,7 @@ gox \ -arch="${XC_ARCH}" \ -osarch="!linux/arm !darwin/386" \ -ldflags "-X main.GitCommit ${GIT_COMMIT}${GIT_DIRTY}" \ + -cgo \ -output "pkg/{{.OS}}_{{.Arch}}/nomad" \ .