diff --git a/api/tasks.go b/api/tasks.go index 92f98b571..5f9b24cb6 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -153,24 +153,27 @@ type TaskState struct { } const ( - TaskDriverFailure = "Driver Failure" - TaskReceived = "Received" - TaskStarted = "Started" - TaskTerminated = "Terminated" - TaskKilled = "Killed" - TaskRestarting = "Restarting" - TaskNotRestarting = "Restarts Exceeded" + TaskDriverFailure = "Driver Failure" + TaskReceived = "Received" + TaskStarted = "Started" + TaskTerminated = "Terminated" + TaskKilled = "Killed" + TaskRestarting = "Restarting" + TaskNotRestarting = "Restarts Exceeded" + TaskDownloadingArtifacts = "Downloading Artifacts" + TaskArtifactDownloadFailed = "Failed Artifact Download" ) // TaskEvent is an event that effects the state of a task and contains meta-data // appropriate to the events type. type TaskEvent struct { - Type string - Time int64 - DriverError string - ExitCode int - Signal int - Message string - KillError string - StartDelay int64 + Type string + Time int64 + DriverError string + ExitCode int + Signal int + Message string + KillError string + StartDelay int64 + DownloadError string } diff --git a/client/driver/exec.go b/client/driver/exec.go index b134cc3de..c832ab9de 100644 --- a/client/driver/exec.go +++ b/client/driver/exec.go @@ -15,7 +15,6 @@ import ( "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/executor" cstructs "github.com/hashicorp/nomad/client/driver/structs" - "github.com/hashicorp/nomad/client/getter" "github.com/hashicorp/nomad/helper/discover" "github.com/hashicorp/nomad/nomad/structs" "github.com/mitchellh/mapstructure" @@ -28,10 +27,8 @@ type ExecDriver struct { } type ExecDriverConfig struct { - ArtifactSource string `mapstructure:"artifact_source"` - Checksum string `mapstructure:"checksum"` - Command string `mapstructure:"command"` - Args []string `mapstructure:"args"` + Command string `mapstructure:"command"` + Args []string `mapstructure:"args"` } // execHandle is returned from Start/Open as a handle to the PID @@ -89,21 +86,6 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) } - // Check if an artificat is specified and attempt to download it - source, ok := task.Config["artifact_source"] - if ok && source != "" { - // Proceed to download an artifact to be executed. - _, err := getter.GetArtifact( - taskDir, - driverConfig.ArtifactSource, - driverConfig.Checksum, - d.logger, - ) - if err != nil { - return nil, err - } - } - bin, err := discover.NomadExecutable() if err != nil { return nil, fmt.Errorf("unable to find the nomad binary: %v", err) diff --git a/client/driver/java.go b/client/driver/java.go index e66205267..d3128c01a 100644 --- a/client/driver/java.go +++ b/client/driver/java.go @@ -21,7 +21,6 @@ import ( "github.com/hashicorp/nomad/client/driver/executor" cstructs "github.com/hashicorp/nomad/client/driver/structs" "github.com/hashicorp/nomad/client/fingerprint" - "github.com/hashicorp/nomad/client/getter" "github.com/hashicorp/nomad/helper/discover" "github.com/hashicorp/nomad/nomad/structs" ) @@ -34,10 +33,9 @@ type JavaDriver struct { } type JavaDriverConfig struct { - JvmOpts []string `mapstructure:"jvm_options"` - ArtifactSource string `mapstructure:"artifact_source"` - Checksum string `mapstructure:"checksum"` - Args []string `mapstructure:"args"` + JarPath string `mapstructure:"jar_path"` + JvmOpts []string `mapstructure:"jvm_options"` + Args []string `mapstructure:"args"` } // javaHandle is returned from Start/Open as a handle to the PID @@ -124,19 +122,10 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) } - // Proceed to download an artifact to be executed. - path, err := getter.GetArtifact( - taskDir, - driverConfig.ArtifactSource, - driverConfig.Checksum, - d.logger, - ) - if err != nil { - return nil, err + if driverConfig.JarPath == "" { + return nil, fmt.Errorf("jar_path must be specified") } - jarName := filepath.Base(path) - args := []string{} // Look for jvm options if len(driverConfig.JvmOpts) != 0 { @@ -145,7 +134,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, } // Build the argument list. - args = append(args, "-jar", jarName) + args = append(args, "-jar", driverConfig.JarPath) if len(driverConfig.Args) != 0 { args = append(args, driverConfig.Args...) } diff --git a/client/driver/qemu.go b/client/driver/qemu.go index e4efe5858..180a91006 100644 --- a/client/driver/qemu.go +++ b/client/driver/qemu.go @@ -17,7 +17,6 @@ import ( "github.com/hashicorp/nomad/client/driver/executor" cstructs "github.com/hashicorp/nomad/client/driver/structs" "github.com/hashicorp/nomad/client/fingerprint" - "github.com/hashicorp/nomad/client/getter" "github.com/hashicorp/nomad/helper/discover" "github.com/hashicorp/nomad/nomad/structs" "github.com/mitchellh/mapstructure" @@ -36,10 +35,9 @@ type QemuDriver struct { } type QemuDriverConfig struct { - ArtifactSource string `mapstructure:"artifact_source"` - Checksum string `mapstructure:"checksum"` - Accelerator string `mapstructure:"accelerator"` - PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports. + ImagePath string `mapstructure:"image_path"` + Accelerator string `mapstructure:"accelerator"` + PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports. } // qemuHandle is returned from Start/Open as a handle to the PID @@ -98,36 +96,19 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, } // Get the image source - source, ok := task.Config["artifact_source"] - if !ok || source == "" { - return nil, fmt.Errorf("Missing source image Qemu driver") - } - - // Qemu defaults to 128M of RAM for a given VM. Instead, we force users to - // supply a memory size in the tasks resources - if task.Resources == nil || task.Resources.MemoryMB == 0 { - return nil, fmt.Errorf("Missing required Task Resource: Memory") + vmPath := driverConfig.ImagePath + if vmPath == "" { + return nil, fmt.Errorf("image_path must be set") } + vmID := filepath.Base(vmPath) // Get the tasks local directory. - taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName] + taskName := d.DriverContext.taskName + taskDir, ok := ctx.AllocDir.TaskDirs[taskName] if !ok { return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) } - // Proceed to download an artifact to be executed. - vmPath, err := getter.GetArtifact( - taskDir, - driverConfig.ArtifactSource, - driverConfig.Checksum, - d.logger, - ) - if err != nil { - return nil, err - } - - vmID := filepath.Base(vmPath) - // Parse configuration arguments // Create the base arguments accelerator := "tcg" diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go index f029525b5..3b1c87372 100644 --- a/client/driver/raw_exec.go +++ b/client/driver/raw_exec.go @@ -14,7 +14,6 @@ import ( "github.com/hashicorp/nomad/client/driver/executor" cstructs "github.com/hashicorp/nomad/client/driver/structs" "github.com/hashicorp/nomad/client/fingerprint" - "github.com/hashicorp/nomad/client/getter" "github.com/hashicorp/nomad/helper/discover" "github.com/hashicorp/nomad/nomad/structs" "github.com/mitchellh/mapstructure" @@ -83,21 +82,6 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl return nil, err } - // Check if an artificat is specified and attempt to download it - source, ok := task.Config["artifact_source"] - if ok && source != "" { - // Proceed to download an artifact to be executed. - _, err := getter.GetArtifact( - taskDir, - driverConfig.ArtifactSource, - driverConfig.Checksum, - d.logger, - ) - if err != nil { - return nil, err - } - } - bin, err := discover.NomadExecutable() if err != nil { return nil, fmt.Errorf("unable to find the nomad binary: %v", err) diff --git a/client/getter/getter.go b/client/getter/getter.go index 9d635f52e..92d295f47 100644 --- a/client/getter/getter.go +++ b/client/getter/getter.go @@ -68,7 +68,7 @@ func GetArtifact(artifact *structs.TaskArtifact, destDir string, logger *log.Log // Download the artifact if err := getClient(url, destDir).Get(); err != nil { - return fmt.Errorf("error downloading artifact (url: %q): %v", url, err) + return err } return nil diff --git a/client/task_runner.go b/client/task_runner.go index 20d1c9843..4531696f7 100644 --- a/client/task_runner.go +++ b/client/task_runner.go @@ -13,6 +13,7 @@ import ( "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver" + "github.com/hashicorp/nomad/client/getter" "github.com/hashicorp/nomad/nomad/structs" "github.com/mitchellh/hashstructure" @@ -48,6 +49,10 @@ type TaskRunner struct { handle driver.DriverHandle handleLock sync.Mutex + // artifactsDownloaded tracks whether the tasks artifacts have been + // downloaded + artifactsDownloaded bool + destroy bool destroyCh chan struct{} destroyLock sync.Mutex @@ -146,6 +151,10 @@ func (r *TaskRunner) RestoreState() error { } r.handleLock.Lock() r.handle = handle + + // If we have previously created the driver, the artifacts have been + // downloaded. + r.artifactsDownloaded = true r.handleLock.Unlock() } return nil @@ -214,12 +223,40 @@ func (r *TaskRunner) Run() { } func (r *TaskRunner) run() { + // Predeclare things so we an jump to the RESTART + var handleEmpty bool + for { + // Download the task's artifacts + if !r.artifactsDownloaded { + r.setState(structs.TaskStatePending, structs.NewTaskEvent(structs.TaskDownloadingArtifacts)) + taskDir, ok := r.ctx.AllocDir.TaskDirs[r.task.Name] + if !ok { + err := fmt.Errorf("task directory couldn't be found") + r.setState(structs.TaskStateDead, structs.NewTaskEvent(structs.TaskDriverFailure).SetDriverError(err)) + r.logger.Printf("[ERR] client: task directory for alloc %q task %q couldn't be found", r.alloc.ID, r.task.Name) + + // Non-restartable error + return + } + + for _, artifact := range r.task.Artifacts { + if err := getter.GetArtifact(artifact, taskDir, r.logger); err != nil { + r.setState(structs.TaskStateDead, + structs.NewTaskEvent(structs.TaskArtifactDownloadFailed).SetDownloadError(err)) + r.restartTracker.SetStartError(cstructs.NewRecoverableError(err, true)) + goto RESTART + } + } + + r.artifactsDownloaded = true + } + // Start the task if not yet started or it is being forced. This logic // is necessary because in the case of a restore the handle already // exists. r.handleLock.Lock() - handleEmpty := r.handle == nil + handleEmpty = r.handle == nil r.handleLock.Unlock() if handleEmpty { startErr := r.startTask() diff --git a/command/alloc_status.go b/command/alloc_status.go index ecba378f9..49e3844fa 100644 --- a/command/alloc_status.go +++ b/command/alloc_status.go @@ -205,6 +205,14 @@ func (c *AllocStatusCommand) taskStatus(alloc *api.Allocation) { } else { desc = "Failed to start task" } + case api.TaskDownloadingArtifacts: + desc = "Client is downloading artifacts" + case api.TaskArtifactDownloadFailed: + if event.DownloadError != "" { + desc = event.DownloadError + } else { + desc = "Failed to download artifacts" + } case api.TaskKilled: if event.KillError != "" { desc = event.KillError diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 4378155ff..293a9c9b6 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -1801,6 +1801,14 @@ const ( // TaskNotRestarting indicates that the task has failed and is not being // restarted because it has exceeded its restart policy. TaskNotRestarting = "Restarts Exceeded" + + // Task Downloading Artifacts means the task is downloading the artifacts + // specified in the task. + TaskDownloadingArtifacts = "Downloading Artifacts" + + // TaskArtifactDownloadFailed indicates that downloading the artifacts + // failed. + TaskArtifactDownloadFailed = "Failed Artifact Download" ) // TaskEvent is an event that effects the state of a task and contains meta-data @@ -1822,6 +1830,9 @@ type TaskEvent struct { // TaskRestarting fields. StartDelay int64 // The sleep period before restarting the task in unix nanoseconds. + + // Artifact Download fields + DownloadError string // Error downloading artifacts } func (te *TaskEvent) GoString() string { @@ -1880,6 +1891,13 @@ func (e *TaskEvent) SetRestartDelay(delay time.Duration) *TaskEvent { return e } +func (e *TaskEvent) SetDownloadError(err error) *TaskEvent { + if err != nil { + e.DownloadError = err.Error() + } + return e +} + // TaskArtifact is an artifact to download before running the task. type TaskArtifact struct { // GetterSource is the source to download an artifact using go-getter