From 715e576dfb9fdc1911065d0016d565402629d00b Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 25 Jan 2017 13:48:33 -0800 Subject: [PATCH 01/17] Dispatch command --- command/job_dispatch.go | 6 + .../docs/commands/job-dispatch.html.md.erb | 106 ++++++++++++++++++ .../source/docs/commands/status.html.md.erb | 2 + website/source/layouts/docs.erb | 3 + 4 files changed, 117 insertions(+) create mode 100644 website/source/docs/commands/job-dispatch.html.md.erb diff --git a/command/job_dispatch.go b/command/job_dispatch.go index 39e92a9f7..763260ec8 100644 --- a/command/job_dispatch.go +++ b/command/job_dispatch.go @@ -32,6 +32,12 @@ General Options: Dispatch Options: + -meta = + Meta takes a key/value pair seperated by "=". The metadata key will be + injected into the job's metadata. The flag can be provided more than + once to inject multiple metadata key/value pairs. Arbitrary injection is not + allowed. The parameterized job must allow the key to be injected. + -detach Return immediately instead of entering monitor mode. After job dispatch, the evaluation ID will be printed to the screen, which can be used to diff --git a/website/source/docs/commands/job-dispatch.html.md.erb b/website/source/docs/commands/job-dispatch.html.md.erb new file mode 100644 index 000000000..551aa2d45 --- /dev/null +++ b/website/source/docs/commands/job-dispatch.html.md.erb @@ -0,0 +1,106 @@ +--- +layout: "docs" +page_title: "Commands: job dispatch" +sidebar_current: "docs-commands-job-dispatch" +description: > + The dispatch command is used to create an instance of a parameterized job. +--- + +# Command: job dispatch + +~> The [`job dispatch`](/docs/commands/snapshot/agent.html) subcommand described here is + available only in version 0.5.3 and later. The release canidate is + downloadable on the [releases + page.](https://releases.hashicorp.com/nomad/0.5.3-rc1/) + +The `job dispatch` command is used to create new instances of a [parameterized +job](TODO). The parameterized job captures a job's configuration and runtime +requirements in a generic way and `dispatch` is used to provide the input for +the job to run against. One can think of the parameterized job as a function +definition and dispatch is used to invoke the function. + +## Usage + +``` +nomad job dispatch [options] [input source] +``` + +Dispatch creates an instance of a parameterized job. A data payload to the +dispatched instance can be provided via stdin by using "-" for the input source +or by specifiying a path to a file. Metadata can be supplied by using the meta +flag one or more times. + +Upon successfully creation, the dispatched job ID will be printed and the +triggered evaluation will be monitored. This can be disabled by supplying the +detach flag. + +On successful job submission and scheduling, exit code 0 will be returned. If +there are job placement issues encountered (unsatisfiable constraints, resource +exhaustion, etc), then the exit code will be 2. Any other errors, including +client connection issues or internal errors, are indicated by exit code 1. + +## General Options + +<%= partial "docs/commands/_general_options" %> + +## Run Options + +* `-meta`: Meta takes a key/value pair seperated by "=". The metadata key will + be injected into the job's metadata. The flag can be provided more than once + to inject multiple metadata key/value pairs. Arbitrary injection is not + allowed. The parameterized job must allow the key to be injected. + +* `-detach`: Return immediately instead of monitoring. A new evaluation ID + will be output, which can be used to examine the evaluation using the + [eval-status](/docs/commands/eval-status.html) command + +* `-verbose`: Show full information. + +## Examples + +Dispatch against a parameterized job with the ID "video-encode" and +passing in a configuration payload via stdin: + +``` +$ cat << EOF | nomad job dispatch video-encode - +{ + "s3-input": "https://s3-us-west-1.amazonaws.com/video-bucket/cb31dabb1", + "s3-output": "https://s3-us-west-1.amazonaws.com/video-bucket/a149adbe3", + "input-codec": "mp4", + "output-codec": "webm", + "quality": "1080p" +} +EOF +Dispatched Job ID = video-encode/dispatch-1485379325-cb38d00d +Evaluation ID = 31199841 + +==> Monitoring evaluation "31199841" + Evaluation triggered by job "example/dispatch-1485379325-cb38d00d" + Allocation "8254b85f" created: node "82ff9c50", group "cache" + Evaluation status changed: "pending" -> "complete" +==> Evaluation "31199841" finished with status "complete" +``` + +Dispatch against a parameterized job with the ID "video-encode" and +passing in a configuration payload via a file: + +``` +$ nomad job dispatch video-encode video-config.json +Dispatched Job ID = video-encode/dispatch-1485379325-cb38d00d +Evaluation ID = 31199841 + +==> Monitoring evaluation "31199841" + Evaluation triggered by job "example/dispatch-1485379325-cb38d00d" + Allocation "8254b85f" created: node "82ff9c50", group "cache" + Evaluation status changed: "pending" -> "complete" +==> Evaluation "31199841" finished with status "complete" +``` + +Dispatch against a parameterized job with the ID "video-encode" using the detach +flag: + +``` +$ nomad job dispatch -detach video-encode video-config.json +Dispatched Job ID = example/dispatch-1485380684-c37b3dba +Evaluation ID = d9034c4e +``` diff --git a/website/source/docs/commands/status.html.md.erb b/website/source/docs/commands/status.html.md.erb index 80e3e046d..0fb643ba0 100644 --- a/website/source/docs/commands/status.html.md.erb +++ b/website/source/docs/commands/status.html.md.erb @@ -152,3 +152,5 @@ b82f58b6 8bf94335 8d0331e9 cache run running 08/08/16 21:03:19 CD ed3665f5 8bf94335 8d0331e9 cache run running 08/08/16 21:03:19 CDT 24cfd201 8bf94335 8d0331e9 cache run running 08/08/16 21:03:19 CDT ``` + +TODO: CRON AND DISPATCH diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index 0fb9bd356..dad25d2b3 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -204,6 +204,9 @@ > inspect + > + job dispatch + > keygen From 78bfc6a4af0d1946884a3101e0ec39ffd4d437a7 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 25 Jan 2017 19:14:58 -0800 Subject: [PATCH 02/17] Parameterized page --- .../docs/commands/job-dispatch.html.md.erb | 8 +- .../job-specification/parameterized.html.md | 159 ++++++++++++++++++ .../docs/job-specification/periodic.html.md | 5 + 3 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 website/source/docs/job-specification/parameterized.html.md diff --git a/website/source/docs/commands/job-dispatch.html.md.erb b/website/source/docs/commands/job-dispatch.html.md.erb index 551aa2d45..cd5b24ffb 100644 --- a/website/source/docs/commands/job-dispatch.html.md.erb +++ b/website/source/docs/commands/job-dispatch.html.md.erb @@ -14,11 +14,15 @@ description: > page.](https://releases.hashicorp.com/nomad/0.5.3-rc1/) The `job dispatch` command is used to create new instances of a [parameterized -job](TODO). The parameterized job captures a job's configuration and runtime +job]. The parameterized job captures a job's configuration and runtime requirements in a generic way and `dispatch` is used to provide the input for the job to run against. One can think of the parameterized job as a function definition and dispatch is used to invoke the function. +Each time a job is dispatched, a unique job ID is generated. This allows a +caller to track the status of the job, much like a future or promise in some +programming languages. + ## Usage ``` @@ -104,3 +108,5 @@ $ nomad job dispatch -detach video-encode video-config.json Dispatched Job ID = example/dispatch-1485380684-c37b3dba Evaluation ID = d9034c4e ``` + +[parameterized job]: /docs/job-specification/parameterized.html "Nomad parameterized Job Specification" diff --git a/website/source/docs/job-specification/parameterized.html.md b/website/source/docs/job-specification/parameterized.html.md new file mode 100644 index 000000000..72ea3e297 --- /dev/null +++ b/website/source/docs/job-specification/parameterized.html.md @@ -0,0 +1,159 @@ +--- +layout: "docs" +page_title: "parameterized Stanza - Job Specification" +sidebar_current: "docs-job-specification-parameterized" +description: |- + A parameterized job is used to encapsulate a set of work that can be carried + out on various inputs much like a function definition. When the + `parameterized` stanza is added to a job, the job acts as a function to the + cluster as a whole. +--- + +# `parameterized` Stanza + + + + + + +
Placement + job -> **parameterized** +
+ +A parameterized job is used to encapsulate a set of work that can be carried out +on various inputs much like a function definition. When the `parameterized` +stanza is added to a job, the job acts as a function to the cluster as a whole. + +The `parameterized` stanza allows job operators to configure a job that carries +out a particular action, define its resource requirements and configure how +inputs and configuration are retreived by the tasks within the job. + +To invoke a parameterized job, [`nomad job +dispatch`][dispatch command] or the equivalent HTTP APIs are +used. When dispatching against a parameterized job, an opaque payload and +metadata may be injected into the job. These inputs to the parameterized job act +like arguments to a function. The job consumes them to change it's behavior, +without exposing the implementation details to the caller. + +To that end, tasks within the job can add a [`dispatch_input`][dispatch_input] stanza that +defines where on the filesystem this payload gets written to. An example payload +would be a task's JSON configuration. + +Further, certain metadata may be marked as required when dispatching a job so it +can be used to inject configuration directly into a task's arguments using +[interpolation]. An example of this would be to require a run ID key that +could be used to lookup the work the job is suppose to do from a management +service or database. + +Each time a job is dispatched, a unique job ID is generated. This allows a +caller to track the status of the job, much like a future or promise in some +programming languages. + +```hcl +job "docs" { + parameterized { + payload = "required" + required_meta = ["dispatcher_email"] + optional_meta = ["pager_email"] + } +} +``` + +## `parameterized` Requirements + + - The job's [scheduler type][batch-type] must be `batch`. + +## `parameterized` Parameters + +- `payload` `(string: "optional")` - Specifies the requirement of providing a + payload when dispatching against the parameterized job. The options for this + field are: + + - `"optional"` - A payload is optional when dispatching against the job. + + - `"required"` - A payload must be provided when dispatching against the job. + + - `"forbidden"` - A payload is forbidden when dispatching against the job. + +- `required_meta` `([]string: nil)` - Specifies the set of metadata keys that + must be provided when dispatching against the job. + +- `optional_meta` `([]string: nil)` - Specifies the set of metadata keys that + may be provided when dispatching against the job. + +## `parameterized` Examples + +The following examples show non-runnable example parameterized jobs: + +### Required Inputs + +This example shows a parameterized job that requires both a payload and +metadata: + +```hcl +job "video-encode" { + ... + type = "batch" + + parameterized { + payload = "required" + required_meta = ["dispatcher_email"] + } + + group "encode" { + ... + + task "ffmpeg" { + driver = "exec" + + config { + command = "ffmpeg-wrapper" + + # When dispatched, the payload is written to a file that is then read by + # the created task upon startup + args = ["-config=${NOMAD_TASK_DIR}/config.json"] + } + + dispatch_input { + file = "config.json" + } + } + } +} +``` + +### Metadata Interpolation + +```hcl +job "email-blast" { + ... + type = "batch" + + parameterized { + payload = "forbidden" + required_meta = ["CAMPAIGN_ID"] + } + + group "emails" { + ... + + task "emailer" { + driver = "exec" + + config { + command = "emailer" + + # The campagain ID is interpolated and injected into the task's + # arguments + args = ["-campaign=${NOMAD_META_CAMPAIGN_ID}"] + } + } + } +} +``` + +[batch-type]: /docs/job-specification/job.html#type "Batch scheduler type" +[dispatch command]: /docs/commands/job-dispatch.html "Nomad Job Dispatch Command" +[resources]: /docs/job-specification/resources.html "Nomad resources Job Specification" +[interpolation]: /docs/runtime/interpolation.html "Nomad Runtime Interpolation" +[dispatch_input]: /docs/job-specification/dispatch-input.html "Nomad dispatch_input Job Specification" diff --git a/website/source/docs/job-specification/periodic.html.md b/website/source/docs/job-specification/periodic.html.md index 8b010c149..4f480448f 100644 --- a/website/source/docs/job-specification/periodic.html.md +++ b/website/source/docs/job-specification/periodic.html.md @@ -35,6 +35,10 @@ job "docs" { The periodic expression is always evaluated in the **UTC timezone** to ensure consistent evaluation when Nomad spans multiple time zones. +## `periodic` Requirements + + - The job's [scheduler type][batch-type] must be `batch`. + ## `periodic` Parameters - `cron` `(string: )` - Specifies a cron expression configuring the @@ -60,4 +64,5 @@ periodic { } ``` +[batch-type]: /docs/job-specification/job.html#type "Batch scheduler type" [cron]: https://github.com/gorhill/cronexpr#implementation "List of cron expressions" From b456e7fff1a2f8a9ac9b2449ae9db93b78c0d92c Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 25 Jan 2017 20:49:06 -0800 Subject: [PATCH 03/17] dispatch_payload --- .../dispatch-payload.html.md | 62 +++++++++++++++++++ .../job-specification/parameterized.html.md | 7 ++- website/source/layouts/docs.erb | 6 ++ 3 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 website/source/docs/job-specification/dispatch-payload.html.md diff --git a/website/source/docs/job-specification/dispatch-payload.html.md b/website/source/docs/job-specification/dispatch-payload.html.md new file mode 100644 index 000000000..70fed94d6 --- /dev/null +++ b/website/source/docs/job-specification/dispatch-payload.html.md @@ -0,0 +1,62 @@ +--- +layout: "docs" +page_title: "dispatch_payload Stanza - Job Specification" +sidebar_current: "docs-job-specification-dispatch-payload" +description: |- + The "dispatch_payload" stanza allows a task to access dispatch payloads. + to +--- + +# `dispatch_payload` Stanza + + + + + + +
Placement + job -> group -> task -> **dispatch_payload** +
+ +The `dispatch_payload` stanza is used in conjuction with a [`paramterized`][parameterized] job +that expects a payload. When the job is dispatched with a payload, the payload +will be made available to any task that has a `dispatch_payload` stanza. The +payload will be written to the configured file before the task is started. This +allows the task to use the payload as input or configuration. + +```hcl +job "docs" { + group "example" { + task "server" { + dispatch_payload { + file = "config.json" + } + } + } +} +``` + +## `dispatch_payload` Parameters + +- `file` `(string: "")` - Specifies the file name to write the content of + dispatch payload to. The file is written to a [task's local + directory][localdir]. + +## `dispatch_payload` Examples + +The following examples only show the `dispatch_payload` stanzas. Remember that the +`dispatch_payload` stanza is only valid in the placements listed above. + +### Write Payload to a File + +This example shows a `dispatch_payload` block in a parameterized job that writes +the payload to a `config.json` file. + +```hcl +dispatch_payload { + file = "config.json" +} +``` + +[localdir]: /docs/runtime/environment.html#local_ "Task Local Directory" +[parameterized]: /docs/job-specification/parameterized.html "Nomad parameterized Job Specification" diff --git a/website/source/docs/job-specification/parameterized.html.md b/website/source/docs/job-specification/parameterized.html.md index 72ea3e297..ee33867ce 100644 --- a/website/source/docs/job-specification/parameterized.html.md +++ b/website/source/docs/job-specification/parameterized.html.md @@ -35,7 +35,8 @@ metadata may be injected into the job. These inputs to the parameterized job act like arguments to a function. The job consumes them to change it's behavior, without exposing the implementation details to the caller. -To that end, tasks within the job can add a [`dispatch_input`][dispatch_input] stanza that +To that end, tasks within the job can add a +[`dispatch_payload`][dispatch_payload] stanza that defines where on the filesystem this payload gets written to. An example payload would be a task's JSON configuration. @@ -114,7 +115,7 @@ job "video-encode" { args = ["-config=${NOMAD_TASK_DIR}/config.json"] } - dispatch_input { + dispatch_payload { file = "config.json" } } @@ -156,4 +157,4 @@ job "email-blast" { [dispatch command]: /docs/commands/job-dispatch.html "Nomad Job Dispatch Command" [resources]: /docs/job-specification/resources.html "Nomad resources Job Specification" [interpolation]: /docs/runtime/interpolation.html "Nomad Runtime Interpolation" -[dispatch_input]: /docs/job-specification/dispatch-input.html "Nomad dispatch_input Job Specification" +[dispatch_payload]: /docs/job-specification/dispatch-payload.html "Nomad dispatch_payload Job Specification" diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index dad25d2b3..f3b00a5fa 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -48,6 +48,9 @@ > constraint + > + dispatch_payload + > env @@ -69,6 +72,9 @@ > network + > + parameterized + > periodic From 8f8215c09365e07ce6278a249b9a168b7b328444 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 25 Jan 2017 20:51:00 -0800 Subject: [PATCH 04/17] Fix bad link --- website/source/docs/commands/job-dispatch.html.md.erb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/website/source/docs/commands/job-dispatch.html.md.erb b/website/source/docs/commands/job-dispatch.html.md.erb index cd5b24ffb..cce195491 100644 --- a/website/source/docs/commands/job-dispatch.html.md.erb +++ b/website/source/docs/commands/job-dispatch.html.md.erb @@ -8,10 +8,9 @@ description: > # Command: job dispatch -~> The [`job dispatch`](/docs/commands/snapshot/agent.html) subcommand described here is - available only in version 0.5.3 and later. The release canidate is - downloadable on the [releases - page.](https://releases.hashicorp.com/nomad/0.5.3-rc1/) +~> The `job dispatch` subcommand described here is available only in version +0.5.3 and later. The release canidate is downloadable on the [releases +page.](https://releases.hashicorp.com/nomad/0.5.3-rc1/) The `job dispatch` command is used to create new instances of a [parameterized job]. The parameterized job captures a job's configuration and runtime From 314435261dc609016507caaed64e36d8937c277e Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 25 Jan 2017 21:06:16 -0800 Subject: [PATCH 05/17] Rename dispatch_input to dispatch_payload --- api/tasks.go | 34 ++++++++++----------- client/task_runner.go | 4 +-- client/task_runner_test.go | 2 +- jobspec/parse.go | 16 +++++----- jobspec/parse_test.go | 2 +- jobspec/test-fixtures/parameterized_job.hcl | 2 +- nomad/structs/diff.go | 4 +-- nomad/structs/diff_test.go | 28 ++++++++--------- nomad/structs/structs.go | 24 +++++++-------- nomad/structs/structs_test.go | 4 +-- 10 files changed, 60 insertions(+), 60 deletions(-) diff --git a/api/tasks.go b/api/tasks.go index 7ba6ea0dd..b6b184072 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -141,28 +141,28 @@ type LogConfig struct { MaxFileSizeMB int } -// DispatchInputConfig configures how a task gets its input from a job dispatch -type DispatchInputConfig struct { +// DispatchPayloadConfig configures how a task gets its input from a job dispatch +type DispatchPayloadConfig struct { File string } // Task is a single process in a task group. type Task struct { - Name string - Driver string - User string - Config map[string]interface{} - Constraints []*Constraint - Env map[string]string - Services []Service - Resources *Resources - Meta map[string]string - KillTimeout time.Duration - LogConfig *LogConfig - Artifacts []*TaskArtifact - Vault *Vault - Templates []*Template - DispatchInput *DispatchInputConfig + Name string + Driver string + User string + Config map[string]interface{} + Constraints []*Constraint + Env map[string]string + Services []Service + Resources *Resources + Meta map[string]string + KillTimeout time.Duration + LogConfig *LogConfig + Artifacts []*TaskArtifact + Vault *Vault + Templates []*Template + DispatchPayload *DispatchPayloadConfig } // TaskArtifact is used to download artifacts before running a task. diff --git a/client/task_runner.go b/client/task_runner.go index 61ca21ccc..e872e434d 100644 --- a/client/task_runner.go +++ b/client/task_runner.go @@ -753,9 +753,9 @@ func (r *TaskRunner) prestart(resultCh chan bool) { // If the job is a dispatch job and there is a payload write it to disk requirePayload := len(r.alloc.Job.Payload) != 0 && - (r.task.DispatchInput != nil && r.task.DispatchInput.File != "") + (r.task.DispatchPayload != nil && r.task.DispatchPayload.File != "") if !r.payloadRendered && requirePayload { - renderTo := filepath.Join(r.taskDir.LocalDir, r.task.DispatchInput.File) + renderTo := filepath.Join(r.taskDir.LocalDir, r.task.DispatchPayload.File) decoded, err := snappy.Decode(nil, r.alloc.Job.Payload) if err != nil { r.setState( diff --git a/client/task_runner_test.go b/client/task_runner_test.go index b1afea5d7..e7e2b19db 100644 --- a/client/task_runner_test.go +++ b/client/task_runner_test.go @@ -1232,7 +1232,7 @@ func TestTaskRunner_SimpleRun_Dispatch(t *testing.T) { "run_for": "1s", } fileName := "test" - task.DispatchInput = &structs.DispatchInputConfig{ + task.DispatchPayload = &structs.DispatchPayloadConfig{ File: fileName, } alloc.Job.ParameterizedJob = &structs.ParameterizedJobConfig{} diff --git a/jobspec/parse.go b/jobspec/parse.go index a4fc88e15..7c4215698 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -564,7 +564,7 @@ func parseTasks(jobName string, taskGroupName string, result *[]*structs.Task, l "artifact", "config", "constraint", - "dispatch_input", + "dispatch_payload", "driver", "env", "kill_timeout", @@ -587,7 +587,7 @@ func parseTasks(jobName string, taskGroupName string, result *[]*structs.Task, l delete(m, "artifact") delete(m, "config") delete(m, "constraint") - delete(m, "dispatch_input") + delete(m, "dispatch_payload") delete(m, "env") delete(m, "logs") delete(m, "meta") @@ -747,10 +747,10 @@ func parseTasks(jobName string, taskGroupName string, result *[]*structs.Task, l t.Vault = v } - // If we have a dispatch_input block parse that - if o := listVal.Filter("dispatch_input"); len(o.Items) > 0 { + // If we have a dispatch_payload block parse that + if o := listVal.Filter("dispatch_payload"); len(o.Items) > 0 { if len(o.Items) > 1 { - return fmt.Errorf("only one dispatch_input block is allowed in a task. Number of dispatch_input blocks found: %d", len(o.Items)) + return fmt.Errorf("only one dispatch_payload block is allowed in a task. Number of dispatch_payload blocks found: %d", len(o.Items)) } var m map[string]interface{} dispatchBlock := o.Items[0] @@ -760,15 +760,15 @@ func parseTasks(jobName string, taskGroupName string, result *[]*structs.Task, l "file", } if err := checkHCLKeys(dispatchBlock.Val, valid); err != nil { - return multierror.Prefix(err, fmt.Sprintf("'%s', dispatch_input ->", n)) + return multierror.Prefix(err, fmt.Sprintf("'%s', dispatch_payload ->", n)) } if err := hcl.DecodeObject(&m, dispatchBlock.Val); err != nil { return err } - t.DispatchInput = &structs.DispatchInputConfig{} - if err := mapstructure.WeakDecode(m, t.DispatchInput); err != nil { + t.DispatchPayload = &structs.DispatchPayloadConfig{} + if err := mapstructure.WeakDecode(m, t.DispatchPayload); err != nil { return err } } diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index e9c6c8944..7bd404a55 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -571,7 +571,7 @@ func TestParse(t *testing.T) { MaxFiles: 10, MaxFileSizeMB: 10, }, - DispatchInput: &structs.DispatchInputConfig{ + DispatchPayload: &structs.DispatchPayloadConfig{ File: "foo/bar", }, }, diff --git a/jobspec/test-fixtures/parameterized_job.hcl b/jobspec/test-fixtures/parameterized_job.hcl index 1b95a7de2..1f31e583d 100644 --- a/jobspec/test-fixtures/parameterized_job.hcl +++ b/jobspec/test-fixtures/parameterized_job.hcl @@ -11,7 +11,7 @@ job "parameterized_job" { driver = "docker" resources {} - dispatch_input { + dispatch_payload { file = "foo/bar" } } diff --git a/nomad/structs/diff.go b/nomad/structs/diff.go index 9fca44c8b..82dc8ea89 100644 --- a/nomad/structs/diff.go +++ b/nomad/structs/diff.go @@ -375,8 +375,8 @@ func (t *Task) Diff(other *Task, contextual bool) (*TaskDiff, error) { diff.Objects = append(diff.Objects, lDiff) } - // Dispatch Input diff - dDiff := primitiveObjectDiff(t.DispatchInput, other.DispatchInput, nil, "DispatchInput", contextual) + // Dispatch payload diff + dDiff := primitiveObjectDiff(t.DispatchPayload, other.DispatchPayload, nil, "DispatchPayload", contextual) if dDiff != nil { diff.Objects = append(diff.Objects, dDiff) } diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index 5cd258a55..0fa6e18dc 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -3666,10 +3666,10 @@ func TestTaskDiff(t *testing.T) { }, }, { - // DispatchInput added + // DispatchPayload added Old: &Task{}, New: &Task{ - DispatchInput: &DispatchInputConfig{ + DispatchPayload: &DispatchPayloadConfig{ File: "foo", }, }, @@ -3678,7 +3678,7 @@ func TestTaskDiff(t *testing.T) { Objects: []*ObjectDiff{ { Type: DiffTypeAdded, - Name: "DispatchInput", + Name: "DispatchPayload", Fields: []*FieldDiff{ { Type: DiffTypeAdded, @@ -3692,9 +3692,9 @@ func TestTaskDiff(t *testing.T) { }, }, { - // DispatchInput deleted + // DispatchPayload deleted Old: &Task{ - DispatchInput: &DispatchInputConfig{ + DispatchPayload: &DispatchPayloadConfig{ File: "foo", }, }, @@ -3704,7 +3704,7 @@ func TestTaskDiff(t *testing.T) { Objects: []*ObjectDiff{ { Type: DiffTypeDeleted, - Name: "DispatchInput", + Name: "DispatchPayload", Fields: []*FieldDiff{ { Type: DiffTypeDeleted, @@ -3718,14 +3718,14 @@ func TestTaskDiff(t *testing.T) { }, }, { - // Dispatch input edited + // Dispatch payload edited Old: &Task{ - DispatchInput: &DispatchInputConfig{ + DispatchPayload: &DispatchPayloadConfig{ File: "foo", }, }, New: &Task{ - DispatchInput: &DispatchInputConfig{ + DispatchPayload: &DispatchPayloadConfig{ File: "bar", }, }, @@ -3734,7 +3734,7 @@ func TestTaskDiff(t *testing.T) { Objects: []*ObjectDiff{ { Type: DiffTypeEdited, - Name: "DispatchInput", + Name: "DispatchPayload", Fields: []*FieldDiff{ { Type: DiffTypeEdited, @@ -3748,16 +3748,16 @@ func TestTaskDiff(t *testing.T) { }, }, { - // DispatchInput edited with context. Place holder for if more + // DispatchPayload edited with context. Place holder for if more // fields are added Contextual: true, Old: &Task{ - DispatchInput: &DispatchInputConfig{ + DispatchPayload: &DispatchPayloadConfig{ File: "foo", }, }, New: &Task{ - DispatchInput: &DispatchInputConfig{ + DispatchPayload: &DispatchPayloadConfig{ File: "bar", }, }, @@ -3766,7 +3766,7 @@ func TestTaskDiff(t *testing.T) { Objects: []*ObjectDiff{ { Type: DiffTypeEdited, - Name: "DispatchInput", + Name: "DispatchPayload", Fields: []*FieldDiff{ { Type: DiffTypeEdited, diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index ceb7df38a..71c437d65 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -1694,22 +1694,22 @@ func DispatchedID(templateID string, t time.Time) string { return fmt.Sprintf("%s%s%d-%s", templateID, DispatchLaunchSuffix, t.Unix(), u) } -// DispatchInputConfig configures how a task gets its input from a job dispatch -type DispatchInputConfig struct { +// DispatchPayloadConfig configures how a task gets its input from a job dispatch +type DispatchPayloadConfig struct { // File specifies a relative path to where the input data should be written File string } -func (d *DispatchInputConfig) Copy() *DispatchInputConfig { +func (d *DispatchPayloadConfig) Copy() *DispatchPayloadConfig { if d == nil { return nil } - nd := new(DispatchInputConfig) + nd := new(DispatchPayloadConfig) *nd = *d return nd } -func (d *DispatchInputConfig) Validate() error { +func (d *DispatchPayloadConfig) Validate() error { // Verify the destination doesn't escape escaped, err := PathEscapesAllocDir("task/local/", d.File) if err != nil { @@ -2272,8 +2272,8 @@ type Task struct { // Resources is the resources needed by this task Resources *Resources - // DispatchInput configures how the task retrieves its input from a dispatch - DispatchInput *DispatchInputConfig + // DispatchPayload configures how the task retrieves its input from a dispatch + DispatchPayload *DispatchPayloadConfig // Meta is used to associate arbitrary metadata with this // task. This is opaque to Nomad. @@ -2312,7 +2312,7 @@ func (t *Task) Copy() *Task { nt.Vault = nt.Vault.Copy() nt.Resources = nt.Resources.Copy() nt.Meta = helper.CopyMapStringString(nt.Meta) - nt.DispatchInput = nt.DispatchInput.Copy() + nt.DispatchPayload = nt.DispatchPayload.Copy() if t.Artifacts != nil { artifacts := make([]*TaskArtifact, 0, len(t.Artifacts)) @@ -2477,10 +2477,10 @@ func (t *Task) Validate(ephemeralDisk *EphemeralDisk) error { } } - // Validate the dispatch input block if there - if t.DispatchInput != nil { - if err := t.DispatchInput.Validate(); err != nil { - mErr.Errors = append(mErr.Errors, fmt.Errorf("Dispatch Input validation failed: %v", err)) + // Validate the dispatch payload block if there + if t.DispatchPayload != nil { + if err := t.DispatchPayload.Validate(); err != nil { + mErr.Errors = append(mErr.Errors, fmt.Errorf("Dispatch Payload validation failed: %v", err)) } } diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index 72c097224..24bbc83d1 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -1518,8 +1518,8 @@ func TestParameterizedJobConfig_Canonicalize(t *testing.T) { } } -func TestDispatchInputConfig_Validate(t *testing.T) { - d := &DispatchInputConfig{ +func TestDispatchPayloadConfig_Validate(t *testing.T) { + d := &DispatchPayloadConfig{ File: "foo", } From 98c17e519571aade164f3fd0a35143c135baf3fe Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 25 Jan 2017 21:13:18 -0800 Subject: [PATCH 06/17] Rename meta/parsing --- jobspec/parse.go | 36 ++----------------- jobspec/test-fixtures/parameterized_job.hcl | 6 ++-- nomad/structs/structs.go | 4 +-- .../job-specification/parameterized.html.md | 20 +++++------ 4 files changed, 16 insertions(+), 50 deletions(-) diff --git a/jobspec/parse.go b/jobspec/parse.go index 7c4215698..1bb6c2bd3 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -1259,12 +1259,11 @@ func parseParameterizedJob(result **structs.ParameterizedJobConfig, list *ast.Ob return err } - delete(m, "meta") - // Check for invalid keys valid := []string{ "payload", - "meta_keys", + "meta_required", + "meta_optional", } if err := checkHCLKeys(o.Val, valid); err != nil { return err @@ -1276,37 +1275,6 @@ func parseParameterizedJob(result **structs.ParameterizedJobConfig, list *ast.Ob return err } - var listVal *ast.ObjectList - if ot, ok := o.Val.(*ast.ObjectType); ok { - listVal = ot.List - } else { - return fmt.Errorf("parameterized block should be an object") - } - - // Parse the meta block - if metaList := listVal.Filter("meta_keys"); len(metaList.Items) > 0 { - // Get our resource object - o := metaList.Items[0] - - var m map[string]interface{} - if err := hcl.DecodeObject(&m, o.Val); err != nil { - return err - } - - // Check for invalid keys - valid := []string{ - "optional", - "required", - } - if err := checkHCLKeys(o.Val, valid); err != nil { - return err - } - - if err := mapstructure.WeakDecode(m, &d); err != nil { - return err - } - } - *result = &d return nil } diff --git a/jobspec/test-fixtures/parameterized_job.hcl b/jobspec/test-fixtures/parameterized_job.hcl index 1f31e583d..a8c5bfe5c 100644 --- a/jobspec/test-fixtures/parameterized_job.hcl +++ b/jobspec/test-fixtures/parameterized_job.hcl @@ -1,10 +1,8 @@ job "parameterized_job" { parameterized { payload = "required" - meta_keys { - required = ["foo", "bar"] - optional = ["baz", "bam"] - } + meta_required = ["foo", "bar"] + meta_optional = ["baz", "bam"] } group "foo" { task "bar" { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 71c437d65..582d0f3ba 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -1647,10 +1647,10 @@ type ParameterizedJobConfig struct { Payload string // MetaRequired is metadata keys that must be specified by the dispatcher - MetaRequired []string `mapstructure:"required"` + MetaRequired []string `mapstructure:"meta_required"` // MetaOptional is metadata keys that may be specified by the dispatcher - MetaOptional []string `mapstructure:"optional"` + MetaOptional []string `mapstructure:"meta_optional"` } func (d *ParameterizedJobConfig) Validate() error { diff --git a/website/source/docs/job-specification/parameterized.html.md b/website/source/docs/job-specification/parameterized.html.md index ee33867ce..59e0cfdb6 100644 --- a/website/source/docs/job-specification/parameterized.html.md +++ b/website/source/docs/job-specification/parameterized.html.md @@ -54,8 +54,8 @@ programming languages. job "docs" { parameterized { payload = "required" - required_meta = ["dispatcher_email"] - optional_meta = ["pager_email"] + meta_required = ["dispatcher_email"] + meta_optional = ["pager_email"] } } ``` @@ -66,6 +66,12 @@ job "docs" { ## `parameterized` Parameters +- `meta_optional` `([]string: nil)` - Specifies the set of metadata keys that + may be provided when dispatching against the job. + +- `meta_required` `([]string: nil)` - Specifies the set of metadata keys that + must be provided when dispatching against the job. + - `payload` `(string: "optional")` - Specifies the requirement of providing a payload when dispatching against the parameterized job. The options for this field are: @@ -76,12 +82,6 @@ job "docs" { - `"forbidden"` - A payload is forbidden when dispatching against the job. -- `required_meta` `([]string: nil)` - Specifies the set of metadata keys that - must be provided when dispatching against the job. - -- `optional_meta` `([]string: nil)` - Specifies the set of metadata keys that - may be provided when dispatching against the job. - ## `parameterized` Examples The following examples show non-runnable example parameterized jobs: @@ -98,7 +98,7 @@ job "video-encode" { parameterized { payload = "required" - required_meta = ["dispatcher_email"] + meta_required = ["dispatcher_email"] } group "encode" { @@ -132,7 +132,7 @@ job "email-blast" { parameterized { payload = "forbidden" - required_meta = ["CAMPAIGN_ID"] + meta_required = ["CAMPAIGN_ID"] } group "emails" { From f49912cbabf44a298d129c8aad816ea55d205a9b Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 25 Jan 2017 21:16:18 -0800 Subject: [PATCH 07/17] alphabetical --- .../docs/job-specification/artifact.html.md | 8 +++---- .../docs/job-specification/template.html.md | 22 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/website/source/docs/job-specification/artifact.html.md b/website/source/docs/job-specification/artifact.html.md index e773241d2..bcd4ef394 100644 --- a/website/source/docs/job-specification/artifact.html.md +++ b/website/source/docs/job-specification/artifact.html.md @@ -51,15 +51,15 @@ before the starting the task. default value is to place the binary in `local/`. The destination is treated as a directory and source files will be downloaded into that directory path. -- `source` `(string: )` - Specifies the URL of the artifact to download. - Only `http`, `https`, and `s3` URLs are supported. See [`go-getter`][go-getter] - for details. - - `options` `(map: nil)` - Specifies configuration parameters to fetch the artifact. The key-value pairs map directly to parameters appended to the supplied `source` URL. Please see the [`go-getter` documentation][go-getter] for a complete list of options and examples +- `source` `(string: )` - Specifies the URL of the artifact to download. + Only `http`, `https`, and `s3` URLs are supported. See [`go-getter`][go-getter] + for details. + ## `artifact` Examples The following examples only show the `artifact` stanzas. Remember that the diff --git a/website/source/docs/job-specification/template.html.md b/website/source/docs/job-specification/template.html.md index 319e2c0cd..d1d674492 100644 --- a/website/source/docs/job-specification/template.html.md +++ b/website/source/docs/job-specification/template.html.md @@ -47,19 +47,17 @@ README][ct]. ## `template` Parameters -- `source` `(string: "")` - Specifies the path to the template to be rendered. - One of `source` or `data` must be specified, but not both. This source can - optionally be fetched using an [`artifact`][artifact] resource. This template - must exist on the machine prior to starting the task; it is not possible to - reference a template inside of a Docker container, for example. - -- `destination` `(string: )` - Specifies the location where the - resulting template should be rendered, relative to the task directory. +- `change_signal` `(string: "")` - Specifies the signal to send to the task as a + string like `"SIGUSR1"` or `"SIGINT"`. This option is required if the + `change_mode` is `signal`. - `data` `(string: "")` - Specifies the raw template to execute. One of `source` or `data` must be specified, but not both. This is useful for smaller templates, but we recommend using `source` for larger templates. +- `destination` `(string: )` - Specifies the location where the + resulting template should be rendered, relative to the task directory. + - `change_mode` `(string: "restart")` - Specifies the behavior Nomad should take if the rendered template changes. The possible values are: @@ -67,9 +65,11 @@ README][ct]. - `"restart"` - restart the task - `"signal"` - send a configurable signal to the task -- `change_signal` `(string: "")` - Specifies the signal to send to the task as a - string like `"SIGUSR1"` or `"SIGINT"`. This option is required if the - `change_mode` is `signal`. +- `source` `(string: "")` - Specifies the path to the template to be rendered. + One of `source` or `data` must be specified, but not both. This source can + optionally be fetched using an [`artifact`][artifact] resource. This template + must exist on the machine prior to starting the task; it is not possible to + reference a template inside of a Docker container, for example. - `splay` `(string: "5s")` - Specifies a random amount of time to wait between 0ms and the given splay value before invoking the change mode. This is From 82fb381fc5184d74607e69071e297da8ef458b6e Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 25 Jan 2017 21:20:50 -0800 Subject: [PATCH 08/17] missing tests --- nomad/structs/diff.go | 4 ++-- nomad/structs/diff_test.go | 36 ++++++++++++++++++------------------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/nomad/structs/diff.go b/nomad/structs/diff.go index 82dc8ea89..f4ec05e69 100644 --- a/nomad/structs/diff.go +++ b/nomad/structs/diff.go @@ -667,11 +667,11 @@ func parameterizedJobDiff(old, new *ParameterizedJobConfig, contextual bool) *Ob diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual) // Meta diffs - if optionalDiff := stringSetDiff(old.MetaOptional, new.MetaOptional, "OptionalMeta", contextual); optionalDiff != nil { + if optionalDiff := stringSetDiff(old.MetaOptional, new.MetaOptional, "MetaOptional", contextual); optionalDiff != nil { diff.Objects = append(diff.Objects, optionalDiff) } - if requiredDiff := stringSetDiff(old.MetaRequired, new.MetaRequired, "RequiredMeta", contextual); requiredDiff != nil { + if requiredDiff := stringSetDiff(old.MetaRequired, new.MetaRequired, "MetaRequired", contextual); requiredDiff != nil { diff.Objects = append(diff.Objects, requiredDiff) } diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index 0fa6e18dc..b5ca58a78 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -904,11 +904,11 @@ func TestJobDiff(t *testing.T) { Objects: []*ObjectDiff{ { Type: DiffTypeAdded, - Name: "OptionalMeta", + Name: "MetaOptional", Fields: []*FieldDiff{ { Type: DiffTypeAdded, - Name: "OptionalMeta", + Name: "MetaOptional", Old: "", New: "foo", }, @@ -916,11 +916,11 @@ func TestJobDiff(t *testing.T) { }, { Type: DiffTypeAdded, - Name: "RequiredMeta", + Name: "MetaRequired", Fields: []*FieldDiff{ { Type: DiffTypeAdded, - Name: "RequiredMeta", + Name: "MetaRequired", Old: "", New: "bar", }, @@ -958,11 +958,11 @@ func TestJobDiff(t *testing.T) { Objects: []*ObjectDiff{ { Type: DiffTypeDeleted, - Name: "OptionalMeta", + Name: "MetaOptional", Fields: []*FieldDiff{ { Type: DiffTypeDeleted, - Name: "OptionalMeta", + Name: "MetaOptional", Old: "foo", New: "", }, @@ -970,11 +970,11 @@ func TestJobDiff(t *testing.T) { }, { Type: DiffTypeDeleted, - Name: "RequiredMeta", + Name: "MetaRequired", Fields: []*FieldDiff{ { Type: DiffTypeDeleted, - Name: "RequiredMeta", + Name: "MetaRequired", Old: "bar", New: "", }, @@ -1018,17 +1018,17 @@ func TestJobDiff(t *testing.T) { Objects: []*ObjectDiff{ { Type: DiffTypeEdited, - Name: "OptionalMeta", + Name: "MetaOptional", Fields: []*FieldDiff{ { Type: DiffTypeAdded, - Name: "OptionalMeta", + Name: "MetaOptional", Old: "", New: "bam", }, { Type: DiffTypeDeleted, - Name: "OptionalMeta", + Name: "MetaOptional", Old: "foo", New: "", }, @@ -1036,17 +1036,17 @@ func TestJobDiff(t *testing.T) { }, { Type: DiffTypeEdited, - Name: "RequiredMeta", + Name: "MetaRequired", Fields: []*FieldDiff{ { Type: DiffTypeAdded, - Name: "RequiredMeta", + Name: "MetaRequired", Old: "", New: "bang", }, { Type: DiffTypeDeleted, - Name: "RequiredMeta", + Name: "MetaRequired", Old: "bar", New: "", }, @@ -1091,11 +1091,11 @@ func TestJobDiff(t *testing.T) { Objects: []*ObjectDiff{ { Type: DiffTypeNone, - Name: "OptionalMeta", + Name: "MetaOptional", Fields: []*FieldDiff{ { Type: DiffTypeNone, - Name: "OptionalMeta", + Name: "MetaOptional", Old: "foo", New: "foo", }, @@ -1103,11 +1103,11 @@ func TestJobDiff(t *testing.T) { }, { Type: DiffTypeNone, - Name: "RequiredMeta", + Name: "MetaRequired", Fields: []*FieldDiff{ { Type: DiffTypeNone, - Name: "RequiredMeta", + Name: "MetaRequired", Old: "bar", New: "bar", }, From 62e6596ea1d9bd7f4c4bf7df5e5303b48c12f2b3 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 25 Jan 2017 21:35:53 -0800 Subject: [PATCH 09/17] API --- website/source/docs/http/job.html.md | 47 +++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/website/source/docs/http/job.html.md b/website/source/docs/http/job.html.md index 2e067f9ca..ca240d964 100644 --- a/website/source/docs/http/job.html.md +++ b/website/source/docs/http/job.html.md @@ -137,7 +137,6 @@ region is used; another region can be specified using the `?region=` query param "ModifyIndex": 14 } ``` - @@ -333,6 +332,52 @@ region is used; another region can be specified using the `?region=` query param +
+
Description
+
+ Dispatch a new instance of a parameterized job. +
+ +
Method
+
PUT or POST
+ +
URL
+
`/v1/job//dispatch`
+ +
Parameters
+
+
    +
  • + Payload + optional + A `[]byte` array encoded as a base64 string. +
    • +
  • + Meta + optional + A `map[string]string` of metadata keys to their values. +
    • +
+
+ +
Returns
+
+ + ```javascript + { + "KnownLeader": false, + "LastContact": 0, + "Index": 13, + "JobCreateIndex": 12, + "EvalCreateIndex": 13, + "EvalID": "e5f55fac-bc69-119d-528a-1fc7ade5e02c", + "DispatchedJobID": "example/dispatch-1485408778-81644024" + } + ``` + +
+
+
Description
From 30abdb09cc4c768aab07491cf5e5002426259956 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 25 Jan 2017 22:15:00 -0800 Subject: [PATCH 10/17] JSON job --- website/source/docs/http/json-jobs.html.md | 41 ++++++++++++++++++- .../dispatch-payload.html.md | 2 +- .../source/docs/job-specification/job.html.md | 10 ++++- .../job-specification/parameterized.html.md | 4 +- .../docs/job-specification/task.html.md | 4 ++ 5 files changed, 55 insertions(+), 6 deletions(-) diff --git a/website/source/docs/http/json-jobs.html.md b/website/source/docs/http/json-jobs.html.md index f86aa7559..ed279db5a 100644 --- a/website/source/docs/http/json-jobs.html.md +++ b/website/source/docs/http/json-jobs.html.md @@ -137,7 +137,10 @@ Below is an example of a JSON object that submits a `periodic` job to Nomad: }, "RelativeDest":"local/" } - ] + ], + "DispatchPayload": { + "File": "config.json" + } } ], "RestartPolicy":{ @@ -165,7 +168,17 @@ Below is an example of a JSON object that submits a `periodic` job to Nomad: "Meta":{ "foo":"bar", "baz":"pipe" - } + }, + "ParameterizedJob": { + "Payload": "required", + "MetaRequired": [ + "foo" + ], + "MetaOptional": [ + "bar" + ] + }, + "Payload": null } } ``` @@ -194,6 +207,24 @@ The `Job` object supports the following keys: * `Meta` - Annotates the job with opaque metadata. +* `ParameterizedJob` - Specifies the job as a paramterized job such that it can + be dispatched against. The `ParamaterizedJob` object supports the following + attributes: + + * `MetaOptional` - Specifies the set of metadata keys that may be provided + when dispatching against the job as a string array. + + * `MetaRequired` - Specifies the set of metadata keys that must be provided + when dispatching against the job as a string array. + + * `Payload` - Specifies the requirement of providing a payload when + dispatching against the parameterized job. The options for this field are + "optional", "required" and "forbidden" + +* `Payload` - The payload may not be set when submitting a job but may appear in + a dispatched job. The `Payload` will be a base64 encoded string containing the + payload that the job was dispatched with. + * `Priority` - Specifies the job priority which is used to prioritize scheduling and access to resources. Must be between 1 and 100 inclusively, and defaults to 50. @@ -295,6 +326,12 @@ The `Task` object supports the following keys: * `Constraints` - This is a list of `Constraint` objects. See the constraint reference for more details. +- `DispatchPayload` - Configures the task to have access to dispatch payloads. + The `DispatchPayload` object supports the following attributes: + + * `File` - Specifies the file name to write the content of dispatch payload + to. The file is written relative to the task's local directory. + * `Driver` - Specifies the task driver that should be used to run the task. See the [driver documentation](/docs/drivers/index.html) for what is available. Examples include `docker`, `qemu`, `java`, and `exec`. diff --git a/website/source/docs/job-specification/dispatch-payload.html.md b/website/source/docs/job-specification/dispatch-payload.html.md index 70fed94d6..357d0583b 100644 --- a/website/source/docs/job-specification/dispatch-payload.html.md +++ b/website/source/docs/job-specification/dispatch-payload.html.md @@ -39,7 +39,7 @@ job "docs" { ## `dispatch_payload` Parameters - `file` `(string: "")` - Specifies the file name to write the content of - dispatch payload to. The file is written to a [task's local + dispatch payload to. The file is written relative to the [task's local directory][localdir]. ## `dispatch_payload` Examples diff --git a/website/source/docs/job-specification/job.html.md b/website/source/docs/job-specification/job.html.md index 844dba133..9c95ad918 100644 --- a/website/source/docs/job-specification/job.html.md +++ b/website/source/docs/job-specification/job.html.md @@ -42,6 +42,10 @@ job "docs" { "my-key" = "my-value" } + parameterized { + # ... + } + periodic { # ... } @@ -74,13 +78,16 @@ job "docs" { - `datacenters` `(array: )` - A list of datacenters in the region which are eligible for task placement. This must be provided, and does not have a default. -- `group` ([Group][group]: ) - Specifies the start of a +- `group` ([Group][group]: \) - Specifies the start of a group of tasks. This can be provided multiple times to define additional groups. Group names must be unique within the job file. - `meta` ([Meta][]: nil) - Specifies a key-value map that annotates with user-defined metadata. +- `parameterized` ([Parameterized][parameterized]: nil) - Specifies + the job as a paramterized job such that it can be dispatched against. + - `periodic` ([Periodic][]: nil) - Allows the job to be scheduled at fixed times, dates or intervals. @@ -215,6 +222,7 @@ $ VAULT_TOKEN="..." nomad run example.nomad [constraint]: /docs/job-specification/constraint.html "Nomad constraint Job Specification" [group]: /docs/job-specification/group.html "Nomad group Job Specification" [meta]: /docs/job-specification/meta.html "Nomad meta Job Specification" +[parameterized]: /docs/job-specification/parameterized.html "Nomad parameterized Job Specification" [periodic]: /docs/job-specification/periodic.html "Nomad periodic Job Specification" [task]: /docs/job-specification/task.html "Nomad task Job Specification" [update]: /docs/job-specification/update.html "Nomad update Job Specification" diff --git a/website/source/docs/job-specification/parameterized.html.md b/website/source/docs/job-specification/parameterized.html.md index 59e0cfdb6..1dcb5cd6c 100644 --- a/website/source/docs/job-specification/parameterized.html.md +++ b/website/source/docs/job-specification/parameterized.html.md @@ -66,10 +66,10 @@ job "docs" { ## `parameterized` Parameters -- `meta_optional` `([]string: nil)` - Specifies the set of metadata keys that +- `meta_optional` `(array: nil)` - Specifies the set of metadata keys that may be provided when dispatching against the job. -- `meta_required` `([]string: nil)` - Specifies the set of metadata keys that +- `meta_required` `(array: nil)` - Specifies the set of metadata keys that must be provided when dispatching against the job. - `payload` `(string: "optional")` - Specifies the requirement of providing a diff --git a/website/source/docs/job-specification/task.html.md b/website/source/docs/job-specification/task.html.md index 38227ce5c..790c17be9 100644 --- a/website/source/docs/job-specification/task.html.md +++ b/website/source/docs/job-specification/task.html.md @@ -37,6 +37,9 @@ job "docs" { constraints on the task. This can be provided multiple times to define additional constraints. +- `dispatch_payload` ([DispatchPayload][]: nil) - Configures the + task to have access to dispatch payloads. + - `driver` - Specifies the task driver that should be used to run the task. See the [driver documentation](/docs/drivers/index.html) for what is available. Examples include `docker`, `qemu`, `java`, and `exec`. @@ -163,6 +166,7 @@ task "server" { [artifact]: /docs/job-specification/artifact.html "Nomad artifact Job Specification" [consul]: https://www.consul.io/ "Consul by HashiCorp" [constraint]: /docs/job-specification/constraint.html "Nomad constraint Job Specification" +[dispatchpayload]: /docs/job-specification/dispatch-payload.html "Nomad dispatch_payload Job Specification" [env]: /docs/job-specification/env.html "Nomad env Job Specification" [meta]: /docs/job-specification/meta.html "Nomad meta Job Specification" [resources]: /docs/job-specification/resources.html "Nomad resources Job Specification" From 7e6fcfc7928b9cfdfd36d93f20536e0f2066293e Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 25 Jan 2017 22:16:20 -0800 Subject: [PATCH 11/17] move file --- .../{dispatch-payload.html.md => dispatch_payload.html.md} | 0 website/source/docs/job-specification/parameterized.html.md | 2 +- website/source/docs/job-specification/task.html.md | 2 +- website/source/layouts/docs.erb | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename website/source/docs/job-specification/{dispatch-payload.html.md => dispatch_payload.html.md} (100%) diff --git a/website/source/docs/job-specification/dispatch-payload.html.md b/website/source/docs/job-specification/dispatch_payload.html.md similarity index 100% rename from website/source/docs/job-specification/dispatch-payload.html.md rename to website/source/docs/job-specification/dispatch_payload.html.md diff --git a/website/source/docs/job-specification/parameterized.html.md b/website/source/docs/job-specification/parameterized.html.md index 1dcb5cd6c..2aff51470 100644 --- a/website/source/docs/job-specification/parameterized.html.md +++ b/website/source/docs/job-specification/parameterized.html.md @@ -157,4 +157,4 @@ job "email-blast" { [dispatch command]: /docs/commands/job-dispatch.html "Nomad Job Dispatch Command" [resources]: /docs/job-specification/resources.html "Nomad resources Job Specification" [interpolation]: /docs/runtime/interpolation.html "Nomad Runtime Interpolation" -[dispatch_payload]: /docs/job-specification/dispatch-payload.html "Nomad dispatch_payload Job Specification" +[dispatch_payload]: /docs/job-specification/dispatch_payload.html "Nomad dispatch_payload Job Specification" diff --git a/website/source/docs/job-specification/task.html.md b/website/source/docs/job-specification/task.html.md index 790c17be9..0bc47c42c 100644 --- a/website/source/docs/job-specification/task.html.md +++ b/website/source/docs/job-specification/task.html.md @@ -166,7 +166,7 @@ task "server" { [artifact]: /docs/job-specification/artifact.html "Nomad artifact Job Specification" [consul]: https://www.consul.io/ "Consul by HashiCorp" [constraint]: /docs/job-specification/constraint.html "Nomad constraint Job Specification" -[dispatchpayload]: /docs/job-specification/dispatch-payload.html "Nomad dispatch_payload Job Specification" +[dispatchpayload]: /docs/job-specification/dispatch_payload.html "Nomad dispatch_payload Job Specification" [env]: /docs/job-specification/env.html "Nomad env Job Specification" [meta]: /docs/job-specification/meta.html "Nomad meta Job Specification" [resources]: /docs/job-specification/resources.html "Nomad resources Job Specification" diff --git a/website/source/layouts/docs.erb b/website/source/layouts/docs.erb index f3b00a5fa..48fb8ebdd 100644 --- a/website/source/layouts/docs.erb +++ b/website/source/layouts/docs.erb @@ -49,7 +49,7 @@ constraint > - dispatch_payload + dispatch_payload > env From aa51fe6f2e78809309c9e619475962879cf8980a Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 25 Jan 2017 22:21:51 -0800 Subject: [PATCH 12/17] Status examples --- .../source/docs/commands/status.html.md.erb | 115 +++++++++++++----- 1 file changed, 85 insertions(+), 30 deletions(-) diff --git a/website/source/docs/commands/status.html.md.erb b/website/source/docs/commands/status.html.md.erb index 0fb643ba0..5735b63ce 100644 --- a/website/source/docs/commands/status.html.md.erb +++ b/website/source/docs/commands/status.html.md.erb @@ -54,26 +54,28 @@ Short view of a specific job: ``` $ nomad status -short job1 -ID = job1 -Name = Test Job -Type = service -Priority = 3 -Datacenters = dc1,dc2,dc3 -Status = pending -Periodic = false +ID = job1 +Name = Test Job +Type = service +Priority = 3 +Datacenters = dc1,dc2,dc3 +Status = pending +Periodic = false +Parameterized = false ``` Full status information of a job: ``` $ nomad status example -ID = example -Name = example -Type = service -Priority = 50 -Datacenters = dc1 -Status = running -Periodic = false +ID = example +Name = example +Type = service +Priority = 50 +Datacenters = dc1 +Status = running +Periodic = false +Parameterized = false Summary Task Group Queued Starting Running Failed Complete Lost @@ -84,17 +86,71 @@ ID Eval ID Node ID Task Group Desired Status Created At 24cfd201 81efc2fa 8d0331e9 cache run running 08/08/16 21:03:19 CDT ``` +Full status information of a perodic job: + +``` +ID = example +Name = example +Type = batch +Priority = 50 +Datacenters = dc1 +Status = running +Periodic = true +Parameterized = false +Next Periodic Launch = 01/26/17 06:19:46 UTC (1s from now) + +Children Job Summary +Pending Running Dead +0 5 0 + +Previously Launched Jobs +ID Status +example/periodic-1485411581 running +example/periodic-1485411582 running +example/periodic-1485411583 running +example/periodic-1485411584 running +example/periodic-1485411585 running +``` + +Full status information of a parameterized job: + +``` +ID = example +Name = example +Type = batch +Priority = 50 +Datacenters = dc1 +Status = running +Periodic = false +Parameterized = true + +Parameterized Job +Payload = required +Required Metadata = foo +Optional Metadata = bar + +Parameterized Job Summary +Pending Running Dead +0 2 0 + +Dispatched Jobs +ID Status +example/dispatch-1485411496-58f24d2d running +example/dispatch-1485411499-fa2ee40e running +``` + Full status information of a job with placement failures: ``` $ nomad status example -ID = example -Name = example -Type = service -Priority = 50 -Datacenters = dc1 -Status = running -Periodic = false +ID = example +Name = example +Type = service +Priority = 50 +Datacenters = dc1 +Status = running +Periodic = false +Parameterized = false Summary Task Group Queued Starting Running Failed Complete Lost @@ -120,13 +176,14 @@ become availables so that it can place the remaining allocations. ``` $ nomad status -evals example -ID = example -Name = example -Type = service -Priority = 50 -Datacenters = dc1 -Status = running -Periodic = false +ID = example +Name = example +Type = service +Priority = 50 +Datacenters = dc1 +Status = running +Periodic = false +Parameterized = false Summary Task Group Queued Starting Running Failed Complete Lost @@ -152,5 +209,3 @@ b82f58b6 8bf94335 8d0331e9 cache run running 08/08/16 21:03:19 CD ed3665f5 8bf94335 8d0331e9 cache run running 08/08/16 21:03:19 CDT 24cfd201 8bf94335 8d0331e9 cache run running 08/08/16 21:03:19 CDT ``` - -TODO: CRON AND DISPATCH From 36dad988965169e3737db7d8c443e7c8e28a4496 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 25 Jan 2017 22:27:15 -0800 Subject: [PATCH 13/17] Children api --- CHANGELOG.md | 5 +++-- website/source/docs/http/job.html.md | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a16113f9..2ff654764 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,10 @@ ## 0.5.3 (Unreleased) IMPROVEMENTS: - * api: Added APIs for requesting GC of allocations [GH-2192] - * core: Introduce Parameterized Jobs and Dispatch command/API [GH-2128] + * core: Introduce parameterized jobs and dispatch command/API [GH-2128] * core: Cancel blocked evals upon successful one for job [GH-2155] + * api: Added APIs for requesting GC of allocations [GH-2192] + * api: Job summary endpoint includes summary status for child jobs [GH-2128] * api/client: Plain text log streaming suitable for viewing logs in a browser [GH-2235] * cli: Defaulting to showing allocations which belong to currently registered diff --git a/website/source/docs/http/job.html.md b/website/source/docs/http/job.html.md index ca240d964..ee48655ab 100644 --- a/website/source/docs/http/job.html.md +++ b/website/source/docs/http/job.html.md @@ -274,6 +274,11 @@ region is used; another region can be specified using the `?region=` query param ```javascript { "JobID": "example", + "Children": { + "Dead": 0, + "Running": 7, + "Pending": 2 + }, "Summary": { "cache": { "Queued": 0, From 7e549679e97a151d2be8790dc5ff260a77e8d7a0 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 26 Jan 2017 11:31:47 -0800 Subject: [PATCH 14/17] Respond to comments --- command/job_dispatch.go | 9 +++++---- .../source/docs/commands/job-dispatch.html.md.erb | 13 ++++++++----- website/source/docs/http/job.html.md | 2 +- website/source/docs/http/json-jobs.html.md | 5 +++-- .../docs/job-specification/parameterized.html.md | 3 ++- 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/command/job_dispatch.go b/command/job_dispatch.go index 763260ec8..c6a334e98 100644 --- a/command/job_dispatch.go +++ b/command/job_dispatch.go @@ -33,10 +33,11 @@ General Options: Dispatch Options: -meta = - Meta takes a key/value pair seperated by "=". The metadata key will be - injected into the job's metadata. The flag can be provided more than - once to inject multiple metadata key/value pairs. Arbitrary injection is not - allowed. The parameterized job must allow the key to be injected. + Meta takes a key/value pair seperated by "=". The metadata key will be + merged into the job's metadata. The job may define a default value for the + key which is overriden when dispatching. The flag can be provided more than + once to inject multiple metadata key/value pairs. Arbitrary keys are not + allowed. The parameterized job must allow the key to be merged. -detach Return immediately instead of entering monitor mode. After job dispatch, diff --git a/website/source/docs/commands/job-dispatch.html.md.erb b/website/source/docs/commands/job-dispatch.html.md.erb index cce195491..cbd17f640 100644 --- a/website/source/docs/commands/job-dispatch.html.md.erb +++ b/website/source/docs/commands/job-dispatch.html.md.erb @@ -15,8 +15,8 @@ page.](https://releases.hashicorp.com/nomad/0.5.3-rc1/) The `job dispatch` command is used to create new instances of a [parameterized job]. The parameterized job captures a job's configuration and runtime requirements in a generic way and `dispatch` is used to provide the input for -the job to run against. One can think of the parameterized job as a function -definition and dispatch is used to invoke the function. +the job to run against. A parameterized job is similar to a function definition, +and dispatch is used to invoke the function. Each time a job is dispatched, a unique job ID is generated. This allows a caller to track the status of the job, much like a future or promise in some @@ -33,6 +33,8 @@ dispatched instance can be provided via stdin by using "-" for the input source or by specifiying a path to a file. Metadata can be supplied by using the meta flag one or more times. +The payload has a **size limit of 16KiB**. + Upon successfully creation, the dispatched job ID will be printed and the triggered evaluation will be monitored. This can be disabled by supplying the detach flag. @@ -49,9 +51,10 @@ client connection issues or internal errors, are indicated by exit code 1. ## Run Options * `-meta`: Meta takes a key/value pair seperated by "=". The metadata key will - be injected into the job's metadata. The flag can be provided more than once - to inject multiple metadata key/value pairs. Arbitrary injection is not - allowed. The parameterized job must allow the key to be injected. + be merged into the job's metadata. The job may define a default value for the + key which is overriden when dispatching. The flag can be provided more than + once to inject multiple metadata key/value pairs. Arbitrary keys are not + allowed. The parameterized job must allow the key to be merged. * `-detach`: Return immediately instead of monitoring. A new evaluation ID will be output, which can be used to examine the evaluation using the diff --git a/website/source/docs/http/job.html.md b/website/source/docs/http/job.html.md index ee48655ab..dfdc01c65 100644 --- a/website/source/docs/http/job.html.md +++ b/website/source/docs/http/job.html.md @@ -355,7 +355,7 @@ region is used; another region can be specified using the `?region=` query param
  • Payload optional - A `[]byte` array encoded as a base64 string. + A `[]byte` array encoded as a base64 string with a maximum size of 16KiB.
  • Meta diff --git a/website/source/docs/http/json-jobs.html.md b/website/source/docs/http/json-jobs.html.md index ed279db5a..7d8cc3bbe 100644 --- a/website/source/docs/http/json-jobs.html.md +++ b/website/source/docs/http/json-jobs.html.md @@ -219,11 +219,12 @@ The `Job` object supports the following keys: * `Payload` - Specifies the requirement of providing a payload when dispatching against the parameterized job. The options for this field are - "optional", "required" and "forbidden" + "optional", "required" and "forbidden". The default value is "optional". * `Payload` - The payload may not be set when submitting a job but may appear in a dispatched job. The `Payload` will be a base64 encoded string containing the - payload that the job was dispatched with. + payload that the job was dispatched with. The `payload` has a **maximum size + of 16 KiB**. * `Priority` - Specifies the job priority which is used to prioritize scheduling and access to resources. Must be between 1 and 100 inclusively, diff --git a/website/source/docs/job-specification/parameterized.html.md b/website/source/docs/job-specification/parameterized.html.md index 2aff51470..6d0cb126c 100644 --- a/website/source/docs/job-specification/parameterized.html.md +++ b/website/source/docs/job-specification/parameterized.html.md @@ -73,7 +73,8 @@ job "docs" { must be provided when dispatching against the job. - `payload` `(string: "optional")` - Specifies the requirement of providing a - payload when dispatching against the parameterized job. The options for this + payload when dispatching against the parameterized job. The **maximum size of a + `payload` is 16 KiB**. The options for this field are: - `"optional"` - A payload is optional when dispatching against the job. From 87c9859db65e0eec22e0730e0c9eb0b91cc5afbb Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 26 Jan 2017 13:02:13 -0800 Subject: [PATCH 15/17] Fix flaky test --- nomad/vault_test.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/nomad/vault_test.go b/nomad/vault_test.go index e17c3bce0..56e609b99 100644 --- a/nomad/vault_test.go +++ b/nomad/vault_test.go @@ -591,9 +591,6 @@ func TestVaultClient_LookupToken_RateLimit(t *testing.T) { unblock := make(chan struct{}) for i := 0; i < numRequests; i++ { go func() { - // Ensure all the goroutines are made - time.Sleep(10 * time.Millisecond) - // Lookup ourselves _, err := client.LookupToken(ctx, v.Config.Token) if err != nil { @@ -607,7 +604,7 @@ func TestVaultClient_LookupToken_RateLimit(t *testing.T) { // Cancel the context cancel() - time.AfterFunc(1*time.Second, func() { close(unblock) }) + close(unblock) }() } @@ -618,9 +615,15 @@ func TestVaultClient_LookupToken_RateLimit(t *testing.T) { } desired := numRequests - 1 - if cancels != desired { - t.Fatalf("Incorrect number of cancels; got %d; want %d", cancels, desired) - } + testutil.WaitForResult(func() (bool, error) { + if cancels != desired { + return false, fmt.Errorf("Incorrect number of cancels; got %d; want %d", cancels, desired) + } + + return true, nil + }, func(err error) { + t.Fatalf("Connection not established") + }) } func TestVaultClient_CreateToken_Root(t *testing.T) { From 6c4a89989e23efe76b9fcc51ea1a65d18ee57983 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 26 Jan 2017 13:07:50 -0800 Subject: [PATCH 16/17] Fix typo --- command/job_dispatch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/job_dispatch.go b/command/job_dispatch.go index c6a334e98..702fa4a97 100644 --- a/command/job_dispatch.go +++ b/command/job_dispatch.go @@ -51,7 +51,7 @@ Dispatch Options: } func (c *JobDispatchCommand) Synopsis() string { - return "Dispatch an instance of a parametereized job" + return "Dispatch an instance of a parameterized job" } func (c *JobDispatchCommand) Run(args []string) int { From 3c7e018438414a55d085488f4e966f90e4d96634 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 26 Jan 2017 13:25:37 -0800 Subject: [PATCH 17/17] up timing --- client/task_runner_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/task_runner_test.go b/client/task_runner_test.go index e7e2b19db..e2cb99209 100644 --- a/client/task_runner_test.go +++ b/client/task_runner_test.go @@ -536,7 +536,7 @@ func TestTaskRunner_RestartTask(t *testing.T) { task.Driver = "mock_driver" task.Config = map[string]interface{}{ "exit_code": "0", - "run_for": "10s", + "run_for": "100s", } ctx := testTaskRunnerFromAlloc(t, true, alloc)