From b2b999249b8945a0564e0c04ef8552f9f4a250d1 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Wed, 11 Apr 2018 14:57:12 -0400 Subject: [PATCH 01/11] command/agent: add /v1/jobs/parse endpoint The parse endpoint accepts a hcl jobspec body within a json object and returns the parsed json object for the job. This allows users to register jobs with the nomad json api without specifically needing a nomad binary to parse their hcl encoded jobspec file. --- api/jobs.go | 13 +++++++ command/agent/http.go | 1 + command/agent/job_endpoint.go | 24 +++++++++++++ command/agent/job_endpoint_test.go | 54 ++++++++++++++++++++++++++++++ 4 files changed, 92 insertions(+) diff --git a/api/jobs.go b/api/jobs.go index 5fcecf403..9219cfc96 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -36,11 +36,24 @@ type Jobs struct { client *Client } +type JobsParseRequest struct { + JobHCL string +} + // Jobs returns a handle on the jobs endpoints. func (c *Client) Jobs() *Jobs { return &Jobs{client: c} } +// Parse is used to convert the HCL repesentation of a Job to JSON server side +// To parse the HCL client side see package github.com/hashicorp/nomad/jobspec +func (j *Jobs) Parse(jobHCL string) (*Job, error) { + var job *Job + req := &JobsParseRequest{JobHCL: jobHCL} + _, err := j.client.write("/v1/jobs/parse", req, job, nil) + return job, err +} + func (j *Jobs) Validate(job *Job, q *WriteOptions) (*JobValidateResponse, *WriteMeta, error) { var resp JobValidateResponse req := &JobValidateRequest{Job: job} diff --git a/command/agent/http.go b/command/agent/http.go index 0321a7b22..6e9b6a802 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -143,6 +143,7 @@ func (s *HTTPServer) Shutdown() { // registerHandlers is used to attach our handlers to the mux func (s *HTTPServer) registerHandlers(enableDebug bool) { s.mux.HandleFunc("/v1/jobs", s.wrap(s.JobsRequest)) + s.mux.HandleFunc("/v1/jobs/parse", s.wrap(s.JobsParseRequest)) s.mux.HandleFunc("/v1/job/", s.wrap(s.JobSpecificRequest)) s.mux.HandleFunc("/v1/nodes", s.wrap(s.NodesRequest)) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index ce1605728..3639cef51 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -8,6 +8,7 @@ import ( "github.com/golang/snappy" "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/jobspec" "github.com/hashicorp/nomad/nomad/structs" ) @@ -544,6 +545,29 @@ func (s *HTTPServer) jobDispatchRequest(resp http.ResponseWriter, req *http.Requ return out, nil } +// JobsParseRequest parses a hcl jobspec and returns a api.Job +func (s *HTTPServer) JobsParseRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + if req.Method != http.MethodPut && req.Method != http.MethodPost { + return nil, CodedError(405, ErrInvalidMethod) + } + + args := &api.JobsParseRequest{} + if err := decodeBody(req, &args); err != nil { + return nil, CodedError(400, err.Error()) + } + if args.JobHCL == "" { + return nil, CodedError(400, "Job spec is empty") + } + + jobfile := strings.NewReader(args.JobHCL) + jobStruct, err := jobspec.Parse(jobfile) + if err != nil { + return nil, CodedError(400, err.Error()) + } + + return jobStruct, nil +} + func ApiJobToStructJob(job *api.Job) *structs.Job { job.Canonicalize() diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 3a950f91c..64821f247 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -272,6 +272,60 @@ func TestHTTP_JobsRegister_Defaulting(t *testing.T) { }) } +func TestHTTP_JobsParse(t *testing.T) { + t.Parallel() + httpTest(t, nil, func(s *TestAgent) { + jobspec := ` +job "example" { + datacenters = ["dc1"] + type = "service" + group "cache" { + count = 1 + ephemeral_disk { + size = 300 + } + task "redis" { + driver = "docker" + config { + image = "redis:3.2" + port_map { + db = 6379 + } + } + resources { + cpu = 500 + memory = 256 + network { + mbits = 10 + port "db" {} + } + } + } + } +} +` + buf := encodeReq(api.JobsParseRequest{JobHCL: jobspec}) + req, err := http.NewRequest("POST", "/v1/jobs/render", buf) + if err != nil { + t.Fatalf("err: %v", err) + } + + respW := httptest.NewRecorder() + + obj, err := s.Server.JobsParseRequest(respW, req) + if err != nil { + t.Fatalf("err: %v", err) + } + if obj == nil { + t.Fatal("response should not be nil") + } + + job := obj.(*api.Job) + if job.Name == nil || *job.Name != "example" { + t.Fatalf("job name is '%s', expected 'example'", *job.Name) + } + }) +} func TestHTTP_JobQuery(t *testing.T) { t.Parallel() httpTest(t, nil, func(s *TestAgent) { From 03f5c2f6f9882728f799aad97eb35039c1b397a1 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 12 Apr 2018 14:45:38 -0400 Subject: [PATCH 02/11] command/agent: add HCL mock for parse endpoint --- command/agent/job_endpoint_test.go | 42 +++++++----------------------- nomad/mock/mock.go | 32 +++++++++++++++++++++++ 2 files changed, 42 insertions(+), 32 deletions(-) diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 64821f247..187615bcf 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -275,36 +275,7 @@ func TestHTTP_JobsRegister_Defaulting(t *testing.T) { func TestHTTP_JobsParse(t *testing.T) { t.Parallel() httpTest(t, nil, func(s *TestAgent) { - jobspec := ` -job "example" { - datacenters = ["dc1"] - type = "service" - group "cache" { - count = 1 - ephemeral_disk { - size = 300 - } - task "redis" { - driver = "docker" - config { - image = "redis:3.2" - port_map { - db = 6379 - } - } - resources { - cpu = 500 - memory = 256 - network { - mbits = 10 - port "db" {} - } - } - } - } -} -` - buf := encodeReq(api.JobsParseRequest{JobHCL: jobspec}) + buf := encodeReq(api.JobsParseRequest{JobHCL: mock.HCL()}) req, err := http.NewRequest("POST", "/v1/jobs/render", buf) if err != nil { t.Fatalf("err: %v", err) @@ -321,8 +292,15 @@ job "example" { } job := obj.(*api.Job) - if job.Name == nil || *job.Name != "example" { - t.Fatalf("job name is '%s', expected 'example'", *job.Name) + expected := mock.Job() + if job.Name == nil || *job.Name != expected.Name { + t.Fatalf("job name is '%s', expected '%s'", *job.Name, expected.Name) + } + + if job.Datacenters == nil || + *job.Datacenters[0] != expected.Datacenters[0] { + t.Fatalf("job datacenters is '%s', expected '%s'", + *job.Datacenters[0], expected.Datacenters[0]) } }) } diff --git a/nomad/mock/mock.go b/nomad/mock/mock.go index 1d39384a8..2d410d1a1 100644 --- a/nomad/mock/mock.go +++ b/nomad/mock/mock.go @@ -63,6 +63,38 @@ func Node() *structs.Node { return node } +func HCL() string { + return `job "my-job" { + datacenters = ["dc1"] + type = "service" + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "web" { + count = 10 + restart { + attempts = 3 + interval = "10m" + delay = "1m" + mode = "delay" + } + task "web" { + driver = "exec" + config { + command = "/bin/date" + } + resources { + cpu = 500 + memory = 256 + } + } + } +} +` +} + func Job() *structs.Job { job := &structs.Job{ Region: "global", From 14c44da757f4f87b76bf9a994d05ba7ff2c9db8f Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 12 Apr 2018 14:46:15 -0400 Subject: [PATCH 03/11] command/agent: add Canonicalize option to parse args --- api/jobs.go | 5 +++++ command/agent/job_endpoint.go | 3 +++ 2 files changed, 8 insertions(+) diff --git a/api/jobs.go b/api/jobs.go index 9219cfc96..5633ea6a2 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -36,8 +36,13 @@ type Jobs struct { client *Client } +// JobsParseRequest is used for arguments of the /vi/jobs/parse endpoint type JobsParseRequest struct { + //JobHCL is an hcl jobspec JobHCL string + //Canonicalize is a flag as to if the server should return default values + //for unset fields + Canonicalize bool } // Jobs returns a handle on the jobs endpoints. diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 3639cef51..2a0a92b17 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -565,6 +565,9 @@ func (s *HTTPServer) JobsParseRequest(resp http.ResponseWriter, req *http.Reques return nil, CodedError(400, err.Error()) } + if args.Canonicalize { + jobStruct.Canonicalize() + } return jobStruct, nil } From a5ffa2271adab5f428d9e233b92ee75b9898b0d2 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 12 Apr 2018 15:35:39 -0400 Subject: [PATCH 04/11] website: add api docs for v1/jobs/parse endpoint --- website/source/api/jobs.html.md | 69 +++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/website/source/api/jobs.html.md b/website/source/api/jobs.html.md index d3316df58..cd4191891 100644 --- a/website/source/api/jobs.html.md +++ b/website/source/api/jobs.html.md @@ -231,6 +231,75 @@ $ curl \ } ``` +## Parse Job + +This endpoint will parse an hcl jobspec and produce the equivalent json encoded +job. + +| Method | Path | Produces | +| ------ | ------------------------- | -------------------------- | +| `POST` | `/v1/jobs/parse` | `application/json` | + +### Parameters + +- `JobHCL` `(string: )` - Specifies the HCL definition of the job + encoded in a JSON string. +- `Canonicalize` `(bool: false)` - Flag to enable setting any unset fields to + their default values. + +## Sample Payload + +```json +{ + "JobHCL":"job \"example\" { type = \"service\" group \"cache\" {} }", + "Canonicalize": true +} +``` + +### Sample Request + +```text +$ curl \ + --request POST \ + --data '{"Canonicalize": true, "JobHCL": "job \"my-job\" {}"}' \ + https://localhost:4646/v1/jobs/parse +``` + +### Sample Response + +```json +{ + "AllAtOnce": false, + "Constraints": null, + "CreateIndex": 0, + "Datacenters": null, + "ID": "my-job", + "JobModifyIndex": 0, + "Meta": null, + "Migrate": null, + "ModifyIndex": 0, + "Name": "my-job", + "Namespace": "default", + "ParameterizedJob": null, + "ParentID": "", + "Payload": null, + "Periodic": null, + "Priority": 50, + "Region": "global", + "Reschedule": null, + "Stable": false, + "Status": "", + "StatusDescription": "", + "Stop": false, + "SubmitTime": null, + "TaskGroups": null, + "Type": "service", + "Update": null, + "VaultToken": "", + "Version": 0 +} +``` + ## Read Job This endpoint reads information about a single job for its specification and From c5945204a6f0c4f4bd6f7daecd50b880def39b3c Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Mon, 16 Apr 2018 13:54:01 -0400 Subject: [PATCH 05/11] command/agent: fix ptr ref in job endpoint test --- command/agent/job_endpoint_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 187615bcf..6323e371b 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -298,9 +298,9 @@ func TestHTTP_JobsParse(t *testing.T) { } if job.Datacenters == nil || - *job.Datacenters[0] != expected.Datacenters[0] { + job.Datacenters[0] != expected.Datacenters[0] { t.Fatalf("job datacenters is '%s', expected '%s'", - *job.Datacenters[0], expected.Datacenters[0]) + job.Datacenters[0], expected.Datacenters[0]) } }) } From 4952c52d4969835ae29b6ea1b4af40a46d75d5df Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Mon, 16 Apr 2018 19:17:58 -0400 Subject: [PATCH 06/11] api: add test for canonicalized jobs/parse --- api/jobs.go | 16 ++++++++++------ api/jobs_test.go | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/api/jobs.go b/api/jobs.go index 5633ea6a2..c710842f1 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -40,6 +40,7 @@ type Jobs struct { type JobsParseRequest struct { //JobHCL is an hcl jobspec JobHCL string + //Canonicalize is a flag as to if the server should return default values //for unset fields Canonicalize bool @@ -50,13 +51,16 @@ func (c *Client) Jobs() *Jobs { return &Jobs{client: c} } -// Parse is used to convert the HCL repesentation of a Job to JSON server side +// Parse is used to convert the HCL repesentation of a Job to JSON server side. // To parse the HCL client side see package github.com/hashicorp/nomad/jobspec -func (j *Jobs) Parse(jobHCL string) (*Job, error) { - var job *Job - req := &JobsParseRequest{JobHCL: jobHCL} - _, err := j.client.write("/v1/jobs/parse", req, job, nil) - return job, err +func (j *Jobs) Parse(jobHCL string, canonicalize bool) (*Job, error) { + var job Job + req := &JobsParseRequest{ + JobHCL: jobHCL, + Canonicalize: canonicalize, + } + _, err := j.client.write("/v1/jobs/parse", req, &job, nil) + return &job, err } func (j *Jobs) Validate(job *Job, q *WriteOptions) (*JobValidateResponse, *WriteMeta, error) { diff --git a/api/jobs_test.go b/api/jobs_test.go index 66f157c54..ceef8242d 100644 --- a/api/jobs_test.go +++ b/api/jobs_test.go @@ -47,6 +47,43 @@ func TestJobs_Register(t *testing.T) { } } +func TestJobs_Parse(t *testing.T) { + t.Parallel() + c, s := makeClient(t, nil, nil) + defer s.Stop() + + jobs := c.Jobs() + + checkJob := func(job *Job, expected string) { + if job == nil { + t.Fatal("job should not be nil") + } + + region := job.Region + + if region == nil { + if expected != "" { + t.Fatalf("expected job region to be '%s' but was unset", expected) + } + } else { + if expected != *region { + t.Fatalf("expected job region '%s', but got '%s'", expected, *region) + } + } + } + job, err := jobs.Parse(mock.HCL(), true) + if err != nil { + t.Fatalf("err: %s", err) + } + checkJob(job, "global") + + job, err = jobs.Parse(mock.HCL(), false) + if err != nil { + t.Fatalf("err: %s", err) + } + checkJob(job, "") +} + func TestJobs_Validate(t *testing.T) { t.Parallel() c, s := makeClient(t, nil, nil) From 4303941098d39597b9b92d8f6b0c93309ffd212f Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Mon, 16 Apr 2018 19:18:33 -0400 Subject: [PATCH 07/11] command/agent: fix url in jobs parse ep test --- command/agent/job_endpoint_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 6323e371b..1c80d6fac 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -276,7 +276,7 @@ func TestHTTP_JobsParse(t *testing.T) { t.Parallel() httpTest(t, nil, func(s *TestAgent) { buf := encodeReq(api.JobsParseRequest{JobHCL: mock.HCL()}) - req, err := http.NewRequest("POST", "/v1/jobs/render", buf) + req, err := http.NewRequest("POST", "/v1/jobs/parse", buf) if err != nil { t.Fatalf("err: %v", err) } From 2e31bbe333619db96d9fa3b29584efd61a58ae15 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Mon, 16 Apr 2018 19:19:02 -0400 Subject: [PATCH 08/11] website: add standard blocking query/ACL table to jobs/parse ep --- website/source/api/jobs.html.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/website/source/api/jobs.html.md b/website/source/api/jobs.html.md index cd4191891..64faa690f 100644 --- a/website/source/api/jobs.html.md +++ b/website/source/api/jobs.html.md @@ -240,6 +240,14 @@ job. | ------ | ------------------------- | -------------------------- | | `POST` | `/v1/jobs/parse` | `application/json` | +The table below shows this endpoint's support for +[blocking queries](/api/index.html#blocking-queries) and +[required ACLs](/api/index.html#acls). + +| Blocking Queries | ACL Required | +| ---------------- | ------------ | +| `NO` | `NONE` | + ### Parameters - `JobHCL` `(string: )` - Specifies the HCL definition of the job From 815c9fcce235cde15e71b9314f9977cc71bcb11b Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Mon, 16 Apr 2018 19:24:05 -0400 Subject: [PATCH 09/11] add changelog entry for new jobs/parse endpoint --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfd695588..7bf620f1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## 0.8.1 (Unreleased) IMPROVEMENTS: + * api: Add /v1/jobs/parse api endpoint for rendering HCL jobs files as JSON [[GH-2782](https://github.com/hashicorp/nomad/issues/2782)] * client: Create new process group on process startup. [[GH-3572](https://github.com/hashicorp/nomad/issues/3572)] ## 0.8.0 (April 12, 2018) From 8ef7d208c906f5713ba476fa06658d85c4cb226d Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Mon, 16 Apr 2018 19:32:37 -0400 Subject: [PATCH 10/11] website: fix type in api/jobs/parse --- website/source/api/jobs.html.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/source/api/jobs.html.md b/website/source/api/jobs.html.md index 64faa690f..6ae2df1e4 100644 --- a/website/source/api/jobs.html.md +++ b/website/source/api/jobs.html.md @@ -246,7 +246,7 @@ The table below shows this endpoint's support for | Blocking Queries | ACL Required | | ---------------- | ------------ | -| `NO` | `NONE` | +| `NO` | `none` | ### Parameters From 059ea949bef8844022ec040472b9e15087071d6c Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Tue, 17 Apr 2018 10:18:36 -0400 Subject: [PATCH 11/11] minor code review fixes to api/jobs --- api/jobs.go | 8 ++++---- api/jobs_test.go | 14 +++++++------- website/source/api/jobs.html.md | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/api/jobs.go b/api/jobs.go index c710842f1..7c2b7c723 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -38,11 +38,11 @@ type Jobs struct { // JobsParseRequest is used for arguments of the /vi/jobs/parse endpoint type JobsParseRequest struct { - //JobHCL is an hcl jobspec + // JobHCL is an hcl jobspec JobHCL string - //Canonicalize is a flag as to if the server should return default values - //for unset fields + // Canonicalize is a flag as to if the server should return default values + // for unset fields Canonicalize bool } @@ -53,7 +53,7 @@ func (c *Client) Jobs() *Jobs { // Parse is used to convert the HCL repesentation of a Job to JSON server side. // To parse the HCL client side see package github.com/hashicorp/nomad/jobspec -func (j *Jobs) Parse(jobHCL string, canonicalize bool) (*Job, error) { +func (j *Jobs) ParseHCL(jobHCL string, canonicalize bool) (*Job, error) { var job Job req := &JobsParseRequest{ JobHCL: jobHCL, diff --git a/api/jobs_test.go b/api/jobs_test.go index ceef8242d..9119f5118 100644 --- a/api/jobs_test.go +++ b/api/jobs_test.go @@ -54,7 +54,7 @@ func TestJobs_Parse(t *testing.T) { jobs := c.Jobs() - checkJob := func(job *Job, expected string) { + checkJob := func(job *Job, expectedRegion string) { if job == nil { t.Fatal("job should not be nil") } @@ -62,22 +62,22 @@ func TestJobs_Parse(t *testing.T) { region := job.Region if region == nil { - if expected != "" { - t.Fatalf("expected job region to be '%s' but was unset", expected) + if expectedRegion != "" { + t.Fatalf("expected job region to be '%s' but was unset", expectedRegion) } } else { - if expected != *region { - t.Fatalf("expected job region '%s', but got '%s'", expected, *region) + if expectedRegion != *region { + t.Fatalf("expected job region '%s', but got '%s'", expectedRegion, *region) } } } - job, err := jobs.Parse(mock.HCL(), true) + job, err := jobs.ParseHCL(mock.HCL(), true) if err != nil { t.Fatalf("err: %s", err) } checkJob(job, "global") - job, err = jobs.Parse(mock.HCL(), false) + job, err = jobs.ParseHCL(mock.HCL(), false) if err != nil { t.Fatalf("err: %s", err) } diff --git a/website/source/api/jobs.html.md b/website/source/api/jobs.html.md index 6ae2df1e4..284830937 100644 --- a/website/source/api/jobs.html.md +++ b/website/source/api/jobs.html.md @@ -233,7 +233,7 @@ $ curl \ ## Parse Job -This endpoint will parse an hcl jobspec and produce the equivalent json encoded +This endpoint will parse a HCL jobspec and produce the equivalent JSON encoded job. | Method | Path | Produces |