From 9220836cb4e58f2da385817454e6ce980838e349 Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Thu, 29 Jun 2017 17:16:20 -0700 Subject: [PATCH] JobVersions returns struct with optional diff --- api/jobs.go | 21 ++-- command/agent/job_endpoint.go | 16 +++- command/agent/job_endpoint_test.go | 9 +- command/job_history.go | 81 ++++++++++++++++ commands.go | 5 + main.go | 2 +- nomad/deployment_watcher_shims.go | 4 +- nomad/deploymentwatcher/deployment_watcher.go | 2 +- .../deploymentwatcher/deployments_watcher.go | 2 +- nomad/job_endpoint.go | 13 ++- nomad/job_endpoint_test.go | 95 ++++++++++++++++++- nomad/structs/structs.go | 8 ++ 12 files changed, 238 insertions(+), 20 deletions(-) create mode 100644 command/job_history.go diff --git a/api/jobs.go b/api/jobs.go index 9981577d4..3daa438f8 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -107,15 +107,15 @@ func (j *Jobs) Info(jobID string, q *QueryOptions) (*Job, *QueryMeta, error) { return &resp, qm, nil } -// Versions is used to retrieve all versions of a particular -// job given its unique ID. -func (j *Jobs) Versions(jobID string, q *QueryOptions) ([]*Job, *QueryMeta, error) { - var resp []*Job - qm, err := j.client.query("/v1/job/"+jobID+"/versions", &resp, q) +// Versions is used to retrieve all versions of a particular job given its +// unique ID. +func (j *Jobs) Versions(jobID string, diffs bool, q *QueryOptions) ([]*Job, []*JobDiff, *QueryMeta, error) { + var resp JobVersionsResponse + qm, err := j.client.query(fmt.Sprintf("/v1/job/%s/versions?diffs=%v", jobID, diffs), &resp, q) if err != nil { - return nil, nil, err + return nil, nil, nil, err } - return resp, qm, nil + return resp.Versions, resp.Diffs, qm, nil } // Allocations is used to return the allocs for a given job ID. @@ -831,3 +831,10 @@ type JobDispatchResponse struct { JobCreateIndex uint64 WriteMeta } + +// JobVersionsResponse is used for a job get versions request +type JobVersionsResponse struct { + Versions []*Job + Diffs []*JobDiff + QueryMeta +} diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 4583070b5..df2c630e1 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -394,8 +394,20 @@ func (s *HTTPServer) jobDelete(resp http.ResponseWriter, req *http.Request, func (s *HTTPServer) jobVersions(resp http.ResponseWriter, req *http.Request, jobName string) (interface{}, error) { - args := structs.JobSpecificRequest{ + + diffsStr := req.URL.Query().Get("diffs") + var diffsBool bool + if diffsStr != "" { + var err error + diffsBool, err = strconv.ParseBool(diffsStr) + if err != nil { + return nil, fmt.Errorf("Failed to parse value of %q (%v) as a bool: %v", "diffs", diffsStr, err) + } + } + + args := structs.JobVersionsRequest{ JobID: jobName, + Diffs: diffsBool, } if s.parse(resp, req, &args.Region, &args.QueryOptions) { return nil, nil @@ -411,7 +423,7 @@ func (s *HTTPServer) jobVersions(resp http.ResponseWriter, req *http.Request, return nil, CodedError(404, "job versions not found") } - return out.Versions, nil + return out, nil } func (s *HTTPServer) jobRevert(resp http.ResponseWriter, req *http.Request, diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index fccc33a2e..95915c06a 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -706,7 +706,7 @@ func TestHTTP_JobVersions(t *testing.T) { } // Make the HTTP request - req, err := http.NewRequest("GET", "/v1/job/"+job.ID+"/versions", nil) + req, err := http.NewRequest("GET", "/v1/job/"+job.ID+"/versions?diffs=true", nil) if err != nil { t.Fatalf("err: %v", err) } @@ -719,7 +719,8 @@ func TestHTTP_JobVersions(t *testing.T) { } // Check the response - versions := obj.([]*structs.Job) + vResp := obj.(structs.JobVersionsResponse) + versions := vResp.Versions if len(versions) != 2 { t.Fatalf("got %d versions; want 2", len(versions)) } @@ -732,6 +733,10 @@ func TestHTTP_JobVersions(t *testing.T) { t.Fatalf("bad %v", v) } + if len(vResp.Diffs) != 1 { + t.Fatalf("bad %v", vResp) + } + // Check for the index if respW.HeaderMap.Get("X-Nomad-Index") == "" { t.Fatalf("missing index") diff --git a/command/job_history.go b/command/job_history.go new file mode 100644 index 000000000..fbaf2e51e --- /dev/null +++ b/command/job_history.go @@ -0,0 +1,81 @@ +package command + +import ( + "fmt" + "strings" +) + +type JobHistoryCommand struct { + Meta +} + +func (c *JobHistoryCommand) Help() string { + helpText := ` +Usage: nomad job history [options] + +History is used to display the known versions of a particular job. The command +can display the diff between job versions and can be useful for understanding +the changes that occured to the job as well as deciding job versions to revert +to. + +General Options: + + ` + generalOptionsUsage() + ` + +History Options: + + -p + Display the difference between each job and its predecessor. + + -full + Display the full job definition for each version. + + -version + Display only the history for the given job version. +` + return strings.TrimSpace(helpText) +} + +func (c *JobHistoryCommand) Synopsis() string { + return "Display all tracked versions of a job" +} + +func (c *JobHistoryCommand) Run(args []string) int { + var diff, full bool + var version uint64 + + flags := c.Meta.FlagSet("job history", FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + flags.BoolVar(&diff, "p", false, "") + flags.BoolVar(&full, "full", false, "") + flags.Uint64Var(&version, "version", 0, "") + + if err := flags.Parse(args); err != nil { + return 1 + } + + // Check that we got exactly one node + args = flags.Args() + if l := len(args); l < 1 || l > 2 { + c.Ui.Error(c.Help()) + return 1 + } + + // Get the HTTP client + client, err := c.Meta.Client() + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + jobID := args[0] + versions, _, err := client.Jobs().Versions(jobID, nil) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error retrieving job versions: %s", err)) + return 1 + } + + c.Ui.Output(jobID) + c.Ui.Output(fmt.Sprintf("%d", len(versions))) + return 0 +} diff --git a/commands.go b/commands.go index 9bc568dea..aabdc90c4 100644 --- a/commands.go +++ b/commands.go @@ -99,6 +99,11 @@ func Commands(metaPtr *command.Meta) map[string]cli.CommandFactory { Meta: meta, }, nil }, + "job history": func() (cli.Command, error) { + return &command.JobHistoryCommand{ + Meta: meta, + }, nil + }, "logs": func() (cli.Command, error) { return &command.LogsCommand{ Meta: meta, diff --git a/main.go b/main.go index cc42fd279..7bb9afe47 100644 --- a/main.go +++ b/main.go @@ -40,7 +40,7 @@ func RunCustom(args []string, commands map[string]cli.CommandFactory) int { case "executor": case "syslog": case "operator raft", "operator raft list-peers", "operator raft remove-peer": - case "job dispatch": + case "job dispatch", "job history": case "fs ls", "fs cat", "fs stat": case "check": default: diff --git a/nomad/deployment_watcher_shims.go b/nomad/deployment_watcher_shims.go index 25de8fb2a..f21ed6af0 100644 --- a/nomad/deployment_watcher_shims.go +++ b/nomad/deployment_watcher_shims.go @@ -27,7 +27,7 @@ type deploymentWatcherStateShim struct { // getJobVersions is used to lookup the versions of a job. This is used when // rolling back to find the latest stable job - getJobVersions func(args *structs.JobSpecificRequest, reply *structs.JobVersionsResponse) error + getJobVersions func(args *structs.JobVersionsRequest, reply *structs.JobVersionsResponse) error // getJob is used to lookup a particular job. getJob func(args *structs.JobSpecificRequest, reply *structs.SingleJobResponse) error @@ -65,7 +65,7 @@ func (d *deploymentWatcherStateShim) GetDeployment(args *structs.DeploymentSpeci return d.getDeployment(args, reply) } -func (d *deploymentWatcherStateShim) GetJobVersions(args *structs.JobSpecificRequest, reply *structs.JobVersionsResponse) error { +func (d *deploymentWatcherStateShim) GetJobVersions(args *structs.JobVersionsRequest, reply *structs.JobVersionsResponse) error { if args.Region == "" { args.Region = d.region } diff --git a/nomad/deploymentwatcher/deployment_watcher.go b/nomad/deploymentwatcher/deployment_watcher.go index 295fb29b4..3151c910b 100644 --- a/nomad/deploymentwatcher/deployment_watcher.go +++ b/nomad/deploymentwatcher/deployment_watcher.go @@ -366,7 +366,7 @@ func (w *deploymentWatcher) watch() { // latestStableJob returns the latest stable job. It may be nil if none exist func (w *deploymentWatcher) latestStableJob() (*structs.Job, error) { - args := &structs.JobSpecificRequest{JobID: w.d.JobID} + args := &structs.JobVersionsRequest{JobID: w.d.JobID} var resp structs.JobVersionsResponse if err := w.watchers.GetJobVersions(args, &resp); err != nil { return nil, err diff --git a/nomad/deploymentwatcher/deployments_watcher.go b/nomad/deploymentwatcher/deployments_watcher.go index 8cd5fa428..3eb6a281d 100644 --- a/nomad/deploymentwatcher/deployments_watcher.go +++ b/nomad/deploymentwatcher/deployments_watcher.go @@ -67,7 +67,7 @@ type DeploymentStateWatchers interface { // GetJobVersions is used to lookup the versions of a job. This is used when // rolling back to find the latest stable job - GetJobVersions(args *structs.JobSpecificRequest, reply *structs.JobVersionsResponse) error + GetJobVersions(args *structs.JobVersionsRequest, reply *structs.JobVersionsResponse) error // GetJob is used to lookup a particular job. GetJob(args *structs.JobSpecificRequest, reply *structs.SingleJobResponse) error diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 9431b058d..5ae2cd6ef 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -558,7 +558,7 @@ func (j *Job) GetJob(args *structs.JobSpecificRequest, } // GetJobVersions is used to retrieve all tracked versions of a job. -func (j *Job) GetJobVersions(args *structs.JobSpecificRequest, +func (j *Job) GetJobVersions(args *structs.JobVersionsRequest, reply *structs.JobVersionsResponse) error { if done, err := j.srv.forward("Job.GetJobVersions", args, args, reply); done { return err @@ -580,6 +580,17 @@ func (j *Job) GetJobVersions(args *structs.JobSpecificRequest, reply.Versions = out if len(out) != 0 { reply.Index = out[0].ModifyIndex + + // Compute the diffs + for i := 0; i < len(out)-1; i++ { + old, new := out[i+1], out[i] + d, err := old.Diff(new, true) + if err != nil { + return fmt.Errorf("failed to create job diff: %v", err) + } + reply.Diffs = append(reply.Diffs, d) + } + } else { // Use the last index that affected the nodes table index, err := state.Index("job_version") diff --git a/nomad/job_endpoint_test.go b/nomad/job_endpoint_test.go index 03edb0f9d..b6a31f5ca 100644 --- a/nomad/job_endpoint_test.go +++ b/nomad/job_endpoint_test.go @@ -1503,7 +1503,7 @@ func TestJobEndpoint_GetJobVersions(t *testing.T) { } // Lookup the job - get := &structs.JobSpecificRequest{ + get := &structs.JobVersionsRequest{ JobID: job.ID, QueryOptions: structs.QueryOptions{Region: "global"}, } @@ -1541,6 +1541,95 @@ func TestJobEndpoint_GetJobVersions(t *testing.T) { } } +func TestJobEndpoint_GetJobVersions_Diff(t *testing.T) { + s1 := testServer(t, nil) + defer s1.Shutdown() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + + // Create the register request + job := mock.Job() + job.Priority = 88 + reg := &structs.JobRegisterRequest{ + Job: job, + WriteRequest: structs.WriteRequest{Region: "global"}, + } + + // Fetch the response + var resp structs.JobRegisterResponse + if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { + t.Fatalf("err: %v", err) + } + + // Register the job again to create another version + job.Priority = 90 + if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { + t.Fatalf("err: %v", err) + } + + // Register the job again to create another version + job.Priority = 100 + if err := msgpackrpc.CallWithCodec(codec, "Job.Register", reg, &resp); err != nil { + t.Fatalf("err: %v", err) + } + + // Lookup the job + get := &structs.JobVersionsRequest{ + JobID: job.ID, + Diffs: true, + QueryOptions: structs.QueryOptions{Region: "global"}, + } + var versionsResp structs.JobVersionsResponse + if err := msgpackrpc.CallWithCodec(codec, "Job.GetJobVersions", get, &versionsResp); err != nil { + t.Fatalf("err: %v", err) + } + if versionsResp.Index != resp.JobModifyIndex { + t.Fatalf("Bad index: %d %d", versionsResp.Index, resp.Index) + } + + // Make sure there are two job versions + versions := versionsResp.Versions + if l := len(versions); l != 3 { + t.Fatalf("Got %d versions; want 3", l) + } + + if v := versions[0]; v.Priority != 100 || v.ID != job.ID || v.Version != 2 { + t.Fatalf("bad: %+v", v) + } + if v := versions[1]; v.Priority != 90 || v.ID != job.ID || v.Version != 1 { + t.Fatalf("bad: %+v", v) + } + if v := versions[2]; v.Priority != 88 || v.ID != job.ID || v.Version != 0 { + t.Fatalf("bad: %+v", v) + } + + // Ensure we got diffs + diffs := versionsResp.Diffs + if l := len(diffs); l != 2 { + t.Fatalf("Got %d diffs; want 2", l) + } + d1 := diffs[0] + if len(d1.Fields) != 1 { + t.Fatalf("Got too many diffs: %#v", d1) + } + if d1.Fields[0].Name != "Priority" { + t.Fatalf("Got wrong field: %#v", d1) + } + if d1.Fields[0].Old != "90" && d1.Fields[0].New != "100" { + t.Fatalf("Got wrong field values: %#v", d1) + } + d2 := diffs[1] + if len(d2.Fields) != 1 { + t.Fatalf("Got too many diffs: %#v", d2) + } + if d2.Fields[0].Name != "Priority" { + t.Fatalf("Got wrong field: %#v", d2) + } + if d2.Fields[0].Old != "88" && d1.Fields[0].New != "90" { + t.Fatalf("Got wrong field values: %#v", d2) + } +} + func TestJobEndpoint_GetJobVersions_Blocking(t *testing.T) { s1 := testServer(t, nil) defer s1.Shutdown() @@ -1569,7 +1658,7 @@ func TestJobEndpoint_GetJobVersions_Blocking(t *testing.T) { } }) - req := &structs.JobSpecificRequest{ + req := &structs.JobVersionsRequest{ JobID: job2.ID, QueryOptions: structs.QueryOptions{ Region: "global", @@ -1599,7 +1688,7 @@ func TestJobEndpoint_GetJobVersions_Blocking(t *testing.T) { } }) - req2 := &structs.JobSpecificRequest{ + req2 := &structs.JobVersionsRequest{ JobID: job3.ID, QueryOptions: structs.QueryOptions{ Region: "global", diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index caae8f782..6c7f80c65 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -734,9 +734,17 @@ type JobListResponse struct { QueryMeta } +// JobVersionsRequest is used to get a jobs versions +type JobVersionsRequest struct { + JobID string + Diffs bool + QueryOptions +} + // JobVersionsResponse is used for a job get versions request type JobVersionsResponse struct { Versions []*Job + Diffs []*JobDiff QueryMeta }