From ce8f230cabcf539c9ebf22a02a6f4a81b42b0d65 Mon Sep 17 00:00:00 2001 From: Drew Bailey <2614075+drewbailey@users.noreply.github.com> Date: Mon, 15 Jun 2020 10:05:31 -0400 Subject: [PATCH] Multiregion deploy status and job status CLI --- api/deployments.go | 3 + command/agent/job_endpoint.go | 2 +- command/agent/job_endpoint_test.go | 2 +- command/deployment_status.go | 84 +++++++++++++- command/deployment_status_test.go | 104 ++++++++++++++++++ command/job_deployments.go | 2 +- command/job_status.go | 15 ++- command/job_status_test.go | 101 +++++++++++++++++ command/util_test.go | 37 +++++++ jobspec/parse_multiregion.go | 2 +- jobspec/parse_test.go | 4 +- nomad/structs/structs.go | 4 + .../hashicorp/nomad/api/deployments.go | 3 + vendor/github.com/hashicorp/nomad/api/jobs.go | 7 +- 14 files changed, 355 insertions(+), 15 deletions(-) diff --git a/api/deployments.go b/api/deployments.go index 8a241243c..af8487782 100644 --- a/api/deployments.go +++ b/api/deployments.go @@ -150,6 +150,9 @@ type Deployment struct { // present the correct list of deployments for the job and not old ones. JobCreateIndex uint64 + // IsMultiregion specifies if this deployment is part of a multi-region deployment + IsMultiregion bool + // TaskGroups is the set of task groups effected by the deployment and their // current deployment status. TaskGroups map[string]*DeploymentState diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index d8f3e470f..fb9573905 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -772,7 +772,7 @@ func ApiJobToStructJob(job *api.Job) *structs.Job { j.Multiregion.Regions = []*structs.MultiregionRegion{} for _, region := range job.Multiregion.Regions { r := &structs.MultiregionRegion{} - r.Name = *region.Name + r.Name = region.Name r.Count = *region.Count r.Datacenters = region.Datacenters r.Meta = region.Meta diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 7856330d4..0e6948860 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -1582,7 +1582,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { }, Regions: []*api.MultiregionRegion{ { - Name: helper.StringToPtr("west"), + Name: "west", Count: helper.IntToPtr(1), Datacenters: []string{"dc1", "dc2"}, Meta: map[string]string{"region_code": "W"}, diff --git a/command/deployment_status.go b/command/deployment_status.go index 61b0b5130..d1a4adfb1 100644 --- a/command/deployment_status.go +++ b/command/deployment_status.go @@ -1,6 +1,7 @@ package command import ( + "errors" "fmt" "sort" "strings" @@ -140,7 +141,7 @@ func (c *DeploymentStatusCommand) Run(args []string) int { return 0 } - c.Ui.Output(c.Colorize().Color(formatDeployment(deploy, length))) + c.Ui.Output(c.Colorize().Color(formatDeployment(client, deploy, length))) return 0 } @@ -182,7 +183,7 @@ func getDeployment(client *api.Deployments, dID string) (match *api.Deployment, } } -func formatDeployment(d *api.Deployment, uuidLength int) string { +func formatDeployment(c *api.Client, d *api.Deployment, uuidLength int) string { if d == nil { return "No deployment found" } @@ -196,6 +197,18 @@ func formatDeployment(d *api.Deployment, uuidLength int) string { } base := formatKV(high) + + // Fetch and Format Multi-region info + if d.IsMultiregion { + regions, err := fetchMultiRegionDeployments(c, d) + if err != nil { + base += "\n\nError fetching Multiregion deployments\n\n" + } else if len(regions) > 0 { + base += "\n\n[bold]Multiregion Deployment[reset]\n" + base += formatMultiregionDeployment(regions, uuidLength) + } + } + if len(d.TaskGroups) == 0 { return base } @@ -204,6 +217,73 @@ func formatDeployment(d *api.Deployment, uuidLength int) string { return base } +type regionResult struct { + region string + d *api.Deployment + err error +} + +func fetchMultiRegionDeployments(c *api.Client, d *api.Deployment) (map[string]*api.Deployment, error) { + results := make(map[string]*api.Deployment) + + job, _, err := c.Jobs().Info(d.JobID, &api.QueryOptions{}) + if err != nil { + return nil, err + } + + requests := make(chan regionResult, len(job.Multiregion.Regions)) + for i := 0; i < cap(requests); i++ { + go func(itr int) { + region := job.Multiregion.Regions[itr] + d, err := fetchRegionDeployment(c, d, region) + requests <- regionResult{d: d, err: err, region: region.Name} + }(i) + } + for i := 0; i < cap(requests); i++ { + res := <-requests + if res.err != nil { + key := fmt.Sprintf("%s (error)", res.region) + results[key] = &api.Deployment{} + continue + } + results[res.region] = res.d + + } + return results, nil +} + +func fetchRegionDeployment(c *api.Client, d *api.Deployment, region *api.MultiregionRegion) (*api.Deployment, error) { + if region == nil { + return nil, errors.New("Region not found") + } + + opts := &api.QueryOptions{Region: region.Name} + deploys, _, err := c.Jobs().Deployments(d.JobID, false, opts) + if err != nil { + return nil, err + } + for _, dep := range deploys { + if dep.JobVersion == d.JobVersion { + return dep, nil + } + } + return nil, fmt.Errorf("Could not find job version %d for region", d.JobVersion) +} + +func formatMultiregionDeployment(regions map[string]*api.Deployment, uuidLength int) string { + rowString := "Region|ID|Status" + rows := make([]string, len(regions)+1) + rows[0] = rowString + i := 1 + for k, v := range regions { + row := fmt.Sprintf("%s|%s|%s", k, limit(v.ID, uuidLength), v.Status) + rows[i] = row + i++ + } + sort.Strings(rows) + return formatList(rows) +} + func formatDeploymentGroups(d *api.Deployment, uuidLength int) string { // Detect if we need to add these columns var canaries, autorevert, progressDeadline bool diff --git a/command/deployment_status_test.go b/command/deployment_status_test.go index 89ad05f4a..59ea74c2c 100644 --- a/command/deployment_status_test.go +++ b/command/deployment_status_test.go @@ -1,9 +1,13 @@ package command import ( + "fmt" "testing" + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/command/agent" "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/testutil" "github.com/mitchellh/cli" "github.com/posener/complete" "github.com/stretchr/testify/assert" @@ -64,3 +68,103 @@ func TestDeploymentStatusCommand_AutocompleteArgs(t *testing.T) { assert.Equal(1, len(res)) assert.Equal(d.ID, res[0]) } + +func TestDeploymentStatusCommand_Multiregion(t *testing.T) { + t.Parallel() + + cbe := func(config *agent.Config) { + config.Region = "east" + config.Datacenter = "east-1" + } + cbw := func(config *agent.Config) { + config.Region = "west" + config.Datacenter = "west-1" + } + + srv, clientEast, url := testServer(t, true, cbe) + defer srv.Shutdown() + + srv2, clientWest, _ := testServer(t, true, cbw) + defer srv2.Shutdown() + + // Join with srv1 + addr1 := fmt.Sprintf("127.0.0.1:%d", + srv.Agent.Server().GetConfig().SerfConfig.MemberlistConfig.BindPort) + + if _, err := srv2.Agent.Server().Join([]string{addr1}); err != nil { + t.Fatalf("Join err: %v", err) + } + + // wait for client node + testutil.WaitForResult(func() (bool, error) { + nodes, _, err := clientEast.Nodes().List(nil) + if err != nil { + return false, err + } + if len(nodes) == 0 { + return false, fmt.Errorf("missing node") + } + if _, ok := nodes[0].Drivers["mock_driver"]; !ok { + return false, fmt.Errorf("mock_driver not ready") + } + return true, nil + }, func(err error) { + t.Fatalf("err: %s", err) + }) + + ui := new(cli.MockUi) + cmd := &DeploymentStatusCommand{Meta: Meta{Ui: ui, flagAddress: url}} + + // Register multiregion job in east + jobEast := testMultiRegionJob("job1_sfxx", "east", "east-1") + resp, _, err := clientEast.Jobs().Register(jobEast, nil) + require.NoError(t, err) + if code := waitForSuccess(ui, clientEast, fullId, t, resp.EvalID); code != 0 { + t.Fatalf("status code non zero saw %d", code) + } + + // Register multiregion job in west + jobWest := testMultiRegionJob("job1_sfxx", "west", "west-1") + resp2, _, err := clientWest.Jobs().Register(jobWest, &api.WriteOptions{Region: "west"}) + require.NoError(t, err) + if code := waitForSuccess(ui, clientWest, fullId, t, resp2.EvalID); code != 0 { + t.Fatalf("status code non zero saw %d", code) + } + + jobs, _, err := clientEast.Jobs().List(&api.QueryOptions{}) + require.NoError(t, err) + require.Len(t, jobs, 1) + + deploys, _, err := clientEast.Jobs().Deployments(jobs[0].ID, true, &api.QueryOptions{}) + require.NoError(t, err) + require.Len(t, deploys, 1) + + // Grab both deployments to verify output + eastDeploys, _, err := clientEast.Jobs().Deployments(jobs[0].ID, true, &api.QueryOptions{Region: "east"}) + require.NoError(t, err) + require.Len(t, eastDeploys, 1) + + westDeploys, _, err := clientWest.Jobs().Deployments(jobs[0].ID, true, &api.QueryOptions{Region: "west"}) + require.NoError(t, err) + require.Len(t, westDeploys, 1) + + // Run command for specific deploy + if code := cmd.Run([]string{"-region=east", "-address=" + url, deploys[0].ID}); code != 0 { + t.Fatalf("expected exit 0, got: %d", code) + } + + // Verify Multi-region Deployment info populated + out := ui.OutputWriter.String() + require.Contains(t, out, "Multiregion Deployment") + require.Contains(t, out, "Region") + require.Contains(t, out, "ID") + require.Contains(t, out, "Status") + require.Contains(t, out, "east") + require.Contains(t, out, eastDeploys[0].ID[0:7]) + require.Contains(t, out, "west") + require.Contains(t, out, westDeploys[0].ID[0:7]) + require.Contains(t, out, "running") + + require.NotContains(t, out, "") + +} diff --git a/command/job_deployments.go b/command/job_deployments.go index 66ca76ca8..39de2f94d 100644 --- a/command/job_deployments.go +++ b/command/job_deployments.go @@ -148,7 +148,7 @@ func (c *JobDeploymentsCommand) Run(args []string) int { return 0 } - c.Ui.Output(c.Colorize().Color(formatDeployment(deploy, length))) + c.Ui.Output(c.Colorize().Color(formatDeployment(client, deploy, length))) return 0 } diff --git a/command/job_status.go b/command/job_status.go index 7448cd0bf..3dd505830 100644 --- a/command/job_status.go +++ b/command/job_status.go @@ -381,7 +381,7 @@ func (c *JobStatusCommand) outputJobInfo(client *api.Client, job *api.Job) error if latestDeployment != nil { c.Ui.Output(c.Colorize().Color("\n[bold]Latest Deployment[reset]")) - c.Ui.Output(c.Colorize().Color(c.formatDeployment(latestDeployment))) + c.Ui.Output(c.Colorize().Color(c.formatDeployment(client, latestDeployment))) } // Format the allocs @@ -390,7 +390,7 @@ func (c *JobStatusCommand) outputJobInfo(client *api.Client, job *api.Job) error return nil } -func (c *JobStatusCommand) formatDeployment(d *api.Deployment) string { +func (c *JobStatusCommand) formatDeployment(client *api.Client, d *api.Deployment) string { // Format the high-level elements high := []string{ fmt.Sprintf("ID|%s", limit(d.ID, c.length)), @@ -399,6 +399,17 @@ func (c *JobStatusCommand) formatDeployment(d *api.Deployment) string { } base := formatKV(high) + + if d.IsMultiregion { + regions, err := fetchMultiRegionDeployments(client, d) + if err != nil { + base += "\n\nError fetching Multiregion deployments\n\n" + } else if len(regions) > 0 { + base += "\n\n[bold]Multiregion Deployment[reset]\n" + base += formatMultiregionDeployment(regions, 8) + } + } + if len(d.TaskGroups) == 0 { return base } diff --git a/command/job_status_test.go b/command/job_status_test.go index 745af018e..116ffe624 100644 --- a/command/job_status_test.go +++ b/command/job_status_test.go @@ -385,6 +385,107 @@ func TestJobStatusCommand_RescheduleEvals(t *testing.T) { require.Contains(out, e.ID[:8]) } +// TestJobStatusCommand_Multiregion tests multiregion deployment output +func TestJobStatusCommand_Multiregion(t *testing.T) { + t.Parallel() + + cbe := func(config *agent.Config) { + config.Region = "east" + config.Datacenter = "east-1" + } + cbw := func(config *agent.Config) { + config.Region = "west" + config.Datacenter = "west-1" + } + + srv, clientEast, url := testServer(t, true, cbe) + defer srv.Shutdown() + + srv2, clientWest, _ := testServer(t, true, cbw) + defer srv2.Shutdown() + + // Join with srv1 + addr := fmt.Sprintf("127.0.0.1:%d", + srv.Agent.Server().GetConfig().SerfConfig.MemberlistConfig.BindPort) + + if _, err := srv2.Agent.Server().Join([]string{addr}); err != nil { + t.Fatalf("Join err: %v", err) + } + + // wait for client node + testutil.WaitForResult(func() (bool, error) { + nodes, _, err := clientEast.Nodes().List(nil) + if err != nil { + return false, err + } + if len(nodes) == 0 { + return false, fmt.Errorf("missing node") + } + if _, ok := nodes[0].Drivers["mock_driver"]; !ok { + return false, fmt.Errorf("mock_driver not ready") + } + return true, nil + }, func(err error) { + t.Fatalf("err: %s", err) + }) + + ui := new(cli.MockUi) + cmd := &JobStatusCommand{Meta: Meta{Ui: ui, flagAddress: url}} + + // Register multiregion job + // Register multiregion job in east + jobEast := testMultiRegionJob("job1_sfxx", "east", "east-1") + resp, _, err := clientEast.Jobs().Register(jobEast, nil) + require.NoError(t, err) + if code := waitForSuccess(ui, clientEast, fullId, t, resp.EvalID); code != 0 { + t.Fatalf("status code non zero saw %d", code) + } + + // Register multiregion job in west + jobWest := testMultiRegionJob("job1_sfxx", "west", "west-1") + resp2, _, err := clientWest.Jobs().Register(jobWest, &api.WriteOptions{Region: "west"}) + require.NoError(t, err) + if code := waitForSuccess(ui, clientWest, fullId, t, resp2.EvalID); code != 0 { + t.Fatalf("status code non zero saw %d", code) + } + + jobs, _, err := clientEast.Jobs().List(&api.QueryOptions{}) + require.NoError(t, err) + require.Len(t, jobs, 1) + + deploys, _, err := clientEast.Jobs().Deployments(jobs[0].ID, true, &api.QueryOptions{}) + require.NoError(t, err) + require.Len(t, deploys, 1) + + // Grab both deployments to verify output + eastDeploys, _, err := clientEast.Jobs().Deployments(jobs[0].ID, true, &api.QueryOptions{Region: "east"}) + require.NoError(t, err) + require.Len(t, eastDeploys, 1) + + westDeploys, _, err := clientWest.Jobs().Deployments(jobs[0].ID, true, &api.QueryOptions{Region: "west"}) + require.NoError(t, err) + // require.Len(t, westDeploys, 1) + + // Run command for specific deploy + if code := cmd.Run([]string{"-address=" + url, jobs[0].ID}); code != 0 { + t.Fatalf("expected exit 0, got: %d", code) + } + + // Verify Multi-region Deployment info populated + out := ui.OutputWriter.String() + require.Contains(t, out, "Multiregion Deployment") + require.Contains(t, out, "Region") + require.Contains(t, out, "ID") + require.Contains(t, out, "Status") + require.Contains(t, out, "east") + require.Contains(t, out, eastDeploys[0].ID[0:7]) + require.Contains(t, out, "west") + require.Contains(t, out, westDeploys[0].ID[0:7]) + require.Contains(t, out, "running") + + require.NotContains(t, out, "") + +} func waitForSuccess(ui cli.Ui, client *api.Client, length int, t *testing.T, evalId string) int { mon := newMonitor(ui, client, length) monErr := mon.monitor(evalId, false) diff --git a/command/util_test.go b/command/util_test.go index a989e532e..a7060742a 100644 --- a/command/util_test.go +++ b/command/util_test.go @@ -48,3 +48,40 @@ func testJob(jobID string) *api.Job { return job } + +func testMultiRegionJob(jobID, region, datacenter string) *api.Job { + task := api.NewTask("task1", "mock_driver"). + SetConfig("kill_after", "10s"). + SetConfig("run_for", "15s"). + SetConfig("exit_code", 0). + Require(&api.Resources{ + MemoryMB: helper.IntToPtr(256), + CPU: helper.IntToPtr(100), + }). + SetLogConfig(&api.LogConfig{ + MaxFiles: helper.IntToPtr(1), + MaxFileSizeMB: helper.IntToPtr(2), + }) + + group := api.NewTaskGroup("group1", 1). + AddTask(task). + RequireDisk(&api.EphemeralDisk{ + SizeMB: helper.IntToPtr(20), + }) + + job := api.NewServiceJob(jobID, jobID, region, 1).AddDatacenter(datacenter).AddTaskGroup(group) + job.Multiregion = &api.Multiregion{ + Regions: []*api.MultiregionRegion{ + { + Name: "east", + Datacenters: []string{"east-1"}, + }, + { + Name: "west", + Datacenters: []string{"west-1"}, + }, + }, + } + + return job +} diff --git a/jobspec/parse_multiregion.go b/jobspec/parse_multiregion.go index 725576fc8..00179929a 100644 --- a/jobspec/parse_multiregion.go +++ b/jobspec/parse_multiregion.go @@ -144,7 +144,7 @@ func parseMultiregionRegions(result *api.Multiregion, list *ast.ObjectList) erro // Build the region with the basic decode var r api.MultiregionRegion - r.Name = helper.StringToPtr(n) + r.Name = n dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ WeaklyTypedInput: true, Result: &r, diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 4c2e66139..f0daa5c8d 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -1350,13 +1350,13 @@ func TestParse(t *testing.T) { }, Regions: []*api.MultiregionRegion{ { - Name: helper.StringToPtr("west"), + Name: "west", Count: helper.IntToPtr(2), Datacenters: []string{"west-1"}, Meta: map[string]string{"region_code": "W"}, }, { - Name: helper.StringToPtr("east"), + Name: "east", Count: helper.IntToPtr(1), Datacenters: []string{"east-1", "east-2"}, Meta: map[string]string{"region_code": "E"}, diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index a57f30a1b..76a15753b 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -7948,6 +7948,9 @@ type Deployment struct { // present the correct list of deployments for the job and not old ones. JobCreateIndex uint64 + // Multiregion specifies if deployment is part of multiregion deployment + IsMultiregion bool + // TaskGroups is the set of task groups effected by the deployment and their // current deployment status. TaskGroups map[string]*DeploymentState @@ -7973,6 +7976,7 @@ func NewDeployment(job *Job) *Deployment { JobModifyIndex: job.ModifyIndex, JobSpecModifyIndex: job.JobModifyIndex, JobCreateIndex: job.CreateIndex, + IsMultiregion: job.IsMultiregion(), Status: DeploymentStatusRunning, StatusDescription: DeploymentStatusDescriptionRunning, TaskGroups: make(map[string]*DeploymentState, len(job.TaskGroups)), diff --git a/vendor/github.com/hashicorp/nomad/api/deployments.go b/vendor/github.com/hashicorp/nomad/api/deployments.go index 8a241243c..af8487782 100644 --- a/vendor/github.com/hashicorp/nomad/api/deployments.go +++ b/vendor/github.com/hashicorp/nomad/api/deployments.go @@ -150,6 +150,9 @@ type Deployment struct { // present the correct list of deployments for the job and not old ones. JobCreateIndex uint64 + // IsMultiregion specifies if this deployment is part of a multi-region deployment + IsMultiregion bool + // TaskGroups is the set of task groups effected by the deployment and their // current deployment status. TaskGroups map[string]*DeploymentState diff --git a/vendor/github.com/hashicorp/nomad/api/jobs.go b/vendor/github.com/hashicorp/nomad/api/jobs.go index b37ce4004..fbb3b2417 100644 --- a/vendor/github.com/hashicorp/nomad/api/jobs.go +++ b/vendor/github.com/hashicorp/nomad/api/jobs.go @@ -645,9 +645,6 @@ func (m *Multiregion) Canonicalize() { m.Regions = []*MultiregionRegion{} } for _, region := range m.Regions { - if region.Name == nil { - region.Name = stringToPtr("") - } if region.Count == nil { region.Count = intToPtr(1) } @@ -672,7 +669,7 @@ func (m *Multiregion) Copy() *Multiregion { } for _, region := range m.Regions { copyRegion := new(MultiregionRegion) - copyRegion.Name = stringToPtr(*region.Name) + copyRegion.Name = region.Name copyRegion.Count = intToPtr(*region.Count) for _, dc := range region.Datacenters { copyRegion.Datacenters = append(copyRegion.Datacenters, dc) @@ -691,7 +688,7 @@ type MultiregionStrategy struct { } type MultiregionRegion struct { - Name *string + Name string Count *int Datacenters []string Meta map[string]string