diff --git a/command/alloc_status.go b/command/alloc_status.go index 58a87877d..0c6e78d4f 100644 --- a/command/alloc_status.go +++ b/command/alloc_status.go @@ -73,20 +73,20 @@ func dumpAllocStatus(ui cli.Ui, alloc *api.Allocation) { // Print exhaustion info if ne := alloc.Metrics.NodesExhausted; ne > 0 { - ui.Output(fmt.Sprintf("Resources exhausted on %d nodes", ne)) + ui.Output(fmt.Sprintf(" * Resources exhausted on %d nodes", ne)) } for class, num := range alloc.Metrics.ClassExhausted { - ui.Output(fmt.Sprintf("Class %q exhausted on %d nodes", class, num)) + ui.Output(fmt.Sprintf(" * Class %q exhausted on %d nodes", class, num)) } for dim, num := range alloc.Metrics.DimensionExhausted { - ui.Output(fmt.Sprintf("Dimension %q exhausted on %d nodes", dim, num)) + ui.Output(fmt.Sprintf(" * Dimension %q exhausted on %d nodes", dim, num)) } // Print filter info for class, num := range alloc.Metrics.ClassFiltered { - ui.Output(fmt.Sprintf("Class %q filtered %d nodes", class, num)) + ui.Output(fmt.Sprintf(" * Class %q filtered %d nodes", class, num)) } for cs, num := range alloc.Metrics.ConstraintFiltered { - ui.Output(fmt.Sprintf("Constraint %q filtered %d nodes", cs, num)) + ui.Output(fmt.Sprintf(" * Constraint %q filtered %d nodes", cs, num)) } } diff --git a/command/alloc_status_test.go b/command/alloc_status_test.go new file mode 100644 index 000000000..d76cb3672 --- /dev/null +++ b/command/alloc_status_test.go @@ -0,0 +1,47 @@ +package command + +import ( + "strings" + "testing" + + "github.com/mitchellh/cli" +) + +func TestAllocStatusCommand_Implements(t *testing.T) { + var _ cli.Command = &AllocStatusCommand{} +} + +func TestAllocStatusCommand_Run(t *testing.T) { + srv, _, url := testServer(t, nil) + defer srv.Stop() + + ui := new(cli.MockUi) + cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}} + + code := cmd.Run([]string{"-address=" + url}) + if code != 0 { + t.Fatalf("expected exit 0, got: %d %s", code) + } +} + +func TestAllocStatusCommand_Fails(t *testing.T) { + ui := new(cli.MockUi) + cmd := &AllocStatusCommand{Meta: Meta{Ui: ui}} + + // Fails on misuse + if code := cmd.Run([]string{"some", "bad", "args"}); code != 1 { + t.Fatalf("expected exit code 1, got: %d", code) + } + if out := ui.ErrorWriter.String(); !strings.Contains(out, cmd.Help()) { + t.Fatalf("expected help output, got: %s", out) + } + ui.ErrorWriter.Reset() + + // Fails on connection failure + if code := cmd.Run([]string{"-address=nope", "nope"}); code != 1 { + t.Fatalf("expected exit code 1, got: %d", code) + } + if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error querying allocation") { + t.Fatalf("expected failed query error, got: %s", out) + } +} diff --git a/command/monitor.go b/command/monitor.go index ab367684c..da48a29b5 100644 --- a/command/monitor.go +++ b/command/monitor.go @@ -22,7 +22,7 @@ const ( type evalState struct { status string desc string - nodeID string + node string allocs map[string]*allocState wait time.Duration index uint64 @@ -37,6 +37,11 @@ type allocState struct { desiredDesc string client string index uint64 + + // full is the allocation struct with full details. This + // must be queried for explicitly so it is only included + // if there is important error information inside. + full *api.Allocation } // monitor wraps an evaluation monitor and holds metadata and @@ -75,33 +80,16 @@ func (m *monitor) init() { // update is used to update our monitor with new state. It can be // called whether the passed information is new or not, and will // only dump update messages when state changes. -func (m *monitor) update(eval *api.Evaluation, allocs []*api.AllocationListStub) { +func (m *monitor) update(update *evalState) { m.Lock() defer m.Unlock() existing := m.state - // Create the new state - update := &evalState{ - status: eval.Status, - desc: eval.StatusDescription, - nodeID: eval.NodeID, - allocs: make(map[string]*allocState), - wait: eval.Wait, - index: eval.CreateIndex, - } - for _, alloc := range allocs { - update.allocs[alloc.ID] = &allocState{ - id: alloc.ID, - group: alloc.TaskGroup, - node: alloc.NodeID, - desired: alloc.DesiredStatus, - desiredDesc: alloc.DesiredDescription, - client: alloc.ClientStatus, - index: alloc.CreateIndex, - } - } - defer func() { m.state = update }() + // Swap in the new state at the end + defer func() { + m.state = update + }() // Check the allocations for allocID, alloc := range update.allocs { @@ -115,12 +103,9 @@ func (m *monitor) update(eval *api.Evaluation, allocs []*api.AllocationListStub) // Generate a more descriptive error for why the allocation // failed and dump it to the screen - fullAlloc, _, err := m.client.Allocations().Info(allocID, nil) - if err != nil { - m.ui.Output(fmt.Sprintf("Error querying alloc: %s", err)) - continue + if alloc.full != nil { + dumpAllocStatus(m.ui, alloc.full) } - dumpAllocStatus(m.ui, fullAlloc) case alloc.index < update.index: // New alloc with create index lower than the eval @@ -149,19 +134,19 @@ func (m *monitor) update(eval *api.Evaluation, allocs []*api.AllocationListStub) // Check if the status changed if existing.status != update.status { m.ui.Output(fmt.Sprintf("Evaluation status changed: %q -> %q", - existing.status, eval.Status)) + existing.status, update.status)) } // Check if the wait time is different if existing.wait == 0 && update.wait != 0 { m.ui.Output(fmt.Sprintf("Waiting %s before running eval", - eval.Wait)) + update.wait)) } - // Check if the nodeID changed - if existing.nodeID == "" && update.nodeID != "" { + // Check if the node changed + if existing.node == "" && update.node != "" { m.ui.Output(fmt.Sprintf("Evaluation was assigned node ID %q", - eval.NodeID)) + update.node)) } } @@ -180,6 +165,16 @@ func (m *monitor) monitor(evalID string) int { return 1 } + // Create the new eval state. + state := &evalState{ + status: eval.Status, + desc: eval.StatusDescription, + node: eval.NodeID, + allocs: make(map[string]*allocState), + wait: eval.Wait, + index: eval.CreateIndex, + } + // Query the allocations associated with the evaluation allocs, _, err := m.client.Evaluations().Allocations(evalID, nil) if err != nil { @@ -187,8 +182,32 @@ func (m *monitor) monitor(evalID string) int { return 1 } + // Add the allocs to the state + for _, alloc := range allocs { + state.allocs[alloc.ID] = &allocState{ + id: alloc.ID, + group: alloc.TaskGroup, + node: alloc.NodeID, + desired: alloc.DesiredStatus, + desiredDesc: alloc.DesiredDescription, + client: alloc.ClientStatus, + index: alloc.CreateIndex, + } + + // If we have a scheduling error, query the full allocation + // to get the details. + if alloc.DesiredStatus == structs.AllocDesiredStatusFailed { + failed, _, err := m.client.Allocations().Info(alloc.ID, nil) + if err != nil { + m.ui.Error(fmt.Sprintf("Error querying allocation: %s", err)) + return 1 + } + state.allocs[alloc.ID].full = failed + } + } + // Update the state - m.update(eval, allocs) + m.update(state) switch eval.Status { case structs.EvalStatusComplete, structs.EvalStatusFailed: diff --git a/command/monitor_test.go b/command/monitor_test.go index d885c9be1..87eb186f8 100644 --- a/command/monitor_test.go +++ b/command/monitor_test.go @@ -10,18 +10,17 @@ import ( "github.com/mitchellh/cli" ) -func TestMonitor_Update(t *testing.T) { +func TestMonitor_Update_Eval(t *testing.T) { ui := new(cli.MockUi) mon := newMonitor(ui, nil) - // Basic eval updates work - eval := &api.Evaluation{ - Status: "pending", - NodeID: "node1", - Wait: 10 * time.Second, - CreateIndex: 2, + state := &evalState{ + status: structs.EvalStatusPending, + node: "node1", + wait: 10 * time.Second, + index: 2, } - mon.update(eval, nil) + mon.update(state) // Logs were output out := ui.OutputWriter.String() @@ -36,36 +35,47 @@ func TestMonitor_Update(t *testing.T) { } ui.OutputWriter.Reset() - // No logs sent if no state update - mon.update(eval, nil) + // No logs sent if no update + mon.update(state) if out := ui.OutputWriter.String(); out != "" { t.Fatalf("expected no output\n\n%s", out) } - // Updates cause more logs to output - eval.Status = "complete" - mon.update(eval, nil) + // Status change sends more logs + state = &evalState{ + status: structs.EvalStatusComplete, + node: "node1", + wait: 10 * time.Second, + index: 3, + } + mon.update(state) out = ui.OutputWriter.String() - if !strings.Contains(out, "complete") { + if !strings.Contains(out, structs.EvalStatusComplete) { t.Fatalf("missing status\n\n%s", out) } - ui.OutputWriter.Reset() +} + +func TestMonitor_Update_Allocs(t *testing.T) { + ui := new(cli.MockUi) + mon := newMonitor(ui, nil) // New allocations write new logs - allocs := []*api.AllocationListStub{ - &api.AllocationListStub{ - ID: "alloc1", - TaskGroup: "group1", - NodeID: "node1", - DesiredStatus: structs.AllocDesiredStatusRun, - ClientStatus: structs.AllocClientStatusPending, - CreateIndex: 3, + state := &evalState{ + allocs: map[string]*allocState{ + "alloc1": &allocState{ + id: "alloc1", + group: "group1", + node: "node1", + desired: structs.AllocDesiredStatusRun, + client: structs.AllocClientStatusPending, + index: 1, + }, }, } - mon.update(eval, allocs) + mon.update(state) // Logs were output - out = ui.OutputWriter.String() + out := ui.OutputWriter.String() if !strings.Contains(out, "alloc1") { t.Fatalf("missing alloc\n\n%s", out) } @@ -81,15 +91,28 @@ func TestMonitor_Update(t *testing.T) { ui.OutputWriter.Reset() // No change yields no logs - mon.update(eval, allocs) + mon.update(state) if out := ui.OutputWriter.String(); out != "" { t.Fatalf("expected no output\n\n%s", out) } ui.OutputWriter.Reset() - // Updates cause more log lines - allocs[0].ClientStatus = "running" - mon.update(eval, allocs) + // Alloc updates cause more log lines + state = &evalState{ + allocs: map[string]*allocState{ + "alloc1": &allocState{ + id: "alloc1", + group: "group1", + node: "node1", + desired: structs.AllocDesiredStatusRun, + client: structs.AllocClientStatusRunning, + index: 2, + }, + }, + } + mon.update(state) + + // Updates were logged out = ui.OutputWriter.String() if !strings.Contains(out, "alloc1") { t.Fatalf("missing alloc\n\n%s", out) @@ -100,20 +123,43 @@ func TestMonitor_Update(t *testing.T) { if !strings.Contains(out, "running") { t.Fatalf("missing new status\n\n%s", out) } - ui.OutputWriter.Reset() +} + +func TestMonitor_Update_SchedulingFailure(t *testing.T) { + ui := new(cli.MockUi) + mon := newMonitor(ui, nil) // New allocs with desired status failed warns - allocs = append(allocs, &api.AllocationListStub{ - ID: "alloc2", - TaskGroup: "group2", - DesiredStatus: structs.AllocDesiredStatusFailed, - DesiredDescription: "something failed", - CreateIndex: 4, - }) - mon.update(eval, allocs) + state := &evalState{ + allocs: map[string]*allocState{ + "alloc2": &allocState{ + id: "alloc2", + group: "group2", + desired: structs.AllocDesiredStatusFailed, + desiredDesc: "something failed", + index: 1, + + // Attach the full failed allocation + full: &api.Allocation{ + ID: "alloc2", + TaskGroup: "group2", + ClientStatus: structs.AllocClientStatusFailed, + DesiredStatus: structs.AllocDesiredStatusFailed, + Metrics: &api.AllocationMetric{ + NodesEvaluated: 3, + NodesFiltered: 3, + ConstraintFiltered: map[string]int{ + "$attr.kernel.name = linux": 3, + }, + }, + }, + }, + }, + } + mon.update(state) // Scheduling failure was logged - out = ui.OutputWriter.String() + out := ui.OutputWriter.String() if !strings.Contains(out, "group2") { t.Fatalf("missing group\n\n%s", out) } @@ -123,20 +169,40 @@ func TestMonitor_Update(t *testing.T) { if !strings.Contains(out, "something failed") { t.Fatalf("missing reason\n\n%s", out) } - ui.OutputWriter.Reset() + + // Check that the allocation details were dumped + if !strings.Contains(out, "3/3") { + t.Fatalf("missing filter stats\n\n%s", out) + } + if !strings.Contains(out, structs.AllocDesiredStatusFailed) { + t.Fatalf("missing alloc status\n\n%s", out) + } + if !strings.Contains(out, "$attr.kernel.name = linux") { + t.Fatalf("missing constraint\n\n%s", out) + } +} + +func TestMonitor_Update_AllocModification(t *testing.T) { + ui := new(cli.MockUi) + mon := newMonitor(ui, nil) // New allocs with a create index lower than the // eval create index are logged as modifications - allocs = append(allocs, &api.AllocationListStub{ - ID: "alloc3", - NodeID: "node1", - TaskGroup: "group2", - CreateIndex: 1, - }) - mon.update(eval, allocs) + state := &evalState{ + index: 2, + allocs: map[string]*allocState{ + "alloc3": &allocState{ + id: "alloc3", + node: "node1", + group: "group2", + index: 1, + }, + }, + } + mon.update(state) // Modification was logged - out = ui.OutputWriter.String() + out := ui.OutputWriter.String() if !strings.Contains(out, "alloc3") { t.Fatalf("missing alloc\n\n%s", out) }