Add eval-status and remove eval-monitor

command/alloc_status.go

@@ -27,6 +27,7 @@ General Options:
 
   ` + generalOptionsUsage() + `
 
+Alloc Status Options:
 
   -short
     Display short output. Shows only the most recent task event.

command/eval_monitor.go

@@ -1,81 +0,0 @@
-package command
-
-import (
-	"fmt"
-	"strings"
-)
-
-type EvalMonitorCommand struct {
-	Meta
-}
-
-func (c *EvalMonitorCommand) Help() string {
-	helpText := `
-Usage: nomad eval-monitor [options] <evaluation>
-
-  Start an interactive monitoring session for an existing evaluation.
-  The monitor command periodically polls for information about the
-  provided evaluation, including status updates, new allocations,
-  updates to allocations, and failures. Status is printed in near
-  real-time to the terminal.
-
-  The command will exit when the given evaluation reaches a terminal
-  state (completed or failed). Exit code 0 is returned on successful
-  evaluation, and if there are no scheduling problems. If there are
-  job placement issues encountered (unsatisfiable constraints,
-  resource exhaustion, etc), then the exit code will be 2. Any other
-  errors, including client connection issues or internal errors, are
-  indicated by exit code 1.
-
-General Options:
-
-  ` + generalOptionsUsage() + `
-
-Eval Monitor Options:
-
-  -verbose
-    Show full information.
-`
-	return strings.TrimSpace(helpText)
-}
-
-func (c *EvalMonitorCommand) Synopsis() string {
-	return "Monitor an evaluation interactively"
-}
-
-func (c *EvalMonitorCommand) Run(args []string) int {
-	var verbose bool
-
-	flags := c.Meta.FlagSet("eval-monitor", FlagSetClient)
-	flags.Usage = func() { c.Ui.Output(c.Help()) }
-	flags.BoolVar(&verbose, "verbose", false, "")
-
-	if err := flags.Parse(args); err != nil {
-		return 1
-	}
-
-	// Truncate the id unless full length is requested
-	length := shortId
-	if verbose {
-		length = fullId
-	}
-
-	// Check that we got exactly one eval ID
-	args = flags.Args()
-	if len(args) != 1 {
-		c.Ui.Error(c.Help())
-		return 1
-	}
-	evalID := args[0]
-
-	// Get the HTTP client
-	client, err := c.Meta.Client()
-	if err != nil {
-		c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
-		return 1
-	}
-
-	// Start monitoring
-	mon := newMonitor(c.Ui, client, length)
-	return mon.monitor(evalID, true)
-}

command/eval_status.go

@@ -0,0 +1,184 @@
+package command
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/nomad/api"
+)
+
+type EvalStatusCommand struct {
+	Meta
+}
+
+func (c *EvalStatusCommand) Help() string {
+	helpText := `
+Usage: nomad eval-status [options] <evaluation-id>
+
+  Display information about evaluations. This command can be used to inspect the
+  current status of an evaluation as well as determine the reason an evaluation
+  did not place all allocations.
+
+General Options:
+
+  ` + generalOptionsUsage() + `
+
+Eval Status Options:
+
+  -monitor
+    Monitor an outstanding evaluation
+
+  -verbose
+    Show full information.
+`
+
+	return strings.TrimSpace(helpText)
+}
+
+func (c *EvalStatusCommand) Synopsis() string {
+	return "Display evaluation status information and placement failure reasons"
+}
+
+func (c *EvalStatusCommand) Run(args []string) int {
+	var monitor, verbose bool
+
+	flags := c.Meta.FlagSet("eval-status", FlagSetClient)
+	flags.Usage = func() { c.Ui.Output(c.Help()) }
+	flags.BoolVar(&monitor, "monitor", false, "")
+	flags.BoolVar(&verbose, "verbose", false, "")
+
+	if err := flags.Parse(args); err != nil {
+		return 1
+	}
+
+	// Check that we got exactly one evaluation ID
+	args = flags.Args()
+	if len(args) != 1 {
+		c.Ui.Error(c.Help())
+		return 1
+	}
+	evalID := args[0]
+
+	// Get the HTTP client
+	client, err := c.Meta.Client()
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
+		return 1
+	}
+
+	// Truncate the id unless full length is requested
+	length := shortId
+	if verbose {
+		length = fullId
+	}
+
+	// Query the allocation info
+	if len(evalID) == 1 {
+		c.Ui.Error(fmt.Sprintf("Identifier must contain at least two characters."))
+		return 1
+	}
+	if len(evalID)%2 == 1 {
+		// Identifiers must be of even length, so we strip off the last byte
+		// to provide a consistent user experience.
+		evalID = evalID[:len(evalID)-1]
+	}
+
+	evals, _, err := client.Evaluations().PrefixList(evalID)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error querying evaluation: %v", err))
+		return 1
+	}
+	if len(evals) == 0 {
+		c.Ui.Error(fmt.Sprintf("No evaluation(s) with prefix or id %q found", evalID))
+		return 1
+	}
+	if len(evals) > 1 {
+		// Format the evals
+		out := make([]string, len(evals)+1)
+		out[0] = "ID|Priority|Triggered By|Status|Placement Failures"
+		for i, eval := range evals {
+			out[i+1] = fmt.Sprintf("%s|%d|%s|%s|%t",
+				limit(eval.ID, length),
+				eval.Priority,
+				eval.TriggeredBy,
+				eval.Status,
+				len(eval.FailedTGAllocs) != 0,
+			)
+		}
+		c.Ui.Output(fmt.Sprintf("Prefix matched multiple evaluations\n\n%s", formatList(out)))
+		return 0
+	}
+
+	// If we are in monitor mode, monitor and exit
+	if monitor {
+		mon := newMonitor(c.Ui, client, length)
+		return mon.monitor(evals[0].ID, true)
+	}
+
+	// Prefix lookup matched a single evaluation
+	eval, _, err := client.Evaluations().Info(evals[0].ID, nil)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error querying evaluation: %s", err))
+		return 1
+	}
+
+	failures := len(eval.FailedTGAllocs) != 0
+	triggerNoun, triggerSubj := getTriggerDetails(eval)
+	statusDesc := eval.StatusDescription
+	if statusDesc == "" {
+		statusDesc = eval.Status
+	}
+
+	// Format the allocation data
+	basic := []string{
+		fmt.Sprintf("ID|%s", limit(eval.ID, length)),
+		fmt.Sprintf("Status|%s", eval.Status),
+		fmt.Sprintf("Status Description|%s", statusDesc),
+		fmt.Sprintf("Type|%s", eval.Type),
+		fmt.Sprintf("TriggeredBy|%s", eval.TriggeredBy),
+		fmt.Sprintf("%s|%s", triggerNoun, triggerSubj),
+		fmt.Sprintf("Priority|%d", eval.Priority),
+		fmt.Sprintf("Placement Failures|%t", failures),
+	}
+
+	if verbose {
+		// NextEval, PreviousEval, BlockedEval
+		basic = append(basic,
+			fmt.Sprintf("Previous Eval|%s", eval.PreviousEval),
+			fmt.Sprintf("Next Eval|%s", eval.NextEval),
+			fmt.Sprintf("Blocked Eval|%s", eval.BlockedEval))
+	}
+	c.Ui.Output(formatKV(basic))
+
+	if failures {
+		c.Ui.Output("\n==> Failed Allocations")
+		for tg, metrics := range eval.FailedTGAllocs {
+			noun := "allocation"
+			if metrics.CoalescedFailures > 0 {
+				noun += "s"
+			}
+			c.Ui.Output(fmt.Sprintf("Task Group %q (failed to place %d %s):", tg, metrics.CoalescedFailures+1, noun))
+			dumpAllocMetrics(c.Ui, metrics, false)
+		}
+
+		if eval.BlockedEval != "" {
+			c.Ui.Output(fmt.Sprintf("\nEvaluation %q waiting for additional capacity to place remainder",
+				limit(eval.BlockedEval, length)))
+		}
+	}
+
+	return 0
+}
+
+func getTriggerDetails(eval *api.Evaluation) (noun, subject string) {
+	switch eval.TriggeredBy {
+	case "job-register", "job-deregister", "periodic-job", "rolling-update":
+		return "Job ID", eval.JobID
+	case "node-update":
+		return "Node ID", eval.NodeID
+	case "max-plan-attempts":
+		return "Previous Eval", eval.PreviousEval
+	default:
+		return "", ""
+	}
+}

command/eval_status_test.go

@@ -7,16 +7,16 @@ import (
 	"github.com/mitchellh/cli"
 )
 
-func TestEvalMonitorCommand_Implements(t *testing.T) {
-	var _ cli.Command = &EvalMonitorCommand{}
+func TestEvalStatusCommand_Implements(t *testing.T) {
+	var _ cli.Command = &EvalStatusCommand{}
 }
 
-func TestEvalMonitorCommand_Fails(t *testing.T) {
+func TestEvalStatusCommand_Fails(t *testing.T) {
 	srv, _, url := testServer(t, nil)
 	defer srv.Stop()
 
 	ui := new(cli.MockUi)
-	cmd := &EvalMonitorCommand{Meta: Meta{Ui: ui}}
+	cmd := &EvalStatusCommand{Meta: Meta{Ui: ui}}
 
 	// Fails on misuse
 	if code := cmd.Run([]string{"some", "bad", "args"}); code != 1 {
@@ -40,7 +40,7 @@ func TestEvalMonitorCommand_Fails(t *testing.T) {
 	if code := cmd.Run([]string{"-address=nope", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 {
 		t.Fatalf("expected exit code 1, got: %d", code)
 	}
-	if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error reading evaluation") {
+	if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error querying evaluation") {
 		t.Fatalf("expected failed query error, got: %s", out)
 	}
 }

command/fs.go

@@ -28,6 +28,8 @@ General Options:
 
   ` + generalOptionsUsage() + `
 
+FS Specific Options:
+
   -H
     Machine friendly output.
 

command/plan.go

@@ -47,7 +47,7 @@ General Options:
 
   ` + generalOptionsUsage() + `
 
-Run Options:
+Plan Options:
 
   -diff
     Defaults to true, but can be toggled off to omit diff output.

command/server_members.go

@@ -23,7 +23,7 @@ General Options:
 
   ` + generalOptionsUsage() + `
 
-Agent Members Options:
+Server Members Options:
 
   -detailed
     Show detailed information about each member. This dumps

commands.go

@@ -54,8 +54,8 @@ func Commands(metaPtr *command.Meta) map[string]cli.CommandFactory {
 				Meta: meta,
 			}, nil
 		},
-		"eval-monitor": func() (cli.Command, error) {
-			return &command.EvalMonitorCommand{
+		"eval-status": func() (cli.Command, error) {
+			return &command.EvalStatusCommand{
 				Meta: meta,
 			}, nil
 		},

scheduler/generic_sched.go

@@ -29,6 +29,14 @@ const (
 
 	// allocInPlace is the status used when speculating on an in-place update
 	allocInPlace = "alloc updating in-place"
+
+	// blockedEvalMaxPlanDesc is the description used for blocked evals that are
+	// a result of hitting the max number of plan attempts
+	blockedEvalMaxPlanDesc = "created due to placement conflicts"
+
+	// blockedEvalFailedPlacements is the description used for blocked evals
+	// that are a result of failing to place all allocations.
+	blockedEvalFailedPlacements = "created to place remaining allocations"
 )
 
 // SetStatusError is used to set the status of the evaluation to the given error
@@ -154,6 +162,9 @@ func (s *GenericScheduler) createBlockedEval(planFailure bool) error {
 	s.blocked = s.eval.CreateBlockedEval(classEligibility, escaped)
 	if planFailure {
 		s.blocked.TriggeredBy = structs.EvalTriggerMaxPlans
+		s.blocked.StatusDescription = blockedEvalMaxPlanDesc
+	} else {
+		s.blocked.StatusDescription = blockedEvalFailedPlacements
 	}
 
 	return s.planner.CreateEval(s.blocked)