From 38dbe768e7f813f198abbc4f2ea2c321d9200cfd Mon Sep 17 00:00:00 2001 From: Alex Dadgar Date: Wed, 25 May 2016 14:11:14 -0700 Subject: [PATCH] Add eval-status and remove eval-monitor --- command/alloc_status.go | 1 + command/eval_monitor.go | 81 -------- command/eval_status.go | 184 ++++++++++++++++++ ...al_monitor_test.go => eval_status_test.go} | 10 +- command/fs.go | 2 + command/plan.go | 2 +- command/server_members.go | 2 +- commands.go | 4 +- scheduler/generic_sched.go | 11 ++ 9 files changed, 207 insertions(+), 90 deletions(-) delete mode 100644 command/eval_monitor.go create mode 100644 command/eval_status.go rename command/{eval_monitor_test.go => eval_status_test.go} (83%) diff --git a/command/alloc_status.go b/command/alloc_status.go index 700a17546..76e6d84cb 100644 --- a/command/alloc_status.go +++ b/command/alloc_status.go @@ -27,6 +27,7 @@ General Options: ` + generalOptionsUsage() + ` +Alloc Status Options: -short Display short output. Shows only the most recent task event. diff --git a/command/eval_monitor.go b/command/eval_monitor.go deleted file mode 100644 index 72d4ffa6f..000000000 --- a/command/eval_monitor.go +++ /dev/null @@ -1,81 +0,0 @@ -package command - -import ( - "fmt" - "strings" -) - -type EvalMonitorCommand struct { - Meta -} - -func (c *EvalMonitorCommand) Help() string { - helpText := ` -Usage: nomad eval-monitor [options] - - Start an interactive monitoring session for an existing evaluation. - The monitor command periodically polls for information about the - provided evaluation, including status updates, new allocations, - updates to allocations, and failures. Status is printed in near - real-time to the terminal. - - The command will exit when the given evaluation reaches a terminal - state (completed or failed). Exit code 0 is returned on successful - evaluation, and if there are no scheduling problems. If there are - job placement issues encountered (unsatisfiable constraints, - resource exhaustion, etc), then the exit code will be 2. Any other - errors, including client connection issues or internal errors, are - indicated by exit code 1. - -General Options: - - ` + generalOptionsUsage() + ` - -Eval Monitor Options: - - -verbose - Show full information. -` - return strings.TrimSpace(helpText) -} - -func (c *EvalMonitorCommand) Synopsis() string { - return "Monitor an evaluation interactively" -} - -func (c *EvalMonitorCommand) Run(args []string) int { - var verbose bool - - flags := c.Meta.FlagSet("eval-monitor", FlagSetClient) - flags.Usage = func() { c.Ui.Output(c.Help()) } - flags.BoolVar(&verbose, "verbose", false, "") - - if err := flags.Parse(args); err != nil { - return 1 - } - - // Truncate the id unless full length is requested - length := shortId - if verbose { - length = fullId - } - - // Check that we got exactly one eval ID - args = flags.Args() - if len(args) != 1 { - c.Ui.Error(c.Help()) - return 1 - } - evalID := args[0] - - // Get the HTTP client - client, err := c.Meta.Client() - if err != nil { - c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) - return 1 - } - - // Start monitoring - mon := newMonitor(c.Ui, client, length) - return mon.monitor(evalID, true) -} diff --git a/command/eval_status.go b/command/eval_status.go new file mode 100644 index 000000000..4d1728069 --- /dev/null +++ b/command/eval_status.go @@ -0,0 +1,184 @@ +package command + +import ( + "fmt" + "strings" + + "github.com/hashicorp/nomad/api" +) + +type EvalStatusCommand struct { + Meta +} + +func (c *EvalStatusCommand) Help() string { + helpText := ` +Usage: nomad eval-status [options] + + Display information about evaluations. This command can be used to inspect the + current status of an evaluation as well as determine the reason an evaluation + did not place all allocations. + +General Options: + + ` + generalOptionsUsage() + ` + +Eval Status Options: + + -monitor + Monitor an outstanding evaluation + + -verbose + Show full information. +` + + return strings.TrimSpace(helpText) +} + +func (c *EvalStatusCommand) Synopsis() string { + return "Display evaluation status information and placement failure reasons" +} + +func (c *EvalStatusCommand) Run(args []string) int { + var monitor, verbose bool + + flags := c.Meta.FlagSet("eval-status", FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + flags.BoolVar(&monitor, "monitor", false, "") + flags.BoolVar(&verbose, "verbose", false, "") + + if err := flags.Parse(args); err != nil { + return 1 + } + + // Check that we got exactly one evaluation ID + args = flags.Args() + if len(args) != 1 { + c.Ui.Error(c.Help()) + return 1 + } + evalID := args[0] + + // Get the HTTP client + client, err := c.Meta.Client() + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + // Truncate the id unless full length is requested + length := shortId + if verbose { + length = fullId + } + + // Query the allocation info + if len(evalID) == 1 { + c.Ui.Error(fmt.Sprintf("Identifier must contain at least two characters.")) + return 1 + } + if len(evalID)%2 == 1 { + // Identifiers must be of even length, so we strip off the last byte + // to provide a consistent user experience. + evalID = evalID[:len(evalID)-1] + } + + evals, _, err := client.Evaluations().PrefixList(evalID) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error querying evaluation: %v", err)) + return 1 + } + if len(evals) == 0 { + c.Ui.Error(fmt.Sprintf("No evaluation(s) with prefix or id %q found", evalID)) + return 1 + } + if len(evals) > 1 { + // Format the evals + out := make([]string, len(evals)+1) + out[0] = "ID|Priority|Triggered By|Status|Placement Failures" + for i, eval := range evals { + out[i+1] = fmt.Sprintf("%s|%d|%s|%s|%t", + limit(eval.ID, length), + eval.Priority, + eval.TriggeredBy, + eval.Status, + len(eval.FailedTGAllocs) != 0, + ) + } + c.Ui.Output(fmt.Sprintf("Prefix matched multiple evaluations\n\n%s", formatList(out))) + return 0 + } + + // If we are in monitor mode, monitor and exit + if monitor { + mon := newMonitor(c.Ui, client, length) + return mon.monitor(evals[0].ID, true) + } + + // Prefix lookup matched a single evaluation + eval, _, err := client.Evaluations().Info(evals[0].ID, nil) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error querying evaluation: %s", err)) + return 1 + } + + failures := len(eval.FailedTGAllocs) != 0 + triggerNoun, triggerSubj := getTriggerDetails(eval) + statusDesc := eval.StatusDescription + if statusDesc == "" { + statusDesc = eval.Status + } + + // Format the allocation data + basic := []string{ + fmt.Sprintf("ID|%s", limit(eval.ID, length)), + fmt.Sprintf("Status|%s", eval.Status), + fmt.Sprintf("Status Description|%s", statusDesc), + fmt.Sprintf("Type|%s", eval.Type), + fmt.Sprintf("TriggeredBy|%s", eval.TriggeredBy), + fmt.Sprintf("%s|%s", triggerNoun, triggerSubj), + fmt.Sprintf("Priority|%d", eval.Priority), + fmt.Sprintf("Placement Failures|%t", failures), + } + + if verbose { + // NextEval, PreviousEval, BlockedEval + basic = append(basic, + fmt.Sprintf("Previous Eval|%s", eval.PreviousEval), + fmt.Sprintf("Next Eval|%s", eval.NextEval), + fmt.Sprintf("Blocked Eval|%s", eval.BlockedEval)) + } + c.Ui.Output(formatKV(basic)) + + if failures { + c.Ui.Output("\n==> Failed Allocations") + for tg, metrics := range eval.FailedTGAllocs { + noun := "allocation" + if metrics.CoalescedFailures > 0 { + noun += "s" + } + c.Ui.Output(fmt.Sprintf("Task Group %q (failed to place %d %s):", tg, metrics.CoalescedFailures+1, noun)) + dumpAllocMetrics(c.Ui, metrics, false) + } + + if eval.BlockedEval != "" { + c.Ui.Output(fmt.Sprintf("\nEvaluation %q waiting for additional capacity to place remainder", + limit(eval.BlockedEval, length))) + } + } + + return 0 +} + +func getTriggerDetails(eval *api.Evaluation) (noun, subject string) { + switch eval.TriggeredBy { + case "job-register", "job-deregister", "periodic-job", "rolling-update": + return "Job ID", eval.JobID + case "node-update": + return "Node ID", eval.NodeID + case "max-plan-attempts": + return "Previous Eval", eval.PreviousEval + default: + return "", "" + } +} diff --git a/command/eval_monitor_test.go b/command/eval_status_test.go similarity index 83% rename from command/eval_monitor_test.go rename to command/eval_status_test.go index 26231b1ea..29ea4634f 100644 --- a/command/eval_monitor_test.go +++ b/command/eval_status_test.go @@ -7,16 +7,16 @@ import ( "github.com/mitchellh/cli" ) -func TestEvalMonitorCommand_Implements(t *testing.T) { - var _ cli.Command = &EvalMonitorCommand{} +func TestEvalStatusCommand_Implements(t *testing.T) { + var _ cli.Command = &EvalStatusCommand{} } -func TestEvalMonitorCommand_Fails(t *testing.T) { +func TestEvalStatusCommand_Fails(t *testing.T) { srv, _, url := testServer(t, nil) defer srv.Stop() ui := new(cli.MockUi) - cmd := &EvalMonitorCommand{Meta: Meta{Ui: ui}} + cmd := &EvalStatusCommand{Meta: Meta{Ui: ui}} // Fails on misuse if code := cmd.Run([]string{"some", "bad", "args"}); code != 1 { @@ -40,7 +40,7 @@ func TestEvalMonitorCommand_Fails(t *testing.T) { if code := cmd.Run([]string{"-address=nope", "12345678-abcd-efab-cdef-123456789abc"}); code != 1 { t.Fatalf("expected exit code 1, got: %d", code) } - if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error reading evaluation") { + if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error querying evaluation") { t.Fatalf("expected failed query error, got: %s", out) } } diff --git a/command/fs.go b/command/fs.go index 2c66b0f41..9d73e7310 100644 --- a/command/fs.go +++ b/command/fs.go @@ -28,6 +28,8 @@ General Options: ` + generalOptionsUsage() + ` +FS Specific Options: + -H Machine friendly output. diff --git a/command/plan.go b/command/plan.go index 677bfcfb8..7742d5a25 100644 --- a/command/plan.go +++ b/command/plan.go @@ -47,7 +47,7 @@ General Options: ` + generalOptionsUsage() + ` -Run Options: +Plan Options: -diff Defaults to true, but can be toggled off to omit diff output. diff --git a/command/server_members.go b/command/server_members.go index 40e375f3f..94f5aadfc 100644 --- a/command/server_members.go +++ b/command/server_members.go @@ -23,7 +23,7 @@ General Options: ` + generalOptionsUsage() + ` -Agent Members Options: +Server Members Options: -detailed Show detailed information about each member. This dumps diff --git a/commands.go b/commands.go index b402f429b..69d96273f 100644 --- a/commands.go +++ b/commands.go @@ -54,8 +54,8 @@ func Commands(metaPtr *command.Meta) map[string]cli.CommandFactory { Meta: meta, }, nil }, - "eval-monitor": func() (cli.Command, error) { - return &command.EvalMonitorCommand{ + "eval-status": func() (cli.Command, error) { + return &command.EvalStatusCommand{ Meta: meta, }, nil }, diff --git a/scheduler/generic_sched.go b/scheduler/generic_sched.go index bad43d21c..0a942cd13 100644 --- a/scheduler/generic_sched.go +++ b/scheduler/generic_sched.go @@ -29,6 +29,14 @@ const ( // allocInPlace is the status used when speculating on an in-place update allocInPlace = "alloc updating in-place" + + // blockedEvalMaxPlanDesc is the description used for blocked evals that are + // a result of hitting the max number of plan attempts + blockedEvalMaxPlanDesc = "created due to placement conflicts" + + // blockedEvalFailedPlacements is the description used for blocked evals + // that are a result of failing to place all allocations. + blockedEvalFailedPlacements = "created to place remaining allocations" ) // SetStatusError is used to set the status of the evaluation to the given error @@ -154,6 +162,9 @@ func (s *GenericScheduler) createBlockedEval(planFailure bool) error { s.blocked = s.eval.CreateBlockedEval(classEligibility, escaped) if planFailure { s.blocked.TriggeredBy = structs.EvalTriggerMaxPlans + s.blocked.StatusDescription = blockedEvalMaxPlanDesc + } else { + s.blocked.StatusDescription = blockedEvalFailedPlacements } return s.planner.CreateEval(s.blocked)