scheduler: add reconciler annotations to completed evals (#26188)

The output of the reconciler stage of scheduling is only visible via debug-level
logs, typically accessible only to the cluster admin. We can give job authors
better ability to understand what's happening to their jobs if we expose this
information to them in the `eval status` command.

Add the reconciler's desired updates to the evaluation struct so it can be
exposed in the API. This increases the size of evals by roughly 15% in the state
store, or a bit more when there are preemptions (but we expect this will be a
small minority of evals).

Ref: https://hashicorp.atlassian.net/browse/NMD-818
Fixes: https://github.com/hashicorp/nomad/issues/15564

command/eval_status.go

@@ -12,6 +12,7 @@ import (
 	"github.com/hashicorp/nomad/api"
 	"github.com/hashicorp/nomad/api/contexts"
 	"github.com/posener/complete"
+	"github.com/ryanuber/columnize"
 )
 
 type EvalStatusCommand struct {
@@ -257,6 +258,36 @@ func (c *EvalStatusCommand) formatEvalStatus(eval *api.Evaluation, placedAllocs
 		c.Ui.Output(c.Colorize().Color("\n[bold]Related Evaluations[reset]"))
 		c.Ui.Output(formatRelatedEvalStubs(eval.RelatedEvals, length))
 	}
+	if eval.PlanAnnotations != nil {
+
+		if len(eval.PlanAnnotations.DesiredTGUpdates) > 0 {
+			c.Ui.Output(c.Colorize().Color("\n[bold]Reconciler Annotations[reset]"))
+			annotations := make([]string, len(eval.PlanAnnotations.DesiredTGUpdates)+1)
+			annotations[0] = "Task Group|Ignore|Place|Stop|Migrate|InPlace|Destructive|Canary|Preemptions"
+			i := 1
+			for tg, updates := range eval.PlanAnnotations.DesiredTGUpdates {
+				annotations[i] = fmt.Sprintf("%s|%d|%d|%d|%d|%d|%d|%d|%d",
+					tg,
+					updates.Ignore,
+					updates.Place,
+					updates.Stop,
+					updates.Migrate,
+					updates.InPlaceUpdate,
+					updates.DestructiveUpdate,
+					updates.Canary,
+					updates.Preemptions,
+				)
+				i++
+			}
+			c.Ui.Output(columnize.SimpleFormat(annotations))
+		}
+
+		if len(eval.PlanAnnotations.PreemptedAllocs) > 0 {
+			c.Ui.Output(c.Colorize().Color("\n[bold]Preempted Allocations[reset]"))
+			allocsOut := formatPreemptedAllocListStubs(eval.PlanAnnotations.PreemptedAllocs, length)
+			c.Ui.Output(allocsOut)
+		}
+	}
 	if len(placedAllocs) > 0 {
 		c.Ui.Output(c.Colorize().Color("\n[bold]Placed Allocations[reset]"))
 		allocsOut := formatAllocListStubs(placedAllocs, false, length)
@@ -323,3 +354,27 @@ func formatRelatedEvalStubs(evals []*api.EvaluationStub, length int) string {
 
 	return formatList(out)
 }
+
+// formatPreemptedAllocListStubs formats alloc stubs but assumes they don't all
+// belong to the same job, as is the case when allocs are preempted by another
+// job
+func formatPreemptedAllocListStubs(stubs []*api.AllocationListStub, uuidLength int) string {
+	allocs := make([]string, len(stubs)+1)
+	allocs[0] = "ID|Job ID|Node ID|Task Group|Version|Desired|Status|Created|Modified"
+	for i, alloc := range stubs {
+		now := time.Now()
+		createTimePretty := prettyTimeDiff(time.Unix(0, alloc.CreateTime), now)
+		modTimePretty := prettyTimeDiff(time.Unix(0, alloc.ModifyTime), now)
+		allocs[i+1] = fmt.Sprintf("%s|%s|%s|%s|%d|%s|%s|%s|%s",
+			limit(alloc.ID, uuidLength),
+			alloc.JobID,
+			limit(alloc.NodeID, uuidLength),
+			alloc.TaskGroup,
+			alloc.JobVersion,
+			alloc.DesiredStatus,
+			alloc.ClientStatus,
+			createTimePretty,
+			modTimePretty)
+	}
+	return formatList(allocs)
+}

command/eval_status_test.go

@@ -150,6 +150,22 @@ func TestEvalStatusCommand_Format(t *testing.T) {
 			CoalescedFailures: 0,
 			ScoreMetaData:     []*api.NodeScoreMeta{},
 		}},
+		PlanAnnotations: &api.PlanAnnotations{
+			DesiredTGUpdates: map[string]*api.DesiredUpdates{"web": {Place: 10}},
+			PreemptedAllocs: []*api.AllocationListStub{
+				{
+					ID:            uuid.Generate(),
+					JobID:         "another",
+					NodeID:        uuid.Generate(),
+					TaskGroup:     "db",
+					DesiredStatus: "evict",
+					JobVersion:    3,
+					ClientStatus:  "complete",
+					CreateTime:    now.Add(-10 * time.Minute).UnixNano(),
+					ModifyTime:    now.Add(-2 * time.Second).UnixNano(),
+				},
+			},
+		},
 		ClassEligibility:     map[string]bool{},
 		EscapedComputedClass: true,
 		QuotaLimitReached:    "",
@@ -207,4 +223,6 @@ Task Group "web" (failed to place 1 allocation):
 
 	must.StrContains(t, out, `Related Evaluations`)
 	must.StrContains(t, out, `Placed Allocations`)
+	must.StrContains(t, out, `Reconciler Annotations`)
+	must.StrContains(t, out, `Preempted Allocations`)
 }