scheduler: add disconnect and reschedule info to reconciler output (#26255)

The `DesiredUpdates` struct that we send to the Read Eval API doesn't include
information about disconnect/reconnect and rescheduling. Annotate the
`DesiredUpdates` with this data, and adjust the `eval status` command to display
only those fields that have non-zero values in order to make the output width
manageable.

Ref: https://hashicorp.atlassian.net/browse/NMD-815

command/eval_status.go

@@ -6,6 +6,7 @@ package command
 import (
 	"fmt"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -37,7 +38,7 @@ Eval Status Options:
     Monitor an outstanding evaluation
 
   -verbose
-    Show full-length IDs and exact timestamps.
+    Show full-length IDs, exact timestamps, and all plan annotation fields.
 
   -json
     Output the evaluation in its JSON format. This format will not include
@@ -261,25 +262,9 @@ func (c *EvalStatusCommand) formatEvalStatus(eval *api.Evaluation, placedAllocs
 	if eval.PlanAnnotations != nil {
 
 		if len(eval.PlanAnnotations.DesiredTGUpdates) > 0 {
-			c.Ui.Output(c.Colorize().Color("\n[bold]Reconciler Annotations[reset]"))
-			annotations := make([]string, len(eval.PlanAnnotations.DesiredTGUpdates)+1)
-			annotations[0] = "Task Group|Ignore|Place|Stop|Migrate|InPlace|Destructive|Canary|Preemptions"
-			i := 1
-			for tg, updates := range eval.PlanAnnotations.DesiredTGUpdates {
-				annotations[i] = fmt.Sprintf("%s|%d|%d|%d|%d|%d|%d|%d|%d",
-					tg,
-					updates.Ignore,
-					updates.Place,
-					updates.Stop,
-					updates.Migrate,
-					updates.InPlaceUpdate,
-					updates.DestructiveUpdate,
-					updates.Canary,
-					updates.Preemptions,
-				)
-				i++
-			}
-			c.Ui.Output(columnize.SimpleFormat(annotations))
+			c.Ui.Output(c.Colorize().Color("\n[bold]Plan Annotations[reset]"))
+			c.Ui.Output(formatPlanAnnotations(
+				eval.PlanAnnotations.DesiredTGUpdates, verbose))
 		}
 
 		if len(eval.PlanAnnotations.PreemptedAllocs) > 0 {
@@ -378,3 +363,62 @@ func formatPreemptedAllocListStubs(stubs []*api.AllocationListStub, uuidLength i
 	}
 	return formatList(allocs)
 }
+
+// formatPlanAnnotations produces a table with one row per task group where the
+// columns are all the changes (ignore, place, stop, etc.) plus all the non-zero
+// causes of those changes (migrate, canary, reschedule, etc)
+func formatPlanAnnotations(desiredTGUpdates map[string]*api.DesiredUpdates, verbose bool) string {
+	annotations := make([]string, len(desiredTGUpdates)+1)
+
+	annotations[0] = "Task Group|Ignore|Place|Stop|InPlace|Destructive"
+	optCols := []string{
+		"Migrate", "Canary", "Preemptions",
+		"Reschedule Now", "Reschedule Later", "Disconnect", "Reconnect"}
+
+	byCol := make([][]uint64, len(optCols))
+	for i := range byCol {
+		for j := range len(desiredTGUpdates) + 1 {
+			byCol[i] = make([]uint64, j+1)
+		}
+	}
+
+	i := 1
+	for tg, updates := range desiredTGUpdates {
+		// we always show the first 5 columns
+		annotations[i] = fmt.Sprintf("%s|%d|%d|%d|%d|%d",
+			tg,
+			updates.Ignore,
+			updates.Place,
+			updates.Stop,
+			updates.InPlaceUpdate,
+			updates.DestructiveUpdate,
+		)
+
+		// we record how many we have of the other columns so we can show them
+		// only if populated
+		byCol[0][i] = updates.Migrate
+		byCol[1][i] = updates.Canary
+		byCol[2][i] = updates.Preemptions
+		byCol[3][i] = updates.RescheduleNow
+		byCol[4][i] = updates.RescheduleLater
+		byCol[5][i] = updates.Disconnect
+		byCol[6][i] = updates.Reconnect
+		i++
+	}
+
+	// the remaining columns only show if they're populated or if we're in
+	// verbose mode
+	for i, col := range optCols {
+		for tgIdx := range len(desiredTGUpdates) + 1 {
+			byCol[i][0] += byCol[i][tgIdx]
+		}
+		if verbose || byCol[i][0] > 0 {
+			annotations[0] += "|" + col
+			for tgIdx := 1; tgIdx < len(desiredTGUpdates)+1; tgIdx++ {
+				annotations[tgIdx] += "|" + strconv.FormatUint(byCol[i][tgIdx], 10)
+			}
+		}
+	}
+
+	return columnize.SimpleFormat(annotations)
+}

command/eval_status_test.go

@@ -223,6 +223,19 @@ Task Group "web" (failed to place 1 allocation):
 
 	must.StrContains(t, out, `Related Evaluations`)
 	must.StrContains(t, out, `Placed Allocations`)
-	must.StrContains(t, out, `Reconciler Annotations`)
+	must.StrContains(t, out, `Plan Annotations`)
 	must.StrContains(t, out, `Preempted Allocations`)
 }
+
+func TestEvalStatus_FormatPlanAnnotations(t *testing.T) {
+
+	updates := map[string]*api.DesiredUpdates{
+		"foo": {Place: 1, Ignore: 2, Canary: 1},
+		"bar": {Place: 1, Stop: 3, Reconnect: 2},
+	}
+
+	out := formatPlanAnnotations(updates, false)
+	must.Eq(t, `Task Group  Ignore  Place  Stop  InPlace  Destructive  Canary  Reconnect
+foo         2       1      0     0        0            1       0
+bar         0       1      3     0        0            0       2`, out)
+}