Add custom status description for rescheduling follow up evals, and make unit test robust

nomad/node_endpoint.go

@@ -27,10 +27,6 @@ const (
 	// maxParallelRequestsPerDerive  is the maximum number of parallel Vault
 	// create token requests that may be outstanding per derive request
 	maxParallelRequestsPerDerive = 16
-
-	// failedAllocsEvalStatusDesc is the description used for evaluations created
-	// when updating reschedulable allocs that have failed
-	failedAllocsEvalStatusDesc = "created to trigger rescheduling failed allocations"
 )
 
 // Node endpoint is used for client interactions
@@ -973,14 +969,13 @@ func (n *Node) UpdateAlloc(args *structs.AllocUpdateRequest, reply *structs.Gene
 				taskGroup := job.LookupTaskGroup(existingAlloc.TaskGroup)
 				if taskGroup != nil && existingAlloc.FollowupEvalID == "" && existingAlloc.RescheduleEligible(taskGroup.ReschedulePolicy, now) {
 					eval := &structs.Evaluation{
-						ID:                uuid.Generate(),
-						Namespace:         existingAlloc.Namespace,
-						TriggeredBy:       structs.EvalTriggerRetryFailedAlloc,
-						JobID:             existingAlloc.JobID,
-						Type:              job.Type,
-						Priority:          job.Priority,
-						Status:            structs.EvalStatusPending,
-						StatusDescription: failedAllocsEvalStatusDesc,
+						ID:          uuid.Generate(),
+						Namespace:   existingAlloc.Namespace,
+						TriggeredBy: structs.EvalTriggerRetryFailedAlloc,
+						JobID:       existingAlloc.JobID,
+						Type:        job.Type,
+						Priority:    job.Priority,
+						Status:      structs.EvalStatusPending,
 					}
 					evals = append(evals, eval)
 				}

nomad/node_endpoint_test.go

@@ -1954,7 +1954,13 @@ func TestClientEndpoint_GetAllocs_Blocking(t *testing.T) {
 
 func TestClientEndpoint_UpdateAlloc(t *testing.T) {
 	t.Parallel()
-	s1 := TestServer(t, nil)
+	s1 := TestServer(t, func(c *Config) {
+		// Disabling scheduling in this test so that we can
+		// ensure that the state store doesn't accumulate more evals
+		// than what we expect the unit test to add
+		c.NumSchedulers = 0
+	})
+
 	defer s1.Shutdown()
 	codec := rpcClient(t, s1)
 	testutil.WaitForLeader(t, s1.RPC)
@@ -2035,7 +2041,7 @@ func TestClientEndpoint_UpdateAlloc(t *testing.T) {
 	require.True(len(evaluations) != 0)
 	foundCount := 0
 	for _, resultEval := range evaluations {
-		if resultEval.TriggeredBy == structs.EvalTriggerRetryFailedAlloc && resultEval.WaitUntil.IsZero() && resultEval.StatusDescription == failedAllocsEvalStatusDesc {
+		if resultEval.TriggeredBy == structs.EvalTriggerRetryFailedAlloc && resultEval.WaitUntil.IsZero() {
 			foundCount++
 		}
 	}

scheduler/generic_sched.go

@@ -47,6 +47,10 @@ const (
 	// that are a result of failing to place all allocations.
 	blockedEvalFailedPlacements = "created to place remaining allocations"
 
+	// reschedulingFollowupEvalDesc is the description used when creating follow
+	// up evals for delayed rescheduling
+	reschedulingFollowupEvalDesc = "created for delayed rescheduling"
+
 	// maxPastRescheduleEvents is the maximum number of past reschedule event
 	// that we track when unlimited rescheduling is enabled
 	maxPastRescheduleEvents = 5

scheduler/reconcile.go

@@ -849,15 +849,16 @@ func (a *allocReconciler) handleDelayedReschedules(rescheduleLater []*delayedRes
 
 	// Create a new eval for the first batch
 	eval := &structs.Evaluation{
-		ID:             uuid.Generate(),
-		Namespace:      a.job.Namespace,
-		Priority:       a.job.Priority,
-		Type:           a.job.Type,
-		TriggeredBy:    structs.EvalTriggerRetryFailedAlloc,
-		JobID:          a.job.ID,
-		JobModifyIndex: a.job.ModifyIndex,
-		Status:         structs.EvalStatusPending,
-		WaitUntil:      nextReschedTime,
+		ID:                uuid.Generate(),
+		Namespace:         a.job.Namespace,
+		Priority:          a.job.Priority,
+		Type:              a.job.Type,
+		TriggeredBy:       structs.EvalTriggerRetryFailedAlloc,
+		JobID:             a.job.ID,
+		JobModifyIndex:    a.job.ModifyIndex,
+		Status:            structs.EvalStatusPending,
+		StatusDescription: reschedulingFollowupEvalDesc,
+		WaitUntil:         nextReschedTime,
 	}
 	evals = append(evals, eval)