Merge pull request #1255 from hashicorp/b-infinite-requeue

Only unblock if missed class was added after eval snapshot index

nomad/blocked_evals.go

@@ -207,9 +207,10 @@ func (b *BlockedEvals) missedUnblock(eval *structs.Evaluation) bool {
 		}
 
 		elig, ok := eval.ClassEligibility[class]
-		if !ok {
-			// The evaluation was processed and did not encounter this class.
-			// Thus for correctness we need to unblock it.
+		if !ok && eval.SnapshotIndex < index {
+			// The evaluation was processed and did not encounter this class
+			// because it was added after it was processed. Thus for correctness
+			// we need to unblock it.
 			return true
 		}
 

nomad/blocked_evals_test.go

@@ -291,8 +291,9 @@ func TestBlockedEvals_Block_ImmediateUnblock_Escaped(t *testing.T) {
 }
 
 // Test the block case in which the eval should be immediately unblocked since
-// it there is an unblock on an unseen class
-func TestBlockedEvals_Block_ImmediateUnblock_UnseenClass(t *testing.T) {
+// there is an unblock on an unseen class that occured while it was in the
+// scheduler
+func TestBlockedEvals_Block_ImmediateUnblock_UnseenClass_After(t *testing.T) {
 	blocked, broker := testBlockedEvals(t)
 
 	// Do an unblock prior to blocking
@@ -325,6 +326,30 @@ func TestBlockedEvals_Block_ImmediateUnblock_UnseenClass(t *testing.T) {
 	})
 }
 
+// Test the block case in which the eval should not immediately unblock since
+// there is an unblock on an unseen class that occured before it was in the
+// scheduler
+func TestBlockedEvals_Block_ImmediateUnblock_UnseenClass_Before(t *testing.T) {
+	blocked, _ := testBlockedEvals(t)
+
+	// Do an unblock prior to blocking
+	blocked.Unblock("v1:123", 500)
+
+	// Create a blocked eval that is eligible on a specific node class and add
+	// it to the blocked tracker.
+	e := mock.Eval()
+	e.Status = structs.EvalStatusBlocked
+	e.EscapedComputedClass = false
+	e.SnapshotIndex = 900
+	blocked.Block(e)
+
+	// Verify block caused the eval to be immediately unblocked
+	blockedStats := blocked.Stats()
+	if blockedStats.TotalBlocked != 1 && blockedStats.TotalEscaped != 0 {
+		t.Fatalf("bad: %#v", blockedStats)
+	}
+}
+
 // Test the block case in which the eval should be immediately unblocked since
 // it a class it is eligible for has been unblocked
 func TestBlockedEvals_Block_ImmediateUnblock_SeenClass(t *testing.T) {

scheduler/generic_sched.go

@@ -139,7 +139,11 @@ func (s *GenericScheduler) Process(eval *structs.Evaluation) error {
 	// If the current evaluation is a blocked evaluation and we didn't place
 	// everything, do not update the status to complete.
 	if s.eval.Status == structs.EvalStatusBlocked && len(s.failedTGAllocs) != 0 {
-		return s.planner.ReblockEval(s.eval)
+		e := s.ctx.Eligibility()
+		newEval := s.eval.Copy()
+		newEval.EscapedComputedClass = e.HasEscaped()
+		newEval.ClassEligibility = e.GetClasses()
+		return s.planner.ReblockEval(newEval)
 	}
 
 	// Update the status to complete