diff --git a/scheduler/generic_sched.go b/scheduler/generic_sched.go index f368b731c..b514d6410 100644 --- a/scheduler/generic_sched.go +++ b/scheduler/generic_sched.go @@ -438,9 +438,9 @@ func (s *GenericScheduler) computePlacements(destructive, place []placementResul if prevAllocation != nil { var penaltyNodes []string penaltyNodes = append(penaltyNodes, prevAllocation.NodeID) - if prevAllocation.RescheduleTrackers != nil { - for _, reschedTracker := range prevAllocation.RescheduleTrackers { - penaltyNodes = append(penaltyNodes, reschedTracker.PrevNodeID) + if prevAllocation.RescheduleTracker != nil { + for _, reschedEvent := range prevAllocation.RescheduleTracker.Events { + penaltyNodes = append(penaltyNodes, reschedEvent.PrevNodeID) } } selectOptions.PenaltyNodeIDs = penaltyNodes @@ -482,14 +482,14 @@ func (s *GenericScheduler) computePlacements(destructive, place []placementResul // set the record the older allocation id so that they are chained if prev := prevAllocation; prev != nil { alloc.PreviousAllocation = prev.ID - var rescheduleTrackers []*structs.RescheduleTracker - if prev.RescheduleTrackers != nil { - for _, reschedInfo := range prev.RescheduleTrackers { - rescheduleTrackers = append(rescheduleTrackers, reschedInfo.Copy()) + var rescheduleEvents []*structs.RescheduleEvent + if prev.RescheduleTracker != nil { + for _, reschedEvent := range prev.RescheduleTracker.Events { + rescheduleEvents = append(rescheduleEvents, reschedEvent.Copy()) } } - rescheduleTrackers = append(rescheduleTrackers, &structs.RescheduleTracker{RescheduleTime: time.Now().UTC().UnixNano(), PrevAllocID: prev.ID, PrevNodeID: alloc.NodeID}) - alloc.RescheduleTrackers = rescheduleTrackers + rescheduleEvents = append(rescheduleEvents, &structs.RescheduleEvent{RescheduleTime: time.Now().UTC().UnixNano(), PrevAllocID: prev.ID, PrevNodeID: alloc.NodeID}) + alloc.RescheduleTracker = &structs.RescheduleTracker{Events: rescheduleEvents} } // If we are placing a canary and we found a match, add the canary diff --git a/scheduler/generic_sched_test.go b/scheduler/generic_sched_test.go index 00f40f63c..6cf9bd595 100644 --- a/scheduler/generic_sched_test.go +++ b/scheduler/generic_sched_test.go @@ -2788,8 +2788,8 @@ func TestServiceSched_Reschedule_Once(t *testing.T) { } } assert.Equal(failedAllocID, newAlloc.PreviousAllocation) - assert.Equal(1, len(newAlloc.RescheduleTrackers)) - assert.Equal(failedAllocID, newAlloc.RescheduleTrackers[0].PrevAllocID) + assert.Equal(1, len(newAlloc.RescheduleTracker.Events)) + assert.Equal(failedAllocID, newAlloc.RescheduleTracker.Events[0].PrevAllocID) // Mark this alloc as failed again, should not get rescheduled newAlloc.ClientStatus = structs.AllocClientStatusFailed @@ -2890,14 +2890,13 @@ func TestServiceSched_Reschedule_Multiple(t *testing.T) { var pendingAllocs []*structs.Allocation fmt.Println("Iteration: ", i) for _, alloc := range out { - fmt.Println(alloc.ID, alloc.ClientStatus, len(alloc.RescheduleTrackers), alloc.PreviousAllocation) if alloc.ClientStatus == structs.AllocClientStatusPending { pendingAllocs = append(pendingAllocs, alloc) } } assert.Equal(1, len(pendingAllocs)) newAlloc := pendingAllocs[0] - assert.Equal(expectedNumReschedTrackers, len(newAlloc.RescheduleTrackers)) + assert.Equal(expectedNumReschedTrackers, len(newAlloc.RescheduleTracker.Events)) // Mark this alloc as failed again newAlloc.ClientStatus = structs.AllocClientStatusFailed diff --git a/scheduler/reconcile_util.go b/scheduler/reconcile_util.go index 6cae14aa4..bdb9b473c 100644 --- a/scheduler/reconcile_util.go +++ b/scheduler/reconcile_util.go @@ -5,6 +5,8 @@ import ( "sort" "strings" + "time" + "github.com/hashicorp/nomad/nomad/structs" ) @@ -227,6 +229,7 @@ func (a allocSet) filterByRescheduleable(isBatch bool, reschedulePolicy *structs rescheduledPrevAllocs := make(map[string]struct{}) // Track previous allocs from any restart trackers + now := time.Now() for _, alloc := range a { if isBatch { // Allocs from batch jobs should be filtered when the desired status @@ -241,7 +244,7 @@ func (a allocSet) filterByRescheduleable(isBatch bool, reschedulePolicy *structs continue default: } - if alloc.ShouldReschedule(reschedulePolicy) { + if alloc.ShouldReschedule(reschedulePolicy, now) { reschedule[alloc.ID] = alloc } else { untainted[alloc.ID] = alloc @@ -249,7 +252,7 @@ func (a allocSet) filterByRescheduleable(isBatch bool, reschedulePolicy *structs } else { // ignore allocs whose desired state is stop/evict // everything else is either rescheduleable or untainted - if alloc.ShouldReschedule(reschedulePolicy) { + if alloc.ShouldReschedule(reschedulePolicy, now) { reschedule[alloc.ID] = alloc } else if alloc.DesiredStatus != structs.AllocDesiredStatusStop && alloc.DesiredStatus != structs.AllocDesiredStatusEvict { untainted[alloc.ID] = alloc @@ -257,11 +260,12 @@ func (a allocSet) filterByRescheduleable(isBatch bool, reschedulePolicy *structs } } - // Find allocs that exist in restart trackers from other allocs + // Find allocs that exist in reschedule events from other allocs + // This needs another pass through allocs we marked as reschedulable for _, alloc := range reschedule { - if alloc.RescheduleTrackers != nil { - for _, reschedTrack := range alloc.RescheduleTrackers { - rescheduledPrevAllocs[reschedTrack.PrevAllocID] = struct{}{} + if alloc.RescheduleTracker != nil { + for _, rescheduleEvent := range alloc.RescheduleTracker.Events { + rescheduledPrevAllocs[rescheduleEvent.PrevAllocID] = struct{}{} } } }