diff --git a/nomad/fsm_test.go b/nomad/fsm_test.go index 0c3a846a8..06177a083 100644 --- a/nomad/fsm_test.go +++ b/nomad/fsm_test.go @@ -56,7 +56,7 @@ func testFSM(t *testing.T) *nomadFSM { Logger: logger, Region: "global", } - fsm, err := NewFSM(fsmConfig) + fsm, err := NewFSM(fsmConfig, TestServer(t, nil)) if err != nil { t.Fatalf("err: %v", err) } diff --git a/nomad/plan_apply_test.go b/nomad/plan_apply_test.go index 4dfa7a43b..619b7b907 100644 --- a/nomad/plan_apply_test.go +++ b/nomad/plan_apply_test.go @@ -3,6 +3,7 @@ package nomad import ( "reflect" "testing" + "time" memdb "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper/testlog" @@ -62,6 +63,7 @@ func testRegisterJob(t *testing.T, s *Server, j *structs.Job) { } } +// Deprecated: Tests the older unoptimized code path for applyPlan func TestPlanApply_applyPlan(t *testing.T) { t.Parallel() s1 := TestServer(t, nil) @@ -228,6 +230,154 @@ func TestPlanApply_applyPlan(t *testing.T) { assert.Equal(index, evalOut.ModifyIndex) } +func TestPlanApply_applyPlanWithNormalizedAllocs(t *testing.T) { + t.Parallel() + s1 := TestServer(t, func(c *Config) { + c.Build = "0.9.1" + }) + defer s1.Shutdown() + testutil.WaitForLeader(t, s1.RPC) + + // Register node + node := mock.Node() + testRegisterNode(t, s1, node) + + // Register a fake deployment + oldDeployment := mock.Deployment() + if err := s1.State().UpsertDeployment(900, oldDeployment); err != nil { + t.Fatalf("UpsertDeployment failed: %v", err) + } + + // Create a deployment + dnew := mock.Deployment() + + // Create a deployment update for the old deployment id + desiredStatus, desiredStatusDescription := "foo", "bar" + updates := []*structs.DeploymentStatusUpdate{ + { + DeploymentID: oldDeployment.ID, + Status: desiredStatus, + StatusDescription: desiredStatusDescription, + }, + } + + // Register allocs, deployment and deployment update + alloc := mock.Alloc() + stoppedAlloc := mock.Alloc() + stoppedAllocDiff := &structs.Allocation{ + ID: stoppedAlloc.ID, + DesiredDescription: "Desired Description", + ClientStatus: structs.AllocClientStatusLost, + } + preemptedAlloc := mock.Alloc() + preemptedAllocDiff := &structs.Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: alloc.ID, + } + s1.State().UpsertJobSummary(1000, mock.JobSummary(alloc.JobID)) + s1.State().UpsertAllocs(1100, []*structs.Allocation{stoppedAlloc, preemptedAlloc}) + // Create an eval + eval := mock.Eval() + eval.JobID = alloc.JobID + if err := s1.State().UpsertEvals(1, []*structs.Evaluation{eval}); err != nil { + t.Fatalf("err: %v", err) + } + + timestampBeforeCommit := time.Now().UTC().UnixNano() + planRes := &structs.PlanResult{ + NodeAllocation: map[string][]*structs.Allocation{ + node.ID: {alloc}, + }, + NodeUpdate: map[string][]*structs.Allocation{ + stoppedAlloc.NodeID: {stoppedAllocDiff}, + }, + NodePreemptions: map[string][]*structs.Allocation{ + preemptedAlloc.NodeID: {preemptedAllocDiff}, + }, + Deployment: dnew, + DeploymentUpdates: updates, + } + + // Snapshot the state + snap, err := s1.State().Snapshot() + if err != nil { + t.Fatalf("err: %v", err) + } + + // Create the plan with a deployment + plan := &structs.Plan{ + Job: alloc.Job, + Deployment: dnew, + DeploymentUpdates: updates, + EvalID: eval.ID, + } + + // Apply the plan + future, err := s1.applyPlan(plan, planRes, snap) + assert := assert.New(t) + assert.Nil(err) + + // Verify our optimistic snapshot is updated + ws := memdb.NewWatchSet() + allocOut, err := snap.AllocByID(ws, alloc.ID) + assert.Nil(err) + assert.NotNil(allocOut) + + deploymentOut, err := snap.DeploymentByID(ws, plan.Deployment.ID) + assert.Nil(err) + assert.NotNil(deploymentOut) + + // Check plan does apply cleanly + index, err := planWaitFuture(future) + assert.Nil(err) + assert.NotEqual(0, index) + + // Lookup the allocation + fsmState := s1.fsm.State() + allocOut, err = fsmState.AllocByID(ws, alloc.ID) + assert.Nil(err) + assert.NotNil(allocOut) + assert.True(allocOut.CreateTime > 0) + assert.True(allocOut.ModifyTime > 0) + assert.Equal(allocOut.CreateTime, allocOut.ModifyTime) + + // Verify stopped alloc diff applied cleanly + updatedStoppedAlloc, err := fsmState.AllocByID(ws, stoppedAlloc.ID) + assert.Nil(err) + assert.NotNil(updatedStoppedAlloc) + assert.True(updatedStoppedAlloc.ModifyTime > timestampBeforeCommit) + assert.Equal(updatedStoppedAlloc.DesiredDescription, stoppedAllocDiff.DesiredDescription) + assert.Equal(updatedStoppedAlloc.ClientStatus, stoppedAllocDiff.ClientStatus) + assert.Equal(updatedStoppedAlloc.DesiredStatus, structs.AllocDesiredStatusStop) + + // Verify preempted alloc diff applied cleanly + updatedPreemptedAlloc, err := fsmState.AllocByID(ws, preemptedAlloc.ID) + assert.Nil(err) + assert.NotNil(updatedPreemptedAlloc) + assert.True(updatedPreemptedAlloc.ModifyTime > timestampBeforeCommit) + assert.Equal(updatedPreemptedAlloc.DesiredDescription, + "Preempted by alloc ID " + preemptedAllocDiff.PreemptedByAllocation) + assert.Equal(updatedPreemptedAlloc.DesiredStatus, structs.AllocDesiredStatusEvict) + + // Lookup the new deployment + dout, err := fsmState.DeploymentByID(ws, plan.Deployment.ID) + assert.Nil(err) + assert.NotNil(dout) + + // Lookup the updated deployment + dout2, err := fsmState.DeploymentByID(ws, oldDeployment.ID) + assert.Nil(err) + assert.NotNil(dout2) + assert.Equal(desiredStatus, dout2.Status) + assert.Equal(desiredStatusDescription, dout2.StatusDescription) + + // Lookup updated eval + evalOut, err := fsmState.EvalByID(ws, eval.ID) + assert.Nil(err) + assert.NotNil(evalOut) + assert.Equal(index, evalOut.ModifyIndex) +} + func TestPlanApply_EvalPlan_Simple(t *testing.T) { t.Parallel() state := testStateStore(t) diff --git a/nomad/state/state_store_test.go b/nomad/state/state_store_test.go index 3cf3b9776..af248a557 100644 --- a/nomad/state/state_store_test.go +++ b/nomad/state/state_store_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" @@ -88,6 +88,7 @@ func TestStateStore_Blocking_MinQuery(t *testing.T) { } } +// COMPAT 0.11: Uses AllocUpdateRequest.Alloc // This test checks that: // 1) The job is denormalized // 2) Allocations are created @@ -140,6 +141,94 @@ func TestStateStore_UpsertPlanResults_AllocationsCreated_Denormalized(t *testing assert.EqualValues(1000, evalOut.ModifyIndex) } +// This test checks that: +// 1) The job is denormalized +// 2) Allocations are denormalized and updated with the diff +func TestStateStore_UpsertPlanResults_AllocationsDenormalized(t *testing.T) { + state := testStateStore(t) + alloc := mock.Alloc() + job := alloc.Job + alloc.Job = nil + + stoppedAlloc := mock.Alloc() + stoppedAlloc.Job = job + stoppedAllocDiff := &structs.Allocation{ + ID: stoppedAlloc.ID, + DesiredDescription: "desired desc", + ClientStatus: structs.AllocClientStatusLost, + } + preemptedAlloc := mock.Alloc() + preemptedAlloc.Job = job + preemptedAllocDiff := &structs.Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: alloc.ID, + } + + if err := state.UpsertAllocs(900, []*structs.Allocation{stoppedAlloc, preemptedAlloc}); err != nil { + t.Fatalf("err: %v", err) + } + + if err := state.UpsertJob(999, job); err != nil { + t.Fatalf("err: %v", err) + } + + eval := mock.Eval() + eval.JobID = job.ID + + // Create an eval + if err := state.UpsertEvals(1, []*structs.Evaluation{eval}); err != nil { + t.Fatalf("err: %v", err) + } + + // Create a plan result + res := structs.ApplyPlanResultsRequest{ + AllocUpdateRequest: structs.AllocUpdateRequest{ + AllocsUpdated: []*structs.Allocation{alloc}, + AllocsStopped: []*structs.Allocation{stoppedAllocDiff}, + Job: job, + }, + EvalID: eval.ID, + NodePreemptions: []*structs.Allocation{preemptedAllocDiff}, + } + assert := assert.New(t) + planModifyIndex := uint64(1000) + err := state.UpsertPlanResults(planModifyIndex, &res) + assert.Nil(err) + + ws := memdb.NewWatchSet() + out, err := state.AllocByID(ws, alloc.ID) + assert.Nil(err) + assert.Equal(alloc, out) + + updatedStoppedAlloc, err := state.AllocByID(ws, stoppedAlloc.ID) + assert.Nil(err) + assert.Equal(stoppedAllocDiff.DesiredDescription, updatedStoppedAlloc.DesiredDescription) + assert.Equal(structs.AllocDesiredStatusStop, updatedStoppedAlloc.DesiredStatus) + assert.Equal(stoppedAllocDiff.ClientStatus, updatedStoppedAlloc.ClientStatus) + assert.Equal(planModifyIndex, updatedStoppedAlloc.AllocModifyIndex) + assert.Equal(planModifyIndex, updatedStoppedAlloc.AllocModifyIndex) + + updatedPreemptedAlloc, err := state.AllocByID(ws, preemptedAlloc.ID) + assert.Nil(err) + assert.Equal(structs.AllocDesiredStatusEvict, updatedPreemptedAlloc.DesiredStatus) + assert.Equal(preemptedAllocDiff.PreemptedByAllocation, updatedPreemptedAlloc.PreemptedByAllocation) + assert.Equal(planModifyIndex, updatedPreemptedAlloc.AllocModifyIndex) + assert.Equal(planModifyIndex, updatedPreemptedAlloc.AllocModifyIndex) + + index, err := state.Index("allocs") + assert.Nil(err) + assert.EqualValues(planModifyIndex, index) + + if watchFired(ws) { + t.Fatalf("bad") + } + + evalOut, err := state.EvalByID(ws, eval.ID) + assert.Nil(err) + assert.NotNil(evalOut) + assert.EqualValues(planModifyIndex, evalOut.ModifyIndex) +} + // This test checks that the deployment is created and allocations count towards // the deployment func TestStateStore_UpsertPlanResults_Deployment(t *testing.T) { @@ -271,11 +360,9 @@ func TestStateStore_UpsertPlanResults_PreemptedAllocs(t *testing.T) { require.NoError(err) minimalPreemptedAlloc := &structs.Allocation{ - ID: preemptedAlloc.ID, - Namespace: preemptedAlloc.Namespace, - DesiredStatus: structs.AllocDesiredStatusEvict, - ModifyTime: time.Now().Unix(), - DesiredDescription: fmt.Sprintf("Preempted by allocation %v", alloc.ID), + ID: preemptedAlloc.ID, + PreemptedByAllocation: alloc.ID, + ModifyTime: time.Now().Unix(), } // Create eval for preempted job @@ -316,7 +403,7 @@ func TestStateStore_UpsertPlanResults_PreemptedAllocs(t *testing.T) { preempted, err := state.AllocByID(ws, preemptedAlloc.ID) require.NoError(err) require.Equal(preempted.DesiredStatus, structs.AllocDesiredStatusEvict) - require.Equal(preempted.DesiredDescription, fmt.Sprintf("Preempted by allocation %v", alloc.ID)) + require.Equal(preempted.DesiredDescription, fmt.Sprintf("Preempted by alloc ID %v", alloc.ID)) // Verify eval for preempted job preemptedJobEval, err := state.EvalByID(ws, eval2.ID) diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index 90b33b198..2cd3922ae 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/hashicorp/consul/api" - multierror "github.com/hashicorp/go-multierror" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/helper/uuid" "github.com/kr/pretty" "github.com/stretchr/testify/assert" @@ -2842,6 +2842,151 @@ func TestTaskArtifact_Validate_Checksum(t *testing.T) { } } +func TestPlan_NormalizeAllocationsWhenNormalizeAllocsIsTrue(t *testing.T) { + t.Parallel() + plan := &Plan{ + NodeUpdate: make(map[string][]*Allocation), + NodePreemptions: make(map[string][]*Allocation), + } + plan.NormalizeAllocs = true + stoppedAlloc := MockAlloc() + desiredDesc := "Desired desc" + plan.AppendStoppedAlloc(stoppedAlloc, desiredDesc, AllocClientStatusLost) + preemptedAlloc := MockAlloc() + preemptingAllocID := uuid.Generate() + plan.AppendPreemptedAlloc(preemptedAlloc, preemptingAllocID) + + plan.NormalizeAllocations() + + actualStoppedAlloc := plan.NodeUpdate[stoppedAlloc.NodeID][0] + expectedStoppedAlloc := &Allocation{ + ID: stoppedAlloc.ID, + DesiredDescription: desiredDesc, + ClientStatus: AllocClientStatusLost, + } + assert.Equal(t, expectedStoppedAlloc, actualStoppedAlloc) + actualPreemptedAlloc := plan.NodePreemptions[preemptedAlloc.NodeID][0] + expectedPreemptedAlloc := &Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: preemptingAllocID, + } + assert.Equal(t, expectedPreemptedAlloc, actualPreemptedAlloc) +} + +func TestPlan_NormalizeAllocationsWhenNormalizeAllocsIsFalse(t *testing.T) { + t.Parallel() + plan := &Plan{ + NodeUpdate: make(map[string][]*Allocation), + NodePreemptions: make(map[string][]*Allocation), + } + plan.NormalizeAllocs = false + stoppedAlloc := MockAlloc() + desiredDesc := "Desired desc" + plan.AppendStoppedAlloc(stoppedAlloc, desiredDesc, AllocClientStatusLost) + preemptedAlloc := MockAlloc() + preemptingAllocID := uuid.Generate() + plan.AppendPreemptedAlloc(preemptedAlloc, preemptingAllocID) + + plan.NormalizeAllocations() + + actualStoppedAlloc := plan.NodeUpdate[stoppedAlloc.NodeID][0] + expectedStoppedAlloc := new(Allocation) + *expectedStoppedAlloc = *stoppedAlloc + expectedStoppedAlloc.DesiredDescription = desiredDesc + expectedStoppedAlloc.DesiredStatus = AllocDesiredStatusStop + expectedStoppedAlloc.ClientStatus = AllocClientStatusLost + expectedStoppedAlloc.Job = nil + assert.Equal(t, expectedStoppedAlloc, actualStoppedAlloc) + actualPreemptedAlloc := plan.NodePreemptions[preemptedAlloc.NodeID][0] + expectedPreemptedAlloc := &Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: preemptingAllocID, + JobID: preemptedAlloc.JobID, + Namespace: preemptedAlloc.Namespace, + DesiredStatus: AllocDesiredStatusEvict, + DesiredDescription: fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocID), + AllocatedResources: preemptedAlloc.AllocatedResources, + TaskResources: preemptedAlloc.TaskResources, + SharedResources: preemptedAlloc.SharedResources, + } + assert.Equal(t, expectedPreemptedAlloc, actualPreemptedAlloc) +} + +func TestPlan_AppendStoppedAllocAppendsAllocWithUpdatedAttrs(t *testing.T) { + t.Parallel() + plan := &Plan{ + NodeUpdate: make(map[string][]*Allocation), + } + alloc := MockAlloc() + desiredDesc := "Desired desc" + + plan.AppendStoppedAlloc(alloc, desiredDesc, AllocClientStatusLost) + + appendedAlloc := plan.NodeUpdate[alloc.NodeID][0] + expectedAlloc := new(Allocation) + *expectedAlloc = *alloc + expectedAlloc.DesiredDescription = desiredDesc + expectedAlloc.DesiredStatus = AllocDesiredStatusStop + expectedAlloc.ClientStatus = AllocClientStatusLost + expectedAlloc.Job = nil + assert.Equal(t, expectedAlloc, appendedAlloc) + assert.Equal(t, alloc.Job, plan.Job) +} + +func TestPlan_AppendPreemptedAllocAppendsAllocWithUpdatedAttrs(t *testing.T) { + t.Parallel() + plan := &Plan{ + NodePreemptions: make(map[string][]*Allocation), + } + alloc := MockAlloc() + preemptingAllocID := uuid.Generate() + + plan.AppendPreemptedAlloc(alloc, preemptingAllocID) + + appendedAlloc := plan.NodePreemptions[alloc.NodeID][0] + expectedAlloc := &Allocation{ + ID: alloc.ID, + PreemptedByAllocation: preemptingAllocID, + JobID: alloc.JobID, + Namespace: alloc.Namespace, + DesiredStatus: AllocDesiredStatusEvict, + DesiredDescription: fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocID), + AllocatedResources: alloc.AllocatedResources, + TaskResources: alloc.TaskResources, + SharedResources: alloc.SharedResources, + } + assert.Equal(t, expectedAlloc, appendedAlloc) +} + +func TestPlan_MsgPackTags(t *testing.T) { + t.Parallel() + planType := reflect.TypeOf(Plan{}) + + msgPackTags, _ := planType.FieldByName("_struct") + normalizeTag, _ := planType.FieldByName("NormalizeAllocs") + + assert.Equal(t, msgPackTags.Tag, reflect.StructTag(`codec:",omitempty"`)) + assert.Equal(t, normalizeTag.Tag, reflect.StructTag(`codec:"-"`)) +} + +func TestAllocation_MsgPackTags(t *testing.T) { + t.Parallel() + planType := reflect.TypeOf(Allocation{}) + + msgPackTags, _ := planType.FieldByName("_struct") + + assert.Equal(t, msgPackTags.Tag, reflect.StructTag(`codec:",omitempty"`)) +} + +func TestEvaluation_MsgPackTags(t *testing.T) { + t.Parallel() + planType := reflect.TypeOf(Evaluation{}) + + msgPackTags, _ := planType.FieldByName("_struct") + + assert.Equal(t, msgPackTags.Tag, reflect.StructTag(`codec:",omitempty"`)) +} + func TestAllocation_Terminated(t *testing.T) { type desiredState struct { ClientStatus string diff --git a/nomad/util_test.go b/nomad/util_test.go index 561406760..f216d5e42 100644 --- a/nomad/util_test.go +++ b/nomad/util_test.go @@ -86,23 +86,8 @@ func TestIsNomadServer(t *testing.T) { } } -func TestServersMeetMinimumVersion(t *testing.T) { +func TestServersMeetMinimumVersionExcludingFailed(t *testing.T) { t.Parallel() - makeMember := func(version string) serf.Member { - return serf.Member{ - Name: "foo", - Addr: net.IP([]byte{127, 0, 0, 1}), - Tags: map[string]string{ - "role": "nomad", - "region": "aws", - "dc": "east-aws", - "port": "10000", - "build": version, - "vsn": "1", - }, - Status: serf.StatusAlive, - } - } cases := []struct { members []serf.Member @@ -112,7 +97,7 @@ func TestServersMeetMinimumVersion(t *testing.T) { // One server, meets reqs { members: []serf.Member{ - makeMember("0.7.5"), + makeMember("0.7.5", serf.StatusAlive), }, ver: version.Must(version.NewVersion("0.7.5")), expected: true, @@ -120,7 +105,7 @@ func TestServersMeetMinimumVersion(t *testing.T) { // One server in dev, meets reqs { members: []serf.Member{ - makeMember("0.8.5-dev"), + makeMember("0.8.5-dev", serf.StatusAlive), }, ver: version.Must(version.NewVersion("0.7.5")), expected: true, @@ -128,7 +113,7 @@ func TestServersMeetMinimumVersion(t *testing.T) { // One server with meta, meets reqs { members: []serf.Member{ - makeMember("0.7.5+ent"), + makeMember("0.7.5+ent", serf.StatusAlive), }, ver: version.Must(version.NewVersion("0.7.5")), expected: true, @@ -136,16 +121,17 @@ func TestServersMeetMinimumVersion(t *testing.T) { // One server, doesn't meet reqs { members: []serf.Member{ - makeMember("0.7.5"), + makeMember("0.7.5", serf.StatusAlive), }, ver: version.Must(version.NewVersion("0.8.0")), expected: false, }, - // Multiple servers, meets req version + // Multiple servers, meets req version, includes failed that doesn't meet req { members: []serf.Member{ - makeMember("0.7.5"), - makeMember("0.8.0"), + makeMember("0.7.5", serf.StatusAlive), + makeMember("0.8.0", serf.StatusAlive), + makeMember("0.7.0", serf.StatusFailed), }, ver: version.Must(version.NewVersion("0.7.5")), expected: true, @@ -153,8 +139,8 @@ func TestServersMeetMinimumVersion(t *testing.T) { // Multiple servers, doesn't meet req version { members: []serf.Member{ - makeMember("0.7.5"), - makeMember("0.8.0"), + makeMember("0.7.5", serf.StatusAlive), + makeMember("0.8.0", serf.StatusAlive), }, ver: version.Must(version.NewVersion("0.8.0")), expected: false, @@ -169,6 +155,60 @@ func TestServersMeetMinimumVersion(t *testing.T) { } } +func TestServersMeetMinimumVersionIncludingFailed(t *testing.T) { + t.Parallel() + + cases := []struct { + members []serf.Member + ver *version.Version + expected bool + }{ + // Multiple servers, meets req version + { + members: []serf.Member{ + makeMember("0.7.5", serf.StatusAlive), + makeMember("0.8.0", serf.StatusAlive), + makeMember("0.7.5", serf.StatusFailed), + }, + ver: version.Must(version.NewVersion("0.7.5")), + expected: true, + }, + // Multiple servers, doesn't meet req version + { + members: []serf.Member{ + makeMember("0.7.5", serf.StatusAlive), + makeMember("0.8.0", serf.StatusAlive), + makeMember("0.7.0", serf.StatusFailed), + }, + ver: version.Must(version.NewVersion("0.7.5")), + expected: false, + }, + } + + for _, tc := range cases { + result := ServersMeetMinimumVersion(tc.members, tc.ver, true) + if result != tc.expected { + t.Fatalf("bad: %v, %v, %v", result, tc.ver.String(), tc) + } + } +} + +func makeMember(version string, status serf.MemberStatus) serf.Member { + return serf.Member{ + Name: "foo", + Addr: net.IP([]byte{127, 0, 0, 1}), + Tags: map[string]string{ + "role": "nomad", + "region": "aws", + "dc": "east-aws", + "port": "10000", + "build": version, + "vsn": "1", + }, + Status: status, + } +} + func TestShuffleStrings(t *testing.T) { t.Parallel() // Generate input diff --git a/nomad/worker_test.go b/nomad/worker_test.go index b7a9e526a..b2dc94f22 100644 --- a/nomad/worker_test.go +++ b/nomad/worker_test.go @@ -8,20 +8,22 @@ import ( "time" log "github.com/hashicorp/go-hclog" - memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/scheduler" "github.com/hashicorp/nomad/testutil" + "github.com/stretchr/testify/assert" ) type NoopScheduler struct { - state scheduler.State - planner scheduler.Planner - eval *structs.Evaluation - err error + state scheduler.State + planner scheduler.Planner + eval *structs.Evaluation + allowPlanOptimization bool + err error } func (n *NoopScheduler) Process(eval *structs.Evaluation) error { @@ -38,8 +40,9 @@ func (n *NoopScheduler) Process(eval *structs.Evaluation) error { func init() { scheduler.BuiltinSchedulers["noop"] = func(logger log.Logger, s scheduler.State, p scheduler.Planner, allowPlanOptimization bool) scheduler.Scheduler { n := &NoopScheduler{ - state: s, - planner: p, + state: s, + planner: p, + allowPlanOptimization: allowPlanOptimization, } return n } @@ -390,6 +393,57 @@ func TestWorker_SubmitPlan(t *testing.T) { } } +func TestWorker_SubmitPlanNormalizedAllocations(t *testing.T) { + t.Parallel() + s1 := TestServer(t, func(c *Config) { + c.NumSchedulers = 0 + c.EnabledSchedulers = []string{structs.JobTypeService} + }) + defer s1.Shutdown() + testutil.WaitForLeader(t, s1.RPC) + + // Register node + node := mock.Node() + testRegisterNode(t, s1, node) + + job := mock.Job() + eval1 := mock.Eval() + eval1.JobID = job.ID + s1.fsm.State().UpsertJob(0, job) + s1.fsm.State().UpsertEvals(0, []*structs.Evaluation{eval1}) + + stoppedAlloc := mock.Alloc() + preemptedAlloc := mock.Alloc() + s1.fsm.State().UpsertAllocs(5, []*structs.Allocation{stoppedAlloc, preemptedAlloc}) + + // Create an allocation plan + plan := &structs.Plan{ + Job: job, + EvalID: eval1.ID, + NodeUpdate: make(map[string][]*structs.Allocation), + NodePreemptions: make(map[string][]*structs.Allocation), + NormalizeAllocs: true, + } + desiredDescription := "desired desc" + plan.AppendStoppedAlloc(stoppedAlloc, desiredDescription, structs.AllocClientStatusLost) + preemptingAllocID := uuid.Generate() + plan.AppendPreemptedAlloc(preemptedAlloc, preemptingAllocID) + + // Attempt to submit a plan + w := &Worker{srv: s1, logger: s1.logger} + w.SubmitPlan(plan) + + assert.Equal(t, &structs.Allocation{ + ID: preemptedAlloc.ID, + PreemptedByAllocation: preemptingAllocID, + }, plan.NodePreemptions[preemptedAlloc.NodeID][0]) + assert.Equal(t, &structs.Allocation{ + ID: stoppedAlloc.ID, + DesiredDescription: desiredDescription, + ClientStatus: structs.AllocClientStatusLost, + }, plan.NodeUpdate[stoppedAlloc.NodeID][0]) +} + func TestWorker_SubmitPlan_MissingNodeRefresh(t *testing.T) { t.Parallel() s1 := TestServer(t, func(c *Config) { diff --git a/scheduler/generic_sched_test.go b/scheduler/generic_sched_test.go index 70e54d237..3a4ec76ce 100644 --- a/scheduler/generic_sched_test.go +++ b/scheduler/generic_sched_test.go @@ -16,614 +16,471 @@ import ( "github.com/stretchr/testify/require" ) -func TestServiceSched_JobRegister(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } - - // Ensure the eval has no spawned blocked eval - if len(h.CreateEvals) != 0 { - t.Fatalf("bad: %#v", h.CreateEvals) - if h.Evals[0].BlockedEval != "" { - t.Fatalf("bad: %#v", h.Evals[0]) - } - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - - // Ensure different ports were used. - used := make(map[int]map[string]struct{}) - for _, alloc := range out { - for _, resource := range alloc.TaskResources { - for _, port := range resource.Networks[0].DynamicPorts { - nodeMap, ok := used[port.Value] - if !ok { - nodeMap = make(map[string]struct{}) - used[port.Value] = nodeMap - } - if _, ok := nodeMap[alloc.NodeID]; ok { - t.Fatalf("Port collision on node %q %v", alloc.NodeID, port.Value) - } - nodeMap[alloc.NodeID] = struct{}{} - } - } - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} - -func TestServiceSched_JobRegister_StickyAllocs(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a job - job := mock.Job() - job.TaskGroups[0].EphemeralDisk.Sticky = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - if err := h.Process(NewServiceScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure the plan allocated - plan := h.Plans[0] - planned := make(map[string]*structs.Allocation) - for _, allocList := range plan.NodeAllocation { - for _, alloc := range allocList { - planned[alloc.ID] = alloc - } - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - - // Update the job to force a rolling upgrade - updated := job.Copy() - updated.TaskGroups[0].Tasks[0].Resources.CPU += 10 - noErr(t, h.State.UpsertJob(h.NextIndex(), updated)) - - // Create a mock evaluation to handle the update - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - h1 := NewHarnessWithState(t, h.State) - if err := h1.Process(NewServiceScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure we have created only one new allocation - // Ensure a single plan - if len(h1.Plans) != 1 { - t.Fatalf("bad: %#v", h1.Plans) - } - plan = h1.Plans[0] - var newPlanned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - newPlanned = append(newPlanned, allocList...) - } - if len(newPlanned) != 10 { - t.Fatalf("bad plan: %#v", plan) - } - // Ensure that the new allocations were placed on the same node as the older - // ones - for _, new := range newPlanned { - if new.PreviousAllocation == "" { - t.Fatalf("new alloc %q doesn't have a previous allocation", new.ID) - } - - old, ok := planned[new.PreviousAllocation] - if !ok { - t.Fatalf("new alloc %q previous allocation doesn't match any prior placed alloc (%q)", new.ID, new.PreviousAllocation) - } - if new.NodeID != old.NodeID { - t.Fatalf("new alloc and old alloc node doesn't match; got %q; want %q", new.NodeID, old.NodeID) - } - } -} - -func TestServiceSched_JobRegister_DiskConstraints(t *testing.T) { - h := NewHarness(t) - - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a job with count 2 and disk as 60GB so that only one allocation - // can fit - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].EphemeralDisk.SizeMB = 88 * 1024 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } - - // Ensure the eval has a blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } - - if h.CreateEvals[0].TriggeredBy != structs.EvalTriggerQueuedAllocs { - t.Fatalf("bad: %#v", h.CreateEvals[0]) - } - - // Ensure the plan allocated only one allocation - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure only one allocation was placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) +func IsPlanOptimizedStr(allowPlanOptimization bool) string { + return fmt.Sprintf("Is plan optimized: %v", allowPlanOptimization) } func TestServiceSched_JobRegister_DistinctHosts(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Create a job that uses distinct host and has count 1 higher than what is + // possible. + job := mock.Job() + job.TaskGroups[0].Count = 11 + job.Constraints = append(job.Constraints, &structs.Constraint{Operand: structs.ConstraintDistinctHosts}) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the eval has spawned blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } + + // Ensure the plan failed to alloc + outEval := h.Evals[0] + if len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %+v", outEval) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + + // Ensure different node was used per. + used := make(map[string]struct{}) + for _, alloc := range out { + if _, ok := used[alloc.NodeID]; ok { + t.Fatalf("Node collision %v", alloc.NodeID) + } + used[alloc.NodeID] = struct{}{} + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - // Create a job that uses distinct host and has count 1 higher than what is - // possible. - job := mock.Job() - job.TaskGroups[0].Count = 11 - job.Constraints = append(job.Constraints, &structs.Constraint{Operand: structs.ConstraintDistinctHosts}) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the eval has spawned blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } - - // Ensure the plan failed to alloc - outEval := h.Evals[0] - if len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %+v", outEval) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - - // Ensure different node was used per. - used := make(map[string]struct{}) - for _, alloc := range out { - if _, ok := used[alloc.NodeID]; ok { - t.Fatalf("Node collision %v", alloc.NodeID) - } - used[alloc.NodeID] = struct{}{} - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobRegister_DistinctProperty(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - rack := "rack2" - if i < 5 { - rack = "rack1" - } - node.Meta["rack"] = rack - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + rack := "rack2" + if i < 5 { + rack = "rack1" + } + node.Meta["rack"] = rack + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job that uses distinct property and has count higher than what is - // possible. - job := mock.Job() - job.TaskGroups[0].Count = 8 - job.Constraints = append(job.Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${meta.rack}", - RTarget: "2", + // Create a job that uses distinct property and has count higher than what is + // possible. + job := mock.Job() + job.TaskGroups[0].Count = 8 + job.Constraints = append(job.Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${meta.rack}", + RTarget: "2", + }) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } + + // Ensure the eval has spawned blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } + + // Ensure the plan failed to alloc + outEval := h.Evals[0] + if len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %+v", outEval) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 4 { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + if len(out) != 4 { + t.Fatalf("bad: %#v", out) + } + + // Ensure each node was only used twice + used := make(map[string]uint64) + for _, alloc := range out { + if count, _ := used[alloc.NodeID]; count > 2 { + t.Fatalf("Node %v used too much: %d", alloc.NodeID, count) + } + used[alloc.NodeID]++ + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) }) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } - - // Ensure the eval has spawned blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } - - // Ensure the plan failed to alloc - outEval := h.Evals[0] - if len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %+v", outEval) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 4 { - t.Fatalf("bad: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - if len(out) != 4 { - t.Fatalf("bad: %#v", out) - } - - // Ensure each node was only used twice - used := make(map[string]uint64) - for _, alloc := range out { - if count, _ := used[alloc.NodeID]; count > 2 { - t.Fatalf("Node %v used too much: %d", alloc.NodeID, count) - } - used[alloc.NodeID]++ - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobRegister_DistinctProperty_TaskGroup(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 2; i++ { - node := mock.Node() - node.Meta["ssd"] = "true" - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 2; i++ { + node := mock.Node() + node.Meta["ssd"] = "true" + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job that uses distinct property only on one task group. - job := mock.Job() - job.TaskGroups = append(job.TaskGroups, job.TaskGroups[0].Copy()) - job.TaskGroups[0].Count = 1 - job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${meta.ssd}", + // Create a job that uses distinct property only on one task group. + job := mock.Job() + job.TaskGroups = append(job.TaskGroups, job.TaskGroups[0].Copy()) + job.TaskGroups[0].Count = 1 + job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${meta.ssd}", + }) + + job.TaskGroups[1].Name = "tg2" + job.TaskGroups[1].Count = 2 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } + + // Ensure the eval hasn't spawned blocked eval + if len(h.CreateEvals) != 0 { + t.Fatalf("bad: %#v", h.CreateEvals[0]) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 3 { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + if len(out) != 3 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) }) - - job.TaskGroups[1].Name = "tg2" - job.TaskGroups[1].Count = 2 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } - - // Ensure the eval hasn't spawned blocked eval - if len(h.CreateEvals) != 0 { - t.Fatalf("bad: %#v", h.CreateEvals[0]) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 3 { - t.Fatalf("bad: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - if len(out) != 3 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobRegister_DistinctProperty_TaskGroup_Incr(t *testing.T) { - h := NewHarness(t) - assert := assert.New(t) + for _, allowPlanOptimization := range []bool{false, true} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + assert := assert.New(t) - // Create a job that uses distinct property over the node-id - job := mock.Job() - job.TaskGroups[0].Count = 3 - job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${node.unique.id}", + // Create a job that uses distinct property over the node-id + job := mock.Job() + job.TaskGroups[0].Count = 3 + job.TaskGroups[0].Constraints = append(job.TaskGroups[0].Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${node.unique.id}", + }) + assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 6; i++ { + node := mock.Node() + nodes = append(nodes, node) + assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") + } + + // Create some allocations + var allocs []*structs.Allocation + for i := 0; i < 3; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + assert.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs), "UpsertAllocs") + + // Update the count + job2 := job.Copy() + job2.TaskGroups[0].Count = 6 + assert.Nil(h.State.UpsertJob(h.NextIndex(), job2), "UpsertJob") + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + assert.Nil(h.Process(NewServiceScheduler, eval), "Process") + + // Ensure a single plan + assert.Len(h.Plans, 1, "Number of plans") + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + assert.Nil(plan.Annotations, "Plan.Annotations") + + // Ensure the eval hasn't spawned blocked eval + assert.Len(h.CreateEvals, 0, "Created Evals") + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + assert.Len(planned, 6, "Planned Allocations") + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + assert.Nil(err, "AllocsByJob") + + // Ensure all allocations placed + assert.Len(out, 6, "Placed Allocations") + + h.AssertEvalStatus(t, structs.EvalStatusComplete) }) - assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 6; i++ { - node := mock.Node() - nodes = append(nodes, node) - assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") } - - // Create some allocations - var allocs []*structs.Allocation - for i := 0; i < 3; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - assert.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs), "UpsertAllocs") - - // Update the count - job2 := job.Copy() - job2.TaskGroups[0].Count = 6 - assert.Nil(h.State.UpsertJob(h.NextIndex(), job2), "UpsertJob") - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - assert.Nil(h.Process(NewServiceScheduler, eval), "Process") - - // Ensure a single plan - assert.Len(h.Plans, 1, "Number of plans") - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - assert.Nil(plan.Annotations, "Plan.Annotations") - - // Ensure the eval hasn't spawned blocked eval - assert.Len(h.CreateEvals, 0, "Created Evals") - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - assert.Len(planned, 6, "Planned Allocations") - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - assert.Nil(err, "AllocsByJob") - - // Ensure all allocations placed - assert.Len(out, 6, "Placed Allocations") - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } // Test job registration with spread configured func TestServiceSched_Spread(t *testing.T) { - assert := assert.New(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + assert := assert.New(t) - start := uint8(100) - step := uint8(10) + start := uint8(100) + step := uint8(10) - for i := 0; i < 10; i++ { - name := fmt.Sprintf("%d%% in dc1", start) - t.Run(name, func(t *testing.T) { - h := NewHarness(t) - remaining := uint8(100 - start) - // Create a job that uses spread over data center + for i := 0; i < 10; i++ { + name := fmt.Sprintf("%d%% in dc1", start) + t.Run(name, func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + remaining := uint8(100 - start) + // Create a job that uses spread over data center + job := mock.Job() + job.Datacenters = []string{"dc1", "dc2"} + job.TaskGroups[0].Count = 10 + job.TaskGroups[0].Spreads = append(job.TaskGroups[0].Spreads, + &structs.Spread{ + Attribute: "${node.datacenter}", + Weight: 100, + SpreadTarget: []*structs.SpreadTarget{ + { + Value: "dc1", + Percent: start, + }, + { + Value: "dc2", + Percent: remaining, + }, + }, + }) + assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") + // Create some nodes, half in dc2 + var nodes []*structs.Node + nodeMap := make(map[string]*structs.Node) + for i := 0; i < 10; i++ { + node := mock.Node() + if i%2 == 0 { + node.Datacenter = "dc2" + } + nodes = append(nodes, node) + assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") + nodeMap[node.ID] = node + } + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + assert.Nil(h.Process(NewServiceScheduler, eval), "Process") + + // Ensure a single plan + assert.Len(h.Plans, 1, "Number of plans") + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + assert.Nil(plan.Annotations, "Plan.Annotations") + + // Ensure the eval hasn't spawned blocked eval + assert.Len(h.CreateEvals, 0, "Created Evals") + + // Ensure the plan allocated + var planned []*structs.Allocation + dcAllocsMap := make(map[string]int) + for nodeId, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + dc := nodeMap[nodeId].Datacenter + c := dcAllocsMap[dc] + c += len(allocList) + dcAllocsMap[dc] = c + } + assert.Len(planned, 10, "Planned Allocations") + + expectedCounts := make(map[string]int) + expectedCounts["dc1"] = 10 - i + if i > 0 { + expectedCounts["dc2"] = i + } + require.Equal(t, expectedCounts, dcAllocsMap) + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + start = start - step + } + }) + } +} + +// Test job registration with even spread across dc +func TestServiceSched_EvenSpread(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + assert := assert.New(t) + + h := NewHarness(t, allowPlanOptimization) + // Create a job that uses even spread over data center job := mock.Job() job.Datacenters = []string{"dc1", "dc2"} job.TaskGroups[0].Count = 10 @@ -631,16 +488,6 @@ func TestServiceSched_Spread(t *testing.T) { &structs.Spread{ Attribute: "${node.datacenter}", Weight: 100, - SpreadTarget: []*structs.SpreadTarget{ - { - Value: "dc1", - Percent: start, - }, - { - Value: "dc2", - Percent: remaining, - }, - }, }) assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") // Create some nodes, half in dc2 @@ -692,2634 +539,687 @@ func TestServiceSched_Spread(t *testing.T) { } assert.Len(planned, 10, "Planned Allocations") + // Expect even split allocs across datacenter expectedCounts := make(map[string]int) - expectedCounts["dc1"] = 10 - i - if i > 0 { - expectedCounts["dc2"] = i - } + expectedCounts["dc1"] = 5 + expectedCounts["dc2"] = 5 + require.Equal(t, expectedCounts, dcAllocsMap) h.AssertEvalStatus(t, structs.EvalStatusComplete) }) - start = start - step } } -// Test job registration with even spread across dc -func TestServiceSched_EvenSpread(t *testing.T) { - assert := assert.New(t) - - h := NewHarness(t) - // Create a job that uses even spread over data center - job := mock.Job() - job.Datacenters = []string{"dc1", "dc2"} - job.TaskGroups[0].Count = 10 - job.TaskGroups[0].Spreads = append(job.TaskGroups[0].Spreads, - &structs.Spread{ - Attribute: "${node.datacenter}", - Weight: 100, - }) - assert.Nil(h.State.UpsertJob(h.NextIndex(), job), "UpsertJob") - // Create some nodes, half in dc2 - var nodes []*structs.Node - nodeMap := make(map[string]*structs.Node) - for i := 0; i < 10; i++ { - node := mock.Node() - if i%2 == 0 { - node.Datacenter = "dc2" - } - nodes = append(nodes, node) - assert.Nil(h.State.UpsertNode(h.NextIndex(), node), "UpsertNode") - nodeMap[node.ID] = node - } - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - assert.Nil(h.Process(NewServiceScheduler, eval), "Process") - - // Ensure a single plan - assert.Len(h.Plans, 1, "Number of plans") - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - assert.Nil(plan.Annotations, "Plan.Annotations") - - // Ensure the eval hasn't spawned blocked eval - assert.Len(h.CreateEvals, 0, "Created Evals") - - // Ensure the plan allocated - var planned []*structs.Allocation - dcAllocsMap := make(map[string]int) - for nodeId, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - dc := nodeMap[nodeId].Datacenter - c := dcAllocsMap[dc] - c += len(allocList) - dcAllocsMap[dc] = c - } - assert.Len(planned, 10, "Planned Allocations") - - // Expect even split allocs across datacenter - expectedCounts := make(map[string]int) - expectedCounts["dc1"] = 5 - expectedCounts["dc2"] = 5 - - require.Equal(t, expectedCounts, dcAllocsMap) - - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} - func TestServiceSched_JobRegister_Annotate(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - AnnotatePlan: true, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + AnnotatePlan: true, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Ensure the plan had annotations. - if plan.Annotations == nil { - t.Fatalf("expected annotations") - } + // Ensure the plan had annotations. + if plan.Annotations == nil { + t.Fatalf("expected annotations") + } - desiredTGs := plan.Annotations.DesiredTGUpdates - if l := len(desiredTGs); l != 1 { - t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) - } + desiredTGs := plan.Annotations.DesiredTGUpdates + if l := len(desiredTGs); l != 1 { + t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) + } - desiredChanges, ok := desiredTGs["web"] - if !ok { - t.Fatalf("expected task group web to have desired changes") - } + desiredChanges, ok := desiredTGs["web"] + if !ok { + t.Fatalf("expected task group web to have desired changes") + } - expected := &structs.DesiredUpdates{Place: 10} - if !reflect.DeepEqual(desiredChanges, expected) { - t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) + expected := &structs.DesiredUpdates{Place: 10} + if !reflect.DeepEqual(desiredChanges, expected) { + t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) + } + }) } } func TestServiceSched_JobRegister_CountZero(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Create a job and set the task group count to zero. + job := mock.Job() + job.TaskGroups[0].Count = 0 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure there was no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure no allocations placed + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - // Create a job and set the task group count to zero. - job := mock.Job() - job.TaskGroups[0].Count = 0 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure there was no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure no allocations placed - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobRegister_AllocFail(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create NO nodes - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create NO nodes + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Ensure there is a follow up eval. + if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { + t.Fatalf("bad: %#v", h.CreateEvals) + } + + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] + + // Ensure the eval has its spawned blocked eval + if outEval.BlockedEval != h.CreateEvals[0].ID { + t.Fatalf("bad: %#v", outEval) + } + + // Ensure the plan failed to alloc + if outEval == nil || len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %#v", outEval) + } + + metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) + } + + // Check the coalesced failures + if metrics.CoalescedFailures != 9 { + t.Fatalf("bad: %#v", metrics) + } + + // Check the available nodes + if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 0 { + t.Fatalf("bad: %#v", metrics) + } + + // Check queued allocations + queued := outEval.QueuedAllocations["web"] + if queued != 10 { + t.Fatalf("expected queued: %v, actual: %v", 10, queued) + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Ensure there is a follow up eval. - if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { - t.Fatalf("bad: %#v", h.CreateEvals) - } - - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] - - // Ensure the eval has its spawned blocked eval - if outEval.BlockedEval != h.CreateEvals[0].ID { - t.Fatalf("bad: %#v", outEval) - } - - // Ensure the plan failed to alloc - if outEval == nil || len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %#v", outEval) - } - - metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) - } - - // Check the coalesced failures - if metrics.CoalescedFailures != 9 { - t.Fatalf("bad: %#v", metrics) - } - - // Check the available nodes - if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 0 { - t.Fatalf("bad: %#v", metrics) - } - - // Check queued allocations - queued := outEval.QueuedAllocations["web"] - if queued != 10 { - t.Fatalf("expected queued: %v, actual: %v", 10, queued) - } - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobRegister_CreateBlockedEval(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create a full node - node := mock.Node() - node.ReservedResources = &structs.NodeReservedResources{ - Cpu: structs.NodeReservedCpuResources{ - CpuShares: node.NodeResources.Cpu.CpuShares, - }, + // Create a full node + node := mock.Node() + node.ReservedResources = &structs.NodeReservedResources{ + Cpu: structs.NodeReservedCpuResources{ + CpuShares: node.NodeResources.Cpu.CpuShares, + }, + } + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create an ineligible node + node2 := mock.Node() + node2.Attributes["kernel.name"] = "windows" + node2.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a jobs + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Ensure the plan has created a follow up eval. + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } + + created := h.CreateEvals[0] + if created.Status != structs.EvalStatusBlocked { + t.Fatalf("bad: %#v", created) + } + + classes := created.ClassEligibility + if len(classes) != 2 || !classes[node.ComputedClass] || classes[node2.ComputedClass] { + t.Fatalf("bad: %#v", classes) + } + + if created.EscapedComputedClass { + t.Fatalf("bad: %#v", created) + } + + // Ensure there is a follow up eval. + if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { + t.Fatalf("bad: %#v", h.CreateEvals) + } + + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] + + // Ensure the plan failed to alloc + if outEval == nil || len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %#v", outEval) + } + + metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) + } + + // Check the coalesced failures + if metrics.CoalescedFailures != 9 { + t.Fatalf("bad: %#v", metrics) + } + + // Check the available nodes + if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 2 { + t.Fatalf("bad: %#v", metrics) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create an ineligible node - node2 := mock.Node() - node2.Attributes["kernel.name"] = "windows" - node2.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a jobs - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Ensure the plan has created a follow up eval. - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } - - created := h.CreateEvals[0] - if created.Status != structs.EvalStatusBlocked { - t.Fatalf("bad: %#v", created) - } - - classes := created.ClassEligibility - if len(classes) != 2 || !classes[node.ComputedClass] || classes[node2.ComputedClass] { - t.Fatalf("bad: %#v", classes) - } - - if created.EscapedComputedClass { - t.Fatalf("bad: %#v", created) - } - - // Ensure there is a follow up eval. - if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusBlocked { - t.Fatalf("bad: %#v", h.CreateEvals) - } - - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] - - // Ensure the plan failed to alloc - if outEval == nil || len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %#v", outEval) - } - - metrics, ok := outEval.FailedTGAllocs[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) - } - - // Check the coalesced failures - if metrics.CoalescedFailures != 9 { - t.Fatalf("bad: %#v", metrics) - } - - // Check the available nodes - if count, ok := metrics.NodesAvailable["dc1"]; !ok || count != 2 { - t.Fatalf("bad: %#v", metrics) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_JobRegister_FeasibleAndInfeasibleTG(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create one node - node := mock.Node() - node.NodeClass = "class_0" - noErr(t, node.ComputeClass()) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create one node + node := mock.Node() + node.NodeClass = "class_0" + noErr(t, node.ComputeClass()) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a job that constrains on a node class - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].Constraints = append(job.Constraints, - &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "class_0", - Operand: "=", - }, - ) - tg2 := job.TaskGroups[0].Copy() - tg2.Name = "web2" - tg2.Constraints[1].RTarget = "class_1" - job.TaskGroups = append(job.TaskGroups, tg2) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job that constrains on a node class + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].Constraints = append(job.Constraints, + &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "class_0", + Operand: "=", + }, + ) + tg2 := job.TaskGroups[0].Copy() + tg2.Name = "web2" + tg2.Constraints[1].RTarget = "class_1" + job.TaskGroups = append(job.TaskGroups, tg2) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 2 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure two allocations placed + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + if len(out) != 2 { + t.Fatalf("bad: %#v", out) + } + + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] + + // Ensure the eval has its spawned blocked eval + if outEval.BlockedEval != h.CreateEvals[0].ID { + t.Fatalf("bad: %#v", outEval) + } + + // Ensure the plan failed to alloc one tg + if outEval == nil || len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %#v", outEval) + } + + metrics, ok := outEval.FailedTGAllocs[tg2.Name] + if !ok { + t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) + } + + // Check the coalesced failures + if metrics.CoalescedFailures != tg2.Count-1 { + t.Fatalf("bad: %#v", metrics) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 2 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure two allocations placed - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - if len(out) != 2 { - t.Fatalf("bad: %#v", out) - } - - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] - - // Ensure the eval has its spawned blocked eval - if outEval.BlockedEval != h.CreateEvals[0].ID { - t.Fatalf("bad: %#v", outEval) - } - - // Ensure the plan failed to alloc one tg - if outEval == nil || len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %#v", outEval) - } - - metrics, ok := outEval.FailedTGAllocs[tg2.Name] - if !ok { - t.Fatalf("no failed metrics: %#v", outEval.FailedTGAllocs) - } - - // Check the coalesced failures - if metrics.CoalescedFailures != tg2.Count-1 { - t.Fatalf("bad: %#v", metrics) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This test just ensures the scheduler handles the eval type to avoid // regressions. func TestServiceSched_EvaluateMaxPlanEval(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create a job and set the task group count to zero. - job := mock.Job() - job.TaskGroups[0].Count = 0 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job and set the task group count to zero. + job := mock.Job() + job.TaskGroups[0].Count = 0 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock blocked evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Status: structs.EvalStatusBlocked, - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerMaxPlans, - JobID: job.ID, + // Create a mock blocked evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Status: structs.EvalStatusBlocked, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerMaxPlans, + JobID: job.ID, + } + + // Insert it into the state store + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure there was no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - // Insert it into the state store - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure there was no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_Plan_Partial_Progress(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a job with a high resource ask so that all the allocations can't - // be placed on a single node. - job := mock.Job() - job.TaskGroups[0].Count = 3 - job.TaskGroups[0].Tasks[0].Resources.CPU = 3600 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job with a high resource ask so that all the allocations can't + // be placed on a single node. + job := mock.Job() + job.TaskGroups[0].Count = 3 + job.TaskGroups[0].Tasks[0].Resources.CPU = 3600 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure only one allocations placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } + + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 2 { + t.Fatalf("expected: %v, actual: %v", 2, queued) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure only one allocations placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } - - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 2 { - t.Fatalf("expected: %v, actual: %v", 2, queued) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestServiceSched_EvaluateBlockedEval(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock blocked evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Status: structs.EvalStatusBlocked, - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - } + // Create a mock blocked evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Status: structs.EvalStatusBlocked, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + } - // Insert it into the state store - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Insert it into the state store + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure there was no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } + // Ensure there was no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure that the eval was reblocked - if len(h.ReblockEvals) != 1 { - t.Fatalf("bad: %#v", h.ReblockEvals) - } - if h.ReblockEvals[0].ID != eval.ID { - t.Fatalf("expect same eval to be reblocked; got %q; want %q", h.ReblockEvals[0].ID, eval.ID) - } + // Ensure that the eval was reblocked + if len(h.ReblockEvals) != 1 { + t.Fatalf("bad: %#v", h.ReblockEvals) + } + if h.ReblockEvals[0].ID != eval.ID { + t.Fatalf("expect same eval to be reblocked; got %q; want %q", h.ReblockEvals[0].ID, eval.ID) + } - // Ensure the eval status was not updated - if len(h.Evals) != 0 { - t.Fatalf("Existing eval should not have status set") + // Ensure the eval status was not updated + if len(h.Evals) != 0 { + t.Fatalf("Existing eval should not have status set") + } + }) } } func TestServiceSched_EvaluateBlockedEval_Finished(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job and set the task group count to zero. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job and set the task group count to zero. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock blocked evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Status: structs.EvalStatusBlocked, - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - } + // Create a mock blocked evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Status: structs.EvalStatusBlocked, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + } - // Insert it into the state store - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Insert it into the state store + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } - // Ensure the eval has no spawned blocked eval - if len(h.Evals) != 1 { - t.Fatalf("bad: %#v", h.Evals) - if h.Evals[0].BlockedEval != "" { - t.Fatalf("bad: %#v", h.Evals[0]) - } - } + // Ensure the eval has no spawned blocked eval + if len(h.Evals) != 1 { + t.Fatalf("bad: %#v", h.Evals) + if h.Evals[0].BlockedEval != "" { + t.Fatalf("bad: %#v", h.Evals[0]) + } + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } - // Ensure the eval was not reblocked - if len(h.ReblockEvals) != 0 { - t.Fatalf("Existing eval should not have been reblocked as it placed all allocations") - } + // Ensure the eval was not reblocked + if len(h.ReblockEvals) != 0 { + t.Fatalf("Existing eval should not have been reblocked as it placed all allocations") + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Ensure queued allocations is zero - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected queued: %v, actual: %v", 0, queued) + // Ensure queued allocations is zero + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected queued: %v, actual: %v", 0, queued) + } + }) } } func TestServiceSched_JobModify(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Add a few terminal status allocations, these should be ignored - var terminal []*structs.Allocation - for i := 0; i < 5; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DesiredStatus = structs.AllocDesiredStatusStop - terminal = append(terminal, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) - - // Update the job - job2 := mock.Job() - job2.ID = job.ID - - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted all allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} - -// Have a single node and submit a job. Increment the count such that all fit -// on the node but the node doesn't have enough resources to fit the new count + -// 1. This tests that we properly discount the resources of existing allocs. -func TestServiceSched_JobModify_IncrCount_NodeLimit(t *testing.T) { - h := NewHarness(t) - - // Create one node - node := mock.Node() - node.NodeResources.Cpu.CpuShares = 1000 - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job with one allocation - job := mock.Job() - job.TaskGroups[0].Tasks[0].Resources.CPU = 256 - job2 := job.Copy() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.AllocatedResources.Tasks["web"].Cpu.CpuShares = 256 - allocs = append(allocs, alloc) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job to count 3 - job2.TaskGroups[0].Count = 3 - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan didn't evicted the alloc - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 3 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan had no failures - if len(h.Evals) != 1 { - t.Fatalf("incorrect number of updated eval: %#v", h.Evals) - } - outEval := h.Evals[0] - if outEval == nil || len(outEval.FailedTGAllocs) != 0 { - t.Fatalf("bad: %#v", outEval) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 3 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} - -func TestServiceSched_JobModify_CountZero(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Add a few terminal status allocations, these should be ignored - var terminal []*structs.Allocation - for i := 0; i < 5; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) - alloc.DesiredStatus = structs.AllocDesiredStatusStop - terminal = append(terminal, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) - - // Update the job to be count zero - job2 := mock.Job() - job2.ID = job.ID - job2.TaskGroups[0].Count = 0 - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted all allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan didn't allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} - -func TestServiceSched_JobModify_Rolling(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.Job() - job2.ID = job.ID - desiredUpdates := 4 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: desiredUpdates, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } - - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted only MaxParallel - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != desiredUpdates { - t.Fatalf("bad: got %d; want %d: %#v", len(update), desiredUpdates, plan) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != desiredUpdates { - t.Fatalf("bad: %#v", plan) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - - // Check that the deployment id is attached to the eval - if h.Evals[0].DeploymentID == "" { - t.Fatalf("Eval not annotated with deployment id") - } - - // Ensure a deployment was created - if plan.Deployment == nil { - t.Fatalf("bad: %#v", plan) - } - state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("bad: %#v", plan) - } - if state.DesiredTotal != 10 && state.DesiredCanaries != 0 { - t.Fatalf("bad: %#v", state) - } -} - -// This tests that the old allocation is stopped before placing. -// It is critical to test that the updated job attempts to place more -// allocations as this allows us to assert that destructive changes are done -// first. -func TestServiceSched_JobModify_Rolling_FullNode(t *testing.T) { - h := NewHarness(t) - - // Create a node and clear the reserved resources - node := mock.Node() - node.ReservedResources = nil - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a resource ask that is the same as the resources available on the - // node - cpu := node.NodeResources.Cpu.CpuShares - mem := node.NodeResources.Memory.MemoryMB - - request := &structs.Resources{ - CPU: int(cpu), - MemoryMB: int(mem), - } - allocated := &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: cpu, - }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: mem, - }, - }, - }, - } - - // Generate a fake job with one alloc that consumes the whole node - job := mock.Job() - job.TaskGroups[0].Count = 1 - job.TaskGroups[0].Tasks[0].Resources = request - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - alloc := mock.Alloc() - alloc.AllocatedResources = allocated - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Update the job to place more versions of the task group, drop the count - // and force destructive updates - job2 := job.Copy() - job2.TaskGroups[0].Count = 5 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: 5, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } - job2.TaskGroups[0].Tasks[0].Resources = mock.Job().TaskGroups[0].Tasks[0].Resources - - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted only MaxParallel - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 1 { - t.Fatalf("bad: got %d; want %d: %#v", len(update), 1, plan) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 5 { - t.Fatalf("bad: %#v", plan) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - - // Check that the deployment id is attached to the eval - if h.Evals[0].DeploymentID == "" { - t.Fatalf("Eval not annotated with deployment id") - } - - // Ensure a deployment was created - if plan.Deployment == nil { - t.Fatalf("bad: %#v", plan) - } - state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("bad: %#v", plan) - } - if state.DesiredTotal != 5 || state.DesiredCanaries != 0 { - t.Fatalf("bad: %#v", state) - } -} - -func TestServiceSched_JobModify_Canaries(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.Job() - job2.ID = job.ID - desiredUpdates := 2 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: desiredUpdates, - Canary: desiredUpdates, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } - - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted nothing - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: got %d; want %d: %#v", len(update), 0, plan) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != desiredUpdates { - t.Fatalf("bad: %#v", plan) - } - for _, canary := range planned { - if canary.DeploymentStatus == nil || !canary.DeploymentStatus.Canary { - t.Fatalf("expected canary field to be set on canary alloc %q", canary.ID) - } - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - - // Check that the deployment id is attached to the eval - if h.Evals[0].DeploymentID == "" { - t.Fatalf("Eval not annotated with deployment id") - } - - // Ensure a deployment was created - if plan.Deployment == nil { - t.Fatalf("bad: %#v", plan) - } - state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] - if !ok { - t.Fatalf("bad: %#v", plan) - } - if state.DesiredTotal != 10 && state.DesiredCanaries != desiredUpdates { - t.Fatalf("bad: %#v", state) - } - - // Assert the canaries were added to the placed list - if len(state.PlacedCanaries) != desiredUpdates { - t.Fatalf("bad: %#v", state) - } -} - -func TestServiceSched_JobModify_InPlace(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations and create an older deployment - job := mock.Job() - d := mock.Deployment() - d.JobID = job.ID - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) - - // Create allocs that are part of the old deployment - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DeploymentID = d.ID - alloc.DeploymentStatus = &structs.AllocDeploymentStatus{Healthy: helper.BoolToPtr(true)} - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.Job() - job2.ID = job.ID - desiredUpdates := 4 - job2.TaskGroups[0].Update = &structs.UpdateStrategy{ - MaxParallel: desiredUpdates, - HealthCheck: structs.UpdateStrategyHealthCheck_Checks, - MinHealthyTime: 10 * time.Second, - HealthyDeadline: 10 * time.Minute, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan did not evict any allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan updated the existing allocs - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - for _, p := range planned { - if p.Job != job2 { - t.Fatalf("should update job") - } - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - - // Verify the network did not change - rp := structs.Port{Label: "admin", Value: 5000} - for _, alloc := range out { - for _, resources := range alloc.TaskResources { - if resources.Networks[0].ReservedPorts[0] != rp { - t.Fatalf("bad: %#v", alloc) - } - } - } - - // Verify the deployment id was changed and health cleared - for _, alloc := range out { - if alloc.DeploymentID == d.ID { - t.Fatalf("bad: deployment id not cleared") - } else if alloc.DeploymentStatus != nil { - t.Fatalf("bad: deployment status not cleared") - } - } -} - -func TestServiceSched_JobModify_DistinctProperty(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - node.Meta["rack"] = fmt.Sprintf("rack%d", i) - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a job that uses distinct property and has count higher than what is - // possible. - job := mock.Job() - job.TaskGroups[0].Count = 11 - job.Constraints = append(job.Constraints, - &structs.Constraint{ - Operand: structs.ConstraintDistinctProperty, - LTarget: "${meta.rack}", - }) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - oldJob := job.Copy() - oldJob.JobModifyIndex -= 1 - oldJob.TaskGroups[0].Count = 4 - - // Place 4 of 10 - var allocs []*structs.Allocation - for i := 0; i < 4; i++ { - alloc := mock.Alloc() - alloc.Job = oldJob - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } - - // Ensure the eval hasn't spawned blocked eval - if len(h.CreateEvals) != 1 { - t.Fatalf("bad: %#v", h.CreateEvals) - } - - // Ensure the plan failed to alloc - outEval := h.Evals[0] - if len(outEval.FailedTGAllocs) != 1 { - t.Fatalf("bad: %+v", outEval) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", planned) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - - // Ensure different node was used per. - used := make(map[string]struct{}) - for _, alloc := range out { - if _, ok := used[alloc.NodeID]; ok { - t.Fatalf("Node collision %v", alloc.NodeID) - } - used[alloc.NodeID] = struct{}{} - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} - -func TestServiceSched_JobDeregister_Purged(t *testing.T) { - h := NewHarness(t) - - // Generate a fake job with allocations - job := mock.Job() - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - allocs = append(allocs, alloc) - } - for _, alloc := range allocs { - h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID)) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted all nodes - if len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"]) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure that the job field on the allocation is still populated - for _, alloc := range out { - if alloc.Job == nil { - t.Fatalf("bad: %#v", alloc) - } - } - - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} - -func TestServiceSched_JobDeregister_Stopped(t *testing.T) { - h := NewHarness(t) - require := require.New(t) - - // Generate a fake job with allocations - job := mock.Job() - job.Stop = true - require.NoError(h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - allocs = append(allocs, alloc) - } - require.NoError(h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a summary where the queued allocs are set as we want to assert - // they get zeroed out. - summary := mock.JobSummary(job.ID) - web := summary.Summary["web"] - web.Queued = 2 - require.NoError(h.State.UpsertJobSummary(h.NextIndex(), summary)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - require.NoError(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - require.NoError(h.Process(NewServiceScheduler, eval)) - - // Ensure a single plan - require.Len(h.Plans, 1) - plan := h.Plans[0] - - // Ensure the plan evicted all nodes - require.Len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"], len(allocs)) - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - require.NoError(err) - - // Ensure that the job field on the allocation is still populated - for _, alloc := range out { - require.NotNil(alloc.Job) - } - - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - require.Empty(out) - - // Assert the job summary is cleared out - sout, err := h.State.JobSummaryByID(ws, job.Namespace, job.ID) - require.NoError(err) - require.NotNil(sout) - require.Contains(sout.Summary, "web") - webOut := sout.Summary["web"] - require.Zero(webOut.Queued) - - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} - -func TestServiceSched_NodeDown(t *testing.T) { - h := NewHarness(t) - - // Register a node - node := mock.Node() - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - - // Cover each terminal case and ensure it doesn't change to lost - allocs[7].DesiredStatus = structs.AllocDesiredStatusRun - allocs[7].ClientStatus = structs.AllocClientStatusLost - allocs[8].DesiredStatus = structs.AllocDesiredStatusRun - allocs[8].ClientStatus = structs.AllocClientStatusFailed - allocs[9].DesiredStatus = structs.AllocDesiredStatusRun - allocs[9].ClientStatus = structs.AllocClientStatusComplete - - // Mark some allocs as running - for i := 0; i < 4; i++ { - out := allocs[i] - out.ClientStatus = structs.AllocClientStatusRunning - } - - // Mark appropriate allocs for migration - for i := 0; i < 7; i++ { - out := allocs[i] - out.DesiredTransition.Migrate = helper.BoolToPtr(true) - } - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Test the scheduler marked all non-terminal allocations as lost - if len(plan.NodeUpdate[node.ID]) != 7 { - t.Fatalf("bad: %#v", plan) - } - - for _, out := range plan.NodeUpdate[node.ID] { - if out.ClientStatus != structs.AllocClientStatusLost && out.DesiredStatus != structs.AllocDesiredStatusStop { - t.Fatalf("bad alloc: %#v", out) - } - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} - -func TestServiceSched_NodeUpdate(t *testing.T) { - h := NewHarness(t) - - // Register a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Mark some allocs as running - ws := memdb.NewWatchSet() - for i := 0; i < 4; i++ { - out, _ := h.State.AllocByID(ws, allocs[i].ID) - out.ClientStatus = structs.AllocClientStatusRunning - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{out})) - } - - // Create a mock evaluation which won't trigger any new placements - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { - t.Fatalf("bad queued allocations: %v", h.Evals[0].QueuedAllocations) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} - -func TestServiceSched_NodeDrain(t *testing.T) { - h := NewHarness(t) - - // Register a draining node - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} - -func TestServiceSched_NodeDrain_Down(t *testing.T) { - h := NewHarness(t) - - // Register a draining node - node := mock.Node() - node.Drain = true - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job with allocations - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Set the desired state of the allocs to stop - var stop []*structs.Allocation - for i := 0; i < 6; i++ { - newAlloc := allocs[i].Copy() - newAlloc.ClientStatus = structs.AllocDesiredStatusStop - newAlloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - stop = append(stop, newAlloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), stop)) - - // Mark some of the allocations as running - var running []*structs.Allocation - for i := 4; i < 6; i++ { - newAlloc := stop[i].Copy() - newAlloc.ClientStatus = structs.AllocClientStatusRunning - running = append(running, newAlloc) - } - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), running)) - - // Mark some of the allocations as complete - var complete []*structs.Allocation - for i := 6; i < 10; i++ { - newAlloc := allocs[i].Copy() - newAlloc.TaskStates = make(map[string]*structs.TaskState) - newAlloc.TaskStates["web"] = &structs.TaskState{ - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, - }, - }, - } - newAlloc.ClientStatus = structs.AllocClientStatusComplete - complete = append(complete, newAlloc) - } - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), complete)) - - // Create a mock evaluation to deal with the node update - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted non terminal allocs - if len(plan.NodeUpdate[node.ID]) != 6 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure that all the allocations which were in running or pending state - // has been marked as lost - var lostAllocs []string - for _, alloc := range plan.NodeUpdate[node.ID] { - lostAllocs = append(lostAllocs, alloc.ID) - } - sort.Strings(lostAllocs) - - var expectedLostAllocs []string - for i := 0; i < 6; i++ { - expectedLostAllocs = append(expectedLostAllocs, allocs[i].ID) - } - sort.Strings(expectedLostAllocs) - - if !reflect.DeepEqual(expectedLostAllocs, lostAllocs) { - t.Fatalf("expected: %v, actual: %v", expectedLostAllocs, lostAllocs) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} - -func TestServiceSched_NodeDrain_Queued_Allocations(t *testing.T) { - h := NewHarness(t) - - // Register a draining node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 2 { - t.Fatalf("expected: %v, actual: %v", 2, queued) - } -} - -func TestServiceSched_RetryLimit(t *testing.T) { - h := NewHarness(t) - h.Planner = &RejectPlan{h} - - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure no allocations placed - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } - - // Should hit the retry limit - h.AssertEvalStatus(t, structs.EvalStatusFailed) -} - -func TestServiceSched_Reschedule_OnceNow(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: 1, - Interval: 15 * time.Minute, - Delay: 5 * time.Second, - MaxDelay: 1 * time.Minute, - DelayFunction: "constant", - } - tgName := job.TaskGroups[0].Name - now := time.Now() - - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - failedAllocID := allocs[1].ID - successAllocID := allocs[0].ID - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Verify that one new allocation got created with its restart tracker info - assert := assert.New(t) - assert.Equal(3, len(out)) - var newAlloc *structs.Allocation - for _, alloc := range out { - if alloc.ID != successAllocID && alloc.ID != failedAllocID { - newAlloc = alloc - } - } - assert.Equal(failedAllocID, newAlloc.PreviousAllocation) - assert.Equal(1, len(newAlloc.RescheduleTracker.Events)) - assert.Equal(failedAllocID, newAlloc.RescheduleTracker.Events[0].PrevAllocID) - - // Mark this alloc as failed again, should not get rescheduled - newAlloc.ClientStatus = structs.AllocClientStatusFailed - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) - - // Create another mock evaluation - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err = h.Process(NewServiceScheduler, eval) - assert.Nil(err) - // Verify no new allocs were created this time - out, err = h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - assert.Equal(3, len(out)) - -} - -// Tests that alloc reschedulable at a future time creates a follow up eval -func TestServiceSched_Reschedule_Later(t *testing.T) { - h := NewHarness(t) - require := require.New(t) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - delayDuration := 15 * time.Second - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: 1, - Interval: 15 * time.Minute, - Delay: delayDuration, - MaxDelay: 1 * time.Minute, - DelayFunction: "constant", - } - tgName := job.TaskGroups[0].Name - now := time.Now() - - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now}} - failedAllocID := allocs[1].ID - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Verify no new allocs were created - require.Equal(2, len(out)) - - // Verify follow up eval was created for the failed alloc - alloc, err := h.State.AllocByID(ws, failedAllocID) - require.Nil(err) - require.NotEmpty(alloc.FollowupEvalID) - - // Ensure there is a follow up eval. - if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusPending { - t.Fatalf("bad: %#v", h.CreateEvals) - } - followupEval := h.CreateEvals[0] - require.Equal(now.Add(delayDuration), followupEval.WaitUntil) -} - -func TestServiceSched_Reschedule_MultipleNow(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - maxRestartAttempts := 3 - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: maxRestartAttempts, - Interval: 30 * time.Minute, - Delay: 5 * time.Second, - DelayFunction: "constant", - } - tgName := job.TaskGroups[0].Name - now := time.Now() - - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.ClientStatus = structs.AllocClientStatusRunning - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - // Mark one of the allocations as failed - allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - expectedNumAllocs := 3 - expectedNumReschedTrackers := 1 - - failedAllocId := allocs[1].ID - failedNodeID := allocs[1].NodeID - - assert := assert.New(t) - for i := 0; i < maxRestartAttempts; i++ { - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - noErr(t, err) - - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Verify that a new allocation got created with its restart tracker info - assert.Equal(expectedNumAllocs, len(out)) - - // Find the new alloc with ClientStatusPending - var pendingAllocs []*structs.Allocation - var prevFailedAlloc *structs.Allocation - - for _, alloc := range out { - if alloc.ClientStatus == structs.AllocClientStatusPending { - pendingAllocs = append(pendingAllocs, alloc) - } - if alloc.ID == failedAllocId { - prevFailedAlloc = alloc - } - } - assert.Equal(1, len(pendingAllocs)) - newAlloc := pendingAllocs[0] - assert.Equal(expectedNumReschedTrackers, len(newAlloc.RescheduleTracker.Events)) - - // Verify the previous NodeID in the most recent reschedule event - reschedEvents := newAlloc.RescheduleTracker.Events - assert.Equal(failedAllocId, reschedEvents[len(reschedEvents)-1].PrevAllocID) - assert.Equal(failedNodeID, reschedEvents[len(reschedEvents)-1].PrevNodeID) - - // Verify that the next alloc of the failed alloc is the newly rescheduled alloc - assert.Equal(newAlloc.ID, prevFailedAlloc.NextAllocation) - - // Mark this alloc as failed again - newAlloc.ClientStatus = structs.AllocClientStatusFailed - newAlloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-12 * time.Second), - FinishedAt: now.Add(-10 * time.Second)}} - - failedAllocId = newAlloc.ID - failedNodeID = newAlloc.NodeID - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) - - // Create another mock evaluation - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - expectedNumAllocs += 1 - expectedNumReschedTrackers += 1 - } - - // Process last eval again, should not reschedule - err := h.Process(NewServiceScheduler, eval) - assert.Nil(err) - - // Verify no new allocs were created because restart attempts were exhausted - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - assert.Equal(5, len(out)) // 2 original, plus 3 reschedule attempts -} - -// Tests that old reschedule attempts are pruned -func TestServiceSched_Reschedule_PruneEvents(t *testing.T) { - h := NewHarness(t) - - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations and an update policy. - job := mock.Job() - job.TaskGroups[0].Count = 2 - job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - DelayFunction: "exponential", - MaxDelay: 1 * time.Hour, - Delay: 5 * time.Second, - Unlimited: true, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for i := 0; i < 2; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - allocs = append(allocs, alloc) - } - now := time.Now() - // Mark allocations as failed with restart info - allocs[1].TaskStates = map[string]*structs.TaskState{job.TaskGroups[0].Name: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-15 * time.Minute)}} - allocs[1].ClientStatus = structs.AllocClientStatusFailed - - allocs[1].RescheduleTracker = &structs.RescheduleTracker{ - Events: []*structs.RescheduleEvent{ - {RescheduleTime: now.Add(-1 * time.Hour).UTC().UnixNano(), - PrevAllocID: uuid.Generate(), - PrevNodeID: uuid.Generate(), - Delay: 5 * time.Second, - }, - {RescheduleTime: now.Add(-40 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 10 * time.Second, - }, - {RescheduleTime: now.Add(-30 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 20 * time.Second, - }, - {RescheduleTime: now.Add(-20 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 40 * time.Second, - }, - {RescheduleTime: now.Add(-10 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 80 * time.Second, - }, - {RescheduleTime: now.Add(-3 * time.Minute).UTC().UnixNano(), - PrevAllocID: allocs[0].ID, - PrevNodeID: uuid.Generate(), - Delay: 160 * time.Second, - }, - }, - } - expectedFirstRescheduleEvent := allocs[1].RescheduleTracker.Events[1] - expectedDelay := 320 * time.Second - failedAllocID := allocs[1].ID - successAllocID := allocs[0].ID - - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Verify that one new allocation got created with its restart tracker info - assert := assert.New(t) - assert.Equal(3, len(out)) - var newAlloc *structs.Allocation - for _, alloc := range out { - if alloc.ID != successAllocID && alloc.ID != failedAllocID { - newAlloc = alloc - } - } - - assert.Equal(failedAllocID, newAlloc.PreviousAllocation) - // Verify that the new alloc copied the last 5 reschedule attempts - assert.Equal(6, len(newAlloc.RescheduleTracker.Events)) - assert.Equal(expectedFirstRescheduleEvent, newAlloc.RescheduleTracker.Events[0]) - - mostRecentRescheduleEvent := newAlloc.RescheduleTracker.Events[5] - // Verify that the failed alloc ID is in the most recent reschedule event - assert.Equal(failedAllocID, mostRecentRescheduleEvent.PrevAllocID) - // Verify that the delay value was captured correctly - assert.Equal(expectedDelay, mostRecentRescheduleEvent.Delay) - -} - -// Tests that deployments with failed allocs result in placements as long as the -// deployment is running. -func TestDeployment_FailedAllocs_Reschedule(t *testing.T) { - for _, failedDeployment := range []bool{false, true} { - t.Run(fmt.Sprintf("Failed Deployment: %v", failedDeployment), func(t *testing.T) { - h := NewHarness(t) - require := require.New(t) // Create some nodes var nodes []*structs.Node for i := 0; i < 10; i++ { @@ -3328,25 +1228,1522 @@ func TestDeployment_FailedAllocs_Reschedule(t *testing.T) { noErr(t, h.State.UpsertNode(h.NextIndex(), node)) } - // Generate a fake job with allocations and a reschedule policy. + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Add a few terminal status allocations, these should be ignored + var terminal []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DesiredStatus = structs.AllocDesiredStatusStop + terminal = append(terminal, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) + + // Update the job + job2 := mock.Job() + job2.ID = job.ID + + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted all allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +// Have a single node and submit a job. Increment the count such that all fit +// on the node but the node doesn't have enough resources to fit the new count + +// 1. This tests that we properly discount the resources of existing allocs. +func TestServiceSched_JobModify_IncrCount_NodeLimit(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create one node + node := mock.Node() + node.NodeResources.Cpu.CpuShares = 1000 + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job with one allocation + job := mock.Job() + job.TaskGroups[0].Tasks[0].Resources.CPU = 256 + job2 := job.Copy() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.AllocatedResources.Tasks["web"].Cpu.CpuShares = 256 + allocs = append(allocs, alloc) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job to count 3 + job2.TaskGroups[0].Count = 3 + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan didn't evicted the alloc + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 3 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan had no failures + if len(h.Evals) != 1 { + t.Fatalf("incorrect number of updated eval: %#v", h.Evals) + } + outEval := h.Evals[0] + if outEval == nil || len(outEval.FailedTGAllocs) != 0 { + t.Fatalf("bad: %#v", outEval) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 3 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_JobModify_CountZero(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Add a few terminal status allocations, these should be ignored + var terminal []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = structs.AllocName(alloc.JobID, alloc.TaskGroup, uint(i)) + alloc.DesiredStatus = structs.AllocDesiredStatusStop + terminal = append(terminal, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) + + // Update the job to be count zero + job2 := mock.Job() + job2.ID = job.ID + job2.TaskGroups[0].Count = 0 + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted all allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan didn't allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_JobModify_Rolling(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.Job() + job2.ID = job.ID + desiredUpdates := 4 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: desiredUpdates, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } + + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted only MaxParallel + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != desiredUpdates { + t.Fatalf("bad: got %d; want %d: %#v", len(update), desiredUpdates, plan) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != desiredUpdates { + t.Fatalf("bad: %#v", plan) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + + // Check that the deployment id is attached to the eval + if h.Evals[0].DeploymentID == "" { + t.Fatalf("Eval not annotated with deployment id") + } + + // Ensure a deployment was created + if plan.Deployment == nil { + t.Fatalf("bad: %#v", plan) + } + state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("bad: %#v", plan) + } + if state.DesiredTotal != 10 && state.DesiredCanaries != 0 { + t.Fatalf("bad: %#v", state) + } + }) + } +} + +// This tests that the old allocation is stopped before placing. +// It is critical to test that the updated job attempts to place more +// allocations as this allows us to assert that destructive changes are done +// first. +func TestServiceSched_JobModify_Rolling_FullNode(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create a node and clear the reserved resources + node := mock.Node() + node.ReservedResources = nil + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a resource ask that is the same as the resources available on the + // node + cpu := node.NodeResources.Cpu.CpuShares + mem := node.NodeResources.Memory.MemoryMB + + request := &structs.Resources{ + CPU: int(cpu), + MemoryMB: int(mem), + } + allocated := &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: cpu, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: mem, + }, + }, + }, + } + + // Generate a fake job with one alloc that consumes the whole node + job := mock.Job() + job.TaskGroups[0].Count = 1 + job.TaskGroups[0].Tasks[0].Resources = request + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + alloc := mock.Alloc() + alloc.AllocatedResources = allocated + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Update the job to place more versions of the task group, drop the count + // and force destructive updates + job2 := job.Copy() + job2.TaskGroups[0].Count = 5 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: 5, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } + job2.TaskGroups[0].Tasks[0].Resources = mock.Job().TaskGroups[0].Tasks[0].Resources + + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted only MaxParallel + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 1 { + t.Fatalf("bad: got %d; want %d: %#v", len(update), 1, plan) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 5 { + t.Fatalf("bad: %#v", plan) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + + // Check that the deployment id is attached to the eval + if h.Evals[0].DeploymentID == "" { + t.Fatalf("Eval not annotated with deployment id") + } + + // Ensure a deployment was created + if plan.Deployment == nil { + t.Fatalf("bad: %#v", plan) + } + state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("bad: %#v", plan) + } + if state.DesiredTotal != 5 || state.DesiredCanaries != 0 { + t.Fatalf("bad: %#v", state) + } + }) + } +} + +func TestServiceSched_JobModify_Canaries(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.Job() + job2.ID = job.ID + desiredUpdates := 2 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: desiredUpdates, + Canary: desiredUpdates, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } + + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted nothing + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: got %d; want %d: %#v", len(update), 0, plan) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != desiredUpdates { + t.Fatalf("bad: %#v", plan) + } + for _, canary := range planned { + if canary.DeploymentStatus == nil || !canary.DeploymentStatus.Canary { + t.Fatalf("expected canary field to be set on canary alloc %q", canary.ID) + } + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + + // Check that the deployment id is attached to the eval + if h.Evals[0].DeploymentID == "" { + t.Fatalf("Eval not annotated with deployment id") + } + + // Ensure a deployment was created + if plan.Deployment == nil { + t.Fatalf("bad: %#v", plan) + } + state, ok := plan.Deployment.TaskGroups[job.TaskGroups[0].Name] + if !ok { + t.Fatalf("bad: %#v", plan) + } + if state.DesiredTotal != 10 && state.DesiredCanaries != desiredUpdates { + t.Fatalf("bad: %#v", state) + } + + // Assert the canaries were added to the placed list + if len(state.PlacedCanaries) != desiredUpdates { + t.Fatalf("bad: %#v", state) + } + }) + } +} + +func TestServiceSched_JobModify_InPlace(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations and create an older deployment + job := mock.Job() + d := mock.Deployment() + d.JobID = job.ID + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) + + // Create allocs that are part of the old deployment + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DeploymentID = d.ID + alloc.DeploymentStatus = &structs.AllocDeploymentStatus{Healthy: helper.BoolToPtr(true)} + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.Job() + job2.ID = job.ID + desiredUpdates := 4 + job2.TaskGroups[0].Update = &structs.UpdateStrategy{ + MaxParallel: desiredUpdates, + HealthCheck: structs.UpdateStrategyHealthCheck_Checks, + MinHealthyTime: 10 * time.Second, + HealthyDeadline: 10 * time.Minute, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan did not evict any allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan updated the existing allocs + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + for _, p := range planned { + if p.Job != job2 { + t.Fatalf("should update job") + } + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) + + // Verify the network did not change + rp := structs.Port{Label: "admin", Value: 5000} + for _, alloc := range out { + for _, resources := range alloc.TaskResources { + if resources.Networks[0].ReservedPorts[0] != rp { + t.Fatalf("bad: %#v", alloc) + } + } + } + + // Verify the deployment id was changed and health cleared + for _, alloc := range out { + if alloc.DeploymentID == d.ID { + t.Fatalf("bad: deployment id not cleared") + } else if alloc.DeploymentStatus != nil { + t.Fatalf("bad: deployment status not cleared") + } + } + }) + } +} + +func TestServiceSched_JobModify_DistinctProperty(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + node.Meta["rack"] = fmt.Sprintf("rack%d", i) + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Create a job that uses distinct property and has count higher than what is + // possible. + job := mock.Job() + job.TaskGroups[0].Count = 11 + job.Constraints = append(job.Constraints, + &structs.Constraint{ + Operand: structs.ConstraintDistinctProperty, + LTarget: "${meta.rack}", + }) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + oldJob := job.Copy() + oldJob.JobModifyIndex -= 1 + oldJob.TaskGroups[0].Count = 4 + + // Place 4 of 10 + var allocs []*structs.Allocation + for i := 0; i < 4; i++ { + alloc := mock.Alloc() + alloc.Job = oldJob + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } + + // Ensure the eval hasn't spawned blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } + + // Ensure the plan failed to alloc + outEval := h.Evals[0] + if len(outEval.FailedTGAllocs) != 1 { + t.Fatalf("bad: %+v", outEval) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", planned) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + + // Ensure different node was used per. + used := make(map[string]struct{}) + for _, alloc := range out { + if _, ok := used[alloc.NodeID]; ok { + t.Fatalf("Node collision %v", alloc.NodeID) + } + used[alloc.NodeID] = struct{}{} + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_JobDeregister_Purged(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Generate a fake job with allocations + job := mock.Job() + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + allocs = append(allocs, alloc) + } + for _, alloc := range allocs { + h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID)) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted all nodes + if len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"]) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure that the job field on the allocation is still populated + for _, alloc := range out { + if alloc.Job == nil { + t.Fatalf("bad: %#v", alloc) + } + } + + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_JobDeregister_Stopped(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + require := require.New(t) + + // Generate a fake job with allocations + job := mock.Job() + job.Stop = true + require.NoError(h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + allocs = append(allocs, alloc) + } + require.NoError(h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a summary where the queued allocs are set as we want to assert + // they get zeroed out. + summary := mock.JobSummary(job.ID) + web := summary.Summary["web"] + web.Queued = 2 + require.NoError(h.State.UpsertJobSummary(h.NextIndex(), summary)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + require.NoError(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + require.NoError(h.Process(NewServiceScheduler, eval)) + + // Ensure a single plan + require.Len(h.Plans, 1) + plan := h.Plans[0] + + // Ensure the plan evicted all nodes + require.Len(plan.NodeUpdate["12345678-abcd-efab-cdef-123456789abc"], len(allocs)) + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + require.NoError(err) + + // Ensure that the job field on the allocation is still populated + for _, alloc := range out { + require.NotNil(alloc.Job) + } + + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + require.Empty(out) + + // Assert the job summary is cleared out + sout, err := h.State.JobSummaryByID(ws, job.Namespace, job.ID) + require.NoError(err) + require.NotNil(sout) + require.Contains(sout.Summary, "web") + webOut := sout.Summary["web"] + require.Zero(webOut.Queued) + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_NodeDown(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a node + node := mock.Node() + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job with allocations and an update policy. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + + // Cover each terminal case and ensure it doesn't change to lost + allocs[7].DesiredStatus = structs.AllocDesiredStatusRun + allocs[7].ClientStatus = structs.AllocClientStatusLost + allocs[8].DesiredStatus = structs.AllocDesiredStatusRun + allocs[8].ClientStatus = structs.AllocClientStatusFailed + allocs[9].DesiredStatus = structs.AllocDesiredStatusRun + allocs[9].ClientStatus = structs.AllocClientStatusComplete + + // Mark some allocs as running + for i := 0; i < 4; i++ { + out := allocs[i] + out.ClientStatus = structs.AllocClientStatusRunning + } + + // Mark appropriate allocs for migration + for i := 0; i < 7; i++ { + out := allocs[i] + out.DesiredTransition.Migrate = helper.BoolToPtr(true) + } + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Test the scheduler marked all non-terminal allocations as lost + if len(plan.NodeUpdate[node.ID]) != 7 { + t.Fatalf("bad: %#v", plan) + } + + for _, out := range plan.NodeUpdate[node.ID] { + if out.ClientStatus != structs.AllocClientStatusLost && out.DesiredStatus != structs.AllocDesiredStatusStop { + t.Fatalf("bad alloc: %#v", out) + } + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_NodeUpdate(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job with allocations and an update policy. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Mark some allocs as running + ws := memdb.NewWatchSet() + for i := 0; i < 4; i++ { + out, _ := h.State.AllocByID(ws, allocs[i].ID) + out.ClientStatus = structs.AllocClientStatusRunning + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{out})) + } + + // Create a mock evaluation which won't trigger any new placements + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { + t.Fatalf("bad queued allocations: %v", h.Evals[0].QueuedAllocations) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_NodeDrain(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a draining node + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations and an update policy. + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_NodeDrain_Down(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a draining node + node := mock.Node() + node.Drain = true + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job with allocations + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Set the desired state of the allocs to stop + var stop []*structs.Allocation + for i := 0; i < 6; i++ { + newAlloc := allocs[i].Copy() + newAlloc.ClientStatus = structs.AllocDesiredStatusStop + newAlloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + stop = append(stop, newAlloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), stop)) + + // Mark some of the allocations as running + var running []*structs.Allocation + for i := 4; i < 6; i++ { + newAlloc := stop[i].Copy() + newAlloc.ClientStatus = structs.AllocClientStatusRunning + running = append(running, newAlloc) + } + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), running)) + + // Mark some of the allocations as complete + var complete []*structs.Allocation + for i := 6; i < 10; i++ { + newAlloc := allocs[i].Copy() + newAlloc.TaskStates = make(map[string]*structs.TaskState) + newAlloc.TaskStates["web"] = &structs.TaskState{ + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, + }, + }, + } + newAlloc.ClientStatus = structs.AllocClientStatusComplete + complete = append(complete, newAlloc) + } + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), complete)) + + // Create a mock evaluation to deal with the node update + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted non terminal allocs + if len(plan.NodeUpdate[node.ID]) != 6 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure that all the allocations which were in running or pending state + // has been marked as lost + var lostAllocs []string + for _, alloc := range plan.NodeUpdate[node.ID] { + lostAllocs = append(lostAllocs, alloc.ID) + } + sort.Strings(lostAllocs) + + var expectedLostAllocs []string + for i := 0; i < 6; i++ { + expectedLostAllocs = append(expectedLostAllocs, allocs[i].ID) + } + sort.Strings(expectedLostAllocs) + + if !reflect.DeepEqual(expectedLostAllocs, lostAllocs) { + t.Fatalf("expected: %v, actual: %v", expectedLostAllocs, lostAllocs) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } +} + +func TestServiceSched_NodeDrain_Queued_Allocations(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Register a draining node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 2 { + t.Fatalf("expected: %v, actual: %v", 2, queued) + } + }) + } +} + +func TestServiceSched_RetryLimit(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + h.Planner = &RejectPlan{h} + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure no allocations placed + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } + + // Should hit the retry limit + h.AssertEvalStatus(t, structs.EvalStatusFailed) + }) + } +} + +func TestServiceSched_Reschedule_OnceNow(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations and an update policy. job := mock.Job() job.TaskGroups[0].Count = 2 job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ - Attempts: 1, - Interval: 15 * time.Minute, + Attempts: 1, + Interval: 15 * time.Minute, + Delay: 5 * time.Second, + MaxDelay: 1 * time.Minute, + DelayFunction: "constant", } - jobIndex := h.NextIndex() - require.Nil(h.State.UpsertJob(jobIndex, job)) + tgName := job.TaskGroups[0].Name + now := time.Now() - deployment := mock.Deployment() - deployment.JobID = job.ID - deployment.JobCreateIndex = jobIndex - deployment.JobVersion = job.Version - if failedDeployment { - deployment.Status = structs.DeploymentStatusFailed - } - - require.Nil(h.State.UpsertDeployment(h.NextIndex(), deployment)) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) var allocs []*structs.Allocation for i := 0; i < 2; i++ { @@ -3355,17 +2752,17 @@ func TestDeployment_FailedAllocs_Reschedule(t *testing.T) { alloc.JobID = job.ID alloc.NodeID = nodes[i].ID alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.DeploymentID = deployment.ID allocs = append(allocs, alloc) } - // Mark one of the allocations as failed in the past + // Mark one of the allocations as failed allocs[1].ClientStatus = structs.AllocClientStatusFailed - allocs[1].TaskStates = map[string]*structs.TaskState{"web": {State: "start", - StartedAt: time.Now().Add(-12 * time.Hour), - FinishedAt: time.Now().Add(-10 * time.Hour)}} - allocs[1].DesiredTransition.Reschedule = helper.BoolToPtr(true) + allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + failedAllocID := allocs[1].ID + successAllocID := allocs[0].ID - require.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs)) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) // Create a mock evaluation eval := &structs.Evaluation{ @@ -3376,471 +2773,1404 @@ func TestDeployment_FailedAllocs_Reschedule(t *testing.T) { JobID: job.ID, Status: structs.EvalStatusPending, } - require.Nil(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) // Process the evaluation - require.Nil(h.Process(NewServiceScheduler, eval)) + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - if failedDeployment { - // Verify no plan created - require.Len(h.Plans, 0) - } else { - require.Len(h.Plans, 1) - plan := h.Plans[0] + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Verify that one new allocation got created with its restart tracker info + assert := assert.New(t) + assert.Equal(3, len(out)) + var newAlloc *structs.Allocation + for _, alloc := range out { + if alloc.ID != successAllocID && alloc.ID != failedAllocID { + newAlloc = alloc } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) + } + assert.Equal(failedAllocID, newAlloc.PreviousAllocation) + assert.Equal(1, len(newAlloc.RescheduleTracker.Events)) + assert.Equal(failedAllocID, newAlloc.RescheduleTracker.Events[0].PrevAllocID) + + // Mark this alloc as failed again, should not get rescheduled + newAlloc.ClientStatus = structs.AllocClientStatusFailed + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) + + // Create another mock evaluation + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err = h.Process(NewServiceScheduler, eval) + assert.Nil(err) + // Verify no new allocs were created this time + out, err = h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + assert.Equal(3, len(out)) + }) + } +} + +// Tests that alloc reschedulable at a future time creates a follow up eval +func TestServiceSched_Reschedule_Later(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + require := require.New(t) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + delayDuration := 15 * time.Second + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 15 * time.Minute, + Delay: delayDuration, + MaxDelay: 1 * time.Minute, + DelayFunction: "constant", + } + tgName := job.TaskGroups[0].Name + now := time.Now() + + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now}} + failedAllocID := allocs[1].ID + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Verify no new allocs were created + require.Equal(2, len(out)) + + // Verify follow up eval was created for the failed alloc + alloc, err := h.State.AllocByID(ws, failedAllocID) + require.Nil(err) + require.NotEmpty(alloc.FollowupEvalID) + + // Ensure there is a follow up eval. + if len(h.CreateEvals) != 1 || h.CreateEvals[0].Status != structs.EvalStatusPending { + t.Fatalf("bad: %#v", h.CreateEvals) + } + followupEval := h.CreateEvals[0] + require.Equal(now.Add(delayDuration), followupEval.WaitUntil) + }) + } +} + +func TestServiceSched_Reschedule_MultipleNow(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + maxRestartAttempts := 3 + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: maxRestartAttempts, + Interval: 30 * time.Minute, + Delay: 5 * time.Second, + DelayFunction: "constant", + } + tgName := job.TaskGroups[0].Name + now := time.Now() + + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.ClientStatus = structs.AllocClientStatusRunning + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + expectedNumAllocs := 3 + expectedNumReschedTrackers := 1 + + failedAllocId := allocs[1].ID + failedNodeID := allocs[1].NodeID + + assert := assert.New(t) + for i := 0; i < maxRestartAttempts; i++ { + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + noErr(t, err) + + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Verify that a new allocation got created with its restart tracker info + assert.Equal(expectedNumAllocs, len(out)) + + // Find the new alloc with ClientStatusPending + var pendingAllocs []*structs.Allocation + var prevFailedAlloc *structs.Allocation + + for _, alloc := range out { + if alloc.ClientStatus == structs.AllocClientStatusPending { + pendingAllocs = append(pendingAllocs, alloc) + } + if alloc.ID == failedAllocId { + prevFailedAlloc = alloc + } + } + assert.Equal(1, len(pendingAllocs)) + newAlloc := pendingAllocs[0] + assert.Equal(expectedNumReschedTrackers, len(newAlloc.RescheduleTracker.Events)) + + // Verify the previous NodeID in the most recent reschedule event + reschedEvents := newAlloc.RescheduleTracker.Events + assert.Equal(failedAllocId, reschedEvents[len(reschedEvents)-1].PrevAllocID) + assert.Equal(failedNodeID, reschedEvents[len(reschedEvents)-1].PrevNodeID) + + // Verify that the next alloc of the failed alloc is the newly rescheduled alloc + assert.Equal(newAlloc.ID, prevFailedAlloc.NextAllocation) + + // Mark this alloc as failed again + newAlloc.ClientStatus = structs.AllocClientStatusFailed + newAlloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-12 * time.Second), + FinishedAt: now.Add(-10 * time.Second)}} + + failedAllocId = newAlloc.ID + failedNodeID = newAlloc.NodeID + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{newAlloc})) + + // Create another mock evaluation + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + expectedNumAllocs += 1 + expectedNumReschedTrackers += 1 + } + + // Process last eval again, should not reschedule + err := h.Process(NewServiceScheduler, eval) + assert.Nil(err) + + // Verify no new allocs were created because restart attempts were exhausted + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + assert.Equal(5, len(out)) // 2 original, plus 3 reschedule attempts + }) + } +} + +// Tests that old reschedule attempts are pruned +func TestServiceSched_Reschedule_PruneEvents(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations and an update policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + DelayFunction: "exponential", + MaxDelay: 1 * time.Hour, + Delay: 5 * time.Second, + Unlimited: true, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + allocs = append(allocs, alloc) + } + now := time.Now() + // Mark allocations as failed with restart info + allocs[1].TaskStates = map[string]*structs.TaskState{job.TaskGroups[0].Name: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-15 * time.Minute)}} + allocs[1].ClientStatus = structs.AllocClientStatusFailed + + allocs[1].RescheduleTracker = &structs.RescheduleTracker{ + Events: []*structs.RescheduleEvent{ + {RescheduleTime: now.Add(-1 * time.Hour).UTC().UnixNano(), + PrevAllocID: uuid.Generate(), + PrevNodeID: uuid.Generate(), + Delay: 5 * time.Second, + }, + {RescheduleTime: now.Add(-40 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 10 * time.Second, + }, + {RescheduleTime: now.Add(-30 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 20 * time.Second, + }, + {RescheduleTime: now.Add(-20 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 40 * time.Second, + }, + {RescheduleTime: now.Add(-10 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 80 * time.Second, + }, + {RescheduleTime: now.Add(-3 * time.Minute).UTC().UnixNano(), + PrevAllocID: allocs[0].ID, + PrevNodeID: uuid.Generate(), + Delay: 160 * time.Second, + }, + }, + } + expectedFirstRescheduleEvent := allocs[1].RescheduleTracker.Events[1] + expectedDelay := 320 * time.Second + failedAllocID := allocs[1].ID + successAllocID := allocs[0].ID + + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Verify that one new allocation got created with its restart tracker info + assert := assert.New(t) + assert.Equal(3, len(out)) + var newAlloc *structs.Allocation + for _, alloc := range out { + if alloc.ID != successAllocID && alloc.ID != failedAllocID { + newAlloc = alloc + } + } + + assert.Equal(failedAllocID, newAlloc.PreviousAllocation) + // Verify that the new alloc copied the last 5 reschedule attempts + assert.Equal(6, len(newAlloc.RescheduleTracker.Events)) + assert.Equal(expectedFirstRescheduleEvent, newAlloc.RescheduleTracker.Events[0]) + + mostRecentRescheduleEvent := newAlloc.RescheduleTracker.Events[5] + // Verify that the failed alloc ID is in the most recent reschedule event + assert.Equal(failedAllocID, mostRecentRescheduleEvent.PrevAllocID) + // Verify that the delay value was captured correctly + assert.Equal(expectedDelay, mostRecentRescheduleEvent.Delay) + }) + } +} + +// Tests that deployments with failed allocs result in placements as long as the +// deployment is running. +func TestDeployment_FailedAllocs_Reschedule(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + for _, failedDeployment := range []bool{false, true} { + t.Run(fmt.Sprintf("Failed Deployment: %v", failedDeployment), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + require := require.New(t) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations and a reschedule policy. + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].ReschedulePolicy = &structs.ReschedulePolicy{ + Attempts: 1, + Interval: 15 * time.Minute, + } + jobIndex := h.NextIndex() + require.Nil(h.State.UpsertJob(jobIndex, job)) + + deployment := mock.Deployment() + deployment.JobID = job.ID + deployment.JobCreateIndex = jobIndex + deployment.JobVersion = job.Version + if failedDeployment { + deployment.Status = structs.DeploymentStatusFailed + } + + require.Nil(h.State.UpsertDeployment(h.NextIndex(), deployment)) + + var allocs []*structs.Allocation + for i := 0; i < 2; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.DeploymentID = deployment.ID + allocs = append(allocs, alloc) + } + // Mark one of the allocations as failed in the past + allocs[1].ClientStatus = structs.AllocClientStatusFailed + allocs[1].TaskStates = map[string]*structs.TaskState{"web": {State: "start", + StartedAt: time.Now().Add(-12 * time.Hour), + FinishedAt: time.Now().Add(-10 * time.Hour)}} + allocs[1].DesiredTransition.Reschedule = helper.BoolToPtr(true) + + require.Nil(h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + require.Nil(h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + require.Nil(h.Process(NewServiceScheduler, eval)) + + if failedDeployment { + // Verify no plan created + require.Len(h.Plans, 0) + } else { + require.Len(h.Plans, 1) + plan := h.Plans[0] + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } + } + }) } }) } } func TestBatchSched_Run_CompleteAlloc(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a complete alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusComplete - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + // Create a complete alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusComplete + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure no plan as it should be a no-op + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure no allocations placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure no plan as it should be a no-op - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure no allocations placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestBatchSched_Run_FailedAlloc(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - tgName := job.TaskGroups[0].Name - now := time.Now() + tgName := job.TaskGroups[0].Name + now := time.Now() - // Create a failed alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusFailed - alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + // Create a failed alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusFailed + alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure a replacement alloc was placed. + if len(out) != 2 { + t.Fatalf("bad: %#v", out) + } + + // Ensure that the scheduler is recording the correct number of queued + // allocations + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected: %v, actual: %v", 1, queued) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure a replacement alloc was placed. - if len(out) != 2 { - t.Fatalf("bad: %#v", out) - } - - // Ensure that the scheduler is recording the correct number of queued - // allocations - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected: %v, actual: %v", 1, queued) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestBatchSched_Run_LostAlloc(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a job - job := mock.Job() - job.ID = "my-job" - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 3 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + job.ID = "my-job" + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 3 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Desired = 3 - // Mark one as lost and then schedule - // [(0, run, running), (1, run, running), (1, stop, lost)] + // Desired = 3 + // Mark one as lost and then schedule + // [(0, run, running), (1, run, running), (1, stop, lost)] - // Create two running allocations - var allocs []*structs.Allocation - for i := 0; i <= 1; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusRunning - allocs = append(allocs, alloc) + // Create two running allocations + var allocs []*structs.Allocation + for i := 0; i <= 1; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusRunning + allocs = append(allocs, alloc) + } + + // Create a failed alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[1]" + alloc.DesiredStatus = structs.AllocDesiredStatusStop + alloc.ClientStatus = structs.AllocClientStatusComplete + allocs = append(allocs, alloc) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure a replacement alloc was placed. + if len(out) != 4 { + t.Fatalf("bad: %#v", out) + } + + // Assert that we have the correct number of each alloc name + expected := map[string]int{ + "my-job.web[0]": 1, + "my-job.web[1]": 2, + "my-job.web[2]": 1, + } + actual := make(map[string]int, 3) + for _, alloc := range out { + actual[alloc.Name] += 1 + } + require.Equal(t, actual, expected) + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } +} - // Create a failed alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[1]" - alloc.DesiredStatus = structs.AllocDesiredStatusStop - alloc.ClientStatus = structs.AllocClientStatusComplete - allocs = append(allocs, alloc) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) +func TestServiceSched_JobRegister(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } + + // Ensure the eval has no spawned blocked eval + if len(h.CreateEvals) != 0 { + t.Fatalf("bad: %#v", h.CreateEvals) + if h.Evals[0].BlockedEval != "" { + t.Fatalf("bad: %#v", h.Evals[0]) + } + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + + // Ensure different ports were used. + used := make(map[int]map[string]struct{}) + for _, alloc := range out { + for _, resource := range alloc.TaskResources { + for _, port := range resource.Networks[0].DynamicPorts { + nodeMap, ok := used[port.Value] + if !ok { + nodeMap = make(map[string]struct{}) + used[port.Value] = nodeMap + } + if _, ok := nodeMap[alloc.NodeID]; ok { + t.Fatalf("Port collision on node %q %v", alloc.NodeID, port.Value) + } + nodeMap[alloc.NodeID] = struct{}{} + } + } + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) +} - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) +func TestServiceSched_JobRegister_StickyAllocs(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Create a job + job := mock.Job() + job.TaskGroups[0].EphemeralDisk.Sticky = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + if err := h.Process(NewServiceScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure the plan allocated + plan := h.Plans[0] + planned := make(map[string]*structs.Allocation) + for _, allocList := range plan.NodeAllocation { + for _, alloc := range allocList { + planned[alloc.ID] = alloc + } + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + + // Update the job to force a rolling upgrade + updated := job.Copy() + updated.TaskGroups[0].Tasks[0].Resources.CPU += 10 + noErr(t, h.State.UpsertJob(h.NextIndex(), updated)) + + // Create a mock evaluation to handle the update + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) + if err := h1.Process(NewServiceScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure we have created only one new allocation + // Ensure a single plan + if len(h1.Plans) != 1 { + t.Fatalf("bad: %#v", h1.Plans) + } + plan = h1.Plans[0] + var newPlanned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + newPlanned = append(newPlanned, allocList...) + } + if len(newPlanned) != 10 { + t.Fatalf("bad plan: %#v", plan) + } + // Ensure that the new allocations were placed on the same node as the older + // ones + for _, new := range newPlanned { + if new.PreviousAllocation == "" { + t.Fatalf("new alloc %q doesn't have a previous allocation", new.ID) + } + + old, ok := planned[new.PreviousAllocation] + if !ok { + t.Fatalf("new alloc %q previous allocation doesn't match any prior placed alloc (%q)", new.ID, new.PreviousAllocation) + } + if new.NodeID != old.NodeID { + t.Fatalf("new alloc and old alloc node doesn't match; got %q; want %q", new.NodeID, old.NodeID) + } + } + }) } +} - // Ensure a plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) +func TestServiceSched_JobRegister_DiskConstraints(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + + // Create a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a job with count 2 and disk as 60GB so that only one allocation + // can fit + job := mock.Job() + job.TaskGroups[0].Count = 2 + job.TaskGroups[0].EphemeralDisk.SizeMB = 88 * 1024 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + ID: uuid.Generate(), + } + + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } + + // Ensure the eval has a blocked eval + if len(h.CreateEvals) != 1 { + t.Fatalf("bad: %#v", h.CreateEvals) + } + + if h.CreateEvals[0].TriggeredBy != structs.EvalTriggerQueuedAllocs { + t.Fatalf("bad: %#v", h.CreateEvals[0]) + } + + // Ensure the plan allocated only one allocation + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure only one allocation was placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure a replacement alloc was placed. - if len(out) != 4 { - t.Fatalf("bad: %#v", out) - } - - // Assert that we have the correct number of each alloc name - expected := map[string]int{ - "my-job.web[0]": 1, - "my-job.web[1]": 2, - "my-job.web[2]": 1, - } - actual := make(map[string]int, 3) - for _, alloc := range out { - actual[alloc.Name] += 1 - } - require.Equal(t, actual, expected) - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestBatchSched_Run_FailedAllocQueuedAllocations(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - tgName := job.TaskGroups[0].Name - now := time.Now() + tgName := job.TaskGroups[0].Name + now := time.Now() - // Create a failed alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusFailed - alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", - StartedAt: now.Add(-1 * time.Hour), - FinishedAt: now.Add(-10 * time.Second)}} - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + // Create a failed alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusFailed + alloc.TaskStates = map[string]*structs.TaskState{tgName: {State: "dead", + StartedAt: now.Add(-1 * time.Hour), + FinishedAt: now.Add(-10 * time.Second)}} + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure that the scheduler is recording the correct number of queued - // allocations - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 1 { - t.Fatalf("expected: %v, actual: %v", 1, queued) + // Ensure that the scheduler is recording the correct number of queued + // allocations + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 1 { + t.Fatalf("expected: %v, actual: %v", 1, queued) + } + }) } } func TestBatchSched_ReRun_SuccessfullyFinishedAlloc(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create two nodes, one that is drained and has a successfully finished - // alloc and a fresh undrained one - node := mock.Node() - node.Drain = true - node2 := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + // Create two nodes, one that is drained and has a successfully finished + // alloc and a fresh undrained one + node := mock.Node() + node.Drain = true + node2 := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a successful alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusComplete - alloc.TaskStates = map[string]*structs.TaskState{ - "web": { - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, + // Create a successful alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusComplete + alloc.TaskStates = map[string]*structs.TaskState{ + "web": { + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, + }, + }, }, - }, - }, + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to rerun the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure no replacement alloc was placed. + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to rerun the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure no replacement alloc was placed. - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This test checks that terminal allocations that receive an in-place updated // are not added to the plan func TestBatchSched_JobModify_InPlace_Terminal(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.Job() - job.Type = structs.JobTypeBatch - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations + job := mock.Job() + job.Type = structs.JobTypeBatch + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusComplete - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusComplete + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Create a mock evaluation to trigger the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to trigger the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans[0]) + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans[0]) + } + }) } } // This test ensures that terminal jobs from older versions are ignored. func TestBatchSched_JobModify_Destructive_Terminal(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations + job := mock.Job() + job.Type = structs.JobTypeBatch + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusComplete + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.Job() + job2.ID = job.ID + job2.Type = structs.JobTypeBatch + job2.Version++ + job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + allocs = nil + for i := 0; i < 10; i++ { + alloc := mock.Alloc() + alloc.Job = job2 + alloc.JobID = job2.ID + alloc.NodeID = nodes[i].ID + alloc.Name = fmt.Sprintf("my-job.web[%d]", i) + alloc.ClientStatus = structs.AllocClientStatusComplete + alloc.TaskStates = map[string]*structs.TaskState{ + "web": { + State: structs.TaskStateDead, + Events: []*structs.TaskEvent{ + { + Type: structs.TaskTerminated, + ExitCode: 0, + }, + }, + }, + } + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } + }) } +} - // Generate a fake job with allocations - job := mock.Job() - job.Type = structs.JobTypeBatch - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) +// This test asserts that an allocation from an old job that is running on a +// drained node is cleaned up. +func TestBatchSched_NodeDrain_Running_OldJob(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - var allocs []*structs.Allocation - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusComplete - allocs = append(allocs, alloc) + // Create two nodes, one that is drained and has a successfully finished + // alloc and a fresh undrained one + node := mock.Node() + node.Drain = true + node2 := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a running alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusRunning + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create an update job + job2 := job.Copy() + job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} + job2.Version++ + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + + plan := h.Plans[0] + + // Ensure the plan evicted 1 + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan places 1 + if len(plan.NodeAllocation[node2.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) +} - // Update the job - job2 := mock.Job() - job2.ID = job.ID - job2.Type = structs.JobTypeBatch - job2.Version++ - job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) +// This test asserts that an allocation from a job that is complete on a +// drained node is ignored up. +func TestBatchSched_NodeDrain_Complete(t *testing.T) { + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - allocs = nil - for i := 0; i < 10; i++ { - alloc := mock.Alloc() - alloc.Job = job2 - alloc.JobID = job2.ID - alloc.NodeID = nodes[i].ID - alloc.Name = fmt.Sprintf("my-job.web[%d]", i) - alloc.ClientStatus = structs.AllocClientStatusComplete - alloc.TaskStates = map[string]*structs.TaskState{ - "web": { + // Create two nodes, one that is drained and has a successfully finished + // alloc and a fresh undrained one + node := mock.Node() + node.Drain = true + node2 := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + + // Create a job + job := mock.Job() + job.Type = structs.JobTypeBatch + job.TaskGroups[0].Count = 1 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a complete alloc + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.ClientStatus = structs.AllocClientStatusComplete + alloc.TaskStates = make(map[string]*structs.TaskState) + alloc.TaskStates["web"] = &structs.TaskState{ State: structs.TaskStateDead, Events: []*structs.TaskEvent{ { @@ -3848,169 +4178,35 @@ func TestBatchSched_JobModify_Destructive_Terminal(t *testing.T) { ExitCode: 0, }, }, - }, - } - allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewBatchScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure no plan + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } -} - -// This test asserts that an allocation from an old job that is running on a -// drained node is cleaned up. -func TestBatchSched_NodeDrain_Running_OldJob(t *testing.T) { - h := NewHarness(t) - - // Create two nodes, one that is drained and has a successfully finished - // alloc and a fresh undrained one - node := mock.Node() - node.Drain = true - node2 := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a running alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusRunning - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create an update job - job2 := job.Copy() - job2.TaskGroups[0].Tasks[0].Env = map[string]string{"foo": "bar"} - job2.Version++ - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - - plan := h.Plans[0] - - // Ensure the plan evicted 1 - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan places 1 - if len(plan.NodeAllocation[node2.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) -} - -// This test asserts that an allocation from a job that is complete on a -// drained node is ignored up. -func TestBatchSched_NodeDrain_Complete(t *testing.T) { - h := NewHarness(t) - - // Create two nodes, one that is drained and has a successfully finished - // alloc and a fresh undrained one - node := mock.Node() - node.Drain = true - node2 := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - - // Create a job - job := mock.Job() - job.Type = structs.JobTypeBatch - job.TaskGroups[0].Count = 1 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a complete alloc - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.ClientStatus = structs.AllocClientStatusComplete - alloc.TaskStates = make(map[string]*structs.TaskState) - alloc.TaskStates["web"] = &structs.TaskState{ - State: structs.TaskStateDead, - Events: []*structs.TaskEvent{ - { - Type: structs.TaskTerminated, - ExitCode: 0, - }, - }, - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewBatchScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure no plan - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This is a slightly odd test but it ensures that we handle a scale down of a @@ -4099,303 +4295,318 @@ func TestBatchSched_ScaleDown_SameName(t *testing.T) { } func TestGenericSched_ChainedAlloc(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewServiceScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } - - var allocIDs []string - for _, allocList := range h.Plans[0].NodeAllocation { - for _, alloc := range allocList { - allocIDs = append(allocIDs, alloc.ID) - } - } - sort.Strings(allocIDs) - - // Create a new harness to invoke the scheduler again - h1 := NewHarnessWithState(t, h.State) - job1 := mock.Job() - job1.ID = job.ID - job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" - job1.TaskGroups[0].Count = 12 - noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) - - // Create a mock evaluation to update the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job1.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job1.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - - // Process the evaluation - if err := h1.Process(NewServiceScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } - - plan := h1.Plans[0] - - // Collect all the chained allocation ids and the new allocations which - // don't have any chained allocations - var prevAllocs []string - var newAllocs []string - for _, allocList := range plan.NodeAllocation { - for _, alloc := range allocList { - if alloc.PreviousAllocation == "" { - newAllocs = append(newAllocs, alloc.ID) - continue + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) } - prevAllocs = append(prevAllocs, alloc.PreviousAllocation) - } - } - sort.Strings(prevAllocs) - // Ensure that the new allocations has their corresponding original - // allocation ids - if !reflect.DeepEqual(prevAllocs, allocIDs) { - t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) - } + // Create a job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensuring two new allocations don't have any chained allocations - if len(newAllocs) != 2 { - t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + if err := h.Process(NewServiceScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } + + var allocIDs []string + for _, allocList := range h.Plans[0].NodeAllocation { + for _, alloc := range allocList { + allocIDs = append(allocIDs, alloc.ID) + } + } + sort.Strings(allocIDs) + + // Create a new harness to invoke the scheduler again + h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) + job1 := mock.Job() + job1.ID = job.ID + job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" + job1.TaskGroups[0].Count = 12 + noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) + + // Create a mock evaluation to update the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job1.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job1.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + + // Process the evaluation + if err := h1.Process(NewServiceScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } + + plan := h1.Plans[0] + + // Collect all the chained allocation ids and the new allocations which + // don't have any chained allocations + var prevAllocs []string + var newAllocs []string + for _, allocList := range plan.NodeAllocation { + for _, alloc := range allocList { + if alloc.PreviousAllocation == "" { + newAllocs = append(newAllocs, alloc.ID) + continue + } + prevAllocs = append(prevAllocs, alloc.PreviousAllocation) + } + } + sort.Strings(prevAllocs) + + // Ensure that the new allocations has their corresponding original + // allocation ids + if !reflect.DeepEqual(prevAllocs, allocIDs) { + t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) + } + + // Ensuring two new allocations don't have any chained allocations + if len(newAllocs) != 2 { + t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) + } + }) } } func TestServiceSched_NodeDrain_Sticky(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Register a draining node - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Register a draining node + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create an alloc on the draining node - alloc := mock.Alloc() - alloc.Name = "my-job.web[0]" - alloc.NodeID = node.ID - alloc.Job.TaskGroups[0].Count = 1 - alloc.Job.TaskGroups[0].EphemeralDisk.Sticky = true - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - noErr(t, h.State.UpsertJob(h.NextIndex(), alloc.Job)) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + // Create an alloc on the draining node + alloc := mock.Alloc() + alloc.Name = "my-job.web[0]" + alloc.NodeID = node.ID + alloc.Job.TaskGroups[0].Count = 1 + alloc.Job.TaskGroups[0].EphemeralDisk.Sticky = true + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + noErr(t, h.State.UpsertJob(h.NextIndex(), alloc.Job)) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: alloc.Job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: alloc.Job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan didn't create any new allocations + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan didn't create any new allocations - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This test ensures that when a job is stopped, the scheduler properly cancels // an outstanding deployment. func TestServiceSched_CancelDeployment_Stopped(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Generate a fake job - job := mock.Job() - job.JobModifyIndex = job.CreateIndex + 1 - job.ModifyIndex = job.CreateIndex + 1 - job.Stop = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job + job := mock.Job() + job.JobModifyIndex = job.CreateIndex + 1 + job.ModifyIndex = job.CreateIndex + 1 + job.Stop = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a deployment - d := mock.Deployment() - d.JobID = job.ID - d.JobCreateIndex = job.CreateIndex - d.JobModifyIndex = job.JobModifyIndex - 1 - noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) + // Create a deployment + d := mock.Deployment() + d.JobID = job.ID + d.JobCreateIndex = job.CreateIndex + d.JobModifyIndex = job.JobModifyIndex - 1 + noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan cancelled the existing deployment + ws := memdb.NewWatchSet() + out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) + noErr(t, err) + + if out == nil { + t.Fatalf("No deployment for job") + } + if out.ID != d.ID { + t.Fatalf("Latest deployment for job is different than original deployment") + } + if out.Status != structs.DeploymentStatusCancelled { + t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) + } + if out.StatusDescription != structs.DeploymentStatusDescriptionStoppedJob { + t.Fatalf("Deployment status description is %q, want %q", + out.StatusDescription, structs.DeploymentStatusDescriptionStoppedJob) + } + + // Ensure the plan didn't allocate anything + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan cancelled the existing deployment - ws := memdb.NewWatchSet() - out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) - noErr(t, err) - - if out == nil { - t.Fatalf("No deployment for job") - } - if out.ID != d.ID { - t.Fatalf("Latest deployment for job is different than original deployment") - } - if out.Status != structs.DeploymentStatusCancelled { - t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) - } - if out.StatusDescription != structs.DeploymentStatusDescriptionStoppedJob { - t.Fatalf("Deployment status description is %q, want %q", - out.StatusDescription, structs.DeploymentStatusDescriptionStoppedJob) - } - - // Ensure the plan didn't allocate anything - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } // This test ensures that when a job is updated and had an old deployment, the scheduler properly cancels // the deployment. func TestServiceSched_CancelDeployment_NewerJob(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Generate a fake job - job := mock.Job() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job + job := mock.Job() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a deployment for an old version of the job - d := mock.Deployment() - d.JobID = job.ID - noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) + // Create a deployment for an old version of the job + d := mock.Deployment() + d.JobID = job.ID + noErr(t, h.State.UpsertDeployment(h.NextIndex(), d)) - // Upsert again to bump job version - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Upsert again to bump job version + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to kick the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, + // Create a mock evaluation to kick the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan cancelled the existing deployment + ws := memdb.NewWatchSet() + out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) + noErr(t, err) + + if out == nil { + t.Fatalf("No deployment for job") + } + if out.ID != d.ID { + t.Fatalf("Latest deployment for job is different than original deployment") + } + if out.Status != structs.DeploymentStatusCancelled { + t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) + } + if out.StatusDescription != structs.DeploymentStatusDescriptionNewerJob { + t.Fatalf("Deployment status description is %q, want %q", + out.StatusDescription, structs.DeploymentStatusDescriptionNewerJob) + } + // Ensure the plan didn't allocate anything + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 0 { + t.Fatalf("bad: %#v", plan) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan cancelled the existing deployment - ws := memdb.NewWatchSet() - out, err := h.State.LatestDeploymentByJobID(ws, job.Namespace, job.ID) - noErr(t, err) - - if out == nil { - t.Fatalf("No deployment for job") - } - if out.ID != d.ID { - t.Fatalf("Latest deployment for job is different than original deployment") - } - if out.Status != structs.DeploymentStatusCancelled { - t.Fatalf("Deployment status is %q, want %q", out.Status, structs.DeploymentStatusCancelled) - } - if out.StatusDescription != structs.DeploymentStatusDescriptionNewerJob { - t.Fatalf("Deployment status description is %q, want %q", - out.StatusDescription, structs.DeploymentStatusDescriptionNewerJob) - } - // Ensure the plan didn't allocate anything - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 0 { - t.Fatalf("bad: %#v", plan) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } // Various table driven tests for carry forward // of past reschedule events func Test_updateRescheduleTracker(t *testing.T) { - t1 := time.Now().UTC() alloc := mock.Alloc() prevAlloc := mock.Alloc() @@ -4626,5 +4837,4 @@ func Test_updateRescheduleTracker(t *testing.T) { require.Equal(tc.expectedRescheduleEvents, alloc.RescheduleTracker.Events) }) } - } diff --git a/scheduler/system_sched_test.go b/scheduler/system_sched_test.go index 9461fd85e..2509697de 100644 --- a/scheduler/system_sched_test.go +++ b/scheduler/system_sched_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" @@ -15,1870 +15,1964 @@ import ( "github.com/stretchr/testify/require" ) +// COMPAT 0.11: Currently, all the tests run for 2 cases: +// 1) Allow plan optimization +// 2) Not allowing plan optimization +// The code for not allowing plan optimizations is in place to allow for safer cluster upgrades, +// and backwards compatibility with the existing raft logs. The older code will be removed later, +// and these tests should then no longer include the testing for case (2) + func TestSystemSched_JobRegister(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + if plan.Annotations != nil { + t.Fatalf("expected no annotations") + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + + // Check the available nodes + if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { + t.Fatalf("bad: %#v", out[0].Metrics) + } + + // Ensure no allocations are queued + queued := h.Evals[0].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected queued allocations: %v, actual: %v", 0, queued) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - if plan.Annotations != nil { - t.Fatalf("expected no annotations") - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - - // Check the available nodes - if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { - t.Fatalf("bad: %#v", out[0].Metrics) - } - - // Ensure no allocations are queued - queued := h.Evals[0].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected queued allocations: %v, actual: %v", 0, queued) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_JobRegister_StickyAllocs(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job - job := mock.SystemJob() - job.TaskGroups[0].EphemeralDisk.Sticky = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.SystemJob() + job.TaskGroups[0].EphemeralDisk.Sticky = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Ensure the plan allocated - plan := h.Plans[0] - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + plan := h.Plans[0] + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } - // Get an allocation and mark it as failed - alloc := planned[4].Copy() - alloc.ClientStatus = structs.AllocClientStatusFailed - noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{alloc})) + // Get an allocation and mark it as failed + alloc := planned[4].Copy() + alloc.ClientStatus = structs.AllocClientStatusFailed + noErr(t, h.State.UpdateAllocsFromClient(h.NextIndex(), []*structs.Allocation{alloc})) - // Create a mock evaluation to handle the update - eval = &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - h1 := NewHarnessWithState(t, h.State) - if err := h1.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to handle the update + eval = &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) + if err := h1.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Ensure we have created only one new allocation - plan = h1.Plans[0] - var newPlanned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - newPlanned = append(newPlanned, allocList...) - } - if len(newPlanned) != 1 { - t.Fatalf("bad plan: %#v", plan) - } - // Ensure that the new allocation was placed on the same node as the older - // one - if newPlanned[0].NodeID != alloc.NodeID || newPlanned[0].PreviousAllocation != alloc.ID { - t.Fatalf("expected: %#v, actual: %#v", alloc, newPlanned[0]) + // Ensure we have created only one new allocation + plan = h1.Plans[0] + var newPlanned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + newPlanned = append(newPlanned, allocList...) + } + if len(newPlanned) != 1 { + t.Fatalf("bad plan: %#v", plan) + } + // Ensure that the new allocation was placed on the same node as the older + // one + if newPlanned[0].NodeID != alloc.NodeID || newPlanned[0].PreviousAllocation != alloc.ID { + t.Fatalf("expected: %#v, actual: %#v", alloc, newPlanned[0]) + } + }) } } func TestSystemSched_JobRegister_EphemeralDiskConstraint(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create a nodes - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create a nodes + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Create a job - job := mock.SystemJob() - job.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job + job := mock.SystemJob() + job.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create another job with a lot of disk resource ask so that it doesn't fit - // the node - job1 := mock.SystemJob() - job1.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 - noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) + // Create another job with a lot of disk resource ask so that it doesn't fit + // the node + job1 := mock.SystemJob() + job1.TaskGroups[0].EphemeralDisk.SizeMB = 60 * 1024 + noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 1 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 1 { + t.Fatalf("bad: %#v", out) + } - // Create a new harness to test the scheduling result for the second job - h1 := NewHarnessWithState(t, h.State) - // Create a mock evaluation to register the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job1.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job1.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Create a new harness to test the scheduling result for the second job + h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) + // Create a mock evaluation to register the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job1.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job1.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - // Process the evaluation - if err := h1.Process(NewSystemScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + if err := h1.Process(NewSystemScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - out, err = h1.State.AllocsByJob(ws, job.Namespace, job1.ID, false) - noErr(t, err) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) + out, err = h1.State.AllocsByJob(ws, job.Namespace, job1.ID, false) + noErr(t, err) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } + }) } } func TestSystemSched_ExhaustResources(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create a nodes - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create a nodes + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Enable Preemption - h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ - PreemptionConfig: structs.PreemptionConfig{ - SystemSchedulerEnabled: true, - }, - }) + // Enable Preemption + h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ + PreemptionConfig: structs.PreemptionConfig{ + SystemSchedulerEnabled: true, + }, + }) - // Create a service job which consumes most of the system resources - svcJob := mock.Job() - svcJob.TaskGroups[0].Count = 1 - svcJob.TaskGroups[0].Tasks[0].Resources.CPU = 3600 - noErr(t, h.State.UpsertJob(h.NextIndex(), svcJob)) + // Create a service job which consumes most of the system resources + svcJob := mock.Job() + svcJob.TaskGroups[0].Count = 1 + svcJob.TaskGroups[0].Tasks[0].Resources.CPU = 3600 + noErr(t, h.State.UpsertJob(h.NextIndex(), svcJob)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: svcJob.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: svcJob.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: svcJob.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: svcJob.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Create a system job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a system job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to register the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } - // System scheduler will preempt the service job and would have placed eval1 - require := require.New(t) + // System scheduler will preempt the service job and would have placed eval1 + require := require.New(t) - newPlan := h.Plans[1] - require.Len(newPlan.NodeAllocation, 1) - require.Len(newPlan.NodePreemptions, 1) + newPlan := h.Plans[1] + require.Len(newPlan.NodeAllocation, 1) + require.Len(newPlan.NodePreemptions, 1) - for _, allocList := range newPlan.NodeAllocation { - require.Len(allocList, 1) - require.Equal(job.ID, allocList[0].JobID) - } + for _, allocList := range newPlan.NodeAllocation { + require.Len(allocList, 1) + require.Equal(job.ID, allocList[0].JobID) + } - for _, allocList := range newPlan.NodePreemptions { - require.Len(allocList, 1) - require.Equal(svcJob.ID, allocList[0].JobID) - } - // Ensure that we have no queued allocations on the second eval - queued := h.Evals[1].QueuedAllocations["web"] - if queued != 0 { - t.Fatalf("expected: %v, actual: %v", 1, queued) + for _, allocList := range newPlan.NodePreemptions { + require.Len(allocList, 1) + alloc, err := h.State.AllocByID(nil, allocList[0].ID) + noErr(t, err) + require.Equal(svcJob.ID, alloc.JobID) + } + // Ensure that we have no queued allocations on the second eval + queued := h.Evals[1].QueuedAllocations["web"] + if queued != 0 { + t.Fatalf("expected: %v, actual: %v", 1, queued) + } + }) } } func TestSystemSched_JobRegister_Annotate(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - if i < 9 { - node.NodeClass = "foo" - } else { - node.NodeClass = "bar" - } - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + if i < 9 { + node.NodeClass = "foo" + } else { + node.NodeClass = "bar" + } + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Create a job constraining on node class - job := mock.SystemJob() - fooConstraint := &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "foo", - Operand: "==", - } - job.Constraints = append(job.Constraints, fooConstraint) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create a job constraining on node class + job := mock.SystemJob() + fooConstraint := &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "foo", + Operand: "==", + } + job.Constraints = append(job.Constraints, fooConstraint) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - AnnotatePlan: true, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + AnnotatePlan: true, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 9 { - t.Fatalf("bad: %#v %d", planned, len(planned)) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 9 { + t.Fatalf("bad: %#v %d", planned, len(planned)) + } - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) - // Ensure all allocations placed - if len(out) != 9 { - t.Fatalf("bad: %#v", out) - } + // Ensure all allocations placed + if len(out) != 9 { + t.Fatalf("bad: %#v", out) + } - // Check the available nodes - if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { - t.Fatalf("bad: %#v", out[0].Metrics) - } + // Check the available nodes + if count, ok := out[0].Metrics.NodesAvailable["dc1"]; !ok || count != 10 { + t.Fatalf("bad: %#v", out[0].Metrics) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Ensure the plan had annotations. - if plan.Annotations == nil { - t.Fatalf("expected annotations") - } + // Ensure the plan had annotations. + if plan.Annotations == nil { + t.Fatalf("expected annotations") + } - desiredTGs := plan.Annotations.DesiredTGUpdates - if l := len(desiredTGs); l != 1 { - t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) - } + desiredTGs := plan.Annotations.DesiredTGUpdates + if l := len(desiredTGs); l != 1 { + t.Fatalf("incorrect number of task groups; got %v; want %v", l, 1) + } - desiredChanges, ok := desiredTGs["web"] - if !ok { - t.Fatalf("expected task group web to have desired changes") - } + desiredChanges, ok := desiredTGs["web"] + if !ok { + t.Fatalf("expected task group web to have desired changes") + } - expected := &structs.DesiredUpdates{Place: 9} - if !reflect.DeepEqual(desiredChanges, expected) { - t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) + expected := &structs.DesiredUpdates{Place: 9} + if !reflect.DeepEqual(desiredChanges, expected) { + t.Fatalf("Unexpected desired updates; got %#v; want %#v", desiredChanges, expected) + + } + }) } } func TestSystemSched_JobRegister_AddNode(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Add a new node. + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + + // Create a mock evaluation to deal with the node update + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan had no node updates + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Log(len(update)) + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan allocated on the new node + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure it allocated on the right node + if _, ok := plan.NodeAllocation[node.ID]; !ok { + t.Fatalf("allocated on wrong node: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 11 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Add a new node. - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a mock evaluation to deal with the node update - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan had no node updates - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Log(len(update)) - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan allocated on the new node - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure it allocated on the right node - if _, ok := plan.NodeAllocation[node.ID]; !ok { - t.Fatalf("allocated on wrong node: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 11 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_JobRegister_AllocFail(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create NO nodes - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Create NO nodes + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure no plan as this should be a no-op. + if len(h.Plans) != 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure no plan as this should be a no-op. - if len(h.Plans) != 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_JobModify(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Add a few terminal status allocations, these should be ignored + var terminal []*structs.Allocation + for i := 0; i < 5; i++ { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = nodes[i].ID + alloc.Name = "my-job.web[0]" + alloc.DesiredStatus = structs.AllocDesiredStatusStop + terminal = append(terminal, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) + + // Update the job + job2 := mock.SystemJob() + job2.ID = job.ID + + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted all allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != len(allocs) { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Add a few terminal status allocations, these should be ignored - var terminal []*structs.Allocation - for i := 0; i < 5; i++ { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = nodes[i].ID - alloc.Name = "my-job.web[0]" - alloc.DesiredStatus = structs.AllocDesiredStatusStop - terminal = append(terminal, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), terminal)) - - // Update the job - job2 := mock.SystemJob() - job2.ID = job.ID - - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted all allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != len(allocs) { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_JobModify_Rolling(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - // Update the job - job2 := mock.SystemJob() - job2.ID = job.ID - job2.Update = structs.UpdateStrategy{ - Stagger: 30 * time.Second, - MaxParallel: 5, - } + // Update the job + job2 := mock.SystemJob() + job2.ID = job.ID + job2.Update = structs.UpdateStrategy{ + Stagger: 30 * time.Second, + MaxParallel: 5, + } - // Update the task, such that it cannot be done in-place - job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + // Update the task, such that it cannot be done in-place + job2.TaskGroups[0].Tasks[0].Config["command"] = "/bin/other" + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] - // Ensure the plan evicted only MaxParallel - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != job2.Update.MaxParallel { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan evicted only MaxParallel + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != job2.Update.MaxParallel { + t.Fatalf("bad: %#v", plan) + } - // Ensure the plan allocated - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != job2.Update.MaxParallel { - t.Fatalf("bad: %#v", plan) - } + // Ensure the plan allocated + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != job2.Update.MaxParallel { + t.Fatalf("bad: %#v", plan) + } - h.AssertEvalStatus(t, structs.EvalStatusComplete) + h.AssertEvalStatus(t, structs.EvalStatusComplete) - // Ensure a follow up eval was created - eval = h.Evals[0] - if eval.NextEval == "" { - t.Fatalf("missing next eval") - } + // Ensure a follow up eval was created + eval = h.Evals[0] + if eval.NextEval == "" { + t.Fatalf("missing next eval") + } - // Check for create - if len(h.CreateEvals) == 0 { - t.Fatalf("missing created eval") - } - create := h.CreateEvals[0] - if eval.NextEval != create.ID { - t.Fatalf("ID mismatch") - } - if create.PreviousEval != eval.ID { - t.Fatalf("missing previous eval") - } + // Check for create + if len(h.CreateEvals) == 0 { + t.Fatalf("missing created eval") + } + create := h.CreateEvals[0] + if eval.NextEval != create.ID { + t.Fatalf("ID mismatch") + } + if create.PreviousEval != eval.ID { + t.Fatalf("missing previous eval") + } - if create.TriggeredBy != structs.EvalTriggerRollingUpdate { - t.Fatalf("bad: %#v", create) + if create.TriggeredBy != structs.EvalTriggerRollingUpdate { + t.Fatalf("bad: %#v", create) + } + }) } } func TestSystemSched_JobModify_InPlace(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Generate a fake job with allocations - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Update the job - job2 := mock.SystemJob() - job2.ID = job.ID - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan did not evict any allocs - var update []*structs.Allocation - for _, updateList := range plan.NodeUpdate { - update = append(update, updateList...) - } - if len(update) != 0 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan updated the existing allocs - var planned []*structs.Allocation - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - } - if len(planned) != 10 { - t.Fatalf("bad: %#v", plan) - } - for _, p := range planned { - if p.Job != job2 { - t.Fatalf("should update job") - } - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - if len(out) != 10 { - t.Fatalf("bad: %#v", out) - } - h.AssertEvalStatus(t, structs.EvalStatusComplete) - - // Verify the network did not change - rp := structs.Port{Label: "admin", Value: 5000} - for _, alloc := range out { - for _, resources := range alloc.TaskResources { - if resources.Networks[0].ReservedPorts[0] != rp { - t.Fatalf("bad: %#v", alloc) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) } - } + + // Generate a fake job with allocations + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Update the job + job2 := mock.SystemJob() + job2.ID = job.ID + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan did not evict any allocs + var update []*structs.Allocation + for _, updateList := range plan.NodeUpdate { + update = append(update, updateList...) + } + if len(update) != 0 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan updated the existing allocs + var planned []*structs.Allocation + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + } + if len(planned) != 10 { + t.Fatalf("bad: %#v", plan) + } + for _, p := range planned { + if p.Job != job2 { + t.Fatalf("should update job") + } + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + if len(out) != 10 { + t.Fatalf("bad: %#v", out) + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) + + // Verify the network did not change + rp := structs.Port{Label: "admin", Value: 5000} + for _, alloc := range out { + for _, resources := range alloc.TaskResources { + if resources.Networks[0].ReservedPorts[0] != rp { + t.Fatalf("bad: %#v", alloc) + } + } + } + }) } } func TestSystemSched_JobDeregister_Purged(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations + job := mock.SystemJob() + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + for _, alloc := range allocs { + noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted the job from all nodes. + for _, node := range nodes { + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - // Generate a fake job with allocations - job := mock.SystemJob() - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - for _, alloc := range allocs { - noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted the job from all nodes. - for _, node := range nodes { - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_JobDeregister_Stopped(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - var nodes []*structs.Node - for i := 0; i < 10; i++ { - node := mock.Node() - nodes = append(nodes, node) - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create some nodes + var nodes []*structs.Node + for i := 0; i < 10; i++ { + node := mock.Node() + nodes = append(nodes, node) + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Generate a fake job with allocations + job := mock.SystemJob() + job.Stop = true + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + var allocs []*structs.Allocation + for _, node := range nodes { + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + allocs = append(allocs, alloc) + } + for _, alloc := range allocs { + noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerJobDeregister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted the job from all nodes. + for _, node := range nodes { + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure no remaining allocations + out, _ = structs.FilterTerminalAllocs(out) + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - // Generate a fake job with allocations - job := mock.SystemJob() - job.Stop = true - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - var allocs []*structs.Allocation - for _, node := range nodes { - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - allocs = append(allocs, alloc) - } - for _, alloc := range allocs { - noErr(t, h.State.UpsertJobSummary(h.NextIndex(), mock.JobSummary(alloc.JobID))) - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), allocs)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerJobDeregister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted the job from all nodes. - for _, node := range nodes { - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure no remaining allocations - out, _ = structs.FilterTerminalAllocs(out) - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_NodeDown(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Register a down node - node := mock.Node() - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Register a down node + node := mock.Node() + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan updated the allocation. + var planned []*structs.Allocation + for _, allocList := range plan.NodeUpdate { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the allocations is stopped + if p := planned[0]; p.DesiredDescription != allocNodeTainted { + t.Fatalf("bad: %#v", planned[0]) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan updated the allocation. - var planned []*structs.Allocation - for _, allocList := range plan.NodeUpdate { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the allocations is stopped - if p := planned[0]; p.DesiredStatus != structs.AllocDesiredStatusStop && - p.ClientStatus != structs.AllocClientStatusLost { - t.Fatalf("bad: %#v", planned[0]) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_NodeDrain_Down(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Register a draining node - node := mock.Node() - node.Drain = true - node.Status = structs.NodeStatusDown - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Register a draining node + node := mock.Node() + node.Drain = true + node.Status = structs.NodeStatusDown + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - // Create a mock evaluation to deal with the node update - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, + // Create a mock evaluation to deal with the node update + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted non terminal allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure that the allocation is marked as lost + var lostAllocs []string + for _, alloc := range plan.NodeUpdate[node.ID] { + lostAllocs = append(lostAllocs, alloc.ID) + } + expected := []string{alloc.ID} + + if !reflect.DeepEqual(lostAllocs, expected) { + t.Fatalf("expected: %v, actual: %v", expected, lostAllocs) + } + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted non terminal allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure that the allocation is marked as lost - var lostAllocs []string - for _, alloc := range plan.NodeUpdate[node.ID] { - lostAllocs = append(lostAllocs, alloc.ID) - } - expected := []string{alloc.ID} - - if !reflect.DeepEqual(lostAllocs, expected) { - t.Fatalf("expected: %v, actual: %v", expected, lostAllocs) - } - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_NodeDrain(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Register a draining node - node := mock.Node() - node.Drain = true - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Register a draining node + node := mock.Node() + node.Drain = true + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted all allocs + if len(plan.NodeUpdate[node.ID]) != 1 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan updated the allocation. + var planned []*structs.Allocation + for _, allocList := range plan.NodeUpdate { + planned = append(planned, allocList...) + } + if len(planned) != 1 { + t.Log(len(planned)) + t.Fatalf("bad: %#v", plan) + } + + // Ensure the allocations is stopped + if planned[0].DesiredDescription != allocNodeTainted { + t.Fatalf("bad: %#v", planned[0]) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted all allocs - if len(plan.NodeUpdate[node.ID]) != 1 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan updated the allocation. - var planned []*structs.Allocation - for _, allocList := range plan.NodeUpdate { - planned = append(planned, allocList...) - } - if len(planned) != 1 { - t.Log(len(planned)) - t.Fatalf("bad: %#v", plan) - } - - // Ensure the allocations is stopped - if planned[0].DesiredStatus != structs.AllocDesiredStatusStop { - t.Fatalf("bad: %#v", planned[0]) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_NodeUpdate(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Register a node - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Register a node + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Generate a fake job allocated on that node. - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a fake job allocated on that node. + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc})) - // Create a mock evaluation to deal - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, + // Create a mock evaluation to deal + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure that queued allocations is zero + if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { + t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure that queued allocations is zero - if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { - t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_RetryLimit(t *testing.T) { - h := NewHarness(t) - h.Planner = &RejectPlan{h} + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) + h.Planner = &RejectPlan{h} - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deregister the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure multiple plans + if len(h.Plans) == 0 { + t.Fatalf("bad: %#v", h.Plans) + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure no allocations placed + if len(out) != 0 { + t.Fatalf("bad: %#v", out) + } + + // Should hit the retry limit + h.AssertEvalStatus(t, structs.EvalStatusFailed) + }) } - - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deregister the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure multiple plans - if len(h.Plans) == 0 { - t.Fatalf("bad: %#v", h.Plans) - } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure no allocations placed - if len(out) != 0 { - t.Fatalf("bad: %#v", out) - } - - // Should hit the retry limit - h.AssertEvalStatus(t, structs.EvalStatusFailed) } // This test ensures that the scheduler doesn't increment the queued allocation // count for a task group when allocations can't be created on currently // available nodes because of constrain mismatches. func TestSystemSched_Queued_With_Constraints(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Register a node - node := mock.Node() - node.Attributes["kernel.name"] = "darwin" - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Register a node + node := mock.Node() + node.Attributes["kernel.name"] = "darwin" + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - // Generate a system job which can't be placed on the node - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + // Generate a system job which can't be placed on the node + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Create a mock evaluation to deal - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Create a mock evaluation to deal + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } - // Ensure that queued allocations is zero - if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { - t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) + // Ensure that queued allocations is zero + if val, ok := h.Evals[0].QueuedAllocations["web"]; !ok || val != 0 { + t.Fatalf("bad queued allocations: %#v", h.Evals[0].QueuedAllocations) + } + }) } } func TestSystemSched_ChainedAlloc(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create some nodes - for i := 0; i < 10; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a job - job := mock.SystemJob() - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Process the evaluation - if err := h.Process(NewSystemScheduler, eval); err != nil { - t.Fatalf("err: %v", err) - } - - var allocIDs []string - for _, allocList := range h.Plans[0].NodeAllocation { - for _, alloc := range allocList { - allocIDs = append(allocIDs, alloc.ID) - } - } - sort.Strings(allocIDs) - - // Create a new harness to invoke the scheduler again - h1 := NewHarnessWithState(t, h.State) - job1 := mock.SystemJob() - job1.ID = job.ID - job1.TaskGroups[0].Tasks[0].Env = make(map[string]string) - job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" - noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) - - // Insert two more nodes - for i := 0; i < 2; i++ { - node := mock.Node() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - } - - // Create a mock evaluation to update the job - eval1 := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job1.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job1.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) - // Process the evaluation - if err := h1.Process(NewSystemScheduler, eval1); err != nil { - t.Fatalf("err: %v", err) - } - - plan := h1.Plans[0] - - // Collect all the chained allocation ids and the new allocations which - // don't have any chained allocations - var prevAllocs []string - var newAllocs []string - for _, allocList := range plan.NodeAllocation { - for _, alloc := range allocList { - if alloc.PreviousAllocation == "" { - newAllocs = append(newAllocs, alloc.ID) - continue + // Create some nodes + for i := 0; i < 10; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) } - prevAllocs = append(prevAllocs, alloc.PreviousAllocation) - } - } - sort.Strings(prevAllocs) - // Ensure that the new allocations has their corresponding original - // allocation ids - if !reflect.DeepEqual(prevAllocs, allocIDs) { - t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) - } + // Create a job + job := mock.SystemJob() + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensuring two new allocations don't have any chained allocations - if len(newAllocs) != 2 { - t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + // Process the evaluation + if err := h.Process(NewSystemScheduler, eval); err != nil { + t.Fatalf("err: %v", err) + } + + var allocIDs []string + for _, allocList := range h.Plans[0].NodeAllocation { + for _, alloc := range allocList { + allocIDs = append(allocIDs, alloc.ID) + } + } + sort.Strings(allocIDs) + + // Create a new harness to invoke the scheduler again + h1 := NewHarnessWithState(t, h.State, allowPlanOptimization) + job1 := mock.SystemJob() + job1.ID = job.ID + job1.TaskGroups[0].Tasks[0].Env = make(map[string]string) + job1.TaskGroups[0].Tasks[0].Env["foo"] = "bar" + noErr(t, h1.State.UpsertJob(h1.NextIndex(), job1)) + + // Insert two more nodes + for i := 0; i < 2; i++ { + node := mock.Node() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + } + + // Create a mock evaluation to update the job + eval1 := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job1.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job1.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval1})) + // Process the evaluation + if err := h1.Process(NewSystemScheduler, eval1); err != nil { + t.Fatalf("err: %v", err) + } + + plan := h1.Plans[0] + + // Collect all the chained allocation ids and the new allocations which + // don't have any chained allocations + var prevAllocs []string + var newAllocs []string + for _, allocList := range plan.NodeAllocation { + for _, alloc := range allocList { + if alloc.PreviousAllocation == "" { + newAllocs = append(newAllocs, alloc.ID) + continue + } + prevAllocs = append(prevAllocs, alloc.PreviousAllocation) + } + } + sort.Strings(prevAllocs) + + // Ensure that the new allocations has their corresponding original + // allocation ids + if !reflect.DeepEqual(prevAllocs, allocIDs) { + t.Fatalf("expected: %v, actual: %v", len(allocIDs), len(prevAllocs)) + } + + // Ensuring two new allocations don't have any chained allocations + if len(newAllocs) != 2 { + t.Fatalf("expected: %v, actual: %v", 2, len(newAllocs)) + } + }) } } func TestSystemSched_PlanWithDrainedNode(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Register two nodes with two different classes - node := mock.Node() - node.NodeClass = "green" - node.Drain = true - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Register two nodes with two different classes + node := mock.Node() + node.NodeClass = "green" + node.Drain = true + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - node2 := mock.Node() - node2.NodeClass = "blue" - node2.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + node2 := mock.Node() + node2.NodeClass = "blue" + node2.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - // Create a Job with two task groups, each constrained on node class - job := mock.SystemJob() - tg1 := job.TaskGroups[0] - tg1.Constraints = append(tg1.Constraints, - &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "green", - Operand: "==", + // Create a Job with two task groups, each constrained on node class + job := mock.SystemJob() + tg1 := job.TaskGroups[0] + tg1.Constraints = append(tg1.Constraints, + &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "green", + Operand: "==", + }) + + tg2 := tg1.Copy() + tg2.Name = "web2" + tg2.Constraints[0].RTarget = "blue" + job.TaskGroups = append(job.TaskGroups, tg2) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create an allocation on each node + alloc := mock.Alloc() + alloc.Job = job + alloc.JobID = job.ID + alloc.NodeID = node.ID + alloc.Name = "my-job.web[0]" + alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) + alloc.TaskGroup = "web" + + alloc2 := mock.Alloc() + alloc2.Job = job + alloc2.JobID = job.ID + alloc2.NodeID = node2.ID + alloc2.Name = "my-job.web2[0]" + alloc2.TaskGroup = "web2" + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc, alloc2})) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + plan := h.Plans[0] + + // Ensure the plan evicted the alloc on the failed node + planned := plan.NodeUpdate[node.ID] + if len(planned) != 1 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the plan didn't place + if len(plan.NodeAllocation) != 0 { + t.Fatalf("bad: %#v", plan) + } + + // Ensure the allocations is stopped + if planned[0].DesiredDescription != allocNodeTainted { + t.Fatalf("bad: %#v", planned[0]) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) }) - - tg2 := tg1.Copy() - tg2.Name = "web2" - tg2.Constraints[0].RTarget = "blue" - job.TaskGroups = append(job.TaskGroups, tg2) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create an allocation on each node - alloc := mock.Alloc() - alloc.Job = job - alloc.JobID = job.ID - alloc.NodeID = node.ID - alloc.Name = "my-job.web[0]" - alloc.DesiredTransition.Migrate = helper.BoolToPtr(true) - alloc.TaskGroup = "web" - - alloc2 := mock.Alloc() - alloc2.Job = job - alloc2.JobID = job.ID - alloc2.NodeID = node2.ID - alloc2.Name = "my-job.web2[0]" - alloc2.TaskGroup = "web2" - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc, alloc2})) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - plan := h.Plans[0] - - // Ensure the plan evicted the alloc on the failed node - planned := plan.NodeUpdate[node.ID] - if len(planned) != 1 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the plan didn't place - if len(plan.NodeAllocation) != 0 { - t.Fatalf("bad: %#v", plan) - } - - // Ensure the allocations is stopped - if planned[0].DesiredStatus != structs.AllocDesiredStatusStop { - t.Fatalf("bad: %#v", planned[0]) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_QueuedAllocsMultTG(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Register two nodes with two different classes - node := mock.Node() - node.NodeClass = "green" - node.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + // Register two nodes with two different classes + node := mock.Node() + node.NodeClass = "green" + node.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - node2 := mock.Node() - node2.NodeClass = "blue" - node2.ComputeClass() - noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) + node2 := mock.Node() + node2.NodeClass = "blue" + node2.ComputeClass() + noErr(t, h.State.UpsertNode(h.NextIndex(), node2)) - // Create a Job with two task groups, each constrained on node class - job := mock.SystemJob() - tg1 := job.TaskGroups[0] - tg1.Constraints = append(tg1.Constraints, - &structs.Constraint{ - LTarget: "${node.class}", - RTarget: "green", - Operand: "==", + // Create a Job with two task groups, each constrained on node class + job := mock.SystemJob() + tg1 := job.TaskGroups[0] + tg1.Constraints = append(tg1.Constraints, + &structs.Constraint{ + LTarget: "${node.class}", + RTarget: "green", + Operand: "==", + }) + + tg2 := tg1.Copy() + tg2.Name = "web2" + tg2.Constraints[0].RTarget = "blue" + job.TaskGroups = append(job.TaskGroups, tg2) + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to deal with drain + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: 50, + TriggeredBy: structs.EvalTriggerNodeUpdate, + JobID: job.ID, + NodeID: node.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure a single plan + if len(h.Plans) != 1 { + t.Fatalf("bad: %#v", h.Plans) + } + + qa := h.Evals[0].QueuedAllocations + if qa["web"] != 0 || qa["web2"] != 0 { + t.Fatalf("bad queued allocations %#v", qa) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) }) - - tg2 := tg1.Copy() - tg2.Name = "web2" - tg2.Constraints[0].RTarget = "blue" - job.TaskGroups = append(job.TaskGroups, tg2) - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to deal with drain - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: 50, - TriggeredBy: structs.EvalTriggerNodeUpdate, - JobID: job.ID, - NodeID: node.ID, - Status: structs.EvalStatusPending, } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - if err != nil { - t.Fatalf("err: %v", err) - } - - // Ensure a single plan - if len(h.Plans) != 1 { - t.Fatalf("bad: %#v", h.Plans) - } - - qa := h.Evals[0].QueuedAllocations - if qa["web"] != 0 || qa["web2"] != 0 { - t.Fatalf("bad queued allocations %#v", qa) - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) } func TestSystemSched_Preemption(t *testing.T) { - h := NewHarness(t) + for _, allowPlanOptimization := range []bool{true, false} { + t.Run(IsPlanOptimizedStr(allowPlanOptimization), func(t *testing.T) { + h := NewHarness(t, allowPlanOptimization) - // Create nodes - var nodes []*structs.Node - for i := 0; i < 2; i++ { - node := mock.Node() - //TODO(preetha): remove in 0.11 - node.Resources = &structs.Resources{ - CPU: 3072, - MemoryMB: 5034, - DiskMB: 20 * 1024, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - CIDR: "192.168.0.100/32", - MBits: 1000, - }, - }, - } - node.NodeResources = &structs.NodeResources{ - Cpu: structs.NodeCpuResources{ - CpuShares: 3072, - }, - Memory: structs.NodeMemoryResources{ - MemoryMB: 5034, - }, - Disk: structs.NodeDiskResources{ - DiskMB: 20 * 1024, - }, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - CIDR: "192.168.0.100/32", - MBits: 1000, - }, - }, - } - noErr(t, h.State.UpsertNode(h.NextIndex(), node)) - nodes = append(nodes, node) - } - - // Enable Preemption - h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ - PreemptionConfig: structs.PreemptionConfig{ - SystemSchedulerEnabled: true, - }, - }) - - // Create some low priority batch jobs and allocations for them - // One job uses a reserved port - job1 := mock.BatchJob() - job1.Type = structs.JobTypeBatch - job1.Priority = 20 - job1.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 512, - MemoryMB: 1024, - Networks: []*structs.NetworkResource{ - { - MBits: 200, - ReservedPorts: []structs.Port{ - { - Label: "web", - Value: 80, + // Create nodes + var nodes []*structs.Node + for i := 0; i < 2; i++ { + node := mock.Node() + //TODO(preetha): remove in 0.11 + node.Resources = &structs.Resources{ + CPU: 3072, + MemoryMB: 5034, + DiskMB: 20 * 1024, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + CIDR: "192.168.0.100/32", + MBits: 1000, + }, }, - }, - }, - }, - } + } + node.NodeResources = &structs.NodeResources{ + Cpu: structs.NodeCpuResources{ + CpuShares: 3072, + }, + Memory: structs.NodeMemoryResources{ + MemoryMB: 5034, + }, + Disk: structs.NodeDiskResources{ + DiskMB: 20 * 1024, + }, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + CIDR: "192.168.0.100/32", + MBits: 1000, + }, + }, + } + noErr(t, h.State.UpsertNode(h.NextIndex(), node)) + nodes = append(nodes, node) + } - alloc1 := mock.Alloc() - alloc1.Job = job1 - alloc1.JobID = job1.ID - alloc1.NodeID = nodes[0].ID - alloc1.Name = "my-job[0]" - alloc1.TaskGroup = job1.TaskGroups[0].Name - alloc1.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 512, - }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 1024, + // Enable Preemption + h.State.SchedulerSetConfig(h.NextIndex(), &structs.SchedulerConfiguration{ + PreemptionConfig: structs.PreemptionConfig{ + SystemSchedulerEnabled: true, }, + }) + + // Create some low priority batch jobs and allocations for them + // One job uses a reserved port + job1 := mock.BatchJob() + job1.Type = structs.JobTypeBatch + job1.Priority = 20 + job1.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 512, + MemoryMB: 1024, Networks: []*structs.NetworkResource{ { - Device: "eth0", - IP: "192.168.0.100", - ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, - MBits: 200, + MBits: 200, + ReservedPorts: []structs.Port{ + { + Label: "web", + Value: 80, + }, + }, }, }, - }, - }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 5 * 1024, - }, - } + } - noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) - - job2 := mock.BatchJob() - job2.Type = structs.JobTypeBatch - job2.Priority = 20 - job2.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 512, - MemoryMB: 1024, - Networks: []*structs.NetworkResource{ - { - MBits: 200, - }, - }, - } - - alloc2 := mock.Alloc() - alloc2.Job = job2 - alloc2.JobID = job2.ID - alloc2.NodeID = nodes[0].ID - alloc2.Name = "my-job[2]" - alloc2.TaskGroup = job2.TaskGroups[0].Name - alloc2.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 512, + alloc1 := mock.Alloc() + alloc1.Job = job1 + alloc1.JobID = job1.ID + alloc1.NodeID = nodes[0].ID + alloc1.Name = "my-job[0]" + alloc1.TaskGroup = job1.TaskGroups[0].Name + alloc1.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 512, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 1024, + }, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + IP: "192.168.0.100", + ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, + MBits: 200, + }, + }, + }, }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 1024, + Shared: structs.AllocatedSharedResources{ + DiskMB: 5 * 1024, }, + } + + noErr(t, h.State.UpsertJob(h.NextIndex(), job1)) + + job2 := mock.BatchJob() + job2.Type = structs.JobTypeBatch + job2.Priority = 20 + job2.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 512, + MemoryMB: 1024, + Networks: []*structs.NetworkResource{ + { + MBits: 200, + }, + }, + } + + alloc2 := mock.Alloc() + alloc2.Job = job2 + alloc2.JobID = job2.ID + alloc2.NodeID = nodes[0].ID + alloc2.Name = "my-job[2]" + alloc2.TaskGroup = job2.TaskGroups[0].Name + alloc2.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 512, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 1024, + }, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + IP: "192.168.0.100", + MBits: 200, + }, + }, + }, + }, + Shared: structs.AllocatedSharedResources{ + DiskMB: 5 * 1024, + }, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) + + job3 := mock.Job() + job3.Type = structs.JobTypeBatch + job3.Priority = 40 + job3.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 1024, + MemoryMB: 2048, Networks: []*structs.NetworkResource{ { Device: "eth0", - IP: "192.168.0.100", - MBits: 200, + MBits: 400, }, }, - }, - }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 5 * 1024, - }, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job2)) - - job3 := mock.Job() - job3.Type = structs.JobTypeBatch - job3.Priority = 40 - job3.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 1024, - MemoryMB: 2048, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - MBits: 400, - }, - }, - } - - alloc3 := mock.Alloc() - alloc3.Job = job3 - alloc3.JobID = job3.ID - alloc3.NodeID = nodes[0].ID - alloc3.Name = "my-job[0]" - alloc3.TaskGroup = job3.TaskGroups[0].Name - alloc3.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 1024, - }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 25, - }, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - IP: "192.168.0.100", - ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, - MBits: 400, - }, - }, - }, - }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 5 * 1024, - }, - } - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc1, alloc2, alloc3})) - - // Create a high priority job and allocs for it - // These allocs should not be preempted - - job4 := mock.BatchJob() - job4.Type = structs.JobTypeBatch - job4.Priority = 100 - job4.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 1024, - MemoryMB: 2048, - Networks: []*structs.NetworkResource{ - { - MBits: 100, - }, - }, - } - - alloc4 := mock.Alloc() - alloc4.Job = job4 - alloc4.JobID = job4.ID - alloc4.NodeID = nodes[0].ID - alloc4.Name = "my-job4[0]" - alloc4.TaskGroup = job4.TaskGroups[0].Name - alloc4.AllocatedResources = &structs.AllocatedResources{ - Tasks: map[string]*structs.AllocatedTaskResources{ - "web": { - Cpu: structs.AllocatedCpuResources{ - CpuShares: 1024, - }, - Memory: structs.AllocatedMemoryResources{ - MemoryMB: 2048, - }, - Networks: []*structs.NetworkResource{ - { - Device: "eth0", - IP: "192.168.0.100", - ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, - MBits: 100, - }, - }, - }, - }, - Shared: structs.AllocatedSharedResources{ - DiskMB: 2 * 1024, - }, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job4)) - noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc4})) - - // Create a system job such that it would need to preempt both allocs to succeed - job := mock.SystemJob() - job.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ - CPU: 1948, - MemoryMB: 256, - Networks: []*structs.NetworkResource{ - { - MBits: 800, - DynamicPorts: []structs.Port{{Label: "http"}}, - }, - }, - } - noErr(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, - } - noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - - // Process the evaluation - err := h.Process(NewSystemScheduler, eval) - require := require.New(t) - require.Nil(err) - - // Ensure a single plan - require.Equal(1, len(h.Plans)) - plan := h.Plans[0] - - // Ensure the plan doesn't have annotations. - require.Nil(plan.Annotations) - - // Ensure the plan allocated on both nodes - var planned []*structs.Allocation - preemptingAllocId := "" - require.Equal(2, len(plan.NodeAllocation)) - - // The alloc that got placed on node 1 is the preemptor - for _, allocList := range plan.NodeAllocation { - planned = append(planned, allocList...) - for _, alloc := range allocList { - if alloc.NodeID == nodes[0].ID { - preemptingAllocId = alloc.ID } - } + + alloc3 := mock.Alloc() + alloc3.Job = job3 + alloc3.JobID = job3.ID + alloc3.NodeID = nodes[0].ID + alloc3.Name = "my-job[0]" + alloc3.TaskGroup = job3.TaskGroups[0].Name + alloc3.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 1024, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 25, + }, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + IP: "192.168.0.100", + ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, + MBits: 400, + }, + }, + }, + }, + Shared: structs.AllocatedSharedResources{ + DiskMB: 5 * 1024, + }, + } + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc1, alloc2, alloc3})) + + // Create a high priority job and allocs for it + // These allocs should not be preempted + + job4 := mock.BatchJob() + job4.Type = structs.JobTypeBatch + job4.Priority = 100 + job4.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 1024, + MemoryMB: 2048, + Networks: []*structs.NetworkResource{ + { + MBits: 100, + }, + }, + } + + alloc4 := mock.Alloc() + alloc4.Job = job4 + alloc4.JobID = job4.ID + alloc4.NodeID = nodes[0].ID + alloc4.Name = "my-job4[0]" + alloc4.TaskGroup = job4.TaskGroups[0].Name + alloc4.AllocatedResources = &structs.AllocatedResources{ + Tasks: map[string]*structs.AllocatedTaskResources{ + "web": { + Cpu: structs.AllocatedCpuResources{ + CpuShares: 1024, + }, + Memory: structs.AllocatedMemoryResources{ + MemoryMB: 2048, + }, + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + IP: "192.168.0.100", + ReservedPorts: []structs.Port{{Label: "web", Value: 80}}, + MBits: 100, + }, + }, + }, + }, + Shared: structs.AllocatedSharedResources{ + DiskMB: 2 * 1024, + }, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job4)) + noErr(t, h.State.UpsertAllocs(h.NextIndex(), []*structs.Allocation{alloc4})) + + // Create a system job such that it would need to preempt both allocs to succeed + job := mock.SystemJob() + job.TaskGroups[0].Tasks[0].Resources = &structs.Resources{ + CPU: 1948, + MemoryMB: 256, + Networks: []*structs.NetworkResource{ + { + MBits: 800, + DynamicPorts: []structs.Port{{Label: "http"}}, + }, + }, + } + noErr(t, h.State.UpsertJob(h.NextIndex(), job)) + + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + noErr(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + + // Process the evaluation + err := h.Process(NewSystemScheduler, eval) + require := require.New(t) + require.Nil(err) + + // Ensure a single plan + require.Equal(1, len(h.Plans)) + plan := h.Plans[0] + + // Ensure the plan doesn't have annotations. + require.Nil(plan.Annotations) + + // Ensure the plan allocated on both nodes + var planned []*structs.Allocation + preemptingAllocId := "" + require.Equal(2, len(plan.NodeAllocation)) + + // The alloc that got placed on node 1 is the preemptor + for _, allocList := range plan.NodeAllocation { + planned = append(planned, allocList...) + for _, alloc := range allocList { + if alloc.NodeID == nodes[0].ID { + preemptingAllocId = alloc.ID + } + } + } + + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + noErr(t, err) + + // Ensure all allocations placed + require.Equal(2, len(out)) + + // Verify that one node has preempted allocs + require.NotNil(plan.NodePreemptions[nodes[0].ID]) + preemptedAllocs := plan.NodePreemptions[nodes[0].ID] + + // Verify that three jobs have preempted allocs + require.Equal(3, len(preemptedAllocs)) + + expectedPreemptedAllocIDs := []string{alloc1.ID, alloc2.ID, alloc3.ID} + + // We expect job1, job2 and job3 to have preempted allocations + // job4 should not have any allocs preempted + for _, alloc := range preemptedAllocs { + require.Contains(expectedPreemptedAllocIDs, alloc.ID) + } + // Look up the preempted allocs by job ID + ws = memdb.NewWatchSet() + + for _, allocID := range expectedPreemptedAllocIDs { + evictedAlloc, err := h.State.AllocByID(ws, allocID) + noErr(t, err) + require.Equal(structs.AllocDesiredStatusEvict, evictedAlloc.DesiredStatus) + require.Equal(fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocId), evictedAlloc.DesiredDescription) + } + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) } - - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - noErr(t, err) - - // Ensure all allocations placed - require.Equal(2, len(out)) - - // Verify that one node has preempted allocs - require.NotNil(plan.NodePreemptions[nodes[0].ID]) - preemptedAllocs := plan.NodePreemptions[nodes[0].ID] - - // Verify that three jobs have preempted allocs - require.Equal(3, len(preemptedAllocs)) - - expectedPreemptedJobIDs := []string{job1.ID, job2.ID, job3.ID} - - // We expect job1, job2 and job3 to have preempted allocations - // job4 should not have any allocs preempted - for _, alloc := range preemptedAllocs { - require.Contains(expectedPreemptedJobIDs, alloc.JobID) - } - // Look up the preempted allocs by job ID - ws = memdb.NewWatchSet() - - for _, jobId := range expectedPreemptedJobIDs { - out, err = h.State.AllocsByJob(ws, structs.DefaultNamespace, jobId, false) - noErr(t, err) - for _, alloc := range out { - require.Equal(structs.AllocDesiredStatusEvict, alloc.DesiredStatus) - require.Equal(fmt.Sprintf("Preempted by alloc ID %v", preemptingAllocId), alloc.DesiredDescription) - } - } - - h.AssertEvalStatus(t, structs.EvalStatusComplete) - } diff --git a/scheduler/testing.go b/scheduler/testing.go index 61fa7f79c..dfe4f7286 100644 --- a/scheduler/testing.go +++ b/scheduler/testing.go @@ -5,9 +5,9 @@ import ( "sync" "time" - testing "github.com/mitchellh/go-testing-interface" + "github.com/mitchellh/go-testing-interface" - memdb "github.com/hashicorp/go-memdb" + "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/state" "github.com/hashicorp/nomad/nomad/structs" @@ -53,26 +53,30 @@ type Harness struct { nextIndex uint64 nextIndexLock sync.Mutex + + allowPlanOptimization bool } // NewHarness is used to make a new testing harness -func NewHarness(t testing.T) *Harness { +func NewHarness(t testing.T, allowPlanOptimization bool) *Harness { state := state.TestStateStore(t) h := &Harness{ - t: t, - State: state, - nextIndex: 1, + t: t, + State: state, + nextIndex: 1, + allowPlanOptimization: allowPlanOptimization, } return h } // NewHarnessWithState creates a new harness with the given state for testing // purposes. -func NewHarnessWithState(t testing.T, state *state.StateStore) *Harness { +func NewHarnessWithState(t testing.T, state *state.StateStore, allowPlanOptimization bool) *Harness { return &Harness{ - t: t, - State: state, - nextIndex: 1, + t: t, + State: state, + nextIndex: 1, + allowPlanOptimization: allowPlanOptimization, } } @@ -101,22 +105,17 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er result.AllocIndex = index // Flatten evicts and allocs - var allocs []*structs.Allocation + now := time.Now().UTC().UnixNano() + allocsStopped := make([]*structs.Allocation, 0, len(result.NodeUpdate)) for _, updateList := range plan.NodeUpdate { - allocs = append(allocs, updateList...) - } - for _, allocList := range plan.NodeAllocation { - allocs = append(allocs, allocList...) + allocsStopped = append(allocsStopped, updateList...) } - // Set the time the alloc was applied for the first time. This can be used - // to approximate the scheduling time. - now := time.Now().UTC().UnixNano() - for _, alloc := range allocs { - if alloc.CreateTime == 0 { - alloc.CreateTime = now - } + allocsUpdated := make([]*structs.Allocation, 0, len(result.NodeAllocation)) + for _, allocList := range plan.NodeAllocation { + allocsUpdated = append(allocsUpdated, allocList...) } + updateCreateTimestamp(allocsUpdated, now) // Set modify time for preempted allocs and flatten them var preemptedAllocs []*structs.Allocation @@ -130,8 +129,7 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er // Setup the update request req := structs.ApplyPlanResultsRequest{ AllocUpdateRequest: structs.AllocUpdateRequest{ - Job: plan.Job, - Alloc: allocs, + Job: plan.Job, }, Deployment: plan.Deployment, DeploymentUpdates: plan.DeploymentUpdates, @@ -139,11 +137,33 @@ func (h *Harness) SubmitPlan(plan *structs.Plan) (*structs.PlanResult, State, er NodePreemptions: preemptedAllocs, } + if h.allowPlanOptimization { + req.AllocsStopped = allocsStopped + req.AllocsUpdated = allocsUpdated + } else { + // Deprecated: Handles unoptimized log format + var allocs []*structs.Allocation + allocs = append(allocs, allocsStopped...) + allocs = append(allocs, allocsUpdated...) + updateCreateTimestamp(allocs, now) + req.Alloc = allocs + } + // Apply the full plan err := h.State.UpsertPlanResults(index, &req) return result, nil, err } +func updateCreateTimestamp(allocations []*structs.Allocation, now int64) { + // Set the time the alloc was applied for the first time. This can be used + // to approximate the scheduling time. + for _, alloc := range allocations { + if alloc.CreateTime == 0 { + alloc.CreateTime = now + } + } +} + func (h *Harness) UpdateEval(eval *structs.Evaluation) error { // Ensure sequential plan application h.planLock.Lock() @@ -214,15 +234,15 @@ func (h *Harness) Snapshot() State { // Scheduler is used to return a new scheduler from // a snapshot of current state using the harness for planning. -func (h *Harness) Scheduler(factory Factory) Scheduler { +func (h *Harness) Scheduler(factory Factory, allowPlanOptimization bool) Scheduler { logger := testlog.HCLogger(h.t) - return factory(logger, h.Snapshot(), h, false) + return factory(logger, h.Snapshot(), h, allowPlanOptimization) } // Process is used to process an evaluation given a factory // function to create the scheduler func (h *Harness) Process(factory Factory, eval *structs.Evaluation) error { - sched := h.Scheduler(factory) + sched := h.Scheduler(factory, h.allowPlanOptimization) return sched.Process(eval) } diff --git a/scheduler/util_test.go b/scheduler/util_test.go index 08f5812aa..13864b432 100644 --- a/scheduler/util_test.go +++ b/scheduler/util_test.go @@ -621,7 +621,7 @@ func TestEvictAndPlace_LimitEqualToAllocs(t *testing.T) { } func TestSetStatus(t *testing.T) { - h := NewHarness(t) + h := NewHarness(t, true) logger := testlog.HCLogger(t) eval := mock.Eval() status := "a" @@ -640,7 +640,7 @@ func TestSetStatus(t *testing.T) { } // Test next evals - h = NewHarness(t) + h = NewHarness(t, true) next := mock.Eval() if err := setStatus(logger, h, eval, next, nil, nil, status, desc, nil, ""); err != nil { t.Fatalf("setStatus() failed: %v", err) @@ -656,7 +656,7 @@ func TestSetStatus(t *testing.T) { } // Test blocked evals - h = NewHarness(t) + h = NewHarness(t, true) blocked := mock.Eval() if err := setStatus(logger, h, eval, nil, blocked, nil, status, desc, nil, ""); err != nil { t.Fatalf("setStatus() failed: %v", err) @@ -672,7 +672,7 @@ func TestSetStatus(t *testing.T) { } // Test metrics - h = NewHarness(t) + h = NewHarness(t, true) metrics := map[string]*structs.AllocMetric{"foo": nil} if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, nil, ""); err != nil { t.Fatalf("setStatus() failed: %v", err) @@ -688,7 +688,7 @@ func TestSetStatus(t *testing.T) { } // Test queued allocations - h = NewHarness(t) + h = NewHarness(t, true) queuedAllocs := map[string]int{"web": 1} if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, queuedAllocs, ""); err != nil { @@ -704,7 +704,7 @@ func TestSetStatus(t *testing.T) { t.Fatalf("setStatus() didn't set failed task group metrics correctly: %v", newEval) } - h = NewHarness(t) + h = NewHarness(t, true) dID := uuid.Generate() if err := setStatus(logger, h, eval, nil, nil, metrics, status, desc, queuedAllocs, dID); err != nil { t.Fatalf("setStatus() failed: %v", err)