From 0dea3a39b00e078a08215f65c2a853fe3879b4f2 Mon Sep 17 00:00:00 2001 From: Jasmine Dahilig Date: Thu, 9 Jan 2020 11:34:46 -0800 Subject: [PATCH] add test cases for scheduler alloc placement with lifecycle resources --- nomad/mock/mock.go | 99 +++++++++++++++++++++++- scheduler/generic_sched_test.go | 128 ++++++++++++++++++++++++-------- 2 files changed, 192 insertions(+), 35 deletions(-) diff --git a/nomad/mock/mock.go b/nomad/mock/mock.go index ce1d3d874..3fe0ee374 100644 --- a/nomad/mock/mock.go +++ b/nomad/mock/mock.go @@ -269,6 +269,96 @@ func Job() *structs.Job { job.Canonicalize() return job } + +func LifecycleSideTask(resources structs.Resources) *structs.Task { + return &structs.Task{ + Name: "sidecar", + Driver: "exec", + Config: map[string]interface{}{ + "command": "/bin/date", + }, + Lifecycle: &structs.TaskLifecycleConfig{ + Hook: structs.TaskLifecycleHookPrestart, + BlockUntil: structs.TaskLifecycleBlockUntilRunning, + }, + LogConfig: structs.DefaultLogConfig(), + Resources: &resources, + } +} + +func LifecycleInitTask(resources structs.Resources) *structs.Task { + return &structs.Task{ + Name: "init", + Driver: "exec", + Config: map[string]interface{}{ + "command": "/bin/date", + }, + Lifecycle: &structs.TaskLifecycleConfig{ + Hook: structs.TaskLifecycleHookPrestart, + BlockUntil: structs.TaskLifecycleBlockUntilCompleted, + }, + LogConfig: structs.DefaultLogConfig(), + Resources: &resources, + } +} + +func LifecycleMainTask(resources structs.Resources) *structs.Task { + return &structs.Task{ + Name: "main", + Driver: "exec", + Config: map[string]interface{}{ + "command": "/bin/date", + }, + LogConfig: structs.DefaultLogConfig(), + Resources: &resources, + } +} +func VariableLifecycleJob(resources structs.Resources, main int, init int, side int) *structs.Job { + tasks := []*structs.Task{} + for i := 0; i < main; i++ { + tasks = append(tasks, LifecycleMainTask(resources)) + } + for i := 0; i < init; i++ { + tasks = append(tasks, LifecycleInitTask(resources)) + } + for i := 0; i < side; i++ { + tasks = append(tasks, LifecycleSideTask(resources)) + } + job := &structs.Job{ + Region: "global", + ID: fmt.Sprintf("mock-service-%s", uuid.Generate()), + Name: "my-job", + Namespace: structs.DefaultNamespace, + Type: structs.JobTypeService, + Priority: 50, + AllAtOnce: false, + Datacenters: []string{"dc1"}, + Constraints: []*structs.Constraint{ + { + LTarget: "${attr.kernel.name}", + RTarget: "linux", + Operand: "=", + }, + }, + TaskGroups: []*structs.TaskGroup{ + { + Name: "web", + Count: 1, + Tasks: tasks, + }, + }, + Meta: map[string]string{ + "owner": "armon", + }, + Status: structs.JobStatusPending, + Version: 0, + CreateIndex: 42, + ModifyIndex: 99, + JobModifyIndex: 99, + } + job.Canonicalize() + return job +} func LifecycleJob() *structs.Job { job := &structs.Job{ Region: "global", @@ -288,7 +378,8 @@ func LifecycleJob() *structs.Job { }, TaskGroups: []*structs.TaskGroup{ { - Name: "web", + Name: "web", + Count: 1, Tasks: []*structs.Task{ { Name: "web", @@ -298,7 +389,7 @@ func LifecycleJob() *structs.Job { }, LogConfig: structs.DefaultLogConfig(), Resources: &structs.Resources{ - CPU: 500, + CPU: 1000, MemoryMB: 256, }, }, @@ -314,7 +405,7 @@ func LifecycleJob() *structs.Job { }, LogConfig: structs.DefaultLogConfig(), Resources: &structs.Resources{ - CPU: 500, + CPU: 1000, MemoryMB: 256, }, }, @@ -330,7 +421,7 @@ func LifecycleJob() *structs.Job { }, LogConfig: structs.DefaultLogConfig(), Resources: &structs.Resources{ - CPU: 500, + CPU: 1000, MemoryMB: 256, }, }, diff --git a/scheduler/generic_sched_test.go b/scheduler/generic_sched_test.go index 78f007386..012259eaf 100644 --- a/scheduler/generic_sched_test.go +++ b/scheduler/generic_sched_test.go @@ -4361,46 +4361,112 @@ func TestBatchSched_ScaleDown_SameName(t *testing.T) { } func TestGenericSched_AllocFit(t *testing.T) { + testCases := []struct { + Name string + NodeCpu int64 + TaskResources structs.Resources + MainTaskCount int + InitTaskCount int + SideTaskCount int + ShouldPlaceAlloc bool + }{ + { + Name: "simple init + sidecar", + NodeCpu: 1200, + TaskResources: structs.Resources{ + CPU: 500, + MemoryMB: 256, + }, + MainTaskCount: 1, + InitTaskCount: 1, + SideTaskCount: 1, + ShouldPlaceAlloc: true, + }, + { + Name: "too big init + sidecar", + NodeCpu: 1200, + TaskResources: structs.Resources{ + CPU: 700, + MemoryMB: 256, + }, + MainTaskCount: 1, + InitTaskCount: 1, + SideTaskCount: 1, + ShouldPlaceAlloc: false, + }, + { + Name: "many init + sidecar", + NodeCpu: 1200, + TaskResources: structs.Resources{ + CPU: 100, + MemoryMB: 100, + }, + MainTaskCount: 3, + InitTaskCount: 5, + SideTaskCount: 5, + ShouldPlaceAlloc: true, + }, - h := NewHarness(t) - node := mock.Node() - node.Resources.CPU = 1200 - require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) - - // Create a job with sidecar & init tasks - job := mock.LifecycleJob() - require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) - - // Create a mock evaluation to register the job - eval := &structs.Evaluation{ - Namespace: structs.DefaultNamespace, - ID: uuid.Generate(), - Priority: job.Priority, - TriggeredBy: structs.EvalTriggerJobRegister, - JobID: job.ID, - Status: structs.EvalStatusPending, + // TODO: This test case currently places the allocation, + // but it should not be placed. Investigate placement. + { + Name: "too many init + sidecar", + NodeCpu: 1200, + TaskResources: structs.Resources{ + CPU: 100, + MemoryMB: 100, + }, + MainTaskCount: 10, + InitTaskCount: 10, + SideTaskCount: 10, + ShouldPlaceAlloc: false, + }, } - require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) + for _, testCase := range testCases { + t.Run(testCase.Name, func(t *testing.T) { + h := NewHarness(t) + node := mock.Node() + node.NodeResources.Cpu.CpuShares = testCase.NodeCpu + require.NoError(t, h.State.UpsertNode(h.NextIndex(), node)) - // Process the evaluation - err := h.Process(NewServiceScheduler, eval) - require.NoError(t, err) + // Create a job with sidecar & init tasks + job := mock.VariableLifecycleJob(testCase.TaskResources, testCase.MainTaskCount, testCase.InitTaskCount, testCase.SideTaskCount) - fmt.Printf("%#+v\n", h.Evals[0]) + require.NoError(t, h.State.UpsertJob(h.NextIndex(), job)) - // Ensure no plan as it should be a no-op - require.Len(t, h.Plans, 1) + // Create a mock evaluation to register the job + eval := &structs.Evaluation{ + Namespace: structs.DefaultNamespace, + ID: uuid.Generate(), + Priority: job.Priority, + TriggeredBy: structs.EvalTriggerJobRegister, + JobID: job.ID, + Status: structs.EvalStatusPending, + } + require.NoError(t, h.State.UpsertEvals(h.NextIndex(), []*structs.Evaluation{eval})) - // Lookup the allocations by JobID - ws := memdb.NewWatchSet() - out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) - require.NoError(t, err) + // Process the evaluation + err := h.Process(NewServiceScheduler, eval) + require.NoError(t, err) - // Ensure no allocations placed - require.Len(t, out, 1) + allocs := 0 + if testCase.ShouldPlaceAlloc { + allocs = 1 + } + // Ensure no plan as it should be a no-op + require.Len(t, h.Plans, allocs) - h.AssertEvalStatus(t, structs.EvalStatusComplete) + // Lookup the allocations by JobID + ws := memdb.NewWatchSet() + out, err := h.State.AllocsByJob(ws, job.Namespace, job.ID, false) + require.NoError(t, err) + // Ensure no allocations placed + require.Len(t, out, allocs) + + h.AssertEvalStatus(t, structs.EvalStatusComplete) + }) + } } func TestGenericSched_ChainedAlloc(t *testing.T) {