From d430e0ff6c33aa844279abd9ee88e94f1966707e Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 12 Mar 2020 14:17:51 -0400 Subject: [PATCH] tests: deflake deploymentwatcher package This deflake the tests in the deploymentwatcher package. The package uses a mock deployment watcher backend, where the watcher in a background goroutine calls UpdateDeploymentStatus . If the mock isn't configured to expect the call, the background goroutine will fail. One UpdateDeploymentStatus call is made at the end of the background goroutine, which may occur after the test completes, thus explaining the flakiness. --- .../deployments_watcher_test.go | 66 +++++++++++++++++++ nomad/deploymentwatcher/testutil_test.go | 4 +- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/nomad/deploymentwatcher/deployments_watcher_test.go b/nomad/deploymentwatcher/deployments_watcher_test.go index eec8b1c95..78c7aeb0a 100644 --- a/nomad/deploymentwatcher/deployments_watcher_test.go +++ b/nomad/deploymentwatcher/deployments_watcher_test.go @@ -33,6 +33,10 @@ func TestWatcher_WatchDeployments(t *testing.T) { require := require.New(t) w, m := defaultTestDeploymentWatcher(t) + m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool { + return true + })).Return(nil).Maybe() + // Create three jobs j1, j2, j3 := mock.Job(), mock.Job(), mock.Job() require.Nil(m.state.UpsertJob(100, j1)) @@ -86,6 +90,10 @@ func TestWatcher_UnknownDeployment(t *testing.T) { w, m := defaultTestDeploymentWatcher(t) w.SetEnabled(true, m.state) + m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool { + return true + })).Return(nil).Maybe() + // The expected error is that it should be an unknown deployment dID := uuid.Generate() expected := fmt.Sprintf("unknown deployment %q", dID) @@ -138,6 +146,10 @@ func TestWatcher_SetAllocHealth_Unknown(t *testing.T) { require := require.New(t) w, m := defaultTestDeploymentWatcher(t) + m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool { + return true + })).Return(nil).Maybe() + // Create a job, and a deployment j := mock.Job() d := mock.Deployment() @@ -178,6 +190,10 @@ func TestWatcher_SetAllocHealth_Healthy(t *testing.T) { require := require.New(t) w, m := defaultTestDeploymentWatcher(t) + m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool { + return true + })).Return(nil).Maybe() + // Create a job, alloc, and a deployment j := mock.Job() d := mock.Deployment() @@ -219,6 +235,10 @@ func TestWatcher_SetAllocHealth_Unhealthy(t *testing.T) { require := require.New(t) w, m := defaultTestDeploymentWatcher(t) + m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool { + return true + })).Return(nil).Maybe() + // Create a job, alloc, and a deployment j := mock.Job() d := mock.Deployment() @@ -267,6 +287,10 @@ func TestWatcher_SetAllocHealth_Unhealthy_Rollback(t *testing.T) { require := require.New(t) w, m := defaultTestDeploymentWatcher(t) + m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool { + return true + })).Return(nil).Maybe() + // Create a job, alloc, and a deployment j := mock.Job() j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() @@ -330,6 +354,10 @@ func TestWatcher_SetAllocHealth_Unhealthy_NoRollback(t *testing.T) { require := require.New(t) w, m := defaultTestDeploymentWatcher(t) + m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool { + return true + })).Return(nil).Maybe() + // Create a job, alloc, and a deployment j := mock.Job() j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() @@ -391,6 +419,10 @@ func TestWatcher_PromoteDeployment_HealthyCanaries(t *testing.T) { require := require.New(t) w, m := defaultTestDeploymentWatcher(t) + m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool { + return true + })).Return(nil).Maybe() + // Create a job, canary alloc, and a deployment j := mock.Job() j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() @@ -447,6 +479,10 @@ func TestWatcher_PromoteDeployment_UnhealthyCanaries(t *testing.T) { require := require.New(t) w, m := defaultTestDeploymentWatcher(t) + m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool { + return true + })).Return(nil).Maybe() + // Create a job, canary alloc, and a deployment j := mock.Job() j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() @@ -544,6 +580,10 @@ func TestWatcher_AutoPromoteDeployment(t *testing.T) { // ============================================================= // Support method calls + + // clear UpdateDeploymentStatus default expectation + m.Mock.ExpectedCalls = nil + matchConfig0 := &matchDeploymentStatusUpdateConfig{ DeploymentID: d.ID, Status: structs.DeploymentStatusFailed, @@ -620,6 +660,9 @@ func TestWatcher_PauseDeployment_Pause_Running(t *testing.T) { require := require.New(t) w, m := defaultTestDeploymentWatcher(t) + // clear UpdateDeploymentStatus default expectation + m.Mock.ExpectedCalls = nil + // Create a job and a deployment j := mock.Job() d := mock.Deployment() @@ -659,6 +702,9 @@ func TestWatcher_PauseDeployment_Pause_Paused(t *testing.T) { require := require.New(t) w, m := defaultTestDeploymentWatcher(t) + // clear UpdateDeploymentStatus default expectation + m.Mock.ExpectedCalls = nil + // Create a job and a deployment j := mock.Job() d := mock.Deployment() @@ -1016,6 +1062,10 @@ func TestDeploymentWatcher_ProgressCutoff(t *testing.T) { require := require.New(t) w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond) + m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool { + return true + })).Return(nil).Maybe() + // Create a job, alloc, and a deployment j := mock.Job() j.TaskGroups[0].Count = 1 @@ -1118,6 +1168,10 @@ func TestDeploymentWatcher_Watch_ProgressDeadline_Canaries(t *testing.T) { require := require.New(t) w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond) + m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool { + return true + })).Return(nil).Maybe() + // Create a job, alloc, and a deployment j := mock.Job() j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() @@ -1196,6 +1250,10 @@ func TestDeploymentWatcher_PromotedCanary_UpdatedAllocs(t *testing.T) { require := require.New(t) w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond) + m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool { + return true + })).Return(nil).Maybe() + // Create a job, alloc, and a deployment j := mock.Job() j.TaskGroups[0].Count = 2 @@ -1283,6 +1341,10 @@ func TestDeploymentWatcher_Watch_StartWithoutProgressDeadline(t *testing.T) { require := require.New(t) w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond) + m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool { + return true + })).Return(nil).Maybe() + // Create a job, and a deployment j := mock.Job() j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() @@ -1451,6 +1513,10 @@ func TestWatcher_BatchAllocUpdates(t *testing.T) { require := require.New(t) w, m := testDeploymentWatcher(t, 1000.0, 1*time.Second) + m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool { + return true + })).Return(nil).Maybe() + // Create a job, alloc, for two deployments j1 := mock.Job() j1.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy() diff --git a/nomad/deploymentwatcher/testutil_test.go b/nomad/deploymentwatcher/testutil_test.go index 73ea07ae0..0542c4469 100644 --- a/nomad/deploymentwatcher/testutil_test.go +++ b/nomad/deploymentwatcher/testutil_test.go @@ -19,10 +19,12 @@ type mockBackend struct { } func newMockBackend(t *testing.T) *mockBackend { - return &mockBackend{ + m := &mockBackend{ index: 10000, state: state.TestStateStore(t), } + m.Test(t) + return m } func (m *mockBackend) nextIndex() uint64 {