tests: deflake deploymentwatcher package

This deflake the tests in the deploymentwatcher package.  The package
uses a mock deployment watcher backend, where the watcher in a
background goroutine calls UpdateDeploymentStatus .  If the mock isn't
configured to expect the call, the background goroutine will fail.  One
UpdateDeploymentStatus call is made at the end of the background
goroutine, which may occur after the test completes, thus explaining the
flakiness.

nomad/deploymentwatcher/deployments_watcher_test.go

@@ -33,6 +33,10 @@ func TestWatcher_WatchDeployments(t *testing.T) {
 	require := require.New(t)
 	w, m := defaultTestDeploymentWatcher(t)
 
+	m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool {
+		return true
+	})).Return(nil).Maybe()
+
 	// Create three jobs
 	j1, j2, j3 := mock.Job(), mock.Job(), mock.Job()
 	require.Nil(m.state.UpsertJob(100, j1))
@@ -86,6 +90,10 @@ func TestWatcher_UnknownDeployment(t *testing.T) {
 	w, m := defaultTestDeploymentWatcher(t)
 	w.SetEnabled(true, m.state)
 
+	m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool {
+		return true
+	})).Return(nil).Maybe()
+
 	// The expected error is that it should be an unknown deployment
 	dID := uuid.Generate()
 	expected := fmt.Sprintf("unknown deployment %q", dID)
@@ -138,6 +146,10 @@ func TestWatcher_SetAllocHealth_Unknown(t *testing.T) {
 	require := require.New(t)
 	w, m := defaultTestDeploymentWatcher(t)
 
+	m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool {
+		return true
+	})).Return(nil).Maybe()
+
 	// Create a job, and a deployment
 	j := mock.Job()
 	d := mock.Deployment()
@@ -178,6 +190,10 @@ func TestWatcher_SetAllocHealth_Healthy(t *testing.T) {
 	require := require.New(t)
 	w, m := defaultTestDeploymentWatcher(t)
 
+	m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool {
+		return true
+	})).Return(nil).Maybe()
+
 	// Create a job, alloc, and a deployment
 	j := mock.Job()
 	d := mock.Deployment()
@@ -219,6 +235,10 @@ func TestWatcher_SetAllocHealth_Unhealthy(t *testing.T) {
 	require := require.New(t)
 	w, m := defaultTestDeploymentWatcher(t)
 
+	m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool {
+		return true
+	})).Return(nil).Maybe()
+
 	// Create a job, alloc, and a deployment
 	j := mock.Job()
 	d := mock.Deployment()
@@ -267,6 +287,10 @@ func TestWatcher_SetAllocHealth_Unhealthy_Rollback(t *testing.T) {
 	require := require.New(t)
 	w, m := defaultTestDeploymentWatcher(t)
 
+	m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool {
+		return true
+	})).Return(nil).Maybe()
+
 	// Create a job, alloc, and a deployment
 	j := mock.Job()
 	j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
@@ -330,6 +354,10 @@ func TestWatcher_SetAllocHealth_Unhealthy_NoRollback(t *testing.T) {
 	require := require.New(t)
 	w, m := defaultTestDeploymentWatcher(t)
 
+	m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool {
+		return true
+	})).Return(nil).Maybe()
+
 	// Create a job, alloc, and a deployment
 	j := mock.Job()
 	j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
@@ -391,6 +419,10 @@ func TestWatcher_PromoteDeployment_HealthyCanaries(t *testing.T) {
 	require := require.New(t)
 	w, m := defaultTestDeploymentWatcher(t)
 
+	m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool {
+		return true
+	})).Return(nil).Maybe()
+
 	// Create a job, canary alloc, and a deployment
 	j := mock.Job()
 	j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
@@ -447,6 +479,10 @@ func TestWatcher_PromoteDeployment_UnhealthyCanaries(t *testing.T) {
 	require := require.New(t)
 	w, m := defaultTestDeploymentWatcher(t)
 
+	m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool {
+		return true
+	})).Return(nil).Maybe()
+
 	// Create a job, canary alloc, and a deployment
 	j := mock.Job()
 	j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
@@ -544,6 +580,10 @@ func TestWatcher_AutoPromoteDeployment(t *testing.T) {
 
 	// =============================================================
 	// Support method calls
+
+	// clear UpdateDeploymentStatus default expectation
+	m.Mock.ExpectedCalls = nil
+
 	matchConfig0 := &matchDeploymentStatusUpdateConfig{
 		DeploymentID:      d.ID,
 		Status:            structs.DeploymentStatusFailed,
@@ -620,6 +660,9 @@ func TestWatcher_PauseDeployment_Pause_Running(t *testing.T) {
 	require := require.New(t)
 	w, m := defaultTestDeploymentWatcher(t)
 
+	// clear UpdateDeploymentStatus default expectation
+	m.Mock.ExpectedCalls = nil
+
 	// Create a job and a deployment
 	j := mock.Job()
 	d := mock.Deployment()
@@ -659,6 +702,9 @@ func TestWatcher_PauseDeployment_Pause_Paused(t *testing.T) {
 	require := require.New(t)
 	w, m := defaultTestDeploymentWatcher(t)
 
+	// clear UpdateDeploymentStatus default expectation
+	m.Mock.ExpectedCalls = nil
+
 	// Create a job and a deployment
 	j := mock.Job()
 	d := mock.Deployment()
@@ -1016,6 +1062,10 @@ func TestDeploymentWatcher_ProgressCutoff(t *testing.T) {
 	require := require.New(t)
 	w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond)
 
+	m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool {
+		return true
+	})).Return(nil).Maybe()
+
 	// Create a job, alloc, and a deployment
 	j := mock.Job()
 	j.TaskGroups[0].Count = 1
@@ -1118,6 +1168,10 @@ func TestDeploymentWatcher_Watch_ProgressDeadline_Canaries(t *testing.T) {
 	require := require.New(t)
 	w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond)
 
+	m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool {
+		return true
+	})).Return(nil).Maybe()
+
 	// Create a job, alloc, and a deployment
 	j := mock.Job()
 	j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
@@ -1196,6 +1250,10 @@ func TestDeploymentWatcher_PromotedCanary_UpdatedAllocs(t *testing.T) {
 	require := require.New(t)
 	w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond)
 
+	m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool {
+		return true
+	})).Return(nil).Maybe()
+
 	// Create a job, alloc, and a deployment
 	j := mock.Job()
 	j.TaskGroups[0].Count = 2
@@ -1283,6 +1341,10 @@ func TestDeploymentWatcher_Watch_StartWithoutProgressDeadline(t *testing.T) {
 	require := require.New(t)
 	w, m := testDeploymentWatcher(t, 1000.0, 1*time.Millisecond)
 
+	m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool {
+		return true
+	})).Return(nil).Maybe()
+
 	// Create a job, and a deployment
 	j := mock.Job()
 	j.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()
@@ -1451,6 +1513,10 @@ func TestWatcher_BatchAllocUpdates(t *testing.T) {
 	require := require.New(t)
 	w, m := testDeploymentWatcher(t, 1000.0, 1*time.Second)
 
+	m.On("UpdateDeploymentStatus", mocker.MatchedBy(func(args *structs.DeploymentStatusUpdateRequest) bool {
+		return true
+	})).Return(nil).Maybe()
+
 	// Create a job, alloc, for two deployments
 	j1 := mock.Job()
 	j1.TaskGroups[0].Update = structs.DefaultUpdateStrategy.Copy()

nomad/deploymentwatcher/testutil_test.go

@@ -19,10 +19,12 @@ type mockBackend struct {
 }
 
 func newMockBackend(t *testing.T) *mockBackend {
-	return &mockBackend{
+	m := &mockBackend{
 		index: 10000,
 		state: state.TestStateStore(t),
 	}
+	m.Test(t)
+	return m
 }
 
 func (m *mockBackend) nextIndex() uint64 {