From b63d3ffc543993cd574adca87f5e3755bcf2111d Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Wed, 14 Oct 2020 11:35:30 -0400 Subject: [PATCH] e2e: eliminate race condition causing rescheduling test flake (#9085) The autorevert test checks for reverted allocations to be placed and running before checking the deployment status, but the deployment can be completed and marked "successful" before we check it for "running" status. Instead, just wait for it to be marked "successful" and assert we have the expected count of deployment statuses. --- e2e/rescheduling/rescheduling.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/e2e/rescheduling/rescheduling.go b/e2e/rescheduling/rescheduling.go index 640730ddc..258bc9f18 100644 --- a/e2e/rescheduling/rescheduling.go +++ b/e2e/rescheduling/rescheduling.go @@ -374,20 +374,29 @@ func (tc *RescheduleE2ETest) TestRescheduleMaxParallelAutoRevert(f *framework.F) "should have one successful, one failed, and 3 reverted allocs", ) + // at this point the allocs have been checked but we need to wait for the + // deployment to be marked complete before we can assert that it's successful + // and verify the count of deployments + f.NoError( + e2e.WaitForLastDeploymentStatus(jobID, ns, "successful", nil), + "most recent deployment should be successful") + out, err := e2e.Command("nomad", "deployment", "status") f.NoError(err, "could not get deployment status") results, err := e2e.ParseColumns(out) f.NoError(err, "could not parse deployment status") - statuses := []string{} + statuses := map[string]int{} for _, row := range results { if row["Job ID"] == jobID { - statuses = append(statuses, row["Status"]) + statuses[row["Status"]]++ } } - f.True(reflect.DeepEqual([]string{"running", "failed", "successful"}, statuses), - fmt.Sprintf("deployment status was: %#v", statuses), - ) + + f.Equal(1, statuses["failed"], + fmt.Sprintf("expected only 1 failed deployment, got:\n%s", out)) + f.Equal(2, statuses["successful"], + fmt.Sprintf("expected 2 successful deployments, got:\n%s", out)) } // TestRescheduleProgressDeadline verifies the progress deadline is reset with