From b91d0e73cb7d3a3cec6982d97b52044638858f4d Mon Sep 17 00:00:00 2001
From: Tim Gross <tgross@hashicorp.com>
Date: Mon, 4 Apr 2022 09:44:55 -0400
Subject: [PATCH] E2E: ensure that CSI EBS tests are isolated from each other
 (#12443)

Tear down the volume-consuming job between subtests, rather than after
all the tests are complete. For good measure, use a different ID for
the volume-consuming job as well.
---
 e2e/csi/ebs.go | 30 +++++++++++++++++++-----------
 1 file changed, 19 insertions(+), 11 deletions(-)

diff --git a/e2e/csi/ebs.go b/e2e/csi/ebs.go
index 6397657fe..24e9ff097 100644
--- a/e2e/csi/ebs.go
+++ b/e2e/csi/ebs.go
@@ -76,8 +76,16 @@ func (tc *CSIControllerPluginEBSTest) BeforeAll(f *framework.F) {
 	tc.volumeIDs = append(tc.volumeIDs, volID)
 }
 
-// AfterAll cleans up the volumes and plugin jobs created by the test.
-func (tc *CSIControllerPluginEBSTest) AfterAll(f *framework.F) {
+func (tc *CSIControllerPluginEBSTest) AfterEach(f *framework.F) {
+
+	// Ensure nodes are all restored
+	for _, id := range tc.nodeIDs {
+		_, err := e2eutil.Command("nomad", "node", "drain", "-disable", "-yes", id)
+		f.Assert().NoError(err)
+		_, err = e2eutil.Command("nomad", "node", "eligibility", "-enable", id)
+		f.Assert().NoError(err)
+	}
+	tc.nodeIDs = []string{}
 
 	// Stop all jobs in test
 	for _, id := range tc.testJobIDs {
@@ -86,6 +94,14 @@ func (tc *CSIControllerPluginEBSTest) AfterAll(f *framework.F) {
 	}
 	tc.testJobIDs = []string{}
 
+	// Garbage collect
+	out, err := e2eutil.Command("nomad", "system", "gc")
+	f.Assert().NoError(err, out)
+}
+
+// AfterAll cleans up the volumes and plugin jobs created by the test.
+func (tc *CSIControllerPluginEBSTest) AfterAll(f *framework.F) {
+
 	for _, volID := range tc.volumeIDs {
 		err := waitForVolumeClaimRelease(volID, reapWait)
 		f.Assert().NoError(err, "volume claims were not released")
@@ -102,14 +118,6 @@ func (tc *CSIControllerPluginEBSTest) AfterAll(f *framework.F) {
 	}
 	tc.pluginJobIDs = []string{}
 
-	for _, id := range tc.nodeIDs {
-		_, err := e2eutil.Command("nomad", "node", "drain", "-disable", "-yes", id)
-		f.Assert().NoError(err)
-		_, err = e2eutil.Command("nomad", "node", "eligibility", "-enable", id)
-		f.Assert().NoError(err)
-	}
-	tc.nodeIDs = []string{}
-
 	// Garbage collect
 	out, err := e2eutil.Command("nomad", "system", "gc")
 	f.Assert().NoError(err, out)
@@ -208,7 +216,7 @@ func (tc *CSIControllerPluginEBSTest) TestNodeDrain(f *framework.F) {
 	expectedHealthyNodePlugins := len(pluginAllocs)
 
 	// deploy a job that writes to the volume
-	writeJobID := "write-ebs-" + tc.uuid
+	writeJobID := "write-ebs-for-drain" + tc.uuid
 	f.NoError(e2eutil.Register(writeJobID, "csi/input/use-ebs-volume.nomad"))
 	f.NoError(
 		e2eutil.WaitForAllocStatusExpected(writeJobID, ns, []string{"running"}),