e2e/framework: move *testing.T access to a context *framework.F which is scoped to each test

e2e/example/example.go

@@ -18,8 +18,8 @@ type SimpleExampleTestCase struct {
 	framework.TC
 }
 
-func (tc *SimpleExampleTestCase) TestExample() {
+func (tc *SimpleExampleTestCase) TestExample(f *framework.F) {
 	jobs, _, err := tc.Nomad().Jobs().List(nil)
-	tc.NoError(err)
-	tc.Empty(jobs)
+	f.NoError(err)
+	f.Empty(jobs)
 }

e2e/framework/case.go

@@ -5,8 +5,6 @@ import (
 	"testing"
 
 	"github.com/hashicorp/nomad/api"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 // TestSuite defines a set of test cases and under what conditions to run them
@@ -59,17 +57,10 @@ func (c Constraints) matches(env Environment) error {
 }
 
 // TC is the base test case which should be embedded in TestCase implementations.
-// It also embeds testify assertions configured with the current *testing.T
-// context. For more information on assertions:
-// https://godoc.org/github.com/stretchr/testify/require#Assertions
 type TC struct {
-	*require.Assertions
-	assert *assert.Assertions
-	t      *testing.T
+	t *testing.T
 
 	cluster *ClusterInfo
-	prefix  string
-	name    string
 }
 
 // Nomad returns a configured nomad api client
@@ -77,36 +68,12 @@ func (tc *TC) Nomad() *api.Client {
 	return tc.cluster.NomadClient
 }
 
-// Prefix will return a test case unique prefix which can be used to scope resources
-// during parallel tests.
-func (tc *TC) Prefix() string {
-	return fmt.Sprintf("%s-", tc.cluster.ID)
-}
-
 // Name returns the name of the test case which is set to the name of the
 // implementing type.
 func (tc *TC) Name() string {
 	return tc.cluster.Name
 }
 
-// T retrieves the current *testing.T context
-func (tc *TC) T() *testing.T {
-	return tc.t
-}
-
-// SetT sets the current *testing.T context
-func (tc *TC) SetT(t *testing.T) {
-	tc.t = t
-	tc.Assertions = require.New(t)
-	tc.assert = assert.New(t)
-}
-
-// Require fetches an assert flavor of testify assertions
-// https://godoc.org/github.com/stretchr/testify/assert
-func (tc *TC) Assert() *assert.Assertions {
-	return tc.assert
-}
-
 func (tc *TC) setClusterInfo(info *ClusterInfo) {
 	tc.cluster = info
 }

e2e/framework/doc.go

@@ -8,16 +8,17 @@ before/after each and all tests.
 Writing Tests
 
 Tests follow a similar patterns as go tests. They are functions that must start
-with 'Test' and instead of a *testing.T argument, they must have a receiver that
-implements the TestCase interface. A crude example as follows:
+with 'Test' and instead of a *testing.T argument, a *framework.F is passed and
+they must have a receiver that implements the TestCase interface.
+A crude example as follows:
 
 	// foo_test.go
 	type MyTestCase struct {
 		framework.TC
 	}
 
-	func (tc *MyTestCase) TestMyFoo() {
-		tc.T().Log("bar")
+	func (tc *MyTestCase) TestMyFoo(f *framework.F) {
+		f.T().Log("bar")
 	}
 
 	func TestCalledFromGoTest(t *testing.T){
@@ -30,7 +31,7 @@ implements the TestCase interface. A crude example as follows:
 	}
 
 Test cases should embed the TC struct which satisfies the TestCase interface.
-Optionally a TestCase can also override the Name() function of TC which returns
+Optionally a TestCase can also implement the Name() function which returns
 a string to name the test case. By default the name is the name of the struct
 type, which in the above example would be "MyTestCase"
 
@@ -50,24 +51,27 @@ can be consumed by the tests. For example:
 		jobID string
 	}
 
-	func (tc *ComplexNomadTC) BeforeEach(){
+	func (tc *ComplexNomadTC) BeforeEach(f *framework.F){
 		// Do some complex job setup with a unique prefix string
-		jobID, err := doSomeComplexSetup(tc.Nomad(), tc.Prefix())
-		tc.NoError(err)
-		tc.jobID = jobID
+		jobID, err := doSomeComplexSetup(tc.Nomad(), f.ID())
+		f.NoError(err)
+		f.Set("jobID", jobID)
 	}
 
-	func (tc *ComplexNomadTC) TestSomeScenario(){
-		doTestThingWithJob(tc.T(), tc.Nomad(), tc.jobID)
+	func (tc *ComplexNomadTC) TestSomeScenario(f *framework.F){
+		jobID := f.Value("jobID").(string)
+		doTestThingWithJob(f, tc.Nomad(), jobID)
 	}
 
-	func (tc *ComplexNomadTC) TestOtherScenario(){
-		doOtherTestThingWithJob(tc.T(), tc.Nomad(), tc.jobID)
+	func (tc *ComplexNomadTC) TestOtherScenario(f *framework.F){
+		jobID := f.Value("jobID").(string)
+		doOtherTestThingWithJob(f, tc.Nomad(), jobID)
 	}
 
-	func (tc *ComplexNomadTC) AfterEach(){
+	func (tc *ComplexNomadTC) AfterEach(f *framework.F){
+		jobID := f.Value("jobID").(string)
 		_, _, err := tc.Nomad().Jobs().Deregister(jobID, true, nil)
-		tc.Require().NoError(err)
+		f.NoError(err)
 	}
 
 As demonstrated in the previous example, TC also exposes functions that return
@@ -94,16 +98,27 @@ Parallelism
 The test framework honors go test's parallel feature under certain conditions.
 A TestSuite can be created with the Parallel field set to true to enable
 parallel execution of the test cases of the suite. Tests within a test case
-will always be executed sequentially. TC.T() is NOT safe to call from multiple
-goroutines, therefore TC.T().Parallel() should NEVER be called from a test of a
-TestCase
+will be executed sequentially unless f.T().Parallel() is called. Note that if
+multiple tests are to be executed in parallel, access to TC is note syncronized.
+The *framework.F offers a way to store state between before/after each method if
+desired.
+
+	func (tc *MyTestCase) BeforeEach(f *framework.F){
+		jobID, _ := doSomeComplexSetup(tc.Nomad(), f.ID())
+		f.Set("jobID", jobID)
+	}
+
+	func (tc *MyTestCase) TestParallel(f *framework.F){
+		f.T().Parallel()
+		jobID := f.Value("jobID").(string)
+	}
 
 Since test cases have the potential to work with a shared Nomad cluster in parallel
 any resources created or destroyed must be prefixed with a unique identifier for
-each test case. The TC struct exposes a Parallel() function that will return a
-string that is unique with in a test cases, so multiple tests with in the case
+each test case. The framework.F struct exposes an ID() function that will return a
+string that is unique with in a test. Therefore, multiple tests with in the case
 can reliably create unique IDs between tests and setup/teardown. The string
-returned is 8 alpha numeric characters with a '-' appended.
+returned is 8 alpha numeric characters.
 
 */
 package framework

e2e/framework/framework.go

@@ -145,21 +145,22 @@ func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error)
 
 		// Each TestCase runs as a subtest of the TestSuite
 		t.Run(c.Name(), func(t *testing.T) {
-			c.SetT(t)
 			// If the TestSuite has Parallel set, all cases run in parallel
 			if s.Parallel {
 				t.Parallel()
 			}
 
+			f := newF(t)
+
 			// Check if the case includes a before all function
 			if beforeAllTests, ok := c.(BeforeAllTests); ok {
-				beforeAllTests.BeforeAll()
+				beforeAllTests.BeforeAll(f)
 			}
 
 			// Check if the case includes an after all function at the end
 			defer func() {
 				if afterAllTests, ok := c.(AfterAllTests); ok {
-					afterAllTests.AfterAll()
+					afterAllTests.AfterAll(f)
 				}
 			}()
 
@@ -175,23 +176,18 @@ func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error)
 				// Test cases are never parallel
 				t.Run(method.Name, func(t *testing.T) {
 
-					// Since the test function interacts with testing.T through
-					// the test case struct, we need to swap the test context for
-					// the duration of the test.
-					parentT := c.T()
-					c.SetT(t)
+					cF := newF(t)
 					if BeforeEachTest, ok := c.(BeforeEachTest); ok {
-						BeforeEachTest.BeforeEach()
+						BeforeEachTest.BeforeEach(cF)
 					}
 					defer func() {
 						if afterEachTest, ok := c.(AfterEachTest); ok {
-							afterEachTest.AfterEach()
+							afterEachTest.AfterEach(cF)
 						}
-						c.SetT(parentT)
 					}()
 
 					//Call the method
-					method.Func.Call([]reflect.Value{reflect.ValueOf(c)})
+					method.Func.Call([]reflect.Value{reflect.ValueOf(c), reflect.ValueOf(cF)})
 				})
 			}
 		})

e2e/framework/interfaces.go

@@ -1,9 +1,5 @@
 package framework
 
-import (
-	"testing"
-)
-
 // TestCase is the interface which an E2E test case implements.
 // It is not meant to be implemented directly, instead the struct should embed
 // the 'framework.TC' struct
@@ -11,8 +7,6 @@ type TestCase interface {
 	internalTestCase
 
 	Name() string
-	T() *testing.T
-	SetT(*testing.T)
 }
 
 type internalTestCase interface {
@@ -22,21 +16,21 @@ type internalTestCase interface {
 // BeforeAllTests is used to define a method to be called before the execution
 // of all tests.
 type BeforeAllTests interface {
-	BeforeAll()
+	BeforeAll(*F)
 }
 
 // AfterAllTests is used to define a method to be called after the execution of
 // all tests.
 type AfterAllTests interface {
-	AfterAll()
+	AfterAll(*F)
 }
 
 // BeforeEachTest is used to define a method to be called before each test.
 type BeforeEachTest interface {
-	BeforeEach()
+	BeforeEach(*F)
 }
 
 // AfterEachTest is used to degine a method to be called after each test.
 type AfterEachTest interface {
-	AfterEach()
+	AfterEach(*F)
 }