diff --git a/e2e/example/example.go b/e2e/example/example.go index ccfc6d706..3884e8b51 100644 --- a/e2e/example/example.go +++ b/e2e/example/example.go @@ -18,8 +18,8 @@ type SimpleExampleTestCase struct { framework.TC } -func (tc *SimpleExampleTestCase) TestExample() { +func (tc *SimpleExampleTestCase) TestExample(f *framework.F) { jobs, _, err := tc.Nomad().Jobs().List(nil) - tc.NoError(err) - tc.Empty(jobs) + f.NoError(err) + f.Empty(jobs) } diff --git a/e2e/framework/case.go b/e2e/framework/case.go index 9587b28bf..81a314ee4 100644 --- a/e2e/framework/case.go +++ b/e2e/framework/case.go @@ -5,8 +5,6 @@ import ( "testing" "github.com/hashicorp/nomad/api" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) // TestSuite defines a set of test cases and under what conditions to run them @@ -59,17 +57,10 @@ func (c Constraints) matches(env Environment) error { } // TC is the base test case which should be embedded in TestCase implementations. -// It also embeds testify assertions configured with the current *testing.T -// context. For more information on assertions: -// https://godoc.org/github.com/stretchr/testify/require#Assertions type TC struct { - *require.Assertions - assert *assert.Assertions - t *testing.T + t *testing.T cluster *ClusterInfo - prefix string - name string } // Nomad returns a configured nomad api client @@ -77,36 +68,12 @@ func (tc *TC) Nomad() *api.Client { return tc.cluster.NomadClient } -// Prefix will return a test case unique prefix which can be used to scope resources -// during parallel tests. -func (tc *TC) Prefix() string { - return fmt.Sprintf("%s-", tc.cluster.ID) -} - // Name returns the name of the test case which is set to the name of the // implementing type. func (tc *TC) Name() string { return tc.cluster.Name } -// T retrieves the current *testing.T context -func (tc *TC) T() *testing.T { - return tc.t -} - -// SetT sets the current *testing.T context -func (tc *TC) SetT(t *testing.T) { - tc.t = t - tc.Assertions = require.New(t) - tc.assert = assert.New(t) -} - -// Require fetches an assert flavor of testify assertions -// https://godoc.org/github.com/stretchr/testify/assert -func (tc *TC) Assert() *assert.Assertions { - return tc.assert -} - func (tc *TC) setClusterInfo(info *ClusterInfo) { tc.cluster = info } diff --git a/e2e/framework/doc.go b/e2e/framework/doc.go index 9ab5a0a8b..755573754 100644 --- a/e2e/framework/doc.go +++ b/e2e/framework/doc.go @@ -8,16 +8,17 @@ before/after each and all tests. Writing Tests Tests follow a similar patterns as go tests. They are functions that must start -with 'Test' and instead of a *testing.T argument, they must have a receiver that -implements the TestCase interface. A crude example as follows: +with 'Test' and instead of a *testing.T argument, a *framework.F is passed and +they must have a receiver that implements the TestCase interface. +A crude example as follows: // foo_test.go type MyTestCase struct { framework.TC } - func (tc *MyTestCase) TestMyFoo() { - tc.T().Log("bar") + func (tc *MyTestCase) TestMyFoo(f *framework.F) { + f.T().Log("bar") } func TestCalledFromGoTest(t *testing.T){ @@ -30,7 +31,7 @@ implements the TestCase interface. A crude example as follows: } Test cases should embed the TC struct which satisfies the TestCase interface. -Optionally a TestCase can also override the Name() function of TC which returns +Optionally a TestCase can also implement the Name() function which returns a string to name the test case. By default the name is the name of the struct type, which in the above example would be "MyTestCase" @@ -50,24 +51,27 @@ can be consumed by the tests. For example: jobID string } - func (tc *ComplexNomadTC) BeforeEach(){ + func (tc *ComplexNomadTC) BeforeEach(f *framework.F){ // Do some complex job setup with a unique prefix string - jobID, err := doSomeComplexSetup(tc.Nomad(), tc.Prefix()) - tc.NoError(err) - tc.jobID = jobID + jobID, err := doSomeComplexSetup(tc.Nomad(), f.ID()) + f.NoError(err) + f.Set("jobID", jobID) } - func (tc *ComplexNomadTC) TestSomeScenario(){ - doTestThingWithJob(tc.T(), tc.Nomad(), tc.jobID) + func (tc *ComplexNomadTC) TestSomeScenario(f *framework.F){ + jobID := f.Value("jobID").(string) + doTestThingWithJob(f, tc.Nomad(), jobID) } - func (tc *ComplexNomadTC) TestOtherScenario(){ - doOtherTestThingWithJob(tc.T(), tc.Nomad(), tc.jobID) + func (tc *ComplexNomadTC) TestOtherScenario(f *framework.F){ + jobID := f.Value("jobID").(string) + doOtherTestThingWithJob(f, tc.Nomad(), jobID) } - func (tc *ComplexNomadTC) AfterEach(){ + func (tc *ComplexNomadTC) AfterEach(f *framework.F){ + jobID := f.Value("jobID").(string) _, _, err := tc.Nomad().Jobs().Deregister(jobID, true, nil) - tc.Require().NoError(err) + f.NoError(err) } As demonstrated in the previous example, TC also exposes functions that return @@ -94,16 +98,27 @@ Parallelism The test framework honors go test's parallel feature under certain conditions. A TestSuite can be created with the Parallel field set to true to enable parallel execution of the test cases of the suite. Tests within a test case -will always be executed sequentially. TC.T() is NOT safe to call from multiple -goroutines, therefore TC.T().Parallel() should NEVER be called from a test of a -TestCase +will be executed sequentially unless f.T().Parallel() is called. Note that if +multiple tests are to be executed in parallel, access to TC is note syncronized. +The *framework.F offers a way to store state between before/after each method if +desired. + + func (tc *MyTestCase) BeforeEach(f *framework.F){ + jobID, _ := doSomeComplexSetup(tc.Nomad(), f.ID()) + f.Set("jobID", jobID) + } + + func (tc *MyTestCase) TestParallel(f *framework.F){ + f.T().Parallel() + jobID := f.Value("jobID").(string) + } Since test cases have the potential to work with a shared Nomad cluster in parallel any resources created or destroyed must be prefixed with a unique identifier for -each test case. The TC struct exposes a Parallel() function that will return a -string that is unique with in a test cases, so multiple tests with in the case +each test case. The framework.F struct exposes an ID() function that will return a +string that is unique with in a test. Therefore, multiple tests with in the case can reliably create unique IDs between tests and setup/teardown. The string -returned is 8 alpha numeric characters with a '-' appended. +returned is 8 alpha numeric characters. */ package framework diff --git a/e2e/framework/framework.go b/e2e/framework/framework.go index 9972b64f3..5c14bdf89 100644 --- a/e2e/framework/framework.go +++ b/e2e/framework/framework.go @@ -145,21 +145,22 @@ func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error) // Each TestCase runs as a subtest of the TestSuite t.Run(c.Name(), func(t *testing.T) { - c.SetT(t) // If the TestSuite has Parallel set, all cases run in parallel if s.Parallel { t.Parallel() } + f := newF(t) + // Check if the case includes a before all function if beforeAllTests, ok := c.(BeforeAllTests); ok { - beforeAllTests.BeforeAll() + beforeAllTests.BeforeAll(f) } // Check if the case includes an after all function at the end defer func() { if afterAllTests, ok := c.(AfterAllTests); ok { - afterAllTests.AfterAll() + afterAllTests.AfterAll(f) } }() @@ -175,23 +176,18 @@ func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error) // Test cases are never parallel t.Run(method.Name, func(t *testing.T) { - // Since the test function interacts with testing.T through - // the test case struct, we need to swap the test context for - // the duration of the test. - parentT := c.T() - c.SetT(t) + cF := newF(t) if BeforeEachTest, ok := c.(BeforeEachTest); ok { - BeforeEachTest.BeforeEach() + BeforeEachTest.BeforeEach(cF) } defer func() { if afterEachTest, ok := c.(AfterEachTest); ok { - afterEachTest.AfterEach() + afterEachTest.AfterEach(cF) } - c.SetT(parentT) }() //Call the method - method.Func.Call([]reflect.Value{reflect.ValueOf(c)}) + method.Func.Call([]reflect.Value{reflect.ValueOf(c), reflect.ValueOf(cF)}) }) } }) diff --git a/e2e/framework/interfaces.go b/e2e/framework/interfaces.go index 8f3ae8f31..fc862ab80 100644 --- a/e2e/framework/interfaces.go +++ b/e2e/framework/interfaces.go @@ -1,9 +1,5 @@ package framework -import ( - "testing" -) - // TestCase is the interface which an E2E test case implements. // It is not meant to be implemented directly, instead the struct should embed // the 'framework.TC' struct @@ -11,8 +7,6 @@ type TestCase interface { internalTestCase Name() string - T() *testing.T - SetT(*testing.T) } type internalTestCase interface { @@ -22,21 +16,21 @@ type internalTestCase interface { // BeforeAllTests is used to define a method to be called before the execution // of all tests. type BeforeAllTests interface { - BeforeAll() + BeforeAll(*F) } // AfterAllTests is used to define a method to be called after the execution of // all tests. type AfterAllTests interface { - AfterAll() + AfterAll(*F) } // BeforeEachTest is used to define a method to be called before each test. type BeforeEachTest interface { - BeforeEach() + BeforeEach(*F) } // AfterEachTest is used to degine a method to be called after each test. type AfterEachTest interface { - AfterEach() + AfterEach(*F) }