diff --git a/e2e/framework/case.go b/e2e/framework/case.go index 2b8e08a8f..8950919e0 100644 --- a/e2e/framework/case.go +++ b/e2e/framework/case.go @@ -9,26 +9,27 @@ import ( "github.com/stretchr/testify/require" ) +// TestSuite defines a set of test cases and under what conditions to run them type TestSuite struct { - Component string + Component string // Name of the component/system/feature tested - CanRunLocal bool - Cases []TestCase - Constraints Constraints - Parallel bool - Slow bool + CanRunLocal bool // Flags if the cases are safe to run on a local nomad cluster + Cases []TestCase // Cases to run + Constraints Constraints // Environment constraints to follow + Parallel bool // If true, will run test cases in parallel + Slow bool // Slow test suites don't run by default } type Constraints struct { - CloudProvider string - OS string - Arch string - Environment string - Tags []string + Provider string // Cloud provider ex. 'aws', 'azure', 'gcp' + OS string // Operating system ex. 'windows', 'linux' + Arch string // CPU architecture ex. 'amd64', 'arm64' + Environment string // Environment name ex. 'simple' + Tags []string // Generic tags that must all exist in the environment } func (c Constraints) matches(env Environment) error { - if len(c.CloudProvider) != 0 && c.CloudProvider != env.Provider { + if len(c.Provider) != 0 && c.Provider != env.Provider { return fmt.Errorf("provider constraint does not match environment") } @@ -52,6 +53,7 @@ func (c Constraints) matches(env Environment) error { return nil } +// TC is the base test case which should be embedded in TestCase implementations type TC struct { *assert.Assertions require *require.Assertions @@ -62,22 +64,28 @@ type TC struct { name string } +// Nomad returns a configured nomad api client func (tc *TC) Nomad() *api.Client { return tc.cluster.NomadClient } +// Prefix will return a test case unique prefix which can be used to scope resources +// during parallel tests. func (tc *TC) Prefix() string { return fmt.Sprintf("%s-", tc.cluster.ID) } +// Name is the Name of the test cluster. func (tc *TC) Name() string { return tc.cluster.Name } +// T retrieves the current *testing.T context func (tc *TC) T() *testing.T { return tc.t } +// SetT sets the current *testing.T context func (tc *TC) SetT(t *testing.T) { tc.t = t tc.Assertions = assert.New(t) diff --git a/e2e/framework/doc.go b/e2e/framework/doc.go new file mode 100644 index 000000000..67a8bf7ab --- /dev/null +++ b/e2e/framework/doc.go @@ -0,0 +1,8 @@ +/* +Package framework implements a model for developing end-to-end test suites. The +model includes a top level Framework which TestSuites can be added to. TestSuites +include conditions under which the suite will run and a list of TestCase +implementations to run. TestCases can be implemented with methods that run +before/after each and all tests. +*/ +package framework diff --git a/e2e/framework/framework.go b/e2e/framework/framework.go index 2f10f7601..3d56b7793 100644 --- a/e2e/framework/framework.go +++ b/e2e/framework/framework.go @@ -8,11 +8,11 @@ import ( "testing" ) -var fProvider = flag.String("nomad.env.provider", "", "cloud provider for which environment is executing against") var fEnv = flag.String("nomad.env", "", "name of the environment executing against") -var fOS = flag.String("nomad.os", "", "operating system for which the environment is executing against") -var fArch = flag.String("nomad.arch", "", "cpu architecture for which the environment is executing against") -var fTags = flag.String("nomad.tags", "", "comma delimited list of tags associated with the environment") +var fProvider = flag.String("nomad.env.provider", "", "cloud provider for which environment is executing against") +var fOS = flag.String("nomad.env.os", "", "operating system for which the environment is executing against") +var fArch = flag.String("nomad.env.arch", "", "cpu architecture for which the environment is executing against") +var fTags = flag.String("nomad.env.tags", "", "comma delimited list of tags associated with the environment") var fLocal = flag.Bool("nomad.local", false, "denotes execution is against a local environment") var fSlow = flag.Bool("nomad.slow", false, "toggles execution of slow test suites") var fForceAll = flag.Bool("nomad.force", false, "if set, skips all environment checks when filtering test suites") @@ -29,6 +29,17 @@ type Framework struct { force bool } +// Environment includes information about the target environment the test +// framework is targeting. During 'go run', these fields are populated by +// the following flags: +// +// -nomad.env "name of the environment executing against" +// -nomad.env.provider "cloud provider for which the environment is executing against" +// -nomad.env.os "operating system of environment" +// -nomad.env.arch "cpu architecture of environment" +// -nomad.env.tags "comma delimited list of environment tags" +// +// These flags are not needed when executing locally type Environment struct { Name string Provider string @@ -37,6 +48,7 @@ type Environment struct { Tags map[string]struct{} } +// New creates a Framework func New() *Framework { env := Environment{ Name: *fEnv, @@ -57,17 +69,19 @@ func New() *Framework { } } +// AddSuites adds a set of test suites to a Framework func (f *Framework) AddSuites(s ...*TestSuite) *Framework { f.suites = append(f.suites, s...) return f } +// AddSuites adds a set of test suites to the package scoped Framework func AddSuites(s ...*TestSuite) *Framework { pkgFramework.AddSuites(s...) return pkgFramework } -// Run starts the test framework and runs each TestSuite +// Run starts the test framework, running each TestSuite func (f *Framework) Run(t *testing.T) { for _, s := range f.suites { t.Run(s.Component, func(t *testing.T) { @@ -84,13 +98,18 @@ func (f *Framework) Run(t *testing.T) { } +// Run starts the package scoped Framework, running each TestSuite func Run(t *testing.T) { pkgFramework.Run(t) } -// runSuite is called from Framework.Run inside of a sub test for each TestSuite +// runSuite is called from Framework.Run inside of a sub test for each TestSuite. +// If skip is returned as true, the test suite is skipped with the error text added +// to the Skip reason +// If skip is false and an error is returned, the test suite is failed. func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error) { + // If -nomad.force is set, skip all constraint checks if !f.force { // If this is a local run, check that the suite supports running locally if !s.CanRunLocal && f.isLocalRun { @@ -109,15 +128,17 @@ func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error) } for _, c := range s.Cases { + // The test name is set to the name of the implementing type, including package name := fmt.Sprintf("%T", c) + // Each TestCase is provisioned a nomad cluster info, err := f.provisioner.ProvisionCluster(ProvisionerOptions{Name: name}) if err != nil { return false, fmt.Errorf("could not provision cluster for case: %v", err) } defer f.provisioner.DestroyCluster(info.ID) - c.setClusterInfo(info) + // Each TestCase runs as a subtest of the TestSuite t.Run(c.Name(), func(t *testing.T) { c.SetT(t) @@ -139,14 +160,15 @@ func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error) }() // Here we need to iterate through the methods of the case to find - // ones that at test functions + // ones that are test functions reflectC := reflect.TypeOf(c) for i := 0; i < reflectC.NumMethod(); i++ { method := reflectC.Method(i) - if ok, _ := isTestMethod(method.Name); !ok { + if ok := isTestMethod(method.Name); !ok { continue } // Each step is run as its own sub test of the case + // Test cases are never parallel t.Run(method.Name, func(t *testing.T) { // Since the test function interacts with testing.T through @@ -163,6 +185,7 @@ func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error) } c.SetT(parentT) }() + //Call the method method.Func.Call([]reflect.Value{reflect.ValueOf(c)}) }) @@ -174,10 +197,10 @@ func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error) return false, nil } -func isTestMethod(m string) (bool, error) { +func isTestMethod(m string) bool { if !strings.HasPrefix(m, "Test") { - return false, nil + return false } - - return true, nil + // THINKING: adding flag to target a specific step or step regex? + return true } diff --git a/e2e/framework/interfaces.go b/e2e/framework/interfaces.go index bb1b9b6ad..2abd56556 100644 --- a/e2e/framework/interfaces.go +++ b/e2e/framework/interfaces.go @@ -4,16 +4,13 @@ import ( "testing" ) -// Named exists simply to make sure the Name() method was implemented since it -// is the only required method implementation of a test case -type Named interface { - Name() string -} - +// TestCase is the interface which an E2E test case implements. +// It is not meant to be implemented directly, instead the struct should embed +// the 'framework.TC' struct type TestCase interface { - Named internalTestCase + Name() string T() *testing.T SetT(*testing.T) } @@ -22,18 +19,24 @@ type internalTestCase interface { setClusterInfo(*ClusterInfo) } +// BeforeAllSteps is used to define a method to be called before the execution +// of all test steps. type BeforeAllSteps interface { BeforeAllSteps() } +// AfterAllSteps is used to define a method to be called after the execution of +// all test steps. type AfterAllSteps interface { AfterAllSteps() } +// BeforeEachStep is used to define a method to be called before each test step. type BeforeEachStep interface { BeforeEachStep() } +// AfterEachStep is used to degine a method to be called after each test step. type AfterEachStep interface { AfterEachStep() }