diff --git a/e2e/e2e.go b/e2e/e2e.go new file mode 100644 index 000000000..b2be4e86d --- /dev/null +++ b/e2e/e2e.go @@ -0,0 +1,11 @@ +package e2e + +import ( + "testing" + + "github.com/hashicorp/nomad/e2e/framework" +) + +func RunE2ETests(t *testing.T) { + framework.Run(t) +} diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go new file mode 100644 index 000000000..c2a3896ac --- /dev/null +++ b/e2e/e2e_test.go @@ -0,0 +1,11 @@ +package e2e + +import ( + "testing" + + _ "github.com/hashicorp/nomad/e2e/example" +) + +func TestE2E(t *testing.T) { + RunE2ETests(t) +} diff --git a/e2e/example/e2e_test.go b/e2e/example/e2e_test.go new file mode 100644 index 000000000..0bc767cc2 --- /dev/null +++ b/e2e/example/e2e_test.go @@ -0,0 +1,17 @@ +package example + +import ( + "testing" + + "github.com/hashicorp/nomad/e2e/framework" +) + +func TestE2E(t *testing.T) { + framework.New().AddSuites(&framework.TestSuite{ + Component: "simple", + CanRunLocal: true, + Cases: []framework.TestCase{ + new(SimpleExampleTestCase), + }, + }).Run(t) +} diff --git a/e2e/example/example.go b/e2e/example/example.go new file mode 100644 index 000000000..ccfc6d706 --- /dev/null +++ b/e2e/example/example.go @@ -0,0 +1,25 @@ +package example + +import ( + "github.com/hashicorp/nomad/e2e/framework" +) + +func init() { + framework.AddSuites(&framework.TestSuite{ + Component: "simple", + CanRunLocal: true, + Cases: []framework.TestCase{ + new(SimpleExampleTestCase), + }, + }) +} + +type SimpleExampleTestCase struct { + framework.TC +} + +func (tc *SimpleExampleTestCase) TestExample() { + jobs, _, err := tc.Nomad().Jobs().List(nil) + tc.NoError(err) + tc.Empty(jobs) +} diff --git a/e2e/framework/case.go b/e2e/framework/case.go new file mode 100644 index 000000000..2b8e08a8f --- /dev/null +++ b/e2e/framework/case.go @@ -0,0 +1,89 @@ +package framework + +import ( + "fmt" + "testing" + + "github.com/hashicorp/nomad/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type TestSuite struct { + Component string + + CanRunLocal bool + Cases []TestCase + Constraints Constraints + Parallel bool + Slow bool +} + +type Constraints struct { + CloudProvider string + OS string + Arch string + Environment string + Tags []string +} + +func (c Constraints) matches(env Environment) error { + if len(c.CloudProvider) != 0 && c.CloudProvider != env.Provider { + return fmt.Errorf("provider constraint does not match environment") + } + + if len(c.OS) != 0 && c.OS != env.OS { + return fmt.Errorf("os constraint does not match environment") + } + + if len(c.Arch) != 0 && c.Arch != env.Arch { + return fmt.Errorf("arch constraint does not match environment") + } + + if len(c.Environment) != 0 && c.Environment != env.Name { + return fmt.Errorf("environment constraint does not match environment name") + } + + for _, t := range c.Tags { + if _, ok := env.Tags[t]; !ok { + return fmt.Errorf("tags constraint failed, tag '%s' is not included in environment") + } + } + return nil +} + +type TC struct { + *assert.Assertions + require *require.Assertions + t *testing.T + + cluster *ClusterInfo + prefix string + name string +} + +func (tc *TC) Nomad() *api.Client { + return tc.cluster.NomadClient +} + +func (tc *TC) Prefix() string { + return fmt.Sprintf("%s-", tc.cluster.ID) +} + +func (tc *TC) Name() string { + return tc.cluster.Name +} + +func (tc *TC) T() *testing.T { + return tc.t +} + +func (tc *TC) SetT(t *testing.T) { + tc.t = t + tc.Assertions = assert.New(t) + tc.require = require.New(t) +} + +func (tc *TC) setClusterInfo(info *ClusterInfo) { + tc.cluster = info +} diff --git a/e2e/framework/framework.go b/e2e/framework/framework.go new file mode 100644 index 000000000..2f10f7601 --- /dev/null +++ b/e2e/framework/framework.go @@ -0,0 +1,183 @@ +package framework + +import ( + "flag" + "fmt" + "reflect" + "strings" + "testing" +) + +var fProvider = flag.String("nomad.env.provider", "", "cloud provider for which environment is executing against") +var fEnv = flag.String("nomad.env", "", "name of the environment executing against") +var fOS = flag.String("nomad.os", "", "operating system for which the environment is executing against") +var fArch = flag.String("nomad.arch", "", "cpu architecture for which the environment is executing against") +var fTags = flag.String("nomad.tags", "", "comma delimited list of tags associated with the environment") +var fLocal = flag.Bool("nomad.local", false, "denotes execution is against a local environment") +var fSlow = flag.Bool("nomad.slow", false, "toggles execution of slow test suites") +var fForceAll = flag.Bool("nomad.force", false, "if set, skips all environment checks when filtering test suites") + +var pkgFramework = New() + +type Framework struct { + suites []*TestSuite + provisioner Provisioner + env Environment + + isLocalRun bool + slow bool + force bool +} + +type Environment struct { + Name string + Provider string + OS string + Arch string + Tags map[string]struct{} +} + +func New() *Framework { + env := Environment{ + Name: *fEnv, + Provider: *fProvider, + OS: *fOS, + Arch: *fArch, + Tags: map[string]struct{}{}, + } + for _, tag := range strings.Split(*fTags, ",") { + env.Tags[tag] = struct{}{} + } + return &Framework{ + provisioner: DefaultProvisioner, + env: env, + isLocalRun: *fLocal, + slow: *fSlow, + force: *fForceAll, + } +} + +func (f *Framework) AddSuites(s ...*TestSuite) *Framework { + f.suites = append(f.suites, s...) + return f +} + +func AddSuites(s ...*TestSuite) *Framework { + pkgFramework.AddSuites(s...) + return pkgFramework +} + +// Run starts the test framework and runs each TestSuite +func (f *Framework) Run(t *testing.T) { + for _, s := range f.suites { + t.Run(s.Component, func(t *testing.T) { + skip, err := f.runSuite(t, s) + if skip { + t.Skipf("skipping suite '%s': %v", s.Component, err) + return + } + if err != nil { + t.Errorf("error starting suite '%s': %v", s.Component, err) + } + }) + } + +} + +func Run(t *testing.T) { + pkgFramework.Run(t) +} + +// runSuite is called from Framework.Run inside of a sub test for each TestSuite +func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error) { + + if !f.force { + // If this is a local run, check that the suite supports running locally + if !s.CanRunLocal && f.isLocalRun { + return true, fmt.Errorf("local run detected and suite cannot run locally") + } + + // Check that constraints are met + if err := s.Constraints.matches(f.env); err != nil { + return true, fmt.Errorf("constraint failed: %v", err) + } + + // Check the slow toggle and if the suite's slow flag is that same + if f.slow != s.Slow { + return true, fmt.Errorf("framework slow suite configuration is %v but suite is %v", f.slow, s.Slow) + } + } + + for _, c := range s.Cases { + name := fmt.Sprintf("%T", c) + // Each TestCase is provisioned a nomad cluster + info, err := f.provisioner.ProvisionCluster(ProvisionerOptions{Name: name}) + if err != nil { + return false, fmt.Errorf("could not provision cluster for case: %v", err) + } + defer f.provisioner.DestroyCluster(info.ID) + + c.setClusterInfo(info) + // Each TestCase runs as a subtest of the TestSuite + t.Run(c.Name(), func(t *testing.T) { + c.SetT(t) + // If the TestSuite has Parallel set, all cases run in parallel + if s.Parallel { + t.Parallel() + } + + // Check if the case includes a before all function + if beforeAllSteps, ok := c.(BeforeAllSteps); ok { + beforeAllSteps.BeforeAllSteps() + } + + // Check if the case includes an after all function at the end + defer func() { + if afterAllSteps, ok := c.(AfterAllSteps); ok { + afterAllSteps.AfterAllSteps() + } + }() + + // Here we need to iterate through the methods of the case to find + // ones that at test functions + reflectC := reflect.TypeOf(c) + for i := 0; i < reflectC.NumMethod(); i++ { + method := reflectC.Method(i) + if ok, _ := isTestMethod(method.Name); !ok { + continue + } + // Each step is run as its own sub test of the case + t.Run(method.Name, func(t *testing.T) { + + // Since the test function interacts with testing.T through + // the test case struct, we need to swap the test context for + // the duration of the step. + parentT := c.T() + c.SetT(t) + if BeforeEachStep, ok := c.(BeforeEachStep); ok { + BeforeEachStep.BeforeEachStep() + } + defer func() { + if afterEachStep, ok := c.(AfterEachStep); ok { + afterEachStep.AfterEachStep() + } + c.SetT(parentT) + }() + //Call the method + method.Func.Call([]reflect.Value{reflect.ValueOf(c)}) + }) + } + }) + + } + + return false, nil +} + +func isTestMethod(m string) (bool, error) { + if !strings.HasPrefix(m, "Test") { + return false, nil + } + + return true, nil +} diff --git a/e2e/framework/interfaces.go b/e2e/framework/interfaces.go new file mode 100644 index 000000000..bb1b9b6ad --- /dev/null +++ b/e2e/framework/interfaces.go @@ -0,0 +1,39 @@ +package framework + +import ( + "testing" +) + +// Named exists simply to make sure the Name() method was implemented since it +// is the only required method implementation of a test case +type Named interface { + Name() string +} + +type TestCase interface { + Named + internalTestCase + + T() *testing.T + SetT(*testing.T) +} + +type internalTestCase interface { + setClusterInfo(*ClusterInfo) +} + +type BeforeAllSteps interface { + BeforeAllSteps() +} + +type AfterAllSteps interface { + AfterAllSteps() +} + +type BeforeEachStep interface { + BeforeEachStep() +} + +type AfterEachStep interface { + AfterEachStep() +} diff --git a/e2e/framework/provisioner.go b/e2e/framework/provisioner.go new file mode 100644 index 000000000..f3e41d3d4 --- /dev/null +++ b/e2e/framework/provisioner.go @@ -0,0 +1,66 @@ +package framework + +import ( + "crypto/md5" + "encoding/hex" + "fmt" + "os" + + capi "github.com/hashicorp/consul/api" + napi "github.com/hashicorp/nomad/api" + vapi "github.com/hashicorp/vault/api" +) + +type ProvisionerOptions struct { + Name string + Servers int + Clients int +} + +type ClusterInfo struct { + ID string + Name string + Servers []string + Clients []string + NomadClient *napi.Client + ConsulClient *capi.Client + VaultClient *vapi.Client +} + +type Provisioner interface { + ProvisionCluster(opts ProvisionerOptions) (*ClusterInfo, error) + DestroyCluster(clusterID string) error +} + +var DefaultProvisioner Provisioner = new(singleClusterProvisioner) + +type singleClusterProvisioner struct{} + +func (p *singleClusterProvisioner) ProvisionCluster(opts ProvisionerOptions) (*ClusterInfo, error) { + h := md5.New() + h.Write([]byte(opts.Name)) + info := &ClusterInfo{ + ID: hex.EncodeToString(h.Sum(nil))[:8], + Name: opts.Name, + } + nomadAddr := os.Getenv("NOMAD_ADDR") + if len(nomadAddr) == 0 { + return nil, fmt.Errorf("environment variable NOMAD_ADDR not set") + } + + nomadConfig := napi.DefaultConfig() + nomadConfig.Address = nomadAddr + nomadClient, err := napi.NewClient(nomadConfig) + if err != nil { + return nil, err + } + + info.NomadClient = nomadClient + + return info, err +} + +func (p *singleClusterProvisioner) DestroyCluster(_ string) error { + //Maybe try to GC things based on id? + return nil +}