e2e/framework: initial e2e framework implementation

e2e/e2e.go

@@ -0,0 +1,11 @@
+package e2e
+
+import (
+	"testing"
+
+	"github.com/hashicorp/nomad/e2e/framework"
+)
+
+func RunE2ETests(t *testing.T) {
+	framework.Run(t)
+}

e2e/e2e_test.go

@@ -0,0 +1,11 @@
+package e2e
+
+import (
+	"testing"
+
+	_ "github.com/hashicorp/nomad/e2e/example"
+)
+
+func TestE2E(t *testing.T) {
+	RunE2ETests(t)
+}

e2e/example/e2e_test.go

@@ -0,0 +1,17 @@
+package example
+
+import (
+	"testing"
+
+	"github.com/hashicorp/nomad/e2e/framework"
+)
+
+func TestE2E(t *testing.T) {
+	framework.New().AddSuites(&framework.TestSuite{
+		Component:   "simple",
+		CanRunLocal: true,
+		Cases: []framework.TestCase{
+			new(SimpleExampleTestCase),
+		},
+	}).Run(t)
+}

e2e/example/example.go

@@ -0,0 +1,25 @@
+package example
+
+import (
+	"github.com/hashicorp/nomad/e2e/framework"
+)
+
+func init() {
+	framework.AddSuites(&framework.TestSuite{
+		Component:   "simple",
+		CanRunLocal: true,
+		Cases: []framework.TestCase{
+			new(SimpleExampleTestCase),
+		},
+	})
+}
+
+type SimpleExampleTestCase struct {
+	framework.TC
+}
+
+func (tc *SimpleExampleTestCase) TestExample() {
+	jobs, _, err := tc.Nomad().Jobs().List(nil)
+	tc.NoError(err)
+	tc.Empty(jobs)
+}

e2e/framework/case.go

@@ -0,0 +1,89 @@
+package framework
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/hashicorp/nomad/api"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type TestSuite struct {
+	Component string
+
+	CanRunLocal bool
+	Cases       []TestCase
+	Constraints Constraints
+	Parallel    bool
+	Slow        bool
+}
+
+type Constraints struct {
+	CloudProvider string
+	OS            string
+	Arch          string
+	Environment   string
+	Tags          []string
+}
+
+func (c Constraints) matches(env Environment) error {
+	if len(c.CloudProvider) != 0 && c.CloudProvider != env.Provider {
+		return fmt.Errorf("provider constraint does not match environment")
+	}
+
+	if len(c.OS) != 0 && c.OS != env.OS {
+		return fmt.Errorf("os constraint does not match environment")
+	}
+
+	if len(c.Arch) != 0 && c.Arch != env.Arch {
+		return fmt.Errorf("arch constraint does not match environment")
+	}
+
+	if len(c.Environment) != 0 && c.Environment != env.Name {
+		return fmt.Errorf("environment constraint does not match environment name")
+	}
+
+	for _, t := range c.Tags {
+		if _, ok := env.Tags[t]; !ok {
+			return fmt.Errorf("tags constraint failed, tag '%s' is not included in environment")
+		}
+	}
+	return nil
+}
+
+type TC struct {
+	*assert.Assertions
+	require *require.Assertions
+	t       *testing.T
+
+	cluster *ClusterInfo
+	prefix  string
+	name    string
+}
+
+func (tc *TC) Nomad() *api.Client {
+	return tc.cluster.NomadClient
+}
+
+func (tc *TC) Prefix() string {
+	return fmt.Sprintf("%s-", tc.cluster.ID)
+}
+
+func (tc *TC) Name() string {
+	return tc.cluster.Name
+}
+
+func (tc *TC) T() *testing.T {
+	return tc.t
+}
+
+func (tc *TC) SetT(t *testing.T) {
+	tc.t = t
+	tc.Assertions = assert.New(t)
+	tc.require = require.New(t)
+}
+
+func (tc *TC) setClusterInfo(info *ClusterInfo) {
+	tc.cluster = info
+}

e2e/framework/framework.go

@@ -0,0 +1,183 @@
+package framework
+
+import (
+	"flag"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+var fProvider = flag.String("nomad.env.provider", "", "cloud provider for which environment is executing against")
+var fEnv = flag.String("nomad.env", "", "name of the environment executing against")
+var fOS = flag.String("nomad.os", "", "operating system for which the environment is executing against")
+var fArch = flag.String("nomad.arch", "", "cpu architecture for which the environment is executing against")
+var fTags = flag.String("nomad.tags", "", "comma delimited list of tags associated with the environment")
+var fLocal = flag.Bool("nomad.local", false, "denotes execution is against a local environment")
+var fSlow = flag.Bool("nomad.slow", false, "toggles execution of slow test suites")
+var fForceAll = flag.Bool("nomad.force", false, "if set, skips all environment checks when filtering test suites")
+
+var pkgFramework = New()
+
+type Framework struct {
+	suites      []*TestSuite
+	provisioner Provisioner
+	env         Environment
+
+	isLocalRun bool
+	slow       bool
+	force      bool
+}
+
+type Environment struct {
+	Name     string
+	Provider string
+	OS       string
+	Arch     string
+	Tags     map[string]struct{}
+}
+
+func New() *Framework {
+	env := Environment{
+		Name:     *fEnv,
+		Provider: *fProvider,
+		OS:       *fOS,
+		Arch:     *fArch,
+		Tags:     map[string]struct{}{},
+	}
+	for _, tag := range strings.Split(*fTags, ",") {
+		env.Tags[tag] = struct{}{}
+	}
+	return &Framework{
+		provisioner: DefaultProvisioner,
+		env:         env,
+		isLocalRun:  *fLocal,
+		slow:        *fSlow,
+		force:       *fForceAll,
+	}
+}
+
+func (f *Framework) AddSuites(s ...*TestSuite) *Framework {
+	f.suites = append(f.suites, s...)
+	return f
+}
+
+func AddSuites(s ...*TestSuite) *Framework {
+	pkgFramework.AddSuites(s...)
+	return pkgFramework
+}
+
+// Run starts the test framework and runs each TestSuite
+func (f *Framework) Run(t *testing.T) {
+	for _, s := range f.suites {
+		t.Run(s.Component, func(t *testing.T) {
+			skip, err := f.runSuite(t, s)
+			if skip {
+				t.Skipf("skipping suite '%s': %v", s.Component, err)
+				return
+			}
+			if err != nil {
+				t.Errorf("error starting suite '%s': %v", s.Component, err)
+			}
+		})
+	}
+
+}
+
+func Run(t *testing.T) {
+	pkgFramework.Run(t)
+}
+
+// runSuite is called from Framework.Run inside of a sub test for each TestSuite
+func (f *Framework) runSuite(t *testing.T, s *TestSuite) (skip bool, err error) {
+
+	if !f.force {
+		// If this is a local run, check that the suite supports running locally
+		if !s.CanRunLocal && f.isLocalRun {
+			return true, fmt.Errorf("local run detected and suite cannot run locally")
+		}
+
+		// Check that constraints are met
+		if err := s.Constraints.matches(f.env); err != nil {
+			return true, fmt.Errorf("constraint failed: %v", err)
+		}
+
+		// Check the slow toggle and if the suite's slow flag is that same
+		if f.slow != s.Slow {
+			return true, fmt.Errorf("framework slow suite configuration is %v but suite is %v", f.slow, s.Slow)
+		}
+	}
+
+	for _, c := range s.Cases {
+		name := fmt.Sprintf("%T", c)
+		// Each TestCase is provisioned a nomad cluster
+		info, err := f.provisioner.ProvisionCluster(ProvisionerOptions{Name: name})
+		if err != nil {
+			return false, fmt.Errorf("could not provision cluster for case: %v", err)
+		}
+		defer f.provisioner.DestroyCluster(info.ID)
+
+		c.setClusterInfo(info)
+		// Each TestCase runs as a subtest of the TestSuite
+		t.Run(c.Name(), func(t *testing.T) {
+			c.SetT(t)
+			// If the TestSuite has Parallel set, all cases run in parallel
+			if s.Parallel {
+				t.Parallel()
+			}
+
+			// Check if the case includes a before all function
+			if beforeAllSteps, ok := c.(BeforeAllSteps); ok {
+				beforeAllSteps.BeforeAllSteps()
+			}
+
+			// Check if the case includes an after all function at the end
+			defer func() {
+				if afterAllSteps, ok := c.(AfterAllSteps); ok {
+					afterAllSteps.AfterAllSteps()
+				}
+			}()
+
+			// Here we need to iterate through the methods of the case to find
+			// ones that at test functions
+			reflectC := reflect.TypeOf(c)
+			for i := 0; i < reflectC.NumMethod(); i++ {
+				method := reflectC.Method(i)
+				if ok, _ := isTestMethod(method.Name); !ok {
+					continue
+				}
+				// Each step is run as its own sub test of the case
+				t.Run(method.Name, func(t *testing.T) {
+
+					// Since the test function interacts with testing.T through
+					// the test case struct, we need to swap the test context for
+					// the duration of the step.
+					parentT := c.T()
+					c.SetT(t)
+					if BeforeEachStep, ok := c.(BeforeEachStep); ok {
+						BeforeEachStep.BeforeEachStep()
+					}
+					defer func() {
+						if afterEachStep, ok := c.(AfterEachStep); ok {
+							afterEachStep.AfterEachStep()
+						}
+						c.SetT(parentT)
+					}()
+					//Call the method
+					method.Func.Call([]reflect.Value{reflect.ValueOf(c)})
+				})
+			}
+		})
+
+	}
+
+	return false, nil
+}
+
+func isTestMethod(m string) (bool, error) {
+	if !strings.HasPrefix(m, "Test") {
+		return false, nil
+	}
+
+	return true, nil
+}

e2e/framework/interfaces.go

@@ -0,0 +1,39 @@
+package framework
+
+import (
+	"testing"
+)
+
+// Named exists simply to make sure the Name() method was implemented since it
+// is the only required method implementation of a test case
+type Named interface {
+	Name() string
+}
+
+type TestCase interface {
+	Named
+	internalTestCase
+
+	T() *testing.T
+	SetT(*testing.T)
+}
+
+type internalTestCase interface {
+	setClusterInfo(*ClusterInfo)
+}
+
+type BeforeAllSteps interface {
+	BeforeAllSteps()
+}
+
+type AfterAllSteps interface {
+	AfterAllSteps()
+}
+
+type BeforeEachStep interface {
+	BeforeEachStep()
+}
+
+type AfterEachStep interface {
+	AfterEachStep()
+}

e2e/framework/provisioner.go

@@ -0,0 +1,66 @@
+package framework
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"os"
+
+	capi "github.com/hashicorp/consul/api"
+	napi "github.com/hashicorp/nomad/api"
+	vapi "github.com/hashicorp/vault/api"
+)
+
+type ProvisionerOptions struct {
+	Name    string
+	Servers int
+	Clients int
+}
+
+type ClusterInfo struct {
+	ID           string
+	Name         string
+	Servers      []string
+	Clients      []string
+	NomadClient  *napi.Client
+	ConsulClient *capi.Client
+	VaultClient  *vapi.Client
+}
+
+type Provisioner interface {
+	ProvisionCluster(opts ProvisionerOptions) (*ClusterInfo, error)
+	DestroyCluster(clusterID string) error
+}
+
+var DefaultProvisioner Provisioner = new(singleClusterProvisioner)
+
+type singleClusterProvisioner struct{}
+
+func (p *singleClusterProvisioner) ProvisionCluster(opts ProvisionerOptions) (*ClusterInfo, error) {
+	h := md5.New()
+	h.Write([]byte(opts.Name))
+	info := &ClusterInfo{
+		ID:   hex.EncodeToString(h.Sum(nil))[:8],
+		Name: opts.Name,
+	}
+	nomadAddr := os.Getenv("NOMAD_ADDR")
+	if len(nomadAddr) == 0 {
+		return nil, fmt.Errorf("environment variable NOMAD_ADDR not set")
+	}
+
+	nomadConfig := napi.DefaultConfig()
+	nomadConfig.Address = nomadAddr
+	nomadClient, err := napi.NewClient(nomadConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	info.NomadClient = nomadClient
+
+	return info, err
+}
+
+func (p *singleClusterProvisioner) DestroyCluster(_ string) error {
+	//Maybe try to GC things based on id?
+	return nil
+}