e2e/cli: implemented run logic

e2e/cli/command/provision.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"strings"
 
 	getter "github.com/hashicorp/go-getter"
@@ -87,8 +88,7 @@ func (c *Provision) Run(args []string) int {
 
 	args = cmdFlags.Args()
 	if len(args) != 2 {
-		log.Fatalf("expected 2 args, but got: %v", args)
-		log.Fatal(c.Help())
+		log.Fatalf("expected 2 args (provider and environment), but got: %v", args)
 	}
 
 	env, err := newEnv(envPath, args[0], args[1], tfPath)
@@ -101,12 +101,12 @@ func (c *Provision) Run(args []string) int {
 			log.Fatal(err)
 			return 1
 		}
-		fmt.Println("Environment successfully destroyed")
+		log.Println("Environment successfully destroyed")
 		return 0
 	}
 
 	// Use go-getter to fetch the nomad binary
-	nomadPath, err := c.fetchBinary(nomadBinary)
+	nomadPath, err := fetchBinary(nomadBinary)
 	defer os.RemoveAll(nomadPath)
 
 	results, err := env.provision(nomadPath)
@@ -122,7 +122,7 @@ NOMAD_ADDR=%s
 }
 
 // Fetches the nomad binary and returns the temporary directory where it exists
-func (c *Provision) fetchBinary(bin string) (string, error) {
+func fetchBinary(bin string) (string, error) {
 	nomadBinaryDir, err := ioutil.TempDir("", "")
 	if err != nil {
 		return "", fmt.Errorf("failed to create temp dir: %v", err)
@@ -152,7 +152,9 @@ type environment struct {
 }
 
 type envResults struct {
-	nomadAddr string
+	nomadAddr  string
+	consulAddr string
+	vaultAddr  string
 }
 
 func newEnv(envPath, provider, name, tfStatePath string) (*environment, error) {
@@ -183,6 +185,31 @@ func newEnv(envPath, provider, name, tfStatePath string) (*environment, error) {
 	return env, nil
 }
 
+// envsFromGlob allows for the discovery of multiple environments using globs (*).
+// ex. aws/* for all environments in aws.
+func envsFromGlob(envPath, glob, tfStatePath string) ([]*environment, error) {
+	results, err := filepath.Glob(filepath.Join(envPath, glob))
+	if err != nil {
+		return nil, err
+	}
+
+	envs := []*environment{}
+
+	for _, p := range results {
+		elems := strings.Split(p, "/")
+		name := elems[len(elems)-1]
+		provider := elems[len(elems)-2]
+		env, err := newEnv(envPath, provider, name, tfStatePath)
+		if err != nil {
+			return nil, err
+		}
+
+		envs = append(envs, env)
+	}
+
+	return envs, nil
+}
+
 // provision calls terraform to setup the environment with the given nomad binary
 func (env *environment) provision(nomadPath string) (*envResults, error) {
 	tfArgs := []string{"apply", "-auto-approve", "-input=false", "-no-color",

e2e/cli/command/run.go

@@ -1,10 +1,16 @@
 package command
 
 import (
+	"flag"
 	"fmt"
+	"io"
+	"log"
+	"os"
 	"os/exec"
 	"strings"
 
+	capi "github.com/hashicorp/consul/api"
+	vapi "github.com/hashicorp/vault/api"
 	"github.com/mitchellh/cli"
 )
 
@@ -27,44 +33,180 @@ func (c *Run) Synopsis() string {
 }
 
 func (c *Run) Run(args []string) int {
-	if err := c.run(); err != nil {
-		fmt.Println(err)
-		return 1
+	var envPath string
+	var nomadBinary string
+	var tfPath string
+	var slow bool
+	var run string
+	var verbose bool
+	cmdFlags := flag.NewFlagSet("run", flag.ContinueOnError)
+	cmdFlags.Usage = func() { log.Println(c.Help()) }
+	cmdFlags.StringVar(&envPath, "env-path", "./environments/", "Path to e2e environment terraform configs")
+	cmdFlags.StringVar(&nomadBinary, "nomad-binary", "", "")
+	cmdFlags.StringVar(&tfPath, "tf-path", "", "")
+	cmdFlags.StringVar(&run, "run", "", "Regex to target specific test suites/cases")
+	cmdFlags.BoolVar(&slow, "slow", false, "Toggle slow running suites")
+	cmdFlags.BoolVar(&verbose, "v", false, "Toggle verbose output")
+
+	if err := cmdFlags.Parse(args); err != nil {
+		log.Fatalf("failed to parse flags: %v", err)
+	}
+
+	args = cmdFlags.Args()
+
+	if len(args) == 0 {
+		log.Println("No environments specified, running test suite locally...")
+		var report *TestReport
+		var err error
+		if report, err = c.run(&runOpts{
+			slow:    slow,
+			verbose: verbose,
+		}); err != nil {
+			log.Fatalf("failed to run test suite: %v", err)
+		}
+		if report.TotalFailedTests == 0 {
+			log.Println("PASSED!")
+			if verbose {
+				log.Println(report.Summary())
+			}
+		} else {
+			log.Println("***FAILED***")
+			log.Println(report.Summary())
+		}
+		return 0
+	}
+
+	environments := []*environment{}
+	for _, e := range args {
+		if len(strings.Split(e, "/")) != 2 {
+			log.Fatalf("argument %s should be formated as <provider>/<environment>", e)
+		}
+		envs, err := envsFromGlob(envPath, e, tfPath)
+		if err != nil {
+			log.Fatalf("failed to build environment %s: %v", e, err)
+		}
+		environments = append(environments, envs...)
+
+	}
+	envCount := len(environments)
+	// Use go-getter to fetch the nomad binary
+	nomadPath, err := fetchBinary(nomadBinary)
+	defer os.RemoveAll(nomadPath)
+	if err != nil {
+		log.Fatal("failed to fetch nomad binary: %v", err)
+	}
+
+	log.Printf("Running tests against %d environments...", envCount)
+	for i, env := range environments {
+		log.Printf("[%d/%d] provisioning %s environment on %s provider", i+1, envCount, env.name, env.provider)
+		results, err := env.provision(nomadPath)
+		if err != nil {
+			log.Fatalf("failed to provision environment %s/%s: %v", env.provider, env.name, err)
+		}
+
+		opts := &runOpts{
+			provider:   env.provider,
+			env:        env.name,
+			slow:       slow,
+			verbose:    verbose,
+			nomadAddr:  results.nomadAddr,
+			consulAddr: results.consulAddr,
+			vaultAddr:  results.vaultAddr,
+		}
+
+		var report *TestReport
+		if report, err = c.run(opts); err != nil {
+			log.Printf("failed to run tests against environment %s/%s: %v", env.provider, env.name, err)
+			return 1
+		}
+		if report.TotalFailedTests == 0 {
+			log.Printf("[%d/%d] %s/%s: PASSED!\n", i, envCount, env.provider, env.name)
+			if verbose {
+				log.Printf("[%d/%d] %s/%s: %s", i, envCount, env.provider, env.name, report.Summary())
+			}
+		} else {
+			log.Printf("[%d/%d] %s/%s: ***FAILED***\n", i, envCount, env.provider, env.name)
+			log.Printf("[%d/%d] %s/%s: %s", i, envCount, env.provider, env.name, report.Summary())
+		}
 	}
 	return 0
 }
 
-func (c *Run) run() error {
+func (c *Run) run(opts *runOpts) (*TestReport, error) {
 	goBin, err := exec.LookPath("go")
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	goArgs := []string{
-		"test",
-		"-json",
-		"github.com/hashicorp/nomad/e2e",
-	}
-
-	cmd := exec.Command(goBin, goArgs...)
+	cmd := exec.Command(goBin, opts.goArgs()...)
+	cmd.Env = opts.goEnv()
 	out, err := cmd.StdoutPipe()
 	defer out.Close()
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	err = cmd.Start()
 	if err != nil {
-		return err
+		return nil, err
+	}
+
+	var logger io.Writer
+	if opts.verbose {
+		logger = os.Stdout
 	}
 
 	dec := NewDecoder(out)
-	report, err := dec.Decode()
+	report, err := dec.Decode(logger)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	fmt.Println(report.Summary())
+	return report, nil
 
-	return nil
+}
+
+type runOpts struct {
+	nomadAddr  string
+	consulAddr string
+	vaultAddr  string
+	provider   string
+	env        string
+	local      bool
+	slow       bool
+	verbose    bool
+}
+
+func (opts *runOpts) goArgs() []string {
+	a := []string{
+		"test",
+		"-json",
+		"github.com/hashicorp/nomad/e2e",
+		"-env=" + opts.env,
+		"-env.provider=" + opts.provider,
+	}
+
+	if opts.slow {
+		a = append(a, "-slow")
+	}
+
+	if opts.local {
+		a = append(a, "-local")
+	}
+	return a
+}
+
+func (opts *runOpts) goEnv() []string {
+	env := append(os.Environ(), "NOMAD_E2E=1")
+	if opts.nomadAddr != "" {
+		env = append(env, "NOMAD_ADDR="+opts.nomadAddr)
+	}
+	if opts.consulAddr != "" {
+		env = append(env, fmt.Sprintf("%s=%s", capi.HTTPAddrEnvName, opts.consulAddr))
+	}
+	if opts.vaultAddr != "" {
+		env = append(env, fmt.Sprintf("%s=%s", vapi.EnvVaultAddress, opts.consulAddr))
+	}
+
+	return env
 }

e2e/cli/command/test_decoder.go

@@ -79,7 +79,7 @@ func NewDecoder(r io.Reader) *EventDecoder {
 	}
 }
 
-func (d *EventDecoder) Decode() (*TestReport, error) {
+func (d *EventDecoder) Decode(logger io.Writer) (*TestReport, error) {
 	for d.dec.More() {
 		var e TestEvent
 		err := d.dec.Decode(&e)
@@ -88,6 +88,9 @@ func (d *EventDecoder) Decode() (*TestReport, error) {
 		}
 
 		d.report.record(&e)
+		if logger != nil {
+			logger.Write([]byte(e.Output))
+		}
 	}
 	return d.report, nil
 }