diff --git a/e2e/cli/command/environment.go b/e2e/cli/command/environment.go new file mode 100644 index 000000000..da288adb7 --- /dev/null +++ b/e2e/cli/command/environment.go @@ -0,0 +1,222 @@ +package command + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "os/signal" + "path" + "path/filepath" + "strings" + "syscall" + "time" + + hclog "github.com/hashicorp/go-hclog" +) + +// environment captures all the information needed to execute terraform +// in order to setup a test environment +type environment struct { + provider string // provider ex. aws + name string // environment name ex. generic + + tf string // location of terraform binary + tfPath string // path to terraform configuration + tfState string // path to terraform state file + logger hclog.Logger +} + +func (env *environment) canonicalName() string { + return fmt.Sprintf("%s/%s", env.provider, env.name) +} + +// envResults are the fields returned after provisioning a test environment +type envResults struct { + nomadAddr string + consulAddr string + vaultAddr string +} + +// newEnv takes a path to the environments directory, environment name and provider, +// path to terraform state file and a logger and builds the environment stuct used +// to initial terraform calls +func newEnv(envPath, provider, name, tfStatePath string, logger hclog.Logger) (*environment, error) { + // Make sure terraform is on the PATH + tf, err := exec.LookPath("terraform") + if err != nil { + return nil, fmt.Errorf("failed to lookup terraform binary: %v", err) + } + + logger = logger.Named("provision").With("provider", provider, "name", name) + + // set the path to the terraform module + tfPath := path.Join(envPath, provider, name) + logger.Debug("using tf path", "path", tfPath) + if _, err := os.Stat(tfPath); os.IsNotExist(err) { + return nil, fmt.Errorf("failed to lookup terraform configuration dir %s: %v", tfPath, err) + } + + // set the path to state file + tfState := path.Join(tfStatePath, fmt.Sprintf("e2e.%s.%s.tfstate", provider, name)) + + env := &environment{ + provider: provider, + name: name, + tf: tf, + tfPath: tfPath, + tfState: tfState, + logger: logger, + } + return env, nil +} + +// envsFromGlob allows for the discovery of multiple environments using globs (*). +// ex. aws/* for all environments in aws. +func envsFromGlob(envPath, glob, tfStatePath string, logger hclog.Logger) ([]*environment, error) { + results, err := filepath.Glob(filepath.Join(envPath, glob)) + if err != nil { + return nil, err + } + + envs := []*environment{} + + for _, p := range results { + elems := strings.Split(p, "/") + name := elems[len(elems)-1] + provider := elems[len(elems)-2] + env, err := newEnv(envPath, provider, name, tfStatePath, logger) + if err != nil { + return nil, err + } + + envs = append(envs, env) + } + + return envs, nil +} + +// provision calls terraform to setup the environment with the given nomad binary +func (env *environment) provision(nomadPath string) (*envResults, error) { + tfArgs := []string{"apply", "-auto-approve", "-input=false", "-no-color", + "-state", env.tfState, + "-var", fmt.Sprintf("nomad_binary=%s", path.Join(nomadPath, "nomad")), + env.tfPath, + } + + // Setup the 'terraform apply' command + ctx := context.Background() + cmd := exec.CommandContext(ctx, env.tf, tfArgs...) + + // Funnel the stdout/stderr to logging + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, fmt.Errorf("failed to get stderr pipe: %v", err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("failed to get stdout pipe: %v", err) + } + + // Run 'terraform apply' + cmd.Start() + go tfLog(env.logger.Named("tf.stderr"), stderr) + go tfLog(env.logger.Named("tf.stdout"), stdout) + + sigChan := make(chan os.Signal) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + + cmdChan := make(chan error) + go func() { + cmdChan <- cmd.Wait() + }() + + // if an interrupt is received before terraform finished, forward signal to + // child pid + select { + case sig := <-sigChan: + env.logger.Error("interrupt received, forwarding signal to child process", + "pid", cmd.Process.Pid) + cmd.Process.Signal(sig) + if err := procWaitTimeout(cmd.Process, 5*time.Second); err != nil { + env.logger.Error("child process did not exit in time, killing forcefully", + "pid", cmd.Process.Pid) + cmd.Process.Kill() + } + return nil, fmt.Errorf("interrupt received") + case err := <-cmdChan: + if err != nil { + return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err) + } + } + + // Setup and run 'terraform output' to get the module output + cmd = exec.CommandContext(ctx, env.tf, "output", "-json", "-state", env.tfState) + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err) + } + + // Parse the json and pull out results + tfOutput := make(map[string]map[string]interface{}) + err = json.Unmarshal(out, &tfOutput) + if err != nil { + return nil, fmt.Errorf("failed to parse terraform output: %v", err) + } + + results := &envResults{} + if nomadAddr, ok := tfOutput["nomad_addr"]; ok { + results.nomadAddr = nomadAddr["value"].(string) + } else { + return nil, fmt.Errorf("'nomad_addr' field expected in terraform output, but was missing") + } + + return results, nil +} + +// destroy calls terraform to destroy the environment +func (env *environment) destroy() error { + tfArgs := []string{"destroy", "-auto-approve", "-no-color", + "-state", env.tfState, + "-var", "nomad_binary=", + env.tfPath, + } + cmd := exec.Command(env.tf, tfArgs...) + + // Funnel the stdout/stderr to logging + stderr, err := cmd.StderrPipe() + if err != nil { + return fmt.Errorf("failed to get stderr pipe: %v", err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("failed to get stdout pipe: %v", err) + } + + // Run 'terraform destroy' + cmd.Start() + go tfLog(env.logger.Named("tf.stderr"), stderr) + go tfLog(env.logger.Named("tf.stdout"), stdout) + + err = cmd.Wait() + if err != nil { + return fmt.Errorf("terraform exited with a non-zero status: %v", err) + } + + return nil +} + +func tfLog(logger hclog.Logger, r io.ReadCloser) { + defer r.Close() + scanner := bufio.NewScanner(r) + for scanner.Scan() { + logger.Debug(scanner.Text()) + } + if err := scanner.Err(); err != nil { + logger.Error("scan error", "error", err) + } + +} diff --git a/e2e/cli/command/meta.go b/e2e/cli/command/meta.go index 478e57d64..8dbd3cdda 100644 --- a/e2e/cli/command/meta.go +++ b/e2e/cli/command/meta.go @@ -14,6 +14,13 @@ type Meta struct { verbose bool } +func NewMeta(ui cli.Ui, logger hclog.Logger) Meta { + return Meta{ + Ui: ui, + logger: logger, + } +} + func (m *Meta) FlagSet(n string) *flag.FlagSet { f := flag.NewFlagSet(n, flag.ContinueOnError) diff --git a/e2e/cli/command/provision.go b/e2e/cli/command/provision.go index 42951e9c8..926beaf05 100644 --- a/e2e/cli/command/provision.go +++ b/e2e/cli/command/provision.go @@ -1,34 +1,25 @@ package command import ( - "bufio" - "context" - "encoding/json" "fmt" - "io" - "io/ioutil" "os" - "os/exec" - "path" - "path/filepath" "strings" getter "github.com/hashicorp/go-getter" hclog "github.com/hashicorp/go-hclog" - "github.com/hashicorp/nomad/helper/discover" "github.com/mitchellh/cli" ) +const ( + DefaultEnvironmentsPath = "./environments/" +) + func init() { getter.Getters["file"].(*getter.FileGetter).Copy = true } -func ProvisionCommandFactory(ui cli.Ui, logger hclog.Logger) cli.CommandFactory { +func ProvisionCommandFactory(meta Meta) cli.CommandFactory { return func() (cli.Command, error) { - meta := Meta{ - Ui: ui, - logger: logger, - } return &Provision{Meta: meta}, nil } } @@ -51,7 +42,7 @@ Provision Options: -env-path Sets the path for where to search for test environment configuration. - This defaults to './environments/'. + This defaults to './environments/'. -nomad-binary Sets the target nomad-binary to use when provisioning a nomad cluster. @@ -83,7 +74,7 @@ func (c *Provision) Run(args []string) int { var tfPath string cmdFlags := c.FlagSet("provision") cmdFlags.Usage = func() { c.Ui.Output(c.Help()) } - cmdFlags.StringVar(&envPath, "env-path", "./environments/", "Path to e2e environment terraform configs") + cmdFlags.StringVar(&envPath, "env-path", DefaultEnvironmentsPath, "Path to e2e environment terraform configs") cmdFlags.StringVar(&nomadBinary, "nomad-binary", "", "") cmdFlags.BoolVar(&destroy, "destroy", false, "") cmdFlags.StringVar(&tfPath, "tf-path", "", "") @@ -119,6 +110,10 @@ func (c *Provision) Run(args []string) int { // Use go-getter to fetch the nomad binary nomadPath, err := fetchBinary(nomadBinary) defer os.RemoveAll(nomadPath) + if err != nil { + c.logger.Error("failed to fetch nomad binary", "error", err) + return 1 + } results, err := env.provision(nomadPath) if err != nil { @@ -132,194 +127,3 @@ NOMAD_ADDR=%s return 0 } - -// Fetches the nomad binary and returns the temporary directory where it exists -func fetchBinary(bin string) (string, error) { - nomadBinaryDir, err := ioutil.TempDir("", "") - if err != nil { - return "", fmt.Errorf("failed to create temp dir: %v", err) - } - - if bin == "" { - bin, err = discover.NomadExecutable() - if err != nil { - return "", fmt.Errorf("failed to discover nomad binary: %v", err) - } - } - if err = getter.GetFile(path.Join(nomadBinaryDir, "nomad"), bin); err != nil { - return "", fmt.Errorf("failed to get nomad binary: %v", err) - } - - return nomadBinaryDir, nil -} - -type environment struct { - path string - provider string - name string - - tf string - tfPath string - tfState string - logger hclog.Logger -} - -type envResults struct { - nomadAddr string - consulAddr string - vaultAddr string -} - -func newEnv(envPath, provider, name, tfStatePath string, logger hclog.Logger) (*environment, error) { - // Make sure terraform is on the PATH - tf, err := exec.LookPath("terraform") - if err != nil { - return nil, fmt.Errorf("failed to lookup terraform binary: %v", err) - } - - logger = logger.Named("provision").With("provider", provider, "name", name) - - // set the path to the terraform module - tfPath := path.Join(envPath, provider, name) - logger.Debug("using tf path", "path", tfPath) - if _, err := os.Stat(tfPath); os.IsNotExist(err) { - return nil, fmt.Errorf("failed to lookup terraform configuration dir %s: %v", tfPath, err) - } - - // set the path to state file - tfState := path.Join(tfStatePath, fmt.Sprintf("e2e.%s.%s.tfstate", provider, name)) - - env := &environment{ - path: envPath, - provider: provider, - name: name, - tf: tf, - tfPath: tfPath, - tfState: tfState, - logger: logger, - } - return env, nil -} - -// envsFromGlob allows for the discovery of multiple environments using globs (*). -// ex. aws/* for all environments in aws. -func envsFromGlob(envPath, glob, tfStatePath string, logger hclog.Logger) ([]*environment, error) { - results, err := filepath.Glob(filepath.Join(envPath, glob)) - if err != nil { - return nil, err - } - - envs := []*environment{} - - for _, p := range results { - elems := strings.Split(p, "/") - name := elems[len(elems)-1] - provider := elems[len(elems)-2] - env, err := newEnv(envPath, provider, name, tfStatePath, logger) - if err != nil { - return nil, err - } - - envs = append(envs, env) - } - - return envs, nil -} - -// provision calls terraform to setup the environment with the given nomad binary -func (env *environment) provision(nomadPath string) (*envResults, error) { - tfArgs := []string{"apply", "-auto-approve", "-input=false", "-no-color", - "-state", env.tfState, - "-var", fmt.Sprintf("nomad_binary=%s", path.Join(nomadPath, "nomad")), - env.tfPath, - } - - // Setup the 'terraform apply' command - ctx := context.Background() - cmd := exec.CommandContext(ctx, env.tf, tfArgs...) - - // Funnel the stdout/stderr to logging - stderr, err := cmd.StderrPipe() - if err != nil { - return nil, fmt.Errorf("failed to get stderr pipe: %v", err) - } - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, fmt.Errorf("failed to get stdout pipe: %v", err) - } - - // Run 'terraform apply' - cmd.Start() - go tfLog(env.logger.Named("tf.stderr"), stderr) - go tfLog(env.logger.Named("tf.stdout"), stdout) - - err = cmd.Wait() - if err != nil { - return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err) - } - - // Setup and run 'terraform output' to get the module output - cmd = exec.CommandContext(ctx, env.tf, "output", "-json", "-state", env.tfState) - out, err := cmd.Output() - if err != nil { - return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err) - } - - // Parse the json and pull out results - tfOutput := make(map[string]map[string]interface{}) - err = json.Unmarshal(out, &tfOutput) - if err != nil { - return nil, fmt.Errorf("failed to parse terraform output: %v", err) - } - - results := &envResults{} - if nomadAddr, ok := tfOutput["nomad_addr"]; ok { - results.nomadAddr = nomadAddr["value"].(string) - } - - return results, nil -} - -//destroy calls terraform to destroy the environment -func (env *environment) destroy() error { - tfArgs := []string{"destroy", "-auto-approve", "-no-color", - "-state", env.tfState, - "-var", "nomad_binary=", - env.tfPath, - } - cmd := exec.Command(env.tf, tfArgs...) - - // Funnel the stdout/stderr to logging - stderr, err := cmd.StderrPipe() - if err != nil { - return fmt.Errorf("failed to get stderr pipe: %v", err) - } - stdout, err := cmd.StdoutPipe() - if err != nil { - return fmt.Errorf("failed to get stdout pipe: %v", err) - } - - // Run 'terraform destroy' - cmd.Start() - go tfLog(env.logger.Named("tf.stderr"), stderr) - go tfLog(env.logger.Named("tf.stdout"), stdout) - - err = cmd.Wait() - if err != nil { - return fmt.Errorf("terraform exited with a non-zero status: %v", err) - } - - return nil -} - -func tfLog(logger hclog.Logger, r io.ReadCloser) { - defer r.Close() - scanner := bufio.NewScanner(r) - for scanner.Scan() { - logger.Debug(scanner.Text()) - } - if err := scanner.Err(); err != nil { - logger.Error("scan error", "error", err) - } - -} diff --git a/e2e/cli/command/run.go b/e2e/cli/command/run.go index 76001baa4..c7b1f84b9 100644 --- a/e2e/cli/command/run.go +++ b/e2e/cli/command/run.go @@ -12,12 +12,8 @@ import ( "github.com/mitchellh/cli" ) -func RunCommandFactory(ui cli.Ui, logger hclog.Logger) cli.CommandFactory { +func RunCommandFactory(meta Meta) cli.CommandFactory { return func() (cli.Command, error) { - meta := Meta{ - Ui: ui, - logger: logger, - } return &Run{Meta: meta}, nil } } @@ -45,7 +41,7 @@ func (c *Run) Run(args []string) int { var run string cmdFlags := c.FlagSet("run") cmdFlags.Usage = func() { c.Ui.Output(c.Help()) } - cmdFlags.StringVar(&envPath, "env-path", "./environments/", "Path to e2e environment terraform configs") + cmdFlags.StringVar(&envPath, "env-path", DefaultEnvironmentsPath, "Path to e2e environment terraform configs") cmdFlags.StringVar(&nomadBinary, "nomad-binary", "", "") cmdFlags.StringVar(&tfPath, "tf-path", "", "") cmdFlags.StringVar(&run, "run", "", "Regex to target specific test suites/cases") @@ -63,22 +59,21 @@ func (c *Run) Run(args []string) int { if len(args) == 0 { c.logger.Info("no environments specified, running test suite locally") - var report *TestReport - var err error - if report, err = c.run(&runOpts{ + report, err := c.runTest(&runOpts{ slow: slow, verbose: c.verbose, - }); err != nil { + }) + if err != nil { c.logger.Error("failed to run test suite", "error", err) return 1 } - if report.TotalFailedTests == 0 { - c.Ui.Output("PASSED!") - if c.verbose { - c.Ui.Output(report.Summary()) - } - } else { - c.Ui.Output("***FAILED***") + if report.TotalFailedTests > 0 { + c.Ui.Error("***FAILED***") + c.Ui.Error(report.Summary()) + return 1 + } + c.Ui.Output("PASSED!") + if c.verbose { c.Ui.Output(report.Summary()) } return 0 @@ -98,7 +93,7 @@ func (c *Run) Run(args []string) int { environments = append(environments, envs...) } - envCount := len(environments) + // Use go-getter to fetch the nomad binary nomadPath, err := fetchBinary(nomadBinary) defer os.RemoveAll(nomadPath) @@ -107,7 +102,9 @@ func (c *Run) Run(args []string) int { return 1 } + envCount := len(environments) c.logger.Debug("starting tests", "totalEnvironments", envCount) + failedEnvs := map[string]*TestReport{} for i, env := range environments { logger := c.logger.With("name", env.name, "provider", env.provider) logger.Debug("provisioning environment") @@ -128,25 +125,35 @@ func (c *Run) Run(args []string) int { } var report *TestReport - if report, err = c.run(opts); err != nil { + if report, err = c.runTest(opts); err != nil { logger.Error("failed to run tests against environment", "error", err) return 1 } - if report.TotalFailedTests == 0 { + if report.TotalFailedTests > 0 { + c.Ui.Error(fmt.Sprintf("[%d/%d] %s: ***FAILED***", i+1, envCount, env.canonicalName())) + c.Ui.Error(fmt.Sprintf("[%d/%d] %s: %s", i+1, envCount, env.canonicalName(), report.Summary())) + failedEnvs[env.canonicalName()] = report + } - c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: PASSED!", i+1, envCount, env.provider, env.name)) - if c.verbose { - c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: %s", i+1, envCount, env.provider, env.name, report.Summary())) - } - } else { - c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: ***FAILED***", i+1, envCount, env.provider, env.name)) - c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: %s", i+1, envCount, env.provider, env.name, report.Summary())) + c.Ui.Output(fmt.Sprintf("[%d/%d] %s: PASSED!", i+1, envCount, env.canonicalName())) + if c.verbose { + c.Ui.Output(fmt.Sprintf("[%d/%d] %s: %s", i+1, envCount, env.canonicalName(), report.Summary())) } } + + if len(failedEnvs) > 0 { + c.Ui.Error(fmt.Sprintf("The following environments ***FAILED***")) + for name, report := range failedEnvs { + c.Ui.Error(fmt.Sprintf(" [%s]: %d out of %d suite failures", + name, report.TotalFailedSuites, report.TotalSuites)) + } + return 1 + } + c.Ui.Output("All Environments PASSED!") return 0 } -func (c *Run) run(opts *runOpts) (*TestReport, error) { +func (c *Run) runTest(opts *runOpts) (*TestReport, error) { goBin, err := exec.LookPath("go") if err != nil { return nil, err @@ -175,6 +182,8 @@ func (c *Run) run(opts *runOpts) (*TestReport, error) { } +// runOpts contains fields used to build the arguments and environment variabled +// nessicary to run go test and initialize the e2e framework type runOpts struct { nomadAddr string consulAddr string @@ -186,6 +195,8 @@ type runOpts struct { verbose bool } +// goArgs returns the list of arguments passed to the go command to start the +// e2e test framework func (opts *runOpts) goArgs() []string { a := []string{ "test", @@ -205,6 +216,8 @@ func (opts *runOpts) goArgs() []string { return a } +// goEnv returns the list of environment variabled passed to the go command to start +// the e2e test framework func (opts *runOpts) goEnv() []string { env := append(os.Environ(), "NOMAD_E2E=1") if opts.nomadAddr != "" { diff --git a/e2e/cli/command/util.go b/e2e/cli/command/util.go new file mode 100644 index 000000000..daf9ed30e --- /dev/null +++ b/e2e/cli/command/util.go @@ -0,0 +1,55 @@ +package command + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "runtime" + "time" + + getter "github.com/hashicorp/go-getter" + "github.com/hashicorp/nomad/helper/discover" +) + +// Fetches the nomad binary and returns the temporary directory where it exists +func fetchBinary(bin string) (string, error) { + nomadBinaryDir, err := ioutil.TempDir("", "") + if err != nil { + return "", fmt.Errorf("failed to create temp dir: %v", err) + } + + if bin == "" { + bin, err = discover.NomadExecutable() + if err != nil { + return "", fmt.Errorf("failed to discover nomad binary: %v", err) + } + } + + dest := path.Join(nomadBinaryDir, "nomad") + if runtime.GOOS == "windows" { + dest = dest + ".exe" + } + + if err = getter.GetFile(dest, bin); err != nil { + return "", fmt.Errorf("failed to get nomad binary: %v", err) + } + + return nomadBinaryDir, nil +} + +func procWaitTimeout(p *os.Process, d time.Duration) error { + stop := make(chan struct{}) + + go func() { + p.Wait() + stop <- struct{}{} + }() + + select { + case <-stop: + return nil + case <-time.NewTimer(d).C: + return fmt.Errorf("timeout waiting for process %d to exit", p.Pid) + } +} diff --git a/e2e/cli/main.go b/e2e/cli/main.go index f2340d5f0..18670e202 100644 --- a/e2e/cli/main.go +++ b/e2e/cli/main.go @@ -8,6 +8,11 @@ import ( "github.com/mitchellh/cli" ) +const ( + NomadE2ECli = "nomad-e2e" + NomadE2ECliVersion = "0.0.1" +) + func main() { ui := &cli.BasicUi{ @@ -17,15 +22,17 @@ func main() { } logger := hclog.New(&hclog.LoggerOptions{ - Name: "nomad-e2e", - Output: &cli.UiWriter{ui}, + Name: NomadE2ECli, + Output: &cli.UiWriter{Ui: ui}, }) - c := cli.NewCLI("nomad-e2e", "0.0.1") + c := cli.NewCLI(NomadE2ECli, NomadE2ECliVersion) c.Args = os.Args[1:] + + meta := command.NewMeta(ui, logger) c.Commands = map[string]cli.CommandFactory{ - "provision": command.ProvisionCommandFactory(ui, logger), - "run": command.RunCommandFactory(ui, logger), + "provision": command.ProvisionCommandFactory(meta), + "run": command.RunCommandFactory(meta), } exitStatus, err := c.Run()