kemko/nomad
1
0
Fork 0
mirror of https://github.com/kemko/nomad.git synced 2026-01-04 17:35:43 +03:00
Code Issues Packages Projects Releases Wiki Activity

e2e/cli: code review comments, restructing and cleanup

Browse Source
This commit is contained in:
Nick Ethier
2018-07-31 11:41:20 -04:00
parent 580c4c29ff
commit 0691fa0ccc
6 changed files with 348 additions and 240 deletions
Download Patch File Download Diff File Expand all files Collapse all files

222
e2e/cli/command/environment.go Normal file
View File

@@ -0,0 +1,222 @@
package command
import (
"bufio"
"context"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"os/signal"
"path"
"path/filepath"
"strings"
"syscall"
"time"
hclog "github.com/hashicorp/go-hclog"
)
// environment captures all the information needed to execute terraform
// in order to setup a test environment
type environment struct {
provider string // provider ex. aws
name string // environment name ex. generic
tf string // location of terraform binary
tfPath string // path to terraform configuration
tfState string // path to terraform state file
logger hclog.Logger
}
func (env *environment) canonicalName() string {
return fmt.Sprintf("%s/%s", env.provider, env.name)
}
// envResults are the fields returned after provisioning a test environment
type envResults struct {
nomadAddr string
consulAddr string
vaultAddr string
}
// newEnv takes a path to the environments directory, environment name and provider,
// path to terraform state file and a logger and builds the environment stuct used
// to initial terraform calls
func newEnv(envPath, provider, name, tfStatePath string, logger hclog.Logger) (*environment, error) {
// Make sure terraform is on the PATH
tf, err := exec.LookPath("terraform")
if err != nil {
return nil, fmt.Errorf("failed to lookup terraform binary: %v", err)
}
logger = logger.Named("provision").With("provider", provider, "name", name)
// set the path to the terraform module
tfPath := path.Join(envPath, provider, name)
logger.Debug("using tf path", "path", tfPath)
if _, err := os.Stat(tfPath); os.IsNotExist(err) {
return nil, fmt.Errorf("failed to lookup terraform configuration dir %s: %v", tfPath, err)
}
// set the path to state file
tfState := path.Join(tfStatePath, fmt.Sprintf("e2e.%s.%s.tfstate", provider, name))
env := &environment{
provider: provider,
name: name,
tf: tf,
tfPath: tfPath,
tfState: tfState,
logger: logger,
}
return env, nil
}
// envsFromGlob allows for the discovery of multiple environments using globs (*).
// ex. aws/* for all environments in aws.
func envsFromGlob(envPath, glob, tfStatePath string, logger hclog.Logger) ([]*environment, error) {
results, err := filepath.Glob(filepath.Join(envPath, glob))
if err != nil {
return nil, err
}
envs := []*environment{}
for _, p := range results {
elems := strings.Split(p, "/")
name := elems[len(elems)-1]
provider := elems[len(elems)-2]
env, err := newEnv(envPath, provider, name, tfStatePath, logger)
if err != nil {
return nil, err
}
envs = append(envs, env)
}
return envs, nil
}
// provision calls terraform to setup the environment with the given nomad binary
func (env *environment) provision(nomadPath string) (*envResults, error) {
tfArgs := []string{"apply", "-auto-approve", "-input=false", "-no-color",
"-state", env.tfState,
"-var", fmt.Sprintf("nomad_binary=%s", path.Join(nomadPath, "nomad")),
env.tfPath,
}
// Setup the 'terraform apply' command
ctx := context.Background()
cmd := exec.CommandContext(ctx, env.tf, tfArgs...)
// Funnel the stdout/stderr to logging
stderr, err := cmd.StderrPipe()
if err != nil {
return nil, fmt.Errorf("failed to get stderr pipe: %v", err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("failed to get stdout pipe: %v", err)
}
// Run 'terraform apply'
cmd.Start()
go tfLog(env.logger.Named("tf.stderr"), stderr)
go tfLog(env.logger.Named("tf.stdout"), stdout)
sigChan := make(chan os.Signal)
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
cmdChan := make(chan error)
go func() {
cmdChan <- cmd.Wait()
}()
// if an interrupt is received before terraform finished, forward signal to
// child pid
select {
case sig := <-sigChan:
env.logger.Error("interrupt received, forwarding signal to child process",
"pid", cmd.Process.Pid)
cmd.Process.Signal(sig)
if err := procWaitTimeout(cmd.Process, 5*time.Second); err != nil {
env.logger.Error("child process did not exit in time, killing forcefully",
"pid", cmd.Process.Pid)
cmd.Process.Kill()
}
return nil, fmt.Errorf("interrupt received")
case err := <-cmdChan:
if err != nil {
return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err)
}
}
// Setup and run 'terraform output' to get the module output
cmd = exec.CommandContext(ctx, env.tf, "output", "-json", "-state", env.tfState)
out, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err)
}
// Parse the json and pull out results
tfOutput := make(map[string]map[string]interface{})
err = json.Unmarshal(out, &tfOutput)
if err != nil {
return nil, fmt.Errorf("failed to parse terraform output: %v", err)
}
results := &envResults{}
if nomadAddr, ok := tfOutput["nomad_addr"]; ok {
results.nomadAddr = nomadAddr["value"].(string)
} else {
return nil, fmt.Errorf("'nomad_addr' field expected in terraform output, but was missing")
}
return results, nil
}
// destroy calls terraform to destroy the environment
func (env *environment) destroy() error {
tfArgs := []string{"destroy", "-auto-approve", "-no-color",
"-state", env.tfState,
"-var", "nomad_binary=",
env.tfPath,
}
cmd := exec.Command(env.tf, tfArgs...)
// Funnel the stdout/stderr to logging
stderr, err := cmd.StderrPipe()
if err != nil {
return fmt.Errorf("failed to get stderr pipe: %v", err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return fmt.Errorf("failed to get stdout pipe: %v", err)
}
// Run 'terraform destroy'
cmd.Start()
go tfLog(env.logger.Named("tf.stderr"), stderr)
go tfLog(env.logger.Named("tf.stdout"), stdout)
err = cmd.Wait()
if err != nil {
return fmt.Errorf("terraform exited with a non-zero status: %v", err)
}
return nil
}
func tfLog(logger hclog.Logger, r io.ReadCloser) {
defer r.Close()
scanner := bufio.NewScanner(r)
for scanner.Scan() {
logger.Debug(scanner.Text())
}
if err := scanner.Err(); err != nil {
logger.Error("scan error", "error", err)
}
}

7
e2e/cli/command/meta.go
View File

@@ -14,6 +14,13 @@ type Meta struct {
verbose bool
}
func NewMeta(ui cli.Ui, logger hclog.Logger) Meta {
return Meta{
Ui: ui,
logger: logger,
}
}
func (m *Meta) FlagSet(n string) *flag.FlagSet {
f := flag.NewFlagSet(n, flag.ContinueOnError)

218
e2e/cli/command/provision.go
View File

@@ -1,34 +1,25 @@
package command
import (
"bufio"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
getter "github.com/hashicorp/go-getter"
hclog "github.com/hashicorp/go-hclog"
"github.com/hashicorp/nomad/helper/discover"
"github.com/mitchellh/cli"
)
const (
DefaultEnvironmentsPath = "./environments/"
)
func init() {
getter.Getters["file"].(*getter.FileGetter).Copy = true
}
func ProvisionCommandFactory(ui cli.Ui, logger hclog.Logger) cli.CommandFactory {
func ProvisionCommandFactory(meta Meta) cli.CommandFactory {
return func() (cli.Command, error) {
meta := Meta{
Ui: ui,
logger: logger,
}
return &Provision{Meta: meta}, nil
}
}
@@ -51,7 +42,7 @@ Provision Options:
-env-path
Sets the path for where to search for test environment configuration.
This defaults to './environments/'.
This defaults to './environments/'.
-nomad-binary
Sets the target nomad-binary to use when provisioning a nomad cluster.
@@ -83,7 +74,7 @@ func (c *Provision) Run(args []string) int {
var tfPath string
cmdFlags := c.FlagSet("provision")
cmdFlags.Usage = func() { c.Ui.Output(c.Help()) }
cmdFlags.StringVar(&envPath, "env-path", "./environments/", "Path to e2e environment terraform configs")
cmdFlags.StringVar(&envPath, "env-path", DefaultEnvironmentsPath, "Path to e2e environment terraform configs")
cmdFlags.StringVar(&nomadBinary, "nomad-binary", "", "")
cmdFlags.BoolVar(&destroy, "destroy", false, "")
cmdFlags.StringVar(&tfPath, "tf-path", "", "")
@@ -119,6 +110,10 @@ func (c *Provision) Run(args []string) int {
// Use go-getter to fetch the nomad binary
nomadPath, err := fetchBinary(nomadBinary)
defer os.RemoveAll(nomadPath)
if err != nil {
c.logger.Error("failed to fetch nomad binary", "error", err)
return 1
}
results, err := env.provision(nomadPath)
if err != nil {
@@ -132,194 +127,3 @@ NOMAD_ADDR=%s
return 0
}
// Fetches the nomad binary and returns the temporary directory where it exists
func fetchBinary(bin string) (string, error) {
nomadBinaryDir, err := ioutil.TempDir("", "")
if err != nil {
return "", fmt.Errorf("failed to create temp dir: %v", err)
}
if bin == "" {
bin, err = discover.NomadExecutable()
if err != nil {
return "", fmt.Errorf("failed to discover nomad binary: %v", err)
}
}
if err = getter.GetFile(path.Join(nomadBinaryDir, "nomad"), bin); err != nil {
return "", fmt.Errorf("failed to get nomad binary: %v", err)
}
return nomadBinaryDir, nil
}
type environment struct {
path string
provider string
name string
tf string
tfPath string
tfState string
logger hclog.Logger
}
type envResults struct {
nomadAddr string
consulAddr string
vaultAddr string
}
func newEnv(envPath, provider, name, tfStatePath string, logger hclog.Logger) (*environment, error) {
// Make sure terraform is on the PATH
tf, err := exec.LookPath("terraform")
if err != nil {
return nil, fmt.Errorf("failed to lookup terraform binary: %v", err)
}
logger = logger.Named("provision").With("provider", provider, "name", name)
// set the path to the terraform module
tfPath := path.Join(envPath, provider, name)
logger.Debug("using tf path", "path", tfPath)
if _, err := os.Stat(tfPath); os.IsNotExist(err) {
return nil, fmt.Errorf("failed to lookup terraform configuration dir %s: %v", tfPath, err)
}
// set the path to state file
tfState := path.Join(tfStatePath, fmt.Sprintf("e2e.%s.%s.tfstate", provider, name))
env := &environment{
path: envPath,
provider: provider,
name: name,
tf: tf,
tfPath: tfPath,
tfState: tfState,
logger: logger,
}
return env, nil
}
// envsFromGlob allows for the discovery of multiple environments using globs (*).
// ex. aws/* for all environments in aws.
func envsFromGlob(envPath, glob, tfStatePath string, logger hclog.Logger) ([]*environment, error) {
results, err := filepath.Glob(filepath.Join(envPath, glob))
if err != nil {
return nil, err
}
envs := []*environment{}
for _, p := range results {
elems := strings.Split(p, "/")
name := elems[len(elems)-1]
provider := elems[len(elems)-2]
env, err := newEnv(envPath, provider, name, tfStatePath, logger)
if err != nil {
return nil, err
}
envs = append(envs, env)
}
return envs, nil
}
// provision calls terraform to setup the environment with the given nomad binary
func (env *environment) provision(nomadPath string) (*envResults, error) {
tfArgs := []string{"apply", "-auto-approve", "-input=false", "-no-color",
"-state", env.tfState,
"-var", fmt.Sprintf("nomad_binary=%s", path.Join(nomadPath, "nomad")),
env.tfPath,
}
// Setup the 'terraform apply' command
ctx := context.Background()
cmd := exec.CommandContext(ctx, env.tf, tfArgs...)
// Funnel the stdout/stderr to logging
stderr, err := cmd.StderrPipe()
if err != nil {
return nil, fmt.Errorf("failed to get stderr pipe: %v", err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, fmt.Errorf("failed to get stdout pipe: %v", err)
}
// Run 'terraform apply'
cmd.Start()
go tfLog(env.logger.Named("tf.stderr"), stderr)
go tfLog(env.logger.Named("tf.stdout"), stdout)
err = cmd.Wait()
if err != nil {
return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err)
}
// Setup and run 'terraform output' to get the module output
cmd = exec.CommandContext(ctx, env.tf, "output", "-json", "-state", env.tfState)
out, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err)
}
// Parse the json and pull out results
tfOutput := make(map[string]map[string]interface{})
err = json.Unmarshal(out, &tfOutput)
if err != nil {
return nil, fmt.Errorf("failed to parse terraform output: %v", err)
}
results := &envResults{}
if nomadAddr, ok := tfOutput["nomad_addr"]; ok {
results.nomadAddr = nomadAddr["value"].(string)
}
return results, nil
}
//destroy calls terraform to destroy the environment
func (env *environment) destroy() error {
tfArgs := []string{"destroy", "-auto-approve", "-no-color",
"-state", env.tfState,
"-var", "nomad_binary=",
env.tfPath,
}
cmd := exec.Command(env.tf, tfArgs...)
// Funnel the stdout/stderr to logging
stderr, err := cmd.StderrPipe()
if err != nil {
return fmt.Errorf("failed to get stderr pipe: %v", err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return fmt.Errorf("failed to get stdout pipe: %v", err)
}
// Run 'terraform destroy'
cmd.Start()
go tfLog(env.logger.Named("tf.stderr"), stderr)
go tfLog(env.logger.Named("tf.stdout"), stdout)
err = cmd.Wait()
if err != nil {
return fmt.Errorf("terraform exited with a non-zero status: %v", err)
}
return nil
}
func tfLog(logger hclog.Logger, r io.ReadCloser) {
defer r.Close()
scanner := bufio.NewScanner(r)
for scanner.Scan() {
logger.Debug(scanner.Text())
}
if err := scanner.Err(); err != nil {
logger.Error("scan error", "error", err)
}
}

69
e2e/cli/command/run.go
View File

@@ -12,12 +12,8 @@ import (
"github.com/mitchellh/cli"
)
func RunCommandFactory(ui cli.Ui, logger hclog.Logger) cli.CommandFactory {
func RunCommandFactory(meta Meta) cli.CommandFactory {
return func() (cli.Command, error) {
meta := Meta{
Ui: ui,
logger: logger,
}
return &Run{Meta: meta}, nil
}
}
@@ -45,7 +41,7 @@ func (c *Run) Run(args []string) int {
var run string
cmdFlags := c.FlagSet("run")
cmdFlags.Usage = func() { c.Ui.Output(c.Help()) }
cmdFlags.StringVar(&envPath, "env-path", "./environments/", "Path to e2e environment terraform configs")
cmdFlags.StringVar(&envPath, "env-path", DefaultEnvironmentsPath, "Path to e2e environment terraform configs")
cmdFlags.StringVar(&nomadBinary, "nomad-binary", "", "")
cmdFlags.StringVar(&tfPath, "tf-path", "", "")
cmdFlags.StringVar(&run, "run", "", "Regex to target specific test suites/cases")
@@ -63,22 +59,21 @@ func (c *Run) Run(args []string) int {
if len(args) == 0 {
c.logger.Info("no environments specified, running test suite locally")
var report *TestReport
var err error
if report, err = c.run(&runOpts{
report, err := c.runTest(&runOpts{
slow: slow,
verbose: c.verbose,
}); err != nil {
})
if err != nil {
c.logger.Error("failed to run test suite", "error", err)
return 1
}
if report.TotalFailedTests == 0 {
c.Ui.Output("PASSED!")
if c.verbose {
c.Ui.Output(report.Summary())
}
} else {
c.Ui.Output("***FAILED***")
if report.TotalFailedTests > 0 {
c.Ui.Error("***FAILED***")
c.Ui.Error(report.Summary())
return 1
}
c.Ui.Output("PASSED!")
if c.verbose {
c.Ui.Output(report.Summary())
}
return 0
@@ -98,7 +93,7 @@ func (c *Run) Run(args []string) int {
environments = append(environments, envs...)
}
envCount := len(environments)
// Use go-getter to fetch the nomad binary
nomadPath, err := fetchBinary(nomadBinary)
defer os.RemoveAll(nomadPath)
@@ -107,7 +102,9 @@ func (c *Run) Run(args []string) int {
return 1
}
envCount := len(environments)
c.logger.Debug("starting tests", "totalEnvironments", envCount)
failedEnvs := map[string]*TestReport{}
for i, env := range environments {
logger := c.logger.With("name", env.name, "provider", env.provider)
logger.Debug("provisioning environment")
@@ -128,25 +125,35 @@ func (c *Run) Run(args []string) int {
}
var report *TestReport
if report, err = c.run(opts); err != nil {
if report, err = c.runTest(opts); err != nil {
logger.Error("failed to run tests against environment", "error", err)
return 1
}
if report.TotalFailedTests == 0 {
if report.TotalFailedTests > 0 {
c.Ui.Error(fmt.Sprintf("[%d/%d] %s: ***FAILED***", i+1, envCount, env.canonicalName()))
c.Ui.Error(fmt.Sprintf("[%d/%d] %s: %s", i+1, envCount, env.canonicalName(), report.Summary()))
failedEnvs[env.canonicalName()] = report
}
c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: PASSED!", i+1, envCount, env.provider, env.name))
if c.verbose {
c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: %s", i+1, envCount, env.provider, env.name, report.Summary()))
}
} else {
c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: ***FAILED***", i+1, envCount, env.provider, env.name))
c.Ui.Output(fmt.Sprintf("[%d/%d] %s/%s: %s", i+1, envCount, env.provider, env.name, report.Summary()))
c.Ui.Output(fmt.Sprintf("[%d/%d] %s: PASSED!", i+1, envCount, env.canonicalName()))
if c.verbose {
c.Ui.Output(fmt.Sprintf("[%d/%d] %s: %s", i+1, envCount, env.canonicalName(), report.Summary()))
}
}
if len(failedEnvs) > 0 {
c.Ui.Error(fmt.Sprintf("The following environments ***FAILED***"))
for name, report := range failedEnvs {
c.Ui.Error(fmt.Sprintf(" [%s]: %d out of %d suite failures",
name, report.TotalFailedSuites, report.TotalSuites))
}
return 1
}
c.Ui.Output("All Environments PASSED!")
return 0
}
func (c *Run) run(opts *runOpts) (*TestReport, error) {
func (c *Run) runTest(opts *runOpts) (*TestReport, error) {
goBin, err := exec.LookPath("go")
if err != nil {
return nil, err
@@ -175,6 +182,8 @@ func (c *Run) run(opts *runOpts) (*TestReport, error) {
}
// runOpts contains fields used to build the arguments and environment variabled
// nessicary to run go test and initialize the e2e framework
type runOpts struct {
nomadAddr string
consulAddr string
@@ -186,6 +195,8 @@ type runOpts struct {
verbose bool
}
// goArgs returns the list of arguments passed to the go command to start the
// e2e test framework
func (opts *runOpts) goArgs() []string {
a := []string{
"test",
@@ -205,6 +216,8 @@ func (opts *runOpts) goArgs() []string {
return a
}
// goEnv returns the list of environment variabled passed to the go command to start
// the e2e test framework
func (opts *runOpts) goEnv() []string {
env := append(os.Environ(), "NOMAD_E2E=1")
if opts.nomadAddr != "" {

55
e2e/cli/command/util.go Normal file
View File

@@ -0,0 +1,55 @@
package command
import (
"fmt"
"io/ioutil"
"os"
"path"
"runtime"
"time"
getter "github.com/hashicorp/go-getter"
"github.com/hashicorp/nomad/helper/discover"
)
// Fetches the nomad binary and returns the temporary directory where it exists
func fetchBinary(bin string) (string, error) {
nomadBinaryDir, err := ioutil.TempDir("", "")
if err != nil {
return "", fmt.Errorf("failed to create temp dir: %v", err)
}
if bin == "" {
bin, err = discover.NomadExecutable()
if err != nil {
return "", fmt.Errorf("failed to discover nomad binary: %v", err)
}
}
dest := path.Join(nomadBinaryDir, "nomad")
if runtime.GOOS == "windows" {
dest = dest + ".exe"
}
if err = getter.GetFile(dest, bin); err != nil {
return "", fmt.Errorf("failed to get nomad binary: %v", err)
}
return nomadBinaryDir, nil
}
func procWaitTimeout(p *os.Process, d time.Duration) error {
stop := make(chan struct{})
go func() {
p.Wait()
stop <- struct{}{}
}()
select {
case <-stop:
return nil
case <-time.NewTimer(d).C:
return fmt.Errorf("timeout waiting for process %d to exit", p.Pid)
}
}