From a20612c2361f929be896173db0aa91a1204f8422 Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Fri, 20 Oct 2017 14:10:30 -0400 Subject: [PATCH 01/10] add e2e test for testing migrations --- e2e/migrations/Dockerfile | 28 ++++++ e2e/migrations/client1.hcl | 18 ++++ e2e/migrations/client2.hcl | 18 ++++ e2e/migrations/docker-init.sh | 2 + e2e/migrations/docker-run.sh | 3 + e2e/migrations/migrations_test.go | 147 ++++++++++++++++++++++++++++++ e2e/migrations/server.hcl | 10 ++ 7 files changed, 226 insertions(+) create mode 100644 e2e/migrations/Dockerfile create mode 100644 e2e/migrations/client1.hcl create mode 100644 e2e/migrations/client2.hcl create mode 100755 e2e/migrations/docker-init.sh create mode 100755 e2e/migrations/docker-run.sh create mode 100644 e2e/migrations/migrations_test.go create mode 100644 e2e/migrations/server.hcl diff --git a/e2e/migrations/Dockerfile b/e2e/migrations/Dockerfile new file mode 100644 index 000000000..7dfe05c1a --- /dev/null +++ b/e2e/migrations/Dockerfile @@ -0,0 +1,28 @@ +FROM ubuntu:17.10 + +RUN apt-get update -y + +RUN apt-get install -y \ + build-essential \ + git \ + golang \ + liblxc1 \ + libpcre3-dev \ + lxc-dev \ + lxc-templates \ + pkg-config + +ENV NOMAD_VERSION 0.7.0-beta1 +ENV NOMAD_SHA256 174794d96d2617252875e2e2ff9e496120acc4a97be54965c324b9a5d11b37ab + +ENV GOPATH=$HOME/gopkg +ENV PATH=$PATH:$GOPATH/bin:/usr/local/lib +ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib + +COPY nomad /bin/nomad + +RUN mkdir -p /nomad/data && \ + mkdir -p /etc/nomad && \ + mkdir -p gopkg/src/migrations + +RUN go get github.com/stretchr/testify/assert diff --git a/e2e/migrations/client1.hcl b/e2e/migrations/client1.hcl new file mode 100644 index 000000000..aa13d1d14 --- /dev/null +++ b/e2e/migrations/client1.hcl @@ -0,0 +1,18 @@ +log_level = "DEBUG" + +data_dir = "/tmp/client1" + +datacenter = "dc1" + +client { + enabled = true + servers = ["127.0.0.1:4647"] + meta { + secondary = 1 + } +} + +ports { + http = 5656 +} + diff --git a/e2e/migrations/client2.hcl b/e2e/migrations/client2.hcl new file mode 100644 index 000000000..393d3127d --- /dev/null +++ b/e2e/migrations/client2.hcl @@ -0,0 +1,18 @@ +log_level = "DEBUG" + +data_dir = "/tmp/client2" + +datacenter = "dc1" + +client { + enabled = true + servers = ["127.0.0.1:4647"] + meta { + secondary = 0 + } +} + +ports { + http = 5657 +} + diff --git a/e2e/migrations/docker-init.sh b/e2e/migrations/docker-init.sh new file mode 100755 index 000000000..cc736fffe --- /dev/null +++ b/e2e/migrations/docker-init.sh @@ -0,0 +1,2 @@ + +docker build -t nomad-e2e . diff --git a/e2e/migrations/docker-run.sh b/e2e/migrations/docker-run.sh new file mode 100755 index 000000000..ccd13ce5d --- /dev/null +++ b/e2e/migrations/docker-run.sh @@ -0,0 +1,3 @@ +CURRENT_DIRECTORY=` pwd` + +docker run --privileged -v $CURRENT_DIRECTORY:/gopkg/src/migrations -it nomad-e2e /bin/bash -c "cd gopkg/src/migrations && go test" diff --git a/e2e/migrations/migrations_test.go b/e2e/migrations/migrations_test.go new file mode 100644 index 000000000..0cb6d3495 --- /dev/null +++ b/e2e/migrations/migrations_test.go @@ -0,0 +1,147 @@ +package e2e + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "os/exec" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// requires nomad executable on the path +func startCluster() (func(), error) { + serverCmd := exec.Command("nomad", "agent", "-config", "server.hcl") + err := serverCmd.Start() + if err != nil { + return func() {}, err + } + + time.Sleep(10 * time.Second) + + clientCmd := exec.Command("nomad", "agent", "-config", "client1.hcl") + err = clientCmd.Start() + if err != nil { + return func() {}, err + } + + time.Sleep(10 * time.Second) + + secondClientCmd := exec.Command("nomad", "agent", "-config", "client2.hcl") + err = secondClientCmd.Start() + if err != nil { + return func() {}, err + } + + time.Sleep(10 * time.Second) + + f := func() { + clientCmd.Process.Kill() + secondClientCmd.Process.Kill() + serverCmd.Process.Kill() + } + + return f, nil +} + +func TestJobMigrations(t *testing.T) { + t.Parallel() + assert := assert.New(t) + + stopCluster, err := startCluster() + assert.Nil(err) + defer stopCluster() + + fh, err := ioutil.TempFile("", "nomad-sleep-1") + assert.Nil(err) + + defer os.Remove(fh.Name()) + _, err = fh.WriteString(` + job "sleep" { + type = "batch" + datacenters = ["dc1"] + constraint { + attribute = "${meta.secondary}" + value = 1 + } + group "group1" { + restart { + mode = "fail" + } + count = 1 + ephemeral_disk { + migrate = true + sticky = true + } + task "sleep" { + template { + data = "hello world" + destination = "/local/hello-world" + } + driver = "exec" + config { + command = "/bin/sleep" + args = [ "infinity" ] + } + } + } + }`) + + jobCmd := exec.Command("nomad", "run", fh.Name()) + err = jobCmd.Run() + assert.Nil(err) + + time.Sleep(20 * time.Second) + + fh2, err := ioutil.TempFile("", "nomad-sleep-2") + assert.Nil(err) + + defer os.Remove(fh2.Name()) + _, err = fh2.WriteString(` + job "sleep" { + type = "batch" + datacenters = ["dc1"] + constraint { + attribute = "${meta.secondary}" + value = 1 + } + group "group1" { + restart { + mode = "fail" + } + count = 1 + ephemeral_disk { + migrate = true + sticky = true + } + task "sleep" { + driver = "exec" + + config { + command = "test" + args = [ "-f", "/local/hello-world" ] + } + } + } + }`) + + secondJobCmd := exec.Command("nomad", "run", fh2.Name()) + err = secondJobCmd.Run() + assert.Nil(err) + + time.Sleep(20 * time.Second) + + jobStatusCmd := exec.Command("nomad", "job", "status", "sleep") + var jobStatusOut bytes.Buffer + jobStatusCmd.Stdout = &jobStatusOut + + err = jobStatusCmd.Run() + assert.Nil(err) + + jobOutput := jobStatusOut.String() + assert.NotContains(jobOutput, "failed") + assert.Contains(jobOutput, "complete") +} diff --git a/e2e/migrations/server.hcl b/e2e/migrations/server.hcl new file mode 100644 index 000000000..4b51f93e6 --- /dev/null +++ b/e2e/migrations/server.hcl @@ -0,0 +1,10 @@ +log_level = "DEBUG" + +data_dir = "/tmp/server1" + +server { + enabled = true + + bootstrap_expect = 1 +} + From ae7fd585852027ba122daa2aa3cd1b6d45dcf976 Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Fri, 20 Oct 2017 15:09:58 -0400 Subject: [PATCH 02/10] refactoring cluster setup --- e2e/migrations/migrations_test.go | 46 ++++++++++++------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/e2e/migrations/migrations_test.go b/e2e/migrations/migrations_test.go index 0cb6d3495..96fe562d6 100644 --- a/e2e/migrations/migrations_test.go +++ b/e2e/migrations/migrations_test.go @@ -2,7 +2,6 @@ package e2e import ( "bytes" - "fmt" "io/ioutil" "os" "os/exec" @@ -13,35 +12,25 @@ import ( ) // requires nomad executable on the path -func startCluster() (func(), error) { - serverCmd := exec.Command("nomad", "agent", "-config", "server.hcl") - err := serverCmd.Start() - if err != nil { - return func() {}, err +func startCluster(clusterConfig []string) (func(), error) { + cmds := make([]*exec.Cmd, 0) + + for _, agentConfig := range clusterConfig { + cmd := exec.Command("nomad", "agent", "-config", agentConfig) + err := cmd.Start() + + if err != nil { + return func() {}, err + } + + time.Sleep(10 * time.Second) + cmds = append(cmds, cmd) } - time.Sleep(10 * time.Second) - - clientCmd := exec.Command("nomad", "agent", "-config", "client1.hcl") - err = clientCmd.Start() - if err != nil { - return func() {}, err - } - - time.Sleep(10 * time.Second) - - secondClientCmd := exec.Command("nomad", "agent", "-config", "client2.hcl") - err = secondClientCmd.Start() - if err != nil { - return func() {}, err - } - - time.Sleep(10 * time.Second) - f := func() { - clientCmd.Process.Kill() - secondClientCmd.Process.Kill() - serverCmd.Process.Kill() + for _, cmd := range cmds { + cmd.Process.Kill() + } } return f, nil @@ -51,7 +40,8 @@ func TestJobMigrations(t *testing.T) { t.Parallel() assert := assert.New(t) - stopCluster, err := startCluster() + clusterConfig := []string{"server.hcl", "client1.hcl", "client2.hcl"} + stopCluster, err := startCluster(clusterConfig) assert.Nil(err) defer stopCluster() From de7e9a89e1627cd8440db293b4e0e9332aec411d Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Fri, 20 Oct 2017 15:28:49 -0400 Subject: [PATCH 03/10] clean up unused dependencies --- e2e/migrations/Dockerfile | 10 +--------- e2e/migrations/docker-run.sh | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/e2e/migrations/Dockerfile b/e2e/migrations/Dockerfile index 7dfe05c1a..ed9a4469b 100644 --- a/e2e/migrations/Dockerfile +++ b/e2e/migrations/Dockerfile @@ -6,18 +6,10 @@ RUN apt-get install -y \ build-essential \ git \ golang \ - liblxc1 \ - libpcre3-dev \ - lxc-dev \ - lxc-templates \ - pkg-config - -ENV NOMAD_VERSION 0.7.0-beta1 -ENV NOMAD_SHA256 174794d96d2617252875e2e2ff9e496120acc4a97be54965c324b9a5d11b37ab + liblxc1 ENV GOPATH=$HOME/gopkg ENV PATH=$PATH:$GOPATH/bin:/usr/local/lib -ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib COPY nomad /bin/nomad diff --git a/e2e/migrations/docker-run.sh b/e2e/migrations/docker-run.sh index ccd13ce5d..94ad6a47f 100755 --- a/e2e/migrations/docker-run.sh +++ b/e2e/migrations/docker-run.sh @@ -1,3 +1,3 @@ -CURRENT_DIRECTORY=` pwd` +CURRENT_DIRECTORY=`pwd` docker run --privileged -v $CURRENT_DIRECTORY:/gopkg/src/migrations -it nomad-e2e /bin/bash -c "cd gopkg/src/migrations && go test" From 638053717baadea7e795e4d192289c868ee16821 Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Fri, 20 Oct 2017 18:15:24 -0400 Subject: [PATCH 04/10] add polling instead of waits add instructions for running migrations tests --- e2e/migrations/README.md | 16 ++++++++ e2e/migrations/migrations_test.go | 63 +++++++++++++++++++++++++------ 2 files changed, 67 insertions(+), 12 deletions(-) create mode 100644 e2e/migrations/README.md diff --git a/e2e/migrations/README.md b/e2e/migrations/README.md new file mode 100644 index 000000000..afb9f6cc8 --- /dev/null +++ b/e2e/migrations/README.md @@ -0,0 +1,16 @@ +## End to end tests for migrating data in sticky volumes + +These tests run in a docker container to ensure proper setup/teardown. + +To create the testing image: +`./docker-init.sh` + +To run tests: +`./docker-run.sh` + +TODO: + 1. Specify how many servers/clients in the test + 2. Have a callback to specify the client options + 3. Run servers/clients in the docker container, return IP addresses for each + instance, but have the test run on the host. + diff --git a/e2e/migrations/migrations_test.go b/e2e/migrations/migrations_test.go index 96fe562d6..c7a1cee2f 100644 --- a/e2e/migrations/migrations_test.go +++ b/e2e/migrations/migrations_test.go @@ -7,10 +7,49 @@ import ( "os/exec" "testing" "time" + "strings" "github.com/stretchr/testify/assert" ) +func isSuccess(execCmd *exec.Cmd, retries int, keyword string) (bool, string) { + numAttempts := retries + success := false + var out bytes.Buffer + + for numAttempts > 0 { + time.Sleep(2 * time.Second) + + cmd := *execCmd + cmd.Stdout = &out + + err := cmd.Run() + if err != nil { + return false, "" + } + + success = (out.String() != "" && !strings.Contains(out.String(), keyword)) + if success { + return success, out.String() + } + + out.Reset() + numAttempts -= 1 + } + + return success, out.String() +} + +func allNomadNodesAreReady(retries int) (bool, string) { + cmd := exec.Command("nomad", "node-status") + return isSuccess(cmd, retries, "initializing") +} + +func jobIsReady(retries int, jobName string) (bool, string) { + cmd := exec.Command("nomad", "job", "status", jobName) + return isSuccess(cmd, retries, "pending") +} + // requires nomad executable on the path func startCluster(clusterConfig []string) (func(), error) { cmds := make([]*exec.Cmd, 0) @@ -23,7 +62,6 @@ func startCluster(clusterConfig []string) (func(), error) { return func() {}, err } - time.Sleep(10 * time.Second) cmds = append(cmds, cmd) } @@ -45,6 +83,9 @@ func TestJobMigrations(t *testing.T) { assert.Nil(err) defer stopCluster() + isReady, _ := allNomadNodesAreReady(10) + assert.True(isReady) + fh, err := ioutil.TempFile("", "nomad-sleep-1") assert.Nil(err) @@ -80,11 +121,15 @@ func TestJobMigrations(t *testing.T) { } }`) + assert.Nil(err) + jobCmd := exec.Command("nomad", "run", fh.Name()) err = jobCmd.Run() assert.Nil(err) - time.Sleep(20 * time.Second) + isFirstJobReady, firstJoboutput := jobIsReady(20, "sleep") + assert.True(isFirstJobReady) + assert.NotContains(firstJoboutput, "failed") fh2, err := ioutil.TempFile("", "nomad-sleep-2") assert.Nil(err) @@ -118,20 +163,14 @@ func TestJobMigrations(t *testing.T) { } }`) + assert.Nil(err) + secondJobCmd := exec.Command("nomad", "run", fh2.Name()) err = secondJobCmd.Run() assert.Nil(err) - time.Sleep(20 * time.Second) - - jobStatusCmd := exec.Command("nomad", "job", "status", "sleep") - var jobStatusOut bytes.Buffer - jobStatusCmd.Stdout = &jobStatusOut - - err = jobStatusCmd.Run() - assert.Nil(err) - - jobOutput := jobStatusOut.String() + isReady, jobOutput := jobIsReady(20, "sleep") + assert.True(isReady) assert.NotContains(jobOutput, "failed") assert.Contains(jobOutput, "complete") } From fd2bc415b5844f11ee651785c8e05bc9281f5e72 Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Tue, 24 Oct 2017 09:56:46 -0400 Subject: [PATCH 05/10] fix up goimports --- e2e/migrations/migrations_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/migrations/migrations_test.go b/e2e/migrations/migrations_test.go index c7a1cee2f..b698b414a 100644 --- a/e2e/migrations/migrations_test.go +++ b/e2e/migrations/migrations_test.go @@ -5,9 +5,9 @@ import ( "io/ioutil" "os" "os/exec" + "strings" "testing" "time" - "strings" "github.com/stretchr/testify/assert" ) From d61ab4a573a49e0b4d674f6018616c29bd568cce Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Tue, 24 Oct 2017 20:56:19 -0400 Subject: [PATCH 06/10] use testutil helper function --- e2e/migrations/Dockerfile | 2 +- e2e/migrations/docker-run.sh | 2 +- e2e/migrations/migrations_test.go | 51 ++++++++++++++++--------------- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/e2e/migrations/Dockerfile b/e2e/migrations/Dockerfile index ed9a4469b..c3b8dc87a 100644 --- a/e2e/migrations/Dockerfile +++ b/e2e/migrations/Dockerfile @@ -15,6 +15,6 @@ COPY nomad /bin/nomad RUN mkdir -p /nomad/data && \ mkdir -p /etc/nomad && \ - mkdir -p gopkg/src/migrations + mkdir -p gopkg/src/github.com/nomad RUN go get github.com/stretchr/testify/assert diff --git a/e2e/migrations/docker-run.sh b/e2e/migrations/docker-run.sh index 94ad6a47f..babe03747 100755 --- a/e2e/migrations/docker-run.sh +++ b/e2e/migrations/docker-run.sh @@ -1,3 +1,3 @@ CURRENT_DIRECTORY=`pwd` -docker run --privileged -v $CURRENT_DIRECTORY:/gopkg/src/migrations -it nomad-e2e /bin/bash -c "cd gopkg/src/migrations && go test" +docker run --privileged -v $CURRENT_DIRECTORY:/gopkg/src/github.com/hashicorp/nomad -it nomad-e2e /bin/bash -c "cd gopkg/src/github.com/hashicorp/nomad/e2e/migrations && go test" diff --git a/e2e/migrations/migrations_test.go b/e2e/migrations/migrations_test.go index b698b414a..e172138b0 100644 --- a/e2e/migrations/migrations_test.go +++ b/e2e/migrations/migrations_test.go @@ -7,45 +7,46 @@ import ( "os/exec" "strings" "testing" - "time" "github.com/stretchr/testify/assert" + "github.com/hashicorp/nomad/testutil" ) -func isSuccess(execCmd *exec.Cmd, retries int, keyword string) (bool, string) { - numAttempts := retries - success := false - var out bytes.Buffer - - for numAttempts > 0 { - time.Sleep(2 * time.Second) +func isSuccess(execCmd *exec.Cmd, retries int, keyword string) (string, error) { + var successOut string + var err error + testutil.WaitForResultRetries(2000, func() (bool, error) { + var out bytes.Buffer cmd := *execCmd cmd.Stdout = &out - err := cmd.Run() + if err != nil { - return false, "" + return false, err } - success = (out.String() != "" && !strings.Contains(out.String(), keyword)) - if success { - return success, out.String() + success := (out.String() != "" && !strings.Contains(out.String(), keyword)) + if !success { + out.Reset() + return false, err } - out.Reset() - numAttempts -= 1 - } + successOut = out.String() + return true, nil + }, func(cmd_err error) { + err = cmd_err + }) - return success, out.String() + return successOut, err } -func allNomadNodesAreReady(retries int) (bool, string) { +func allNomadNodesAreReady(retries int) (string, error) { cmd := exec.Command("nomad", "node-status") return isSuccess(cmd, retries, "initializing") } -func jobIsReady(retries int, jobName string) (bool, string) { +func jobIsReady(retries int, jobName string) (string, error) { cmd := exec.Command("nomad", "job", "status", jobName) return isSuccess(cmd, retries, "pending") } @@ -83,8 +84,8 @@ func TestJobMigrations(t *testing.T) { assert.Nil(err) defer stopCluster() - isReady, _ := allNomadNodesAreReady(10) - assert.True(isReady) + _, err = allNomadNodesAreReady(10) + assert.Nil(err) fh, err := ioutil.TempFile("", "nomad-sleep-1") assert.Nil(err) @@ -127,8 +128,8 @@ func TestJobMigrations(t *testing.T) { err = jobCmd.Run() assert.Nil(err) - isFirstJobReady, firstJoboutput := jobIsReady(20, "sleep") - assert.True(isFirstJobReady) + firstJoboutput, err := jobIsReady(20, "sleep") + assert.Nil(err) assert.NotContains(firstJoboutput, "failed") fh2, err := ioutil.TempFile("", "nomad-sleep-2") @@ -169,8 +170,8 @@ func TestJobMigrations(t *testing.T) { err = secondJobCmd.Run() assert.Nil(err) - isReady, jobOutput := jobIsReady(20, "sleep") - assert.True(isReady) + jobOutput, err := jobIsReady(20, "sleep") + assert.Nil(err) assert.NotContains(jobOutput, "failed") assert.Contains(jobOutput, "complete") } From aa410d0530e4e16de84f6fef08676ec2ad3631a8 Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Tue, 24 Oct 2017 21:38:15 -0400 Subject: [PATCH 07/10] fixing up code review feedback --- e2e/migrations/migrations_test.go | 130 ++++++++++++++++-------------- 1 file changed, 69 insertions(+), 61 deletions(-) diff --git a/e2e/migrations/migrations_test.go b/e2e/migrations/migrations_test.go index e172138b0..986f113cd 100644 --- a/e2e/migrations/migrations_test.go +++ b/e2e/migrations/migrations_test.go @@ -8,10 +8,67 @@ import ( "strings" "testing" - "github.com/stretchr/testify/assert" "github.com/hashicorp/nomad/testutil" + "github.com/stretchr/testify/assert" ) +const sleepJobOne = `job "sleep" { + type = "batch" + datacenters = ["dc1"] + constraint { + attribute = "${meta.secondary}" + value = 1 + } + group "group1" { + restart { + mode = "fail" + } + count = 1 + ephemeral_disk { + migrate = true + sticky = true + } + task "sleep" { + template { + data = "hello world" + destination = "/local/hello-world" + } + driver = "exec" + config { + command = "/bin/sleep" + args = [ "infinity" ] + } + } + } +}` + +const sleepJobTwo = `job "sleep" { + type = "batch" + datacenters = ["dc1"] + constraint { + attribute = "${meta.secondary}" + value = 0 + } + group "group1" { + restart { + mode = "fail" + } + count = 1 + ephemeral_disk { + migrate = true + sticky = true + } + task "sleep" { + driver = "exec" + + config { + command = "test" + args = [ "-f", "/local/hello-world" ] + } + } + } +}` + func isSuccess(execCmd *exec.Cmd, retries int, keyword string) (string, error) { var successOut string var err error @@ -41,17 +98,23 @@ func isSuccess(execCmd *exec.Cmd, retries int, keyword string) (string, error) { return successOut, err } -func allNomadNodesAreReady(retries int) (string, error) { +// allNodesAreReady attempts to query the status of a cluster a specific number +// of times +func allNodesAreReady(retries int) (string, error) { cmd := exec.Command("nomad", "node-status") return isSuccess(cmd, retries, "initializing") } +// jobIsReady attempts sto query the status of a specific job a fixed number of +// times func jobIsReady(retries int, jobName string) (string, error) { cmd := exec.Command("nomad", "job", "status", jobName) return isSuccess(cmd, retries, "pending") } -// requires nomad executable on the path +// startCluster will create a running cluster, given a list of agent config +// files. In order to have a complete cluster, at least one server and one +// client config file should be included. func startCluster(clusterConfig []string) (func(), error) { cmds := make([]*exec.Cmd, 0) @@ -84,43 +147,14 @@ func TestJobMigrations(t *testing.T) { assert.Nil(err) defer stopCluster() - _, err = allNomadNodesAreReady(10) + _, err = allNodesAreReady(10) assert.Nil(err) fh, err := ioutil.TempFile("", "nomad-sleep-1") assert.Nil(err) defer os.Remove(fh.Name()) - _, err = fh.WriteString(` - job "sleep" { - type = "batch" - datacenters = ["dc1"] - constraint { - attribute = "${meta.secondary}" - value = 1 - } - group "group1" { - restart { - mode = "fail" - } - count = 1 - ephemeral_disk { - migrate = true - sticky = true - } - task "sleep" { - template { - data = "hello world" - destination = "/local/hello-world" - } - driver = "exec" - config { - command = "/bin/sleep" - args = [ "infinity" ] - } - } - } - }`) + _, err = fh.WriteString(sleepJobOne) assert.Nil(err) @@ -136,33 +170,7 @@ func TestJobMigrations(t *testing.T) { assert.Nil(err) defer os.Remove(fh2.Name()) - _, err = fh2.WriteString(` - job "sleep" { - type = "batch" - datacenters = ["dc1"] - constraint { - attribute = "${meta.secondary}" - value = 1 - } - group "group1" { - restart { - mode = "fail" - } - count = 1 - ephemeral_disk { - migrate = true - sticky = true - } - task "sleep" { - driver = "exec" - - config { - command = "test" - args = [ "-f", "/local/hello-world" ] - } - } - } - }`) + _, err = fh2.WriteString(sleepJobTwo) assert.Nil(err) From f4e466f54be7dc93cf07dd248c1e80b27f23d9f4 Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Wed, 25 Oct 2017 15:56:42 -0400 Subject: [PATCH 08/10] exclude integration tests from main build --- e2e/migrations/docker-run.sh | 5 ++++- e2e/migrations/migrations_test.go | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/e2e/migrations/docker-run.sh b/e2e/migrations/docker-run.sh index babe03747..a4a8b28ec 100755 --- a/e2e/migrations/docker-run.sh +++ b/e2e/migrations/docker-run.sh @@ -1,3 +1,6 @@ CURRENT_DIRECTORY=`pwd` -docker run --privileged -v $CURRENT_DIRECTORY:/gopkg/src/github.com/hashicorp/nomad -it nomad-e2e /bin/bash -c "cd gopkg/src/github.com/hashicorp/nomad/e2e/migrations && go test" +docker run --privileged -v \ +$CURRENT_DIRECTORY:/gopkg/src/github.com/hashicorp/nomad \ +-it nomad-e2e /bin/bash \ +-c "cd gopkg/src/github.com/hashicorp/nomad/e2e/migrations && go test -integration" diff --git a/e2e/migrations/migrations_test.go b/e2e/migrations/migrations_test.go index 986f113cd..b1e49378d 100644 --- a/e2e/migrations/migrations_test.go +++ b/e2e/migrations/migrations_test.go @@ -2,6 +2,7 @@ package e2e import ( "bytes" + "flag" "io/ioutil" "os" "os/exec" @@ -12,6 +13,8 @@ import ( "github.com/stretchr/testify/assert" ) +var integration = flag.Bool("integration", false, "run integration tests") + const sleepJobOne = `job "sleep" { type = "batch" datacenters = ["dc1"] @@ -139,6 +142,11 @@ func startCluster(clusterConfig []string) (func(), error) { } func TestJobMigrations(t *testing.T) { + flag.Parse() + if !*integration { + t.Skip("skipping test in non-integration mode.") + } + t.Parallel() assert := assert.New(t) From 3d85928a0438019b35ee29bd091b4984a1a3321b Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Wed, 25 Oct 2017 17:50:47 -0400 Subject: [PATCH 09/10] feedback from code review, improve run script --- e2e/migrations/README.md | 2 ++ e2e/migrations/docker-run.sh | 3 ++- e2e/migrations/migrations_test.go | 6 ++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/e2e/migrations/README.md b/e2e/migrations/README.md index afb9f6cc8..213376ce7 100644 --- a/e2e/migrations/README.md +++ b/e2e/migrations/README.md @@ -13,4 +13,6 @@ TODO: 2. Have a callback to specify the client options 3. Run servers/clients in the docker container, return IP addresses for each instance, but have the test run on the host. + 4. There should be a 1:1 mapping from container to agent, rather than running + the entire cluster in a container. diff --git a/e2e/migrations/docker-run.sh b/e2e/migrations/docker-run.sh index a4a8b28ec..4b3e71f92 100755 --- a/e2e/migrations/docker-run.sh +++ b/e2e/migrations/docker-run.sh @@ -1,6 +1,7 @@ CURRENT_DIRECTORY=`pwd` +ROOT_DIRECTORY="$( dirname "$(dirname "$CURRENT_DIRECTORY")")" docker run --privileged -v \ -$CURRENT_DIRECTORY:/gopkg/src/github.com/hashicorp/nomad \ +$ROOT_DIRECTORY:/gopkg/src/github.com/hashicorp/nomad \ -it nomad-e2e /bin/bash \ -c "cd gopkg/src/github.com/hashicorp/nomad/e2e/migrations && go test -integration" diff --git a/e2e/migrations/migrations_test.go b/e2e/migrations/migrations_test.go index b1e49378d..7cb5cc492 100644 --- a/e2e/migrations/migrations_test.go +++ b/e2e/migrations/migrations_test.go @@ -72,6 +72,11 @@ const sleepJobTwo = `job "sleep" { } }` +// isSuccess waits until a given keyword is not present in the output of a +// command. For example, isSuccess will poll for a given timeperiod as long as +// the output of the command of "nomad node-status" includes the keyword +// "initializing." The absence of this keyword means this command has returned +// successfully. func isSuccess(execCmd *exec.Cmd, retries int, keyword string) (string, error) { var successOut string var err error @@ -189,5 +194,6 @@ func TestJobMigrations(t *testing.T) { jobOutput, err := jobIsReady(20, "sleep") assert.Nil(err) assert.NotContains(jobOutput, "failed") + assert.NotContains(jobOutput, "pending") assert.Contains(jobOutput, "complete") } From c6ef3d3d93a5b4a0bf791ba476b63aed45fa6eff Mon Sep 17 00:00:00 2001 From: Chelsea Holland Komlo Date: Thu, 26 Oct 2017 13:14:58 -0400 Subject: [PATCH 10/10] add assertion for pending --- e2e/migrations/migrations_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/e2e/migrations/migrations_test.go b/e2e/migrations/migrations_test.go index 7cb5cc492..6752800ac 100644 --- a/e2e/migrations/migrations_test.go +++ b/e2e/migrations/migrations_test.go @@ -175,9 +175,10 @@ func TestJobMigrations(t *testing.T) { err = jobCmd.Run() assert.Nil(err) - firstJoboutput, err := jobIsReady(20, "sleep") + firstJobOutput, err := jobIsReady(20, "sleep") assert.Nil(err) - assert.NotContains(firstJoboutput, "failed") + assert.NotContains(firstJobOutput, "failed") + assert.NotContains(firstJobOutput, "pending") fh2, err := ioutil.TempFile("", "nomad-sleep-2") assert.Nil(err)