diff --git a/e2e/README.md b/e2e/README.md index 59c1a0dcb..76fcc0f5e 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -7,7 +7,8 @@ The `terraform` folder has provisioning code to spin up a Nomad cluster on AWS. Local Development ================= -The workflow when developing end to end tests locally is to run the provisioning step described below once, and then run the tests as described below. +The workflow when developing end to end tests locally is to run the provisioning step described below once, and then run the tests as described below. +When making local changes, use `./bin/update $(which nomad) /usr/local/bin/nomad` and `./bin/run sudo systemctl restart nomad` to destructively modify the provisioned cluster. Provisioning ============ diff --git a/e2e/bin/run b/e2e/bin/run new file mode 100755 index 000000000..e4400c7b4 --- /dev/null +++ b/e2e/bin/run @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +if [ "$1" == "" ]; then + echo "./run.sh commands..." + exit 1 +fi + +nodes=$(terraform output -json -state=terraform/terraform.tfstate | jq -r '(.clients,.servers).value[]') +for node in $nodes +do + echo Executing: ssh -i terraform/keys/*.pem ubuntu@$node "$@" + ssh -o StrictHostKeyChecking=accept-new -i terraform/keys/*.pem ubuntu@$node "$@" +done diff --git a/e2e/bin/update b/e2e/bin/update new file mode 100755 index 000000000..34c032cce --- /dev/null +++ b/e2e/bin/update @@ -0,0 +1,17 @@ +#!/bin/bash + +if [ $# -ne 2 ]; then + echo "./upload.sh " + exit 1 +fi + +set -e + +nodes=$(terraform output -json -state=terraform/terraform.tfstate | jq -r '(.clients,.servers).value[]') +for node in $nodes +do + echo Executing: scp -C -i terraform/keys/*.pem "$1" ubuntu@$node:"$2" + # scp -o StrictHostKeyChecking=accept-new -C -i terraform/keys/*.pem "$1" ubuntu@$node:"$2" + scp -o StrictHostKeyChecking=accept-new -C -i terraform/keys/*.pem "$1" ubuntu@$node:/tmp/uploaded + ssh -i terraform/keys/*.pem ubuntu@$node sudo mv /tmp/uploaded "$2" +done diff --git a/e2e/deployment/deployment.go b/e2e/deployment/deployment.go index 459bb5e09..04f818f54 100644 --- a/e2e/deployment/deployment.go +++ b/e2e/deployment/deployment.go @@ -1,9 +1,11 @@ package deployment import ( - "github.com/hashicorp/nomad/api" + "fmt" + "github.com/hashicorp/nomad/e2e/framework" "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/testutil" "github.com/stretchr/testify/require" "github.com/hashicorp/nomad/e2e/e2eutil" @@ -34,29 +36,35 @@ func (tc *DeploymentTest) BeforeAll(f *framework.F) { func (tc *DeploymentTest) TestDeploymentAutoPromote(f *framework.F) { t := f.T() nomadClient := tc.Nomad() + run := structs.DeploymentStatusRunning uuid := uuid.Generate() + // unique each run, cluster could have previous jobs jobId := "deployment" + uuid[0:8] tc.jobIds = append(tc.jobIds, jobId) e2eutil.RegisterAndWaitForAllocs(t, nomadClient, "deployment/input/deployment_auto0.nomad", jobId) + ds := e2eutil.DeploymentsForJob(t, nomadClient, jobId) + require.Equal(t, 1, len(ds)) + deploy := ds[0] // Upgrade e2eutil.RegisterAllocs(t, nomadClient, "deployment/input/deployment_auto1.nomad", jobId) - var deploy *api.Deployment - ds, _, err := nomadClient.Deployments().List(nil) - require.NoError(t, err) - // Find the deployment - for _, d := range ds { - if d.JobID == jobId { - deploy = d - break + // Find the deployment we don't already have + testutil.WaitForResult(func() (bool, error) { + ds = e2eutil.DeploymentsForJob(t, nomadClient, jobId) + for _, d := range ds { + if d.ID != deploy.ID { + deploy = d + return true, nil + } } - } + return false, fmt.Errorf("missing update deployment for job %s", jobId) + }, func(e error) { + require.NoError(t, e) + }) // Deployment is auto pending the upgrade of "two" which has a longer time to health - run := structs.DeploymentStatusRunning - require.Equal(t, run, deploy.Status) - require.Equal(t, structs.DeploymentStatusDescriptionRunningAutoPromotion, deploy.StatusDescription) + e2eutil.WaitForDeployment(t, nomadClient, deploy.ID, run, structs.DeploymentStatusDescriptionRunningAutoPromotion) // Deployment is eventually running e2eutil.WaitForDeployment(t, nomadClient, deploy.ID, run, structs.DeploymentStatusDescriptionRunning) diff --git a/e2e/e2eutil/utils.go b/e2e/e2eutil/utils.go index 212282cda..5bd41875c 100644 --- a/e2e/e2eutil/utils.go +++ b/e2e/e2eutil/utils.go @@ -111,6 +111,20 @@ func WaitForAllocRunning(t *testing.T, nomadClient *api.Client, allocID string) }) } +func DeploymentsForJob(t *testing.T, nomadClient *api.Client, jobID string) []*api.Deployment { + ds, _, err := nomadClient.Deployments().List(nil) + require.NoError(t, err) + + out := []*api.Deployment{} + for _, d := range ds { + if d.JobID == jobID { + out = append(out, d) + } + } + + return out +} + func WaitForDeployment(t *testing.T, nomadClient *api.Client, deployID string, status string, statusDesc string) { testutil.WaitForResultRetries(retries, func() (bool, error) { time.Sleep(time.Millisecond * 100) @@ -123,10 +137,10 @@ func WaitForDeployment(t *testing.T, nomadClient *api.Client, deployID string, s return true, nil } return false, fmt.Errorf("expected status %s \"%s\", but got: %s \"%s\"", - deploy.Status, - deploy.StatusDescription, status, statusDesc, + deploy.Status, + deploy.StatusDescription, ) }, func(err error) { diff --git a/website/source/docs/commands/deployment/promote.html.md.erb b/website/source/docs/commands/deployment/promote.html.md.erb index aafd1ea5d..d3d42b316 100644 --- a/website/source/docs/commands/deployment/promote.html.md.erb +++ b/website/source/docs/commands/deployment/promote.html.md.erb @@ -69,7 +69,7 @@ web 0 0 3 0 0 0 Latest Deployment ID = 9fa81f27 Status = running -Description = Deployment is running but requires promotion +Description = Deployment is running but requires manual promotion Deployed Task Group Promoted Desired Canaries Placed Healthy Unhealthy @@ -157,7 +157,7 @@ web 0 0 3 0 0 0 Latest Deployment ID = a6b87a6c Status = running -Description = Deployment is running but requires promotion +Description = Deployment is running but requires manual promotion Deployed Task Group Promoted Desired Canaries Placed Healthy Unhealthy @@ -202,7 +202,7 @@ web 0 0 3 0 0 0 Latest Deployment ID = a6b87a6c Status = running -Description = Deployment is running but requires promotion +Description = Deployment is running but requires manual promotion Deployed Task Group Promoted Desired Canaries Placed Healthy Unhealthy diff --git a/website/source/docs/commands/deployment/status.html.md.erb b/website/source/docs/commands/deployment/status.html.md.erb index b68345669..5990d5dc6 100644 --- a/website/source/docs/commands/deployment/status.html.md.erb +++ b/website/source/docs/commands/deployment/status.html.md.erb @@ -59,7 +59,7 @@ ID = 0b23b149 Job ID = example Job Version = 1 Status = running -Description = Deployment is running but requires promotion +Description = Deployment is running but requires manual promotion Deployed Task Group Promoted Desired Canaries Placed Healthy Unhealthy diff --git a/website/source/docs/commands/job/deployments.html.md.erb b/website/source/docs/commands/job/deployments.html.md.erb index 56f6ecae5..96e3264a3 100644 --- a/website/source/docs/commands/job/deployments.html.md.erb +++ b/website/source/docs/commands/job/deployments.html.md.erb @@ -44,7 +44,7 @@ List the deployment for a particular job: ``` $ nomad job deployments example ID Job ID Job Version Status Description -0b23b149 example 1 running Deployment is running but requires promotion +0b23b149 example 1 running Deployment is running but requires manual promotion 06ca68a2 example 0 successful Deployment completed successfully ``` diff --git a/website/source/docs/commands/job/promote.html.md.erb b/website/source/docs/commands/job/promote.html.md.erb index 860d223c1..df19080ee 100644 --- a/website/source/docs/commands/job/promote.html.md.erb +++ b/website/source/docs/commands/job/promote.html.md.erb @@ -69,7 +69,7 @@ web 0 0 3 0 0 0 Latest Deployment ID = 9fa81f27 Status = running -Description = Deployment is running but requires promotion +Description = Deployment is running but requires manual promotion Deployed Task Group Promoted Desired Canaries Placed Healthy Unhealthy @@ -157,7 +157,7 @@ web 0 0 3 0 0 0 Latest Deployment ID = a6b87a6c Status = running -Description = Deployment is running but requires promotion +Description = Deployment is running but requires manual promotion Deployed Task Group Promoted Desired Canaries Placed Healthy Unhealthy @@ -202,7 +202,7 @@ web 0 0 3 0 0 0 Latest Deployment ID = a6b87a6c Status = running -Description = Deployment is running but requires promotion +Description = Deployment is running but requires manual promotion Deployed Task Group Promoted Desired Canaries Placed Healthy Unhealthy diff --git a/website/source/guides/operating-a-job/update-strategies/blue-green-and-canary-deployments.html.md b/website/source/guides/operating-a-job/update-strategies/blue-green-and-canary-deployments.html.md index 02e424151..f75cdf9f0 100644 --- a/website/source/guides/operating-a-job/update-strategies/blue-green-and-canary-deployments.html.md +++ b/website/source/guides/operating-a-job/update-strategies/blue-green-and-canary-deployments.html.md @@ -126,7 +126,7 @@ api 0 0 10 0 0 0 Latest Deployment ID = 32a080c1 Status = running -Description = Deployment is running but requires promotion +Description = Deployment is running but requires manual promotion Deployed Task Group Auto Revert Promoted Desired Canaries Placed Healthy Unhealthy @@ -385,7 +385,7 @@ api 0 0 6 0 0 0 Latest Deployment ID = 32a080c1 Status = running -Description = Deployment is running but requires promotion +Description = Deployment is running but requires manual promotion Deployed Task Group Auto Revert Promoted Desired Canaries Placed Healthy Unhealthy