From 22b6c2e6ade62f71c578d08e2ded310630173d5c Mon Sep 17 00:00:00 2001
From: Lang Martin <lang@hashicorp.com>
Date: Tue, 4 Jun 2019 11:25:18 -0400
Subject: [PATCH 1/6] e2e/deployment find the second deployment, use its status

---
 e2e/deployment/deployment.go | 28 +++++++++++++++-------------
 e2e/e2eutil/utils.go         | 20 ++++++++++++++++++--
 2 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/e2e/deployment/deployment.go b/e2e/deployment/deployment.go
index 459bb5e09..238130c40 100644
--- a/e2e/deployment/deployment.go
+++ b/e2e/deployment/deployment.go
@@ -1,9 +1,9 @@
 package deployment
 
 import (
-	"github.com/hashicorp/nomad/api"
 	"github.com/hashicorp/nomad/e2e/framework"
 	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/nomad/testutil"
 	"github.com/stretchr/testify/require"
 
 	"github.com/hashicorp/nomad/e2e/e2eutil"
@@ -34,29 +34,31 @@ func (tc *DeploymentTest) BeforeAll(f *framework.F) {
 func (tc *DeploymentTest) TestDeploymentAutoPromote(f *framework.F) {
 	t := f.T()
 	nomadClient := tc.Nomad()
+	run := structs.DeploymentStatusRunning
 	uuid := uuid.Generate()
+	// unique each run, cluster could have previous jobs
 	jobId := "deployment" + uuid[0:8]
 	tc.jobIds = append(tc.jobIds, jobId)
 	e2eutil.RegisterAndWaitForAllocs(t, nomadClient, "deployment/input/deployment_auto0.nomad", jobId)
+	deploy := e2eutil.DeploymentsForJob(nomadClient, jobId)[0]
 
 	// Upgrade
 	e2eutil.RegisterAllocs(t, nomadClient, "deployment/input/deployment_auto1.nomad", jobId)
-	var deploy *api.Deployment
-	ds, _, err := nomadClient.Deployments().List(nil)
-	require.NoError(t, err)
 
-	// Find the deployment
-	for _, d := range ds {
-		if d.JobID == jobId {
-			deploy = d
-			break
+	// Find the deployment we don't already have
+	testutil.WaitForResult(func() (bool, error) {
+		ds := e2eutil.DeploymentsForJob(nomadClient, jobId)
+		for _, d := range ds {
+			if d.ID != deploy.ID {
+				deploy = d
+				return true, nil
+			}
 		}
-	}
+		return false, nil
+	}, func(e error) {})
 
 	// Deployment is auto pending the upgrade of "two" which has a longer time to health
-	run := structs.DeploymentStatusRunning
-	require.Equal(t, run, deploy.Status)
-	require.Equal(t, structs.DeploymentStatusDescriptionRunningAutoPromotion, deploy.StatusDescription)
+	e2eutil.WaitForDeployment(t, nomadClient, deploy.ID, run, structs.DeploymentStatusDescriptionRunningAutoPromotion)
 
 	// Deployment is eventually running
 	e2eutil.WaitForDeployment(t, nomadClient, deploy.ID, run, structs.DeploymentStatusDescriptionRunning)
diff --git a/e2e/e2eutil/utils.go b/e2e/e2eutil/utils.go
index 212282cda..e98e95aeb 100644
--- a/e2e/e2eutil/utils.go
+++ b/e2e/e2eutil/utils.go
@@ -111,6 +111,22 @@ func WaitForAllocRunning(t *testing.T, nomadClient *api.Client, allocID string)
 	})
 }
 
+func DeploymentsForJob(nomadClient *api.Client, jobID string) []*api.Deployment {
+	ds, _, err := nomadClient.Deployments().List(nil)
+	if err != nil {
+		return nil
+	}
+
+	out := []*api.Deployment{}
+	for _, d := range ds {
+		if d.JobID == jobID {
+			out = append(out, d)
+		}
+	}
+
+	return out
+}
+
 func WaitForDeployment(t *testing.T, nomadClient *api.Client, deployID string, status string, statusDesc string) {
 	testutil.WaitForResultRetries(retries, func() (bool, error) {
 		time.Sleep(time.Millisecond * 100)
@@ -123,10 +139,10 @@ func WaitForDeployment(t *testing.T, nomadClient *api.Client, deployID string, s
 			return true, nil
 		}
 		return false, fmt.Errorf("expected status %s \"%s\", but got: %s \"%s\"",
-			deploy.Status,
-			deploy.StatusDescription,
 			status,
 			statusDesc,
+			deploy.Status,
+			deploy.StatusDescription,
 		)
 
 	}, func(err error) {

From 293b3d4b9fac1f83566f53114e8ba7cb83e9eec8 Mon Sep 17 00:00:00 2001
From: Lang Martin <lang@hashicorp.com>
Date: Tue, 4 Jun 2019 13:40:46 -0400
Subject: [PATCH 2/6] e2e bin/update and bin/run, README

---
 e2e/README.md  |  3 ++-
 e2e/bin/run    | 15 +++++++++++++++
 e2e/bin/update | 22 ++++++++++++++++++++++
 3 files changed, 39 insertions(+), 1 deletion(-)
 create mode 100755 e2e/bin/run
 create mode 100755 e2e/bin/update

diff --git a/e2e/README.md b/e2e/README.md
index 59c1a0dcb..76fcc0f5e 100644
--- a/e2e/README.md
+++ b/e2e/README.md
@@ -7,7 +7,8 @@ The `terraform` folder has provisioning code to spin up a Nomad cluster on AWS.
 
 Local Development
 =================
-The workflow when developing end to end tests locally is to run the provisioning step described below once, and then run the tests as described below. 
+The workflow when developing end to end tests locally is to run the provisioning step described below once, and then run the tests as described below.
+When making local changes, use `./bin/update $(which nomad) /usr/local/bin/nomad` and `./bin/run sudo systemctl restart nomad` to destructively modify the provisioned cluster.
 
 Provisioning
 ============
diff --git a/e2e/bin/run b/e2e/bin/run
new file mode 100755
index 000000000..6edec7afe
--- /dev/null
+++ b/e2e/bin/run
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+if [ "$1" == "" ]; then
+    echo "./run.sh commands..."
+    exit 1
+fi
+
+nodes=$(terraform output -json -state=terraform/terraform.tfstate | jq -r '(.clients,.servers).value[]')
+for node in $nodes
+do
+    echo Executing: ssh -i terraform/keys/*.pem ubuntu@$node $@
+    ssh -o StrictHostKeyChecking=accept-new -i terraform/keys/*.pem ubuntu@$node $@
+done
\ No newline at end of file
diff --git a/e2e/bin/update b/e2e/bin/update
new file mode 100755
index 000000000..bbae0bfe4
--- /dev/null
+++ b/e2e/bin/update
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+
+if [ "$1" == "" ]; then
+    echo "./upload.sh <source> <destination>"
+    exit 1
+fi
+
+if [ "$2" == "" ]; then
+    echo "./upload.sh <source> <destination>"
+    exit 1
+fi
+
+nodes=$(terraform output -json -state=terraform/terraform.tfstate | jq -r '(.clients,.servers).value[]')
+for node in $nodes
+do
+    echo Executing: scp -C -i terraform/keys/*.pem $1 ubuntu@$node:$2
+    #scp -o StrictHostKeyChecking=accept-new -C -i terraform/keys/*.pem "$1" ubuntu@$node:"$2"
+    scp -o StrictHostKeyChecking=accept-new -C -i terraform/keys/*.pem "$1" ubuntu@$node:/tmp/uploaded
+    ssh -i terraform/keys/*.pem ubuntu@$node sudo mv /tmp/uploaded "$2"
+done
\ No newline at end of file

From 9b0ae123587c52c2c63eb1a4df6830c07aa87f4a Mon Sep 17 00:00:00 2001
From: Lang Martin <lang@hashicorp.com>
Date: Tue, 4 Jun 2019 13:45:44 -0400
Subject: [PATCH 3/6] deployment update website examples to say 'requires
 manual promotion'

---
 website/source/docs/commands/deployment/promote.html.md.erb | 6 +++---
 website/source/docs/commands/deployment/status.html.md.erb  | 2 +-
 website/source/docs/commands/job/deployments.html.md.erb    | 2 +-
 website/source/docs/commands/job/promote.html.md.erb        | 6 +++---
 .../blue-green-and-canary-deployments.html.md               | 4 ++--
 5 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/website/source/docs/commands/deployment/promote.html.md.erb b/website/source/docs/commands/deployment/promote.html.md.erb
index aafd1ea5d..d3d42b316 100644
--- a/website/source/docs/commands/deployment/promote.html.md.erb
+++ b/website/source/docs/commands/deployment/promote.html.md.erb
@@ -69,7 +69,7 @@ web         0       0         3        0       0         0
 Latest Deployment
 ID          = 9fa81f27
 Status      = running
-Description = Deployment is running but requires promotion
+Description = Deployment is running but requires manual promotion
 
 Deployed
 Task Group  Promoted  Desired  Canaries  Placed  Healthy  Unhealthy
@@ -157,7 +157,7 @@ web         0       0         3        0       0         0
 Latest Deployment
 ID          = a6b87a6c
 Status      = running
-Description = Deployment is running but requires promotion
+Description = Deployment is running but requires manual promotion
 
 Deployed
 Task Group  Promoted  Desired  Canaries  Placed  Healthy  Unhealthy
@@ -202,7 +202,7 @@ web         0       0         3        0       0         0
 Latest Deployment
 ID          = a6b87a6c
 Status      = running
-Description = Deployment is running but requires promotion
+Description = Deployment is running but requires manual promotion
 
 Deployed
 Task Group  Promoted  Desired  Canaries  Placed  Healthy  Unhealthy
diff --git a/website/source/docs/commands/deployment/status.html.md.erb b/website/source/docs/commands/deployment/status.html.md.erb
index b68345669..5990d5dc6 100644
--- a/website/source/docs/commands/deployment/status.html.md.erb
+++ b/website/source/docs/commands/deployment/status.html.md.erb
@@ -59,7 +59,7 @@ ID          = 0b23b149
 Job ID      = example
 Job Version = 1
 Status      = running
-Description = Deployment is running but requires promotion
+Description = Deployment is running but requires manual promotion
 
 Deployed
 Task Group  Promoted  Desired  Canaries  Placed  Healthy  Unhealthy
diff --git a/website/source/docs/commands/job/deployments.html.md.erb b/website/source/docs/commands/job/deployments.html.md.erb
index 0f3d3e8ac..cb325ad79 100644
--- a/website/source/docs/commands/job/deployments.html.md.erb
+++ b/website/source/docs/commands/job/deployments.html.md.erb
@@ -44,7 +44,7 @@ List the deployment for a particular job:
 ```
 $ nomad job deployments example
 ID        Job ID   Job Version  Status      Description
-0b23b149  example  1            running     Deployment is running but requires promotion
+0b23b149  example  1            running     Deployment is running but requires manual promotion
 06ca68a2  example  0            successful  Deployment completed successfully
 ```
 
diff --git a/website/source/docs/commands/job/promote.html.md.erb b/website/source/docs/commands/job/promote.html.md.erb
index 860d223c1..df19080ee 100644
--- a/website/source/docs/commands/job/promote.html.md.erb
+++ b/website/source/docs/commands/job/promote.html.md.erb
@@ -69,7 +69,7 @@ web         0       0         3        0       0         0
 Latest Deployment
 ID          = 9fa81f27
 Status      = running
-Description = Deployment is running but requires promotion
+Description = Deployment is running but requires manual promotion
 
 Deployed
 Task Group  Promoted  Desired  Canaries  Placed  Healthy  Unhealthy
@@ -157,7 +157,7 @@ web         0       0         3        0       0         0
 Latest Deployment
 ID          = a6b87a6c
 Status      = running
-Description = Deployment is running but requires promotion
+Description = Deployment is running but requires manual promotion
 
 Deployed
 Task Group  Promoted  Desired  Canaries  Placed  Healthy  Unhealthy
@@ -202,7 +202,7 @@ web         0       0         3        0       0         0
 Latest Deployment
 ID          = a6b87a6c
 Status      = running
-Description = Deployment is running but requires promotion
+Description = Deployment is running but requires manual promotion
 
 Deployed
 Task Group  Promoted  Desired  Canaries  Placed  Healthy  Unhealthy
diff --git a/website/source/guides/operating-a-job/update-strategies/blue-green-and-canary-deployments.html.md b/website/source/guides/operating-a-job/update-strategies/blue-green-and-canary-deployments.html.md
index 02e424151..f75cdf9f0 100644
--- a/website/source/guides/operating-a-job/update-strategies/blue-green-and-canary-deployments.html.md
+++ b/website/source/guides/operating-a-job/update-strategies/blue-green-and-canary-deployments.html.md
@@ -126,7 +126,7 @@ api         0       0         10       0       0         0
 Latest Deployment
 ID          = 32a080c1
 Status      = running
-Description = Deployment is running but requires promotion
+Description = Deployment is running but requires manual promotion
 
 Deployed
 Task Group  Auto Revert  Promoted  Desired  Canaries  Placed  Healthy  Unhealthy
@@ -385,7 +385,7 @@ api         0       0         6        0       0         0
 Latest Deployment
 ID          = 32a080c1
 Status      = running
-Description = Deployment is running but requires promotion
+Description = Deployment is running but requires manual promotion
 
 Deployed
 Task Group  Auto Revert  Promoted  Desired  Canaries  Placed  Healthy  Unhealthy

From 6ca33d0f206ab6eda96f9969a31b1b8ea9a945c7 Mon Sep 17 00:00:00 2001
From: Lang Martin <lang@hashicorp.com>
Date: Tue, 4 Jun 2019 14:08:30 -0400
Subject: [PATCH 4/6] e2e/deployment fail if the second deployment times out

---
 e2e/deployment/deployment.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/e2e/deployment/deployment.go b/e2e/deployment/deployment.go
index 238130c40..b48476083 100644
--- a/e2e/deployment/deployment.go
+++ b/e2e/deployment/deployment.go
@@ -55,7 +55,9 @@ func (tc *DeploymentTest) TestDeploymentAutoPromote(f *framework.F) {
 			}
 		}
 		return false, nil
-	}, func(e error) {})
+	}, func(e error) {
+		t.Error("missing update deployment")
+	})
 
 	// Deployment is auto pending the upgrade of "two" which has a longer time to health
 	e2eutil.WaitForDeployment(t, nomadClient, deploy.ID, run, structs.DeploymentStatusDescriptionRunningAutoPromotion)

From 4b8c9428f47d36ef617b6b9f0d94e00e341fac0a Mon Sep 17 00:00:00 2001
From: Lang Martin <lang@hashicorp.com>
Date: Tue, 4 Jun 2019 14:31:42 -0400
Subject: [PATCH 5/6] e2e/deployment DeploymentsForJob fail instead of nil,
 error passing

---
 e2e/deployment/deployment.go | 12 ++++++++----
 e2e/e2eutil/utils.go         |  6 ++----
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/e2e/deployment/deployment.go b/e2e/deployment/deployment.go
index b48476083..04f818f54 100644
--- a/e2e/deployment/deployment.go
+++ b/e2e/deployment/deployment.go
@@ -1,6 +1,8 @@
 package deployment
 
 import (
+	"fmt"
+
 	"github.com/hashicorp/nomad/e2e/framework"
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/testutil"
@@ -40,23 +42,25 @@ func (tc *DeploymentTest) TestDeploymentAutoPromote(f *framework.F) {
 	jobId := "deployment" + uuid[0:8]
 	tc.jobIds = append(tc.jobIds, jobId)
 	e2eutil.RegisterAndWaitForAllocs(t, nomadClient, "deployment/input/deployment_auto0.nomad", jobId)
-	deploy := e2eutil.DeploymentsForJob(nomadClient, jobId)[0]
+	ds := e2eutil.DeploymentsForJob(t, nomadClient, jobId)
+	require.Equal(t, 1, len(ds))
+	deploy := ds[0]
 
 	// Upgrade
 	e2eutil.RegisterAllocs(t, nomadClient, "deployment/input/deployment_auto1.nomad", jobId)
 
 	// Find the deployment we don't already have
 	testutil.WaitForResult(func() (bool, error) {
-		ds := e2eutil.DeploymentsForJob(nomadClient, jobId)
+		ds = e2eutil.DeploymentsForJob(t, nomadClient, jobId)
 		for _, d := range ds {
 			if d.ID != deploy.ID {
 				deploy = d
 				return true, nil
 			}
 		}
-		return false, nil
+		return false, fmt.Errorf("missing update deployment for job %s", jobId)
 	}, func(e error) {
-		t.Error("missing update deployment")
+		require.NoError(t, e)
 	})
 
 	// Deployment is auto pending the upgrade of "two" which has a longer time to health
diff --git a/e2e/e2eutil/utils.go b/e2e/e2eutil/utils.go
index e98e95aeb..5bd41875c 100644
--- a/e2e/e2eutil/utils.go
+++ b/e2e/e2eutil/utils.go
@@ -111,11 +111,9 @@ func WaitForAllocRunning(t *testing.T, nomadClient *api.Client, allocID string)
 	})
 }
 
-func DeploymentsForJob(nomadClient *api.Client, jobID string) []*api.Deployment {
+func DeploymentsForJob(t *testing.T, nomadClient *api.Client, jobID string) []*api.Deployment {
 	ds, _, err := nomadClient.Deployments().List(nil)
-	if err != nil {
-		return nil
-	}
+	require.NoError(t, err)
 
 	out := []*api.Deployment{}
 	for _, d := range ds {

From f7ab4f0f214b5a62b7f255e8cb997651cb1f0b8f Mon Sep 17 00:00:00 2001
From: Lang Martin <lang@hashicorp.com>
Date: Tue, 4 Jun 2019 15:52:32 -0400
Subject: [PATCH 6/6] e2e update shell scripts argument quoting

---
 e2e/bin/run    |  6 +++---
 e2e/bin/update | 21 ++++++++-------------
 2 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/e2e/bin/run b/e2e/bin/run
index 6edec7afe..e4400c7b4 100755
--- a/e2e/bin/run
+++ b/e2e/bin/run
@@ -10,6 +10,6 @@ fi
 nodes=$(terraform output -json -state=terraform/terraform.tfstate | jq -r '(.clients,.servers).value[]')
 for node in $nodes
 do
-    echo Executing: ssh -i terraform/keys/*.pem ubuntu@$node $@
-    ssh -o StrictHostKeyChecking=accept-new -i terraform/keys/*.pem ubuntu@$node $@
-done
\ No newline at end of file
+    echo Executing: ssh -i terraform/keys/*.pem ubuntu@$node "$@"
+    ssh -o StrictHostKeyChecking=accept-new -i terraform/keys/*.pem ubuntu@$node "$@"
+done
diff --git a/e2e/bin/update b/e2e/bin/update
index bbae0bfe4..34c032cce 100755
--- a/e2e/bin/update
+++ b/e2e/bin/update
@@ -1,22 +1,17 @@
 #!/bin/bash
 
+if [ $# -ne 2 ]; then
+    echo "./upload.sh <source> <destination>"
+    exit 1
+fi
+
 set -e
 
-if [ "$1" == "" ]; then
-    echo "./upload.sh <source> <destination>"
-    exit 1
-fi
-
-if [ "$2" == "" ]; then
-    echo "./upload.sh <source> <destination>"
-    exit 1
-fi
-
 nodes=$(terraform output -json -state=terraform/terraform.tfstate | jq -r '(.clients,.servers).value[]')
 for node in $nodes
 do
-    echo Executing: scp -C -i terraform/keys/*.pem $1 ubuntu@$node:$2
-    #scp -o StrictHostKeyChecking=accept-new -C -i terraform/keys/*.pem "$1" ubuntu@$node:"$2"
+    echo Executing: scp -C -i terraform/keys/*.pem "$1" ubuntu@$node:"$2"
+    # scp -o StrictHostKeyChecking=accept-new -C -i terraform/keys/*.pem "$1" ubuntu@$node:"$2"
     scp -o StrictHostKeyChecking=accept-new -C -i terraform/keys/*.pem "$1" ubuntu@$node:/tmp/uploaded
     ssh -i terraform/keys/*.pem ubuntu@$node sudo mv /tmp/uploaded "$2"
-done
\ No newline at end of file
+done