From cd93c1f4da37582a9ec7b3b5bfaa56771dd904c5 Mon Sep 17 00:00:00 2001
From: Tim Gross <tgross@hashicorp.com>
Date: Thu, 20 Aug 2020 16:10:00 -0400
Subject: [PATCH] E2E: move Nomad installation to script on remote hosts
 (#8706)

This changeset moves the installation of Nomad binaries out of the
provisioning framework and into scripts that are installed on the remote host
during AMI builds.

This provides a few advantages:

* The provisioning framework can be reduced in scope (with the goal of moving
  most of it into the Terraform stack entirely).
* The scripts can be arbitrarily complex if we don't have to stuff them into
  ssh commands, so it's easier to make them idempotent. In this changeset, the
  scripts check the version of the existing binary and don't re-download when
  using the `--nomad_sha` or `--nomad_version` flags.
* The scripts can be OS/distro specific, which helps in building new test
  targets.
---
 e2e/framework/provisioning/deploy.go          |  57 +-----
 e2e/terraform/packer/linux/install-nomad      | 103 ++++++++++
 e2e/terraform/packer/linux/setup.sh           |  12 +-
 e2e/terraform/packer/packer-windows.json      |  14 +-
 e2e/terraform/packer/packer.json              |   5 +
 .../packer/windows/install-nomad.ps1          | 178 +++++++++++++++---
 6 files changed, 289 insertions(+), 80 deletions(-)
 create mode 100755 e2e/terraform/packer/linux/install-nomad

diff --git a/e2e/framework/provisioning/deploy.go b/e2e/framework/provisioning/deploy.go
index 8fae45000..479e4ad7d 100644
--- a/e2e/framework/provisioning/deploy.go
+++ b/e2e/framework/provisioning/deploy.go
@@ -2,7 +2,6 @@ package provisioning
 
 import (
 	"fmt"
-	"path/filepath"
 	"testing"
 )
 
@@ -38,35 +37,15 @@ func deployLinux(t *testing.T, target *ProvisioningTarget) error {
 			return fmt.Errorf("copying Nomad failed: %v", err)
 		}
 	} else if deployment.NomadSha != "" {
-		if deployment.RemoteBinaryPath == "" {
-			return fmt.Errorf("remote binary path not set")
-		}
-		s3_url := fmt.Sprintf("s3://nomad-team-dev-test-binaries/builds-oss/nomad_%s_%s.tar.gz",
-			deployment.Platform, deployment.NomadSha,
-		)
-		remoteDir := filepath.Dir(deployment.RemoteBinaryPath)
-		script := fmt.Sprintf(`aws s3 cp --quiet %s nomad.tar.gz
-			sudo tar -zxvf nomad.tar.gz -C %s
-			sudo chmod 0755 %s
-			sudo chown root:root %s`,
-			s3_url, remoteDir, deployment.RemoteBinaryPath, deployment.RemoteBinaryPath)
+		script := fmt.Sprintf(
+			`/opt/install-nomad --nomad_sha %s --nostart`, deployment.NomadSha)
 		err = runner.Run(script)
 		if err != nil {
 			return err
 		}
 	} else if deployment.NomadVersion != "" {
-		if deployment.RemoteBinaryPath == "" {
-			return fmt.Errorf("remote binary path not set")
-		}
-		url := fmt.Sprintf("https://releases.hashicorp.com/nomad/%s/nomad_%s_%s.zip",
-			deployment.NomadVersion, deployment.NomadVersion, deployment.Platform,
-		)
-		remoteDir := filepath.Dir(deployment.RemoteBinaryPath)
-		script := fmt.Sprintf(`curl -L --fail -o /tmp/nomad.zip %s
-			sudo unzip -o /tmp/nomad.zip -d %s
-			sudo chmod 0755 %s
-			sudo chown root:root %s`,
-			url, remoteDir, deployment.RemoteBinaryPath, deployment.RemoteBinaryPath)
+		script := fmt.Sprintf(
+			`/opt/install-nomad --nomad_version %s --nostart`, deployment.NomadVersion)
 		err = runner.Run(script)
 		if err != nil {
 			return err
@@ -82,6 +61,7 @@ func deployLinux(t *testing.T, target *ProvisioningTarget) error {
 			return fmt.Errorf("copying bundle '%s' failed: %v", bundle.Source, err)
 		}
 	}
+
 	for _, step := range deployment.Steps {
 		err = runner.Run(step)
 		if err != nil {
@@ -115,33 +95,15 @@ func deployWindows(t *testing.T, target *ProvisioningTarget) error {
 			return fmt.Errorf("copying Nomad failed: %v", err)
 		}
 	} else if deployment.NomadSha != "" {
-		if deployment.RemoteBinaryPath == "" {
-			return fmt.Errorf("remote binary path not set")
-		}
-		script := fmt.Sprintf(`
-			Read-S3Object -BucketName nomad-team-dev-test-binaries -Key "builds-oss/nomad_windows_amd64_%s.zip" -File ./nomad.zip
-			Expand-Archive ./nomad.zip ./ -Force
-			Remove-Item %s  -ErrorAction Ignore
-			Move-Item -Path .\pkg\windows_amd64\nomad.exe -Destination %s -Force`,
-			deployment.NomadSha, deployment.RemoteBinaryPath,
-			deployment.RemoteBinaryPath)
+		script := fmt.Sprintf(
+			`C:/opt/install-nomad.ps1 -nomad_sha %s -nostart`, deployment.NomadSha)
 		err = runner.Run(script)
 		if err != nil {
 			return err
 		}
 	} else if deployment.NomadVersion != "" {
-		if deployment.RemoteBinaryPath == "" {
-			return fmt.Errorf("remote binary path not set")
-		}
-		url := fmt.Sprintf("https://releases.hashicorp.com/nomad/%s/nomad_%s_%s.zip",
-			deployment.NomadVersion, deployment.NomadVersion, deployment.Platform,
-		)
-		script := fmt.Sprintf(`
-			Invoke-WebRequest -Uri "%s" -Outfile /.nomad.zip
-			Expand-Archive ./nomad.zip ./ -Force
-			Remove-Item %s  -ErrorAction Ignore
-			Move-Item -Path .\pkg\windows_amd64\nomad.exe -Destination %s -Force`,
-			url, deployment.RemoteBinaryPath, deployment.RemoteBinaryPath)
+		script := fmt.Sprintf(
+			`C:/opt/install-nomad.ps1 -nomad_version %s -nostart`, deployment.NomadVersion)
 		err = runner.Run(script)
 		if err != nil {
 			return err
@@ -157,6 +119,7 @@ func deployWindows(t *testing.T, target *ProvisioningTarget) error {
 			return fmt.Errorf("copying bundle '%s' failed: %v", bundle.Source, err)
 		}
 	}
+
 	for _, step := range deployment.Steps {
 		err = runner.Run(step)
 		if err != nil {
diff --git a/e2e/terraform/packer/linux/install-nomad b/e2e/terraform/packer/linux/install-nomad
new file mode 100755
index 000000000..b9fe8b292
--- /dev/null
+++ b/e2e/terraform/packer/linux/install-nomad
@@ -0,0 +1,103 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set +x
+
+usage() {
+    cat <<EOF
+Usage: install-nomad [options...]
+Options (use one of the following):
+ --nomad_sha SHA          full git sha to install from S3
+ --nomad_version VERSION  release version number (ex. 0.12.3+ent)
+ --nomad_binary FILEPATH  path to file on host
+ --nostart                do not start or restart Nomad
+
+EOF
+
+    exit 2
+}
+
+
+INSTALL_DIR=/usr/local/bin
+INSTALL_PATH="${INSTALL_DIR}/nomad"
+PLATFORM=linux_amd64
+START=1
+install_fn=
+
+install_from_s3() {
+    # check that we don't already have this version
+    if [ "$(command -v nomad)" ]; then
+        nomad -version | grep -q "${NOMAD_SHA}" \
+            && echo "$NOMAD_SHA already installed" && exit 0
+    fi
+
+    S3_URL="s3://nomad-team-dev-test-binaries/builds-oss/nomad_linux_${NOMAD_SHA}.tar.gz"
+    aws s3 cp --quiet "$S3_URL" nomad.tar.gz
+    sudo tar -zxvf nomad.tar.gz -C "$INSTALL_DIR"
+    set_ownership
+    if [ $START == "1" ]; then sudo systemctl restart nomad; fi
+}
+
+install_from_uploaded_binary() {
+    # we don't check for reinstallation here because we do it at the user's
+    # end, rather than on the remote host, so that we don't bother copying
+    # if we don't have to
+    sudo cp "$NOMAD_UPLOADED_BINARY" "$INSTALL_PATH"
+    set_ownership
+    if [ $START == "1" ]; then sudo systemctl restart nomad; fi
+}
+
+install_from_release() {
+    # check that we don't already have this version
+    if [ "$(command -v nomad)" ]; then
+        nomad -version | grep -v 'dev' | grep -q "${NOMAD_VERSION}" \
+            && echo "$NOMAD_VERSION already installed" && exit 0
+    fi
+
+    RELEASE_URL="https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_${PLATFORM}.zip"
+    curl -sL --fail -o /tmp/nomad.zip "$RELEASE_URL"
+    sudo unzip -o /tmp/nomad.zip -d "$INSTALL_DIR"
+    set_ownership
+    if [ $START == "1" ]; then sudo systemctl restart nomad; fi
+}
+
+set_ownership() {
+    sudo chmod 0755 "$INSTALL_PATH"
+    sudo chown root:root "$INSTALL_PATH"
+}
+
+while [[ $# -gt 0 ]]
+do
+opt="$1"
+    case $opt in
+        --nomad_sha)
+            if [ -z "$2" ]; then echo "Missing sha parameter"; usage; fi
+            NOMAD_SHA="$2"
+            install_fn=install_from_s3
+            shift 2
+            ;;
+        --nomad_release | --nomad_version)
+            if [ -z "$2" ]; then echo "Missing version parameter"; usage; fi
+            NOMAD_VERSION="$2"
+            install_fn=install_from_release
+            shift 2
+            ;;
+        --nomad_binary)
+            if [ -z "$2" ]; then echo "Missing file parameter"; usage; fi
+            NOMAD_UPLOADED_BINARY="$2"
+            install_fn=install_from_uploaded_binary
+            shift 2
+            ;;
+        --nostart)
+            # for initial packer builds, we don't want to start Nomad
+            START=0
+            shift
+            ;;
+        *) usage ;;
+    esac
+done
+
+# call the appropriate instalation function
+if [ -z "$install_fn" ]; then echo "Missing install option"; usage; fi
+$install_fn
diff --git a/e2e/terraform/packer/linux/setup.sh b/e2e/terraform/packer/linux/setup.sh
index 9f3fcccb0..2674d827d 100755
--- a/e2e/terraform/packer/linux/setup.sh
+++ b/e2e/terraform/packer/linux/setup.sh
@@ -22,7 +22,6 @@ VAULTDIR=/opt/vault
 
 # Will be overwritten by sha specified
 NOMADVERSION=0.9.1
-NOMADDOWNLOAD=https://releases.hashicorp.com/nomad/${NOMADVERSION}/nomad_${NOMADVERSION}_linux_amd64.zip
 NOMADCONFIGDIR=/etc/nomad.d
 NOMADDIR=/opt/nomad
 NOMADPLUGINDIR=/opt/nomad/plugins
@@ -67,12 +66,6 @@ sudo chmod 755 $VAULTCONFIGDIR
 sudo mkdir -p $VAULTDIR
 sudo chmod 755 $VAULTDIR
 
-echo "Install Nomad"
-curl -fsL -o /tmp/nomad.zip $NOMADDOWNLOAD
-sudo unzip -q /tmp/nomad.zip -d /usr/local/bin
-sudo chmod 0755 /usr/local/bin/nomad
-sudo chown root:root /usr/local/bin/nomad
-
 echo "Configure Nomad"
 sudo mkdir -p $NOMADCONFIGDIR
 sudo chmod 755 $NOMADCONFIGDIR
@@ -81,6 +74,11 @@ sudo chmod 755 $NOMADDIR
 sudo mkdir -p $NOMADPLUGINDIR
 sudo chmod 755 $NOMADPLUGINDIR
 
+echo "Install Nomad"
+sudo mv /tmp/install-nomad /opt/install-nomad
+sudo chmod +x /opt/install-nomad
+/opt/install-nomad --nomad_version $NOMADVERSION --nostart
+
 echo "Installing third-party apt repositories"
 
 # Docker
diff --git a/e2e/terraform/packer/packer-windows.json b/e2e/terraform/packer/packer-windows.json
index 66e058ef2..c8d2c5a78 100644
--- a/e2e/terraform/packer/packer-windows.json
+++ b/e2e/terraform/packer/packer-windows.json
@@ -44,14 +44,24 @@
     {
       "type": "windows-restart"
     },
+    {
+      "type": "file",
+      "source": "./windows/install-nomad.ps1",
+      "destination": "/opt/install-nomad.ps1"
+    },
+    {
+      "type": "powershell",
+      "elevated_user": "Administrator",
+      "elevated_password": "{{.WinRMPassword}}",
+      "inline": ["/opt/install-nomad.ps1 -nomad_version 0.9.6 -nostart"]
+    },
     {
       "type": "powershell",
       "elevated_user": "Administrator",
       "elevated_password": "{{.WinRMPassword}}",
       "scripts": [
         "windows/install-consul.ps1",
-        "windows/install-vault.ps1",
-        "windows/install-nomad.ps1"
+        "windows/install-vault.ps1"
       ]
     },
     {
diff --git a/e2e/terraform/packer/packer.json b/e2e/terraform/packer/packer.json
index cd26a2851..3a575e489 100644
--- a/e2e/terraform/packer/packer.json
+++ b/e2e/terraform/packer/packer.json
@@ -14,6 +14,11 @@
     }
   ],
   "provisioners": [
+    {
+      "type": "file",
+      "source": "./linux/install-nomad",
+      "destination": "/tmp/install-nomad"
+    },
     {
       "type": "shell",
       "script": "./linux/setup.sh"
diff --git a/e2e/terraform/packer/windows/install-nomad.ps1 b/e2e/terraform/packer/windows/install-nomad.ps1
index 300197319..183f63925 100755
--- a/e2e/terraform/packer/windows/install-nomad.ps1
+++ b/e2e/terraform/packer/windows/install-nomad.ps1
@@ -1,34 +1,164 @@
+param(
+    [string]$nomad_sha,
+    [string]$nomad_version,
+    [string]$nomad_binary,
+    [switch]$nostart = $false
+)
+
 Set-StrictMode -Version latest
 $ErrorActionPreference = "Stop"
 
+
+
+$usage = @"
+Usage: install-nomad [options...]
+Options (use one of the following):
+ --nomad_sha SHA          full git sha to install from S3
+ --nomad_version VERSION  release version number (ex. 0.12.3+ent)
+ --nomad_binary FILEPATH  path to file on host
+ --nostart                do not start or restart Nomad
+"@
+
+$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")
+if (!$RunningAsAdmin) {
+  Write-Error "Must be executed in Administrator level shell."
+  exit 1
+}
+
 # Force TLS1.2
 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
 
+
+$install_path = "C:\opt\nomad.exe"
+
 Set-Location C:\opt
 
-Try {
-    # we install the most recent stable/GA release; this will be replaced
-    # with the current master when we run e2e tests
-    $releases = "https://releases.hashicorp.com"
-    $version = "0.9.6"
-    $url = "${releases}/nomad/${version}/nomad_${version}_windows_amd64.zip"
-
-    $configDir = "C:\opt\nomad.d"
-    md $configDir
-    md C:\opt\nomad
-
-    # TODO: check sha!
-    Write-Output "Downloading Nomad from: $url"
-    Invoke-WebRequest -Uri $url -Outfile nomad.zip
-    Expand-Archive .\nomad.zip .\
-    mv nomad.exe C:\opt\nomad.exe
-    C:\opt\nomad.exe version
-    rm nomad.zip
-
-} Catch {
-    Write-Error "Failed to install Nomad."
-    $host.SetShouldExit(-1)
-    throw
+function Usage {
+    Write-Output "${usage}"
 }
 
-Write-Output "Installed Nomad."
+function InstallFromS3 {
+    Try {
+        # check that we don't already have this version
+        if (C:\opt\nomad.exe -version `
+          | Select-String -Pattern $nomad_sha -SimpleMatch -Quiet) {
+              Write-Output "${nomad_sha} already installed"
+              return
+          }
+    } Catch {
+        Write-Output "${nomad_sha} not previously installed"
+    }
+
+    Try {
+        $key = "builds-oss/nomad_windows_amd64_${nomad_sha}.zip"
+		Read-S3Object -BucketName nomad-team-dev-test-binaries -Key $key -File ./nomad.zip
+		Remove-Item -Path $install_path -Force -ErrorAction Ignore
+		Expand-Archive ./nomad.zip ./ -Force
+		Move-Item -Path .\pkg\windows_amd64\nomad.exe -Destination $install_path -Force
+		Remove-Item -Path nomad.zip -Force -ErrorAction Ignore
+
+        New-Item -ItemType Directory -Force -Path C:\opt\nomad.d
+        New-Item -ItemType Directory -Force -Path C:\opt\nomad
+        Write-Output "Installed Nomad."
+
+        if (!($nostart)) {
+            StartNomad
+        }
+    } Catch {
+        Write-Error "Failed to install Nomad."
+        $host.SetShouldExit(-1)
+        throw
+    }
+}
+
+function InstallFromUploadedBinary {
+    Try {
+		Remove-Item -Path $install_path -Force -ErrorAction Ignore
+        Move-Item -Path $nomad_binary -Destination $install_path -Force
+
+        New-Item -ItemType Directory -Force -Path C:\opt\nomad.d
+        New-Item -ItemType Directory -Force -Path C:\opt\nomad
+        Write-Output "Installed Nomad."
+
+        if (!($nostart)) {
+            StartNomad
+        }
+    } Catch {
+        Write-Error "Failed to install Nomad."
+        $host.SetShouldExit(-1)
+        throw
+    }
+}
+
+function InstallFromRelease {
+    Try {
+        # check that we don't already have this version
+        if (C:\opt\nomad.exe -version `
+          | Select-String -Pattern $nomad_version -SimpleMatch -Quiet) {
+              if (C:\opt\nomad.exe -version `
+                | Select-String -Pattern dev -SimpleMatch -Quiet -NotMatch) {
+                    Write-Output "${nomad_version} already installed"
+                    return
+                }
+          }
+    } Catch {
+        Write-Output "${nomad_version} not previously installed"
+    }
+
+    Try {
+        $releases = "https://releases.hashicorp.com"
+        $url = "${releases}/nomad/${nomad_version}/nomad_${nomad_version}_windows_amd64.zip"
+
+        Write-Output "Downloading Nomad from: $url"
+        Invoke-WebRequest -Uri $url -Outfile nomad.zip
+		Remove-Item -Path $install_path -Force -ErrorAction Ignore
+        Expand-Archive .\nomad.zip .\
+		Remove-Item nomad.zip -Force -ErrorAction Ignore
+
+        New-Item -ItemType Directory -Force -Path C:\opt\nomad.d
+        New-Item -ItemType Directory -Force -Path C:\opt\nomad
+        Write-Output "Installed Nomad."
+
+        if (!($nostart)) {
+            StartNomad
+        }
+    } Catch {
+        Write-Error "Failed to install Nomad."
+        $host.SetShouldExit(-1)
+        throw
+    }
+}
+
+function StartNomad {
+    New-NetFirewallRule `
+      -DisplayName 'Nomad HTTP Inbound' `
+      -Profile @('Public', 'Domain', 'Private') `
+      -Direction Inbound `
+      -Action Allow `
+      -Protocol TCP `
+      -LocalPort @('4646')
+
+    # idempotently enable as a service
+    New-Service `
+      -Name "Nomad" `
+      -BinaryPathName "C:\opt\nomad.exe agent -config C:\opt\nomad.d" `
+      -StartupType "Automatic" `
+      -ErrorAction Ignore
+
+    Start-Service "Nomad"
+}
+
+if ( "" -ne $nomad_sha ) {
+    InstallFromS3
+    return
+}
+if ( "" -ne $nomad_version ) {
+    InstallFromRelease
+    return
+}
+if ( "" -ne $nomad_binary ) {
+    InstallFromUploadedBinary
+    return
+}
+
+Usage