E2E: move Nomad installation to script on remote hosts (#8706)

This changeset moves the installation of Nomad binaries out of the provisioning framework and into scripts that are installed on the remote host during AMI builds. This provides a few advantages: * The provisioning framework can be reduced in scope (with the goal of moving most of it into the Terraform stack entirely). * The scripts can be arbitrarily complex if we don't have to stuff them into ssh commands, so it's easier to make them idempotent. In this changeset, the scripts check the version of the existing binary and don't re-download when using the `--nomad_sha` or `--nomad_version` flags. * The scripts can be OS/distro specific, which helps in building new test targets.
2026-01-01 16:05:42 +03:00 · 2020-08-20 16:10:00 -04:00
parent 6f08becdae
commit cd93c1f4da
6 changed files with 289 additions and 80 deletions
--- a/e2e/framework/provisioning/deploy.go
+++ b/e2e/framework/provisioning/deploy.go
@@ -2,7 +2,6 @@ package provisioning

 import (
 	"fmt"
-	"path/filepath"
 	"testing"
 )

@@ -38,35 +37,15 @@ func deployLinux(t *testing.T, target *ProvisioningTarget) error {
 			return fmt.Errorf("copying Nomad failed: %v", err)
 		}
 	} else if deployment.NomadSha != "" {
-		if deployment.RemoteBinaryPath == "" {
-			return fmt.Errorf("remote binary path not set")
-		}
-		s3_url := fmt.Sprintf("s3://nomad-team-dev-test-binaries/builds-oss/nomad_%s_%s.tar.gz",
-			deployment.Platform, deployment.NomadSha,
-		)
-		remoteDir := filepath.Dir(deployment.RemoteBinaryPath)
-		script := fmt.Sprintf(`aws s3 cp --quiet %s nomad.tar.gz
-			sudo tar -zxvf nomad.tar.gz -C %s
-			sudo chmod 0755 %s
-			sudo chown root:root %s`,
-			s3_url, remoteDir, deployment.RemoteBinaryPath, deployment.RemoteBinaryPath)
+		script := fmt.Sprintf(
+			`/opt/install-nomad --nomad_sha %s --nostart`, deployment.NomadSha)
 		err = runner.Run(script)
 		if err != nil {
 			return err
 		}
 	} else if deployment.NomadVersion != "" {
-		if deployment.RemoteBinaryPath == "" {
-			return fmt.Errorf("remote binary path not set")
-		}
-		url := fmt.Sprintf("https://releases.hashicorp.com/nomad/%s/nomad_%s_%s.zip",
-			deployment.NomadVersion, deployment.NomadVersion, deployment.Platform,
-		)
-		remoteDir := filepath.Dir(deployment.RemoteBinaryPath)
-		script := fmt.Sprintf(`curl -L --fail -o /tmp/nomad.zip %s
-			sudo unzip -o /tmp/nomad.zip -d %s
-			sudo chmod 0755 %s
-			sudo chown root:root %s`,
-			url, remoteDir, deployment.RemoteBinaryPath, deployment.RemoteBinaryPath)
+		script := fmt.Sprintf(
+			`/opt/install-nomad --nomad_version %s --nostart`, deployment.NomadVersion)
 		err = runner.Run(script)
 		if err != nil {
 			return err
@@ -82,6 +61,7 @@ func deployLinux(t *testing.T, target *ProvisioningTarget) error {
 			return fmt.Errorf("copying bundle '%s' failed: %v", bundle.Source, err)
 		}
 	}
+
 	for _, step := range deployment.Steps {
 		err = runner.Run(step)
 		if err != nil {
@@ -115,33 +95,15 @@ func deployWindows(t *testing.T, target *ProvisioningTarget) error {
 			return fmt.Errorf("copying Nomad failed: %v", err)
 		}
 	} else if deployment.NomadSha != "" {
-		if deployment.RemoteBinaryPath == "" {
-			return fmt.Errorf("remote binary path not set")
-		}
-		script := fmt.Sprintf(`
-			Read-S3Object -BucketName nomad-team-dev-test-binaries -Key "builds-oss/nomad_windows_amd64_%s.zip" -File ./nomad.zip
-			Expand-Archive ./nomad.zip ./ -Force
-			Remove-Item %s  -ErrorAction Ignore
-			Move-Item -Path .\pkg\windows_amd64\nomad.exe -Destination %s -Force`,
-			deployment.NomadSha, deployment.RemoteBinaryPath,
-			deployment.RemoteBinaryPath)
+		script := fmt.Sprintf(
+			`C:/opt/install-nomad.ps1 -nomad_sha %s -nostart`, deployment.NomadSha)
 		err = runner.Run(script)
 		if err != nil {
 			return err
 		}
 	} else if deployment.NomadVersion != "" {
-		if deployment.RemoteBinaryPath == "" {
-			return fmt.Errorf("remote binary path not set")
-		}
-		url := fmt.Sprintf("https://releases.hashicorp.com/nomad/%s/nomad_%s_%s.zip",
-			deployment.NomadVersion, deployment.NomadVersion, deployment.Platform,
-		)
-		script := fmt.Sprintf(`
-			Invoke-WebRequest -Uri "%s" -Outfile /.nomad.zip
-			Expand-Archive ./nomad.zip ./ -Force
-			Remove-Item %s  -ErrorAction Ignore
-			Move-Item -Path .\pkg\windows_amd64\nomad.exe -Destination %s -Force`,
-			url, deployment.RemoteBinaryPath, deployment.RemoteBinaryPath)
+		script := fmt.Sprintf(
+			`C:/opt/install-nomad.ps1 -nomad_version %s -nostart`, deployment.NomadVersion)
 		err = runner.Run(script)
 		if err != nil {
 			return err
@@ -157,6 +119,7 @@ func deployWindows(t *testing.T, target *ProvisioningTarget) error {
 			return fmt.Errorf("copying bundle '%s' failed: %v", bundle.Source, err)
 		}
 	}
+
 	for _, step := range deployment.Steps {
 		err = runner.Run(step)
 		if err != nil {
--- a/e2e/terraform/packer/linux/install-nomad
+++ b/e2e/terraform/packer/linux/install-nomad
@@ -0,0 +1,103 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set +x
+
+usage() {
+    cat <<EOF
+Usage: install-nomad [options...]
+Options (use one of the following):
+ --nomad_sha SHA          full git sha to install from S3
+ --nomad_version VERSION  release version number (ex. 0.12.3+ent)
+ --nomad_binary FILEPATH  path to file on host
+ --nostart                do not start or restart Nomad
+
+EOF
+
+    exit 2
+}
+
+
+INSTALL_DIR=/usr/local/bin
+INSTALL_PATH="${INSTALL_DIR}/nomad"
+PLATFORM=linux_amd64
+START=1
+install_fn=
+
+install_from_s3() {
+    # check that we don't already have this version
+    if [ "$(command -v nomad)" ]; then
+        nomad -version | grep -q "${NOMAD_SHA}" \
+            && echo "$NOMAD_SHA already installed" && exit 0
+    fi
+
+    S3_URL="s3://nomad-team-dev-test-binaries/builds-oss/nomad_linux_${NOMAD_SHA}.tar.gz"
+    aws s3 cp --quiet "$S3_URL" nomad.tar.gz
+    sudo tar -zxvf nomad.tar.gz -C "$INSTALL_DIR"
+    set_ownership
+    if [ $START == "1" ]; then sudo systemctl restart nomad; fi
+}
+
+install_from_uploaded_binary() {
+    # we don't check for reinstallation here because we do it at the user's
+    # end, rather than on the remote host, so that we don't bother copying
+    # if we don't have to
+    sudo cp "$NOMAD_UPLOADED_BINARY" "$INSTALL_PATH"
+    set_ownership
+    if [ $START == "1" ]; then sudo systemctl restart nomad; fi
+}
+
+install_from_release() {
+    # check that we don't already have this version
+    if [ "$(command -v nomad)" ]; then
+        nomad -version | grep -v 'dev' | grep -q "${NOMAD_VERSION}" \
+            && echo "$NOMAD_VERSION already installed" && exit 0
+    fi
+
+    RELEASE_URL="https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_${PLATFORM}.zip"
+    curl -sL --fail -o /tmp/nomad.zip "$RELEASE_URL"
+    sudo unzip -o /tmp/nomad.zip -d "$INSTALL_DIR"
+    set_ownership
+    if [ $START == "1" ]; then sudo systemctl restart nomad; fi
+}
+
+set_ownership() {
+    sudo chmod 0755 "$INSTALL_PATH"
+    sudo chown root:root "$INSTALL_PATH"
+}
+
+while [[ $# -gt 0 ]]
+do
+opt="$1"
+    case $opt in
+        --nomad_sha)
+            if [ -z "$2" ]; then echo "Missing sha parameter"; usage; fi
+            NOMAD_SHA="$2"
+            install_fn=install_from_s3
+            shift 2
+            ;;
+        --nomad_release | --nomad_version)
+            if [ -z "$2" ]; then echo "Missing version parameter"; usage; fi
+            NOMAD_VERSION="$2"
+            install_fn=install_from_release
+            shift 2
+            ;;
+        --nomad_binary)
+            if [ -z "$2" ]; then echo "Missing file parameter"; usage; fi
+            NOMAD_UPLOADED_BINARY="$2"
+            install_fn=install_from_uploaded_binary
+            shift 2
+            ;;
+        --nostart)
+            # for initial packer builds, we don't want to start Nomad
+            START=0
+            shift
+            ;;
+        *) usage ;;
+    esac
+done
+
+# call the appropriate instalation function
+if [ -z "$install_fn" ]; then echo "Missing install option"; usage; fi
+$install_fn
--- a/e2e/terraform/packer/linux/setup.sh
+++ b/e2e/terraform/packer/linux/setup.sh
@@ -22,7 +22,6 @@ VAULTDIR=/opt/vault

 # Will be overwritten by sha specified
 NOMADVERSION=0.9.1
-NOMADDOWNLOAD=https://releases.hashicorp.com/nomad/${NOMADVERSION}/nomad_${NOMADVERSION}_linux_amd64.zip
 NOMADCONFIGDIR=/etc/nomad.d
 NOMADDIR=/opt/nomad
 NOMADPLUGINDIR=/opt/nomad/plugins
@@ -67,12 +66,6 @@ sudo chmod 755 $VAULTCONFIGDIR
 sudo mkdir -p $VAULTDIR
 sudo chmod 755 $VAULTDIR

-echo "Install Nomad"
-curl -fsL -o /tmp/nomad.zip $NOMADDOWNLOAD
-sudo unzip -q /tmp/nomad.zip -d /usr/local/bin
-sudo chmod 0755 /usr/local/bin/nomad
-sudo chown root:root /usr/local/bin/nomad
-
 echo "Configure Nomad"
 sudo mkdir -p $NOMADCONFIGDIR
 sudo chmod 755 $NOMADCONFIGDIR
@@ -81,6 +74,11 @@ sudo chmod 755 $NOMADDIR
 sudo mkdir -p $NOMADPLUGINDIR
 sudo chmod 755 $NOMADPLUGINDIR

+echo "Install Nomad"
+sudo mv /tmp/install-nomad /opt/install-nomad
+sudo chmod +x /opt/install-nomad
+/opt/install-nomad --nomad_version $NOMADVERSION --nostart
+
 echo "Installing third-party apt repositories"

 # Docker
--- a/e2e/terraform/packer/packer-windows.json
+++ b/e2e/terraform/packer/packer-windows.json
@@ -44,14 +44,24 @@
    {
      "type": "windows-restart"
    },
+    {
+      "type": "file",
+      "source": "./windows/install-nomad.ps1",
+      "destination": "/opt/install-nomad.ps1"
+    },
+    {
+      "type": "powershell",
+      "elevated_user": "Administrator",
+      "elevated_password": "{{.WinRMPassword}}",
+      "inline": ["/opt/install-nomad.ps1 -nomad_version 0.9.6 -nostart"]
+    },
    {
      "type": "powershell",
      "elevated_user": "Administrator",
      "elevated_password": "{{.WinRMPassword}}",
      "scripts": [
        "windows/install-consul.ps1",
-        "windows/install-vault.ps1",
-        "windows/install-nomad.ps1"
+        "windows/install-vault.ps1"
      ]
    },
    {
--- a/e2e/terraform/packer/packer.json
+++ b/e2e/terraform/packer/packer.json
@@ -14,6 +14,11 @@
    }
  ],
  "provisioners": [
+    {
+      "type": "file",
+      "source": "./linux/install-nomad",
+      "destination": "/tmp/install-nomad"
+    },
    {
      "type": "shell",
      "script": "./linux/setup.sh"
--- a/e2e/terraform/packer/windows/install-nomad.ps1
+++ b/e2e/terraform/packer/windows/install-nomad.ps1
@@ -1,34 +1,164 @@
+param(
+    [string]$nomad_sha,
+    [string]$nomad_version,
+    [string]$nomad_binary,
+    [switch]$nostart = $false
+)
+
 Set-StrictMode -Version latest
 $ErrorActionPreference = "Stop"

+
+
+$usage = @"
+Usage: install-nomad [options...]
+Options (use one of the following):
+ --nomad_sha SHA          full git sha to install from S3
+ --nomad_version VERSION  release version number (ex. 0.12.3+ent)
+ --nomad_binary FILEPATH  path to file on host
+ --nostart                do not start or restart Nomad
+"@
+
+$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")
+if (!$RunningAsAdmin) {
+  Write-Error "Must be executed in Administrator level shell."
+  exit 1
+}
+
 # Force TLS1.2
 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12

+
+$install_path = "C:\opt\nomad.exe"
+
 Set-Location C:\opt

-Try {
-    # we install the most recent stable/GA release; this will be replaced
-    # with the current master when we run e2e tests
-    $releases = "https://releases.hashicorp.com"
-    $version = "0.9.6"
-    $url = "${releases}/nomad/${version}/nomad_${version}_windows_amd64.zip"
-
-    $configDir = "C:\opt\nomad.d"
-    md $configDir
-    md C:\opt\nomad
-
-    # TODO: check sha!
-    Write-Output "Downloading Nomad from: $url"
-    Invoke-WebRequest -Uri $url -Outfile nomad.zip
-    Expand-Archive .\nomad.zip .\
-    mv nomad.exe C:\opt\nomad.exe
-    C:\opt\nomad.exe version
-    rm nomad.zip
-
-} Catch {
-    Write-Error "Failed to install Nomad."
-    $host.SetShouldExit(-1)
-    throw
+function Usage {
+    Write-Output "${usage}"
 }

-Write-Output "Installed Nomad."
+function InstallFromS3 {
+    Try {
+        # check that we don't already have this version
+        if (C:\opt\nomad.exe -version `
+          | Select-String -Pattern $nomad_sha -SimpleMatch -Quiet) {
+              Write-Output "${nomad_sha} already installed"
+              return
+          }
+    } Catch {
+        Write-Output "${nomad_sha} not previously installed"
+    }
+
+    Try {
+        $key = "builds-oss/nomad_windows_amd64_${nomad_sha}.zip"
+		Read-S3Object -BucketName nomad-team-dev-test-binaries -Key $key -File ./nomad.zip
+		Remove-Item -Path $install_path -Force -ErrorAction Ignore
+		Expand-Archive ./nomad.zip ./ -Force
+		Move-Item -Path .\pkg\windows_amd64\nomad.exe -Destination $install_path -Force
+		Remove-Item -Path nomad.zip -Force -ErrorAction Ignore
+
+        New-Item -ItemType Directory -Force -Path C:\opt\nomad.d
+        New-Item -ItemType Directory -Force -Path C:\opt\nomad
+        Write-Output "Installed Nomad."
+
+        if (!($nostart)) {
+            StartNomad
+        }
+    } Catch {
+        Write-Error "Failed to install Nomad."
+        $host.SetShouldExit(-1)
+        throw
+    }
+}
+
+function InstallFromUploadedBinary {
+    Try {
+		Remove-Item -Path $install_path -Force -ErrorAction Ignore
+        Move-Item -Path $nomad_binary -Destination $install_path -Force
+
+        New-Item -ItemType Directory -Force -Path C:\opt\nomad.d
+        New-Item -ItemType Directory -Force -Path C:\opt\nomad
+        Write-Output "Installed Nomad."
+
+        if (!($nostart)) {
+            StartNomad
+        }
+    } Catch {
+        Write-Error "Failed to install Nomad."
+        $host.SetShouldExit(-1)
+        throw
+    }
+}
+
+function InstallFromRelease {
+    Try {
+        # check that we don't already have this version
+        if (C:\opt\nomad.exe -version `
+          | Select-String -Pattern $nomad_version -SimpleMatch -Quiet) {
+              if (C:\opt\nomad.exe -version `
+                | Select-String -Pattern dev -SimpleMatch -Quiet -NotMatch) {
+                    Write-Output "${nomad_version} already installed"
+                    return
+                }
+          }
+    } Catch {
+        Write-Output "${nomad_version} not previously installed"
+    }
+
+    Try {
+        $releases = "https://releases.hashicorp.com"
+        $url = "${releases}/nomad/${nomad_version}/nomad_${nomad_version}_windows_amd64.zip"
+
+        Write-Output "Downloading Nomad from: $url"
+        Invoke-WebRequest -Uri $url -Outfile nomad.zip
+		Remove-Item -Path $install_path -Force -ErrorAction Ignore
+        Expand-Archive .\nomad.zip .\
+		Remove-Item nomad.zip -Force -ErrorAction Ignore
+
+        New-Item -ItemType Directory -Force -Path C:\opt\nomad.d
+        New-Item -ItemType Directory -Force -Path C:\opt\nomad
+        Write-Output "Installed Nomad."
+
+        if (!($nostart)) {
+            StartNomad
+        }
+    } Catch {
+        Write-Error "Failed to install Nomad."
+        $host.SetShouldExit(-1)
+        throw
+    }
+}
+
+function StartNomad {
+    New-NetFirewallRule `
+      -DisplayName 'Nomad HTTP Inbound' `
+      -Profile @('Public', 'Domain', 'Private') `
+      -Direction Inbound `
+      -Action Allow `
+      -Protocol TCP `
+      -LocalPort @('4646')
+
+    # idempotently enable as a service
+    New-Service `
+      -Name "Nomad" `
+      -BinaryPathName "C:\opt\nomad.exe agent -config C:\opt\nomad.d" `
+      -StartupType "Automatic" `
+      -ErrorAction Ignore
+
+    Start-Service "Nomad"
+}
+
+if ( "" -ne $nomad_sha ) {
+    InstallFromS3
+    return
+}
+if ( "" -ne $nomad_version ) {
+    InstallFromRelease
+    return
+}
+if ( "" -ne $nomad_binary ) {
+    InstallFromUploadedBinary
+    return
+}
+
+Usage