Infrastructure for Windows e2e testing (#6584)

Includes: * baseline Windows AMI * initial pass at Terraform configurations * OpenSSH for Windows Using OpenSSH is a lot nicer for Nomad developers than winrm would be, plus it lets us avoid passing around the Windows password in the clear. Note that now we're copying up all the provisioning scripts and configs as a zipped bundle because TF's file provisioner dies in the middle of pushing up multiple files (whereas `scp -r` works fine). We're also running all the provisioning scripts inside the userdata by polling for the zip file to show up (gross!). This is because `remote-exec` provisioners are failing on Windows with the same symptoms as: https://github.com/hashicorp/terraform/issues/17728 If we can't fix this, it'll prevent us from having multiple Windows clients running until TF supports count interpolation in the `template_file`, which is planned for a later 0.12 release.
2026-01-04 17:35:43 +03:00 · 2019-11-19 11:06:10 -05:00
parent c7db027809
commit 04b588dcf0
24 changed files with 828 additions and 8 deletions
--- a/e2e/terraform/.gitignore
+++ b/e2e/terraform/.gitignore
@@ -0,0 +1 @@
+*.zip
--- a/e2e/terraform/README.md
+++ b/e2e/terraform/README.md
@@ -23,9 +23,15 @@ Terraform will output node IPs that may be accessed via ssh:
 ssh -i keys/nomad-e2e-*.pem ubuntu@${EC2_IP_ADDR}
 ```

+The Windows client runs OpenSSH for conveniences, but has a different user and will drop you into a Powershell shell instead of bash:
+
+```
+ssh -i keys/nomad-e2e-*.pem Administrator@${EC2_IP_ADDR}
+```
+

 ## Teardown
-The terraform state file stores all the info, so the nomad_sha doesn't need to be valid during teardown. 
+The terraform state file stores all the info, so the nomad_sha doesn't need to be valid during teardown.

 ```
 $ cd e2e/terraform/
--- a/e2e/terraform/compute.tf
+++ b/e2e/terraform/compute.tf
@@ -1,5 +1,5 @@
 resource "aws_instance" "server" {
-  ami                    = data.aws_ami.main.image_id
+  ami                    = data.aws_ami.linux.image_id
  instance_type          = var.instance_type
  key_name               = module.keys.key_name
  vpc_security_group_ids = [aws_security_group.primary.id]
@@ -44,7 +44,7 @@ resource "aws_instance" "server" {
 }

 resource "aws_instance" "client" {
-  ami                    = data.aws_ami.main.image_id
+  ami                    = data.aws_ami.linux.image_id
  instance_type          = var.instance_type
  key_name               = module.keys.key_name
  vpc_security_group_ids = [aws_security_group.primary.id]
@@ -95,3 +95,71 @@ resource "aws_instance" "client" {
    }
  }
 }
+data "template_file" "user_data_client_windows" {
+  template = file("${path.root}/shared/config/userdata-windows.ps1")
+  vars = {
+    nomad_sha = var.nomad_sha
+  }
+}
+
+data "archive_file" "windows_configs" {
+  type        = "zip"
+  source_dir  = "./shared"
+  output_path = "./windows_configs.zip"
+}
+
+resource "aws_instance" "client_windows" {
+  ami                    = data.aws_ami.windows.image_id
+  instance_type          = var.instance_type
+  key_name               = module.keys.key_name
+  vpc_security_group_ids = [aws_security_group.primary.id]
+  count                  = var.windows_client_count
+  depends_on             = [aws_instance.server]
+  iam_instance_profile   = "${aws_iam_instance_profile.instance_profile.name}"
+
+  # Instance tags
+  tags = {
+    Name           = "${local.random_name}-client-windows-${count.index}"
+    ConsulAutoJoin = "auto-join"
+  }
+
+  ebs_block_device {
+    device_name           = "xvdd"
+    volume_type           = "gp2"
+    volume_size           = "50"
+    delete_on_termination = "true"
+  }
+
+  # We need this userdata script because Windows machines don't
+  # configure ssh with cloud-init by default.
+  user_data = data.template_file.user_data_client_windows.rendered
+
+  # Note:
+  # we're copying up all the provisioning scripts and configs as
+  # a zipped bundle because TF's file provisioner dies in the middle
+  # of pushing up multiple files (whereas 'scp -r' works fine).
+  #
+  # We're also running all the provisioning scripts inside the
+  # userdata by polling for the zip file to show up. (Gross!)
+  # This is because remote-exec provisioners are failing on Windows
+  # with the same symptoms as:
+  # https://github.com/hashicorp/terraform/issues/17728
+  #
+  # If we can't fix this, it'll prevent us from having multiple
+  # Windows clients running until TF supports count interpolation
+  # in the template_file, which is planned for a later 0.12 release
+  #
+  provisioner "file" {
+    source      = "./windows_configs.zip"
+    destination = "C:/ops/windows_configs.zip"
+
+    connection {
+      host        = coalesce(self.public_ip, self.private_ip)
+      type        = "ssh"
+      user        = "Administrator"
+      private_key = module.keys.private_key_pem
+      timeout     = "10m"
+    }
+  }
+
+}
--- a/e2e/terraform/main.tf
+++ b/e2e/terraform/main.tf
@@ -28,6 +28,11 @@ variable "client_count" {
  default     = "4"
 }

+variable "windows_client_count" {
+  description = "The number of windows clients to provision."
+  default     = "1"
+}
+
 variable "nomad_sha" {
  description = "The sha of Nomad to run"
 }
@@ -39,6 +44,12 @@ provider "aws" {
 resource "random_pet" "e2e" {
 }

+resource "random_password" "windows_admin_password" {
+  length           = 20
+  special          = true
+  override_special = "_%@"
+}
+
 locals {
  random_name = "${var.name}-${random_pet.e2e.id}"
 }
@@ -51,7 +62,7 @@ module "keys" {
  version = "v2.0.0"
 }

-data "aws_ami" "main" {
+data "aws_ami" "linux" {
  most_recent = true
  owners      = ["self"]

@@ -66,6 +77,21 @@ data "aws_ami" "main" {
  }
 }

+data "aws_ami" "windows" {
+  most_recent = true
+  owners      = ["self"]
+
+  filter {
+    name   = "name"
+    values = ["nomad-e2e-windows-2016*"]
+  }
+
+  filter {
+    name   = "tag:OS"
+    values = ["Windows2016"]
+  }
+}
+
 data "aws_caller_identity" "current" {
 }

--- a/e2e/terraform/packer/README.md
+++ b/e2e/terraform/packer/README.md
@@ -18,4 +18,47 @@ $ packer --version

 # build linux AMI
 $ packer build packer.json
+
+# build Windows AMI
+$ packer build packer-windows.json
 ```
+
+## Debugging Packer Builds
+
+You'll need the Windows administrator password in order to access Windows machines via `winrm` as Packer does. You can get this by enabling `-debug` on your Packer build.
+
+```sh
+packer build -debug -on-error=abort packer-windows.json
+...
+==> amazon-ebs: Pausing after run of step 'StepRunSourceInstance'. Press enter to continue.
+==> amazon-ebs: Waiting for auto-generated password for instance...
+    amazon-ebs: Password (since debug is enabled): <redacted>
+```
+
+Alternately, you can follow the steps in the [AWS documentation](https://aws.amazon.com/premiumsupport/knowledge-center/retrieve-windows-admin-password/). Note that you'll need the `ec2_amazon-ebs.pem` file that Packer drops in this directory.
+
+
+Then in powershell (note the leading `$` here indicate variable declarations, not shell prompts!):
+
+```
+$username = "Administrator"
+$password = "<redacted>"
+$securePassword = ConvertTo-SecureString -AsPlainText -Force $password
+$remoteHostname = "54.x.y.z"
+$port = 5986
+$cred = New-Object System.Management.Automation.PSCredential ($username, $securePassword)
+$so = New-PSSessionOption -SkipCACheck -SkipCNCheck
+
+Enter-PsSession `
+    -ComputerName $remoteHostname `
+    -Port $port `
+    -Credential $cred `
+    -UseSSL `
+    -SessionOption $so `
+    -Authentication Basic
+```
+
+Packer doesn't have a cleanup command if you've run `-on-error=abort`. So when you're done, clean up the machine by looking for "Packer" in the AWS console:
+* [EC2 instances](https://console.aws.amazon.com/ec2/home?region=us-east-1#Instances:search=Packer;sort=tag:Name)
+* [Key pairs](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#KeyPairs:search=packer;sort=keyName)
+* [Security groups](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#SecurityGroups:search=packer;sort=groupName)
--- a/e2e/terraform/packer/packer-windows.json
+++ b/e2e/terraform/packer/packer-windows.json
@@ -0,0 +1,69 @@
+{
+  "builders": [
+    {
+      "type": "amazon-ebs",
+      "region": "us-east-1",
+      "source_ami_filter": {
+        "filters": {
+          "virtualization-type": "hvm",
+          "name": "Windows_Server-2016-English-Full-Base-*",
+          "root-device-type": "ebs"
+        },
+        "owners": [
+          "amazon"
+        ],
+        "most_recent": true
+      },
+      "instance_type": "t2.medium",
+      "ami_name": "nomad-e2e-windows-2016-{{timestamp}}",
+      "ami_groups": [
+        "all"
+      ],
+      "communicator": "winrm",
+      "user_data_file": "windows/setupwinrm.ps1",
+      "winrm_username": "Administrator",
+      "winrm_insecure": true,
+      "winrm_use_ssl": true,
+      "tags": {
+        "OS": "Windows2016"
+      }
+    }
+  ],
+  "provisioners": [
+    {
+      "type": "powershell",
+      "elevated_user": "Administrator",
+      "elevated_password": "{{.WinRMPassword}}",
+      "scripts": [
+        "windows/disable-windows-updates.ps1",
+        "windows/fix-tls.ps1",
+        "windows/install-nuget.ps1",
+        "windows/install-tools.ps1",
+        "windows/install-docker.ps1",
+        "windows/setup-directories.ps1",
+        "windows/install-openssh.ps1"
+      ]
+    },
+    {
+      "type": "windows-restart"
+    },
+    {
+      "type": "powershell",
+      "elevated_user": "Administrator",
+      "elevated_password": "{{.WinRMPassword}}",
+      "scripts": [
+        "windows/install-consul.ps1",
+        "windows/install-vault.ps1",
+        "windows/install-nomad.ps1"
+      ]
+    },
+    {
+      "type": "powershell",
+      "inline": [
+        "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\SendWindowsIsReady.ps1 -Schedule",
+        "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\InitializeInstance.ps1 -Schedule",
+        "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\SysprepInstance.ps1 -NoShutdown"
+      ]
+    }
+  ]
+}
--- a/e2e/terraform/packer/windows/README.md
+++ b/e2e/terraform/packer/windows/README.md
@@ -0,0 +1,20 @@
+# Windows Packer Build
+
+There are a few boilerplate items in the Powershell scripts, explained below.
+
+The default TLS protocol in the version of .NET that our Powershell cmdlets are built in it 1.0, which means plenty of properly configured HTTP servers will reject requests. The boilerplate snippet below sets this for the current script:
+
+```
+# Force TLS1.2
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+```
+
+We need to run some of the scripts as an administrator role. The following is a safety check that we're doing so:
+
+```
+$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")
+if (!$RunningAsAdmin) {
+  Write-Error "Must be executed in Administrator level shell."
+  exit 1
+}
+```
--- a/e2e/terraform/packer/windows/disable-windows-updates.ps1
+++ b/e2e/terraform/packer/windows/disable-windows-updates.ps1
@@ -0,0 +1,30 @@
+$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")
+if (!$RunningAsAdmin) {
+  Write-Error "Must be executed in Administrator level shell."
+  exit 1
+}
+
+$service = Get-WmiObject Win32_Service -Filter 'Name="wuauserv"'
+
+if (!$service) {
+  Write-Error "Failed to retrieve the wauserv service"
+  exit 1
+}
+
+if ($service.StartMode -ne "Disabled") {
+  $result = $service.ChangeStartMode("Disabled").ReturnValue
+  if($result) {
+    Write-Error "Failed to disable the 'wuauserv' service. The return value was $result."
+    exit 1
+  }
+}
+
+if ($service.State -eq "Running") {
+  $result = $service.StopService().ReturnValue
+  if ($result) {
+    Write-Error "Failed to stop the 'wuauserv' service. The return value was $result."
+    exit 1
+  }
+}
+
+Write-Output "Automatic Windows Updates disabled."
--- a/e2e/terraform/packer/windows/fix-tls.ps1
+++ b/e2e/terraform/packer/windows/fix-tls.ps1
@@ -0,0 +1,151 @@
+# This script hardens TLS configuration by disabling weak and broken protocols
+# and enabling useful protocols like TLS 1.1 and 1.2.
+
+$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")
+if (!$RunningAsAdmin) {
+  Write-Error "Must be executed in Administrator level shell."
+  exit 1
+}
+
+$weakProtocols = @(
+	'Multi-Protocol Unified Hello',
+	'PCT 1.0',
+	'SSL 2.0',
+	'SSL 3.0'
+)
+
+$strongProtocols = @(
+	'TLS 1.0',
+	'TLS 1.1',
+	'TLS 1.2'
+)
+
+$weakCiphers = @(
+	'DES 56/56',
+	'NULL',
+	'RC2 128/128',
+	'RC2 40/128',
+	'RC2 56/128',
+	'RC4 40/128',
+	'RC4 56/128',
+	'RC4 64/128',
+	'RC4 128/128'
+)
+
+$strongCiphers = @(
+	'AES 128/128',
+	'AES 256/256',
+	'Triple DES 168/168'
+)
+
+$weakHashes = @(
+	'MD5',
+	'SHA'
+)
+
+$strongHashes = @(
+	'SHA 256',
+	'SHA 384',
+	'SHA 512'
+)
+
+$strongKeyExchanges = @(
+	'Diffie-Hellman',
+	'ECDH',
+	'PKCS'
+)
+
+$cipherOrder = @(
+  'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384_P521',
+  'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384_P384',
+  'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384_P256',
+  'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA_P521',
+  'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA_P384',
+  'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA_P256',
+  'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256_P521',
+  'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256_P384',
+  'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256_P256',
+  'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA_P521',
+  'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA_P384',
+  'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA_P256',
+  'TLS_RSA_WITH_AES_256_GCM_SHA384',
+  'TLS_RSA_WITH_AES_128_GCM_SHA256',
+  'TLS_RSA_WITH_AES_256_CBC_SHA256',
+  'TLS_RSA_WITH_AES_256_CBC_SHA',
+  'TLS_RSA_WITH_AES_128_CBC_SHA256',
+  'TLS_RSA_WITH_AES_128_CBC_SHA',
+  'TLS_RSA_WITH_3DES_EDE_CBC_SHA'
+)
+
+# Reset the protocols key
+New-Item 'HKLM:SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols' -Force | Out-Null
+
+# Disable weak protocols
+Foreach ($protocol in $weakProtocols) {
+  New-Item HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -Force | Out-Null
+  New-Item HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -Force | Out-Null
+  New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -name Enabled -value 0 -PropertyType 'DWord' -Force | Out-Null
+  New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -name DisabledByDefault -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null
+  New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -name Enabled -value 0 -PropertyType 'DWord' -Force | Out-Null
+  New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -name DisabledByDefault -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null
+}
+
+# Enable strong protocols
+Foreach ($protocol in $strongProtocols) {
+  New-Item HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -Force | Out-Null
+  New-Item HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -Force | Out-Null
+  New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -name 'Enabled' -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null
+  New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -name 'DisabledByDefault' -value 0 -PropertyType 'DWord' -Force | Out-Null
+  New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -name 'Enabled' -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null
+  New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -name 'DisabledByDefault' -value 0 -PropertyType 'DWord' -Force | Out-Null
+}
+
+# Reset the ciphers key
+New-Item 'HKLM:SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Ciphers' -Force | Out-Null
+
+# Disable Weak Ciphers
+Foreach ($cipher in $weakCiphers) {
+  $key = (get-item HKLM:\).OpenSubKey("SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Ciphers", $true).CreateSubKey($cipher)
+  $key.SetValue('Enabled', 0, 'DWord')
+  $key.Close()
+}
+
+# Enable Strong Ciphers
+Foreach ($cipher in $strongCiphers) {
+  $key = (get-item HKLM:\).OpenSubKey("SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Ciphers", $true).CreateSubKey($cipher)
+  New-ItemProperty -path "HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Ciphers\$cipher" -name 'Enabled' -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null
+  $key.Close()
+}
+
+# Reset the hashes key
+New-Item 'HKLM:SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Hashes' -Force | Out-Null
+
+# Disable weak hashes
+Foreach ($hash in $weakHashes) {
+  $key = (get-item HKLM:\).OpenSubKey("SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Hashes", $true).CreateSubKey($hash)
+  New-ItemProperty -path "HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Hashes\$hash" -name 'Enabled' -value '0' -PropertyType 'DWord' -Force | Out-Null
+  $key.Close()
+}
+
+# Enable Hashes
+Foreach ($hash in $strongHashes) {
+  $key = (get-item HKLM:\).OpenSubKey("SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Hashes", $true).CreateSubKey($hash)
+  New-ItemProperty -path "HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Hashes\$hash" -name 'Enabled' -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null
+  $key.Close()
+}
+
+# Reset the KeyExchangeAlgorithms key
+New-Item 'HKLM:SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\KeyExchangeAlgorithms' -Force | Out-Null
+
+# Enable KeyExchangeAlgorithms
+Foreach ($keyExchange in $strongKeyExchanges) {
+  $key = (get-item HKLM:\).OpenSubKey("SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\KeyExchangeAlgorithms", $true).CreateSubKey($keyExchange)
+  New-ItemProperty -path "HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\KeyExchangeAlgorithms\$keyExchange" -name 'Enabled' -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null
+  $key.Close()
+}
+
+# Set cipher order
+$cipherOrderString = [string]::join(',', $cipherOrder)
+New-ItemProperty -path 'HKLM:\SOFTWARE\Policies\Microsoft\Cryptography\Configuration\SSL\00010002' -name 'Functions' -value $cipherOrderString -PropertyType 'String' -Force | Out-Null
+
+Write-Output "TLS hardened."
--- a/e2e/terraform/packer/windows/install-consul.ps1
+++ b/e2e/terraform/packer/windows/install-consul.ps1
@@ -0,0 +1,32 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = "Stop"
+
+# Force TLS1.2
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+Set-Location C:\opt
+
+Try {
+    $releases = "https://releases.hashicorp.com"
+    $version = "1.6.1"
+    $url = "${releases}/consul/${version}/consul_${version}_windows_amd64.zip"
+
+    $configDir = "C:\opt\consul.d"
+    md $configDir
+    md C:\opt\consul
+
+    # TODO: check sha!
+    Write-Output "Downloading Consul from: $url"
+    Invoke-WebRequest -Uri $url -Outfile consul.zip
+    Expand-Archive .\consul.zip .\
+    mv consul.exe C:\opt\consul.exe
+    C:\opt\consul.exe version
+    rm consul.zip
+
+} Catch {
+    Write-Error "Failed to install Consul."
+    $host.SetShouldExit(-1)
+    throw
+}
+
+Write-Output "Installed Consul."
--- a/e2e/terraform/packer/windows/install-docker.ps1
+++ b/e2e/terraform/packer/windows/install-docker.ps1
@@ -0,0 +1,35 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = "Stop"
+
+$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")
+if (!$RunningAsAdmin) {
+  Write-Error "Must be executed in Administrator level shell."
+  exit 1
+}
+
+# Force TLS1.2
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+Try {
+    Write-Output "Installing containers feature."
+    Install-WindowsFeature -Name Containers
+
+    Write-Output "Creating user for Docker."
+    net localgroup docker /add
+    net localgroup docker $env:USERNAME /add
+
+    Write-Output "Installing Docker."
+    Set-PSRepository -InstallationPolicy Trusted -Name PSGallery
+    Install-Module -Name DockerMsftProvider -Repository PSGallery -Force
+    Install-Package -Name docker -ProviderName DockerMsftProvider -Force
+
+} Catch {
+    Write-Error "Failed to install Docker."
+    $host.SetShouldExit(-1)
+    throw
+} Finally {
+    # clean up by re-securing this package repo
+    Set-PSRepository -InstallationPolicy Untrusted -Name PSGallery
+}
+
+Write-Output "Installed Docker."
--- a/e2e/terraform/packer/windows/install-nomad.ps1
+++ b/e2e/terraform/packer/windows/install-nomad.ps1
@@ -0,0 +1,34 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = "Stop"
+
+# Force TLS1.2
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+Set-Location C:\opt
+
+Try {
+    # we install the most recent stable/GA release; this will be replaced
+    # with the current master when we run e2e tests
+    $releases = "https://releases.hashicorp.com"
+    $version = "0.9.6"
+    $url = "${releases}/nomad/${version}/nomad_${version}_windows_amd64.zip"
+
+    $configDir = "C:\opt\nomad.d"
+    md $configDir
+    md C:\opt\nomad
+
+    # TODO: check sha!
+    Write-Output "Downloading Nomad from: $url"
+    Invoke-WebRequest -Uri $url -Outfile nomad.zip
+    Expand-Archive .\nomad.zip .\
+    mv nomad.exe C:\opt\nomad.exe
+    C:\opt\nomad.exe version
+    rm nomad.zip
+
+} Catch {
+    Write-Error "Failed to install Nomad."
+    $host.SetShouldExit(-1)
+    throw
+}
+
+Write-Output "Installed Nomad."
--- a/e2e/terraform/packer/windows/install-nuget.ps1
+++ b/e2e/terraform/packer/windows/install-nuget.ps1
@@ -0,0 +1,21 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = "Stop"
+
+$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")
+if (!$RunningAsAdmin) {
+  Write-Error "Must be executed in Administrator level shell."
+  exit 1
+}
+
+# Force TLS1.2
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+Try {
+    Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force
+} Catch {
+    Write-Error "Failed to install NuGet package manager."
+    $host.SetShouldExit(-1)
+    throw
+}
+
+Write-Output "Installed NuGet."
--- a/e2e/terraform/packer/windows/install-openssh.ps1
+++ b/e2e/terraform/packer/windows/install-openssh.ps1
@@ -0,0 +1,53 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = "Stop"
+
+$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")
+if (!$RunningAsAdmin) {
+  Write-Error "Must be executed in Administrator level shell."
+  exit 1
+}
+
+# Force TLS1.2
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+Try {
+
+    # install portable SSH instead of the Windows feature because we
+    # need to target 2016
+    $repo = "https://github.com/PowerShell/Win32-OpenSSH"
+    $version = "v8.0.0.0p1-Beta"
+    $url = "${repo}/releases/download/${version}/OpenSSH-Win64.zip"
+
+    # TODO: check sha!
+    Write-Output "Downloading OpenSSH from: $url"
+    Invoke-WebRequest -Uri $url -Outfile "OpenSSH-Win64.zip"
+    Expand-Archive ".\OpenSSH-Win64.zip" "C:\Program Files"
+    Rename-Item -Path "C:\Program Files\OpenSSH-Win64" -NewName "OpenSSH"
+
+    & "C:\Program Files\OpenSSH\install-sshd.ps1"
+
+    # Start the service
+    Start-Service sshd
+    Set-Service -Name sshd -StartupType 'Automatic'
+
+    Start-Service ssh-agent
+    Set-Service -Name ssh-agent -StartupType 'Automatic'
+
+    # Enable host firewall rule if it doesn't exist
+    New-NetFirewallRule -Name sshd -DisplayName 'OpenSSH Server (sshd)' `
+      -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22
+
+    # Set powershell as the OpenSSH login shell
+    New-ItemProperty -Path "HKLM:\SOFTWARE\OpenSSH" `
+      -Name DefaultShell `
+      -Value "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe" `
+      -PropertyType String -Force
+
+
+} Catch {
+    Write-Error "Failed to install OpenSSH."
+    $host.SetShouldExit(-1)
+    throw
+}
+
+Write-Output "Installed OpenSSH."
--- a/e2e/terraform/packer/windows/install-tools.ps1
+++ b/e2e/terraform/packer/windows/install-tools.ps1
@@ -0,0 +1,37 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = "Stop"
+
+$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")
+if (!$RunningAsAdmin) {
+  Write-Error "Must be executed in Administrator level shell."
+  exit 1
+}
+
+# Force TLS1.2
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+# TODO (tgross: some stuff installed on Linux but not here yet
+# - Possible issues: no redis-tools for windows
+# - Possible non-issues: probably don't need tree, curl,tmux
+
+Try {
+    Set-PSRepository -InstallationPolicy Trusted -Name PSGallery
+
+    Write-Output "Installing 7Zip"
+    Install-Package -Force 7Zip4PowerShell
+
+    Write-Output "Installing JQ"
+    Invoke-WebRequest `
+      -Uri https://github.com/stedolan/jq/releases/download/jq-1.6/jq-win64.exe `
+      -Outfile jq-win64.exe
+
+} Catch {
+    Write-Error "Failed to install dependencies."
+    $host.SetShouldExit(-1)
+    throw
+} Finally {
+    # clean up by re-securing this package repo
+    Set-PSRepository -InstallationPolicy Untrusted -Name PSGallery
+}
+
+Write-Output "Installed dependencies"
--- a/e2e/terraform/packer/windows/install-vault.ps1
+++ b/e2e/terraform/packer/windows/install-vault.ps1
@@ -0,0 +1,31 @@
+Set-StrictMode -Version latest
+$ErrorActionPreference = "Stop"
+
+# Force TLS1.2
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+Set-Location C:\opt
+
+Try {
+    $releases = "https://releases.hashicorp.com"
+    $version = "1.2.3"
+    $url = "${releases}/vault/${version}/vault_${version}_windows_amd64.zip"
+
+    $configDir = "C:\opt\vault.d"
+    md $configDir
+
+    # TODO: check sha!
+    Write-Output "Downloading Vault from: $url"
+    Invoke-WebRequest -Uri $url -Outfile vault.zip
+    Expand-Archive .\vault.zip .\
+    mv vault.exe C:\opt\vault.exe
+    C:\opt\vault.exe version
+    rm vault.zip
+
+} Catch {
+    Write-Error "Failed to install Vault."
+    $host.SetShouldExit(-1)
+    throw
+}
+
+Write-Output "Installed Vault."
--- a/e2e/terraform/packer/windows/setup-directories.ps1
+++ b/e2e/terraform/packer/windows/setup-directories.ps1
@@ -0,0 +1,2 @@
+md C:\ops
+md C:\opt
--- a/e2e/terraform/packer/windows/setupwinrm.ps1
+++ b/e2e/terraform/packer/windows/setupwinrm.ps1
@@ -0,0 +1,44 @@
+<powershell>
+
+Write-Output "Running User Data Script"
+Write-Host "(host) Running User Data Script"
+
+Set-ExecutionPolicy Unrestricted -Scope LocalMachine -Force -ErrorAction Ignore
+
+# Don't set this before Set-ExecutionPolicy as it throws an error
+$ErrorActionPreference = "stop"
+
+# Remove HTTP listener
+Remove-Item -Path WSMan:\Localhost\listener\listener* -Recurse
+
+$Cert = New-SelfSignedCertificate `
+  -CertstoreLocation Cert:\LocalMachine\My `
+  -DnsName "packer"
+
+New-Item `
+  -Path WSMan:\LocalHost\Listener `
+  -Transport HTTPS `
+  -Address * `
+  -CertificateThumbPrint $Cert.Thumbprint `
+  -Force
+
+# WinRM
+write-output "Setting up WinRM"
+write-host "(host) setting up WinRM"
+
+cmd.exe /c winrm quickconfig -q
+cmd.exe /c winrm set "winrm/config" '@{MaxTimeoutms="1800000"}'
+cmd.exe /c winrm set "winrm/config/winrs" '@{MaxMemoryPerShellMB="1024"}'
+cmd.exe /c winrm set "winrm/config/service" '@{AllowUnencrypted="true"}'
+cmd.exe /c winrm set "winrm/config/client" '@{AllowUnencrypted="true"}'
+cmd.exe /c winrm set "winrm/config/service/auth" '@{Basic="true"}'
+cmd.exe /c winrm set "winrm/config/client/auth" '@{Basic="true"}'
+cmd.exe /c winrm set "winrm/config/service/auth" '@{CredSSP="true"}'
+cmd.exe /c winrm set "winrm/config/listener?Address=*+Transport=HTTPS" "@{Port=`"5986`";Hostname=`"packer`";CertificateThumbprint=`"$($Cert.Thumbprint)`"}"
+cmd.exe /c netsh advfirewall firewall set rule group="remote administration" new enable=yes
+cmd.exe /c netsh firewall add portopening TCP 5986 "Port 5986"
+cmd.exe /c net stop winrm
+cmd.exe /c sc config winrm start= auto
+cmd.exe /c net start winrm
+
+</powershell>
--- a/e2e/terraform/shared/config/provision-client.sh
+++ b/e2e/terraform/shared/config/provision-client.sh
--- a/e2e/terraform/shared/config/provision-server.sh
+++ b/e2e/terraform/shared/config/provision-server.sh
--- a/e2e/terraform/shared/config/provision-windows-client.ps1
+++ b/e2e/terraform/shared/config/provision-windows-client.ps1
@@ -0,0 +1,49 @@
+param(
+    [string]$Cloud = "aws",
+    [string]$NomadSha = "",
+    [string]$Index=0
+)
+
+# Force TLS1.2
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+# Consul
+cp "C:\ops\shared\consul\base.json" "C:\opt\consul.d\base.json"
+cp "C:\ops\shared\consul\retry_$Cloud.json" "C:\opt\consul.d\retry_$Cloud.json"
+sc.exe create "Consul" binPath= "C:\opt\consul.exe agent -config-dir C:\opt\consul.d -log-file C:\opt\consul\consul.log" start= auto
+sc.exe start "Consul"
+
+# Vault
+# TODO(tgross): we don't need Vault for clients
+# cp "C:\ops\shared\vault\vault.hcl" C:\opt\vault.d\vault.hcl
+# sc.exe create "Vault" binPath= "C:\opt\vault.exe" agent -config-dir "C:\opt\vault.d" start= auto
+
+# Nomad
+
+md C:\opt\nomad
+
+Read-S3Object `
+  -BucketName nomad-team-test-binary `
+  -Key "builds-oss/nomad_windows_amd64_$NomadSha.zip" `
+  -File .\nomad.zip
+
+Expand-Archive .\nomad.zip .\
+rm C:\opt\nomad.exe
+mv nomad.exe C:\opt\nomad.exe
+
+# install config file
+cp "C:\ops\shared\nomad\client-windows.hcl" "C:\opt\nomad.d\nomad.hcl"
+
+# Setup Host Volumes
+md C:\tmp\data
+
+# TODO(tgross): not sure we even support this for Windows?
+# Write-Output "Install CNI"
+# md C:\opt\cni\bin
+# $cni_url = "https://github.com/containernetworking/plugins/releases/download/v0.8.2/cni-plugins-windows-amd64-v0.8.2.tgz"
+# Invoke-WebRequest -Uri "$cni_url" -Outfile cni.tgz
+# Expand-7Zip -ArchiveFileName .\cni.tgz -TargetPath C:\opt\cni\bin\
+
+# enable as a service
+sc.exe create "Nomad" binPath= "C:\opt\nomad.exe agent -config C:\opt\nomad.d" start= auto
+sc.exe start "Nomad"
--- a/e2e/terraform/shared/config/userdata-windows.ps1
+++ b/e2e/terraform/shared/config/userdata-windows.ps1
@@ -0,0 +1,30 @@
+<powershell>
+
+# Bring ebs volume online with read-write access
+Get-Disk | Where-Object IsOffline -Eq $True | Set-Disk -IsOffline $False
+Get-Disk | Where-Object isReadOnly -Eq $True | Set-Disk -IsReadOnly $False
+
+md "C:\Users\Administrator\.ssh\"
+
+$myKey = "C:\Users\Administrator\.ssh\authorized_keys"
+$adminKey = "C:\ProgramData\ssh\administrators_authorized_keys"
+
+Invoke-RestMethod `
+  -Uri "http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key" `
+  -Outfile $myKey
+
+cp $myKey $adminKey
+
+icacls $adminKey /reset
+icacls $adminKey /inheritance:r
+icacls $adminKey /grant BUILTIN\Administrators:`(F`)
+icacls $adminKey /grant SYSTEM:`(F`)
+
+$archiveFile = "C:\ops\windows_configs.zip"
+while (!(Test-Path $archiveFile)) { Start-Sleep 10 }
+
+Expand-Archive $archiveFile "C:\ops\shared"
+
+& C:\ops\shared\config\provision-windows-client.ps1 -Cloud aws -NomadSha ${nomad_sha} -Index 1
+
+</powershell>
--- a/e2e/terraform/shared/nomad/client-windows.hcl
+++ b/e2e/terraform/shared/nomad/client-windows.hcl
@@ -0,0 +1,35 @@
+enable_debug = true
+
+log_level = "debug"
+log_file = true
+
+data_dir = "C:\\opt\\nomad\\data"
+
+bind_addr = "0.0.0.0"
+
+# Enable the client
+client {
+  enabled = true
+
+  options {
+    # Allow rawexec jobs
+    "driver.raw_exec.enable" = "1"
+  }
+}
+
+consul {
+  address = "127.0.0.1:8500"
+}
+
+vault {
+  enabled = true
+  address = "http://active.vault.service.consul:8200"
+}
+
+telemetry {
+  collection_interval        = "1s"
+  disable_hostname           = true
+  prometheus_metrics         = true
+  publish_allocation_metrics = true
+  publish_node_metrics       = true
+}
--- a/e2e/terraform/terraform.tfvars
+++ b/e2e/terraform/terraform.tfvars
@@ -1,4 +1,7 @@
-region            = "us-east-1"
-instance_type     = "t2.medium"
-server_count      = "3"
-client_count      = "4"
+region               = "us-east-1"
+instance_type        = "t2.medium"
+server_count         = "3"
+client_count         = "4"
+
+# TODO(tgross): add only once Windows client is working
+windows_client_count = "0"