From 04b588dcf004cca5ee8ebc0c55f9ecb05a664afc Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Tue, 19 Nov 2019 11:06:10 -0500 Subject: [PATCH] Infrastructure for Windows e2e testing (#6584) Includes: * baseline Windows AMI * initial pass at Terraform configurations * OpenSSH for Windows Using OpenSSH is a lot nicer for Nomad developers than winrm would be, plus it lets us avoid passing around the Windows password in the clear. Note that now we're copying up all the provisioning scripts and configs as a zipped bundle because TF's file provisioner dies in the middle of pushing up multiple files (whereas `scp -r` works fine). We're also running all the provisioning scripts inside the userdata by polling for the zip file to show up (gross!). This is because `remote-exec` provisioners are failing on Windows with the same symptoms as: https://github.com/hashicorp/terraform/issues/17728 If we can't fix this, it'll prevent us from having multiple Windows clients running until TF supports count interpolation in the `template_file`, which is planned for a later 0.12 release. --- e2e/terraform/.gitignore | 1 + e2e/terraform/README.md | 8 +- e2e/terraform/compute.tf | 72 ++++++++- e2e/terraform/main.tf | 28 +++- e2e/terraform/packer/README.md | 43 +++++ e2e/terraform/packer/packer-windows.json | 69 ++++++++ e2e/terraform/packer/windows/README.md | 20 +++ .../windows/disable-windows-updates.ps1 | 30 ++++ e2e/terraform/packer/windows/fix-tls.ps1 | 151 ++++++++++++++++++ .../packer/windows/install-consul.ps1 | 32 ++++ .../packer/windows/install-docker.ps1 | 35 ++++ .../packer/windows/install-nomad.ps1 | 34 ++++ .../packer/windows/install-nuget.ps1 | 21 +++ .../packer/windows/install-openssh.ps1 | 53 ++++++ .../packer/windows/install-tools.ps1 | 37 +++++ .../packer/windows/install-vault.ps1 | 31 ++++ .../packer/windows/setup-directories.ps1 | 2 + e2e/terraform/packer/windows/setupwinrm.ps1 | 44 +++++ .../shared/config/provision-client.sh | 0 .../shared/config/provision-server.sh | 0 .../config/provision-windows-client.ps1 | 49 ++++++ .../shared/config/userdata-windows.ps1 | 30 ++++ e2e/terraform/shared/nomad/client-windows.hcl | 35 ++++ e2e/terraform/terraform.tfvars | 11 +- 24 files changed, 828 insertions(+), 8 deletions(-) create mode 100644 e2e/terraform/.gitignore create mode 100644 e2e/terraform/packer/packer-windows.json create mode 100644 e2e/terraform/packer/windows/README.md create mode 100755 e2e/terraform/packer/windows/disable-windows-updates.ps1 create mode 100755 e2e/terraform/packer/windows/fix-tls.ps1 create mode 100755 e2e/terraform/packer/windows/install-consul.ps1 create mode 100755 e2e/terraform/packer/windows/install-docker.ps1 create mode 100755 e2e/terraform/packer/windows/install-nomad.ps1 create mode 100755 e2e/terraform/packer/windows/install-nuget.ps1 create mode 100755 e2e/terraform/packer/windows/install-openssh.ps1 create mode 100755 e2e/terraform/packer/windows/install-tools.ps1 create mode 100755 e2e/terraform/packer/windows/install-vault.ps1 create mode 100755 e2e/terraform/packer/windows/setup-directories.ps1 create mode 100755 e2e/terraform/packer/windows/setupwinrm.ps1 mode change 100644 => 100755 e2e/terraform/shared/config/provision-client.sh mode change 100644 => 100755 e2e/terraform/shared/config/provision-server.sh create mode 100755 e2e/terraform/shared/config/provision-windows-client.ps1 create mode 100755 e2e/terraform/shared/config/userdata-windows.ps1 create mode 100644 e2e/terraform/shared/nomad/client-windows.hcl diff --git a/e2e/terraform/.gitignore b/e2e/terraform/.gitignore new file mode 100644 index 000000000..c4c4ffc6a --- /dev/null +++ b/e2e/terraform/.gitignore @@ -0,0 +1 @@ +*.zip diff --git a/e2e/terraform/README.md b/e2e/terraform/README.md index e41873c35..e46702709 100644 --- a/e2e/terraform/README.md +++ b/e2e/terraform/README.md @@ -23,9 +23,15 @@ Terraform will output node IPs that may be accessed via ssh: ssh -i keys/nomad-e2e-*.pem ubuntu@${EC2_IP_ADDR} ``` +The Windows client runs OpenSSH for conveniences, but has a different user and will drop you into a Powershell shell instead of bash: + +``` +ssh -i keys/nomad-e2e-*.pem Administrator@${EC2_IP_ADDR} +``` + ## Teardown -The terraform state file stores all the info, so the nomad_sha doesn't need to be valid during teardown. +The terraform state file stores all the info, so the nomad_sha doesn't need to be valid during teardown. ``` $ cd e2e/terraform/ diff --git a/e2e/terraform/compute.tf b/e2e/terraform/compute.tf index 7f1c6ae8d..59ec0fdec 100644 --- a/e2e/terraform/compute.tf +++ b/e2e/terraform/compute.tf @@ -1,5 +1,5 @@ resource "aws_instance" "server" { - ami = data.aws_ami.main.image_id + ami = data.aws_ami.linux.image_id instance_type = var.instance_type key_name = module.keys.key_name vpc_security_group_ids = [aws_security_group.primary.id] @@ -44,7 +44,7 @@ resource "aws_instance" "server" { } resource "aws_instance" "client" { - ami = data.aws_ami.main.image_id + ami = data.aws_ami.linux.image_id instance_type = var.instance_type key_name = module.keys.key_name vpc_security_group_ids = [aws_security_group.primary.id] @@ -95,3 +95,71 @@ resource "aws_instance" "client" { } } } +data "template_file" "user_data_client_windows" { + template = file("${path.root}/shared/config/userdata-windows.ps1") + vars = { + nomad_sha = var.nomad_sha + } +} + +data "archive_file" "windows_configs" { + type = "zip" + source_dir = "./shared" + output_path = "./windows_configs.zip" +} + +resource "aws_instance" "client_windows" { + ami = data.aws_ami.windows.image_id + instance_type = var.instance_type + key_name = module.keys.key_name + vpc_security_group_ids = [aws_security_group.primary.id] + count = var.windows_client_count + depends_on = [aws_instance.server] + iam_instance_profile = "${aws_iam_instance_profile.instance_profile.name}" + + # Instance tags + tags = { + Name = "${local.random_name}-client-windows-${count.index}" + ConsulAutoJoin = "auto-join" + } + + ebs_block_device { + device_name = "xvdd" + volume_type = "gp2" + volume_size = "50" + delete_on_termination = "true" + } + + # We need this userdata script because Windows machines don't + # configure ssh with cloud-init by default. + user_data = data.template_file.user_data_client_windows.rendered + + # Note: + # we're copying up all the provisioning scripts and configs as + # a zipped bundle because TF's file provisioner dies in the middle + # of pushing up multiple files (whereas 'scp -r' works fine). + # + # We're also running all the provisioning scripts inside the + # userdata by polling for the zip file to show up. (Gross!) + # This is because remote-exec provisioners are failing on Windows + # with the same symptoms as: + # https://github.com/hashicorp/terraform/issues/17728 + # + # If we can't fix this, it'll prevent us from having multiple + # Windows clients running until TF supports count interpolation + # in the template_file, which is planned for a later 0.12 release + # + provisioner "file" { + source = "./windows_configs.zip" + destination = "C:/ops/windows_configs.zip" + + connection { + host = coalesce(self.public_ip, self.private_ip) + type = "ssh" + user = "Administrator" + private_key = module.keys.private_key_pem + timeout = "10m" + } + } + +} diff --git a/e2e/terraform/main.tf b/e2e/terraform/main.tf index 13c5838b5..941435b8d 100644 --- a/e2e/terraform/main.tf +++ b/e2e/terraform/main.tf @@ -28,6 +28,11 @@ variable "client_count" { default = "4" } +variable "windows_client_count" { + description = "The number of windows clients to provision." + default = "1" +} + variable "nomad_sha" { description = "The sha of Nomad to run" } @@ -39,6 +44,12 @@ provider "aws" { resource "random_pet" "e2e" { } +resource "random_password" "windows_admin_password" { + length = 20 + special = true + override_special = "_%@" +} + locals { random_name = "${var.name}-${random_pet.e2e.id}" } @@ -51,7 +62,7 @@ module "keys" { version = "v2.0.0" } -data "aws_ami" "main" { +data "aws_ami" "linux" { most_recent = true owners = ["self"] @@ -66,6 +77,21 @@ data "aws_ami" "main" { } } +data "aws_ami" "windows" { + most_recent = true + owners = ["self"] + + filter { + name = "name" + values = ["nomad-e2e-windows-2016*"] + } + + filter { + name = "tag:OS" + values = ["Windows2016"] + } +} + data "aws_caller_identity" "current" { } diff --git a/e2e/terraform/packer/README.md b/e2e/terraform/packer/README.md index bea87daa5..69b04d3a1 100644 --- a/e2e/terraform/packer/README.md +++ b/e2e/terraform/packer/README.md @@ -18,4 +18,47 @@ $ packer --version # build linux AMI $ packer build packer.json + +# build Windows AMI +$ packer build packer-windows.json ``` + +## Debugging Packer Builds + +You'll need the Windows administrator password in order to access Windows machines via `winrm` as Packer does. You can get this by enabling `-debug` on your Packer build. + +```sh +packer build -debug -on-error=abort packer-windows.json +... +==> amazon-ebs: Pausing after run of step 'StepRunSourceInstance'. Press enter to continue. +==> amazon-ebs: Waiting for auto-generated password for instance... + amazon-ebs: Password (since debug is enabled): +``` + +Alternately, you can follow the steps in the [AWS documentation](https://aws.amazon.com/premiumsupport/knowledge-center/retrieve-windows-admin-password/). Note that you'll need the `ec2_amazon-ebs.pem` file that Packer drops in this directory. + + +Then in powershell (note the leading `$` here indicate variable declarations, not shell prompts!): + +``` +$username = "Administrator" +$password = "" +$securePassword = ConvertTo-SecureString -AsPlainText -Force $password +$remoteHostname = "54.x.y.z" +$port = 5986 +$cred = New-Object System.Management.Automation.PSCredential ($username, $securePassword) +$so = New-PSSessionOption -SkipCACheck -SkipCNCheck + +Enter-PsSession ` + -ComputerName $remoteHostname ` + -Port $port ` + -Credential $cred ` + -UseSSL ` + -SessionOption $so ` + -Authentication Basic +``` + +Packer doesn't have a cleanup command if you've run `-on-error=abort`. So when you're done, clean up the machine by looking for "Packer" in the AWS console: +* [EC2 instances](https://console.aws.amazon.com/ec2/home?region=us-east-1#Instances:search=Packer;sort=tag:Name) +* [Key pairs](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#KeyPairs:search=packer;sort=keyName) +* [Security groups](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#SecurityGroups:search=packer;sort=groupName) diff --git a/e2e/terraform/packer/packer-windows.json b/e2e/terraform/packer/packer-windows.json new file mode 100644 index 000000000..52f2575e6 --- /dev/null +++ b/e2e/terraform/packer/packer-windows.json @@ -0,0 +1,69 @@ +{ + "builders": [ + { + "type": "amazon-ebs", + "region": "us-east-1", + "source_ami_filter": { + "filters": { + "virtualization-type": "hvm", + "name": "Windows_Server-2016-English-Full-Base-*", + "root-device-type": "ebs" + }, + "owners": [ + "amazon" + ], + "most_recent": true + }, + "instance_type": "t2.medium", + "ami_name": "nomad-e2e-windows-2016-{{timestamp}}", + "ami_groups": [ + "all" + ], + "communicator": "winrm", + "user_data_file": "windows/setupwinrm.ps1", + "winrm_username": "Administrator", + "winrm_insecure": true, + "winrm_use_ssl": true, + "tags": { + "OS": "Windows2016" + } + } + ], + "provisioners": [ + { + "type": "powershell", + "elevated_user": "Administrator", + "elevated_password": "{{.WinRMPassword}}", + "scripts": [ + "windows/disable-windows-updates.ps1", + "windows/fix-tls.ps1", + "windows/install-nuget.ps1", + "windows/install-tools.ps1", + "windows/install-docker.ps1", + "windows/setup-directories.ps1", + "windows/install-openssh.ps1" + ] + }, + { + "type": "windows-restart" + }, + { + "type": "powershell", + "elevated_user": "Administrator", + "elevated_password": "{{.WinRMPassword}}", + "scripts": [ + "windows/install-consul.ps1", + "windows/install-vault.ps1", + "windows/install-nomad.ps1" + ] + }, + { + "type": "powershell", + "inline": [ + "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\SendWindowsIsReady.ps1 -Schedule", + "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\InitializeInstance.ps1 -Schedule", + "C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\SysprepInstance.ps1 -NoShutdown" + ] + } + ] +} diff --git a/e2e/terraform/packer/windows/README.md b/e2e/terraform/packer/windows/README.md new file mode 100644 index 000000000..071f41227 --- /dev/null +++ b/e2e/terraform/packer/windows/README.md @@ -0,0 +1,20 @@ +# Windows Packer Build + +There are a few boilerplate items in the Powershell scripts, explained below. + +The default TLS protocol in the version of .NET that our Powershell cmdlets are built in it 1.0, which means plenty of properly configured HTTP servers will reject requests. The boilerplate snippet below sets this for the current script: + +``` +# Force TLS1.2 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +``` + +We need to run some of the scripts as an administrator role. The following is a safety check that we're doing so: + +``` +$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator") +if (!$RunningAsAdmin) { + Write-Error "Must be executed in Administrator level shell." + exit 1 +} +``` diff --git a/e2e/terraform/packer/windows/disable-windows-updates.ps1 b/e2e/terraform/packer/windows/disable-windows-updates.ps1 new file mode 100755 index 000000000..1b9f80660 --- /dev/null +++ b/e2e/terraform/packer/windows/disable-windows-updates.ps1 @@ -0,0 +1,30 @@ +$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator") +if (!$RunningAsAdmin) { + Write-Error "Must be executed in Administrator level shell." + exit 1 +} + +$service = Get-WmiObject Win32_Service -Filter 'Name="wuauserv"' + +if (!$service) { + Write-Error "Failed to retrieve the wauserv service" + exit 1 +} + +if ($service.StartMode -ne "Disabled") { + $result = $service.ChangeStartMode("Disabled").ReturnValue + if($result) { + Write-Error "Failed to disable the 'wuauserv' service. The return value was $result." + exit 1 + } +} + +if ($service.State -eq "Running") { + $result = $service.StopService().ReturnValue + if ($result) { + Write-Error "Failed to stop the 'wuauserv' service. The return value was $result." + exit 1 + } +} + +Write-Output "Automatic Windows Updates disabled." diff --git a/e2e/terraform/packer/windows/fix-tls.ps1 b/e2e/terraform/packer/windows/fix-tls.ps1 new file mode 100755 index 000000000..f17a6aea5 --- /dev/null +++ b/e2e/terraform/packer/windows/fix-tls.ps1 @@ -0,0 +1,151 @@ +# This script hardens TLS configuration by disabling weak and broken protocols +# and enabling useful protocols like TLS 1.1 and 1.2. + +$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator") +if (!$RunningAsAdmin) { + Write-Error "Must be executed in Administrator level shell." + exit 1 +} + +$weakProtocols = @( + 'Multi-Protocol Unified Hello', + 'PCT 1.0', + 'SSL 2.0', + 'SSL 3.0' +) + +$strongProtocols = @( + 'TLS 1.0', + 'TLS 1.1', + 'TLS 1.2' +) + +$weakCiphers = @( + 'DES 56/56', + 'NULL', + 'RC2 128/128', + 'RC2 40/128', + 'RC2 56/128', + 'RC4 40/128', + 'RC4 56/128', + 'RC4 64/128', + 'RC4 128/128' +) + +$strongCiphers = @( + 'AES 128/128', + 'AES 256/256', + 'Triple DES 168/168' +) + +$weakHashes = @( + 'MD5', + 'SHA' +) + +$strongHashes = @( + 'SHA 256', + 'SHA 384', + 'SHA 512' +) + +$strongKeyExchanges = @( + 'Diffie-Hellman', + 'ECDH', + 'PKCS' +) + +$cipherOrder = @( + 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384_P521', + 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384_P384', + 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384_P256', + 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA_P521', + 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA_P384', + 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA_P256', + 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256_P521', + 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256_P384', + 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256_P256', + 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA_P521', + 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA_P384', + 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA_P256', + 'TLS_RSA_WITH_AES_256_GCM_SHA384', + 'TLS_RSA_WITH_AES_128_GCM_SHA256', + 'TLS_RSA_WITH_AES_256_CBC_SHA256', + 'TLS_RSA_WITH_AES_256_CBC_SHA', + 'TLS_RSA_WITH_AES_128_CBC_SHA256', + 'TLS_RSA_WITH_AES_128_CBC_SHA', + 'TLS_RSA_WITH_3DES_EDE_CBC_SHA' +) + +# Reset the protocols key +New-Item 'HKLM:SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols' -Force | Out-Null + +# Disable weak protocols +Foreach ($protocol in $weakProtocols) { + New-Item HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -Force | Out-Null + New-Item HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -Force | Out-Null + New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -name Enabled -value 0 -PropertyType 'DWord' -Force | Out-Null + New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -name DisabledByDefault -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null + New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -name Enabled -value 0 -PropertyType 'DWord' -Force | Out-Null + New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -name DisabledByDefault -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null +} + +# Enable strong protocols +Foreach ($protocol in $strongProtocols) { + New-Item HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -Force | Out-Null + New-Item HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -Force | Out-Null + New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -name 'Enabled' -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null + New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -name 'DisabledByDefault' -value 0 -PropertyType 'DWord' -Force | Out-Null + New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -name 'Enabled' -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null + New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -name 'DisabledByDefault' -value 0 -PropertyType 'DWord' -Force | Out-Null +} + +# Reset the ciphers key +New-Item 'HKLM:SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Ciphers' -Force | Out-Null + +# Disable Weak Ciphers +Foreach ($cipher in $weakCiphers) { + $key = (get-item HKLM:\).OpenSubKey("SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Ciphers", $true).CreateSubKey($cipher) + $key.SetValue('Enabled', 0, 'DWord') + $key.Close() +} + +# Enable Strong Ciphers +Foreach ($cipher in $strongCiphers) { + $key = (get-item HKLM:\).OpenSubKey("SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Ciphers", $true).CreateSubKey($cipher) + New-ItemProperty -path "HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Ciphers\$cipher" -name 'Enabled' -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null + $key.Close() +} + +# Reset the hashes key +New-Item 'HKLM:SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Hashes' -Force | Out-Null + +# Disable weak hashes +Foreach ($hash in $weakHashes) { + $key = (get-item HKLM:\).OpenSubKey("SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Hashes", $true).CreateSubKey($hash) + New-ItemProperty -path "HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Hashes\$hash" -name 'Enabled' -value '0' -PropertyType 'DWord' -Force | Out-Null + $key.Close() +} + +# Enable Hashes +Foreach ($hash in $strongHashes) { + $key = (get-item HKLM:\).OpenSubKey("SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Hashes", $true).CreateSubKey($hash) + New-ItemProperty -path "HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Hashes\$hash" -name 'Enabled' -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null + $key.Close() +} + +# Reset the KeyExchangeAlgorithms key +New-Item 'HKLM:SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\KeyExchangeAlgorithms' -Force | Out-Null + +# Enable KeyExchangeAlgorithms +Foreach ($keyExchange in $strongKeyExchanges) { + $key = (get-item HKLM:\).OpenSubKey("SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\KeyExchangeAlgorithms", $true).CreateSubKey($keyExchange) + New-ItemProperty -path "HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\KeyExchangeAlgorithms\$keyExchange" -name 'Enabled' -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null + $key.Close() +} + +# Set cipher order +$cipherOrderString = [string]::join(',', $cipherOrder) +New-ItemProperty -path 'HKLM:\SOFTWARE\Policies\Microsoft\Cryptography\Configuration\SSL\00010002' -name 'Functions' -value $cipherOrderString -PropertyType 'String' -Force | Out-Null + +Write-Output "TLS hardened." diff --git a/e2e/terraform/packer/windows/install-consul.ps1 b/e2e/terraform/packer/windows/install-consul.ps1 new file mode 100755 index 000000000..d2359a9c3 --- /dev/null +++ b/e2e/terraform/packer/windows/install-consul.ps1 @@ -0,0 +1,32 @@ +Set-StrictMode -Version latest +$ErrorActionPreference = "Stop" + +# Force TLS1.2 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +Set-Location C:\opt + +Try { + $releases = "https://releases.hashicorp.com" + $version = "1.6.1" + $url = "${releases}/consul/${version}/consul_${version}_windows_amd64.zip" + + $configDir = "C:\opt\consul.d" + md $configDir + md C:\opt\consul + + # TODO: check sha! + Write-Output "Downloading Consul from: $url" + Invoke-WebRequest -Uri $url -Outfile consul.zip + Expand-Archive .\consul.zip .\ + mv consul.exe C:\opt\consul.exe + C:\opt\consul.exe version + rm consul.zip + +} Catch { + Write-Error "Failed to install Consul." + $host.SetShouldExit(-1) + throw +} + +Write-Output "Installed Consul." diff --git a/e2e/terraform/packer/windows/install-docker.ps1 b/e2e/terraform/packer/windows/install-docker.ps1 new file mode 100755 index 000000000..6dbe21f99 --- /dev/null +++ b/e2e/terraform/packer/windows/install-docker.ps1 @@ -0,0 +1,35 @@ +Set-StrictMode -Version latest +$ErrorActionPreference = "Stop" + +$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator") +if (!$RunningAsAdmin) { + Write-Error "Must be executed in Administrator level shell." + exit 1 +} + +# Force TLS1.2 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +Try { + Write-Output "Installing containers feature." + Install-WindowsFeature -Name Containers + + Write-Output "Creating user for Docker." + net localgroup docker /add + net localgroup docker $env:USERNAME /add + + Write-Output "Installing Docker." + Set-PSRepository -InstallationPolicy Trusted -Name PSGallery + Install-Module -Name DockerMsftProvider -Repository PSGallery -Force + Install-Package -Name docker -ProviderName DockerMsftProvider -Force + +} Catch { + Write-Error "Failed to install Docker." + $host.SetShouldExit(-1) + throw +} Finally { + # clean up by re-securing this package repo + Set-PSRepository -InstallationPolicy Untrusted -Name PSGallery +} + +Write-Output "Installed Docker." diff --git a/e2e/terraform/packer/windows/install-nomad.ps1 b/e2e/terraform/packer/windows/install-nomad.ps1 new file mode 100755 index 000000000..300197319 --- /dev/null +++ b/e2e/terraform/packer/windows/install-nomad.ps1 @@ -0,0 +1,34 @@ +Set-StrictMode -Version latest +$ErrorActionPreference = "Stop" + +# Force TLS1.2 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +Set-Location C:\opt + +Try { + # we install the most recent stable/GA release; this will be replaced + # with the current master when we run e2e tests + $releases = "https://releases.hashicorp.com" + $version = "0.9.6" + $url = "${releases}/nomad/${version}/nomad_${version}_windows_amd64.zip" + + $configDir = "C:\opt\nomad.d" + md $configDir + md C:\opt\nomad + + # TODO: check sha! + Write-Output "Downloading Nomad from: $url" + Invoke-WebRequest -Uri $url -Outfile nomad.zip + Expand-Archive .\nomad.zip .\ + mv nomad.exe C:\opt\nomad.exe + C:\opt\nomad.exe version + rm nomad.zip + +} Catch { + Write-Error "Failed to install Nomad." + $host.SetShouldExit(-1) + throw +} + +Write-Output "Installed Nomad." diff --git a/e2e/terraform/packer/windows/install-nuget.ps1 b/e2e/terraform/packer/windows/install-nuget.ps1 new file mode 100755 index 000000000..e1f38da6d --- /dev/null +++ b/e2e/terraform/packer/windows/install-nuget.ps1 @@ -0,0 +1,21 @@ +Set-StrictMode -Version latest +$ErrorActionPreference = "Stop" + +$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator") +if (!$RunningAsAdmin) { + Write-Error "Must be executed in Administrator level shell." + exit 1 +} + +# Force TLS1.2 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +Try { + Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force +} Catch { + Write-Error "Failed to install NuGet package manager." + $host.SetShouldExit(-1) + throw +} + +Write-Output "Installed NuGet." diff --git a/e2e/terraform/packer/windows/install-openssh.ps1 b/e2e/terraform/packer/windows/install-openssh.ps1 new file mode 100755 index 000000000..767b570ef --- /dev/null +++ b/e2e/terraform/packer/windows/install-openssh.ps1 @@ -0,0 +1,53 @@ +Set-StrictMode -Version latest +$ErrorActionPreference = "Stop" + +$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator") +if (!$RunningAsAdmin) { + Write-Error "Must be executed in Administrator level shell." + exit 1 +} + +# Force TLS1.2 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +Try { + + # install portable SSH instead of the Windows feature because we + # need to target 2016 + $repo = "https://github.com/PowerShell/Win32-OpenSSH" + $version = "v8.0.0.0p1-Beta" + $url = "${repo}/releases/download/${version}/OpenSSH-Win64.zip" + + # TODO: check sha! + Write-Output "Downloading OpenSSH from: $url" + Invoke-WebRequest -Uri $url -Outfile "OpenSSH-Win64.zip" + Expand-Archive ".\OpenSSH-Win64.zip" "C:\Program Files" + Rename-Item -Path "C:\Program Files\OpenSSH-Win64" -NewName "OpenSSH" + + & "C:\Program Files\OpenSSH\install-sshd.ps1" + + # Start the service + Start-Service sshd + Set-Service -Name sshd -StartupType 'Automatic' + + Start-Service ssh-agent + Set-Service -Name ssh-agent -StartupType 'Automatic' + + # Enable host firewall rule if it doesn't exist + New-NetFirewallRule -Name sshd -DisplayName 'OpenSSH Server (sshd)' ` + -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 + + # Set powershell as the OpenSSH login shell + New-ItemProperty -Path "HKLM:\SOFTWARE\OpenSSH" ` + -Name DefaultShell ` + -Value "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe" ` + -PropertyType String -Force + + +} Catch { + Write-Error "Failed to install OpenSSH." + $host.SetShouldExit(-1) + throw +} + +Write-Output "Installed OpenSSH." diff --git a/e2e/terraform/packer/windows/install-tools.ps1 b/e2e/terraform/packer/windows/install-tools.ps1 new file mode 100755 index 000000000..b4535d1d0 --- /dev/null +++ b/e2e/terraform/packer/windows/install-tools.ps1 @@ -0,0 +1,37 @@ +Set-StrictMode -Version latest +$ErrorActionPreference = "Stop" + +$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator") +if (!$RunningAsAdmin) { + Write-Error "Must be executed in Administrator level shell." + exit 1 +} + +# Force TLS1.2 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +# TODO (tgross: some stuff installed on Linux but not here yet +# - Possible issues: no redis-tools for windows +# - Possible non-issues: probably don't need tree, curl,tmux + +Try { + Set-PSRepository -InstallationPolicy Trusted -Name PSGallery + + Write-Output "Installing 7Zip" + Install-Package -Force 7Zip4PowerShell + + Write-Output "Installing JQ" + Invoke-WebRequest ` + -Uri https://github.com/stedolan/jq/releases/download/jq-1.6/jq-win64.exe ` + -Outfile jq-win64.exe + +} Catch { + Write-Error "Failed to install dependencies." + $host.SetShouldExit(-1) + throw +} Finally { + # clean up by re-securing this package repo + Set-PSRepository -InstallationPolicy Untrusted -Name PSGallery +} + +Write-Output "Installed dependencies" diff --git a/e2e/terraform/packer/windows/install-vault.ps1 b/e2e/terraform/packer/windows/install-vault.ps1 new file mode 100755 index 000000000..82498ea08 --- /dev/null +++ b/e2e/terraform/packer/windows/install-vault.ps1 @@ -0,0 +1,31 @@ +Set-StrictMode -Version latest +$ErrorActionPreference = "Stop" + +# Force TLS1.2 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +Set-Location C:\opt + +Try { + $releases = "https://releases.hashicorp.com" + $version = "1.2.3" + $url = "${releases}/vault/${version}/vault_${version}_windows_amd64.zip" + + $configDir = "C:\opt\vault.d" + md $configDir + + # TODO: check sha! + Write-Output "Downloading Vault from: $url" + Invoke-WebRequest -Uri $url -Outfile vault.zip + Expand-Archive .\vault.zip .\ + mv vault.exe C:\opt\vault.exe + C:\opt\vault.exe version + rm vault.zip + +} Catch { + Write-Error "Failed to install Vault." + $host.SetShouldExit(-1) + throw +} + +Write-Output "Installed Vault." diff --git a/e2e/terraform/packer/windows/setup-directories.ps1 b/e2e/terraform/packer/windows/setup-directories.ps1 new file mode 100755 index 000000000..3cb3ad806 --- /dev/null +++ b/e2e/terraform/packer/windows/setup-directories.ps1 @@ -0,0 +1,2 @@ +md C:\ops +md C:\opt diff --git a/e2e/terraform/packer/windows/setupwinrm.ps1 b/e2e/terraform/packer/windows/setupwinrm.ps1 new file mode 100755 index 000000000..030eb6f1c --- /dev/null +++ b/e2e/terraform/packer/windows/setupwinrm.ps1 @@ -0,0 +1,44 @@ + + +Write-Output "Running User Data Script" +Write-Host "(host) Running User Data Script" + +Set-ExecutionPolicy Unrestricted -Scope LocalMachine -Force -ErrorAction Ignore + +# Don't set this before Set-ExecutionPolicy as it throws an error +$ErrorActionPreference = "stop" + +# Remove HTTP listener +Remove-Item -Path WSMan:\Localhost\listener\listener* -Recurse + +$Cert = New-SelfSignedCertificate ` + -CertstoreLocation Cert:\LocalMachine\My ` + -DnsName "packer" + +New-Item ` + -Path WSMan:\LocalHost\Listener ` + -Transport HTTPS ` + -Address * ` + -CertificateThumbPrint $Cert.Thumbprint ` + -Force + +# WinRM +write-output "Setting up WinRM" +write-host "(host) setting up WinRM" + +cmd.exe /c winrm quickconfig -q +cmd.exe /c winrm set "winrm/config" '@{MaxTimeoutms="1800000"}' +cmd.exe /c winrm set "winrm/config/winrs" '@{MaxMemoryPerShellMB="1024"}' +cmd.exe /c winrm set "winrm/config/service" '@{AllowUnencrypted="true"}' +cmd.exe /c winrm set "winrm/config/client" '@{AllowUnencrypted="true"}' +cmd.exe /c winrm set "winrm/config/service/auth" '@{Basic="true"}' +cmd.exe /c winrm set "winrm/config/client/auth" '@{Basic="true"}' +cmd.exe /c winrm set "winrm/config/service/auth" '@{CredSSP="true"}' +cmd.exe /c winrm set "winrm/config/listener?Address=*+Transport=HTTPS" "@{Port=`"5986`";Hostname=`"packer`";CertificateThumbprint=`"$($Cert.Thumbprint)`"}" +cmd.exe /c netsh advfirewall firewall set rule group="remote administration" new enable=yes +cmd.exe /c netsh firewall add portopening TCP 5986 "Port 5986" +cmd.exe /c net stop winrm +cmd.exe /c sc config winrm start= auto +cmd.exe /c net start winrm + + diff --git a/e2e/terraform/shared/config/provision-client.sh b/e2e/terraform/shared/config/provision-client.sh old mode 100644 new mode 100755 diff --git a/e2e/terraform/shared/config/provision-server.sh b/e2e/terraform/shared/config/provision-server.sh old mode 100644 new mode 100755 diff --git a/e2e/terraform/shared/config/provision-windows-client.ps1 b/e2e/terraform/shared/config/provision-windows-client.ps1 new file mode 100755 index 000000000..7eaf29b28 --- /dev/null +++ b/e2e/terraform/shared/config/provision-windows-client.ps1 @@ -0,0 +1,49 @@ +param( + [string]$Cloud = "aws", + [string]$NomadSha = "", + [string]$Index=0 +) + +# Force TLS1.2 +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +# Consul +cp "C:\ops\shared\consul\base.json" "C:\opt\consul.d\base.json" +cp "C:\ops\shared\consul\retry_$Cloud.json" "C:\opt\consul.d\retry_$Cloud.json" +sc.exe create "Consul" binPath= "C:\opt\consul.exe agent -config-dir C:\opt\consul.d -log-file C:\opt\consul\consul.log" start= auto +sc.exe start "Consul" + +# Vault +# TODO(tgross): we don't need Vault for clients +# cp "C:\ops\shared\vault\vault.hcl" C:\opt\vault.d\vault.hcl +# sc.exe create "Vault" binPath= "C:\opt\vault.exe" agent -config-dir "C:\opt\vault.d" start= auto + +# Nomad + +md C:\opt\nomad + +Read-S3Object ` + -BucketName nomad-team-test-binary ` + -Key "builds-oss/nomad_windows_amd64_$NomadSha.zip" ` + -File .\nomad.zip + +Expand-Archive .\nomad.zip .\ +rm C:\opt\nomad.exe +mv nomad.exe C:\opt\nomad.exe + +# install config file +cp "C:\ops\shared\nomad\client-windows.hcl" "C:\opt\nomad.d\nomad.hcl" + +# Setup Host Volumes +md C:\tmp\data + +# TODO(tgross): not sure we even support this for Windows? +# Write-Output "Install CNI" +# md C:\opt\cni\bin +# $cni_url = "https://github.com/containernetworking/plugins/releases/download/v0.8.2/cni-plugins-windows-amd64-v0.8.2.tgz" +# Invoke-WebRequest -Uri "$cni_url" -Outfile cni.tgz +# Expand-7Zip -ArchiveFileName .\cni.tgz -TargetPath C:\opt\cni\bin\ + +# enable as a service +sc.exe create "Nomad" binPath= "C:\opt\nomad.exe agent -config C:\opt\nomad.d" start= auto +sc.exe start "Nomad" diff --git a/e2e/terraform/shared/config/userdata-windows.ps1 b/e2e/terraform/shared/config/userdata-windows.ps1 new file mode 100755 index 000000000..c92039bd2 --- /dev/null +++ b/e2e/terraform/shared/config/userdata-windows.ps1 @@ -0,0 +1,30 @@ + + +# Bring ebs volume online with read-write access +Get-Disk | Where-Object IsOffline -Eq $True | Set-Disk -IsOffline $False +Get-Disk | Where-Object isReadOnly -Eq $True | Set-Disk -IsReadOnly $False + +md "C:\Users\Administrator\.ssh\" + +$myKey = "C:\Users\Administrator\.ssh\authorized_keys" +$adminKey = "C:\ProgramData\ssh\administrators_authorized_keys" + +Invoke-RestMethod ` + -Uri "http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key" ` + -Outfile $myKey + +cp $myKey $adminKey + +icacls $adminKey /reset +icacls $adminKey /inheritance:r +icacls $adminKey /grant BUILTIN\Administrators:`(F`) +icacls $adminKey /grant SYSTEM:`(F`) + +$archiveFile = "C:\ops\windows_configs.zip" +while (!(Test-Path $archiveFile)) { Start-Sleep 10 } + +Expand-Archive $archiveFile "C:\ops\shared" + +& C:\ops\shared\config\provision-windows-client.ps1 -Cloud aws -NomadSha ${nomad_sha} -Index 1 + + diff --git a/e2e/terraform/shared/nomad/client-windows.hcl b/e2e/terraform/shared/nomad/client-windows.hcl new file mode 100644 index 000000000..189a340ba --- /dev/null +++ b/e2e/terraform/shared/nomad/client-windows.hcl @@ -0,0 +1,35 @@ +enable_debug = true + +log_level = "debug" +log_file = true + +data_dir = "C:\\opt\\nomad\\data" + +bind_addr = "0.0.0.0" + +# Enable the client +client { + enabled = true + + options { + # Allow rawexec jobs + "driver.raw_exec.enable" = "1" + } +} + +consul { + address = "127.0.0.1:8500" +} + +vault { + enabled = true + address = "http://active.vault.service.consul:8200" +} + +telemetry { + collection_interval = "1s" + disable_hostname = true + prometheus_metrics = true + publish_allocation_metrics = true + publish_node_metrics = true +} diff --git a/e2e/terraform/terraform.tfvars b/e2e/terraform/terraform.tfvars index b38ab8855..1160e9d7a 100644 --- a/e2e/terraform/terraform.tfvars +++ b/e2e/terraform/terraform.tfvars @@ -1,4 +1,7 @@ -region = "us-east-1" -instance_type = "t2.medium" -server_count = "3" -client_count = "4" +region = "us-east-1" +instance_type = "t2.medium" +server_count = "3" +client_count = "4" + +# TODO(tgross): add only once Windows client is working +windows_client_count = "0"