E2E: include Windows 2022 host in test targets (#26003)

Some time ago the Windows host we were using as a Nomad client agent test target
started failing to allow ssh connections. The underlying problem appears to be
with sysprep but I wasn't able to debug the exact cause as it's not an area I
have a lot of expertise in.

Swap out the deprecated Windows 2016 host for a Windows 2022 host. This will use
a base image provided by Amazon and then we'll use a userdata script to
bootstrap ssh and some target directories for Terraform to upload files to. The
more modern Windows will let us drop some of extra powershell scripts we were
using as well.

Fixes: https://hashicorp.atlassian.net/browse/NMD-151
Fixes: https://github.com/hashicorp/nomad-e2e/issues/125
This commit is contained in:
Tim Gross
2025-06-16 12:12:15 -04:00
committed by GitHub
parent 26004c5407
commit d6800c41c1
23 changed files with 232 additions and 606 deletions

View File

@@ -46,8 +46,6 @@ func artifactCheckLogContents(t *testing.T, nomad *api.Client, group, task strin
}
func testWindows(t *testing.T) {
t.Skip("SKIP WINDOWS TEST") // TODO restore when windows client is fixed
nomad := e2eutil.NomadClient(t)
jobID := "artifact-windows-" + uuid.Short()
jobIDs := []string{jobID}

View File

@@ -73,6 +73,10 @@ func TestMetrics(t *testing.T) {
_, cleanupCaddy := jobs3.Submit(t, "./input/caddy.hcl")
t.Cleanup(cleanupCaddy)
t.Log("running metrics job winagent ...")
jobWin, cleanupWin := jobs3.Submit(t, "./input/winagent.hcl")
t.Cleanup(cleanupWin)
t.Log("let the metrics collect for a bit (10s) ...")
time.Sleep(10 * time.Second)
@@ -89,7 +93,12 @@ func TestMetrics(t *testing.T) {
name: "nomad_client_allocs_cpu_allocated",
filter: "exported_job",
key: jobPy.JobID(),
}})
}, {
name: "nomad_client_allocs_memory_rss",
filter: "exported_job",
key: jobWin.JobID(),
},
})
t.Log("measuring client metrics ...")
testClientMetrics(t, []*metric{{

View File

@@ -8,7 +8,7 @@ custom.tfvars:
echo 'nomad_local_binary = "$(PKG_PATH)"' > custom.tfvars
echo 'volumes = false' >> custom.tfvars
echo 'client_count_linux = 3' >> custom.tfvars
echo 'client_count_windows_2016 = 0' >> custom.tfvars
echo 'client_count_windows_2022 = 0' >> custom.tfvars
echo 'consul_license = "$(shell cat $(CONSUL_LICENSE_PATH))"' >> custom.tfvars
echo 'nomad_license = "$(shell cat $(NOMAD_LICENSE_PATH))"' >> custom.tfvars

View File

@@ -53,7 +53,7 @@ region = "us-east-1"
instance_type = "t2.medium"
server_count = "3"
client_count_linux = "4"
client_count_windows_2016 = "1"
client_count_windows_2022 = "1"
```
You will also need a Consul Enterprise license file and a Nomad Enterprise
@@ -67,7 +67,7 @@ linux).
NOTE: If you want to have a cluster with mixed CPU architectures,
you need to specify the count and also provide the corresponding
binary using `var.nomad_local_binary_client_ubuntu_jammy` and or
`var.nomad_local_binary_client_windows_2016`.
`var.nomad_local_binary_client_windows_2022`.
Run Terraform apply to deploy the infrastructure:

View File

@@ -10,11 +10,11 @@ module "provision-infra" {
server_count = var.server_count
client_count_linux = var.client_count_linux
client_count_windows_2016 = var.client_count_windows_2016
client_count_windows_2022 = var.client_count_windows_2022
nomad_local_binary_server = var.nomad_local_binary_server
nomad_local_binary = var.nomad_local_binary
nomad_local_binary_client_ubuntu_jammy = var.nomad_local_binary_client_ubuntu_jammy
nomad_local_binary_client_windows_2016 = var.nomad_local_binary_client_windows_2016
nomad_local_binary_client_windows_2022 = var.nomad_local_binary_client_windows_2022
nomad_license = var.nomad_license
consul_license = var.consul_license
nomad_region = var.nomad_region

View File

@@ -34,9 +34,6 @@ $ packer --version
# build Ubuntu Jammy AMI
$ ./build ubuntu-jammy-amd64
# build Windows AMI
$ ./build windows-2016-amd64
```
## Debugging Packer Builds
@@ -51,3 +48,7 @@ you're done, clean up the machine by looking for "Packer" in the AWS console:
* [EC2 instances](https://console.aws.amazon.com/ec2/home?region=us-east-1#Instances:search=Packer;sort=tag:Name)
* [Key pairs](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#KeyPairs:search=packer;sort=keyName)
* [Security groups](https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#SecurityGroups:search=packer;sort=groupName)
## Q: What About Windows?
For now, we're using an Amazon base image directly.

View File

@@ -1,64 +0,0 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
variable "build_sha" {
type = string
description = "the revision of the packer scripts building this image"
}
locals {
timestamp = regex_replace(timestamp(), "[- TZ:]", "")
version = "v3"
}
source "amazon-ebs" "latest_windows_2016" {
ami_name = "nomad-e2e-${local.version}-windows-2016-amd64-${local.timestamp}"
communicator = "ssh"
instance_type = "m7a.large"
region = "us-east-1"
user_data_file = "windows-2016-amd64/userdata.ps1" # enables ssh
ssh_timeout = "10m"
ssh_username = "Administrator"
source_ami_filter {
filters = {
name = "Windows_Server-2016-English-Full-ECS_Optimized-*"
root-device-type = "ebs"
virtualization-type = "hvm"
}
most_recent = true
owners = ["amazon"]
}
tags = {
OS = "Windows2016"
BuilderSha = var.build_sha
}
}
build {
sources = ["source.amazon-ebs.latest_windows_2016"]
provisioner "powershell" {
scripts = [
"windows-2016-amd64/disable-windows-updates.ps1",
"windows-2016-amd64/fix-tls.ps1",
"windows-2016-amd64/install-nuget.ps1",
"windows-2016-amd64/install-consul.ps1",
"windows-2016-amd64/install-nomad.ps1"
]
}
# this restart is required for adding the "containers feature", but we can
# wait to do it until right before we do sysprep, which makes debugging
# builds slightly faster
provisioner "windows-restart" {}
provisioner "powershell" {
inline = [
"C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\SendWindowsIsReady.ps1 -Schedule",
"C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\InitializeInstance.ps1 -Schedule",
"C:\\ProgramData\\Amazon\\EC2-Windows\\Launch\\Scripts\\SysprepInstance.ps1 -NoShutdown"
]
}
}

View File

@@ -1,20 +0,0 @@
# Windows Packer Build
There are a few boilerplate items in the Powershell scripts, explained below.
The default TLS protocol in the version of .NET that our Powershell cmdlets are built in it 1.0, which means plenty of properly configured HTTP servers will reject requests. The boilerplate snippet below sets this for the current script:
```
# Force TLS1.2
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
```
We need to run some of the scripts as an administrator role. The following is a safety check that we're doing so:
```
$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")
if (!$RunningAsAdmin) {
Write-Error "Must be executed in Administrator level shell."
exit 1
}
```

View File

@@ -1,33 +0,0 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")
if (!$RunningAsAdmin) {
Write-Error "Must be executed in Administrator level shell."
exit 1
}
$service = Get-WmiObject Win32_Service -Filter 'Name="wuauserv"'
if (!$service) {
Write-Error "Failed to retrieve the wauserv service"
exit 1
}
if ($service.StartMode -ne "Disabled") {
$result = $service.ChangeStartMode("Disabled").ReturnValue
if($result) {
Write-Error "Failed to disable the 'wuauserv' service. The return value was $result."
exit 1
}
}
if ($service.State -eq "Running") {
$result = $service.StopService().ReturnValue
if ($result) {
Write-Error "Failed to stop the 'wuauserv' service. The return value was $result."
exit 1
}
}
Write-Output "Automatic Windows Updates disabled."

View File

@@ -1,150 +0,0 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
# This script hardens TLS configuration by disabling weak and broken protocols
# and enabling useful protocols like TLS 1.1 and 1.2.
$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")
if (!$RunningAsAdmin) {
Write-Error "Must be executed in Administrator level shell."
exit 1
}
$weakProtocols = @(
'Multi-Protocol Unified Hello',
'PCT 1.0',
'SSL 2.0',
'SSL 3.0'
)
$strongProtocols = @(
'TLS 1.0',
'TLS 1.1',
'TLS 1.2'
)
$weakCiphers = @(
'DES 56/56',
'NULL',
'RC2 128/128',
'RC2 40/128',
'RC2 56/128',
'RC4 40/128',
'RC4 56/128',
'RC4 64/128',
'RC4 128/128'
)
$strongCiphers = @(
'AES 128/128',
'AES 256/256',
'Triple DES 168/168'
)
$weakHashes = @(
'MD5',
'SHA'
)
$strongHashes = @(
'SHA 256',
'SHA 384',
'SHA 512'
)
$strongKeyExchanges = @(
'Diffie-Hellman',
'ECDH',
'PKCS'
)
$cipherOrder = @(
'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384_P521',
'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384_P384',
'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384_P256',
'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA_P521',
'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA_P384',
'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA_P256',
'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA_P521',
'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA_P384',
'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA_P256',
'TLS_RSA_WITH_AES_256_GCM_SHA384',
'TLS_RSA_WITH_AES_128_GCM_SHA256',
'TLS_RSA_WITH_AES_256_CBC_SHA256',
'TLS_RSA_WITH_AES_256_CBC_SHA',
'TLS_RSA_WITH_AES_128_CBC_SHA',
'TLS_RSA_WITH_3DES_EDE_CBC_SHA'
)
# Reset the protocols key
New-Item 'HKLM:SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols' -Force | Out-Null
# Disable weak protocols
Foreach ($protocol in $weakProtocols) {
New-Item HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -Force | Out-Null
New-Item HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -Force | Out-Null
New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -name Enabled -value 0 -PropertyType 'DWord' -Force | Out-Null
New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -name DisabledByDefault -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null
New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -name Enabled -value 0 -PropertyType 'DWord' -Force | Out-Null
New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -name DisabledByDefault -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null
}
# Enable strong protocols
Foreach ($protocol in $strongProtocols) {
New-Item HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -Force | Out-Null
New-Item HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -Force | Out-Null
New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -name 'Enabled' -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null
New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Server -name 'DisabledByDefault' -value 0 -PropertyType 'DWord' -Force | Out-Null
New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -name 'Enabled' -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null
New-ItemProperty -path HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Protocols\$protocol\Client -name 'DisabledByDefault' -value 0 -PropertyType 'DWord' -Force | Out-Null
}
# Reset the ciphers key
New-Item 'HKLM:SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Ciphers' -Force | Out-Null
# Disable Weak Ciphers
Foreach ($cipher in $weakCiphers) {
$key = (get-item HKLM:\).OpenSubKey("SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Ciphers", $true).CreateSubKey($cipher)
$key.SetValue('Enabled', 0, 'DWord')
$key.Close()
}
# Enable Strong Ciphers
Foreach ($cipher in $strongCiphers) {
$key = (get-item HKLM:\).OpenSubKey("SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Ciphers", $true).CreateSubKey($cipher)
New-ItemProperty -path "HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Ciphers\$cipher" -name 'Enabled' -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null
$key.Close()
}
# Reset the hashes key
New-Item 'HKLM:SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Hashes' -Force | Out-Null
# Disable weak hashes
Foreach ($hash in $weakHashes) {
$key = (get-item HKLM:\).OpenSubKey("SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Hashes", $true).CreateSubKey($hash)
New-ItemProperty -path "HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Hashes\$hash" -name 'Enabled' -value '0' -PropertyType 'DWord' -Force | Out-Null
$key.Close()
}
# Enable Hashes
Foreach ($hash in $strongHashes) {
$key = (get-item HKLM:\).OpenSubKey("SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Hashes", $true).CreateSubKey($hash)
New-ItemProperty -path "HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\Hashes\$hash" -name 'Enabled' -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null
$key.Close()
}
# Reset the KeyExchangeAlgorithms key
New-Item 'HKLM:SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\KeyExchangeAlgorithms' -Force | Out-Null
# Enable KeyExchangeAlgorithms
Foreach ($keyExchange in $strongKeyExchanges) {
$key = (get-item HKLM:\).OpenSubKey("SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\KeyExchangeAlgorithms", $true).CreateSubKey($keyExchange)
New-ItemProperty -path "HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\KeyExchangeAlgorithms\$keyExchange" -name 'Enabled' -value '0xffffffff' -PropertyType 'DWord' -Force | Out-Null
$key.Close()
}
# Set cipher order
$cipherOrderString = [string]::join(',', $cipherOrder)
New-ItemProperty -path 'HKLM:\SOFTWARE\Policies\Microsoft\Cryptography\Configuration\SSL\00010002' -name 'Functions' -value $cipherOrderString -PropertyType 'String' -Force | Out-Null
Write-Output "TLS hardened."

View File

@@ -1,41 +0,0 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
Set-StrictMode -Version latest
$ErrorActionPreference = "Stop"
# Force TLS1.2
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
Set-Location C:\opt
Try {
$releases = "https://releases.hashicorp.com"
$version = "1.11.4+ent"
$url = "${releases}/consul/${version}/consul_${version}_windows_amd64.zip"
New-Item -ItemType Directory -Force -Path C:\opt\consul
New-Item -ItemType Directory -Force -Path C:\etc\consul.d
# TODO: check sha!
Write-Output "Downloading Consul from: $url"
Invoke-WebRequest -Uri $url -Outfile consul.zip -ErrorAction Stop
Expand-Archive .\consul.zip .\ -ErrorAction Stop
Move-Item consul.exe C:\opt\consul.exe -Force -ErrorAction Stop
C:\opt\consul.exe version
rm consul.zip
New-Service `
-Name "Consul" `
-BinaryPathName "C:\opt\consul.exe agent -config-dir C:\etc\consul.d" `
-StartupType "Automatic" `
-ErrorAction Ignore
} Catch {
Write-Output "Failed to install Consul."
Write-Output $_
$host.SetShouldExit(-1)
throw
}
Write-Output "Installed Consul."

View File

@@ -1,49 +0,0 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
Set-StrictMode -Version latest
$ErrorActionPreference = "Stop"
# Force TLS1.2
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
Set-Location C:\opt
Try {
$releases = "https://releases.hashicorp.com"
$version = "1.2.6"
$url = "${releases}/nomad/${version}/nomad_${version}_windows_amd64.zip"
New-Item -ItemType Directory -Force -Path C:\opt\nomad
New-Item -ItemType Directory -Force -Path C:\etc\nomad.d
# TODO: check sha!
Write-Output "Downloading Nomad from: $url"
Invoke-WebRequest -Uri $url -Outfile nomad.zip -ErrorAction Stop
Expand-Archive .\nomad.zip .\ -ErrorAction Stop
Move-Item nomad.exe C:\opt\nomad.exe -Force -ErrorAction Stop
C:\opt\nomad.exe version
rm nomad.zip
New-NetFirewallRule `
-DisplayName 'Nomad HTTP Inbound' `
-Profile @('Public', 'Domain', 'Private') `
-Direction Inbound `
-Action Allow `
-Protocol TCP `
-LocalPort @('4646')
New-Service `
-Name "Nomad" `
-BinaryPathName "C:\opt\nomad.exe agent -config C:\etc\nomad.d" `
-StartupType "Automatic" `
-ErrorAction Ignore
} Catch {
Write-Output "Failed to install Nomad."
Write-Output $_
$host.SetShouldExit(-1)
throw
}
Write-Output "Installed Nomad."

View File

@@ -1,25 +0,0 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
Set-StrictMode -Version latest
$ErrorActionPreference = "Stop"
$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")
if (!$RunningAsAdmin) {
Write-Error "Must be executed in Administrator level shell."
exit 1
}
# Force TLS1.2
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
Try {
Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force -ErrorAction Stop
} Catch {
Write-Output "Failed to install NuGet package manager."
Write-Output $_
$host.SetShouldExit(-1)
throw
}
Write-Output "Installed NuGet."

View File

@@ -1,137 +0,0 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
<powershell>
Set-StrictMode -Version latest
$ErrorActionPreference = "Stop"
$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")
if (!$RunningAsAdmin) {
Write-Error "Must be executed in Administrator level shell."
exit 1
}
# Force TLS1.2
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
Write-Output "Running User Data Script"
Write-Host "(host) Running User Data Script"
Set-ExecutionPolicy Unrestricted -Scope LocalMachine -Force -ErrorAction Ignore
# Don't set this before Set-ExecutionPolicy as it throws an error
$ErrorActionPreference = "stop"
# -------------------------------------------
# WinRM
# Remove HTTP listener
Remove-Item -Path WSMan:\Localhost\listener\listener* -Recurse
$Cert = New-SelfSignedCertificate `
-CertstoreLocation Cert:\LocalMachine\My `
-DnsName "packer"
New-Item `
-Path WSMan:\LocalHost\Listener `
-Transport HTTPS `
-Address * `
-CertificateThumbPrint $Cert.Thumbprint `
-Force
Write-output "Setting up WinRM"
Write-host "(host) setting up WinRM"
cmd.exe /c winrm quickconfig -q
cmd.exe /c winrm set "winrm/config" '@{MaxTimeoutms="1800000"}'
cmd.exe /c winrm set "winrm/config/winrs" '@{MaxMemoryPerShellMB="1024"}'
cmd.exe /c winrm set "winrm/config/service" '@{AllowUnencrypted="true"}'
cmd.exe /c winrm set "winrm/config/client" '@{AllowUnencrypted="true"}'
cmd.exe /c winrm set "winrm/config/service/auth" '@{Basic="true"}'
cmd.exe /c winrm set "winrm/config/client/auth" '@{Basic="true"}'
cmd.exe /c winrm set "winrm/config/service/auth" '@{CredSSP="true"}'
cmd.exe /c winrm set "winrm/config/listener?Address=*+Transport=HTTPS" "@{Port=`"5986`";Hostname=`"packer`";CertificateThumbprint=`"$($Cert.Thumbprint)`"}"
cmd.exe /c netsh advfirewall firewall set rule group="remote administration" new enable=yes
cmd.exe /c netsh firewall add portopening TCP 5986 "Port 5986"
cmd.exe /c net stop winrm
cmd.exe /c sc config winrm start= auto
cmd.exe /c net start winrm
# -------------------------------------------
# Disks and Directories
# Bring ebs volume online with read-write access
Get-Disk | Where-Object IsOffline -Eq $True | Set-Disk -IsOffline $False
Get-Disk | Where-Object isReadOnly -Eq $True | Set-Disk -IsReadOnly $False
New-Item -ItemType Directory -Force -Path C:\opt -ErrorAction Stop
# -------------------------------------------
# SSH
Try {
# install portable SSH instead of the Windows feature because we
# need to target 2016
$repo = "https://github.com/PowerShell/Win32-OpenSSH"
$version = "v8.0.0.0p1-Beta"
$url = "${repo}/releases/download/${version}/OpenSSH-Win64.zip"
# TODO: check sha!
Write-Output "Downloading OpenSSH from: $url"
Invoke-WebRequest -Uri $url -Outfile "OpenSSH-Win64.zip" -ErrorAction Stop
Expand-Archive ".\OpenSSH-Win64.zip" "C:\Program Files" -ErrorAction Stop
Rename-Item -Path "C:\Program Files\OpenSSH-Win64" -NewName "OpenSSH" -ErrorAction Stop
& "C:\Program Files\OpenSSH\install-sshd.ps1"
# Start the service
Start-Service sshd
Set-Service -Name sshd -StartupType 'Automatic' -ErrorAction Stop
Start-Service ssh-agent
Set-Service -Name ssh-agent -StartupType 'Automatic' -ErrorAction Stop
# Enable host firewall rule if it doesn't exist
New-NetFirewallRule -Name sshd -DisplayName 'OpenSSH Server (sshd)' `
-Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 -ErrorAction Stop
# Note: there appears to be a regression in recent versions of
# Terraform for file provisioning over ssh for Windows with
# powershell as the default shell
# See: https://github.com/hashicorp/terraform/issues/30661
#
# Set powershell as the OpenSSH login shell
# New-ItemProperty -Path "HKLM:\SOFTWARE\OpenSSH" `
# -Name DefaultShell `
# -Value "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe" `
# -PropertyType String -Force -ErrorAction Stop
Write-Output "Installed OpenSSH."
} Catch {
Write-Output "Failed to install OpenSSH."
Write-Output $_
$host.SetShouldExit(-1)
throw
}
md "C:\Users\Administrator\.ssh\"
$myKey = "C:\Users\Administrator\.ssh\authorized_keys"
$adminKey = "C:\ProgramData\ssh\administrators_authorized_keys"
Invoke-RestMethod `
-Uri "http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key" `
-Outfile $myKey
cp $myKey $adminKey
icacls $adminKey /reset
icacls $adminKey /inheritance:r
icacls $adminKey /grant BUILTIN\Administrators:`(F`)
icacls $adminKey /grant SYSTEM:`(F`)
</powershell>

View File

@@ -4,7 +4,6 @@
locals {
ami_prefix = "nomad-e2e-v3"
ubuntu_image_name = "ubuntu-jammy-${var.instance_arch}"
windows_image_name = "windows-2016-${var.instance_arch}"
}
resource "aws_instance" "server" {
@@ -44,20 +43,20 @@ resource "aws_instance" "client_ubuntu_jammy" {
resource "aws_instance" "client_windows_2016" {
ami = data.aws_ami.windows_2016[0].image_id
resource "aws_instance" "client_windows_2022" {
ami = data.aws_ami.windows_2022[0].image_id
instance_type = var.instance_type
key_name = module.keys.key_name
vpc_security_group_ids = [aws_security_group.clients.id]
count = var.client_count_windows_2016
count = var.client_count_windows_2022
iam_instance_profile = data.aws_iam_instance_profile.nomad_e2e_cluster.name
availability_zone = var.availability_zone
user_data = file("${path.module}/userdata/windows-2016.ps1")
user_data = file("${path.module}/userdata/windows-2022.ps1")
# Instance tags
tags = {
Name = "${local.random_name}-client-windows-2016-${count.index}"
Name = "${local.random_name}-client-windows-2022-${count.index}"
ConsulAutoJoin = "auto-join-${local.random_name}"
User = data.aws_caller_identity.current.arn
OS = "windows"
@@ -138,24 +137,14 @@ data "aws_ami" "ubuntu_jammy" {
}
}
data "aws_ami" "windows_2016" {
count = var.client_count_windows_2016 > 0 ? 1 : 0
data "aws_ami" "windows_2022" {
count = var.client_count_windows_2022 > 0 ? 1 : 0
most_recent = true
owners = ["self"]
owners = ["amazon"]
filter {
name = "name"
values = ["${local.ami_prefix}-${local.windows_image_name}-*"]
}
filter {
name = "tag:OS"
values = ["Windows2016"]
}
filter {
name = "tag:BuilderSha"
values = [data.external.packer_sha.result["sha"]]
values = ["Windows_Server-2022-English-Full-ECS_Optimized-2025.*"]
}
}

View File

@@ -4,7 +4,7 @@
locals {
server_binary = var.nomad_local_binary_server != "" ? var.nomad_local_binary_server : var.nomad_local_binary
linux_binary = var.nomad_local_binary_client_ubuntu_jammy != "" ? var.nomad_local_binary_client_ubuntu_jammy : var.nomad_local_binary
windows_binary = var.nomad_local_binary_client_windows_2016 != "" ? var.nomad_local_binary_client_windows_2016 : var.nomad_local_binary
windows_binary = var.nomad_local_binary_client_windows_2022 != "" ? var.nomad_local_binary_client_windows_2022 : var.nomad_local_binary
}
module "nomad_server" {
@@ -70,18 +70,16 @@ module "nomad_client_ubuntu_jammy" {
}
# TODO: split out the different Windows targets (2016, 2019) when they're
# available
module "nomad_client_windows_2016" {
module "nomad_client_windows_2022" {
source = "./provision-nomad"
depends_on = [aws_instance.client_windows_2016]
count = var.client_count_windows_2016
depends_on = [aws_instance.client_windows_2022]
count = var.client_count_windows_2022
platform = "windows"
arch = "windows_${var.instance_arch}"
role = "client"
index = count.index
instance = aws_instance.client_windows_2016[count.index]
instance = aws_instance.client_windows_2022[count.index]
nomad_region = var.nomad_region
nomad_license = var.nomad_license

View File

@@ -10,11 +10,11 @@ output "linux_clients" {
}
output "windows_clients" {
value = aws_instance.client_windows_2016.*.public_ip
value = aws_instance.client_windows_2022.*.public_ip
}
output "clients" {
value = concat(aws_instance.client_ubuntu_jammy.*.public_ip, aws_instance.client_windows_2016.*.public_ip)
value = concat(aws_instance.client_ubuntu_jammy.*.public_ip, aws_instance.client_windows_2022.*.public_ip)
}
output "message" {
@@ -38,7 +38,7 @@ ssh into clients with:
%{for ip in aws_instance.client_ubuntu_jammy.*.public_ip~}
ssh -i ${local.keys_dir}/${local.random_name}.pem ubuntu@${ip}
%{endfor~}
%{for ip in aws_instance.client_windows_2016.*.public_ip~}
%{for ip in aws_instance.client_windows_2022.*.public_ip~}
ssh -i ${local.keys_dir}/${local.random_name}.pem Administrator@${ip}
%{endfor~}

View File

@@ -47,10 +47,10 @@ resource "null_resource" "install_consul_configs_windows" {
"powershell Remove-Item -Force -Recurse -Path C://etc/consul.d",
"powershell New-Item -Force -Path C:// -Name opt -ItemType directory",
"powershell New-Item -Force -Path C://etc -Name consul.d -ItemType directory",
"powershell Move-Item -Force -Path C://tmp/consul_ca.pem C://Windows/System32/ca.pem",
"powershell Move-Item -Force -Path C://tmp/consul_client_acl.json C://etc/consul.d/acl.json",
"powershell Move-Item -Force -Path C://tmp/consul_client.json C://etc/consul.d/consul_client.json",
"powershell Move-Item -Force -Path C://tmp/consul_client_base.json C://etc/consul.d/consul_client_base.json",
"powershell Move-Item -Force -Path C://tmp/consul_ca.crt C://etc/consul.d/ca.pem",
"powershell Move-Item -Force -Path C://tmp/consul_cert.key.pem C://etc/consul.d/cert.key.pem",
"powershell Move-Item -Force -Path C://tmp/consul_cert.pem C://etc/consul.d/cert.pem",
"powershell Move-Item -Force -Path C://tmp/consul_client.hcl C://etc/consul.d/consul_client.hcl",
]
}
}

View File

@@ -1,29 +0,0 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
<powershell>
# Bring ebs volume online with read-write access
Get-Disk | Where-Object IsOffline -Eq $True | Set-Disk -IsOffline $False
Get-Disk | Where-Object isReadOnly -Eq $True | Set-Disk -IsReadOnly $False
md "C:\Users\Administrator\.ssh\"
$myKey = "C:\Users\Administrator\.ssh\authorized_keys"
$adminKey = "C:\ProgramData\ssh\administrators_authorized_keys"
Invoke-RestMethod `
-Uri "http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key" `
-Outfile $myKey
cp $myKey $adminKey
icacls $adminKey /reset
icacls $adminKey /inheritance:r
icacls $adminKey /grant BUILTIN\Administrators:`(F`)
icacls $adminKey /grant SYSTEM:`(F`)
# for host volume testing
New-Item -ItemType Directory -Force -Path C:\tmp\data
</powershell>

View File

@@ -0,0 +1,179 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
<powershell>
Set-StrictMode -Version latest
$ErrorActionPreference = "Stop"
$RunningAsAdmin = ([Security.Principal.WindowsPrincipal] [Security.Principal.WindowsIdentity]::GetCurrent()).IsInRole([Security.Principal.WindowsBuiltInRole] "Administrator")
if (!$RunningAsAdmin) {
Write-Error "Must be executed in Administrator level shell."
exit 1
}
# -------------------------------------------
# Disks and Directories
# Bring ebs volume online with read-write access
Get-Disk | Where-Object IsOffline -Eq $True | Set-Disk -IsOffline $False
Get-Disk | Where-Object isReadOnly -Eq $True | Set-Disk -IsReadOnly $False
New-Item -ItemType Directory -Force -Path C:\opt\nomad
New-Item -ItemType Directory -Force -Path C:\etc\nomad.d
New-Item -ItemType Directory -Force -Path C:\tmp
New-Item -ItemType Directory -Force -Path C:\opt\consul
New-Item -ItemType Directory -Force -Path C:\etc\consul.d
# -------------------------------------------
# Install Consul Agent
Set-Location C:\opt
Try {
$releases = "https://releases.hashicorp.com"
$version = "1.21.1+ent"
$url = "${releases}/consul/${version}/consul_${version}_windows_amd64.zip"
Write-Output "Downloading Consul from: $url"
Invoke-WebRequest -Uri $url -Outfile consul.zip -ErrorAction Stop
Expand-Archive .\consul.zip .\ -ErrorAction Stop
Move-Item consul.exe C:\opt\consul.exe -Force -ErrorAction Stop
C:\opt\consul.exe version
rm consul.zip
New-Service `
-Name "Consul" `
-BinaryPathName "C:\opt\consul.exe agent -config-dir C:\etc\consul.d" `
-StartupType "Automatic" `
-ErrorAction Ignore
} Catch {
Write-Output "Failed to install Consul."
Write-Output $_
$host.SetShouldExit(-1)
throw
}
Write-Output "Installed Consul."
# -------------------------------------------
# Install service and firewall rules for Nomad
# Note the service can't run until we upload Nomad too
Try {
New-NetFirewallRule `
-DisplayName 'Nomad HTTP Inbound' `
-Profile @('Public', 'Domain', 'Private') `
-Direction Inbound `
-Action Allow `
-Protocol TCP `
-LocalPort @('4646')
New-Service `
-Name "Nomad" `
-BinaryPathName "C:\opt\nomad.exe agent -config C:\etc\nomad.d" `
-StartupType "Automatic" `
-ErrorAction Ignore
} Catch {
Write-Output "Failed to install Nomad."
Write-Output $_
$host.SetShouldExit(-1)
throw
}
Write-Output "Installed Nomad."
# --------------------------------------------
# Install firewall rules required to allow tests
Try {
New-NetFirewallRule `
-DisplayName 'Metrics Inbound' `
-Profile @('Public', 'Domain', 'Private') `
-Direction Inbound `
-Action Allow `
-Protocol TCP `
-LocalPort @('6120')
} Catch {
Write-Output "Failed to install firewall rules."
Write-Output $_
$host.SetShouldExit(-1)
throw
}
# -------------------------------------------
# Install and configure ssh
# Note: we don't set powershell as the default ssh shell because of
# https://github.com/hashicorp/terraform/issues/30661
# Note: this is after we install services and binaries so that we can block on
# ssh availability and not race with the provisioning steps in Terraform
Write-Host 'Installing and starting sshd'
Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
Set-Service -Name sshd -StartupType Automatic
Start-Service sshd
Write-Host 'Installing and starting ssh-agent'
Add-WindowsCapability -Online -Name OpenSSH.Client~~~~0.0.1.0
Set-Service -Name ssh-agent -StartupType Automatic
Start-Service ssh-agent
# From https://learn.microsoft.com/en-us/windows-server/administration/openssh/openssh_install_firstuse?tabs=powershell&pivots=windows-server-2022
# Confirm the Firewall rule is configured. It should be created automatically by
# setup. Run the following to verify
if (!(Get-NetFirewallRule -Name "OpenSSH-Server-In-TCP" -ErrorAction SilentlyContinue | Select-Object Name, Enabled)) {
Write-Output "Firewall Rule 'OpenSSH-Server-In-TCP' does not exist, creating it..."
New-NetFirewallRule -Name 'OpenSSH-Server-In-TCP' -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22
} else {
Write-Output "Firewall rule 'OpenSSH-Server-In-TCP' has been created and exists."
}
md "C:\Users\Administrator\.ssh\"
$myKey = "C:\Users\Administrator\.ssh\authorized_keys"
$adminKey = "C:\ProgramData\ssh\administrators_authorized_keys"
Invoke-RestMethod `
-Uri "http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key" `
-Outfile $myKey
cp $myKey $adminKey
icacls $adminKey /reset
icacls $adminKey /inheritance:r
icacls $adminKey /grant BUILTIN\Administrators:`(F`)
icacls $adminKey /grant SYSTEM:`(F`)
# -------------------------------------------
# Disable automatic updates so we don't get restarts in the middle of tests
$service = Get-WmiObject Win32_Service -Filter 'Name="wuauserv"'
if (!$service) {
Write-Error "Failed to retrieve the wauserv service"
exit 1
}
if ($service.StartMode -ne "Disabled") {
$result = $service.ChangeStartMode("Disabled").ReturnValue
if($result) {
Write-Error "Failed to disable the 'wuauserv' service. The return value was $result."
exit 1
}
}
if ($service.State -eq "Running") {
$result = $service.StopService().ReturnValue
if ($result) {
Write-Error "Failed to stop the 'wuauserv' service. The return value was $result."
exit 1
}
}
Write-Output "Automatic Windows Updates disabled."
</powershell>

View File

@@ -36,9 +36,9 @@ variable "client_count_linux" {
default = "4"
}
variable "client_count_windows_2016" {
description = "The number of windows 2016 clients to provision."
default = "0"
variable "client_count_windows_2022" {
description = "The number of windows 2022 clients to provision."
default = "1"
}
variable "restrict_ingress_cidrblock" {
@@ -120,7 +120,7 @@ variable "nomad_local_binary_client_ubuntu_jammy" {
default = ""
}
variable "nomad_local_binary_client_windows_2016" {
variable "nomad_local_binary_client_windows_2022" {
description = "A path to an alternative binary to deploy to windows clients, to override nomad_local_binary"
type = string
default = ""

View File

@@ -7,4 +7,4 @@
# folder
nomad_local_binary = "../../pkg/linux_amd64/nomad"
nomad_local_binary_client_windows_2016 = "../../pkg/windows_amd64/nomad.exe"
nomad_local_binary_client_windows_2022 = "../../pkg/windows_amd64/nomad.exe"

View File

@@ -36,8 +36,8 @@ variable "client_count_linux" {
default = "4"
}
variable "client_count_windows_2016" {
description = "The number of windows 2016 clients to provision."
variable "client_count_windows_2022" {
description = "The number of windows 2022 clients to provision."
default = "0"
}
@@ -111,7 +111,7 @@ variable "nomad_local_binary_client_ubuntu_jammy" {
default = ""
}
variable "nomad_local_binary_client_windows_2016" {
variable "nomad_local_binary_client_windows_2022" {
description = "A path to an alternative binary to deploy to windows clients, to override nomad_local_binary"
type = string
default = ""