[F-net-11478] Make keys directory cluster grouped (#24883)

* func: make windows arch dependant

* func: unify keys and make them cluster grouped

* Update README.md

* Update e2e/terraform/provision-infra/provision-nomad/variables.tf

Co-authored-by: Tim Gross <tgross@hashicorp.com>

* Update .gitignore

* style: add an output with the custer identifier

---------

Co-authored-by: Tim Gross <tgross@hashicorp.com>
This commit is contained in:
Juana De La Cuesta
2025-01-20 10:18:38 +01:00
committed by GitHub
parent af4f31fc5f
commit 039da61d8f
20 changed files with 108 additions and 88 deletions

View File

@@ -8,7 +8,7 @@ custom.tfvars:
echo 'nomad_local_binary = "$(PKG_PATH)"' > custom.tfvars
echo 'volumes = false' >> custom.tfvars
echo 'client_count_linux = 3' >> custom.tfvars
echo 'client_count_windows_2016_amd64 = 0' >> custom.tfvars
echo 'client_count_windows_2016 = 0' >> custom.tfvars
echo 'consul_license = "$(shell cat $(CONSUL_LICENSE_PATH))"' >> custom.tfvars
echo 'nomad_license = "$(shell cat $(NOMAD_LICENSE_PATH))"' >> custom.tfvars
@@ -45,10 +45,10 @@ destroy_full:
# don't run this by default in plan/apply because it prevents you from
# updating a running cluster
tidy:
rm -rf keys
mkdir keys
chmod 0700 keys
rm -rf uploads/*
rm -rf provision-infra/keys
mkdir -p provision-infra/keys
chmod 0700 provision-infra/keys
rm -rf provision-infra/uploads/*
git checkout uploads/README.md
rm -f terraform.tfstate.*.backup
rm custom.tfvars

View File

@@ -52,7 +52,7 @@ region = "us-east-1"
instance_type = "t2.medium"
server_count = "3"
client_count_linux = "4"
client_count_windows_2016_amd64 = "1"
client_count_windows_2016 = "1"
```
You will also need a Consul Enterprise license file and a Nomad Enterprise license file.
@@ -134,20 +134,21 @@ about the cluster:
client node IPs.
- `terraform output windows_clients` will output the list of Windows
client node IPs.
- `cluster_unique_identifier` will output the random name used to identify the cluster's resources
## SSH
You can use Terraform outputs above to access nodes via ssh:
```sh
ssh -i keys/nomad-e2e-*.pem ubuntu@${EC2_IP_ADDR}
ssh -i keys/${CLUSTER_UNIQUE_IDENTIFIER}/nomad-e2e-*.pem ubuntu@${EC2_IP_ADDR}
```
The Windows client runs OpenSSH for convenience, but has a different
user and will drop you into a Powershell shell instead of bash:
```sh
ssh -i keys/nomad-e2e-*.pem Administrator@${EC2_IP_ADDR}
ssh -i keys/${CLUSTER_UNIQUE_IDENTIFIER}/nomad-e2e-*.pem Administrator@${EC2_IP_ADDR}
```
## Teardown

View File

@@ -8,13 +8,13 @@ provider "aws" {
module "provision-infra" {
source = "./provision-infra"
server_count = var.server_count
client_count_linux = var.client_count_linux
client_count_windows_2016_amd64 = var.client_count_windows_2016_amd64
nomad_local_binary = var.nomad_local_binary
nomad_license = var.nomad_license
consul_license = var.consul_license
nomad_region = var.nomad_region
instance_architecture = var.instance_architecture
name = var.name
server_count = var.server_count
client_count_linux = var.client_count_linux
client_count_windows_2016 = var.client_count_windows_2016
nomad_local_binary = var.nomad_local_binary
nomad_license = var.nomad_license
consul_license = var.consul_license
nomad_region = var.nomad_region
instance_arch = var.instance_arch
name = var.name
}

View File

@@ -38,6 +38,10 @@ output "nomad_token" {
sensitive = true
}
output "cluster_unique_identifier" {
value = module.provision-infra.cluster_unique_identifier
}
# Note: Consul and Vault environment needs to be set in test
# environment before the Terraform run, so we don't have that output
# here
@@ -45,4 +49,4 @@ output "environment" {
description = "get connection config by running: $(terraform output environment)"
sensitive = true
value = module.provision-infra.environment
}
}

View File

@@ -2,12 +2,13 @@
# SPDX-License-Identifier: BUSL-1.1
locals {
ami_prefix = "nomad-e2e-v3"
ubuntu_image_name = "ubuntu-jammy-${var.instance_architecture}"
ami_prefix = "nomad-e2e-v3"
ubuntu_image_name = "ubuntu-jammy-${var.instance_arch}"
windows_image_name = "windows-2016-${var.instance_arch}"
}
resource "aws_instance" "server" {
ami = data.aws_ami.ubuntu_jammy_amd64.image_id
ami = data.aws_ami.ubuntu_jammy.image_id
instance_type = var.instance_type
key_name = module.keys.key_name
vpc_security_group_ids = [aws_security_group.servers.id] # see also the secondary ENI
@@ -42,12 +43,12 @@ resource "aws_instance" "client_ubuntu_jammy" {
resource "aws_instance" "client_windows_2016_amd64" {
ami = data.aws_ami.windows_2016_amd64[0].image_id
resource "aws_instance" "client_windows_2016" {
ami = data.aws_ami.windows_2016[0].image_id
instance_type = var.instance_type
key_name = module.keys.key_name
vpc_security_group_ids = [aws_security_group.clients.id]
count = var.client_count_windows_2016_amd64
count = var.client_count_windows_2016
iam_instance_profile = data.aws_iam_instance_profile.nomad_e2e_cluster.name
availability_zone = var.availability_zone
@@ -127,15 +128,15 @@ data "aws_ami" "ubuntu_jammy" {
}
}
data "aws_ami" "windows_2016_amd64" {
count = var.client_count_windows_2016_amd64 > 0 ? 1 : 0
data "aws_ami" "windows_2016" {
count = var.client_count_windows_2016 > 0 ? 1 : 0
most_recent = true
owners = ["self"]
filter {
name = "name"
values = ["${local.ami_prefix}-windows-2016-amd64-*"]
values = ["${local.ami_prefix}-windows-2016-${local.windows_image_name}-*"]
}
filter {

View File

@@ -10,7 +10,7 @@ resource "random_uuid" "consul_initial_management_token" {}
resource "local_sensitive_file" "consul_initial_management_token" {
content = random_uuid.consul_initial_management_token.result
filename = "${path.module}/keys/consul_initial_management_token"
filename = "${local.keys_dir}/consul_initial_management_token"
file_permission = "0600"
}
@@ -91,13 +91,13 @@ resource "null_resource" "upload_consul_server_configs" {
user = "ubuntu"
host = aws_instance.consul_server.public_ip
port = 22
private_key = file("${path.module}/../keys/${local.random_name}.pem")
private_key = file("${local.keys_dir}/${local.random_name}.pem")
target_platform = "unix"
timeout = "15m"
}
provisioner "file" {
source = "${path.module}/keys/tls_ca.crt"
source = "${local.keys_dir}/tls_ca.crt"
destination = "/tmp/consul_ca.pem"
}
provisioner "file" {
@@ -133,7 +133,7 @@ resource "null_resource" "install_consul_server_configs" {
user = "ubuntu"
host = aws_instance.consul_server.public_ip
port = 22
private_key = file("${path.module}/../keys/${local.random_name}.pem")
private_key = file("${local.keys_dir}/${local.random_name}.pem")
target_platform = "unix"
timeout = "15m"
}
@@ -169,7 +169,7 @@ resource "null_resource" "bootstrap_consul_acls" {
command = "${path.module}/scripts/bootstrap-consul.sh"
environment = {
CONSUL_HTTP_ADDR = "https://${aws_instance.consul_server.public_ip}:8501"
CONSUL_CACERT = "${path.module}/keys/tls_ca.crt"
CONSUL_CACERT = "${local.keys_dir}/tls_ca.crt"
CONSUL_HTTP_TOKEN = "${random_uuid.consul_initial_management_token.result}"
CONSUL_AGENT_TOKEN = "${random_uuid.consul_agent_token.result}"
NOMAD_CLUSTER_CONSUL_TOKEN = "${random_uuid.consul_token_for_nomad.result}"

View File

@@ -16,12 +16,13 @@ resource "random_password" "windows_admin_password" {
locals {
random_name = "${var.name}-${random_pet.e2e.id}"
uploads_dir = "${path.module}/provision-nomad/uploads/${random_pet.e2e.id}"
keys_dir = "${path.module}/keys/${random_pet.e2e.id}"
}
# Generates keys to use for provisioning and access
module "keys" {
name = local.random_name
path = "${path.module}/../keys"
path = "${local.keys_dir}"
source = "mitchellh/dynamic-keys/aws"
version = "v2.0.0"
}

View File

@@ -14,16 +14,17 @@ resource "null_resource" "bootstrap_nomad_acls" {
command = "${path.module}/scripts/bootstrap-nomad.sh"
environment = {
NOMAD_ADDR = "https://${aws_instance.server.0.public_ip}:4646"
NOMAD_CACERT = "${path.module}/keys/tls_ca.crt"
NOMAD_CLIENT_CERT = "${path.module}/keys/tls_api_client.crt"
NOMAD_CLIENT_KEY = "${path.module}/keys/tls_api_client.key"
NOMAD_CACERT = "${local.keys_dir}/tls_ca.crt"
NOMAD_CLIENT_CERT = "${local.keys_dir}/tls_api_client.crt"
NOMAD_CLIENT_KEY = "${local.keys_dir}/tls_api_client.key"
NOMAD_TOKEN_PATH = "${local.keys_dir}"
}
}
}
data "local_sensitive_file" "nomad_token" {
depends_on = [null_resource.bootstrap_nomad_acls]
filename = "${path.module}/keys/nomad_root_token"
filename = "${local.keys_dir}/nomad_root_token"
}
# push the token out to the servers for humans to use.
@@ -53,7 +54,7 @@ resource "null_resource" "root_nomad_env_servers" {
user = "ubuntu"
host = aws_instance.server[count.index].public_ip
port = 22
private_key = file("${path.module}/../keys/${local.random_name}.pem")
private_key = file("${local.keys_dir}/${local.random_name}.pem")
timeout = "5m"
}
provisioner "remote-exec" {

View File

@@ -23,12 +23,13 @@ module "nomad_server" {
aws_kms_key_id = data.aws_kms_alias.e2e.target_key_id
uploads_dir = local.uploads_dir
keys_dir = local.keys_dir
connection = {
type = "ssh"
user = "ubuntu"
port = 22
private_key = "${path.module}/../keys/${local.random_name}.pem"
private_key = "${local.keys_dir}/${local.random_name}.pem"
}
}
@@ -40,54 +41,56 @@ module "nomad_client_ubuntu_jammy" {
count = var.client_count_linux
platform = "linux"
arch = "linux_amd64"
arch = "linux_${var.instance_arch}"
role = "client"
index = count.index
instance = aws_instance.client_ubuntu_jammy[count.index]
nomad_license = var.nomad_license
nomad_region = var.nomad_region
nomad_local_binary = count.index < length(var.nomad_local_binary_client_ubuntu_jammy_amd64) ? var.nomad_local_binary_client_ubuntu_jammy_amd64[count.index] : var.nomad_local_binary
nomad_local_binary = count.index < length(var.nomad_local_binary_client_ubuntu_jammy) ? var.nomad_local_binary_client_ubuntu_jammy[count.index] : var.nomad_local_binary
tls_ca_key = tls_private_key.ca.private_key_pem
tls_ca_cert = tls_self_signed_cert.ca.cert_pem
uploads_dir = local.uploads_dir
keys_dir = local.keys_dir
connection = {
type = "ssh"
user = "ubuntu"
port = 22
private_key = "${path.module}/../keys/${local.random_name}.pem"
private_key = "${local.keys_dir}/${local.random_name}.pem"
}
}
# TODO: split out the different Windows targets (2016, 2019) when they're
# available
module "nomad_client_windows_2016_amd64" {
module "nomad_client_windows_2016" {
source = "./provision-nomad"
depends_on = [aws_instance.client_windows_2016_amd64]
count = var.client_count_windows_2016_amd64
depends_on = [aws_instance.client_windows_2016]
count = var.client_count_windows_2016
platform = "windows"
arch = "windows_amd64"
arch = "windows_${var.instance_arch}"
role = "client"
index = count.index
instance = aws_instance.client_windows_2016_amd64[count.index]
instance = aws_instance.client_windows_2016[count.index]
nomad_region = var.nomad_region
nomad_license = var.nomad_license
nomad_local_binary = count.index < length(var.nomad_local_binary_client_windows_2016_amd64) ? var.nomad_local_binary_client_windows_2016_amd64[count.index] : ""
nomad_local_binary = count.index < length(var.nomad_local_binary_client_windows_2016) ? var.nomad_local_binary_client_windows_2016[count.index] : ""
tls_ca_key = tls_private_key.ca.private_key_pem
tls_ca_cert = tls_self_signed_cert.ca.cert_pem
uploads_dir = local.uploads_dir
keys_dir = local.keys_dir
connection = {
type = "ssh"
user = "Administrator"
port = 22
private_key = "${path.module}/../keys/${local.random_name}.pem"
private_key = "${local.keys_dir}/${local.random_name}.pem"
}
}

View File

@@ -10,7 +10,7 @@ output "linux_clients" {
}
output "windows_clients" {
value = aws_instance.client_windows_2016_amd64.*.public_ip
value = aws_instance.client_windows_2016.*.public_ip
}
output "message" {
@@ -26,16 +26,16 @@ Then you can run tests from the e2e directory with:
ssh into servers with:
%{for ip in aws_instance.server.*.public_ip~}
ssh -i keys/${local.random_name}.pem ubuntu@${ip}
ssh -i keys/${local.random_name}/${local.random_name}.pem ubuntu@${ip}
%{endfor~}
ssh into clients with:
%{for ip in aws_instance.client_ubuntu_jammy.*.public_ip~}
ssh -i keys/${local.random_name}.pem ubuntu@${ip}
ssh -i keys/${local.random_name}/${local.random_name}.pem ubuntu@${ip}
%{endfor~}
%{for ip in aws_instance.client_windows_2016_amd64.*.public_ip~}
ssh -i keys/${local.random_name}.pem Administrator@${ip}
%{for ip in aws_instance.client_windows_2016.*.public_ip~}
ssh -i keys/${local.random_name}/${local.random_name}.pem Administrator@${ip}
%{endfor~}
EOM
@@ -49,31 +49,36 @@ output "environment" {
sensitive = true
value = <<EOM
export NOMAD_ADDR=https://${aws_instance.server[0].public_ip}:4646
export NOMAD_CACERT=${abspath(path.module)}/keys/tls_ca.crt
export NOMAD_CLIENT_CERT=${abspath(path.module)}/keys/tls_api_client.crt
export NOMAD_CLIENT_KEY=${abspath(path.module)}/keys/tls_api_client.key
export NOMAD_CACERT=${abspath(local.keys_dir)}/tls_ca.crt
export NOMAD_CLIENT_CERT=${abspath(local.keys_dir)}/tls_api_client.crt
export NOMAD_CLIENT_KEY=${abspath(local.keys_dir)}/tls_api_client.key
export NOMAD_TOKEN=${data.local_sensitive_file.nomad_token.content}
export NOMAD_E2E=1
export CONSUL_HTTP_ADDR=https://${aws_instance.consul_server.public_ip}:8501
export CONSUL_HTTP_TOKEN=${local_sensitive_file.consul_initial_management_token.content}
export CONSUL_CACERT=${abspath(path.module)}/keys/tls_ca.crt
export CONSUL_CACERT=${abspath(local.keys_dir)}/tls_ca.crt
export CLUSTER_UNIQUE_IDENTIFIER=${local.random_name}
EOM
}
output "cluster_unique_identifier" {
value = "${local.random_name}"
}
output "nomad_addr" {
value = "https://${aws_instance.server[0].public_ip}:4646"
}
output "ca_file" {
value = "${abspath(path.module)}/keys/tls_ca.crt"
value = "${abspath(local.keys_dir)}/tls_ca.crt"
}
output "cert_file" {
value = "${abspath(path.module)}/keys/tls_api_client.crt"
value = "${abspath(local.keys_dir)}/tls_api_client.crt"
}
output "key_file" {
value = "${abspath(path.module)}/keys/tls_api_client.key"
value = "${abspath(local.keys_dir)}/tls_api_client.key"
}
output "nomad_token" {

View File

@@ -67,7 +67,7 @@ resource "null_resource" "upload_consul_configs" {
destination = "/tmp/consul_cert.pem"
}
provisioner "file" {
source = "${path.module}/../keys/tls_ca.crt"
source = "${var.keys_dir}/tls_ca.crt"
destination = "/tmp/consul_ca.crt"
}
provisioner "file" {
@@ -136,23 +136,23 @@ resource "null_resource" "upload_nomad_configs" {
destination = "/tmp/agent-${var.instance.public_ip}.crt"
}
provisioner "file" {
source = "${path.module}/../keys/tls_api_client.key"
source = "${var.keys_dir}/tls_api_client.key"
destination = "/tmp/tls_proxy.key"
}
provisioner "file" {
source = "${path.module}/../keys/tls_api_client.crt"
source = "${var.keys_dir}/tls_api_client.crt"
destination = "/tmp/tls_proxy.crt"
}
provisioner "file" {
source = "${path.module}/../keys/tls_ca.crt"
source = "${var.keys_dir}/tls_ca.crt"
destination = "/tmp/ca.crt"
}
provisioner "file" {
source = "${path.module}/../keys/self_signed.key"
source = "${var.keys_dir}/self_signed.key"
destination = "/tmp/self_signed.key"
}
provisioner "file" {
source = "${path.module}/../keys/self_signed.crt"
source = "${var.keys_dir}/self_signed.crt"
destination = "/tmp/self_signed.crt"
}
}

View File

@@ -34,10 +34,10 @@ resource "tls_locally_signed_cert" "nomad" {
resource "local_sensitive_file" "nomad_client_key" {
content = tls_private_key.nomad.private_key_pem
filename = "keys/agent-${var.instance.public_ip}.key"
filename = "${var.keys_dir}/agent-${var.instance.public_ip}.key"
}
resource "local_sensitive_file" "nomad_client_cert" {
content = tls_locally_signed_cert.nomad.cert_pem
filename = "keys/agent-${var.instance.public_ip}.crt"
filename = "${var.keys_dir}/agent-${var.instance.public_ip}.crt"
}

View File

@@ -93,3 +93,9 @@ variable "uploads_dir" {
description = "Directory where all the configuration files for nomad, consul and vault will be taken from to configure each nomad node"
default = ""
}
variable "keys_dir" {
type = string
description = "Directory where all the configuration TLS and SSH keys and certificates will be stored for provisioning"
default = ""
}

View File

@@ -19,8 +19,8 @@ set -e
export NOMAD_TOKEN="$ROOT_TOKEN"
mkdir -p ../keys
echo $NOMAD_TOKEN > "${DIR}/../keys/nomad_root_token"
mkdir -p "$NOMAD_TOKEN_PATH"
echo $NOMAD_TOKEN > "${NOMAD_TOKEN_PATH}/nomad_root_token"
echo NOMAD_TOKEN=$NOMAD_TOKEN
# Our default policy after bootstrapping will be full-access. Without

View File

@@ -23,11 +23,11 @@ resource "tls_self_signed_cert" "ca" {
}
resource "local_sensitive_file" "ca_key" {
filename = "${path.module}/keys/tls_ca.key"
filename = "${local.keys_dir}/tls_ca.key"
content = tls_private_key.ca.private_key_pem
}
resource "local_sensitive_file" "ca_cert" {
filename = "${path.module}/keys/tls_ca.crt"
filename = "${local.keys_dir}/tls_ca.crt"
content = tls_self_signed_cert.ca.cert_pem
}

View File

@@ -34,12 +34,12 @@ resource "tls_locally_signed_cert" "api_client" {
resource "local_sensitive_file" "api_client_key" {
content = tls_private_key.api_client.private_key_pem
filename = "${path.module}/keys/tls_api_client.key"
filename = "${local.keys_dir}/tls_api_client.key"
}
resource "local_sensitive_file" "api_client_cert" {
content = tls_locally_signed_cert.api_client.cert_pem
filename = "${path.module}/keys/tls_api_client.crt"
filename = "${local.keys_dir}/tls_api_client.crt"
}
# Self signed cert for reverse proxy
@@ -66,10 +66,10 @@ resource "tls_self_signed_cert" "self_signed" {
resource "local_sensitive_file" "self_signed_key" {
content = tls_private_key.self_signed.private_key_pem
filename = "${path.module}/keys/self_signed.key"
filename = "${local.keys_dir}/self_signed.key"
}
resource "local_sensitive_file" "self_signed_cert" {
content = tls_self_signed_cert.self_signed.cert_pem
filename = "${path.module}/keys/self_signed.crt"
filename = "${local.keys_dir}/self_signed.crt"
}

View File

@@ -21,7 +21,7 @@ variable "instance_type" {
default = "t3a.medium"
}
variable "instance_architecture" {
variable "instance_arch" {
description = "The architecture for the AWS instance type to use for both clients and servers."
default = "amd64"
}
@@ -36,7 +36,7 @@ variable "client_count_linux" {
default = "4"
}
variable "client_count_windows_2016_amd64" {
variable "client_count_windows_2016" {
description = "The number of windows 2016 clients to provision."
default = "0"
}
@@ -109,13 +109,13 @@ variable "nomad_local_binary_server" {
default = []
}
variable "nomad_local_binary_client_ubuntu_jammy_amd64" {
variable "nomad_local_binary_client_ubuntu_jammy" {
description = "A list of nomad local binary paths to deploy to Ubuntu Jammy clients, to override nomad_local_binary"
type = list(string)
default = []
}
variable "nomad_local_binary_client_windows_2016_amd64" {
variable "nomad_local_binary_client_windows_2016" {
description = "A list of nomad local binary paths to deploy to Windows 2016 clients, to override nomad_local_binary"
type = list(string)
default = []

View File

@@ -7,4 +7,4 @@
# folder
nomad_local_binary = "../../pkg/linux_amd64/nomad"
nomad_local_binary_client_windows_2016_amd64 = ["../../pkg/windows_amd64/nomad.exe"]
nomad_local_binary_client_windows_2016 = ["../../pkg/windows_amd64/nomad.exe"]

View File

@@ -21,7 +21,7 @@ variable "instance_type" {
default = "t3a.medium"
}
variable "instance_architecture" {
variable "instance_arch" {
description = "The architecture for the AWS instance type to use for both clients and servers."
default = "amd64"
}
@@ -36,7 +36,7 @@ variable "client_count_linux" {
default = "4"
}
variable "client_count_windows_2016_amd64" {
variable "client_count_windows_2016" {
description = "The number of windows 2016 clients to provision."
default = "0"
}
@@ -105,13 +105,13 @@ variable "nomad_local_binary_server" {
default = []
}
variable "nomad_local_binary_client_ubuntu_jammy_amd64" {
variable "nomad_local_binary_client_ubuntu_jammy" {
description = "A list of nomad local binary paths to deploy to Ubuntu Jammy clients, to override nomad_local_binary"
type = list(string)
default = []
}
variable "nomad_local_binary_client_windows_2016_amd64" {
variable "nomad_local_binary_client_windows_2016" {
description = "A list of nomad local binary paths to deploy to Windows 2016 clients, to override nomad_local_binary"
type = list(string)
default = []

2
enos/.gitignore vendored
View File

@@ -1,2 +0,0 @@
# enos scenarios
.enos/