From 039da61d8fa132e6177f0b631406d2e59ae1524e Mon Sep 17 00:00:00 2001 From: Juana De La Cuesta Date: Mon, 20 Jan 2025 10:18:38 +0100 Subject: [PATCH] [F-net-11478] Make keys directory cluster grouped (#24883) * func: make windows arch dependant * func: unify keys and make them cluster grouped * Update README.md * Update e2e/terraform/provision-infra/provision-nomad/variables.tf Co-authored-by: Tim Gross * Update .gitignore * style: add an output with the custer identifier --------- Co-authored-by: Tim Gross --- e2e/terraform/Makefile | 10 +++---- e2e/terraform/README.md | 7 +++-- e2e/terraform/main.tf | 18 ++++++------ e2e/terraform/outputs.tf | 6 +++- e2e/terraform/provision-infra/compute.tf | 19 ++++++------ .../provision-infra/consul-servers.tf | 10 +++---- e2e/terraform/provision-infra/main.tf | 3 +- e2e/terraform/provision-infra/nomad-acls.tf | 11 +++---- e2e/terraform/provision-infra/nomad.tf | 25 +++++++++------- e2e/terraform/provision-infra/outputs.tf | 29 +++++++++++-------- .../provision-infra/provision-nomad/main.tf | 12 ++++---- .../provision-infra/provision-nomad/tls.tf | 4 +-- .../provision-nomad/variables.tf | 6 ++++ .../scripts/bootstrap-nomad.sh | 4 +-- e2e/terraform/provision-infra/tls_ca.tf | 4 +-- e2e/terraform/provision-infra/tls_client.tf | 8 ++--- e2e/terraform/provision-infra/variables.tf | 8 ++--- e2e/terraform/terraform.tfvars | 2 +- e2e/terraform/variables.tf | 8 ++--- enos/.gitignore | 2 -- 20 files changed, 108 insertions(+), 88 deletions(-) delete mode 100644 enos/.gitignore diff --git a/e2e/terraform/Makefile b/e2e/terraform/Makefile index 14d7124c2..fa77c60f8 100644 --- a/e2e/terraform/Makefile +++ b/e2e/terraform/Makefile @@ -8,7 +8,7 @@ custom.tfvars: echo 'nomad_local_binary = "$(PKG_PATH)"' > custom.tfvars echo 'volumes = false' >> custom.tfvars echo 'client_count_linux = 3' >> custom.tfvars - echo 'client_count_windows_2016_amd64 = 0' >> custom.tfvars + echo 'client_count_windows_2016 = 0' >> custom.tfvars echo 'consul_license = "$(shell cat $(CONSUL_LICENSE_PATH))"' >> custom.tfvars echo 'nomad_license = "$(shell cat $(NOMAD_LICENSE_PATH))"' >> custom.tfvars @@ -45,10 +45,10 @@ destroy_full: # don't run this by default in plan/apply because it prevents you from # updating a running cluster tidy: - rm -rf keys - mkdir keys - chmod 0700 keys - rm -rf uploads/* + rm -rf provision-infra/keys + mkdir -p provision-infra/keys + chmod 0700 provision-infra/keys + rm -rf provision-infra/uploads/* git checkout uploads/README.md rm -f terraform.tfstate.*.backup rm custom.tfvars diff --git a/e2e/terraform/README.md b/e2e/terraform/README.md index 12a9e1196..e691bb773 100644 --- a/e2e/terraform/README.md +++ b/e2e/terraform/README.md @@ -52,7 +52,7 @@ region = "us-east-1" instance_type = "t2.medium" server_count = "3" client_count_linux = "4" -client_count_windows_2016_amd64 = "1" +client_count_windows_2016 = "1" ``` You will also need a Consul Enterprise license file and a Nomad Enterprise license file. @@ -134,20 +134,21 @@ about the cluster: client node IPs. - `terraform output windows_clients` will output the list of Windows client node IPs. +- `cluster_unique_identifier` will output the random name used to identify the cluster's resources ## SSH You can use Terraform outputs above to access nodes via ssh: ```sh -ssh -i keys/nomad-e2e-*.pem ubuntu@${EC2_IP_ADDR} +ssh -i keys/${CLUSTER_UNIQUE_IDENTIFIER}/nomad-e2e-*.pem ubuntu@${EC2_IP_ADDR} ``` The Windows client runs OpenSSH for convenience, but has a different user and will drop you into a Powershell shell instead of bash: ```sh -ssh -i keys/nomad-e2e-*.pem Administrator@${EC2_IP_ADDR} +ssh -i keys/${CLUSTER_UNIQUE_IDENTIFIER}/nomad-e2e-*.pem Administrator@${EC2_IP_ADDR} ``` ## Teardown diff --git a/e2e/terraform/main.tf b/e2e/terraform/main.tf index e5ab2753c..39800ef31 100644 --- a/e2e/terraform/main.tf +++ b/e2e/terraform/main.tf @@ -8,13 +8,13 @@ provider "aws" { module "provision-infra" { source = "./provision-infra" - server_count = var.server_count - client_count_linux = var.client_count_linux - client_count_windows_2016_amd64 = var.client_count_windows_2016_amd64 - nomad_local_binary = var.nomad_local_binary - nomad_license = var.nomad_license - consul_license = var.consul_license - nomad_region = var.nomad_region - instance_architecture = var.instance_architecture - name = var.name + server_count = var.server_count + client_count_linux = var.client_count_linux + client_count_windows_2016 = var.client_count_windows_2016 + nomad_local_binary = var.nomad_local_binary + nomad_license = var.nomad_license + consul_license = var.consul_license + nomad_region = var.nomad_region + instance_arch = var.instance_arch + name = var.name } diff --git a/e2e/terraform/outputs.tf b/e2e/terraform/outputs.tf index 985f523a5..a6539a7d7 100644 --- a/e2e/terraform/outputs.tf +++ b/e2e/terraform/outputs.tf @@ -38,6 +38,10 @@ output "nomad_token" { sensitive = true } +output "cluster_unique_identifier" { + value = module.provision-infra.cluster_unique_identifier +} + # Note: Consul and Vault environment needs to be set in test # environment before the Terraform run, so we don't have that output # here @@ -45,4 +49,4 @@ output "environment" { description = "get connection config by running: $(terraform output environment)" sensitive = true value = module.provision-infra.environment -} \ No newline at end of file +} diff --git a/e2e/terraform/provision-infra/compute.tf b/e2e/terraform/provision-infra/compute.tf index ed3bd0b4c..6290cb33d 100644 --- a/e2e/terraform/provision-infra/compute.tf +++ b/e2e/terraform/provision-infra/compute.tf @@ -2,12 +2,13 @@ # SPDX-License-Identifier: BUSL-1.1 locals { - ami_prefix = "nomad-e2e-v3" - ubuntu_image_name = "ubuntu-jammy-${var.instance_architecture}" + ami_prefix = "nomad-e2e-v3" + ubuntu_image_name = "ubuntu-jammy-${var.instance_arch}" + windows_image_name = "windows-2016-${var.instance_arch}" } resource "aws_instance" "server" { - ami = data.aws_ami.ubuntu_jammy_amd64.image_id + ami = data.aws_ami.ubuntu_jammy.image_id instance_type = var.instance_type key_name = module.keys.key_name vpc_security_group_ids = [aws_security_group.servers.id] # see also the secondary ENI @@ -42,12 +43,12 @@ resource "aws_instance" "client_ubuntu_jammy" { -resource "aws_instance" "client_windows_2016_amd64" { - ami = data.aws_ami.windows_2016_amd64[0].image_id +resource "aws_instance" "client_windows_2016" { + ami = data.aws_ami.windows_2016[0].image_id instance_type = var.instance_type key_name = module.keys.key_name vpc_security_group_ids = [aws_security_group.clients.id] - count = var.client_count_windows_2016_amd64 + count = var.client_count_windows_2016 iam_instance_profile = data.aws_iam_instance_profile.nomad_e2e_cluster.name availability_zone = var.availability_zone @@ -127,15 +128,15 @@ data "aws_ami" "ubuntu_jammy" { } } -data "aws_ami" "windows_2016_amd64" { - count = var.client_count_windows_2016_amd64 > 0 ? 1 : 0 +data "aws_ami" "windows_2016" { + count = var.client_count_windows_2016 > 0 ? 1 : 0 most_recent = true owners = ["self"] filter { name = "name" - values = ["${local.ami_prefix}-windows-2016-amd64-*"] + values = ["${local.ami_prefix}-windows-2016-${local.windows_image_name}-*"] } filter { diff --git a/e2e/terraform/provision-infra/consul-servers.tf b/e2e/terraform/provision-infra/consul-servers.tf index 67740b2d9..35d96534d 100644 --- a/e2e/terraform/provision-infra/consul-servers.tf +++ b/e2e/terraform/provision-infra/consul-servers.tf @@ -10,7 +10,7 @@ resource "random_uuid" "consul_initial_management_token" {} resource "local_sensitive_file" "consul_initial_management_token" { content = random_uuid.consul_initial_management_token.result - filename = "${path.module}/keys/consul_initial_management_token" + filename = "${local.keys_dir}/consul_initial_management_token" file_permission = "0600" } @@ -91,13 +91,13 @@ resource "null_resource" "upload_consul_server_configs" { user = "ubuntu" host = aws_instance.consul_server.public_ip port = 22 - private_key = file("${path.module}/../keys/${local.random_name}.pem") + private_key = file("${local.keys_dir}/${local.random_name}.pem") target_platform = "unix" timeout = "15m" } provisioner "file" { - source = "${path.module}/keys/tls_ca.crt" + source = "${local.keys_dir}/tls_ca.crt" destination = "/tmp/consul_ca.pem" } provisioner "file" { @@ -133,7 +133,7 @@ resource "null_resource" "install_consul_server_configs" { user = "ubuntu" host = aws_instance.consul_server.public_ip port = 22 - private_key = file("${path.module}/../keys/${local.random_name}.pem") + private_key = file("${local.keys_dir}/${local.random_name}.pem") target_platform = "unix" timeout = "15m" } @@ -169,7 +169,7 @@ resource "null_resource" "bootstrap_consul_acls" { command = "${path.module}/scripts/bootstrap-consul.sh" environment = { CONSUL_HTTP_ADDR = "https://${aws_instance.consul_server.public_ip}:8501" - CONSUL_CACERT = "${path.module}/keys/tls_ca.crt" + CONSUL_CACERT = "${local.keys_dir}/tls_ca.crt" CONSUL_HTTP_TOKEN = "${random_uuid.consul_initial_management_token.result}" CONSUL_AGENT_TOKEN = "${random_uuid.consul_agent_token.result}" NOMAD_CLUSTER_CONSUL_TOKEN = "${random_uuid.consul_token_for_nomad.result}" diff --git a/e2e/terraform/provision-infra/main.tf b/e2e/terraform/provision-infra/main.tf index 27403ebb7..0e7965208 100644 --- a/e2e/terraform/provision-infra/main.tf +++ b/e2e/terraform/provision-infra/main.tf @@ -16,12 +16,13 @@ resource "random_password" "windows_admin_password" { locals { random_name = "${var.name}-${random_pet.e2e.id}" uploads_dir = "${path.module}/provision-nomad/uploads/${random_pet.e2e.id}" + keys_dir = "${path.module}/keys/${random_pet.e2e.id}" } # Generates keys to use for provisioning and access module "keys" { name = local.random_name - path = "${path.module}/../keys" + path = "${local.keys_dir}" source = "mitchellh/dynamic-keys/aws" version = "v2.0.0" } diff --git a/e2e/terraform/provision-infra/nomad-acls.tf b/e2e/terraform/provision-infra/nomad-acls.tf index 846bce89e..fded73880 100644 --- a/e2e/terraform/provision-infra/nomad-acls.tf +++ b/e2e/terraform/provision-infra/nomad-acls.tf @@ -14,16 +14,17 @@ resource "null_resource" "bootstrap_nomad_acls" { command = "${path.module}/scripts/bootstrap-nomad.sh" environment = { NOMAD_ADDR = "https://${aws_instance.server.0.public_ip}:4646" - NOMAD_CACERT = "${path.module}/keys/tls_ca.crt" - NOMAD_CLIENT_CERT = "${path.module}/keys/tls_api_client.crt" - NOMAD_CLIENT_KEY = "${path.module}/keys/tls_api_client.key" + NOMAD_CACERT = "${local.keys_dir}/tls_ca.crt" + NOMAD_CLIENT_CERT = "${local.keys_dir}/tls_api_client.crt" + NOMAD_CLIENT_KEY = "${local.keys_dir}/tls_api_client.key" + NOMAD_TOKEN_PATH = "${local.keys_dir}" } } } data "local_sensitive_file" "nomad_token" { depends_on = [null_resource.bootstrap_nomad_acls] - filename = "${path.module}/keys/nomad_root_token" + filename = "${local.keys_dir}/nomad_root_token" } # push the token out to the servers for humans to use. @@ -53,7 +54,7 @@ resource "null_resource" "root_nomad_env_servers" { user = "ubuntu" host = aws_instance.server[count.index].public_ip port = 22 - private_key = file("${path.module}/../keys/${local.random_name}.pem") + private_key = file("${local.keys_dir}/${local.random_name}.pem") timeout = "5m" } provisioner "remote-exec" { diff --git a/e2e/terraform/provision-infra/nomad.tf b/e2e/terraform/provision-infra/nomad.tf index 8b2b87e9a..cd79a0f6a 100644 --- a/e2e/terraform/provision-infra/nomad.tf +++ b/e2e/terraform/provision-infra/nomad.tf @@ -23,12 +23,13 @@ module "nomad_server" { aws_kms_key_id = data.aws_kms_alias.e2e.target_key_id uploads_dir = local.uploads_dir + keys_dir = local.keys_dir connection = { type = "ssh" user = "ubuntu" port = 22 - private_key = "${path.module}/../keys/${local.random_name}.pem" + private_key = "${local.keys_dir}/${local.random_name}.pem" } } @@ -40,54 +41,56 @@ module "nomad_client_ubuntu_jammy" { count = var.client_count_linux platform = "linux" - arch = "linux_amd64" + arch = "linux_${var.instance_arch}" role = "client" index = count.index instance = aws_instance.client_ubuntu_jammy[count.index] nomad_license = var.nomad_license nomad_region = var.nomad_region - nomad_local_binary = count.index < length(var.nomad_local_binary_client_ubuntu_jammy_amd64) ? var.nomad_local_binary_client_ubuntu_jammy_amd64[count.index] : var.nomad_local_binary + nomad_local_binary = count.index < length(var.nomad_local_binary_client_ubuntu_jammy) ? var.nomad_local_binary_client_ubuntu_jammy[count.index] : var.nomad_local_binary tls_ca_key = tls_private_key.ca.private_key_pem tls_ca_cert = tls_self_signed_cert.ca.cert_pem uploads_dir = local.uploads_dir + keys_dir = local.keys_dir connection = { type = "ssh" user = "ubuntu" port = 22 - private_key = "${path.module}/../keys/${local.random_name}.pem" + private_key = "${local.keys_dir}/${local.random_name}.pem" } } # TODO: split out the different Windows targets (2016, 2019) when they're # available -module "nomad_client_windows_2016_amd64" { +module "nomad_client_windows_2016" { source = "./provision-nomad" - depends_on = [aws_instance.client_windows_2016_amd64] - count = var.client_count_windows_2016_amd64 + depends_on = [aws_instance.client_windows_2016] + count = var.client_count_windows_2016 platform = "windows" - arch = "windows_amd64" + arch = "windows_${var.instance_arch}" role = "client" index = count.index - instance = aws_instance.client_windows_2016_amd64[count.index] + instance = aws_instance.client_windows_2016[count.index] nomad_region = var.nomad_region nomad_license = var.nomad_license - nomad_local_binary = count.index < length(var.nomad_local_binary_client_windows_2016_amd64) ? var.nomad_local_binary_client_windows_2016_amd64[count.index] : "" + nomad_local_binary = count.index < length(var.nomad_local_binary_client_windows_2016) ? var.nomad_local_binary_client_windows_2016[count.index] : "" tls_ca_key = tls_private_key.ca.private_key_pem tls_ca_cert = tls_self_signed_cert.ca.cert_pem uploads_dir = local.uploads_dir + keys_dir = local.keys_dir connection = { type = "ssh" user = "Administrator" port = 22 - private_key = "${path.module}/../keys/${local.random_name}.pem" + private_key = "${local.keys_dir}/${local.random_name}.pem" } } diff --git a/e2e/terraform/provision-infra/outputs.tf b/e2e/terraform/provision-infra/outputs.tf index 012b1cb1f..c330eadb4 100644 --- a/e2e/terraform/provision-infra/outputs.tf +++ b/e2e/terraform/provision-infra/outputs.tf @@ -10,7 +10,7 @@ output "linux_clients" { } output "windows_clients" { - value = aws_instance.client_windows_2016_amd64.*.public_ip + value = aws_instance.client_windows_2016.*.public_ip } output "message" { @@ -26,16 +26,16 @@ Then you can run tests from the e2e directory with: ssh into servers with: %{for ip in aws_instance.server.*.public_ip~} - ssh -i keys/${local.random_name}.pem ubuntu@${ip} + ssh -i keys/${local.random_name}/${local.random_name}.pem ubuntu@${ip} %{endfor~} ssh into clients with: %{for ip in aws_instance.client_ubuntu_jammy.*.public_ip~} - ssh -i keys/${local.random_name}.pem ubuntu@${ip} + ssh -i keys/${local.random_name}/${local.random_name}.pem ubuntu@${ip} %{endfor~} -%{for ip in aws_instance.client_windows_2016_amd64.*.public_ip~} - ssh -i keys/${local.random_name}.pem Administrator@${ip} +%{for ip in aws_instance.client_windows_2016.*.public_ip~} + ssh -i keys/${local.random_name}/${local.random_name}.pem Administrator@${ip} %{endfor~} EOM @@ -49,31 +49,36 @@ output "environment" { sensitive = true value = < "${DIR}/../keys/nomad_root_token" +mkdir -p "$NOMAD_TOKEN_PATH" +echo $NOMAD_TOKEN > "${NOMAD_TOKEN_PATH}/nomad_root_token" echo NOMAD_TOKEN=$NOMAD_TOKEN # Our default policy after bootstrapping will be full-access. Without diff --git a/e2e/terraform/provision-infra/tls_ca.tf b/e2e/terraform/provision-infra/tls_ca.tf index d2aaa9a1b..94481341d 100644 --- a/e2e/terraform/provision-infra/tls_ca.tf +++ b/e2e/terraform/provision-infra/tls_ca.tf @@ -23,11 +23,11 @@ resource "tls_self_signed_cert" "ca" { } resource "local_sensitive_file" "ca_key" { - filename = "${path.module}/keys/tls_ca.key" + filename = "${local.keys_dir}/tls_ca.key" content = tls_private_key.ca.private_key_pem } resource "local_sensitive_file" "ca_cert" { - filename = "${path.module}/keys/tls_ca.crt" + filename = "${local.keys_dir}/tls_ca.crt" content = tls_self_signed_cert.ca.cert_pem } diff --git a/e2e/terraform/provision-infra/tls_client.tf b/e2e/terraform/provision-infra/tls_client.tf index 3b184b734..74dce73f8 100644 --- a/e2e/terraform/provision-infra/tls_client.tf +++ b/e2e/terraform/provision-infra/tls_client.tf @@ -34,12 +34,12 @@ resource "tls_locally_signed_cert" "api_client" { resource "local_sensitive_file" "api_client_key" { content = tls_private_key.api_client.private_key_pem - filename = "${path.module}/keys/tls_api_client.key" + filename = "${local.keys_dir}/tls_api_client.key" } resource "local_sensitive_file" "api_client_cert" { content = tls_locally_signed_cert.api_client.cert_pem - filename = "${path.module}/keys/tls_api_client.crt" + filename = "${local.keys_dir}/tls_api_client.crt" } # Self signed cert for reverse proxy @@ -66,10 +66,10 @@ resource "tls_self_signed_cert" "self_signed" { resource "local_sensitive_file" "self_signed_key" { content = tls_private_key.self_signed.private_key_pem - filename = "${path.module}/keys/self_signed.key" + filename = "${local.keys_dir}/self_signed.key" } resource "local_sensitive_file" "self_signed_cert" { content = tls_self_signed_cert.self_signed.cert_pem - filename = "${path.module}/keys/self_signed.crt" + filename = "${local.keys_dir}/self_signed.crt" } diff --git a/e2e/terraform/provision-infra/variables.tf b/e2e/terraform/provision-infra/variables.tf index d2306b154..44f5cf7eb 100644 --- a/e2e/terraform/provision-infra/variables.tf +++ b/e2e/terraform/provision-infra/variables.tf @@ -21,7 +21,7 @@ variable "instance_type" { default = "t3a.medium" } -variable "instance_architecture" { +variable "instance_arch" { description = "The architecture for the AWS instance type to use for both clients and servers." default = "amd64" } @@ -36,7 +36,7 @@ variable "client_count_linux" { default = "4" } -variable "client_count_windows_2016_amd64" { +variable "client_count_windows_2016" { description = "The number of windows 2016 clients to provision." default = "0" } @@ -109,13 +109,13 @@ variable "nomad_local_binary_server" { default = [] } -variable "nomad_local_binary_client_ubuntu_jammy_amd64" { +variable "nomad_local_binary_client_ubuntu_jammy" { description = "A list of nomad local binary paths to deploy to Ubuntu Jammy clients, to override nomad_local_binary" type = list(string) default = [] } -variable "nomad_local_binary_client_windows_2016_amd64" { +variable "nomad_local_binary_client_windows_2016" { description = "A list of nomad local binary paths to deploy to Windows 2016 clients, to override nomad_local_binary" type = list(string) default = [] diff --git a/e2e/terraform/terraform.tfvars b/e2e/terraform/terraform.tfvars index 20a165111..35ae87fc9 100644 --- a/e2e/terraform/terraform.tfvars +++ b/e2e/terraform/terraform.tfvars @@ -7,4 +7,4 @@ # folder nomad_local_binary = "../../pkg/linux_amd64/nomad" -nomad_local_binary_client_windows_2016_amd64 = ["../../pkg/windows_amd64/nomad.exe"] +nomad_local_binary_client_windows_2016 = ["../../pkg/windows_amd64/nomad.exe"] diff --git a/e2e/terraform/variables.tf b/e2e/terraform/variables.tf index 814e8eb49..d70b6a7eb 100644 --- a/e2e/terraform/variables.tf +++ b/e2e/terraform/variables.tf @@ -21,7 +21,7 @@ variable "instance_type" { default = "t3a.medium" } -variable "instance_architecture" { +variable "instance_arch" { description = "The architecture for the AWS instance type to use for both clients and servers." default = "amd64" } @@ -36,7 +36,7 @@ variable "client_count_linux" { default = "4" } -variable "client_count_windows_2016_amd64" { +variable "client_count_windows_2016" { description = "The number of windows 2016 clients to provision." default = "0" } @@ -105,13 +105,13 @@ variable "nomad_local_binary_server" { default = [] } -variable "nomad_local_binary_client_ubuntu_jammy_amd64" { +variable "nomad_local_binary_client_ubuntu_jammy" { description = "A list of nomad local binary paths to deploy to Ubuntu Jammy clients, to override nomad_local_binary" type = list(string) default = [] } -variable "nomad_local_binary_client_windows_2016_amd64" { +variable "nomad_local_binary_client_windows_2016" { description = "A list of nomad local binary paths to deploy to Windows 2016 clients, to override nomad_local_binary" type = list(string) default = [] diff --git a/enos/.gitignore b/enos/.gitignore deleted file mode 100644 index 4f1020381..000000000 --- a/enos/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# enos scenarios -.enos/ \ No newline at end of file