diff --git a/e2e/terraform/config/dev-cluster/nomad/server/server.hcl b/e2e/terraform/config/dev-cluster/nomad/server/server.hcl index b739d08a7..385e208f1 100644 --- a/e2e/terraform/config/dev-cluster/nomad/server/server.hcl +++ b/e2e/terraform/config/dev-cluster/nomad/server/server.hcl @@ -2,11 +2,3 @@ server { enabled = true bootstrap_expect = 3 } - -vault { - enabled = false - address = "http://active.vault.service.consul:8200" - task_token_ttl = "1h" - create_from_role = "nomad-cluster" - token = "" -} diff --git a/e2e/terraform/config/dev-cluster/vault/server/vault.hcl b/e2e/terraform/config/dev-cluster/vault/server/vault.hcl index b831da022..1e04a13e9 100644 --- a/e2e/terraform/config/dev-cluster/vault/server/vault.hcl +++ b/e2e/terraform/config/dev-cluster/vault/server/vault.hcl @@ -1,12 +1,15 @@ -backend "consul" { - path = "vault/" - address = "{{ GetPrivateIP }}:8500" - cluster_addr = "https://{{ GetPrivateIP }}:8201" - redirect_addr = "http://{{ GetPrivateIP }}:8200" +listener "tcp" { + address = "0.0.0.0:8200" + tls_disable = 1 } -listener "tcp" { - address = "{{ GetPrivateIP }}:8200" - cluster_address = "{{ GetPrivateIP }}:8201" - tls_disable = 1 +# this autounseal key is created by Terraform in the E2E infrastructure repo +# and should be used only for these tests +seal "awskms" { + region = "us-east-1" + kms_key_id = "74b7e226-c745-4ddd-9b7f-2371024ee37d" } + +# Vault 1.5.4 doesn't have autodiscovery for retry_join on its +# integrated storage yet so we'll just use consul for storage +storage "consul" {} diff --git a/e2e/terraform/config/full-cluster/nomad/server/server.hcl b/e2e/terraform/config/full-cluster/nomad/server/server.hcl index b739d08a7..385e208f1 100644 --- a/e2e/terraform/config/full-cluster/nomad/server/server.hcl +++ b/e2e/terraform/config/full-cluster/nomad/server/server.hcl @@ -2,11 +2,3 @@ server { enabled = true bootstrap_expect = 3 } - -vault { - enabled = false - address = "http://active.vault.service.consul:8200" - task_token_ttl = "1h" - create_from_role = "nomad-cluster" - token = "" -} diff --git a/e2e/terraform/config/full-cluster/vault/server/vault.hcl b/e2e/terraform/config/full-cluster/vault/server/vault.hcl index b831da022..1e04a13e9 100644 --- a/e2e/terraform/config/full-cluster/vault/server/vault.hcl +++ b/e2e/terraform/config/full-cluster/vault/server/vault.hcl @@ -1,12 +1,15 @@ -backend "consul" { - path = "vault/" - address = "{{ GetPrivateIP }}:8500" - cluster_addr = "https://{{ GetPrivateIP }}:8201" - redirect_addr = "http://{{ GetPrivateIP }}:8200" +listener "tcp" { + address = "0.0.0.0:8200" + tls_disable = 1 } -listener "tcp" { - address = "{{ GetPrivateIP }}:8200" - cluster_address = "{{ GetPrivateIP }}:8201" - tls_disable = 1 +# this autounseal key is created by Terraform in the E2E infrastructure repo +# and should be used only for these tests +seal "awskms" { + region = "us-east-1" + kms_key_id = "74b7e226-c745-4ddd-9b7f-2371024ee37d" } + +# Vault 1.5.4 doesn't have autodiscovery for retry_join on its +# integrated storage yet so we'll just use consul for storage +storage "consul" {} diff --git a/e2e/terraform/network.tf b/e2e/terraform/network.tf index b8dd606ac..23241c729 100644 --- a/e2e/terraform/network.tf +++ b/e2e/terraform/network.tf @@ -42,6 +42,14 @@ resource "aws_security_group" "primary" { cidr_blocks = ["0.0.0.0/0"] } + # Vault + ingress { + from_port = 8200 + to_port = 8200 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + # HDFS NameNode UI ingress { from_port = 50070 diff --git a/e2e/terraform/nomad-acls.tf b/e2e/terraform/nomad-acls.tf index b8b5feedb..41b84f31c 100644 --- a/e2e/terraform/nomad-acls.tf +++ b/e2e/terraform/nomad-acls.tf @@ -8,19 +8,19 @@ resource "null_resource" "bootstrap_nomad_acls" { depends_on = [module.nomad_server] triggers = { - script = data.template_file.bootstrap_script.rendered + script = data.template_file.bootstrap_nomad_script.rendered } provisioner "local-exec" { - command = data.template_file.bootstrap_script.rendered + command = data.template_file.bootstrap_nomad_script.rendered } } # write the bootstrap token to the keys/ directory (where the ssh key is) # so that we can read it into the data.local_file later. If not set, # ensure that it's empty. -data "template_file" "bootstrap_script" { - template = var.nomad_acls ? "NOMAD_ADDR=http://${aws_instance.server.0.public_ip}:4646 ./acls/bootstrap-nomad.sh" : "mkdir -p ${path.root}/keys; echo > ${path.root}/keys/nomad_root_token" +data "template_file" "bootstrap_nomad_script" { + template = var.nomad_acls ? "NOMAD_ADDR=http://${aws_instance.server.0.public_ip}:4646 ./scripts/bootstrap-nomad.sh" : "mkdir -p ${path.root}/keys; echo > ${path.root}/keys/nomad_root_token" } data "local_file" "nomad_token" { diff --git a/e2e/terraform/outputs.tf b/e2e/terraform/outputs.tf index e393cf933..860de9123 100644 --- a/e2e/terraform/outputs.tf +++ b/e2e/terraform/outputs.tf @@ -38,7 +38,10 @@ output "environment" { value = < "${DIR}/../keys/nomad_root_token" nomad acl policy apply \ -description "Anonymous policy (full-access)" \ anonymous \ - "${DIR}/anonymous.policy.hcl" + "${DIR}/anonymous.nomad_policy.hcl" diff --git a/e2e/terraform/scripts/bootstrap-vault.sh b/e2e/terraform/scripts/bootstrap-vault.sh new file mode 100755 index 000000000..d4247ba2b --- /dev/null +++ b/e2e/terraform/scripts/bootstrap-vault.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# unseal vault and get a root operator token; the vault is configured to +# autounseal with AWS KMS +while true : +do + ROOT_TOKEN=$(vault operator init -recovery-shares=1 -recovery-threshold=1 | awk '/Initial Root Token/{print $4}') + if [ ! -z $ROOT_TOKEN ]; then break; fi + sleep 5 +done +set -e + +export VAULT_TOKEN="$ROOT_TOKEN" + +mkdir -p ../keys +echo $VAULT_TOKEN > "${DIR}/../keys/vault_root_token" + +# write policies for Nomad to Vault, and then configure Nomad to use the +# token from those policies + +vault policy write nomad-server "${DIR}/vault-nomad-server-policy.hcl" +vault write /auth/token/roles/nomad-cluster "@${DIR}/vault-nomad-cluster-role.json" + +NOMAD_VAULT_TOKEN=$(vault token create -policy nomad-server -period 72h -orphan | awk '/token /{print $2}') + +cat < "${DIR}/../keys/nomad_vault.hcl" +vault { + enabled = true + address = "http://active.vault.service.consul:8200" + task_token_ttl = "1h" + create_from_role = "nomad-cluster" + token = "$NOMAD_VAULT_TOKEN" +} + +EOF diff --git a/e2e/terraform/scripts/vault-nomad-cluster-role.json b/e2e/terraform/scripts/vault-nomad-cluster-role.json new file mode 100644 index 000000000..033ea30c1 --- /dev/null +++ b/e2e/terraform/scripts/vault-nomad-cluster-role.json @@ -0,0 +1,8 @@ +{ + "disallowed_policies": "nomad-server", + "token_explicit_max_ttl": 0, + "name": "nomad-cluster", + "orphan": true, + "token_period": 259200, + "renewable": true +} diff --git a/e2e/terraform/scripts/vault-nomad-server-policy.hcl b/e2e/terraform/scripts/vault-nomad-server-policy.hcl new file mode 100644 index 000000000..d93a537d1 --- /dev/null +++ b/e2e/terraform/scripts/vault-nomad-server-policy.hcl @@ -0,0 +1,41 @@ +# Allow creating tokens under "nomad-cluster" role. The role name should be +# updated if "nomad-cluster" is not used. +path "auth/token/create/nomad-cluster" { + capabilities = ["update"] +} + +# Allow looking up "nomad-cluster" role. The role name should be updated if +# "nomad-cluster" is not used. +path "auth/token/roles/nomad-cluster" { + capabilities = ["read"] +} + +# Allow looking up the token passed to Nomad to validate the token has the +# proper capabilities. This is provided by the "default" policy. +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +# Allow looking up incoming tokens to validate they have permissions to access +# the tokens they are requesting. This is only required if +# `allow_unauthenticated` is set to false. +path "auth/token/lookup" { + capabilities = ["update"] +} + +# Allow revoking tokens that should no longer exist. This allows revoking +# tokens for dead tasks. +path "auth/token/revoke-accessor" { + capabilities = ["update"] +} + +# Allow checking the capabilities of our own token. This is used to validate the +# token upon startup. +path "sys/capabilities-self" { + capabilities = ["update"] +} + +# Allow our own token to be renewed. +path "auth/token/renew-self" { + capabilities = ["update"] +} diff --git a/e2e/terraform/terraform.tfvars b/e2e/terraform/terraform.tfvars index 2625b6521..5107285f5 100644 --- a/e2e/terraform/terraform.tfvars +++ b/e2e/terraform/terraform.tfvars @@ -6,3 +6,4 @@ windows_client_count = "1" profile = "full-cluster" nomad_enterprise = true nomad_acls = true +vault = true diff --git a/e2e/terraform/terraform.tfvars.dev b/e2e/terraform/terraform.tfvars.dev index e5e9cc9f1..dad134ccc 100644 --- a/e2e/terraform/terraform.tfvars.dev +++ b/e2e/terraform/terraform.tfvars.dev @@ -6,6 +6,7 @@ windows_client_count = "0" profile = "dev-cluster" nomad_acls = false nomad_enterprise = false +vault = true # Example overrides: # nomad_local_binary = "../../pkg/linux_amd/nomad" diff --git a/e2e/terraform/variables.tf b/e2e/terraform/variables.tf index d1767b462..9c7bf217c 100644 --- a/e2e/terraform/variables.tf +++ b/e2e/terraform/variables.tf @@ -85,6 +85,12 @@ variable "nomad_acls" { default = false } +variable "vault" { + type = bool + description = "Bootstrap Vault" + default = false +} + # ---------------------------------------- # If you want to deploy multiple versions you can use these variables to # provide a list of builds to override the values of nomad_sha, nomad_version, diff --git a/e2e/terraform/vault.tf b/e2e/terraform/vault.tf new file mode 100644 index 000000000..c21b3174b --- /dev/null +++ b/e2e/terraform/vault.tf @@ -0,0 +1,64 @@ +resource "null_resource" "bootstrap_vault" { + depends_on = [ + aws_instance.server, + module.nomad_server + ] + triggers = { + script = data.template_file.bootstrap_vault_script.rendered + } + + provisioner "local-exec" { + command = data.template_file.bootstrap_vault_script.rendered + } +} + +# write the bootstrap token to the keys/ directory (where the ssh key is) +# so that we can read it into the data.local_file later. If not set, +# ensure that it's empty. +data "template_file" "bootstrap_vault_script" { + template = var.vault ? "VAULT_ADDR=http://${aws_instance.server.0.public_ip}:8200 ./scripts/bootstrap-vault.sh" : "mkdir -p ${path.root}/keys; echo > ${path.root}/keys/vault_root_token" +} + +data "local_file" "vault_token" { + depends_on = [null_resource.bootstrap_vault] + filename = "${path.root}/keys/vault_root_token" +} + +data "local_file" "nomad_vault_config" { + depends_on = [null_resource.bootstrap_vault] + filename = "${path.root}/keys/nomad_vault.hcl" +} + +resource "null_resource" "nomad_vault_config" { + + depends_on = [ + aws_instance.server, + null_resource.bootstrap_vault + ] + + triggers = { + data = data.local_file.nomad_vault_config.content + } + + count = var.server_count + + provisioner "file" { + source = "${path.root}/keys/nomad_vault.hcl" + destination = "./nomad_vault.hcl" + } + + provisioner "remote-exec" { + inline = [ + "sudo mv ./nomad_vault.hcl /etc/nomad.d/nomad_vault.hcl", + "sudo systemctl restart nomad" + ] + } + + connection { + type = "ssh" + user = "ubuntu" + host = aws_instance.server[count.index].public_ip + port = 22 + private_key = file("${path.root}/keys/${local.random_name}.pem") + } +}