From 782016fc7884c29988b02338becbf3cc25de7a2c Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 3 May 2019 07:52:03 -0700 Subject: [PATCH 1/5] e2e: enable systemd units so they start on boot --- e2e/terraform/compute.tf | 2 ++ e2e/terraform/shared/scripts/client.sh | 7 +++---- e2e/terraform/shared/scripts/server.sh | 6 ++++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/e2e/terraform/compute.tf b/e2e/terraform/compute.tf index e644ac48a..33da7ce52 100644 --- a/e2e/terraform/compute.tf +++ b/e2e/terraform/compute.tf @@ -60,6 +60,7 @@ resource "aws_instance" "server" { "sudo cp /tmp/server.hcl /etc/nomad.d/nomad.hcl", "sudo chmod 0755 /usr/local/bin/nomad", "sudo chown root:root /usr/local/bin/nomad", + "sudo systemctl enable nomad.service", "sudo systemctl start nomad.service" ] @@ -112,6 +113,7 @@ resource "aws_instance" "client" { "sudo cp /tmp/client.hcl /etc/nomad.d/nomad.hcl", "sudo chmod 0755 /usr/local/bin/nomad", "sudo chown root:root /usr/local/bin/nomad", + "sudo systemctl enable nomad.service", "sudo systemctl start nomad.service" ] diff --git a/e2e/terraform/shared/scripts/client.sh b/e2e/terraform/shared/scripts/client.sh index 1230687c4..ebb53e004 100644 --- a/e2e/terraform/shared/scripts/client.sh +++ b/e2e/terraform/shared/scripts/client.sh @@ -25,10 +25,11 @@ sed -i "s/RETRY_JOIN/$RETRY_JOIN/g" $CONFIGDIR/consul_client.json sudo cp $CONFIGDIR/consul_client.json $CONSULCONFIGDIR/consul.json sudo cp $CONFIGDIR/consul_$CLOUD.service /etc/systemd/system/consul.service -sudo systemctl start consul.service +sudo systemctl enable consul.service +sudo systemctl start consul.service sleep 10 -2export NOMAD_ADDR=http://$IP_ADDRESS:4646 +export NOMAD_ADDR=http://$IP_ADDRESS:4646 # Add hostname to /etc/hosts echo "127.0.0.1 $(hostname)" | sudo tee --append /etc/hosts @@ -53,5 +54,3 @@ echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre" | sudo tee --appe # Update PATH echo "export PATH=$PATH:/usr/local/bin/spark/bin:/usr/local/$HADOOP_VERSION/bin" | sudo tee --append /home/$HOME_DIR/.bashrc - - diff --git a/e2e/terraform/shared/scripts/server.sh b/e2e/terraform/shared/scripts/server.sh index ac729941f..172052967 100644 --- a/e2e/terraform/shared/scripts/server.sh +++ b/e2e/terraform/shared/scripts/server.sh @@ -28,7 +28,8 @@ sed -i "s/RETRY_JOIN/$RETRY_JOIN/g" $CONFIGDIR/consul.json sudo cp $CONFIGDIR/consul.json $CONSULCONFIGDIR sudo cp $CONFIGDIR/consul_$CLOUD.service /etc/systemd/system/consul.service -sudo systemctl start consul.service +sudo systemctl enable consul.service +sudo systemctl start consul.service sleep 10 export CONSUL_HTTP_ADDR=$IP_ADDRESS:8500 export CONSUL_RPC_ADDR=$IP_ADDRESS:8400 @@ -38,7 +39,8 @@ sed -i "s/IP_ADDRESS/$IP_ADDRESS/g" $CONFIGDIR/vault.hcl sudo cp $CONFIGDIR/vault.hcl $VAULTCONFIGDIR sudo cp $CONFIGDIR/vault.service /etc/systemd/system/vault.service -sudo systemctl start vault.service +sudo systemctl enable vault.service +sudo systemctl start vault.service export NOMAD_ADDR=http://$IP_ADDRESS:4646 From 10dfbb391e146998a93752595c06a2e508d9c10b Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 3 May 2019 07:53:08 -0700 Subject: [PATCH 2/5] e2e: update deps and install nomad in packer Nomad on the packer image will be overwritten by the sha specified in the TF var, but including a base version on the packer image makes the image valid for independent use. --- e2e/terraform/shared/scripts/setup.sh | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/e2e/terraform/shared/scripts/setup.sh b/e2e/terraform/shared/scripts/setup.sh index 3106d9248..aacc70fbf 100644 --- a/e2e/terraform/shared/scripts/setup.sh +++ b/e2e/terraform/shared/scripts/setup.sh @@ -9,16 +9,19 @@ cd /ops CONFIGDIR=/ops/shared/config -CONSULVERSION=1.4.0 +CONSULVERSION=1.4.4 CONSULDOWNLOAD=https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip CONSULCONFIGDIR=/etc/consul.d CONSULDIR=/opt/consul -VAULTVERSION=0.11.4 +VAULTVERSION=1.1.1 VAULTDOWNLOAD=https://releases.hashicorp.com/vault/${VAULTVERSION}/vault_${VAULTVERSION}_linux_amd64.zip VAULTCONFIGDIR=/etc/vault.d VAULTDIR=/opt/vault +# Will be overwritten by sha specified +NOMADVERSION=0.9.1 +NOMADDOWNLOAD=https://releases.hashicorp.com/nomad/${NOMADVERSION}/nomad_${NOMADVERSION}_linux_amd64.zip NOMADCONFIGDIR=/etc/nomad.d NOMADDIR=/opt/nomad @@ -38,42 +41,37 @@ sudo pip install numpy sudo ufw disable || echo "ufw not installed" -# Consul - +echo "Install Consul" curl -L $CONSULDOWNLOAD > consul.zip - -## Install sudo unzip consul.zip -d /usr/local/bin sudo chmod 0755 /usr/local/bin/consul sudo chown root:root /usr/local/bin/consul -## Configure +echo "Configure Consul" sudo mkdir -p $CONSULCONFIGDIR sudo chmod 755 $CONSULCONFIGDIR sudo mkdir -p $CONSULDIR sudo chmod 755 $CONSULDIR -# Vault - +echo "Install Vault" curl -L $VAULTDOWNLOAD > vault.zip - -## Install sudo unzip vault.zip -d /usr/local/bin sudo chmod 0755 /usr/local/bin/vault sudo chown root:root /usr/local/bin/vault -## Configure +echo "Configure Vault" sudo mkdir -p $VAULTCONFIGDIR sudo chmod 755 $VAULTCONFIGDIR sudo mkdir -p $VAULTDIR sudo chmod 755 $VAULTDIR -## Install +echo "Install Nomad" +curl -L $NOMADDOWNLOAD > nomad.zip sudo unzip nomad.zip -d /usr/local/bin sudo chmod 0755 /usr/local/bin/nomad sudo chown root:root /usr/local/bin/nomad -## Configure +echo "Configure Nomad" sudo mkdir -p $NOMADCONFIGDIR sudo chmod 755 $NOMADCONFIGDIR sudo mkdir -p $NOMADDIR From ac8d50f937a04e1d43a1c7e37f1363313e6192d7 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 3 May 2019 07:54:34 -0700 Subject: [PATCH 3/5] e2e: ssh instructions + remove redundant naming --- e2e/terraform/README.md | 12 ++++++++++-- e2e/terraform/main.tf | 7 ++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/e2e/terraform/README.md b/e2e/terraform/README.md index 02329c502..6a2ca4b13 100644 --- a/e2e/terraform/README.md +++ b/e2e/terraform/README.md @@ -15,8 +15,16 @@ $ envchain nomadaws TF_VAR_nomad_sha= terraform apply After this step, you should have a nomad client address to point the end to end tests in the `e2e` folder to. -Teardown -======== +## SSH + +Terraform will output node IPs that may be accessed via ssh: + +``` +ssh -i keys/nomad-e2e-*.pem ubuntu@${EC2_IP_ADDR} +``` + + +## Teardown The terraform state file stores all the info, so the nomad_sha doesn't need to be valid during teardown. ``` diff --git a/e2e/terraform/main.tf b/e2e/terraform/main.tf index 012887397..73c4dbf3d 100644 --- a/e2e/terraform/main.tf +++ b/e2e/terraform/main.tf @@ -49,7 +49,7 @@ locals { # Generates keys to use for provisioning and access module "keys" { - name = "nomad-e2e-${local.random_name}" + name = "${local.random_name}" path = "${path.root}/keys" source = "mitchellh/dynamic-keys/aws" } @@ -88,5 +88,10 @@ Then you can run e2e tests with: ``` go test -v ./e2e ``` + +ssh into nodes with: +``` +ssh -i keys/${local.random_name}.pem ubuntu@${aws_instance.client.0.public_ip} +``` EOM } From d477ca50ff5d3db34713c0645e16affe0d5185b8 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 3 May 2019 07:54:55 -0700 Subject: [PATCH 4/5] e2e: let the unindex clients do anything... ...and be debugable! --- e2e/terraform/configs/client.hcl | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/e2e/terraform/configs/client.hcl b/e2e/terraform/configs/client.hcl index 847bcaae8..5b267521c 100644 --- a/e2e/terraform/configs/client.hcl +++ b/e2e/terraform/configs/client.hcl @@ -1,11 +1,19 @@ -data_dir = "/opt/nomad/data" -bind_addr = "0.0.0.0" +enable_debug = true +log_level = "DEBUG" +data_dir = "/opt/nomad/data" +bind_addr = "0.0.0.0" # Enable the client client { enabled = true options { + # Allow jobs to run as root + "user.blacklist" = "" + + # Allow rawexec jobs "driver.raw_exec.enable" = "1" + + # Allow privileged docker jobs "docker.privileged.enabled" = "true" } } From 452d9ab26c0a1a48a73239839626f334a38b2609 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 3 May 2019 07:55:28 -0700 Subject: [PATCH 5/5] e2e: explain these scripts are for packer It took me way too long to figure out these weren't used by TF. --- e2e/terraform/shared/scripts/README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 e2e/terraform/shared/scripts/README.md diff --git a/e2e/terraform/shared/scripts/README.md b/e2e/terraform/shared/scripts/README.md new file mode 100644 index 000000000..b0ab5c528 --- /dev/null +++ b/e2e/terraform/shared/scripts/README.md @@ -0,0 +1 @@ +Scripts used by Packer to create base images