Merge pull request #5647 from hashicorp/e2e-tf

E2E Test Terraform/Packer Improvements

e2e/terraform/README.md

@@ -15,8 +15,16 @@ $ envchain nomadaws TF_VAR_nomad_sha=<nomad_sha> terraform apply
 
 After this step, you should have a nomad client address to point the end to end tests in the `e2e` folder to.
 
-Teardown
-========
+## SSH
+
+Terraform will output node IPs that may be accessed via ssh:
+
+```
+ssh -i keys/nomad-e2e-*.pem ubuntu@${EC2_IP_ADDR}
+```
+
+
+## Teardown
 The terraform state file stores all the info, so the nomad_sha doesn't need to be valid during teardown. 
 
 ```

e2e/terraform/compute.tf

@@ -60,6 +60,7 @@ resource "aws_instance" "server" {
       "sudo cp /tmp/server.hcl /etc/nomad.d/nomad.hcl",
       "sudo chmod 0755 /usr/local/bin/nomad",
       "sudo chown root:root /usr/local/bin/nomad",
+      "sudo systemctl enable nomad.service",
       "sudo systemctl start nomad.service"
     ]
 
@@ -112,6 +113,7 @@ resource "aws_instance" "client" {
       "sudo cp /tmp/client.hcl /etc/nomad.d/nomad.hcl",
       "sudo chmod 0755 /usr/local/bin/nomad",
       "sudo chown root:root /usr/local/bin/nomad",
+			"sudo systemctl enable nomad.service",
       "sudo systemctl start nomad.service"
     ]
 

e2e/terraform/configs/client.hcl

@@ -1,11 +1,19 @@
-data_dir = "/opt/nomad/data"
-bind_addr = "0.0.0.0"
+enable_debug = true
+log_level    = "DEBUG"
+data_dir     = "/opt/nomad/data"
+bind_addr    = "0.0.0.0"
 
 # Enable the client
 client {
   enabled = true
   options {
+    # Allow jobs to run as root
+    "user.blacklist" = ""
+
+    # Allow rawexec jobs
     "driver.raw_exec.enable" = "1"
+
+    # Allow privileged docker jobs
     "docker.privileged.enabled" = "true"
   }
 }

e2e/terraform/main.tf

@@ -49,7 +49,7 @@ locals {
 
 # Generates keys to use for provisioning and access
 module "keys" {
-  name   = "nomad-e2e-${local.random_name}"
+  name   = "${local.random_name}"
   path   = "${path.root}/keys"
   source = "mitchellh/dynamic-keys/aws"
 }
@@ -88,5 +88,10 @@ Then you can run e2e tests with:
 ```
 go test -v ./e2e
 ```
+
+ssh into nodes with:
+```
+ssh -i keys/${local.random_name}.pem ubuntu@${aws_instance.client.0.public_ip}
+```
 EOM
 }

e2e/terraform/shared/scripts/README.md

@@ -0,0 +1 @@
+Scripts used by Packer to create base images

e2e/terraform/shared/scripts/client.sh

@@ -25,10 +25,11 @@ sed -i "s/RETRY_JOIN/$RETRY_JOIN/g" $CONFIGDIR/consul_client.json
 sudo cp $CONFIGDIR/consul_client.json $CONSULCONFIGDIR/consul.json
 sudo cp $CONFIGDIR/consul_$CLOUD.service /etc/systemd/system/consul.service
 
-sudo systemctl start consul.service
+sudo systemctl enable consul.service
+sudo systemctl start  consul.service
 sleep 10
 
-2export NOMAD_ADDR=http://$IP_ADDRESS:4646
+export NOMAD_ADDR=http://$IP_ADDRESS:4646
 
 # Add hostname to /etc/hosts
 echo "127.0.0.1 $(hostname)" | sudo tee --append /etc/hosts
@@ -53,5 +54,3 @@ echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre"  | sudo tee --appe
 
 # Update PATH
 echo "export PATH=$PATH:/usr/local/bin/spark/bin:/usr/local/$HADOOP_VERSION/bin" | sudo tee --append /home/$HOME_DIR/.bashrc
-
-

e2e/terraform/shared/scripts/server.sh

@@ -28,7 +28,8 @@ sed -i "s/RETRY_JOIN/$RETRY_JOIN/g" $CONFIGDIR/consul.json
 sudo cp $CONFIGDIR/consul.json $CONSULCONFIGDIR
 sudo cp $CONFIGDIR/consul_$CLOUD.service /etc/systemd/system/consul.service
 
-sudo systemctl start consul.service
+sudo systemctl enable consul.service
+sudo systemctl start  consul.service
 sleep 10
 export CONSUL_HTTP_ADDR=$IP_ADDRESS:8500
 export CONSUL_RPC_ADDR=$IP_ADDRESS:8400
@@ -38,7 +39,8 @@ sed -i "s/IP_ADDRESS/$IP_ADDRESS/g" $CONFIGDIR/vault.hcl
 sudo cp $CONFIGDIR/vault.hcl $VAULTCONFIGDIR
 sudo cp $CONFIGDIR/vault.service /etc/systemd/system/vault.service
 
-sudo systemctl start vault.service
+sudo systemctl enable vault.service
+sudo systemctl start  vault.service
 
 export NOMAD_ADDR=http://$IP_ADDRESS:4646
 

e2e/terraform/shared/scripts/setup.sh

@@ -9,16 +9,19 @@ cd /ops
 
 CONFIGDIR=/ops/shared/config
 
-CONSULVERSION=1.4.0
+CONSULVERSION=1.4.4
 CONSULDOWNLOAD=https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip
 CONSULCONFIGDIR=/etc/consul.d
 CONSULDIR=/opt/consul
 
-VAULTVERSION=0.11.4
+VAULTVERSION=1.1.1
 VAULTDOWNLOAD=https://releases.hashicorp.com/vault/${VAULTVERSION}/vault_${VAULTVERSION}_linux_amd64.zip
 VAULTCONFIGDIR=/etc/vault.d
 VAULTDIR=/opt/vault
 
+# Will be overwritten by sha specified
+NOMADVERSION=0.9.1
+NOMADDOWNLOAD=https://releases.hashicorp.com/nomad/${NOMADVERSION}/nomad_${NOMADVERSION}_linux_amd64.zip
 NOMADCONFIGDIR=/etc/nomad.d
 NOMADDIR=/opt/nomad
 
@@ -38,42 +41,37 @@ sudo pip install numpy
 
 sudo ufw disable || echo "ufw not installed"
 
-# Consul
-
+echo "Install Consul"
 curl -L $CONSULDOWNLOAD > consul.zip
-
-## Install
 sudo unzip consul.zip -d /usr/local/bin
 sudo chmod 0755 /usr/local/bin/consul
 sudo chown root:root /usr/local/bin/consul
 
-## Configure
+echo "Configure Consul"
 sudo mkdir -p $CONSULCONFIGDIR
 sudo chmod 755 $CONSULCONFIGDIR
 sudo mkdir -p $CONSULDIR
 sudo chmod 755 $CONSULDIR
 
-# Vault
-
+echo "Install Vault"
 curl -L $VAULTDOWNLOAD > vault.zip
-
-## Install
 sudo unzip vault.zip -d /usr/local/bin
 sudo chmod 0755 /usr/local/bin/vault
 sudo chown root:root /usr/local/bin/vault
 
-## Configure
+echo "Configure Vault"
 sudo mkdir -p $VAULTCONFIGDIR
 sudo chmod 755 $VAULTCONFIGDIR
 sudo mkdir -p $VAULTDIR
 sudo chmod 755 $VAULTDIR
 
-## Install
+echo "Install Nomad"
+curl -L $NOMADDOWNLOAD > nomad.zip
 sudo unzip nomad.zip -d /usr/local/bin
 sudo chmod 0755 /usr/local/bin/nomad
 sudo chown root:root /usr/local/bin/nomad
 
-## Configure
+echo "Configure Nomad"
 sudo mkdir -p $NOMADCONFIGDIR
 sudo chmod 755 $NOMADCONFIGDIR
 sudo mkdir -p $NOMADDIR