From 606e7a6d5437d92df1ea6612fbcefefb62c9d283 Mon Sep 17 00:00:00 2001 From: Rob Genova Date: Tue, 20 Mar 2018 17:46:32 +0000 Subject: [PATCH 1/4] Support dynamic replacement of the Nomad binary at run-time --- terraform/aws/README.md | 18 +++++++++++++++--- terraform/aws/env/us-east/main.tf | 6 ++++++ terraform/aws/env/us-east/terraform.tfvars | 2 +- terraform/aws/env/us-east/user-data-client.sh | 2 +- terraform/aws/env/us-east/user-data-server.sh | 2 +- terraform/aws/modules/hashistack/hashistack.tf | 10 ++++++++++ terraform/shared/scripts/client.sh | 10 ++++++++++ terraform/shared/scripts/server.sh | 10 ++++++++++ 8 files changed, 54 insertions(+), 6 deletions(-) diff --git a/terraform/aws/README.md b/terraform/aws/README.md index 81b0a3ce6..e58b30f7e 100644 --- a/terraform/aws/README.md +++ b/terraform/aws/README.md @@ -43,15 +43,27 @@ a custom AMI: ```bash region = "us-east-1" -ami = "ami-d42d74ae" +ami = "ami-540cd929" instance_type = "t2.medium" key_name = "KEY_NAME" server_count = "3" client_count = "4" ``` -You can also modify the `region`, `instance_type`, `server_count`, and `client_count`. -At least one client and one server are required. +Modify the `region`, `instance_type`, `server_count`, and `client_count` variables +as appropriate. At least one client and one server are required. You can +optionally replace the Nomad binary at runtime by adding the `nomad_binary` +variable like so: + +```bash +region = "us-east-1" +ami = "ami-540cd929" +instance_type = "t2.medium" +key_name = "KEY_NAME" +server_count = "3" +client_count = "4" +nomad_binary = "https://releases.hashicorp.com/nomad/0.7.0/nomad_0.7.0_linux_amd64.zip" +``` Provision the cluster: diff --git a/terraform/aws/env/us-east/main.tf b/terraform/aws/env/us-east/main.tf index a24321e77..6896cf4fc 100644 --- a/terraform/aws/env/us-east/main.tf +++ b/terraform/aws/env/us-east/main.tf @@ -27,6 +27,11 @@ variable "retry_join" { default = "provider=aws tag_key=ConsulAutoJoin tag_value=auto-join" } +variable "nomad_binary" { + description = "Used to replace the machine image installed Nomad binary." + default = "none" +} + provider "aws" { region = "${var.region}" } @@ -41,6 +46,7 @@ module "hashistack" { server_count = "${var.server_count}" client_count = "${var.client_count}" retry_join = "${var.retry_join}" + nomad_binary = "${var.nomad_binary}" } output "IP_Addresses" { diff --git a/terraform/aws/env/us-east/terraform.tfvars b/terraform/aws/env/us-east/terraform.tfvars index bf4b90236..4c374f093 100644 --- a/terraform/aws/env/us-east/terraform.tfvars +++ b/terraform/aws/env/us-east/terraform.tfvars @@ -1,5 +1,5 @@ region = "us-east-1" -ami = "ami-d42d74ae" +ami = "ami-540cd929" instance_type = "t2.medium" key_name = "KEY_NAME" server_count = "3" diff --git a/terraform/aws/env/us-east/user-data-client.sh b/terraform/aws/env/us-east/user-data-client.sh index 74933f00d..383b9e87d 100644 --- a/terraform/aws/env/us-east/user-data-client.sh +++ b/terraform/aws/env/us-east/user-data-client.sh @@ -3,4 +3,4 @@ set -e exec > >(sudo tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 -sudo bash /ops/shared/scripts/client.sh "aws" "${retry_join}" +sudo bash /ops/shared/scripts/client.sh "aws" "${retry_join}" "${nomad_binary}" diff --git a/terraform/aws/env/us-east/user-data-server.sh b/terraform/aws/env/us-east/user-data-server.sh index 0199b405b..c7465295d 100644 --- a/terraform/aws/env/us-east/user-data-server.sh +++ b/terraform/aws/env/us-east/user-data-server.sh @@ -3,4 +3,4 @@ set -e exec > >(sudo tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 -sudo bash /ops/shared/scripts/server.sh "aws" "${server_count}" "${retry_join}" +sudo bash /ops/shared/scripts/server.sh "aws" "${server_count}" "${retry_join}" "${nomad_binary}" diff --git a/terraform/aws/modules/hashistack/hashistack.tf b/terraform/aws/modules/hashistack/hashistack.tf index e5a851503..881ffe965 100644 --- a/terraform/aws/modules/hashistack/hashistack.tf +++ b/terraform/aws/modules/hashistack/hashistack.tf @@ -5,6 +5,7 @@ variable "key_name" {} variable "server_count" {} variable "client_count" {} variable "retry_join" {} +variable "nomad_binary" {} data "aws_vpc" "default" { default = true @@ -83,6 +84,7 @@ data "template_file" "user_data_server" { server_count = "${var.server_count}" region = "${var.region}" retry_join = "${var.retry_join}" + nomad_binary = "${var.nomad_binary}" } } @@ -92,6 +94,7 @@ data "template_file" "user_data_client" { vars { region = "${var.region}" retry_join = "${var.retry_join}" + nomad_binary = "${var.nomad_binary}" } } @@ -126,6 +129,13 @@ resource "aws_instance" "client" { ConsulAutoJoin = "auto-join" } + ebs_block_device = { + device_name = "/dev/xvdd" + volume_type = "gp2" + volume_size = "50" + delete_on_termination = "true" + } + user_data = "${data.template_file.user_data_client.rendered}" iam_instance_profile = "${aws_iam_instance_profile.instance_profile.name}" } diff --git a/terraform/shared/scripts/client.sh b/terraform/shared/scripts/client.sh index 9f84d9286..f46b68a4b 100644 --- a/terraform/shared/scripts/client.sh +++ b/terraform/shared/scripts/client.sh @@ -18,6 +18,7 @@ IP_ADDRESS="$(/sbin/ifconfig eth0 | grep 'inet addr:' | cut -d: -f2 | awk '{ pri DOCKER_BRIDGE_IP_ADDRESS=(`ifconfig docker0 2>/dev/null|awk '/inet addr:/ {print $2}'|sed 's/addr://'`) CLOUD=$1 RETRY_JOIN=$2 +NOMAD_BINARY=$3 # Consul sed -i "s/IP_ADDRESS/$IP_ADDRESS/g" $CONFIGDIR/consul_client.json @@ -29,6 +30,15 @@ sudo systemctl start consul.service sleep 10 # Nomad + +## Replace existing Nomad binary if remote file exists +if [[ `wget -S --spider $NOMAD_BINARY 2>&1 | grep 'HTTP/1.1 200 OK'` ]]; then + curl -L $NOMAD_BINARY > nomad.zip + sudo unzip -o nomad.zip -d /usr/local/bin + sudo chmod 0755 /usr/local/bin/nomad + sudo chown root:root /usr/local/bin/nomad +fi + sudo cp $CONFIGDIR/nomad_client.hcl $NOMADCONFIGDIR/nomad.hcl sudo cp $CONFIGDIR/nomad.service /etc/systemd/system/nomad.service diff --git a/terraform/shared/scripts/server.sh b/terraform/shared/scripts/server.sh index c9b27c391..acd8319dd 100644 --- a/terraform/shared/scripts/server.sh +++ b/terraform/shared/scripts/server.sh @@ -20,6 +20,7 @@ DOCKER_BRIDGE_IP_ADDRESS=(`ifconfig docker0 2>/dev/null|awk '/inet addr:/ {print CLOUD=$1 SERVER_COUNT=$2 RETRY_JOIN=$3 +NOMAD_BINARY=$4 # Consul sed -i "s/IP_ADDRESS/$IP_ADDRESS/g" $CONFIGDIR/consul.json @@ -41,6 +42,15 @@ sudo cp $CONFIGDIR/vault.service /etc/systemd/system/vault.service sudo systemctl start vault.service # Nomad + +## Replace existing Nomad binary if remote file exists +if [[ `wget -S --spider $NOMAD_BINARY 2>&1 | grep 'HTTP/1.1 200 OK'` ]]; then + curl -L $NOMAD_BINARY > nomad.zip + sudo unzip -o nomad.zip -d /usr/local/bin + sudo chmod 0755 /usr/local/bin/nomad + sudo chown root:root /usr/local/bin/nomad +fi + sed -i "s/SERVER_COUNT/$SERVER_COUNT/g" $CONFIGDIR/nomad.hcl sudo cp $CONFIGDIR/nomad.hcl $NOMADCONFIGDIR sudo cp $CONFIGDIR/nomad.service /etc/systemd/system/nomad.service From dae97a97081e545a4c14769079cfc5a1628f2e50 Mon Sep 17 00:00:00 2001 From: Rob Genova Date: Tue, 20 Mar 2018 17:47:24 +0000 Subject: [PATCH 2/4] Bump Hadoop version --- terraform/shared/scripts/setup.sh | 36 ++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/terraform/shared/scripts/setup.sh b/terraform/shared/scripts/setup.sh index da6ade089..e6c777ed8 100644 --- a/terraform/shared/scripts/setup.sh +++ b/terraform/shared/scripts/setup.sh @@ -21,7 +21,7 @@ NOMADDOWNLOAD=https://releases.hashicorp.com/nomad/${NOMADVERSION}/nomad_${NOMAD NOMADCONFIGDIR=/etc/nomad.d NOMADDIR=/opt/nomad -HADOOP_VERSION=2.7.4 +HADOOP_VERSION=2.7.5 # Dependencies sudo apt-get install -y software-properties-common @@ -88,6 +88,40 @@ sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58 sudo apt-get update sudo apt-get install -y docker-engine +# rkt +VERSION=1.27.0 +DOWNLOAD=https://github.com/rkt/rkt/releases/download/v${VERSION}/rkt-v${VERSION}.tar.gz + +function install_rkt() { + wget -q -O /tmp/rkt.tar.gz "${DOWNLOAD}" + tar -C /tmp -xvf /tmp/rkt.tar.gz + sudo mv /tmp/rkt-v${VERSION}/rkt /usr/local/bin + sudo mv /tmp/rkt-v${VERSION}/*.aci /usr/local/bin +} + +function configure_rkt_networking() { + sudo mkdir -p /etc/rkt/net.d + sudo bash -c 'cat << EOT > /etc/rkt/net.d/99-network.conf +{ + "name": "default", + "type": "ptp", + "ipMasq": false, + "ipam": { + "type": "host-local", + "subnet": "172.16.28.0/24", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ] + } +} +EOT' +} + +install_rkt +configure_rkt_networking + # Java sudo add-apt-repository -y ppa:openjdk-r/ppa sudo apt-get update From bfd0967acfccc41acf02643c4851d9a6afefa29b Mon Sep 17 00:00:00 2001 From: Rob Genova Date: Tue, 20 Mar 2018 17:48:14 +0000 Subject: [PATCH 3/4] Enable raw_exec and Docker privileged mode --- terraform/shared/config/nomad_client.hcl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/terraform/shared/config/nomad_client.hcl b/terraform/shared/config/nomad_client.hcl index c9cbb14d0..5149bb636 100644 --- a/terraform/shared/config/nomad_client.hcl +++ b/terraform/shared/config/nomad_client.hcl @@ -4,6 +4,10 @@ bind_addr = "0.0.0.0" # Enable the client client { enabled = true + options { + "driver.raw_exec.enable" = "1" + "docker.privileged.enabled" = "true" + } } consul { From a6d620dcabf4e268ea6c5090b34a2afd9eccd850 Mon Sep 17 00:00:00 2001 From: Rob Genova Date: Tue, 20 Mar 2018 19:55:09 +0000 Subject: [PATCH 4/4] Bump Hadoop version in run-time config scripts; update AMI --- terraform/aws/README.md | 4 ++-- terraform/aws/env/us-east/terraform.tfvars | 2 +- terraform/shared/scripts/client.sh | 2 +- terraform/shared/scripts/server.sh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/terraform/aws/README.md b/terraform/aws/README.md index e58b30f7e..d80e71347 100644 --- a/terraform/aws/README.md +++ b/terraform/aws/README.md @@ -43,7 +43,7 @@ a custom AMI: ```bash region = "us-east-1" -ami = "ami-540cd929" +ami = "ami-3330e54e" instance_type = "t2.medium" key_name = "KEY_NAME" server_count = "3" @@ -57,7 +57,7 @@ variable like so: ```bash region = "us-east-1" -ami = "ami-540cd929" +ami = "ami-3330e54e" instance_type = "t2.medium" key_name = "KEY_NAME" server_count = "3" diff --git a/terraform/aws/env/us-east/terraform.tfvars b/terraform/aws/env/us-east/terraform.tfvars index 4c374f093..1c0b6aa9e 100644 --- a/terraform/aws/env/us-east/terraform.tfvars +++ b/terraform/aws/env/us-east/terraform.tfvars @@ -1,5 +1,5 @@ region = "us-east-1" -ami = "ami-540cd929" +ami = "ami-3330e54e" instance_type = "t2.medium" key_name = "KEY_NAME" server_count = "3" diff --git a/terraform/shared/scripts/client.sh b/terraform/shared/scripts/client.sh index f46b68a4b..64a1f42a1 100644 --- a/terraform/shared/scripts/client.sh +++ b/terraform/shared/scripts/client.sh @@ -6,7 +6,7 @@ CONFIGDIR=/ops/shared/config CONSULCONFIGDIR=/etc/consul.d NOMADCONFIGDIR=/etc/nomad.d -HADOOP_VERSION=hadoop-2.7.4 +HADOOP_VERSION=hadoop-2.7.5 HADOOPCONFIGDIR=/usr/local/$HADOOP_VERSION/etc/hadoop HOME_DIR=ubuntu diff --git a/terraform/shared/scripts/server.sh b/terraform/shared/scripts/server.sh index acd8319dd..87b87911b 100644 --- a/terraform/shared/scripts/server.sh +++ b/terraform/shared/scripts/server.sh @@ -7,7 +7,7 @@ CONFIGDIR=/ops/shared/config CONSULCONFIGDIR=/etc/consul.d VAULTCONFIGDIR=/etc/vault.d NOMADCONFIGDIR=/etc/nomad.d -HADOOP_VERSION=hadoop-2.7.4 +HADOOP_VERSION=hadoop-2.7.5 HADOOPCONFIGDIR=/usr/local/$HADOOP_VERSION/etc/hadoop HOME_DIR=ubuntu