From 547e7faae65072d08500cda8478c2acd2f3d0460 Mon Sep 17 00:00:00 2001 From: Rob Genova Date: Sat, 24 Jun 2017 16:25:46 -0700 Subject: [PATCH] refactor build and provisioning scripts to support DNS integration, HDFS --- terraform/shared/config/consul.json | 1 - terraform/shared/config/consul_client.json | 2 +- terraform/shared/config/consul_upstart.conf | 3 +++ terraform/shared/config/core-site.xml | 8 ++++++ terraform/shared/scripts/client.sh | 27 ++++++++++++++++++--- terraform/shared/scripts/server.sh | 11 ++++++--- terraform/shared/scripts/setup.sh | 19 ++++++++------- 7 files changed, 54 insertions(+), 17 deletions(-) create mode 100644 terraform/shared/config/core-site.xml diff --git a/terraform/shared/config/consul.json b/terraform/shared/config/consul.json index 09de947eb..6a99c1b49 100644 --- a/terraform/shared/config/consul.json +++ b/terraform/shared/config/consul.json @@ -6,7 +6,6 @@ "client_addr": "IP_ADDRESS", "advertise_addr": "IP_ADDRESS", "bootstrap_expect": SERVER_COUNT, - "node_name": "consul@IP_ADDRESS", "service": { "name": "consul" }, diff --git a/terraform/shared/config/consul_client.json b/terraform/shared/config/consul_client.json index e43218651..c122340c3 100644 --- a/terraform/shared/config/consul_client.json +++ b/terraform/shared/config/consul_client.json @@ -2,8 +2,8 @@ "log_level": "INFO", "data_dir": "/opt/consul/data", "bind_addr": "0.0.0.0", + "client_addr": "0.0.0.0", "advertise_addr": "IP_ADDRESS", - "node_name": "consul@IP_ADDRESS", "retry_join_ec2": { "tag_key": "ConsulAutoJoin", "tag_value": "CLUSTER_TAG_VALUE", diff --git a/terraform/shared/config/consul_upstart.conf b/terraform/shared/config/consul_upstart.conf index 6a6fe8d73..3d73a1667 100644 --- a/terraform/shared/config/consul_upstart.conf +++ b/terraform/shared/config/consul_upstart.conf @@ -14,9 +14,12 @@ script # Make sure to use all our CPUs, because Consul can block a scheduler thread export GOMAXPROCS=`nproc` + export CONSUL_ALLOW_PRIVILEGED_PORTS=true exec /usr/local/bin/consul agent \ -config-dir="/etc/consul.d" \ + -dns-port="53" \ + -recursor="172.31.0.2" \ \$${CONSUL_FLAGS} \ >>/var/log/consul.log 2>&1 end script diff --git a/terraform/shared/config/core-site.xml b/terraform/shared/config/core-site.xml new file mode 100644 index 000000000..75360fcc2 --- /dev/null +++ b/terraform/shared/config/core-site.xml @@ -0,0 +1,8 @@ + + + + + fs.defaultFS + hdfs://hdfs.service.consul/ + + diff --git a/terraform/shared/scripts/client.sh b/terraform/shared/scripts/client.sh index 12e4fe866..04166e14a 100644 --- a/terraform/shared/scripts/client.sh +++ b/terraform/shared/scripts/client.sh @@ -3,9 +3,14 @@ CONFIGDIR=/ops/shared/config CONSULCONFIGDIR=/etc/consul.d NOMADCONFIGDIR=/etc/nomad.d +HADOOP_VERSION=hadoop-2.7.3 +HADOOPCONFIGDIR=/usr/local/$HADOOP_VERSION/etc/hadoop HOME_DIR=ubuntu +sleep 15 + IP_ADDRESS=$(curl http://instance-data/latest/meta-data/local-ipv4) +DOCKER_BRIDGE_IP_ADDRESS=(`ifconfig docker0 2>/dev/null|awk '/inet addr:/ {print $2}'|sed 's/addr://'`) REGION=$1 CLUSTER_TAG_VALUE=$2 @@ -20,7 +25,7 @@ sudo service consul start sleep 10 # Nomad -sed -i "s/SERVER_IP_ADDRESS/$SERVER_IP_ADDRESS/g" $CONFIGDIR/nomad_client.hcl +# sed -i "s/SERVER_IP_ADDRESS/$SERVER_IP_ADDRESS/g" $CONFIGDIR/nomad_client.hcl sed -i "s/IP_ADDRESS/$IP_ADDRESS/g" $CONFIGDIR/nomad_client.hcl sudo cp $CONFIGDIR/nomad_client.hcl $NOMADCONFIGDIR/nomad.hcl sudo cp $CONFIGDIR/nomad_upstart.conf /etc/init/nomad.conf @@ -29,6 +34,19 @@ sudo service nomad start sleep 10 export NOMAD_ADDR=http://$IP_ADDRESS:4646 +# Add hostname to /etc/hosts + +echo "127.0.0.1 $(hostname)" | sudo tee --append /etc/hosts + +# Add Docker bridge network IP to /etc/resolv.conf (at the top) + +echo "nameserver $DOCKER_BRIDGE_IP_ADDRESS" | sudo tee /etc/resolv.conf.new +cat /etc/resolv.conf | sudo tee --append /etc/resolv.conf.new +sudo mv /etc/resolv.conf.new /etc/resolv.conf + +# Hadoop +sudo cp $CONFIGDIR/core-site.xml $HADOOPCONFIGDIR + # Set env vars in bashrc echo "export VAULT_ADDR=http://$IP_ADDRESS:8200" | sudo tee --append /home/$HOME_DIR/.bashrc @@ -41,5 +59,8 @@ sudo mv /ops/examples /home/$HOME_DIR sudo chown -R $HOME_DIR:$HOME_DIR /home/$HOME_DIR/examples sudo chmod -R 775 /home/$HOME_DIR/examples -# Copy transcode.sh to /usr/bin -# sudo cp /home/$HOME_DIR/examples/nomad/dispatch/bin/transcode.sh /usr/bin/transcode.sh +# Update PATH in .bashrc + +echo "export PATH=$PATH:/usr/local/bin/spark/bin:/usr/local/$HADOOP_VERSION/bin" | sudo tee --append /home/$HOME_DIR/.bashrc + + diff --git a/terraform/shared/scripts/server.sh b/terraform/shared/scripts/server.sh index 6598c87b9..7a960464d 100644 --- a/terraform/shared/scripts/server.sh +++ b/terraform/shared/scripts/server.sh @@ -4,6 +4,8 @@ CONFIGDIR=/ops/shared/config CONSULCONFIGDIR=/etc/consul.d VAULTCONFIGDIR=/etc/vault.d NOMADCONFIGDIR=/etc/nomad.d +HADOOP_VERSION=hadoop-2.7.3 +HADOOPCONFIGDIR=/usr/local/$HADOOP_VERSION/etc/hadoop HOME_DIR=ubuntu IP_ADDRESS=$(curl http://instance-data/latest/meta-data/local-ipv4) @@ -41,13 +43,16 @@ sudo service nomad start sleep 10 export NOMAD_ADDR=http://$IP_ADDRESS:4646 +# Hadoop +sudo cp $CONFIGDIR/core-site.xml $HADOOPCONFIGDIR + +# Set env vars for tool CLIs echo "export CONSUL_RPC_ADDR=$IP_ADDRESS:8400" | sudo tee --append /home/$HOME_DIR/.bashrc echo "export CONSUL_HTTP_ADDR=$IP_ADDRESS:8500" | sudo tee --append /home/$HOME_DIR/.bashrc echo "export VAULT_ADDR=http://$IP_ADDRESS:8200" | sudo tee --append /home/$HOME_DIR/.bashrc echo "export NOMAD_ADDR=http://$IP_ADDRESS:4646" | sudo tee --append /home/$HOME_DIR/.bashrc echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre" | sudo tee --append /home/$HOME_DIR/.bashrc -# Move examples directory to $HOME -sudo mv /ops/examples /home/$HOME_DIR -sudo chown -R $HOME_DIR:$HOME_DIR /home/$HOME_DIR/examples +# Update PATH in .bashrc +echo "export PATH=$PATH:/usr/local/bin/spark/bin:/usr/local/$HADOOP_VERSION/bin" | sudo tee --append /home/$HOME_DIR/.bashrc diff --git a/terraform/shared/scripts/setup.sh b/terraform/shared/scripts/setup.sh index 4fe83d8fc..1c7abadb3 100644 --- a/terraform/shared/scripts/setup.sh +++ b/terraform/shared/scripts/setup.sh @@ -22,20 +22,17 @@ NOMADDIR=/opt/nomad echo Dependencies... sudo apt-get install -y software-properties-common -sudo add-apt-repository ppa:mc3man/trusty-media sudo apt-get update -sudo apt-get install -y unzip tree redis-tools jq s3cmd ffmpeg +sudo apt-get install -y unzip tree redis-tools jq +sudo apt-get install -y upstart-sysv +sudo update-initramfs -u -# Numpy +# Numpy (for Spark) sudo apt-get install -y python-setuptools sudo easy_install pip sudo pip install numpy -# Instead of symlink, move ffmpeg to be inside the chroot for Nomad -sudo rm /usr/bin/ffmpeg -sudo cp /opt/ffmpeg/bin/ffmpeg /usr/bin/ffmpeg - # Disable the firewall sudo ufw disable @@ -106,6 +103,10 @@ JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::") sudo wget -P /ops/examples/spark https://s3.amazonaws.com/rcgenova-nomad-spark/spark-2.1.0-bin-nomad-preview-6.tgz sudo tar -xvf /ops/examples/spark/spark-2.1.0-bin-nomad-preview-6.tgz --directory /ops/examples/spark -sudo mv /ops/examples/spark/spark-2.1.0-bin-nomad-preview-6 /ops/examples/spark/spark -sudo rm /ops/examples/spark/spark-2.1.0-bin-nomad-preview-6.tgz +sudo mv /ops/examples/spark/spark-2.1.0-bin-nomad-preview-6 /usr/local/bin/spark +sudo chown -R root:root /usr/local/bin/spark +## Install Hadoop to enable the HDFS CLI + +export HADOOP_VERSION=2.7.3 +wget -O - http://apache.mirror.iphh.net/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz | sudo tar xz -C /usr/local/