refactor build and provisioning scripts to support DNS integration, HDFS

terraform/shared/config/consul.json

@@ -6,7 +6,6 @@
   "client_addr": "IP_ADDRESS",
   "advertise_addr": "IP_ADDRESS",
   "bootstrap_expect": SERVER_COUNT,
-  "node_name": "consul@IP_ADDRESS",
   "service": {
     "name": "consul"
   },

terraform/shared/config/consul_client.json

@@ -2,8 +2,8 @@
   "log_level": "INFO",
   "data_dir": "/opt/consul/data",
   "bind_addr": "0.0.0.0",
+  "client_addr": "0.0.0.0",
   "advertise_addr": "IP_ADDRESS",
-  "node_name": "consul@IP_ADDRESS",
   "retry_join_ec2": {
     "tag_key": "ConsulAutoJoin",
     "tag_value": "CLUSTER_TAG_VALUE",

terraform/shared/config/consul_upstart.conf

@@ -14,9 +14,12 @@ script
 
   # Make sure to use all our CPUs, because Consul can block a scheduler thread
   export GOMAXPROCS=`nproc`
+  export CONSUL_ALLOW_PRIVILEGED_PORTS=true
 
   exec /usr/local/bin/consul agent \
     -config-dir="/etc/consul.d" \
+    -dns-port="53" \
+    -recursor="172.31.0.2" \
     \$${CONSUL_FLAGS} \
     >>/var/log/consul.log 2>&1
 end script

terraform/shared/config/core-site.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+    <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://hdfs.service.consul/</value>
+    </property>
+</configuration>

terraform/shared/scripts/client.sh

@@ -3,9 +3,14 @@
 CONFIGDIR=/ops/shared/config
 CONSULCONFIGDIR=/etc/consul.d
 NOMADCONFIGDIR=/etc/nomad.d
+HADOOP_VERSION=hadoop-2.7.3
+HADOOPCONFIGDIR=/usr/local/$HADOOP_VERSION/etc/hadoop
 HOME_DIR=ubuntu
 
+sleep 15
+
 IP_ADDRESS=$(curl http://instance-data/latest/meta-data/local-ipv4)
+DOCKER_BRIDGE_IP_ADDRESS=(`ifconfig docker0 2>/dev/null|awk '/inet addr:/ {print $2}'|sed 's/addr://'`)
 REGION=$1
 CLUSTER_TAG_VALUE=$2
 
@@ -20,7 +25,7 @@ sudo service consul start
 sleep 10
 
 # Nomad
-sed -i "s/SERVER_IP_ADDRESS/$SERVER_IP_ADDRESS/g" $CONFIGDIR/nomad_client.hcl
+# sed -i "s/SERVER_IP_ADDRESS/$SERVER_IP_ADDRESS/g" $CONFIGDIR/nomad_client.hcl
 sed -i "s/IP_ADDRESS/$IP_ADDRESS/g" $CONFIGDIR/nomad_client.hcl
 sudo cp $CONFIGDIR/nomad_client.hcl $NOMADCONFIGDIR/nomad.hcl
 sudo cp $CONFIGDIR/nomad_upstart.conf /etc/init/nomad.conf
@@ -29,6 +34,19 @@ sudo service nomad start
 sleep 10
 export NOMAD_ADDR=http://$IP_ADDRESS:4646
 
+# Add hostname to /etc/hosts
+
+echo "127.0.0.1 $(hostname)" | sudo tee --append /etc/hosts
+
+# Add Docker bridge network IP to /etc/resolv.conf (at the top)
+
+echo "nameserver $DOCKER_BRIDGE_IP_ADDRESS" | sudo tee /etc/resolv.conf.new
+cat /etc/resolv.conf | sudo tee --append /etc/resolv.conf.new
+sudo mv /etc/resolv.conf.new /etc/resolv.conf
+
+# Hadoop
+sudo cp $CONFIGDIR/core-site.xml $HADOOPCONFIGDIR
+
 # Set env vars in bashrc
 
 echo "export VAULT_ADDR=http://$IP_ADDRESS:8200" | sudo tee --append /home/$HOME_DIR/.bashrc
@@ -41,5 +59,8 @@ sudo mv /ops/examples /home/$HOME_DIR
 sudo chown -R $HOME_DIR:$HOME_DIR /home/$HOME_DIR/examples
 sudo chmod -R 775 /home/$HOME_DIR/examples
 
-# Copy transcode.sh to /usr/bin
-# sudo cp /home/$HOME_DIR/examples/nomad/dispatch/bin/transcode.sh /usr/bin/transcode.sh
+# Update PATH in .bashrc
+
+echo "export PATH=$PATH:/usr/local/bin/spark/bin:/usr/local/$HADOOP_VERSION/bin" | sudo tee --append /home/$HOME_DIR/.bashrc
+
+

terraform/shared/scripts/server.sh

@@ -4,6 +4,8 @@ CONFIGDIR=/ops/shared/config
 CONSULCONFIGDIR=/etc/consul.d
 VAULTCONFIGDIR=/etc/vault.d
 NOMADCONFIGDIR=/etc/nomad.d
+HADOOP_VERSION=hadoop-2.7.3
+HADOOPCONFIGDIR=/usr/local/$HADOOP_VERSION/etc/hadoop
 HOME_DIR=ubuntu
 
 IP_ADDRESS=$(curl http://instance-data/latest/meta-data/local-ipv4)
@@ -41,13 +43,16 @@ sudo service nomad start
 sleep 10
 export NOMAD_ADDR=http://$IP_ADDRESS:4646
 
+# Hadoop
+sudo cp $CONFIGDIR/core-site.xml $HADOOPCONFIGDIR
+
+# Set env vars for tool CLIs
 echo "export CONSUL_RPC_ADDR=$IP_ADDRESS:8400" | sudo tee --append /home/$HOME_DIR/.bashrc
 echo "export CONSUL_HTTP_ADDR=$IP_ADDRESS:8500" | sudo tee --append /home/$HOME_DIR/.bashrc
 echo "export VAULT_ADDR=http://$IP_ADDRESS:8200" | sudo tee --append /home/$HOME_DIR/.bashrc
 echo "export NOMAD_ADDR=http://$IP_ADDRESS:4646" | sudo tee --append /home/$HOME_DIR/.bashrc
 echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre"  | sudo tee --append /home/$HOME_DIR/.bashrc
 
-# Move examples directory to $HOME
-sudo mv /ops/examples /home/$HOME_DIR
-sudo chown -R $HOME_DIR:$HOME_DIR /home/$HOME_DIR/examples
+# Update PATH in .bashrc
 
+echo "export PATH=$PATH:/usr/local/bin/spark/bin:/usr/local/$HADOOP_VERSION/bin" | sudo tee --append /home/$HOME_DIR/.bashrc

terraform/shared/scripts/setup.sh

@@ -22,20 +22,17 @@ NOMADDIR=/opt/nomad
 
 echo Dependencies...
 sudo apt-get install -y software-properties-common
-sudo add-apt-repository ppa:mc3man/trusty-media
 sudo apt-get update
-sudo apt-get install -y unzip tree redis-tools jq s3cmd ffmpeg
+sudo apt-get install -y unzip tree redis-tools jq
+sudo apt-get install -y upstart-sysv
+sudo update-initramfs -u
 
-# Numpy
+# Numpy (for Spark)
 
 sudo apt-get install -y python-setuptools
 sudo easy_install pip
 sudo pip install numpy
 
-# Instead of symlink, move ffmpeg to be inside the chroot for Nomad
-sudo rm /usr/bin/ffmpeg
-sudo cp /opt/ffmpeg/bin/ffmpeg /usr/bin/ffmpeg
-
 # Disable the firewall
 
 sudo ufw disable
@@ -106,6 +103,10 @@ JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::")
 
 sudo wget -P /ops/examples/spark https://s3.amazonaws.com/rcgenova-nomad-spark/spark-2.1.0-bin-nomad-preview-6.tgz
 sudo tar -xvf /ops/examples/spark/spark-2.1.0-bin-nomad-preview-6.tgz --directory /ops/examples/spark
-sudo mv /ops/examples/spark/spark-2.1.0-bin-nomad-preview-6 /ops/examples/spark/spark
-sudo rm /ops/examples/spark/spark-2.1.0-bin-nomad-preview-6.tgz
+sudo mv /ops/examples/spark/spark-2.1.0-bin-nomad-preview-6 /usr/local/bin/spark
+sudo chown -R root:root /usr/local/bin/spark
 
+## Install Hadoop to enable the HDFS CLI
+
+export HADOOP_VERSION=2.7.3
+wget -O - http://apache.mirror.iphh.net/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz | sudo tar xz -C /usr/local/