mirror of
https://github.com/kemko/nomad.git
synced 2026-01-01 16:05:42 +03:00
build: Rework Vagrant to support multiple OS boxes
This commit reworks the Vagrantfile for Nomad in order to support straightforward testing on more than one operating system, whilst retaining the ability to stand up a test cluster running Ubuntu. The following changes are made: - Scripts have been extracted from the Vagrantfile into their own shell script files, in order that editors lint them. - All scripts have been edited to lint with no warnings or errors for their respective shells. - Scripts are named according to the operating system and privilege level which they run. We prefer to run a whole shell script as root versus prefixing (essentially) every command with `sudo` or an equivalent. - The Linux development box has been separated from the test cluster, removing some of the more gnarly (and less portable) logic. The Linux development box is still primary and autostarts. - A FreeBSD target has been added. The base box works for both Virtualbox and VMWare Fusion. - A target is added to the GNUmakefile to stand up a test cluster, using the default provider, or overriding the provider by setting the PROVIDER variable in make: - `make testcluster` - `make testcluster PROVIDER=vmware_fusion` - Machines in the test cluster have Avahi configured for zeroconf discovery. Each machine can ping each other machine at `hostname.local` - for example `nomad-server02.local`, `nomad-client03.local`.
This commit is contained in:
James Nugent
259
Vagrantfile
vendored
259
Vagrantfile
vendored
@@ -1,155 +1,146 @@
|
||||
# -*- mode: ruby -*-
|
||||
# vi: set ft=ruby :
|
||||
#
|
||||
|
||||
# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
|
||||
VAGRANTFILE_API_VERSION = "2"
|
||||
LINUX_BASE_BOX = "bento/ubuntu-16.04"
|
||||
FREEBSD_BASE_BOX = "jen20/FreeBSD-11.1-RELEASE"
|
||||
|
||||
DEFAULT_CPU_COUNT = 2
|
||||
$script = <<SCRIPT
|
||||
GO_VERSION="1.9"
|
||||
Vagrant.configure(2) do |config|
|
||||
# Compilation and development boxes
|
||||
config.vm.define "linux", autostart: true, primary: true do |vmCfg|
|
||||
vmCfg.vm.box = LINUX_BASE_BOX
|
||||
vmCfg.vm.hostname = "linux"
|
||||
vmCfg = configureProviders vmCfg,
|
||||
cpus: suggestedCPUCores()
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
vmCfg = configureLinuxProvisioners(vmCfg)
|
||||
|
||||
sudo dpkg --add-architecture i386
|
||||
sudo apt-get update
|
||||
vmCfg.vm.synced_folder '.',
|
||||
'/opt/gopath/src/github.com/hashicorp/nomad'
|
||||
|
||||
# Install base dependencies
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential curl git-core mercurial bzr \
|
||||
libpcre3-dev pkg-config zip default-jre qemu silversearcher-ag \
|
||||
jq htop vim unzip tree \
|
||||
liblxc1 lxc-dev lxc-templates \
|
||||
gcc-5-aarch64-linux-gnu binutils-aarch64-linux-gnu \
|
||||
libc6-dev-i386 linux-libc-dev:i386 \
|
||||
gcc-5-arm-linux-gnueabihf gcc-5-multilib-arm-linux-gnueabihf binutils-arm-linux-gnueabihf \
|
||||
gcc-mingw-w64 binutils-mingw-w64
|
||||
vmCfg.vm.provision "shell",
|
||||
privileged: true,
|
||||
path: './scripts/vagrant-linux-unpriv-bootstrap.sh'
|
||||
end
|
||||
|
||||
# Setup go, for development of Nomad
|
||||
SRCROOT="/opt/go"
|
||||
SRCPATH="/opt/gopath"
|
||||
config.vm.define "freebsd", autostart: false, primary: false do |vmCfg|
|
||||
vmCfg.vm.box = FREEBSD_BASE_BOX
|
||||
vmCfg.vm.hostname = "freebsd"
|
||||
vmCfg = configureProviders vmCfg,
|
||||
cpus: suggestedCPUCores()
|
||||
|
||||
# Get the ARCH
|
||||
ARCH=`uname -m | sed 's|i686|386|' | sed 's|x86_64|amd64|'`
|
||||
vmCfg.vm.synced_folder '.',
|
||||
'/opt/gopath/src/github.com/hashicorp/nomad',
|
||||
type: "nfs",
|
||||
bsd__nfs_options: ['noatime']
|
||||
|
||||
# Install Go
|
||||
if [[ $(go version) == "go version go${GO_VERSION} linux/${ARCH}" ]]; then
|
||||
echo "Go ${GO_VERSION} ${ARCH} already installed; Skipping"
|
||||
else
|
||||
cd /tmp
|
||||
wget -q https://storage.googleapis.com/golang/go${GO_VERSION}.linux-${ARCH}.tar.gz
|
||||
tar -xf go${GO_VERSION}.linux-${ARCH}.tar.gz
|
||||
sudo rm -rf $SRCROOT
|
||||
sudo mv go $SRCROOT
|
||||
sudo chmod 775 $SRCROOT
|
||||
sudo chown vagrant:vagrant $SRCROOT
|
||||
fi
|
||||
vmCfg.vm.provision "shell",
|
||||
privileged: true,
|
||||
path: './scripts/vagrant-freebsd-priv-config.sh'
|
||||
|
||||
# Setup the GOPATH; even though the shared folder spec gives the working
|
||||
# directory the right user/group, we need to set it properly on the
|
||||
# parent path to allow subsequent "go get" commands to work.
|
||||
sudo mkdir -p $SRCPATH
|
||||
sudo chown -R vagrant:vagrant $SRCPATH 2>/dev/null || true
|
||||
# ^^ silencing errors here because we expect this to fail for the shared folder
|
||||
vmCfg.vm.provision "shell",
|
||||
privileged: false,
|
||||
path: './scripts/vagrant-freebsd-unpriv-bootstrap.sh'
|
||||
end
|
||||
|
||||
cat <<EOF >/tmp/gopath.sh
|
||||
export GOPATH="$SRCPATH"
|
||||
export GOROOT="$SRCROOT"
|
||||
export PATH="$SRCROOT/bin:$SRCPATH/bin:\$PATH"
|
||||
EOF
|
||||
sudo mv /tmp/gopath.sh /etc/profile.d/gopath.sh
|
||||
sudo chmod 0755 /etc/profile.d/gopath.sh
|
||||
source /etc/profile.d/gopath.sh
|
||||
# Test Cluster (Linux)
|
||||
1.upto(3) do |n|
|
||||
serverName = "nomad-server%02d" % [n]
|
||||
clientName = "nomad-client%02d" % [n]
|
||||
serverIP = "10.199.0.%d" % [10 + n]
|
||||
clientIP = "10.199.0.%d" % [20 + n]
|
||||
|
||||
# Install Docker
|
||||
if [[ -f /etc/apt/sources.list.d/docker.list ]]; then
|
||||
echo "Docker repository already installed; Skipping"
|
||||
else
|
||||
echo deb https://apt.dockerproject.org/repo ubuntu-`lsb_release -c | awk '{print $2}'` main | sudo tee /etc/apt/sources.list.d/docker.list
|
||||
sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
|
||||
sudo apt-get update
|
||||
fi
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y docker-engine
|
||||
config.vm.define serverName, autostart: false, primary: false do |vmCfg|
|
||||
vmCfg.vm.box = LINUX_BASE_BOX
|
||||
vmCfg.vm.hostname = serverName
|
||||
vmCfg = configureProviders(vmCfg)
|
||||
vmCfg = configureLinuxProvisioners(vmCfg)
|
||||
|
||||
# Restart docker to make sure we get the latest version of the daemon if there is an upgrade
|
||||
sudo service docker restart
|
||||
vmCfg.vm.provider "virtualbox" do |_|
|
||||
vmCfg.vm.network :private_network, ip: serverIP
|
||||
end
|
||||
|
||||
vmCfg.vm.synced_folder '.',
|
||||
'/opt/gopath/src/github.com/hashicorp/nomad'
|
||||
|
||||
vmCfg.vm.provision "shell",
|
||||
privileged: true,
|
||||
path: './scripts/vagrant-linux-priv-zeroconf.sh'
|
||||
end
|
||||
|
||||
config.vm.define clientName, autostart: false, primary: false do |vmCfg|
|
||||
vmCfg.vm.box = LINUX_BASE_BOX
|
||||
vmCfg.vm.hostname = clientName
|
||||
vmCfg = configureProviders(vmCfg)
|
||||
vmCfg = configureLinuxProvisioners(vmCfg)
|
||||
|
||||
vmCfg.vm.provider "virtualbox" do |_|
|
||||
vmCfg.vm.network :private_network, ip: clientIP
|
||||
end
|
||||
|
||||
vmCfg.vm.synced_folder '.',
|
||||
'/opt/gopath/src/github.com/hashicorp/nomad'
|
||||
|
||||
# Make sure we can actually use docker as the vagrant user
|
||||
sudo usermod -aG docker vagrant
|
||||
|
||||
# Setup Nomad for development
|
||||
cd /opt/gopath/src/github.com/hashicorp/nomad && make bootstrap
|
||||
|
||||
# Install rkt, consul and vault
|
||||
bash scripts/install_rkt.sh
|
||||
bash scripts/install_rkt_vagrant.sh
|
||||
bash scripts/install_consul.sh
|
||||
bash scripts/install_vault.sh
|
||||
|
||||
# Set hostname's IP to made advertisement Just Work
|
||||
sudo sed -i -e "s/.*nomad.*/$(ip route get 1 | awk '{print $NF;exit}') $(hostname)/" /etc/hosts
|
||||
|
||||
# CD into the nomad working directory when we login to the VM
|
||||
grep "cd /opt/gopath/src/github.com/hashicorp/nomad" ~/.profile || echo "cd /opt/gopath/src/github.com/hashicorp/nomad" >> ~/.profile
|
||||
SCRIPT
|
||||
|
||||
def configureVM(vmCfg, vmParams={
|
||||
numCPUs: DEFAULT_CPU_COUNT,
|
||||
}
|
||||
)
|
||||
# When updating make sure to use a box that supports VMWare and VirtualBox
|
||||
vmCfg.vm.box = "bento/ubuntu-16.04" # 16.04 LTS
|
||||
|
||||
vmCfg.vm.provision "shell", inline: $script, privileged: false
|
||||
vmCfg.vm.synced_folder '.', '/opt/gopath/src/github.com/hashicorp/nomad'
|
||||
|
||||
# We're going to compile go and run a concurrent system, so give ourselves
|
||||
# some extra resources. Nomad will have trouble working correctly with <2
|
||||
# CPUs so we should use at least that many.
|
||||
cpus = vmParams.fetch(:numCPUs, DEFAULT_CPU_COUNT)
|
||||
memory = 2048
|
||||
|
||||
vmCfg.vm.provider "parallels" do |p, o|
|
||||
p.memory = memory
|
||||
p.cpus = cpus
|
||||
end
|
||||
|
||||
vmCfg.vm.provider "virtualbox" do |v|
|
||||
v.memory = memory
|
||||
v.cpus = cpus
|
||||
end
|
||||
|
||||
["vmware_fusion", "vmware_workstation"].each do |p|
|
||||
vmCfg.vm.provider p do |v|
|
||||
v.enable_vmrun_ip_lookup = false
|
||||
v.gui = false
|
||||
v.memory = memory
|
||||
v.cpus = cpus
|
||||
end
|
||||
end
|
||||
return vmCfg
|
||||
vmCfg.vm.provision "shell",
|
||||
privileged: true,
|
||||
path: './scripts/vagrant-linux-priv-zeroconf.sh'
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
|
||||
1.upto(3) do |n|
|
||||
vmName = "nomad-server%02d" % [n]
|
||||
isFirstBox = (n == 1)
|
||||
def configureLinuxProvisioners(vmCfg)
|
||||
vmCfg.vm.provision "shell",
|
||||
privileged: true,
|
||||
inline: 'rm -f /home/vagrant/linux.iso'
|
||||
|
||||
numCPUs = DEFAULT_CPU_COUNT
|
||||
if isFirstBox and Object::RUBY_PLATFORM =~ /darwin/i
|
||||
# Override the max CPUs for the first VM
|
||||
numCPUs = [numCPUs, (`/usr/sbin/sysctl -n hw.ncpu`.to_i - 1)].max
|
||||
end
|
||||
vmCfg.vm.provision "shell",
|
||||
privileged: true,
|
||||
path: './scripts/vagrant-linux-priv-config.sh'
|
||||
|
||||
config.vm.define vmName, autostart: isFirstBox, primary: isFirstBox do |vmCfg|
|
||||
vmCfg.vm.hostname = vmName
|
||||
vmCfg = configureVM(vmCfg, {:numCPUs => numCPUs})
|
||||
end
|
||||
end
|
||||
vmCfg.vm.provision "shell",
|
||||
privileged: true,
|
||||
path: './scripts/vagrant-linux-priv-consul.sh'
|
||||
|
||||
1.upto(3) do |n|
|
||||
vmName = "nomad-client%02d" % [n]
|
||||
config.vm.define vmName, autostart: false, primary: false do |vmCfg|
|
||||
vmCfg.vm.hostname = vmName
|
||||
vmCfg = configureVM(vmCfg)
|
||||
end
|
||||
end
|
||||
vmCfg.vm.provision "shell",
|
||||
privileged: true,
|
||||
path: './scripts/vagrant-linux-priv-vault.sh'
|
||||
|
||||
vmCfg.vm.provision "shell",
|
||||
privileged: true,
|
||||
path: './scripts/vagrant-linux-priv-rkt.sh'
|
||||
|
||||
return vmCfg
|
||||
end
|
||||
|
||||
def configureProviders(vmCfg, cpus: "2", memory: "2048")
|
||||
vmCfg.vm.provider "virtualbox" do |v|
|
||||
v.memory = memory
|
||||
v.cpus = cpus
|
||||
end
|
||||
|
||||
["vmware_fusion", "vmware_workstation"].each do |p|
|
||||
vmCfg.vm.provider p do |v|
|
||||
v.enable_vmrun_ip_lookup = false
|
||||
v.vmx["memsize"] = memory
|
||||
v.vmx["numvcpus"] = cpus
|
||||
end
|
||||
end
|
||||
|
||||
vmCfg.vm.provider "virtualbox" do |v|
|
||||
v.memory = memory
|
||||
v.cpus = cpus
|
||||
end
|
||||
|
||||
return vmCfg
|
||||
end
|
||||
|
||||
def suggestedCPUCores()
|
||||
case RbConfig::CONFIG['host_os']
|
||||
when /darwin/
|
||||
Integer(`sysctl -n hw.ncpu`) / 2
|
||||
when /linux/
|
||||
Integer(`cat /proc/cpuinfo | grep processor | wc -l`) / 2
|
||||
else
|
||||
2
|
||||
end
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user