From e1c544f5e581762a89ff75ce1e5ea19971312a73 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Mon, 11 Nov 2019 22:38:27 +0000 Subject: [PATCH 01/67] low case references --- .circleci/config.yml | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 42e398922..185e07ff9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,11 +1,11 @@ version: 2.1 references: - common_envs: &COMMON_ENVS + common_envs: &common_envs GOMAXPROCS: 1 NOMAD_SLOW_TEST: 1 GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - ignore_for_ui_branches: &IGNORE_FOR_UI_BRANCHES + ignore_for_ui_branches: &ignore_for_ui_branches filters: branches: ignore: /^.-ui\b.*/ @@ -15,30 +15,30 @@ workflows: build-test: jobs: - lint-go: - <<: *IGNORE_FOR_UI_BRANCHES + <<: *ignore_for_ui_branches - build-binaries - test-machine: name: "test-client" test_packages: "./client/..." - <<: *IGNORE_FOR_UI_BRANCHES + <<: *ignore_for_ui_branches - test-machine: name: "test-nomad" test_packages: "./nomad/..." - <<: *IGNORE_FOR_UI_BRANCHES + <<: *ignore_for_ui_branches - test-machine: # API Tests run in a VM rather than container due to the FS tests # requiring `mount` priviliges. name: "test-api" test_packages: "./api/..." - <<: *IGNORE_FOR_UI_BRANCHES + <<: *ignore_for_ui_branches - test-container: name: "test-devices" test_packages: "./devices/..." - <<: *IGNORE_FOR_UI_BRANCHES + <<: *ignore_for_ui_branches - test-machine: name: "test-other" exclude_packages: "./api|./client|./drivers/docker|./drivers/exec|./drivers/rkt|./drivers/shared/executor|./nomad|./devices" - <<: *IGNORE_FOR_UI_BRANCHES + <<: *ignore_for_ui_branches - test-machine: name: "test-docker" test_packages: "./drivers/docker" @@ -46,27 +46,27 @@ workflows: # and we get unexpected failures # e.g. https://circleci.com/gh/hashicorp/nomad/3854 executor: go-machine - <<: *IGNORE_FOR_UI_BRANCHES + <<: *ignore_for_ui_branches - test-machine: name: "test-exec" test_packages: "./drivers/exec" - <<: *IGNORE_FOR_UI_BRANCHES + <<: *ignore_for_ui_branches - test-machine: name: "test-shared-exec" test_packages: "./drivers/shared/executor" - <<: *IGNORE_FOR_UI_BRANCHES + <<: *ignore_for_ui_branches - test-machine: name: "test-32bit" # Currently we only explicitly test fingerprinting on 32bit # architectures. test_packages: "./client/fingerprint" goarch: "386" - <<: *IGNORE_FOR_UI_BRANCHES + <<: *ignore_for_ui_branches - test-e2e: - <<: *IGNORE_FOR_UI_BRANCHES + <<: *ignore_for_ui_branches - test-ui - test-website: - <<: *IGNORE_FOR_UI_BRANCHES + <<: *ignore_for_ui_branches website: jobs: @@ -106,7 +106,7 @@ jobs: lint-go: executor: go environment: - <<: *COMMON_ENVS + <<: *common_envs GOPATH: /go steps: - checkout @@ -150,7 +150,7 @@ jobs: type: string default: "amd64" environment: - <<: *COMMON_ENVS + <<: *common_envs GOTEST_PKGS: "<< parameters.test_packages >>" GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>" GOPATH: /go @@ -171,7 +171,7 @@ jobs: test-e2e: executor: go environment: - <<: *COMMON_ENVS + <<: *common_envs GOPATH: /go steps: - checkout @@ -193,7 +193,7 @@ jobs: test-website: executor: go-machine-recent environment: - <<: *COMMON_ENVS + <<: *common_envs steps: - checkout - run: make test-website @@ -214,7 +214,7 @@ jobs: type: string default: "amd64" environment: - <<: *COMMON_ENVS + <<: *common_envs GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>" GOTEST_PKGS: "<< parameters.test_packages >>" GOPATH: /home/circleci/go From 7136a8466576e5ee5d89b9a11b63189751a5e664 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Mon, 11 Nov 2019 22:41:59 +0000 Subject: [PATCH 02/67] Ignore ci workflow in stable-website `stable-website` branch is only meant for updating the nomadproject.io website, and the backend tests are irrelevant. Also, the ci workflow uses up the plans containers and may delay website deployments by 20 minutes or more while we are cutting a release. --- .circleci/config.yml | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 185e07ff9..28b57f8fd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,40 +5,47 @@ references: GOMAXPROCS: 1 NOMAD_SLOW_TEST: 1 GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - ignore_for_ui_branches: &ignore_for_ui_branches + backend_branches_filter: &backend_branches_filter filters: branches: - ignore: /^.-ui\b.*/ + ignore: + - /^.-ui\b.*/ + - stable-website + ui_branches_filter: &backend_branches_filter + filters: + branches: + ignore: + - stable-website workflows: build-test: jobs: - lint-go: - <<: *ignore_for_ui_branches + <<: *backend_branches_filter - build-binaries - test-machine: name: "test-client" test_packages: "./client/..." - <<: *ignore_for_ui_branches + <<: *backend_branches_filter - test-machine: name: "test-nomad" test_packages: "./nomad/..." - <<: *ignore_for_ui_branches + <<: *backend_branches_filter - test-machine: # API Tests run in a VM rather than container due to the FS tests # requiring `mount` priviliges. name: "test-api" test_packages: "./api/..." - <<: *ignore_for_ui_branches + <<: *backend_branches_filter - test-container: name: "test-devices" test_packages: "./devices/..." - <<: *ignore_for_ui_branches + <<: *backend_branches_filter - test-machine: name: "test-other" exclude_packages: "./api|./client|./drivers/docker|./drivers/exec|./drivers/rkt|./drivers/shared/executor|./nomad|./devices" - <<: *ignore_for_ui_branches + <<: *backend_branches_filter - test-machine: name: "test-docker" test_packages: "./drivers/docker" @@ -46,27 +53,28 @@ workflows: # and we get unexpected failures # e.g. https://circleci.com/gh/hashicorp/nomad/3854 executor: go-machine - <<: *ignore_for_ui_branches + <<: *backend_branches_filter - test-machine: name: "test-exec" test_packages: "./drivers/exec" - <<: *ignore_for_ui_branches + <<: *backend_branches_filter - test-machine: name: "test-shared-exec" test_packages: "./drivers/shared/executor" - <<: *ignore_for_ui_branches + <<: *backend_branches_filter - test-machine: name: "test-32bit" # Currently we only explicitly test fingerprinting on 32bit # architectures. test_packages: "./client/fingerprint" goarch: "386" - <<: *ignore_for_ui_branches + <<: *backend_branches_filter - test-e2e: - <<: *ignore_for_ui_branches - - test-ui + <<: *backend_branches_filter + - test-ui: + <<: *ui_branches_filter - test-website: - <<: *ignore_for_ui_branches + <<: *backend_branches_filter website: jobs: From b6b6b55cc11528a23611b32a94caf3fb41925e0f Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Mon, 11 Nov 2019 22:45:20 +0000 Subject: [PATCH 03/67] build-deps-image no longer runs This is a remenant of the time we used a custom hashicorp docker image for CI. Currently, we use the official golang image, so no longer need the job or manage the dockerhub credentials. --- .circleci/config.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 28b57f8fd..8b6cc0836 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -92,9 +92,6 @@ executors: working_directory: ~/go/src/github.com/hashicorp/nomad machine: image: circleci/classic:201808-01 - docker-builder: - working_directory: ~/go/src/github.com/hashicorp/nomad - machine: true # TODO: Find latest docker image id # uses a more recent image with unattended upgrades disabled properly # but seems to break docker builds @@ -104,13 +101,6 @@ executors: image: ubuntu-1604:201903-01 jobs: - build-deps-image: - executor: docker-builder - steps: - - checkout - - run: docker build -t hashicorpnomad/ci-build-image:$CIRCLE_SHA1 . -f ./Dockerfile.ci - - run: docker push hashicorpnomad/ci-build-image:$CIRCLE_SHA1 - lint-go: executor: go environment: From 90d1c69e9e514a241ab7efb154c6e65d58699cf4 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Wed, 20 Nov 2019 11:11:28 -0500 Subject: [PATCH 04/67] Refactor circleci CI scripts This adopts pattern used by Vault, where we split CircleCI yaml config into multiple files that get packed and translated to 2.0. This has two motivations: First, to ease translating config to CircleCI 2.0 so it can run on Enterprise private repository. Second and most importantly, it also adding Enterprise specific jobs in separate files with reduced config file merging conflict resolution. --- .circleci/.gitattributes | 1 + .circleci/.gitignore | 1 + .circleci/Makefile | 80 ++ .circleci/README.md | 130 ++ .circleci/config.yml | 1231 +++++++++++++----- .circleci/config/commands/install-consul.yml | 11 + .circleci/config/commands/install-golang.yml | 10 + .circleci/config/commands/install-protoc.yml | 6 + .circleci/config/commands/install-vault.yml | 11 + .circleci/config/commands/run-tests.yml | 18 + .circleci/config/config.yml | 39 + .circleci/config/jobs/build-binaries.yml | 17 + .circleci/config/jobs/build-website.yml | 30 + .circleci/config/jobs/lint-go.yml | 8 + .circleci/config/jobs/test-container.yml | 27 + .circleci/config/jobs/test-e2e.yml | 17 + .circleci/config/jobs/test-machine.yml | 39 + .circleci/config/jobs/test-ui.yml | 27 + .circleci/config/jobs/test-website.yml | 4 + .circleci/config/workflows/build-test.yml | 62 + .circleci/config/workflows/website.yml | 6 + 21 files changed, 1414 insertions(+), 361 deletions(-) create mode 100644 .circleci/.gitattributes create mode 100644 .circleci/.gitignore create mode 100644 .circleci/Makefile create mode 100644 .circleci/README.md create mode 100644 .circleci/config/commands/install-consul.yml create mode 100644 .circleci/config/commands/install-golang.yml create mode 100644 .circleci/config/commands/install-protoc.yml create mode 100644 .circleci/config/commands/install-vault.yml create mode 100644 .circleci/config/commands/run-tests.yml create mode 100644 .circleci/config/config.yml create mode 100644 .circleci/config/jobs/build-binaries.yml create mode 100644 .circleci/config/jobs/build-website.yml create mode 100644 .circleci/config/jobs/lint-go.yml create mode 100644 .circleci/config/jobs/test-container.yml create mode 100644 .circleci/config/jobs/test-e2e.yml create mode 100644 .circleci/config/jobs/test-machine.yml create mode 100644 .circleci/config/jobs/test-ui.yml create mode 100644 .circleci/config/jobs/test-website.yml create mode 100644 .circleci/config/workflows/build-test.yml create mode 100644 .circleci/config/workflows/website.yml diff --git a/.circleci/.gitattributes b/.circleci/.gitattributes new file mode 100644 index 000000000..2dd06ee5f --- /dev/null +++ b/.circleci/.gitattributes @@ -0,0 +1 @@ +config.yml linguist-generated diff --git a/.circleci/.gitignore b/.circleci/.gitignore new file mode 100644 index 000000000..3018b3a68 --- /dev/null +++ b/.circleci/.gitignore @@ -0,0 +1 @@ +.tmp/ diff --git a/.circleci/Makefile b/.circleci/Makefile new file mode 100644 index 000000000..3852d19f7 --- /dev/null +++ b/.circleci/Makefile @@ -0,0 +1,80 @@ +# Set SHELL to 'strict mode' without using .SHELLFLAGS for max compatibility. +# See https://fieldnotes.tech/how-to-shell-for-compatible-makefiles/ +SHELL := /usr/bin/env bash -euo pipefail -c + +CIRCLECI := circleci --skip-update-check + +# Set up some documentation/help message variables. +# We do not attempt to install the CircleCI CLI from this Makefile. +CCI_INSTALL_LINK := https://circleci.com/docs/2.0/local-cli/\#installation +CCI_INSTALL_MSG := Please install CircleCI CLI. See $(CCI_INSTALL_LINK) +CCI_VERSION := $(shell $(CIRCLECI) version 2> /dev/null) +ifeq ($(CCI_VERSION),) +# Attempting to use the CLI fails with installation instructions. +CIRCLECI := echo '$(CCI_INSTALL_MSG)'; exit 1; \# +endif + +SOURCE_DIR := config +SOURCE_YML := $(shell [ ! -d $(SOURCE_DIR) ] || find $(SOURCE_DIR) -name '*.yml') +CONFIG_SOURCE := Makefile $(SOURCE_YML) | $(SOURCE_DIR) +OUT := config.yml +TMP := .tmp/config-processed +CONFIG_PACKED := .tmp/config-packed + +default: help + +help: + @echo "Usage:" + @echo " make ci-config: recompile config.yml from $(SOURCE_DIR)/" + @echo " make ci-verify: verify that config.yml is a true mapping from $(SOURCE_DIR)/" + @echo + @echo "Diagnostics:" + @[ -z "$(CCI_VERSION)" ] || echo " circleci-cli version $(CCI_VERSION)" + @[ -n "$(CCI_VERSION)" ] || echo " $(CCI_INSTALL_MSG)" + +$(SOURCE_DIR): + @echo No source directory $(SOURCE_DIR) found.; exit 1 + +# Make sure our .tmp dir exists. +$(shell [ -d .tmp ] || mkdir .tmp) + +.PHONY: ci-config +ci-config: $(OUT) + +.PHONY: ci-verify +ci-verify: config-up-to-date + @$(CIRCLECI) config validate $(OUT) + +define GENERATED_FILE_HEADER +### *** +### WARNING: DO NOT manually EDIT or MERGE this file, it is generated by 'make ci-config'. +### INSTEAD: Edit or merge the source in $(SOURCE_DIR)/ then run 'make ci-config'. +### *** +endef +export GENERATED_FILE_HEADER + +# GEN_CONFIG writes the config to a temporary file. If the whole process succeeds, +# it then moves that file to $@. This makes is an atomic operation, so if it fails +# make doesn't consider a half-baked file up to date. +define GEN_CONFIG + @$(CIRCLECI) config pack $(SOURCE_DIR) > $(CONFIG_PACKED) + @echo "$$GENERATED_FILE_HEADER" > $@.tmp || { rm -f $@; exit 1; } + @$(CIRCLECI) config process $(CONFIG_PACKED) >> $@.tmp || { rm -f $@.tmp; exit 1; } + @mv -f $@.tmp $@ +endef + +$(OUT): $(CONFIG_SOURCE) + $(GEN_CONFIG) + @echo "$@ updated" + +$(TMP): $(CONFIG_SOURCE) + $(GEN_CONFIG) + +.PHONY: config-up-to-date +config-up-to-date: $(TMP) # Note this must not depend on $(OUT)! + @if diff config.yml $<; then \ + echo "Generated $(OUT) is up to date!"; \ + else \ + echo "Generated $(OUT) is out of date, run make $(CONFIG) to update."; \ + exit 1; \ + fi diff --git a/.circleci/README.md b/.circleci/README.md new file mode 100644 index 000000000..1ec75cafa --- /dev/null +++ b/.circleci/README.md @@ -0,0 +1,130 @@ +# How to use CircleCI multi-file config + +This README and the Makefile should be in your `.circleci` directory, +in the root of your repository. +All path references in this README assume we are in this `.circleci` directory. + +The `Makefile` in this directory generates `./config.yml` in CircleCI 2.0 syntax, +from the tree rooted at `./config/`, which contains files in CircleCI 2.0 or 2.1 syntax. + + +## Quickstart + +The basic workflow is: + +- Edit source files in `./config/` +- When you are done, run `make ci-config` to update `./config.yml` +- Commit this entire `.circleci` directory, including that generated file together. +- Run `make ci-verify` to ensure the current `./config.yml` is up to date with the source. + +When merging this `.circleci` directory: + +- Do not merge the generated `./config.yml` file, instead: +- Merge the source files under `./config/`, and then +- Run `make ci-config` to re-generate the merged `./config.yml` + +And that's it, for more detail, read on! + + +## How does it work, roughly? + +CircleCI supports [generating a single config file from many], +using the `$ circleci config pack` command. +It also supports [expanding 2.1 syntax to 2.0 syntax] +using the `$ circleci config process` command. +We use these two commands, stitched together using the `Makefile` +to implement the workflow. + +[generating a single config file from many]: https://circleci.com/docs/2.0/local-cli/#packing-a-config +[expanding 2.1 syntax to 2.0 syntax]: https://circleci.com/docs/2.0/local-cli/#processing-a-config + + +## Prerequisites + +You will need the [CircleCI CLI tool] installed and working, +at least version `0.1.5607`. +You can [download this tool directly from GitHub Releases]. + +``` +$ circleci version +0.1.5607+f705856 +``` + +[CircleCI CLI tool]: https://circleci.com/docs/2.0/local-cli/ +[download this tool directly from GitHub Releases]: https://github.com/CircleCI-Public/circleci-cli/releases + + +## Updating the config source + +Before making changes, be sure to understand the layout +of the `./config/` file tree, as well as circleci 2.1 syntax. +See the [Syntax and layout] section below. + +To update the config, you should edit, add or remove files +in the `./config/` directory, +and then run `make ci-config`. +If that's successful, +you should then commit every `*.yml` file in the tree rooted in this directory. +That is: you should commit both the source under `./config/` +and the generated file `./config.yml` at the same time, in the same commit. +The included git pre-commit hook will help with this. +Do not edit the `./config.yml` file directly, as you will lose your changes +next time `make ci-config` is run. + +[Syntax and layout]: #syntax-and-layout + + +### Verifying `./config.yml` + +To check whether or not the current `./config.yml` is up to date with the source +and valid, run `$ make ci-verify`. +Note that `$ make ci-verify` should be run in CI, +in case not everyone has the git pre-commit hook set up correctly. + + +#### Example shell session + +```sh +$ make ci-config +config.yml updated +$ git add -A . # The -A makes sure to include deletions/renames etc. +$ git commit -m "ci: blah blah blah" +Changes detected in .circleci/, running 'make -C .circleci ci-verify' +--> Generated config.yml is up to date! +--> Config file at config.yml is valid. +``` + + +### Syntax and layout + +It is important to understand the layout of the config directory. +Read the documentation on [packing a config] for a full understanding +of how multiple YAML files are merged by the circleci CLI tool. + +[packing a config]: https://circleci.com/docs/2.0/local-cli/#packing-a-config + +Here is an example file tree (with comments added afterwards): + +```sh +$ tree . +. +├── Makefile +├── README.md # This file. +├── config # The source code for config.yml is rooted here. +│   ├── @config.yml # Files beginning with @ are treated specially by `circleci config pack` +│   ├── commands # Subdirectories of config become top-level keys. +│   │   └── go_test.yml # Filenames (minus .yml) become top-level keys under +│   │   └── go_build.yml # their parent (in this case "commands"). +│ │ # The contents of go_test.yml therefore are placed at: .commands.go_test: +│   └── jobs # jobs also becomes a top-level key under config... +│   ├── build.yml # ...and likewise filenames become keys under their parent. +│   └── test.yml +└── config.yml # The generated file in 2.0 syntax. +``` + +About those `@` files... Preceding a filename with `@` +indicates to `$ circleci config pack` that the contents of this YAML file +should be at the top-level, rather than underneath a key named after their filename. +This naming convention is unfortunate as it breaks autocompletion in bash, +but there we go. + diff --git a/.circleci/config.yml b/.circleci/config.yml index 8b6cc0836..83f5280fc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,369 +1,878 @@ -version: 2.1 +### *** +### WARNING: DO NOT manually EDIT or MERGE this file, it is generated by 'make ci-config'. +### INSTEAD: Edit or merge the source in config/ then run 'make ci-config'. +### *** +version: 2 +jobs: + test-nomad: + machine: + image: ubuntu-1604:201903-01 + working_directory: ~/go/src/github.com/hashicorp/nomad + environment: + - GOLANG_VERSION: 1.12.13 + - GOMAXPROCS: 1 + - GOPATH: /home/circleci/go + - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + - NOMAD_SLOW_TEST: 1 + - GOTEST_PKGS: ./nomad/... + - GOTEST_PKGS_EXCLUDE: '' + - GOTESTARCH: amd64 + steps: + - checkout + - run: + command: | + set -x + echo installing golang ${GOLANG_VERSION} + sudo rm -rf /usr/local/go + wget -O /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf /tmp/golang.tar.gz + rm -rf /tmp/golang.tar.gz + name: install golang + - run: + command: | + sudo rm -rf /usr/bin/protoc + sudo ./scripts/vagrant-linux-priv-protoc.sh + name: install protoc + - run: + command: | + wget -q -O /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.1/consul_1.6.1_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/consul.zip + rm -rf /tmp/consul* + name: Install Consul 1.6.1 + - run: + command: | + wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/1.2.3/vault_1.2.3_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/vault.zip + rm -rf /tmp/vault* + name: Install Vault 1.2.3 + - run: + command: | + if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then + sudo apt-get update + sudo apt-get install -y gcc-multilib + else + echo "Skipping 32bit lib installation while building for not 386" + fi + name: Install 32bit gcc libs + - run: + command: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap + - run: + command: | + if [ -z $GOTEST_PKGS_EXCLUDE ]; + then + unset GOTEST_PKGS_EXCLUDE + else + unset GOTEST_PKGS + fi -references: - common_envs: &common_envs - GOMAXPROCS: 1 - NOMAD_SLOW_TEST: 1 - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - backend_branches_filter: &backend_branches_filter - filters: - branches: - ignore: - - /^.-ui\b.*/ - - stable-website - ui_branches_filter: &backend_branches_filter - filters: - branches: - ignore: - - stable-website + if [ ! -z $GOTESTARCH ]; then + export GOARCH="$GOTESTARCH"; + fi + mkdir -p /tmp/test-reports + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad + name: Running Nomad Tests + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-reports + test-api: + machine: + image: ubuntu-1604:201903-01 + working_directory: ~/go/src/github.com/hashicorp/nomad + environment: + - GOLANG_VERSION: 1.12.13 + - GOMAXPROCS: 1 + - GOPATH: /home/circleci/go + - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + - NOMAD_SLOW_TEST: 1 + - GOTEST_PKGS: ./api/... + - GOTEST_PKGS_EXCLUDE: '' + - GOTESTARCH: amd64 + steps: + - checkout + - run: + command: | + set -x + echo installing golang ${GOLANG_VERSION} + sudo rm -rf /usr/local/go + wget -O /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf /tmp/golang.tar.gz + rm -rf /tmp/golang.tar.gz + name: install golang + - run: + command: | + sudo rm -rf /usr/bin/protoc + sudo ./scripts/vagrant-linux-priv-protoc.sh + name: install protoc + - run: + command: | + wget -q -O /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.1/consul_1.6.1_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/consul.zip + rm -rf /tmp/consul* + name: Install Consul 1.6.1 + - run: + command: | + wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/1.2.3/vault_1.2.3_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/vault.zip + rm -rf /tmp/vault* + name: Install Vault 1.2.3 + - run: + command: | + if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then + sudo apt-get update + sudo apt-get install -y gcc-multilib + else + echo "Skipping 32bit lib installation while building for not 386" + fi + name: Install 32bit gcc libs + - run: + command: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap + - run: + command: | + if [ -z $GOTEST_PKGS_EXCLUDE ]; + then + unset GOTEST_PKGS_EXCLUDE + else + unset GOTEST_PKGS + fi + if [ ! -z $GOTESTARCH ]; then + export GOARCH="$GOTESTARCH"; + fi + + mkdir -p /tmp/test-reports + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad + name: Running Nomad Tests + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-reports + test-exec: + machine: + image: ubuntu-1604:201903-01 + working_directory: ~/go/src/github.com/hashicorp/nomad + environment: + - GOLANG_VERSION: 1.12.13 + - GOMAXPROCS: 1 + - GOPATH: /home/circleci/go + - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + - NOMAD_SLOW_TEST: 1 + - GOTEST_PKGS: ./drivers/exec + - GOTEST_PKGS_EXCLUDE: '' + - GOTESTARCH: amd64 + steps: + - checkout + - run: + command: | + set -x + echo installing golang ${GOLANG_VERSION} + sudo rm -rf /usr/local/go + wget -O /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf /tmp/golang.tar.gz + rm -rf /tmp/golang.tar.gz + name: install golang + - run: + command: | + sudo rm -rf /usr/bin/protoc + sudo ./scripts/vagrant-linux-priv-protoc.sh + name: install protoc + - run: + command: | + wget -q -O /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.1/consul_1.6.1_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/consul.zip + rm -rf /tmp/consul* + name: Install Consul 1.6.1 + - run: + command: | + wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/1.2.3/vault_1.2.3_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/vault.zip + rm -rf /tmp/vault* + name: Install Vault 1.2.3 + - run: + command: | + if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then + sudo apt-get update + sudo apt-get install -y gcc-multilib + else + echo "Skipping 32bit lib installation while building for not 386" + fi + name: Install 32bit gcc libs + - run: + command: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap + - run: + command: | + if [ -z $GOTEST_PKGS_EXCLUDE ]; + then + unset GOTEST_PKGS_EXCLUDE + else + unset GOTEST_PKGS + fi + + if [ ! -z $GOTESTARCH ]; then + export GOARCH="$GOTESTARCH"; + fi + + mkdir -p /tmp/test-reports + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad + name: Running Nomad Tests + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-reports + test-client: + machine: + image: ubuntu-1604:201903-01 + working_directory: ~/go/src/github.com/hashicorp/nomad + environment: + - GOLANG_VERSION: 1.12.13 + - GOMAXPROCS: 1 + - GOPATH: /home/circleci/go + - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + - NOMAD_SLOW_TEST: 1 + - GOTEST_PKGS: ./client/... + - GOTEST_PKGS_EXCLUDE: '' + - GOTESTARCH: amd64 + steps: + - checkout + - run: + command: | + set -x + echo installing golang ${GOLANG_VERSION} + sudo rm -rf /usr/local/go + wget -O /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf /tmp/golang.tar.gz + rm -rf /tmp/golang.tar.gz + name: install golang + - run: + command: | + sudo rm -rf /usr/bin/protoc + sudo ./scripts/vagrant-linux-priv-protoc.sh + name: install protoc + - run: + command: | + wget -q -O /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.1/consul_1.6.1_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/consul.zip + rm -rf /tmp/consul* + name: Install Consul 1.6.1 + - run: + command: | + wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/1.2.3/vault_1.2.3_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/vault.zip + rm -rf /tmp/vault* + name: Install Vault 1.2.3 + - run: + command: | + if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then + sudo apt-get update + sudo apt-get install -y gcc-multilib + else + echo "Skipping 32bit lib installation while building for not 386" + fi + name: Install 32bit gcc libs + - run: + command: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap + - run: + command: | + if [ -z $GOTEST_PKGS_EXCLUDE ]; + then + unset GOTEST_PKGS_EXCLUDE + else + unset GOTEST_PKGS + fi + + if [ ! -z $GOTESTARCH ]; then + export GOARCH="$GOTESTARCH"; + fi + + mkdir -p /tmp/test-reports + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad + name: Running Nomad Tests + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-reports + test-shared-exec: + machine: + image: ubuntu-1604:201903-01 + working_directory: ~/go/src/github.com/hashicorp/nomad + environment: + - GOLANG_VERSION: 1.12.13 + - GOMAXPROCS: 1 + - GOPATH: /home/circleci/go + - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + - NOMAD_SLOW_TEST: 1 + - GOTEST_PKGS: ./drivers/shared/executor + - GOTEST_PKGS_EXCLUDE: '' + - GOTESTARCH: amd64 + steps: + - checkout + - run: + command: | + set -x + echo installing golang ${GOLANG_VERSION} + sudo rm -rf /usr/local/go + wget -O /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf /tmp/golang.tar.gz + rm -rf /tmp/golang.tar.gz + name: install golang + - run: + command: | + sudo rm -rf /usr/bin/protoc + sudo ./scripts/vagrant-linux-priv-protoc.sh + name: install protoc + - run: + command: | + wget -q -O /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.1/consul_1.6.1_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/consul.zip + rm -rf /tmp/consul* + name: Install Consul 1.6.1 + - run: + command: | + wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/1.2.3/vault_1.2.3_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/vault.zip + rm -rf /tmp/vault* + name: Install Vault 1.2.3 + - run: + command: | + if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then + sudo apt-get update + sudo apt-get install -y gcc-multilib + else + echo "Skipping 32bit lib installation while building for not 386" + fi + name: Install 32bit gcc libs + - run: + command: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap + - run: + command: | + if [ -z $GOTEST_PKGS_EXCLUDE ]; + then + unset GOTEST_PKGS_EXCLUDE + else + unset GOTEST_PKGS + fi + + if [ ! -z $GOTESTARCH ]; then + export GOARCH="$GOTESTARCH"; + fi + + mkdir -p /tmp/test-reports + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad + name: Running Nomad Tests + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-reports + test-ui: + docker: + - environment: + JOBS: 2 + image: circleci/node:10-browsers + steps: + - checkout + - restore_cache: + keys: + - v1-deps-{{ checksum "ui/yarn.lock" }} + - v1-deps- + - run: + command: cd ui && yarn install + name: yarn install + - save_cache: + key: v1-deps-{{ checksum "ui/yarn.lock" }} + paths: + - ./ui/node_modules + - run: + command: cd ui && yarn run lint:js + name: lint:js + - run: + command: cd ui && yarn run lint:hbs + name: lint:hbs + - run: + command: cd ui && yarn test + name: Ember tests + lint-go: + docker: + - image: golang:1.12.13 + working_directory: /go/src/github.com/hashicorp/nomad + steps: + - checkout + - run: + command: apt-get update; apt-get install -y shellcheck sudo unzip + - run: + command: | + sudo rm -rf /usr/bin/protoc + sudo ./scripts/vagrant-linux-priv-protoc.sh + name: install protoc + - run: + command: make deps lint-deps + - run: + command: make check + - run: + command: make checkscripts + environment: + - GOMAXPROCS: 1 + - GOPATH: /go + - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + - NOMAD_SLOW_TEST: 1 + build-website: + docker: + - image: hashicorp/middleman-hashicorp:0.3.35 + steps: + - checkout: + path: ~/project + - restore_cache: + key: static-site-gems-v1-{{ checksum "Gemfile.lock" }} + - run: + command: bundle check || bundle install --path vendor/bundle --retry=3 + name: install gems + - save_cache: + key: static-site-gems-v1-{{ checksum "Gemfile.lock" }} + paths: + - ~/project/website/vendor/bundle + - run: + command: bundle exec middleman build + name: middleman build + - run: + command: ./scripts/deploy.sh + name: website deploy + working_directory: ~/project/website + test-other: + machine: + image: ubuntu-1604:201903-01 + working_directory: ~/go/src/github.com/hashicorp/nomad + environment: + - GOLANG_VERSION: 1.12.13 + - GOMAXPROCS: 1 + - GOPATH: /home/circleci/go + - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + - NOMAD_SLOW_TEST: 1 + - GOTEST_PKGS: '' + - GOTEST_PKGS_EXCLUDE: ./api|./client|./drivers/docker|./drivers/exec|./drivers/rkt|./drivers/shared/executor|./nomad|./devices + - GOTESTARCH: amd64 + steps: + - checkout + - run: + command: | + set -x + echo installing golang ${GOLANG_VERSION} + sudo rm -rf /usr/local/go + wget -O /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf /tmp/golang.tar.gz + rm -rf /tmp/golang.tar.gz + name: install golang + - run: + command: | + sudo rm -rf /usr/bin/protoc + sudo ./scripts/vagrant-linux-priv-protoc.sh + name: install protoc + - run: + command: | + wget -q -O /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.1/consul_1.6.1_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/consul.zip + rm -rf /tmp/consul* + name: Install Consul 1.6.1 + - run: + command: | + wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/1.2.3/vault_1.2.3_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/vault.zip + rm -rf /tmp/vault* + name: Install Vault 1.2.3 + - run: + command: | + if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then + sudo apt-get update + sudo apt-get install -y gcc-multilib + else + echo "Skipping 32bit lib installation while building for not 386" + fi + name: Install 32bit gcc libs + - run: + command: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap + - run: + command: | + if [ -z $GOTEST_PKGS_EXCLUDE ]; + then + unset GOTEST_PKGS_EXCLUDE + else + unset GOTEST_PKGS + fi + + if [ ! -z $GOTESTARCH ]; then + export GOARCH="$GOTESTARCH"; + fi + + mkdir -p /tmp/test-reports + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad + name: Running Nomad Tests + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-reports + test-devices: + docker: + - image: golang:1.12.13 + working_directory: /go/src/github.com/hashicorp/nomad + environment: + - GOMAXPROCS: 1 + - GOPATH: /go + - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + - NOMAD_SLOW_TEST: 1 + - GOTEST_PKGS: ./devices/... + - GOTEST_PKGS_EXCLUDE: '' + - GOTESTARCH: amd64 + steps: + - checkout + - run: + command: apt-get update; apt-get install -y shellcheck sudo unzip + - run: + command: make deps + - run: + command: | + sudo rm -rf /usr/bin/protoc + sudo ./scripts/vagrant-linux-priv-protoc.sh + name: install protoc + - run: + command: | + wget -q -O /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.1/consul_1.6.1_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/consul.zip + rm -rf /tmp/consul* + name: Install Consul 1.6.1 + - run: + command: | + wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/1.2.3/vault_1.2.3_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/vault.zip + rm -rf /tmp/vault* + name: Install Vault 1.2.3 + - run: + command: | + if [ -z $GOTEST_PKGS_EXCLUDE ]; + then + unset GOTEST_PKGS_EXCLUDE + else + unset GOTEST_PKGS + fi + + if [ ! -z $GOTESTARCH ]; then + export GOARCH="$GOTESTARCH"; + fi + + mkdir -p /tmp/test-reports + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad + name: Running Nomad Tests + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-reports + test-website: + machine: + image: ubuntu-1604:201903-01 + working_directory: ~/go/src/github.com/hashicorp/nomad + steps: + - checkout + - run: + command: make test-website + environment: + - GOLANG_VERSION: 1.12.13 + - GOMAXPROCS: 1 + - GOPATH: /home/circleci/go + - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + - NOMAD_SLOW_TEST: 1 + test-docker: + machine: + image: circleci/classic:201808-01 + working_directory: ~/go/src/github.com/hashicorp/nomad + environment: + - GOLANG_VERSION: 1.12.13 + - GOMAXPROCS: 1 + - GOPATH: /home/circleci/go + - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + - NOMAD_SLOW_TEST: 1 + - GOTEST_PKGS: ./drivers/docker + - GOTEST_PKGS_EXCLUDE: '' + - GOTESTARCH: amd64 + steps: + - checkout + - run: + command: | + set -x + echo installing golang ${GOLANG_VERSION} + sudo rm -rf /usr/local/go + wget -O /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf /tmp/golang.tar.gz + rm -rf /tmp/golang.tar.gz + name: install golang + - run: + command: | + sudo rm -rf /usr/bin/protoc + sudo ./scripts/vagrant-linux-priv-protoc.sh + name: install protoc + - run: + command: | + wget -q -O /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.1/consul_1.6.1_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/consul.zip + rm -rf /tmp/consul* + name: Install Consul 1.6.1 + - run: + command: | + wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/1.2.3/vault_1.2.3_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/vault.zip + rm -rf /tmp/vault* + name: Install Vault 1.2.3 + - run: + command: | + if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then + sudo apt-get update + sudo apt-get install -y gcc-multilib + else + echo "Skipping 32bit lib installation while building for not 386" + fi + name: Install 32bit gcc libs + - run: + command: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap + - run: + command: | + if [ -z $GOTEST_PKGS_EXCLUDE ]; + then + unset GOTEST_PKGS_EXCLUDE + else + unset GOTEST_PKGS + fi + + if [ ! -z $GOTESTARCH ]; then + export GOARCH="$GOTESTARCH"; + fi + + mkdir -p /tmp/test-reports + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad + name: Running Nomad Tests + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-reports + build-binaries: + docker: + - image: golang:1.12.13 + working_directory: /go/src/github.com/hashicorp/nomad + environment: + - GOMAXPROCS: 1 + - GOPATH: /go + - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + - NOMAD_SLOW_TEST: 1 + - GO_TAGS: codegen_generated + steps: + - checkout + - run: + command: apt-get update; apt-get install -y sudo unzip zip + - run: + command: make deps + - run: + command: | + sudo rm -rf /usr/bin/protoc + sudo ./scripts/vagrant-linux-priv-protoc.sh + name: install protoc + - run: + command: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs + - run: + command: make pkg/windows_amd64.zip pkg/linux_amd64.zip + - store_artifacts: + destination: /builds/nomad_windows_amd64.zip + path: pkg/windows_amd64.zip + - store_artifacts: + destination: /builds/nomad_linux_amd64.zip + path: pkg/linux_amd64.zip + test-e2e: + docker: + - image: golang:1.12.13 + working_directory: /go/src/github.com/hashicorp/nomad + steps: + - checkout + - run: + command: apt-get update; apt-get install -y sudo unzip + - run: + command: | + groupadd --gid 3434 circleci + useradd --uid 3434 --gid circleci --shell /bin/bash --create-home circleci + echo 'circleci ALL=NOPASSWD: ALL' >> /etc/sudoers.d/50-circleci + echo 'Defaults env_keep += "DEBIAN_FRONTEND"' >> /etc/sudoers.d/env_keep + chown -R circleci:circleci /go + name: prepare non-root user + - run: + command: sudo -E -H -u circleci PATH=${PATH} make deps + - run: + command: sudo -E -H -u circleci PATH=${PATH} make e2e-test + environment: + - GOMAXPROCS: 1 + - GOPATH: /go + - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + - NOMAD_SLOW_TEST: 1 + test-32bit: + machine: + image: ubuntu-1604:201903-01 + working_directory: ~/go/src/github.com/hashicorp/nomad + environment: + - GOLANG_VERSION: 1.12.13 + - GOMAXPROCS: 1 + - GOPATH: /home/circleci/go + - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + - NOMAD_SLOW_TEST: 1 + - GOTEST_PKGS: ./client/fingerprint + - GOTEST_PKGS_EXCLUDE: '' + - GOTESTARCH: '386' + steps: + - checkout + - run: + command: | + set -x + echo installing golang ${GOLANG_VERSION} + sudo rm -rf /usr/local/go + wget -O /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf /tmp/golang.tar.gz + rm -rf /tmp/golang.tar.gz + name: install golang + - run: + command: | + sudo rm -rf /usr/bin/protoc + sudo ./scripts/vagrant-linux-priv-protoc.sh + name: install protoc + - run: + command: | + wget -q -O /tmp/consul.zip https://releases.hashicorp.com/consul/1.6.1/consul_1.6.1_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/consul.zip + rm -rf /tmp/consul* + name: Install Consul 1.6.1 + - run: + command: | + wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/1.2.3/vault_1.2.3_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/vault.zip + rm -rf /tmp/vault* + name: Install Vault 1.2.3 + - run: + command: | + if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then + sudo apt-get update + sudo apt-get install -y gcc-multilib + else + echo "Skipping 32bit lib installation while building for not 386" + fi + name: Install 32bit gcc libs + - run: + command: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap + - run: + command: | + if [ -z $GOTEST_PKGS_EXCLUDE ]; + then + unset GOTEST_PKGS_EXCLUDE + else + unset GOTEST_PKGS + fi + + if [ ! -z $GOTESTARCH ]; then + export GOARCH="$GOTESTARCH"; + fi + + mkdir -p /tmp/test-reports + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad + name: Running Nomad Tests + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-reports workflows: build-test: jobs: - - lint-go: - <<: *backend_branches_filter - - build-binaries - - test-machine: - name: "test-client" - test_packages: "./client/..." - <<: *backend_branches_filter - - test-machine: - name: "test-nomad" - test_packages: "./nomad/..." - <<: *backend_branches_filter - - test-machine: - # API Tests run in a VM rather than container due to the FS tests - # requiring `mount` priviliges. - name: "test-api" - test_packages: "./api/..." - <<: *backend_branches_filter - - test-container: - name: "test-devices" - test_packages: "./devices/..." - <<: *backend_branches_filter - - test-machine: - name: "test-other" - exclude_packages: "./api|./client|./drivers/docker|./drivers/exec|./drivers/rkt|./drivers/shared/executor|./nomad|./devices" - <<: *backend_branches_filter - - test-machine: - name: "test-docker" - test_packages: "./drivers/docker" - # docker is misbehaving in docker-machine-recent image - # and we get unexpected failures - # e.g. https://circleci.com/gh/hashicorp/nomad/3854 - executor: go-machine - <<: *backend_branches_filter - - test-machine: - name: "test-exec" - test_packages: "./drivers/exec" - <<: *backend_branches_filter - - test-machine: - name: "test-shared-exec" - test_packages: "./drivers/shared/executor" - <<: *backend_branches_filter - - test-machine: - name: "test-32bit" - # Currently we only explicitly test fingerprinting on 32bit - # architectures. - test_packages: "./client/fingerprint" - goarch: "386" - <<: *backend_branches_filter - - test-e2e: - <<: *backend_branches_filter - - test-ui: - <<: *ui_branches_filter - - test-website: - <<: *backend_branches_filter - + - build-binaries + - lint-go: + filters: + branches: + ignore: + - /^.-ui\b.*/ + - stable-website + - test-client: + filters: + branches: + ignore: + - /^.-ui\b.*/ + - stable-website + - test-nomad: + filters: + branches: + ignore: + - /^.-ui\b.*/ + - stable-website + - test-api: + filters: + branches: + ignore: + - /^.-ui\b.*/ + - stable-website + - test-devices: + filters: + branches: + ignore: + - /^.-ui\b.*/ + - stable-website + - test-other: + filters: + branches: + ignore: + - /^.-ui\b.*/ + - stable-website + - test-docker: + filters: + branches: + ignore: + - /^.-ui\b.*/ + - stable-website + - test-exec: + filters: + branches: + ignore: + - /^.-ui\b.*/ + - stable-website + - test-shared-exec: + filters: + branches: + ignore: + - /^.-ui\b.*/ + - stable-website + - test-32bit: + filters: + branches: + ignore: + - /^.-ui\b.*/ + - stable-website + - test-e2e: + filters: + branches: + ignore: + - /^.-ui\b.*/ + - stable-website + - test-ui: + filters: + branches: + ignore: + - stable-website + - test-website: + filters: + branches: + ignore: + - /^.-ui\b.*/ + - stable-website website: jobs: - - build-website: - context: static-sites - filters: - branches: - only: stable-website -executors: - go: - working_directory: /go/src/github.com/hashicorp/nomad - docker: - - image: golang:1.12.13 - go-machine: - working_directory: ~/go/src/github.com/hashicorp/nomad - machine: - image: circleci/classic:201808-01 - - # uses a more recent image with unattended upgrades disabled properly - # but seems to break docker builds - go-machine-recent: - working_directory: ~/go/src/github.com/hashicorp/nomad - machine: - image: ubuntu-1604:201903-01 - -jobs: - lint-go: - executor: go - environment: - <<: *common_envs - GOPATH: /go - steps: - - checkout - - run: apt-get update; apt-get install -y shellcheck sudo unzip - - install-protoc - - run: make deps lint-deps - - run: make check - - run: make checkscripts - - build-binaries: - executor: go - environment: - <<: *COMMON_ENVS - GOPATH: /go - # TODO: add ui tag here - GO_TAGS: "codegen_generated" - steps: - - checkout - - run: apt-get update; apt-get install -y sudo unzip zip - - run: make deps - - install-protoc - - run: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs - - run: make pkg/windows_amd64.zip pkg/linux_amd64.zip - - store_artifacts: - path: pkg/windows_amd64.zip - destination: /builds/nomad_windows_amd64.zip - - store_artifacts: - path: pkg/linux_amd64.tar.gz - destination: /builds/nomad_linux_amd64.tar.gz - - test-container: - executor: go - parameters: - test_packages: - type: string - default: "" - exclude_packages: - type: string - default: "" - goarch: - type: string - default: "amd64" - environment: - <<: *common_envs - GOTEST_PKGS: "<< parameters.test_packages >>" - GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>" - GOPATH: /go - GOTESTARCH: "<< parameters.goarch >>" - steps: - - checkout - - run: apt-get update; apt-get install -y shellcheck sudo unzip - - run: make deps - - install-protoc - - install-consul - - install-vault - - run-tests - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-reports - - test-e2e: - executor: go - environment: - <<: *common_envs - GOPATH: /go - steps: - - checkout - - run: apt-get update; apt-get install -y sudo unzip - # e2e tests require privileged mount/umount permissions when running as root - # TODO: switch to using machine executor and run as root to test e2e path - - run: - name: prepare non-root user - command: | - groupadd --gid 3434 circleci - useradd --uid 3434 --gid circleci --shell /bin/bash --create-home circleci - echo 'circleci ALL=NOPASSWD: ALL' >> /etc/sudoers.d/50-circleci - echo 'Defaults env_keep += "DEBIAN_FRONTEND"' >> /etc/sudoers.d/env_keep - chown -R circleci:circleci /go - - - run: sudo -E -H -u circleci PATH=${PATH} make deps - - run: sudo -E -H -u circleci PATH=${PATH} make e2e-test - - test-website: - executor: go-machine-recent - environment: - <<: *common_envs - steps: - - checkout - - run: make test-website - - test-machine: - executor: "<< parameters.executor >>" - parameters: - test_packages: - type: string - default: "" - exclude_packages: - type: string - default: "" - executor: - type: string - default: "go-machine-recent" - goarch: - type: string - default: "amd64" - environment: - <<: *common_envs - GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>" - GOTEST_PKGS: "<< parameters.test_packages >>" - GOPATH: /home/circleci/go - GOTESTARCH: "<< parameters.goarch >>" - steps: - - checkout - - install-golang - - install-protoc - - install-consul - - install-vault - - run: - name: Install 32bit gcc libs - command: | - if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then - sudo apt-get update - sudo apt-get install -y gcc-multilib - else - echo "Skipping 32bit lib installation while building for not 386" - fi - - run: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap - - run-tests - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-reports - test-ui: - docker: - - image: circleci/node:10-browsers - environment: - # See https://git.io/vdao3 for details. - JOBS: 2 - steps: - - checkout - - restore_cache: - keys: - - v1-deps-{{ checksum "ui/yarn.lock" }} - - v1-deps- - - run: - name: yarn install - command: cd ui && yarn install - - save_cache: - key: v1-deps-{{ checksum "ui/yarn.lock" }} - paths: - - ./ui/node_modules - - run: - name: lint:js - command: cd ui && yarn run lint:js - - run: - name: lint:hbs - command: cd ui && yarn run lint:hbs - - run: - name: Ember tests - command: cd ui && yarn test - - build-website: - # setting the working_directory along with the checkout path allows us to not have - # to cd into the website/ directory for commands - working_directory: ~/project/website - docker: - - image: hashicorp/middleman-hashicorp:0.3.35 - steps: - - checkout: - path: ~/project - - # restores gem cache - - restore_cache: - key: static-site-gems-v1-{{ checksum "Gemfile.lock" }} - - - run: - name: install gems - command: bundle check || bundle install --path vendor/bundle --retry=3 - - # saves gem cache if we have changed the Gemfile - - save_cache: - key: static-site-gems-v1-{{ checksum "Gemfile.lock" }} - paths: - - ~/project/website/vendor/bundle - - - run: - name: middleman build - command: bundle exec middleman build - - - run: - name: website deploy - command: ./scripts/deploy.sh - -commands: - install-golang: - parameters: - version: - type: string - default: "1.12.13" - steps: - - run: - name: install golang << parameters.version >> - command: | - sudo rm -rf /usr/local/go - wget -q -O /tmp/golang.tar.gz https://dl.google.com/go/go<< parameters.version >>.linux-amd64.tar.gz - sudo tar -C /usr/local -xzf /tmp/golang.tar.gz - rm -rf /tmp/golang.tar.gz - - install-vault: - parameters: - version: - type: string - default: 1.2.3 - steps: - - run: - name: Install Vault << parameters.version >> - command: | - wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/<< parameters.version >>/vault_<< parameters.version>>_linux_amd64.zip - sudo unzip -d /usr/local/bin /tmp/vault.zip - rm -rf /tmp/vault* - - install-consul: - parameters: - version: - type: string - default: 1.6.1 - steps: - - run: - name: Install Consul << parameters.version >> - command: | - wget -q -O /tmp/consul.zip https://releases.hashicorp.com/consul/<< parameters.version >>/consul_<< parameters.version >>_linux_amd64.zip - sudo unzip -d /usr/local/bin /tmp/consul.zip - rm -rf /tmp/consul* - - install-protoc: - steps: - - run: - name: install protoc - command: | - sudo rm -rf /usr/bin/protoc - sudo ./scripts/vagrant-linux-priv-protoc.sh - - run-tests: - steps: - - run: - name: Running Nomad Tests - command: | - if [ -z $GOTEST_PKGS_EXCLUDE ]; - then - unset GOTEST_PKGS_EXCLUDE - else - unset GOTEST_PKGS - fi - - if [ ! -z $GOTESTARCH ]; then - export GOARCH="$GOTESTARCH"; - fi - - mkdir -p /tmp/test-reports - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad + - build-website: + filters: + branches: + only: stable-website + context: static-sites + version: 2 diff --git a/.circleci/config/commands/install-consul.yml b/.circleci/config/commands/install-consul.yml new file mode 100644 index 000000000..75aeb3b81 --- /dev/null +++ b/.circleci/config/commands/install-consul.yml @@ -0,0 +1,11 @@ +parameters: + version: + type: string + default: 1.6.1 +steps: + - run: + name: Install Consul << parameters.version >> + command: | + wget -q -O /tmp/consul.zip https://releases.hashicorp.com/consul/<< parameters.version >>/consul_<< parameters.version >>_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/consul.zip + rm -rf /tmp/consul* diff --git a/.circleci/config/commands/install-golang.yml b/.circleci/config/commands/install-golang.yml new file mode 100644 index 000000000..485ffd9e1 --- /dev/null +++ b/.circleci/config/commands/install-golang.yml @@ -0,0 +1,10 @@ +steps: + - run: + name: install golang + command: | + set -x + echo installing golang ${GOLANG_VERSION} + sudo rm -rf /usr/local/go + wget -O /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.linux-amd64.tar.gz + sudo tar -C /usr/local -xzf /tmp/golang.tar.gz + rm -rf /tmp/golang.tar.gz diff --git a/.circleci/config/commands/install-protoc.yml b/.circleci/config/commands/install-protoc.yml new file mode 100644 index 000000000..db0d0dad7 --- /dev/null +++ b/.circleci/config/commands/install-protoc.yml @@ -0,0 +1,6 @@ +steps: + - run: + name: install protoc + command: | + sudo rm -rf /usr/bin/protoc + sudo ./scripts/vagrant-linux-priv-protoc.sh diff --git a/.circleci/config/commands/install-vault.yml b/.circleci/config/commands/install-vault.yml new file mode 100644 index 000000000..3e52cbee2 --- /dev/null +++ b/.circleci/config/commands/install-vault.yml @@ -0,0 +1,11 @@ +parameters: + version: + type: string + default: 1.2.3 +steps: + - run: + name: Install Vault << parameters.version >> + command: | + wget -q -O /tmp/vault.zip https://releases.hashicorp.com/vault/<< parameters.version >>/vault_<< parameters.version>>_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/vault.zip + rm -rf /tmp/vault* diff --git a/.circleci/config/commands/run-tests.yml b/.circleci/config/commands/run-tests.yml new file mode 100644 index 000000000..f72a1d7a5 --- /dev/null +++ b/.circleci/config/commands/run-tests.yml @@ -0,0 +1,18 @@ +steps: + - run: + name: Running Nomad Tests + command: | + if [ -z $GOTEST_PKGS_EXCLUDE ]; + then + unset GOTEST_PKGS_EXCLUDE + else + unset GOTEST_PKGS + fi + + if [ ! -z $GOTESTARCH ]; then + export GOARCH="$GOTESTARCH"; + fi + + mkdir -p /tmp/test-reports + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad diff --git a/.circleci/config/config.yml b/.circleci/config/config.yml new file mode 100644 index 000000000..9d3168971 --- /dev/null +++ b/.circleci/config/config.yml @@ -0,0 +1,39 @@ +version: 2.1 + +references: + # environment specific references - aim to avoid conflicts + go-machine-image: &go_machine_image + circleci/classic:201808-01 + go-machine-recent-image: &go_machine_recent_image + ubuntu-1604:201903-01 + +executors: + go: + working_directory: /go/src/github.com/hashicorp/nomad + docker: + - image: golang:1.12.13 + environment: &common_envs + GOMAXPROCS: 1 + NOMAD_SLOW_TEST: 1 + GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + GOPATH: /go + + go-machine: + working_directory: ~/go/src/github.com/hashicorp/nomad + machine: + image: *go_machine_image + environment: &machine_env + GOMAXPROCS: 1 + NOMAD_SLOW_TEST: 1 + GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + GOPATH: /home/circleci/go + GOLANG_VERSION: "1.12.13" + + + # uses a more recent image with unattended upgrades disabled properly + # but seems to break docker builds + go-machine-recent: + working_directory: ~/go/src/github.com/hashicorp/nomad + machine: + image: *go_machine_recent_image + environment: *machine_env diff --git a/.circleci/config/jobs/build-binaries.yml b/.circleci/config/jobs/build-binaries.yml new file mode 100644 index 000000000..21a0079c5 --- /dev/null +++ b/.circleci/config/jobs/build-binaries.yml @@ -0,0 +1,17 @@ +executor: go +environment: + # TODO: add ui tag here + GO_TAGS: "codegen_generated" +steps: + - checkout + - run: apt-get update; apt-get install -y sudo unzip zip + - run: make deps + - install-protoc + - run: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs + - run: make pkg/windows_amd64.zip pkg/linux_amd64.zip + - store_artifacts: + path: pkg/windows_amd64.zip + destination: /builds/nomad_windows_amd64.zip + - store_artifacts: + path: pkg/linux_amd64.zip + destination: /builds/nomad_linux_amd64.zip diff --git a/.circleci/config/jobs/build-website.yml b/.circleci/config/jobs/build-website.yml new file mode 100644 index 000000000..3dc1384c3 --- /dev/null +++ b/.circleci/config/jobs/build-website.yml @@ -0,0 +1,30 @@ +# setting the working_directory along with the checkout path allows us to not have +# to cd into the website/ directory for commands +working_directory: ~/project/website +docker: + - image: hashicorp/middleman-hashicorp:0.3.35 +steps: + - checkout: + path: ~/project + + # restores gem cache + - restore_cache: + key: static-site-gems-v1-{{ checksum "Gemfile.lock" }} + + - run: + name: install gems + command: bundle check || bundle install --path vendor/bundle --retry=3 + + # saves gem cache if we have changed the Gemfile + - save_cache: + key: static-site-gems-v1-{{ checksum "Gemfile.lock" }} + paths: + - ~/project/website/vendor/bundle + + - run: + name: middleman build + command: bundle exec middleman build + + - run: + name: website deploy + command: ./scripts/deploy.sh diff --git a/.circleci/config/jobs/lint-go.yml b/.circleci/config/jobs/lint-go.yml new file mode 100644 index 000000000..90a9abcab --- /dev/null +++ b/.circleci/config/jobs/lint-go.yml @@ -0,0 +1,8 @@ +executor: go +steps: + - checkout + - run: apt-get update; apt-get install -y shellcheck sudo unzip + - install-protoc + - run: make deps lint-deps + - run: make check + - run: make checkscripts diff --git a/.circleci/config/jobs/test-container.yml b/.circleci/config/jobs/test-container.yml new file mode 100644 index 000000000..744f931d0 --- /dev/null +++ b/.circleci/config/jobs/test-container.yml @@ -0,0 +1,27 @@ +executor: go +parameters: + test_packages: + type: string + default: "" + exclude_packages: + type: string + default: "" + goarch: + type: string + default: "amd64" +environment: + GOTEST_PKGS: "<< parameters.test_packages >>" + GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>" + GOTESTARCH: "<< parameters.goarch >>" +steps: + - checkout + - run: apt-get update; apt-get install -y shellcheck sudo unzip + - run: make deps + - install-protoc + - install-consul + - install-vault + - run-tests + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-reports diff --git a/.circleci/config/jobs/test-e2e.yml b/.circleci/config/jobs/test-e2e.yml new file mode 100644 index 000000000..103cc7fb9 --- /dev/null +++ b/.circleci/config/jobs/test-e2e.yml @@ -0,0 +1,17 @@ +executor: go +steps: + - checkout + - run: apt-get update; apt-get install -y sudo unzip + # e2e tests require privileged mount/umount permissions when running as root + # TODO: switch to using machine executor and run as root to test e2e path + - run: + name: prepare non-root user + command: | + groupadd --gid 3434 circleci + useradd --uid 3434 --gid circleci --shell /bin/bash --create-home circleci + echo 'circleci ALL=NOPASSWD: ALL' >> /etc/sudoers.d/50-circleci + echo 'Defaults env_keep += "DEBIAN_FRONTEND"' >> /etc/sudoers.d/env_keep + chown -R circleci:circleci /go + + - run: sudo -E -H -u circleci PATH=${PATH} make deps + - run: sudo -E -H -u circleci PATH=${PATH} make e2e-test diff --git a/.circleci/config/jobs/test-machine.yml b/.circleci/config/jobs/test-machine.yml new file mode 100644 index 000000000..755a99fcc --- /dev/null +++ b/.circleci/config/jobs/test-machine.yml @@ -0,0 +1,39 @@ +executor: "<< parameters.executor >>" +parameters: + test_packages: + type: string + default: "" + exclude_packages: + type: string + default: "" + executor: + type: string + default: "go-machine-recent" + goarch: + type: string + default: "amd64" +environment: + GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>" + GOTEST_PKGS: "<< parameters.test_packages >>" + GOTESTARCH: "<< parameters.goarch >>" +steps: + - checkout + - install-golang + - install-protoc + - install-consul + - install-vault + - run: + name: Install 32bit gcc libs + command: | + if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then + sudo apt-get update + sudo apt-get install -y gcc-multilib + else + echo "Skipping 32bit lib installation while building for not 386" + fi + - run: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap + - run-tests + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-reports diff --git a/.circleci/config/jobs/test-ui.yml b/.circleci/config/jobs/test-ui.yml new file mode 100644 index 000000000..d29c0f92b --- /dev/null +++ b/.circleci/config/jobs/test-ui.yml @@ -0,0 +1,27 @@ +docker: + - image: circleci/node:10-browsers + environment: + # See https://git.io/vdao3 for details. + JOBS: 2 +steps: + - checkout + - restore_cache: + keys: + - v1-deps-{{ checksum "ui/yarn.lock" }} + - v1-deps- + - run: + name: yarn install + command: cd ui && yarn install + - save_cache: + key: v1-deps-{{ checksum "ui/yarn.lock" }} + paths: + - ./ui/node_modules + - run: + name: lint:js + command: cd ui && yarn run lint:js + - run: + name: lint:hbs + command: cd ui && yarn run lint:hbs + - run: + name: Ember tests + command: cd ui && yarn test diff --git a/.circleci/config/jobs/test-website.yml b/.circleci/config/jobs/test-website.yml new file mode 100644 index 000000000..0a54aa06a --- /dev/null +++ b/.circleci/config/jobs/test-website.yml @@ -0,0 +1,4 @@ +executor: go-machine-recent +steps: + - checkout + - run: make test-website diff --git a/.circleci/config/workflows/build-test.yml b/.circleci/config/workflows/build-test.yml new file mode 100644 index 000000000..136fd4087 --- /dev/null +++ b/.circleci/config/workflows/build-test.yml @@ -0,0 +1,62 @@ +jobs: +- build-binaries +- lint-go: + filters: &backend_branches_filter + branches: + ignore: + - /^.-ui\b.*/ + - stable-website +- test-machine: + name: "test-client" + test_packages: "./client/..." + filters: *backend_branches_filter +- test-machine: + name: "test-nomad" + test_packages: "./nomad/..." + filters: *backend_branches_filter +- test-machine: + # API Tests run in a VM rather than container due to the FS tests + # requiring `mount` priviliges. + name: "test-api" + test_packages: "./api/..." + filters: *backend_branches_filter +- test-container: + name: "test-devices" + test_packages: "./devices/..." + filters: *backend_branches_filter +- test-machine: + name: "test-other" + exclude_packages: "./api|./client|./drivers/docker|./drivers/exec|./drivers/rkt|./drivers/shared/executor|./nomad|./devices" + filters: *backend_branches_filter +- test-machine: + name: "test-docker" + test_packages: "./drivers/docker" + # docker is misbehaving in docker-machine-recent image + # and we get unexpected failures + # e.g. https://circleci.com/gh/hashicorp/nomad/3854 + executor: go-machine + filters: *backend_branches_filter +- test-machine: + name: "test-exec" + test_packages: "./drivers/exec" + filters: *backend_branches_filter +- test-machine: + name: "test-shared-exec" + test_packages: "./drivers/shared/executor" + filters: *backend_branches_filter +- test-machine: + name: "test-32bit" + # Currently we only explicitly test fingerprinting on 32bit + # architectures. + test_packages: "./client/fingerprint" + goarch: "386" + filters: *backend_branches_filter +- test-e2e: + filters: *backend_branches_filter +- test-ui: + filters: + branches: + ignore: + - stable-website +- test-website: + filters: *backend_branches_filter diff --git a/.circleci/config/workflows/website.yml b/.circleci/config/workflows/website.yml new file mode 100644 index 000000000..519c460e5 --- /dev/null +++ b/.circleci/config/workflows/website.yml @@ -0,0 +1,6 @@ +jobs: + - build-website: + context: static-sites + filters: + branches: + only: stable-website From 96a3742ed961e87cb55e4f32ca742de8b73fd111 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Wed, 20 Nov 2019 14:52:25 -0500 Subject: [PATCH 05/67] simply link to vault .circleci README --- .circleci/README.md | 129 +------------------------------------------- 1 file changed, 1 insertion(+), 128 deletions(-) diff --git a/.circleci/README.md b/.circleci/README.md index 1ec75cafa..c21070097 100644 --- a/.circleci/README.md +++ b/.circleci/README.md @@ -1,130 +1,3 @@ # How to use CircleCI multi-file config -This README and the Makefile should be in your `.circleci` directory, -in the root of your repository. -All path references in this README assume we are in this `.circleci` directory. - -The `Makefile` in this directory generates `./config.yml` in CircleCI 2.0 syntax, -from the tree rooted at `./config/`, which contains files in CircleCI 2.0 or 2.1 syntax. - - -## Quickstart - -The basic workflow is: - -- Edit source files in `./config/` -- When you are done, run `make ci-config` to update `./config.yml` -- Commit this entire `.circleci` directory, including that generated file together. -- Run `make ci-verify` to ensure the current `./config.yml` is up to date with the source. - -When merging this `.circleci` directory: - -- Do not merge the generated `./config.yml` file, instead: -- Merge the source files under `./config/`, and then -- Run `make ci-config` to re-generate the merged `./config.yml` - -And that's it, for more detail, read on! - - -## How does it work, roughly? - -CircleCI supports [generating a single config file from many], -using the `$ circleci config pack` command. -It also supports [expanding 2.1 syntax to 2.0 syntax] -using the `$ circleci config process` command. -We use these two commands, stitched together using the `Makefile` -to implement the workflow. - -[generating a single config file from many]: https://circleci.com/docs/2.0/local-cli/#packing-a-config -[expanding 2.1 syntax to 2.0 syntax]: https://circleci.com/docs/2.0/local-cli/#processing-a-config - - -## Prerequisites - -You will need the [CircleCI CLI tool] installed and working, -at least version `0.1.5607`. -You can [download this tool directly from GitHub Releases]. - -``` -$ circleci version -0.1.5607+f705856 -``` - -[CircleCI CLI tool]: https://circleci.com/docs/2.0/local-cli/ -[download this tool directly from GitHub Releases]: https://github.com/CircleCI-Public/circleci-cli/releases - - -## Updating the config source - -Before making changes, be sure to understand the layout -of the `./config/` file tree, as well as circleci 2.1 syntax. -See the [Syntax and layout] section below. - -To update the config, you should edit, add or remove files -in the `./config/` directory, -and then run `make ci-config`. -If that's successful, -you should then commit every `*.yml` file in the tree rooted in this directory. -That is: you should commit both the source under `./config/` -and the generated file `./config.yml` at the same time, in the same commit. -The included git pre-commit hook will help with this. -Do not edit the `./config.yml` file directly, as you will lose your changes -next time `make ci-config` is run. - -[Syntax and layout]: #syntax-and-layout - - -### Verifying `./config.yml` - -To check whether or not the current `./config.yml` is up to date with the source -and valid, run `$ make ci-verify`. -Note that `$ make ci-verify` should be run in CI, -in case not everyone has the git pre-commit hook set up correctly. - - -#### Example shell session - -```sh -$ make ci-config -config.yml updated -$ git add -A . # The -A makes sure to include deletions/renames etc. -$ git commit -m "ci: blah blah blah" -Changes detected in .circleci/, running 'make -C .circleci ci-verify' ---> Generated config.yml is up to date! ---> Config file at config.yml is valid. -``` - - -### Syntax and layout - -It is important to understand the layout of the config directory. -Read the documentation on [packing a config] for a full understanding -of how multiple YAML files are merged by the circleci CLI tool. - -[packing a config]: https://circleci.com/docs/2.0/local-cli/#packing-a-config - -Here is an example file tree (with comments added afterwards): - -```sh -$ tree . -. -├── Makefile -├── README.md # This file. -├── config # The source code for config.yml is rooted here. -│   ├── @config.yml # Files beginning with @ are treated specially by `circleci config pack` -│   ├── commands # Subdirectories of config become top-level keys. -│   │   └── go_test.yml # Filenames (minus .yml) become top-level keys under -│   │   └── go_build.yml # their parent (in this case "commands"). -│ │ # The contents of go_test.yml therefore are placed at: .commands.go_test: -│   └── jobs # jobs also becomes a top-level key under config... -│   ├── build.yml # ...and likewise filenames become keys under their parent. -│   └── test.yml -└── config.yml # The generated file in 2.0 syntax. -``` - -About those `@` files... Preceding a filename with `@` -indicates to `$ circleci config pack` that the contents of this YAML file -should be at the top-level, rather than underneath a key named after their filename. -This naming convention is unfortunate as it breaks autocompletion in bash, -but there we go. - +Refer to https://github.com/hashicorp/vault/blob/master/.circleci/README.md . From 82263f3b79ccff1dd3677915a6f34e72b6d99244 Mon Sep 17 00:00:00 2001 From: Lang Martin Date: Wed, 20 Nov 2019 14:01:10 -0500 Subject: [PATCH 06/67] command: quota status reports network usage --- command/quota_status.go | 16 +++++++++++++--- .../docs/commands/quota/status.html.md.erb | 4 ++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/command/quota_status.go b/command/quota_status.go index 6c7805e3f..f5a0a3690 100644 --- a/command/quota_status.go +++ b/command/quota_status.go @@ -155,7 +155,7 @@ func formatQuotaLimits(spec *api.QuotaSpec, usages map[string]*api.QuotaUsage) s sort.Sort(api.QuotaLimitSort(spec.Limits)) limits := make([]string, len(spec.Limits)+1) - limits[0] = "Region|CPU Usage|Memory Usage" + limits[0] = "Region|CPU Usage|Memory Usage|Network Usage" i := 0 for _, specLimit := range spec.Limits { i++ @@ -171,17 +171,27 @@ func formatQuotaLimits(spec *api.QuotaSpec, usages map[string]*api.QuotaUsage) s return used, ok } + specBits := 0 + if len(specLimit.RegionLimit.Networks) == 1 { + specBits = *specLimit.RegionLimit.Networks[0].MBits + } + used, ok := lookupUsage() if !ok { cpu := fmt.Sprintf("- / %s", formatQuotaLimitInt(specLimit.RegionLimit.CPU)) memory := fmt.Sprintf("- / %s", formatQuotaLimitInt(specLimit.RegionLimit.MemoryMB)) - limits[i] = fmt.Sprintf("%s|%s|%s", specLimit.Region, cpu, memory) + net := fmt.Sprintf("- / %s", formatQuotaLimitInt(&specBits)) + limits[i] = fmt.Sprintf("%s|%s|%s|%s", specLimit.Region, cpu, memory, net) continue } cpu := fmt.Sprintf("%d / %s", *used.RegionLimit.CPU, formatQuotaLimitInt(specLimit.RegionLimit.CPU)) memory := fmt.Sprintf("%d / %s", *used.RegionLimit.MemoryMB, formatQuotaLimitInt(specLimit.RegionLimit.MemoryMB)) - limits[i] = fmt.Sprintf("%s|%s|%s", specLimit.Region, cpu, memory) + net := fmt.Sprintf("- / %s", formatQuotaLimitInt(&specBits)) + if len(used.RegionLimit.Networks) == 1 { + net = fmt.Sprintf("%d / %s", *used.RegionLimit.Networks[0].MBits, formatQuotaLimitInt(&specBits)) + } + limits[i] = fmt.Sprintf("%s|%s|%s|%s", specLimit.Region, cpu, memory, net) } return formatList(limits) diff --git a/website/source/docs/commands/quota/status.html.md.erb b/website/source/docs/commands/quota/status.html.md.erb index 7de58835c..e568c61fb 100644 --- a/website/source/docs/commands/quota/status.html.md.erb +++ b/website/source/docs/commands/quota/status.html.md.erb @@ -36,6 +36,6 @@ Description = Limit the shared default namespace Limits = 1 Quota Limits -Region CPU Usage Memory Usage -global 500 / 2500 256 / 2000 +Region CPU Usage Memory Usage Network Usage +global 500 / 2500 256 / 2000 30 / 50 ``` From 26b39ffac887e5829fefb6bb1270ee27da4605a8 Mon Sep 17 00:00:00 2001 From: Lang Martin Date: Wed, 20 Nov 2019 14:02:25 -0500 Subject: [PATCH 07/67] command: quota init writes files with a network limit --- command/quota_init.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/command/quota_init.go b/command/quota_init.go index a78a356c7..45d712591 100644 --- a/command/quota_init.go +++ b/command/quota_init.go @@ -115,6 +115,9 @@ limit { region_limit { cpu = 2500 memory = 1000 + network { + mbits = 50 + } } } `) @@ -128,7 +131,10 @@ var defaultJsonQuotaSpec = strings.TrimSpace(` "Region": "global", "RegionLimit": { "CPU": 2500, - "MemoryMB": 1000 + "MemoryMB": 1000, + "Networks": [ + { "MBits": 50 } + ] } } ] From 9210bde91a580b6cd7ffe7299db485c3b3c5783f Mon Sep 17 00:00:00 2001 From: Lang Martin Date: Tue, 19 Nov 2019 16:52:03 -0500 Subject: [PATCH 08/67] docs: update quota docs to include network limits --- website/source/api/quotas.html.md | 45 ++++++++++++++++--- .../docs/commands/quota/inspect.html.md.erb | 12 ++++- .../governance-and-policy/quotas.html.md | 7 ++- 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/website/source/api/quotas.html.md b/website/source/api/quotas.html.md index 290d5ced8..988869bbf 100644 --- a/website/source/api/quotas.html.md +++ b/website/source/api/quotas.html.md @@ -62,7 +62,17 @@ $ curl \ "CPU": 2500, "DiskMB": 0, "MemoryMB": 2000, - "Networks": null + "Networks": [ + { + "CIDR": "", + "Device": "", + "DynamicPorts": null, + "IP": "", + "MBits": 50, + "Mode": "", + "ReservedPorts": null + } + ] } } ], @@ -115,7 +125,17 @@ $ curl \ "CPU": 2500, "DiskMB": 0, "MemoryMB": 2000, - "Networks": null + "Networks": [ + { + "CIDR": "", + "Device": "", + "DynamicPorts": null, + "IP": "", + "MBits": 50, + "Mode": "", + "ReservedPorts": null + } + ] } } ], @@ -157,12 +177,17 @@ object](https://github.com/hashicorp/nomad/blob/master/api/quota.go#L100-L131). "Region": "global", "RegionLimit": { "CPU": 2500, - "MemoryMB": 1000 + "MemoryMB": 1000, + "Networks": [ + { + "Mbits": 50 + } + ] } } ] } -``` +``` ### Sample Request @@ -305,7 +330,17 @@ $ curl \ "CPU": 500, "MemoryMB": 256, "DiskMB": 0, - "Networks": null + "Networks": [ + { + "CIDR": "", + "Device": "", + "DynamicPorts": null, + "IP": "", + "MBits": 50, + "Mode": "", + "ReservedPorts": null + } + ] }, "Hash": "NLOoV2WBU8ieJIrYXXx8NRb5C2xU61pVVWRDLEIMxlU=" } diff --git a/website/source/docs/commands/quota/inspect.html.md.erb b/website/source/docs/commands/quota/inspect.html.md.erb index 403e4f2da..cc843126b 100644 --- a/website/source/docs/commands/quota/inspect.html.md.erb +++ b/website/source/docs/commands/quota/inspect.html.md.erb @@ -68,7 +68,17 @@ $ nomad quota inspect default-quota "CPU": 500, "DiskMB": 0, "MemoryMB": 256, - "Networks": null + "Networks": [ + { + "CIDR": "", + "Device": "", + "DynamicPorts": null, + "IP": "", + "MBits": 0, + "Mode": "", + "ReservedPorts": null + } + ] } } } diff --git a/website/source/guides/governance-and-policy/quotas.html.md b/website/source/guides/governance-and-policy/quotas.html.md index e803226bc..1c7f56df7 100644 --- a/website/source/guides/governance-and-policy/quotas.html.md +++ b/website/source/guides/governance-and-policy/quotas.html.md @@ -65,13 +65,18 @@ limit { region_limit { cpu = 2500 memory = 1000 + networks = [ + { mbits = 50 } + ] } } ``` A quota specification is composed of one or more resource limits. Each limit applies to a particular Nomad region. Within the limit object, operators can -specify the allowed CPU and memory usage. +specify the allowed CPU, memory usage, and network bandwidth. Network bandwidth +limits may only specify a single limit for all interfaces. Network quotas were +introduced in 0.10.2 and are optional, they will not be enforced if omitted. To create the particular quota, it is as simple as running: From 47bc949a8fadc6524d6d1d2354938921da61a910 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Wed, 20 Nov 2019 20:22:14 -0500 Subject: [PATCH 09/67] tests: remove TestClient_RestoreError test TestClient_RestoreError is very slow, taking ~81 seconds. It has few problematic patterns. It's unclear what it tests, it simulates a failure condition where all state db lookup fails and asserts that alloc fails. Though starting from https://github.com/hashicorp/nomad/pull/6216 , we don't fail allocs in that condition but rather restart them. Also, the drivers used in second client `c2` are the same singleton instances used in `c1` and already shutdown. We ought to start healthy new driver instances. --- client/client_test.go | 92 ------------------------------------------- 1 file changed, 92 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index f1a4c0013..4c029eaf4 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -29,7 +29,6 @@ import ( "github.com/hashicorp/nomad/testutil" "github.com/stretchr/testify/assert" - hclog "github.com/hashicorp/go-hclog" cstate "github.com/hashicorp/nomad/client/state" ctestutil "github.com/hashicorp/nomad/client/testutil" "github.com/stretchr/testify/require" @@ -637,97 +636,6 @@ func TestClient_SaveRestoreState(t *testing.T) { } } -func TestClient_RestoreError(t *testing.T) { - t.Parallel() - require := require.New(t) - - s1, _ := testServer(t, nil) - defer s1.Shutdown() - testutil.WaitForLeader(t, s1.RPC) - - c1, cleanup := TestClient(t, func(c *config.Config) { - c.DevMode = false - c.RPCHandler = s1 - }) - defer cleanup() - - // Wait until the node is ready - waitTilNodeReady(c1, t) - - // Create mock allocations - job := mock.Job() - alloc1 := mock.Alloc() - alloc1.NodeID = c1.Node().ID - alloc1.Job = job - alloc1.JobID = job.ID - alloc1.Job.TaskGroups[0].Tasks[0].Driver = "mock_driver" - alloc1.Job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ - "run_for": "10s", - } - alloc1.ClientStatus = structs.AllocClientStatusRunning - - state := s1.State() - err := state.UpsertJob(100, job) - require.Nil(err) - - err = state.UpsertJobSummary(101, mock.JobSummary(alloc1.JobID)) - require.Nil(err) - - err = state.UpsertAllocs(102, []*structs.Allocation{alloc1}) - require.Nil(err) - - // Allocations should get registered - testutil.WaitForResult(func() (bool, error) { - c1.allocLock.RLock() - ar := c1.allocs[alloc1.ID] - c1.allocLock.RUnlock() - if ar == nil { - return false, fmt.Errorf("nil alloc runner") - } - if ar.Alloc().ClientStatus != structs.AllocClientStatusRunning { - return false, fmt.Errorf("client status: got %v; want %v", ar.Alloc().ClientStatus, structs.AllocClientStatusRunning) - } - return true, nil - }, func(err error) { - t.Fatalf("err: %v", err) - }) - - // Shutdown the client, saves state - if err := c1.Shutdown(); err != nil { - t.Fatalf("err: %v", err) - } - - // Create a new client with a stateDB implementation that errors - logger := testlog.HCLogger(t) - c1.config.Logger = logger - catalog := consul.NewMockCatalog(logger) - mockService := consulApi.NewMockConsulServiceClient(t, logger) - - // This stateDB returns errors for all methods called by restore - stateDBFunc := func(hclog.Logger, string) (cstate.StateDB, error) { - return &cstate.ErrDB{Allocs: []*structs.Allocation{alloc1}}, nil - } - c1.config.StateDBFactory = stateDBFunc - - c2, err := NewClient(c1.config, catalog, mockService) - require.Nil(err) - defer c2.Shutdown() - - // Ensure the allocation has been marked as failed on the server - testutil.WaitForResult(func() (bool, error) { - alloc, err := s1.State().AllocByID(nil, alloc1.ID) - require.Nil(err) - failed := alloc.ClientStatus == structs.AllocClientStatusFailed - if !failed { - return false, fmt.Errorf("Expected failed client status, but got %v", alloc.ClientStatus) - } - return true, nil - }, func(err error) { - require.NoError(err) - }) - -} - func TestClient_AddAllocError(t *testing.T) { t.Parallel() require := require.New(t) From 77a0064fcd2a2e21fa3dfa920a105aa9b2e0f914 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Wed, 20 Nov 2019 20:29:27 -0500 Subject: [PATCH 10/67] testS: fix TestClient_RestoreError When spinning a second client, ensure that it uses new driver instances, rather than reuse the already shutdown unhealthy drivers from first instance. This speeds up tests significantly, but cutting ~50 seconds or so, the timeout in NewClient until drivers fingerprints. They never do because drivers were shutdown already. --- client/client_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index 4c029eaf4..8f677f1fe 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -18,6 +18,7 @@ import ( "github.com/hashicorp/nomad/client/state" "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper/pluginutils/catalog" + "github.com/hashicorp/nomad/helper/pluginutils/singleton" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad" @@ -602,10 +603,14 @@ func TestClient_SaveRestoreState(t *testing.T) { // Create a new client logger := testlog.HCLogger(t) c1.config.Logger = logger - catalog := consul.NewMockCatalog(logger) + consulCatalog := consul.NewMockCatalog(logger) mockService := consulApi.NewMockConsulServiceClient(t, logger) - c2, err := NewClient(c1.config, catalog, mockService) + // ensure we use non-shutdown driver instances + c1.config.PluginLoader = catalog.TestPluginLoaderWithOptions(t, "", c1.config.Options, nil) + c1.config.PluginSingletonLoader = singleton.NewSingletonLoader(logger, c1.config.PluginLoader) + + c2, err := NewClient(c1.config, consulCatalog, mockService) if err != nil { t.Fatalf("err: %v", err) } From b886e154879aa3280a058c3670b0e0f156128447 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Wed, 20 Nov 2019 20:32:11 -0500 Subject: [PATCH 11/67] tests: run TestClient_WatchAllocs in non-linux environments --- client/client_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index 8f677f1fe..8899b0fd8 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -31,7 +31,6 @@ import ( "github.com/stretchr/testify/assert" cstate "github.com/hashicorp/nomad/client/state" - ctestutil "github.com/hashicorp/nomad/client/testutil" "github.com/stretchr/testify/require" ) @@ -447,7 +446,6 @@ func TestClient_UpdateAllocStatus(t *testing.T) { func TestClient_WatchAllocs(t *testing.T) { t.Parallel() - ctestutil.ExecCompatible(t) s1, _ := testServer(t, nil) defer s1.Shutdown() testutil.WaitForLeader(t, s1.RPC) @@ -462,6 +460,11 @@ func TestClient_WatchAllocs(t *testing.T) { // Create mock allocations job := mock.Job() + job.TaskGroups[0].Count = 3 + job.TaskGroups[0].Tasks[0].Driver = "mock_driver" + job.TaskGroups[0].Tasks[0].Config = map[string]interface{}{ + "run_for": "10s", + } alloc1 := mock.Alloc() alloc1.JobID = job.ID alloc1.Job = job From bb81fce18e6eb1e127745119c1d64bf7484280bf Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Wed, 20 Nov 2019 20:34:11 -0500 Subject: [PATCH 12/67] tests: don't assume eth0 network is available TestClient_UpdateNodeFromFingerprintKeepsConfig checks a test node network interface, which is hardcoded to `eth0` and is updated asynchronously. This causes flakiness when eth0 isn't available. Here, we hardcode the value to an arbitrary network interface. --- client/client_test.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/client/client_test.go b/client/client_test.go index 8899b0fd8..898ff32f0 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -3,6 +3,7 @@ package client import ( "fmt" "io/ioutil" + "net" "os" "path/filepath" "runtime" @@ -1171,7 +1172,6 @@ func TestClient_UpdateNodeFromFingerprintKeepsConfig(t *testing.T) { client, cleanup := TestClient(t, nil) defer cleanup() // capture the platform fingerprinted device name for the next test - dev := client.config.Node.NodeResources.Networks[0].Device client.updateNodeFromFingerprint(&fingerprint.FingerprintResponse{ NodeResources: &structs.NodeResources{ Cpu: structs.NodeCpuResources{CpuShares: 123}, @@ -1187,6 +1187,14 @@ func TestClient_UpdateNodeFromFingerprintKeepsConfig(t *testing.T) { assert.Equal(t, 80, client.config.Node.Resources.CPU) assert.Equal(t, "any-interface", client.config.Node.Resources.Networks[0].Device) + // lookup an interface. client.Node starts with a hardcoded value, eth0, + // and is only updated async through fingerprinter. + // Let's just lookup network device; anyone will do for this test + interfaces, err := net.Interfaces() + require.NoError(t, err) + require.NotEmpty(t, interfaces) + dev := interfaces[0].Name + // Client with network interface configured keeps the config // setting on update name := "TestClient_UpdateNodeFromFingerprintKeepsConfig2" From 720b41c47ef26206faa89fa37939270b8b51b733 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Wed, 20 Nov 2019 20:39:23 -0500 Subject: [PATCH 13/67] ci: match ci timeout to go test timeout make test-nomad sets 15 minute time out for build. Increase the ci timeout to 20m, so we can get meaningful output and goroutine stack traces rather than have test be simply killed by CircleCI. The extra 5 minutes is a buffer for generating-structs and some unnecessary padding. --- .circleci/config.yml | 9 +++++++++ .circleci/config/commands/run-tests.yml | 1 + 2 files changed, 10 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 83f5280fc..d5a887dc9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -73,6 +73,7 @@ jobs: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad name: Running Nomad Tests + no_output_timeout: 20m - store_test_results: path: /tmp/test-reports - store_artifacts: @@ -146,6 +147,7 @@ jobs: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad name: Running Nomad Tests + no_output_timeout: 20m - store_test_results: path: /tmp/test-reports - store_artifacts: @@ -219,6 +221,7 @@ jobs: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad name: Running Nomad Tests + no_output_timeout: 20m - store_test_results: path: /tmp/test-reports - store_artifacts: @@ -292,6 +295,7 @@ jobs: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad name: Running Nomad Tests + no_output_timeout: 20m - store_test_results: path: /tmp/test-reports - store_artifacts: @@ -365,6 +369,7 @@ jobs: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad name: Running Nomad Tests + no_output_timeout: 20m - store_test_results: path: /tmp/test-reports - store_artifacts: @@ -511,6 +516,7 @@ jobs: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad name: Running Nomad Tests + no_output_timeout: 20m - store_test_results: path: /tmp/test-reports - store_artifacts: @@ -567,6 +573,7 @@ jobs: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad name: Running Nomad Tests + no_output_timeout: 20m - store_test_results: path: /tmp/test-reports - store_artifacts: @@ -654,6 +661,7 @@ jobs: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad name: Running Nomad Tests + no_output_timeout: 20m - store_test_results: path: /tmp/test-reports - store_artifacts: @@ -783,6 +791,7 @@ jobs: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad name: Running Nomad Tests + no_output_timeout: 20m - store_test_results: path: /tmp/test-reports - store_artifacts: diff --git a/.circleci/config/commands/run-tests.yml b/.circleci/config/commands/run-tests.yml index f72a1d7a5..9a90841d3 100644 --- a/.circleci/config/commands/run-tests.yml +++ b/.circleci/config/commands/run-tests.yml @@ -1,6 +1,7 @@ steps: - run: name: Running Nomad Tests + no_output_timeout: 20m command: | if [ -z $GOTEST_PKGS_EXCLUDE ]; then From c661d37ca20aa9a9799a04f1b16e89daeb833c1d Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 21 Nov 2019 08:28:20 -0500 Subject: [PATCH 14/67] fixup! tests: don't assume eth0 network is available --- client/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/client_test.go b/client/client_test.go index 898ff32f0..3b4533b9e 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -1171,7 +1171,7 @@ func TestClient_UpdateNodeFromFingerprintKeepsConfig(t *testing.T) { // Client without network configured updates to match fingerprint client, cleanup := TestClient(t, nil) defer cleanup() - // capture the platform fingerprinted device name for the next test + client.updateNodeFromFingerprint(&fingerprint.FingerprintResponse{ NodeResources: &structs.NodeResources{ Cpu: structs.NodeCpuResources{CpuShares: 123}, From 7b16c98a9c9c47bcd90cdfff063eb43199f8d0a5 Mon Sep 17 00:00:00 2001 From: Lang Martin Date: Thu, 21 Nov 2019 10:34:05 -0500 Subject: [PATCH 15/67] CHANGELOG quota network --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f86ec1c0..37df777f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ IMPROVEMENTS: * client: Enable setting tags on Consul Connect sidecar service [[GH-6448](https://github.com/hashicorp/nomad/issues/6448)] * client: Add support for downloading artifacts from Google Cloud Storage [[GH-6692](https://github.com/hashicorp/nomad/pull/6692)] * command: add -tls-server-name flag [[GH-6370](https://github.com/hashicorp/nomad/issues/6370)] + * quota: Add support for network bandwidth quota limits in Nomad enterprise BUG FIXES: From 1c301bd3680dd1789f7407a597f3178631df09ef Mon Sep 17 00:00:00 2001 From: Drew Bailey <2614075+drewbailey@users.noreply.github.com> Date: Thu, 21 Nov 2019 14:51:48 -0500 Subject: [PATCH 16/67] Allows a node uuid prefix to be passed in --- command/agent_monitor.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/command/agent_monitor.go b/command/agent_monitor.go index 4d50faf6a..4dc466d2a 100644 --- a/command/agent_monitor.go +++ b/command/agent_monitor.go @@ -89,6 +89,28 @@ func (c *MonitorCommand) Run(args []string) int { return 1 } + // Query the node info and lookup prefix + if len(nodeID) == 1 { + c.Ui.Error(fmt.Sprintf("Node identifier must contain at least two characters.")) + return 1 + } + + if nodeID != "" { + nodeID = sanitizeUUIDPrefix(nodeID) + nodes, _, err := client.Nodes().PrefixList(nodeID) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error querying node: %v", err)) + return 1 + } + + if len(nodes) > 1 { + out := formatNodeStubList(nodes, false) + c.Ui.Output(fmt.Sprintf("Prefix matched multiple nodes\n\n%s", out)) + return 1 + } + nodeID = nodes[0].ID + } + params := map[string]string{ "log_level": logLevel, "node_id": nodeID, From 4f1890b5feabcec4f440f6f035f617941317dac0 Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Thu, 21 Nov 2019 14:18:06 -0600 Subject: [PATCH 17/67] docs: vault integration docs should reference new token_period field --- dev/vault/nomad-cluster-role.json | 2 +- website/source/data/vault/nomad-cluster-role.json | 2 +- website/source/docs/vault-integration/index.html.md | 4 ++-- .../guides/integrations/vault-integration/index.html.md | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/vault/nomad-cluster-role.json b/dev/vault/nomad-cluster-role.json index 37728363f..3017ae36e 100644 --- a/dev/vault/nomad-cluster-role.json +++ b/dev/vault/nomad-cluster-role.json @@ -3,6 +3,6 @@ "explicit_max_ttl": 0, "name": "nomad-cluster", "orphan": false, - "period": 259200, + "token_period": 259200, "renewable": true } diff --git a/website/source/data/vault/nomad-cluster-role.json b/website/source/data/vault/nomad-cluster-role.json index dc2621524..21909d50c 100644 --- a/website/source/data/vault/nomad-cluster-role.json +++ b/website/source/data/vault/nomad-cluster-role.json @@ -3,6 +3,6 @@ "explicit_max_ttl": 0, "name": "nomad-cluster", "orphan": true, - "period": 259200, + "token_period": 259200, "renewable": true } diff --git a/website/source/docs/vault-integration/index.html.md b/website/source/docs/vault-integration/index.html.md index fff3434f1..0871043db 100644 --- a/website/source/docs/vault-integration/index.html.md +++ b/website/source/docs/vault-integration/index.html.md @@ -141,7 +141,7 @@ An example token role definition is given below: "explicit_max_ttl": 0, "name": "nomad-cluster", "orphan": true, - "period": 259200, + "token_period": 259200, "renewable": true } ``` @@ -196,7 +196,7 @@ documentation for all possible fields and more complete documentation. making bootstrapping and upgrading simpler. As such, **setting `orphan = true` is the recommended setting**. -* `period` - Specifies the length the TTL is extended by each renewal in +* `token_period` - Specifies the length the TTL is extended by each renewal in seconds. It is suggested to set this value on the order of magnitude of 3 days (259200 seconds) to avoid a large renewal request rate to Vault. **Must be set to a positive value**. diff --git a/website/source/guides/integrations/vault-integration/index.html.md b/website/source/guides/integrations/vault-integration/index.html.md index 07b5c1c8f..3803efcf9 100644 --- a/website/source/guides/integrations/vault-integration/index.html.md +++ b/website/source/guides/integrations/vault-integration/index.html.md @@ -192,7 +192,7 @@ submitted to Nomad. We will use the following token role: "explicit_max_ttl": 0, "name": "nomad-cluster", "orphan": true, - "period": 259200, + "token_period": 259200, "renewable": true } ``` @@ -213,7 +213,7 @@ disallowed policies group. An example of this is shown below: "explicit_max_ttl": 0, "name": "nomad-cluster", "orphan": true, - "period": 259200, + "token_period": 259200, "renewable": true } ``` From 022afcb6023ea39f65262ed948ae53661a25b924 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 21 Nov 2019 15:49:49 -0500 Subject: [PATCH 18/67] changelog GH-6580 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37df777f6..08d510805 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ BUG FIXES: register [[GH-6646](https://github.com/hashicorp/nomad/issues/6646)] * scheduler: Changes to devices in resource stanza should cause rescheduling [[GH-6644](https://github.com/hashicorp/nomad/issues/6644)] * vault: Allow overriding implicit Vault version constraint [[GH-6687](https://github.com/hashicorp/nomad/issues/6687)] - * vault: Supported Vault auth role's new field, `token_period` [[GH-6574](https://github.com/hashicorp/nomad/issues/6574)] + * vault: Supported Vault auth role's new fields, `token_period` and `token_explicit_max_ttl` [[GH-6574](https://github.com/hashicorp/nomad/issues/6574)], [[GH-6580](https://github.com/hashicorp/nomad/issues/6580)] * scheduler: Fixed a bug that allowed inplace updates after a constraint, affinity, or spread was changed [[GH-6703](https://github.com/hashicorp/nomad/issues/6703)] ## 0.10.1 (November 4, 2019) From f2e2efac3843ed4a39adbf6d0a701ec05ed8b072 Mon Sep 17 00:00:00 2001 From: Drew Bailey <2614075+drewbailey@users.noreply.github.com> Date: Thu, 21 Nov 2019 16:01:09 -0500 Subject: [PATCH 19/67] add server-id to -h output --- command/agent_monitor.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/command/agent_monitor.go b/command/agent_monitor.go index 4dc466d2a..8fd2fdf19 100644 --- a/command/agent_monitor.go +++ b/command/agent_monitor.go @@ -30,7 +30,7 @@ Usage: nomad monitor [options] General Options: ` + generalOptionsUsage() + ` - + Monitor Specific Options: -log-level @@ -39,6 +39,9 @@ Monitor Specific Options: -node-id Sets the specific node to monitor + -server-id + Sets the specific server to monitor + -json Sets log output to JSON format ` From e23db928ec6879f0afd5e10c347d1e1104094978 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Thu, 21 Nov 2019 14:36:56 -0800 Subject: [PATCH 20/67] docs: reformat changelog for 0.10.2-rc1 --- CHANGELOG.md | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08d510805..425cfb9cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,19 +1,26 @@ ## 0.10.2 (Unreleased) FEATURES: - * core: Add `nomad monitor` command to stream logs at a specified level for debugging [[GH-6499](https://github.com/hashicorp/nomad/issues/6499)] + + * **Nomad Monitor**: New `nomad monitor` command allows remotely following + the logs of any Nomad Agent (clients or servers). See + https://nomadproject.io/docs/commands/monitor.html + * **Docker Container Cleanup**: Nomad will now automatically remove Docker + containers for tasks leaked due to Nomad or Docker crashes or bugs. IMPROVEMENTS: - * api: Add `StartedAt` field to `Node.DrainStrategy` [[GH-6698](https://github.com/hashicorp/nomad/issues/6698)] + + * agent: Added support for running under Windows Service Manager [[GH-6220](https://github.com/hashicorp/nomad/issues/6220)] + * api: Added `StartedAt` field to `Node.DrainStrategy` [[GH-6698](https://github.com/hashicorp/nomad/issues/6698)] * api: Added JSON representation of rules to policy endpoint response [[GH-6017](https://github.com/hashicorp/nomad/pull/6017)] * api: Update policy endpoint to permit anonymous access [[GH-6021](https://github.com/hashicorp/nomad/issues/6021)] * build: Updated to Go 1.12.13 [[GH-6606](https://github.com/hashicorp/nomad/issues/6606)] - * core: Add support for running under Windows Service Manager [[GH-6220](https://github.com/hashicorp/nomad/issues/6220)] * cli: Show full ID in node and alloc individual status views [[GH-6425](https://github.com/hashicorp/nomad/issues/6425)] * client: Enable setting tags on Consul Connect sidecar service [[GH-6448](https://github.com/hashicorp/nomad/issues/6448)] - * client: Add support for downloading artifacts from Google Cloud Storage [[GH-6692](https://github.com/hashicorp/nomad/pull/6692)] - * command: add -tls-server-name flag [[GH-6370](https://github.com/hashicorp/nomad/issues/6370)] - * quota: Add support for network bandwidth quota limits in Nomad enterprise + * client: Added support for downloading artifacts from Google Cloud Storage [[GH-6692](https://github.com/hashicorp/nomad/pull/6692)] + * command: Added -tls-server-name flag [[GH-6370](https://github.com/hashicorp/nomad/issues/6370)] + * command: Added `nomad monitor` command to stream logs at a specified level for debugging [[GH-6499](https://github.com/hashicorp/nomad/issues/6499)] + * quota: Added support for network bandwidth quota limits in Nomad enterprise BUG FIXES: @@ -23,19 +30,18 @@ BUG FIXES: * api: Decompress web socket response body if gzipped on error responses [[GH-6650](https://github.com/hashicorp/nomad/issues/6650)] * api: Fixed a bug where some FS/Allocation API endpoints didn't return error messages [[GH-6427](https://github.com/hashicorp/nomad/issues/6427)] * api: Return 40X status code for failing ACL requests, rather than 500 [[GH-6421](https://github.com/hashicorp/nomad/issues/6421)] - * cli: Make scoring column orders consistent `nomad alloc status` [[GH-6609](https://github.com/hashicorp/nomad/issues/6609)] + * cli: Made scoring column orders consistent `nomad alloc status` [[GH-6609](https://github.com/hashicorp/nomad/issues/6609)] * cli: Fixed a bug where `nomad alloc exec` fails if stdout is being redirected and not a TTY [[GH-6684](https://github.com/hashicorp/nomad/issues/6684)] * cli: Fixed a bug where a cli user may fail to query FS/Allocation API endpoints if they lack `node:read` capability [[GH-6423](https://github.com/hashicorp/nomad/issues/6423)] - * client: Fixed a bug where a client may not restart dead internal processes upon client's restart on Windows [[GH-6426](https://github.com/hashicorp/nomad/issues/6426)] * client: client: Return empty values when host stats fail [[GH-6349](https://github.com/hashicorp/nomad/issues/6349)] + * client: Fixed a bug where a client may not restart dead internal processes upon client's restart on Windows [[GH-6426](https://github.com/hashicorp/nomad/issues/6426)] * driver/exec: Fixed a bug where exec tasks can spawn processes that live beyond task lifecycle [[GH-6722](https://github.com/hashicorp/nomad/issues/6722)] * driver/docker: Added mechanism for detecting running unexpectedly running docker containers [[GH-6325](https://github.com/hashicorp/nomad/issues/6325)] - * nomad: Multiple connect enabled services in the same taskgroup failed to - register [[GH-6646](https://github.com/hashicorp/nomad/issues/6646)] + * nomad: Fixed registering multiple connect enabled services in the same task group [[GH-6646](https://github.com/hashicorp/nomad/issues/6646)] * scheduler: Changes to devices in resource stanza should cause rescheduling [[GH-6644](https://github.com/hashicorp/nomad/issues/6644)] + * scheduler: Fixed a bug that allowed inplace updates after affinity or spread were changed [[GH-6703](https://github.com/hashicorp/nomad/issues/6703)] * vault: Allow overriding implicit Vault version constraint [[GH-6687](https://github.com/hashicorp/nomad/issues/6687)] * vault: Supported Vault auth role's new fields, `token_period` and `token_explicit_max_ttl` [[GH-6574](https://github.com/hashicorp/nomad/issues/6574)], [[GH-6580](https://github.com/hashicorp/nomad/issues/6580)] - * scheduler: Fixed a bug that allowed inplace updates after a constraint, affinity, or spread was changed [[GH-6703](https://github.com/hashicorp/nomad/issues/6703)] ## 0.10.1 (November 4, 2019) From bb2a482ddde6058652e3f10fd9df112526d29105 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Thu, 21 Nov 2019 14:37:09 -0800 Subject: [PATCH 21/67] docs: add Nomad v0.10.2-rc1 download link --- website/source/downloads.html.erb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/website/source/downloads.html.erb b/website/source/downloads.html.erb index 391440f05..fb58aa0c5 100644 --- a/website/source/downloads.html.erb +++ b/website/source/downloads.html.erb @@ -36,6 +36,11 @@ description: |-

Check out the v<%= latest_version %> CHANGELOG for information on the latest release.

+ +

Nomad 0.10.2 Release Candidate

+

+ A release candidate for Nomad 0.10.2 is also available! The release candidate can be downloaded on the Nomad releases page. +

From 9062380c1144b16920d6e35a98b6d44ca47dc1ce Mon Sep 17 00:00:00 2001 From: Chris Baker <1675087+cgbaker@users.noreply.github.com> Date: Thu, 21 Nov 2019 19:50:50 +0000 Subject: [PATCH 22/67] added the device plugin authoring guide, made minor formatting changes to task driver plugin authoring guide. --- .../docs/internals/plugins/devices.html.md | 77 ++++++++++++++++++- .../internals/plugins/task-drivers.html.md | 16 ++-- 2 files changed, 82 insertions(+), 11 deletions(-) diff --git a/website/source/docs/internals/plugins/devices.html.md b/website/source/docs/internals/plugins/devices.html.md index a7a7e205f..6b137aa10 100644 --- a/website/source/docs/internals/plugins/devices.html.md +++ b/website/source/docs/internals/plugins/devices.html.md @@ -3,10 +3,81 @@ layout: "docs" page_title: "Device Plugins" sidebar_current: "docs-internals-plugins-devices" description: |- - Learn about how to author a Nomad device plugin. + Learn how to author a Nomad device plugin. --- # Devices -Device plugin documentation is currently a work in progress. Until there is -documentation, the [Nvidia GPU plugin](https://github.com/hashicorp/nomad/tree/master/devices/gpu/nvidia) is a useful example. +Nomad has built-in support for scheduling compute resources such as CPU, memory, +and networking. Nomad device plugins are used to support scheduling tasks with +other devices, such as GPUs. They are responsible for fingerprinting these +devices and working with the Nomad client to make them available to assigned +tasks. + +For a real world example of a Nomad device plugin implementation, see the [Nvidia +GPU plugin](https://github.com/hashicorp/nomad/tree/master/devices/gpu/nvidia). + +## Authoring Device Plugins + +Authoring a device plugin in Nomad consists of implementing the +[DevicePlugin][devicePlugin] interface alongside +a main package to launch the plugin. + +The [device plugin skeleton project][skeletonProject] exists to help bootstrap +the development of new device plugins. It provides most of the boilerplate +necessary for a device plugin, along with detailed comments. + +### Lifecycle and State + +A device plugin is long-lived. Nomad will ensure that one instance of the plugin is +running. If the plugin crashes or otherwise terminates, Nomad will launch another +instance of it. + +However, unlike [task drivers](task-drivers.html), device plugins do not currently +have an interface for persisting state to the Nomad client. Instead, the device +plugin API emphasizes fingerprinting devices and reporting their status. After +helping to provision a task with a scheduled device, a device plugin does not +have any responsibility (or ability) to monitor the task. + +## Device Plugin API + +The [base plugin][baseplugin] must be implemented in addition to the following +functions. + +### `Fingerprint(context.Context) (<-chan *FingerprintResponse, error)` + +The `Fingerprint` [function][fingerprintFn] is called by the client when the plugin is started. +It allows the plugin to provide Nomad with a list of discovered devices, along with their +attributes, for the purpose of scheduling workloads using devices. +The channel returned should immediately send an initial +[`FingerprintResponse`][fingerprintResponse], then send periodic updates at +an appropriate interval until the context is canceled. + +Each fingerprint response consists of either an error or a list of device groups. +A device group is a list of detected devices that are identical for the purpose of +scheduling; that is, they will have identical attributes. + +### `Stats(context.Context, time.Duration) (<-chan *StatsResponse, error)` + +The `Stats` [function][statsFn] returns a channel on which the plugin should +emit device statistics, at the specified interval, until either an error is +encountered or the specified context is cancelled. The `StatsReponse` object +allows [dimensioned][dimensioned] statistics to be returned for each device in a device group. + +### `Reserve(deviceIDs []string) (*ContainerReservation, error)` + +The `Reserve` [function][reserveFn] accepts a list of device IDs and returns the information +necessary for the client to make those devices available to a task. Currently, +the `ContainerReservation` object allows the plugin to specify environment +variables for the task, as well as a list of host devices and files to be mounted +into the task's filesystem. Any orchestration required to prepare the device for +use should also be performed in this function. + +[DevicePlugin]: https://github.com/hashicorp/nomad/blob/v0.9.0/plugins/device/device.go#L20-L33 +[baseplugin]: /docs/internals/plugins/base.html +[skeletonProject]: https://github.com/hashicorp/nomad-skeleton-device-plugin +[fingerprintResponse]: https://github.com/hashicorp/nomad/blob/v0.9.0/plugins/device/device.go#L37-L43 +[fingerprintFn]: https://github.com/hashicorp/nomad-skeleton-device-plugin/blob/v0.1.0/device/device.go#L159-L165 +[statsFn]: https://github.com/hashicorp/nomad-skeleton-device-plugin/blob/v0.1.0/device/device.go#L169-L176 +[reserveFn]: https://github.com/hashicorp/nomad-skeleton-device-plugin/blob/v0.1.0/device/device.go#L189-L245 +[dimensioned]: https://github.com/hashicorp/nomad/blob/v0.9.0/plugins/shared/structs/stats.go#L33-L34 diff --git a/website/source/docs/internals/plugins/task-drivers.html.md b/website/source/docs/internals/plugins/task-drivers.html.md index a6dd06bce..8b0d72724 100644 --- a/website/source/docs/internals/plugins/task-drivers.html.md +++ b/website/source/docs/internals/plugins/task-drivers.html.md @@ -3,22 +3,22 @@ layout: "docs" page_title: "Task Driver Plugins" sidebar_current: "docs-internals-plugins-task-drivers" description: |- - Learn about how to author a Nomad plugin. + Learn how to author a Nomad task driver plugin. --- # Task Drivers Task drivers in Nomad are the runtime components that execute workloads. For -a real world example of a Nomad task driver plugin implementation see the [LXC +a real world example of a Nomad task driver plugin implementation, see the [LXC driver source][lxcdriver]. ## Authoring Task Driver Plugins Authoring a task driver (shortened to driver in this documentation) in Nomad consists of implementing the [DriverPlugin][driverplugin] interface and adding -a main package to launch the plugin. A driver plugin is long lived and its +a main package to launch the plugin. A driver plugin is long-lived and its lifetime is not bound to the Nomad client. This means that the Nomad client can -be restarted without the restarting the driver. Nomad will ensure that one +be restarted without restarting the driver. Nomad will ensure that one instance of the driver is running, meaning if the driver crashes or otherwise terminates, Nomad will launch another instance of it. @@ -29,7 +29,7 @@ Nomad client can recover tasks into the driver state. ## Task Driver Plugin API -The [base plugin][baseplugin] must be implement in addition to the following +The [base plugin][baseplugin] must be implemented in addition to the following functions. ### `TaskConfigSchema() (*hclspec.Spec, error)` @@ -123,7 +123,7 @@ returned by the `StartTask` function. If no error was returned, it is expected that the driver can now operate on the task by referencing the task ID. If an error occurs, the Nomad client will mark the task as `lost`. -### `WaitTask(ctx context.Context, id string) (<-chan *ExitResult, error)` +### `WaitTask(context.Context, id string) (<-chan *ExitResult, error)` The `WaitTask` function is expected to return a channel that will send an `*ExitResult` when the task exits or close the channel when the context is @@ -153,7 +153,7 @@ called. The `InspectTask` function returns detailed status information for the referenced `taskID`. -### `TaskStats(ctx context.Context, id string, i time.Duration) (<-chan *cstructs.TaskResourceUsage, error)` +### `TaskStats(context.Context, id string, time.Duration) (<-chan *cstructs.TaskResourceUsage, error)` The `TaskStats` function returns a channel which the driver should send stats to at the given interval. The driver must send stats at the given interval @@ -188,7 +188,7 @@ inside the running container. `ExecTask` is called for Consul script checks. [lxcdriver]: https://github.com/hashicorp/nomad-driver-lxc -[DriverPlugin]: https://github.com/hashicorp/nomad/blob/v0.9.0-beta2/plugins/drivers/driver.go#L39-L57 +[DriverPlugin]: https://github.com/hashicorp/nomad/blob/v0.9.0/plugins/drivers/driver.go#L39-L57 [baseplugin]: /docs/internals/plugins/base.html [taskconfig]: https://godoc.org/github.com/hashicorp/nomad/plugins/drivers#TaskConfig [taskhandle]: https://godoc.org/github.com/hashicorp/nomad/plugins/drivers#TaskHandle From 538695d1c1f4d16ac2db83d86a9167e1c7ddff33 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 08:43:09 -0500 Subject: [PATCH 23/67] acl_endpoint: permission denied for unauthenticated requests If ACL Request is unauthenticated, we should honor the anonymous token. This PR makes few changes: * `GetPolicy` endpoints may return policy if anonymous policy allows it, or return permission denied otherwise. * `ListPolicies` returns an empty policy list, or one with anonymous policy if one exists. Without this PR, the we return an incomprehensible error. Before: ``` $ curl http://localhost:4646/v1/acl/policy/doesntexist; echo acl token lookup failed: index error: UUID must be 36 characters $ curl http://localhost:4646/v1/acl/policies; echo acl token lookup failed: index error: UUID must be 36 characters ``` After: ``` $ curl http://localhost:4646/v1/acl/policy/doesntexist; echo Permission denied $ curl http://localhost:4646/v1/acl/policies; echo [] ``` --- nomad/acl_endpoint.go | 48 ++++++++++++++--------------- nomad/acl_endpoint_test.go | 63 ++++++++++++++++++++++++++++++++++++++ nomad/state/state_store.go | 8 +++++ 3 files changed, 94 insertions(+), 25 deletions(-) diff --git a/nomad/acl_endpoint.go b/nomad/acl_endpoint.go index cfcc0c3c7..f28a45c6a 100644 --- a/nomad/acl_endpoint.go +++ b/nomad/acl_endpoint.go @@ -119,6 +119,7 @@ func (a *ACL) ListPolicies(args *structs.ACLPolicyListRequest, reply *structs.AC if !a.srv.config.ACLEnabled { return aclDisabled } + if done, err := a.srv.forward("ACL.ListPolicies", args, args, reply); done { return err } @@ -136,12 +137,7 @@ func (a *ACL) ListPolicies(args *structs.ACLPolicyListRequest, reply *structs.AC mgt := acl.IsManagement() var policies map[string]struct{} if !mgt { - snap, err := a.srv.fsm.State().Snapshot() - if err != nil { - return err - } - - token, err := snap.ACLTokenBySecretID(nil, args.AuthToken) + token, err := a.requestAuthToken(args.AuthToken) if err != nil { return err } @@ -207,6 +203,7 @@ func (a *ACL) GetPolicy(args *structs.ACLPolicySpecificRequest, reply *structs.S if !a.srv.config.ACLEnabled { return aclDisabled } + if done, err := a.srv.forward("ACL.GetPolicy", args, args, reply); done { return err } @@ -224,12 +221,7 @@ func (a *ACL) GetPolicy(args *structs.ACLPolicySpecificRequest, reply *structs.S // If it is not a management token determine if it can get this policy mgt := acl.IsManagement() if !mgt && args.Name != "anonymous" { - snap, err := a.srv.fsm.State().Snapshot() - if err != nil { - return err - } - - token, err := snap.ACLTokenBySecretID(nil, args.AuthToken) + token, err := a.requestAuthToken(args.AuthToken) if err != nil { return err } @@ -284,6 +276,19 @@ func (a *ACL) GetPolicy(args *structs.ACLPolicySpecificRequest, reply *structs.S return a.srv.blockingRPC(&opts) } +func (a *ACL) requestAuthToken(secretID string) (*structs.ACLToken, error) { + if secretID == "" { + return structs.AnonymousACLToken, nil + } + + snap, err := a.srv.fsm.State().Snapshot() + if err != nil { + return nil, err + } + + return snap.ACLTokenBySecretID(nil, secretID) +} + // GetPolicies is used to get a set of policies func (a *ACL) GetPolicies(args *structs.ACLPolicySetRequest, reply *structs.ACLPolicySetResponse) error { if !a.srv.config.ACLEnabled { @@ -294,19 +299,12 @@ func (a *ACL) GetPolicies(args *structs.ACLPolicySetRequest, reply *structs.ACLP } defer metrics.MeasureSince([]string{"nomad", "acl", "get_policies"}, time.Now()) - var token *structs.ACLToken - var err error - if args.AuthToken == "" { - // No need to look up the anonymous token - token = structs.AnonymousACLToken - } else { - // For client typed tokens, allow them to query any policies associated with that token. - // This is used by clients which are resolving the policies to enforce. Any associated - // policies need to be fetched so that the client can determine what to allow. - token, err = a.srv.State().ACLTokenBySecretID(nil, args.AuthToken) - if err != nil { - return err - } + // For client typed tokens, allow them to query any policies associated with that token. + // This is used by clients which are resolving the policies to enforce. Any associated + // policies need to be fetched so that the client can determine what to allow. + token, err := a.requestAuthToken(args.AuthToken) + if err != nil { + return err } if token == nil { diff --git a/nomad/acl_endpoint_test.go b/nomad/acl_endpoint_test.go index 6357d5bcc..eb388087b 100644 --- a/nomad/acl_endpoint_test.go +++ b/nomad/acl_endpoint_test.go @@ -89,6 +89,18 @@ func TestACLEndpoint_GetPolicy(t *testing.T) { } assert.EqualValues(t, 1001, resp3.Index) assert.Equal(t, anonymousPolicy, resp3.Policy) + + // Lookup non-anonoymous policy with no token + get = &structs.ACLPolicySpecificRequest{ + Name: policy.Name, + QueryOptions: structs.QueryOptions{ + Region: "global", + }, + } + var resp4 structs.SingleACLPolicyResponse + err := msgpackrpc.CallWithCodec(codec, "ACL.GetPolicy", get, &resp4) + require.Error(t, err) + require.Contains(t, err.Error(), structs.ErrPermissionDenied.Error()) } func TestACLEndpoint_GetPolicy_Blocking(t *testing.T) { @@ -395,6 +407,57 @@ func TestACLEndpoint_ListPolicies(t *testing.T) { } } +// TestACLEndpoint_ListPolicies_Unauthenticated asserts that +// unauthenticated ListPolicies returns anonymous policy if one +// exists, otherwise, empty +func TestACLEndpoint_ListPolicies_Unauthenticated(t *testing.T) { + t.Parallel() + s1, _ := TestACLServer(t, nil) + defer s1.Shutdown() + codec := rpcClient(t, s1) + testutil.WaitForLeader(t, s1.RPC) + + listPolicies := func() (*structs.ACLPolicyListResponse, error) { + // Lookup the policies + get := &structs.ACLPolicyListRequest{ + QueryOptions: structs.QueryOptions{ + Region: "global", + }, + } + + var resp structs.ACLPolicyListResponse + err := msgpackrpc.CallWithCodec(codec, "ACL.ListPolicies", get, &resp) + if err != nil { + return nil, err + } + return &resp, nil + } + + p1 := mock.ACLPolicy() + p1.Name = "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9" + s1.fsm.State().UpsertACLPolicies(1000, []*structs.ACLPolicy{p1}) + + t.Run("no anonymous policy", func(t *testing.T) { + resp, err := listPolicies() + require.NoError(t, err) + require.Empty(t, resp.Policies) + require.Equal(t, uint64(1000), resp.Index) + }) + + // now try with anonymous policy + p2 := mock.ACLPolicy() + p2.Name = "anonymous" + s1.fsm.State().UpsertACLPolicies(1001, []*structs.ACLPolicy{p2}) + + t.Run("with anonymous policy", func(t *testing.T) { + resp, err := listPolicies() + require.NoError(t, err) + require.Len(t, resp.Policies, 1) + require.Equal(t, "anonymous", resp.Policies[0].Name) + require.Equal(t, uint64(1001), resp.Index) + }) +} + func TestACLEndpoint_ListPolicies_Blocking(t *testing.T) { t.Parallel() s1, root := TestACLServer(t, nil) diff --git a/nomad/state/state_store.go b/nomad/state/state_store.go index ab61390b5..a04e824a0 100644 --- a/nomad/state/state_store.go +++ b/nomad/state/state_store.go @@ -3769,6 +3769,10 @@ func (s *StateStore) DeleteACLTokens(index uint64, ids []string) error { // ACLTokenByAccessorID is used to lookup a token by accessor ID func (s *StateStore) ACLTokenByAccessorID(ws memdb.WatchSet, id string) (*structs.ACLToken, error) { + if id == "" { + return nil, fmt.Errorf("acl token lookup failed: missing accessor id") + } + txn := s.db.Txn(false) watchCh, existing, err := txn.FirstWatch("acl_token", "id", id) @@ -3785,6 +3789,10 @@ func (s *StateStore) ACLTokenByAccessorID(ws memdb.WatchSet, id string) (*struct // ACLTokenBySecretID is used to lookup a token by secret ID func (s *StateStore) ACLTokenBySecretID(ws memdb.WatchSet, secretID string) (*structs.ACLToken, error) { + if secretID == "" { + return nil, fmt.Errorf("acl token lookup failed: missing secret id") + } + txn := s.db.Txn(false) watchCh, existing, err := txn.FirstWatch("acl_token", "secret", secretID) From bfe08cf8872ba9d2ba4953efe0bf55456fcca371 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 09:58:00 -0500 Subject: [PATCH 24/67] document docker dangling container repeaper --- website/source/docs/drivers/docker.html.md | 48 +++++++++++++++++-- .../guides/upgrade/upgrade-specific.html.md | 11 +++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/website/source/docs/drivers/docker.html.md b/website/source/docs/drivers/docker.html.md index 62d4943d8..7e280628e 100644 --- a/website/source/docs/drivers/docker.html.md +++ b/website/source/docs/drivers/docker.html.md @@ -166,7 +166,7 @@ The `docker` driver supports the following configuration in the job spec. Only } ``` -* `logging` - (Optional) A key-value map of Docker logging options. +* `logging` - (Optional) A key-value map of Docker logging options. Defaults to `json-file` with log rotation (`max-file=2` and `max-size=2m`). ```hcl @@ -648,6 +648,13 @@ plugin "docker" { image = true image_delay = "3m" container = true + + dangling_containers { + enabled = true + dry_run = false + period = "5m" + creation_grace = "5m" + } } volumes { @@ -690,7 +697,7 @@ plugin "docker" { * `config` - Allows an operator to specify a JSON file which is in the dockercfg format containing authentication information for a private registry, from either (in order) `auths`, - `credHelpers` or `credsStore`. + `credHelpers` or `credsStore`. * `helper` - Allows an operator to specify a [credsStore](https://docs.docker.com/engine/reference/commandline/login/#credential-helper-protocol) -like script on $PATH to lookup authentication information from external @@ -719,6 +726,16 @@ plugin "docker" { * `container` - Defaults to `true`. This option can be used to disable Nomad from removing a container when the task exits. Under a name conflict, Nomad may still remove the dead container. + * `dangling_containers` stanza for controlling dangling container detection + and cleanup: + * `enabled` - Defaults to `true`). Enables dangling container handling + * `dry_run` - Defaults to `false`. Enables a mode where nomad logs + potential dangling containers without killing them. + * `period` - Defaults to `"5m"`. A time duration that controls interval + between Nomad scans for dangling containers. + * `creation_grace` - Defaults to `"5m"`. A time duration that controls + how long a container can run before it is tracked by Nomad or gets + marked (and killed) as a dangling container * `volumes` stanza: * `enabled` - Defaults to `true`. Allows tasks to bind host paths @@ -894,7 +911,32 @@ need a higher degree of isolation between processes for security or other reasons, it is recommended to use full virtualization like [QEMU](/docs/drivers/qemu.html). -## Docker for Windows Caveats +## Caveats + +### Dangling Containers + +Nomad 0.10.2 introduces a detector and a reaper for docker dangling containers, +containers that Nomad starts yet does not manage or track. Though rare, they +sometimes in very loaded clusters and lead to unexpectedly running services, +potentially with stale versions. + +When docker daemon becomes unavailable as Nomad starts a task, it is possible +for Docker to successfully start the container and fails the API call with 500 +error code. In such cases, Nomad retries and eventually aims to kill such +containers. However, if the Docker Engine remains unhealthy, subsequent retries +and stop attempts may still fail, and the started container becomes a dangling +container that Nomad no longer manges. + +The newly added reaper periodically scans for such containers. It only targets +containers with a `com.hashicorp.nomad.allocation_id` label, or match Nomad's +conventions for naming and bind-mounts (i.e. `/alloc`, `/secrets`, `local`). +Containers that don't match Nomad container patterns are left untouched. + +Operators can run the reaper in a dry mode, where it only logs dangling +container ids without killing them, or simply disable it through +the `gc.dangling_containers` config stanza. + +### Docker for Windows Docker for Windows only supports running Windows containers. Because Docker for Windows is relatively new and rapidly evolving you may want to consult the diff --git a/website/source/guides/upgrade/upgrade-specific.html.md b/website/source/guides/upgrade/upgrade-specific.html.md index 3b93d8649..9afb6d291 100644 --- a/website/source/guides/upgrade/upgrade-specific.html.md +++ b/website/source/guides/upgrade/upgrade-specific.html.md @@ -15,6 +15,16 @@ details provided for their upgrades as a result of new features or changed behavior. This page is used to document those details separately from the standard upgrade flow. +## Nomad 0.10.2 + +Nomad 0.10.2 addresses an issue occurring in heavily loaded clients, where +containers are started without being properly managed by Nomad. Nomad 0.10.2 +introduced a reaper that detects and kills such containers. + +Operators may opt to run reaper in a dry mode or disabling it through a client config. + +For more information, see [Docker Dangling containers][dangling-containers]. + ## Nomad 0.10.0 ### Deployments @@ -364,6 +374,7 @@ deleted and then Nomad 0.3.0 can be launched. [drain-api]: /api/nodes.html#drain-node [drain-cli]: /docs/commands/node/drain.html +[dangling-containers]: /docs/drivers/docker.html#dangling-containers [hcl2]: https://github.com/hashicorp/hcl2 [lxc]: /docs/drivers/external/lxc.html [migrate]: /docs/job-specification/migrate.html From ac9547e6b2d47a49f8ad27ad2a3a25b7e04b0f38 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 10:44:59 -0500 Subject: [PATCH 25/67] drivers: always initialize taskHandle.logger Looks like the RecoverTask doesn't set taskHandle.logger field causing a panic when the handle attempts to log (e.g. when Shutdown or Signaling fails). --- drivers/exec/driver.go | 1 + drivers/exec/driver_pre09.go | 1 + drivers/java/driver.go | 1 + drivers/java/driver_pre09.go | 1 + drivers/qemu/driver.go | 1 + drivers/qemu/driver_pre09.go | 1 + drivers/rkt/driver.go | 1 + drivers/rkt/driver_pre09.go | 1 + 8 files changed, 8 insertions(+) diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index 6611ec472..6c0bf7c99 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -304,6 +304,7 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { procState: drivers.TaskStateRunning, startedAt: taskState.StartedAt, exitResult: &drivers.ExitResult{}, + logger: d.logger, } d.tasks.Set(taskState.TaskConfig.ID, h) diff --git a/drivers/exec/driver_pre09.go b/drivers/exec/driver_pre09.go index 5b063ea24..607142d9e 100644 --- a/drivers/exec/driver_pre09.go +++ b/drivers/exec/driver_pre09.go @@ -36,6 +36,7 @@ func (d *Driver) recoverPre09Task(h *drivers.TaskHandle) error { procState: drivers.TaskStateRunning, startedAt: time.Now(), exitResult: &drivers.ExitResult{}, + logger: d.logger, } d.tasks.Set(h.Config.ID, th) diff --git a/drivers/java/driver.go b/drivers/java/driver.go index ac35aa82d..02931cdcf 100644 --- a/drivers/java/driver.go +++ b/drivers/java/driver.go @@ -290,6 +290,7 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { procState: drivers.TaskStateRunning, startedAt: taskState.StartedAt, exitResult: &drivers.ExitResult{}, + logger: d.logger, } d.tasks.Set(taskState.TaskConfig.ID, h) diff --git a/drivers/java/driver_pre09.go b/drivers/java/driver_pre09.go index 80d1fc5ca..9f1b3f31f 100644 --- a/drivers/java/driver_pre09.go +++ b/drivers/java/driver_pre09.go @@ -36,6 +36,7 @@ func (d *Driver) recoverPre09Task(h *drivers.TaskHandle) error { procState: drivers.TaskStateRunning, startedAt: time.Now(), exitResult: &drivers.ExitResult{}, + logger: d.logger, } d.tasks.Set(h.Config.ID, th) diff --git a/drivers/qemu/driver.go b/drivers/qemu/driver.go index 156dd1104..f24bda73d 100644 --- a/drivers/qemu/driver.go +++ b/drivers/qemu/driver.go @@ -286,6 +286,7 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { procState: drivers.TaskStateRunning, startedAt: taskState.StartedAt, exitResult: &drivers.ExitResult{}, + logger: d.logger, } d.tasks.Set(taskState.TaskConfig.ID, h) diff --git a/drivers/qemu/driver_pre09.go b/drivers/qemu/driver_pre09.go index 10d810ee0..eeef3d29f 100644 --- a/drivers/qemu/driver_pre09.go +++ b/drivers/qemu/driver_pre09.go @@ -36,6 +36,7 @@ func (d *Driver) recoverPre09Task(h *drivers.TaskHandle) error { procState: drivers.TaskStateRunning, startedAt: time.Now(), exitResult: &drivers.ExitResult{}, + logger: d.logger, } d.tasks.Set(h.Config.ID, th) diff --git a/drivers/rkt/driver.go b/drivers/rkt/driver.go index 238dcfc77..6de9f4dfe 100644 --- a/drivers/rkt/driver.go +++ b/drivers/rkt/driver.go @@ -405,6 +405,7 @@ func (d *Driver) RecoverTask(handle *drivers.TaskHandle) error { procState: drivers.TaskStateRunning, startedAt: taskState.StartedAt, exitResult: &drivers.ExitResult{}, + logger: d.logger, } d.tasks.Set(taskState.TaskConfig.ID, h) diff --git a/drivers/rkt/driver_pre09.go b/drivers/rkt/driver_pre09.go index e974e9c09..a30c80ad4 100644 --- a/drivers/rkt/driver_pre09.go +++ b/drivers/rkt/driver_pre09.go @@ -36,6 +36,7 @@ func (d *Driver) recoverPre09Task(h *drivers.TaskHandle) error { procState: drivers.TaskStateRunning, startedAt: time.Now(), exitResult: &drivers.ExitResult{}, + logger: d.logger, } d.tasks.Set(h.Config.ID, th) From dd46c985d5db314d41a4d29742c560058893a312 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 11:04:24 -0500 Subject: [PATCH 26/67] changelog [ci skip] --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 425cfb9cc..4693be800 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ BUG FIXES: * cli: Fixed a bug where a cli user may fail to query FS/Allocation API endpoints if they lack `node:read` capability [[GH-6423](https://github.com/hashicorp/nomad/issues/6423)] * client: client: Return empty values when host stats fail [[GH-6349](https://github.com/hashicorp/nomad/issues/6349)] * client: Fixed a bug where a client may not restart dead internal processes upon client's restart on Windows [[GH-6426](https://github.com/hashicorp/nomad/issues/6426)] + * drivers: Fixed a bug where client may panic if a restored task failed to shutdown cleanly [[GH-6763](https://github.com/hashicorp/nomad/issues/6763)] * driver/exec: Fixed a bug where exec tasks can spawn processes that live beyond task lifecycle [[GH-6722](https://github.com/hashicorp/nomad/issues/6722)] * driver/docker: Added mechanism for detecting running unexpectedly running docker containers [[GH-6325](https://github.com/hashicorp/nomad/issues/6325)] * nomad: Fixed registering multiple connect enabled services in the same task group [[GH-6646](https://github.com/hashicorp/nomad/issues/6646)] From 275c94b86c3c73933d4a60fb1468f496d2e80641 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 11:24:13 -0500 Subject: [PATCH 27/67] git: only .circleci/config.yml is a generated one --- .circleci/.gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/.gitattributes b/.circleci/.gitattributes index 2dd06ee5f..f7c6b31eb 100644 --- a/.circleci/.gitattributes +++ b/.circleci/.gitattributes @@ -1 +1 @@ -config.yml linguist-generated +/config.yml linguist-generated From ff096037f43c436951ee7ea7aeb3c14aa7ce8305 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 11:29:06 -0500 Subject: [PATCH 28/67] ci: ignore docs only changes Skip running backend and UI changes for docs/website updates, as indicated by branch name. --- .circleci/config.yml | 13 +++++++++++++ .circleci/config/workflows/build-test.yml | 2 ++ 2 files changed, 15 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d5a887dc9..973d2e825 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -805,77 +805,90 @@ workflows: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-client: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-nomad: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-api: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-devices: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-other: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-docker: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-exec: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-shared-exec: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-32bit: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-e2e: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-ui: filters: branches: ignore: - stable-website + - /^docs-.*/ - test-website: filters: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website website: jobs: diff --git a/.circleci/config/workflows/build-test.yml b/.circleci/config/workflows/build-test.yml index 136fd4087..3cbc270d7 100644 --- a/.circleci/config/workflows/build-test.yml +++ b/.circleci/config/workflows/build-test.yml @@ -5,6 +5,7 @@ jobs: branches: ignore: - /^.-ui\b.*/ + - /^docs-.*/ - stable-website - test-machine: name: "test-client" @@ -58,5 +59,6 @@ jobs: branches: ignore: - stable-website + - /^docs-.*/ - test-website: filters: *backend_branches_filter From 78178cb8923669427b4b9da37adac1a9a6d25725 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 11:51:10 -0500 Subject: [PATCH 29/67] ci: avoid paging --- .circleci/config.yml | 26 ++++++++++++++++++++++++++ .circleci/config/config.yml | 23 ++++++++++++++--------- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 973d2e825..02c78ce4e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,11 +9,13 @@ jobs: image: ubuntu-1604:201903-01 working_directory: ~/go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: ./nomad/... - GOTEST_PKGS_EXCLUDE: '' - GOTESTARCH: amd64 @@ -83,11 +85,13 @@ jobs: image: ubuntu-1604:201903-01 working_directory: ~/go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: ./api/... - GOTEST_PKGS_EXCLUDE: '' - GOTESTARCH: amd64 @@ -157,11 +161,13 @@ jobs: image: ubuntu-1604:201903-01 working_directory: ~/go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: ./drivers/exec - GOTEST_PKGS_EXCLUDE: '' - GOTESTARCH: amd64 @@ -231,11 +237,13 @@ jobs: image: ubuntu-1604:201903-01 working_directory: ~/go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: ./client/... - GOTEST_PKGS_EXCLUDE: '' - GOTESTARCH: amd64 @@ -305,11 +313,13 @@ jobs: image: ubuntu-1604:201903-01 working_directory: ~/go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: ./drivers/shared/executor - GOTEST_PKGS_EXCLUDE: '' - GOTESTARCH: amd64 @@ -421,10 +431,12 @@ jobs: - run: command: make checkscripts environment: + - GIT_PAGER: cat - GOMAXPROCS: 1 - GOPATH: /go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat build-website: docker: - image: hashicorp/middleman-hashicorp:0.3.35 @@ -452,11 +464,13 @@ jobs: image: ubuntu-1604:201903-01 working_directory: ~/go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: '' - GOTEST_PKGS_EXCLUDE: ./api|./client|./drivers/docker|./drivers/exec|./drivers/rkt|./drivers/shared/executor|./nomad|./devices - GOTESTARCH: amd64 @@ -526,10 +540,12 @@ jobs: - image: golang:1.12.13 working_directory: /go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOMAXPROCS: 1 - GOPATH: /go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: ./devices/... - GOTEST_PKGS_EXCLUDE: '' - GOTESTARCH: amd64 @@ -587,21 +603,25 @@ jobs: - run: command: make test-website environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat test-docker: machine: image: circleci/classic:201808-01 working_directory: ~/go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: ./drivers/docker - GOTEST_PKGS_EXCLUDE: '' - GOTESTARCH: amd64 @@ -671,10 +691,12 @@ jobs: - image: golang:1.12.13 working_directory: /go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOMAXPROCS: 1 - GOPATH: /go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GO_TAGS: codegen_generated steps: - checkout @@ -718,20 +740,24 @@ jobs: - run: command: sudo -E -H -u circleci PATH=${PATH} make e2e-test environment: + - GIT_PAGER: cat - GOMAXPROCS: 1 - GOPATH: /go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat test-32bit: machine: image: ubuntu-1604:201903-01 working_directory: ~/go/src/github.com/hashicorp/nomad environment: + - GIT_PAGER: cat - GOLANG_VERSION: 1.12.13 - GOMAXPROCS: 1 - GOPATH: /home/circleci/go - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - NOMAD_SLOW_TEST: 1 + - PAGER: cat - GOTEST_PKGS: ./client/fingerprint - GOTEST_PKGS_EXCLUDE: '' - GOTESTARCH: '386' diff --git a/.circleci/config/config.yml b/.circleci/config/config.yml index 9d3168971..c18e455f2 100644 --- a/.circleci/config/config.yml +++ b/.circleci/config/config.yml @@ -7,29 +7,34 @@ references: go-machine-recent-image: &go_machine_recent_image ubuntu-1604:201903-01 + # common references + common_envs: &common_envs + GOMAXPROCS: 1 + NOMAD_SLOW_TEST: 1 + GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + # disable implicit git paging. CircleCI runs commands with in a tty + # making git assume it's an interactive session. + PAGER: cat + GIT_PAGER: cat + executors: go: working_directory: /go/src/github.com/hashicorp/nomad docker: - image: golang:1.12.13 - environment: &common_envs - GOMAXPROCS: 1 - NOMAD_SLOW_TEST: 1 - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + environment: + <<: *common_envs GOPATH: /go - + go-machine: working_directory: ~/go/src/github.com/hashicorp/nomad machine: image: *go_machine_image environment: &machine_env - GOMAXPROCS: 1 - NOMAD_SLOW_TEST: 1 - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + <<: *common_envs GOPATH: /home/circleci/go GOLANG_VERSION: "1.12.13" - # uses a more recent image with unattended upgrades disabled properly # but seems to break docker builds go-machine-recent: From 9da5a4f86dece24c05e5999f9f903cf4414febdb Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 13:03:20 -0500 Subject: [PATCH 30/67] Apply suggestions from code review Co-Authored-By: Michael Schurter --- website/source/docs/drivers/docker.html.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/website/source/docs/drivers/docker.html.md b/website/source/docs/drivers/docker.html.md index 7e280628e..88f991b0b 100644 --- a/website/source/docs/drivers/docker.html.md +++ b/website/source/docs/drivers/docker.html.md @@ -728,7 +728,7 @@ plugin "docker" { Nomad may still remove the dead container. * `dangling_containers` stanza for controlling dangling container detection and cleanup: - * `enabled` - Defaults to `true`). Enables dangling container handling + * `enabled` - Defaults to `true`. Enables dangling container handling. * `dry_run` - Defaults to `false`. Enables a mode where nomad logs potential dangling containers without killing them. * `period` - Defaults to `"5m"`. A time duration that controls interval @@ -915,24 +915,24 @@ reasons, it is recommended to use full virtualization like ### Dangling Containers -Nomad 0.10.2 introduces a detector and a reaper for docker dangling containers, +Nomad 0.10.2 introduces a detector and a reaper for dangling Docker containers, containers that Nomad starts yet does not manage or track. Though rare, they sometimes in very loaded clusters and lead to unexpectedly running services, potentially with stale versions. -When docker daemon becomes unavailable as Nomad starts a task, it is possible +When Docker daemon becomes unavailable as Nomad starts a task, it is possible for Docker to successfully start the container and fails the API call with 500 error code. In such cases, Nomad retries and eventually aims to kill such containers. However, if the Docker Engine remains unhealthy, subsequent retries and stop attempts may still fail, and the started container becomes a dangling -container that Nomad no longer manges. +container that Nomad no longer manages. The newly added reaper periodically scans for such containers. It only targets containers with a `com.hashicorp.nomad.allocation_id` label, or match Nomad's conventions for naming and bind-mounts (i.e. `/alloc`, `/secrets`, `local`). Containers that don't match Nomad container patterns are left untouched. -Operators can run the reaper in a dry mode, where it only logs dangling +Operators can run the reaper in a dry run mode, where it only logs dangling container ids without killing them, or simply disable it through the `gc.dangling_containers` config stanza. From cba071b5d87c9647a0ece7e1b7b2da671225f6f1 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 13:07:54 -0500 Subject: [PATCH 31/67] address more review comments --- website/source/docs/drivers/docker.html.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/website/source/docs/drivers/docker.html.md b/website/source/docs/drivers/docker.html.md index 88f991b0b..6adadd745 100644 --- a/website/source/docs/drivers/docker.html.md +++ b/website/source/docs/drivers/docker.html.md @@ -729,13 +729,15 @@ plugin "docker" { * `dangling_containers` stanza for controlling dangling container detection and cleanup: * `enabled` - Defaults to `true`. Enables dangling container handling. - * `dry_run` - Defaults to `false`. Enables a mode where nomad logs - potential dangling containers without killing them. - * `period` - Defaults to `"5m"`. A time duration that controls interval + * `dry_run` - Defaults to `false`. Only log dangling containers without + cleaning them up. + * `period` - Defaults to `"5m"`. A time duration that controls interval between Nomad scans for dangling containers. - * `creation_grace` - Defaults to `"5m"`. A time duration that controls - how long a container can run before it is tracked by Nomad or gets - marked (and killed) as a dangling container + * `creation_grace` - Defaults to `"5m"`. Grace period after a container is + created during which the GC ignores it. Only used to prevent the GC from + removing newly created containers before they are registered with the + GC. Should not need adjusting higher but may be adjusted lower to GC + more aggressively. * `volumes` stanza: * `enabled` - Defaults to `true`. Allows tasks to bind host paths @@ -917,8 +919,7 @@ reasons, it is recommended to use full virtualization like Nomad 0.10.2 introduces a detector and a reaper for dangling Docker containers, containers that Nomad starts yet does not manage or track. Though rare, they -sometimes in very loaded clusters and lead to unexpectedly running services, -potentially with stale versions. +lead to unexpectedly running services, potentially with stale versions. When Docker daemon becomes unavailable as Nomad starts a task, it is possible for Docker to successfully start the container and fails the API call with 500 From c0e0125fb1b66788e94e3ee03cfc6640f65d4054 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Fri, 22 Nov 2019 10:18:10 -0800 Subject: [PATCH 32/67] docs: update connect limitations --- .../integrations/consul-connect/index.html.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/website/source/guides/integrations/consul-connect/index.html.md b/website/source/guides/integrations/consul-connect/index.html.md index eecc8cc59..2cc8fe5e3 100644 --- a/website/source/guides/integrations/consul-connect/index.html.md +++ b/website/source/guides/integrations/consul-connect/index.html.md @@ -323,13 +323,19 @@ dashes (`-`) are converted to underscores (`_`) in environment variables so - The `consul` binary must be present in Nomad's `$PATH` to run the Envoy proxy sidecar on client nodes. - - Consul Connect Native is not yet supported. - - Consul Connect HTTP and gRPC checks are not yet supported. - - Consul ACLs are not yet supported. - - Only the Docker, exec, raw_exec, and java drivers support network namespaces + - Consul Connect Native is not yet supported ([#6083][gh6083]). + - Consul Connect HTTP and gRPC checks are not yet supported ([#6120][gh6120]). + - Consul ACLs are not yet supported ([#6701][gh6701]). + - Only the Docker, `exec`, `raw_exec`, and `java` drivers support network namespaces and Connect. - - Variable interpolation for group services and checks are not yet supported. + - Changes to the `connect` stanza may not properly trigger a job update + ([#6459][gh6459]). Changing a `meta` variable is the suggested workaround as + this will always cause an update to occur. - Consul Connect and network namespaces are only supported on Linux. [count-dashboard]: /assets/images/count-dashboard.png +[gh6083]: https://github.com/hashicorp/nomad/issues/6083 +[gh6120]: https://github.com/hashicorp/nomad/issues/6120 +[gh6701]: https://github.com/hashicorp/nomad/issues/6701 +[gh6459]: https://github.com/hashicorp/nomad/issues/6459 From 41bf4e8ecf3a1c029a815ba773d6cf81cd174532 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 13:22:14 -0500 Subject: [PATCH 33/67] docs: address more GH-6762 review comments Incorporate suggestions in https://github.com/hashicorp/nomad/pull/6762#pullrequestreview-321716747 [ci skip] --- website/source/docs/drivers/docker.html.md | 10 +++++----- website/source/guides/upgrade/upgrade-specific.html.md | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/website/source/docs/drivers/docker.html.md b/website/source/docs/drivers/docker.html.md index 6adadd745..1ca370818 100644 --- a/website/source/docs/drivers/docker.html.md +++ b/website/source/docs/drivers/docker.html.md @@ -922,8 +922,8 @@ containers that Nomad starts yet does not manage or track. Though rare, they lead to unexpectedly running services, potentially with stale versions. When Docker daemon becomes unavailable as Nomad starts a task, it is possible -for Docker to successfully start the container and fails the API call with 500 -error code. In such cases, Nomad retries and eventually aims to kill such +for Docker to successfully start the container but return a 500 error code from +the API call. In such cases, Nomad retries and eventually aims to kill such containers. However, if the Docker Engine remains unhealthy, subsequent retries and stop attempts may still fail, and the started container becomes a dangling container that Nomad no longer manages. @@ -933,9 +933,9 @@ containers with a `com.hashicorp.nomad.allocation_id` label, or match Nomad's conventions for naming and bind-mounts (i.e. `/alloc`, `/secrets`, `local`). Containers that don't match Nomad container patterns are left untouched. -Operators can run the reaper in a dry run mode, where it only logs dangling -container ids without killing them, or simply disable it through -the `gc.dangling_containers` config stanza. +Operators can run the reaper in a dry-run mode, where it only logs dangling +container ids without killing them, or disable it by setting the +`gc.dangling_containers` config stanza. ### Docker for Windows diff --git a/website/source/guides/upgrade/upgrade-specific.html.md b/website/source/guides/upgrade/upgrade-specific.html.md index 9afb6d291..e626a37c2 100644 --- a/website/source/guides/upgrade/upgrade-specific.html.md +++ b/website/source/guides/upgrade/upgrade-specific.html.md @@ -21,7 +21,7 @@ Nomad 0.10.2 addresses an issue occurring in heavily loaded clients, where containers are started without being properly managed by Nomad. Nomad 0.10.2 introduced a reaper that detects and kills such containers. -Operators may opt to run reaper in a dry mode or disabling it through a client config. +Operators may opt to run reaper in a dry-mode or disabling it through a client config. For more information, see [Docker Dangling containers][dangling-containers]. From a44449161b5deb338fe96eb75cca271f02119afa Mon Sep 17 00:00:00 2001 From: Charlie Voiselle <464492+angrycub@users.noreply.github.com> Date: Fri, 22 Nov 2019 14:29:12 -0500 Subject: [PATCH 34/67] Migrating the Load-balancing guides to Nomad Added redirects and stub-ified the index page --- .../guides/load-balancing/fabio.html.md | 233 -------------- .../guides/load-balancing/haproxy.html.md | 279 ----------------- .../load-balancing/load-balancing.html.md | 17 +- .../guides/load-balancing/nginx.html.md | 293 ------------------ .../guides/load-balancing/traefik.html.md | 265 ---------------- website/source/redirects.txt | 5 + 6 files changed, 12 insertions(+), 1080 deletions(-) delete mode 100644 website/source/guides/load-balancing/fabio.html.md delete mode 100644 website/source/guides/load-balancing/haproxy.html.md delete mode 100644 website/source/guides/load-balancing/nginx.html.md delete mode 100644 website/source/guides/load-balancing/traefik.html.md diff --git a/website/source/guides/load-balancing/fabio.html.md b/website/source/guides/load-balancing/fabio.html.md deleted file mode 100644 index ad379f9f5..000000000 --- a/website/source/guides/load-balancing/fabio.html.md +++ /dev/null @@ -1,233 +0,0 @@ ---- -layout: "guides" -page_title: "Load Balancing with Nomad" -sidebar_current: "guides-load-balancing-fabio" -description: |- - There are multiple approaches to load balancing within a Nomad cluster. - One approach involves using [fabio][fabio]. Fabio integrates natively - with Consul and provides rich features with an optional Web UI. ---- - -# Load Balancing with Fabio - -[Fabio][fabio] integrates natively with Consul and provides an optional Web UI -to visualize routing. - -The main use case for fabio is to distribute incoming HTTP(S) and TCP requests -from the internet to frontend services that can handle these requests. This -guide will show you one such example using [Apache][apache] web server. - -## Reference Material - -- [Fabio](https://github.com/fabiolb/fabio) on GitHub -- [Load Balancing Strategies for Consul](https://www.hashicorp.com/blog/load-balancing-strategies-for-consul) -- [Elastic Load Balancing][elb] - -## Estimated Time to Complete - -20 minutes - -## Challenge - -Think of a scenario where a Nomad operator needs to configure an environment to -make Apache web server highly available behind an endpoint and distribute -incoming traffic evenly. - -## Solution - -Deploy fabio as a -[system][system] -scheduler so that it can route incoming traffic evenly to the Apache web server -group regardless of which client nodes Apache is running on. Place all client nodes -behind an [AWS load balancer][elb] to -provide the end user with a single endpoint for access. - -## Prerequisites - -To perform the tasks described in this guide, you need to have a Nomad -environment with Consul installed. You can use this -[repo](https://github.com/hashicorp/nomad/tree/master/terraform#provision-a-nomad-cluster-in-the-cloud) -to easily provision a sandbox environment. This guide will assume a cluster with -one server node and three client nodes. - --> **Please Note:** This guide is for demo purposes and is only using a single server -node. In a production cluster, 3 or 5 server nodes are recommended. - -## Steps - -### Step 1: Create a Job for Fabio - -Create a job for Fabio and name it `fabio.nomad` - -```hcl -job "fabio" { - datacenters = ["dc1"] - type = "system" - - group "fabio" { - task "fabio" { - driver = "docker" - config { - image = "fabiolb/fabio" - network_mode = "host" - } - - resources { - cpu = 200 - memory = 128 - network { - mbits = 20 - port "lb" { - static = 9999 - } - port "ui" { - static = 9998 - } - } - } - } - } -} -``` - -Setting `type` to [system][system] will ensure that fabio is run on all clients. -Please note that the `network_mode` option is set to `host` so that fabio can -communicate with Consul which is also running on the client nodes. - -### Step 2: Run the Fabio Job - -We can now register our fabio job: - -```shell -$ nomad job run fabio.nomad -==> Monitoring evaluation "fba4f04a" - Evaluation triggered by job "fabio" - Allocation "6e6367d4" created: node "f3739267", group "fabio" - Allocation "d17573b4" created: node "28d7f859", group "fabio" - Allocation "f3ad9b16" created: node "510898b6", group "fabio" - Evaluation status changed: "pending" -> "complete" -==> Evaluation "fba4f04a" finished with status "complete" -``` -At this point, you should be able to visit any one of your client nodes at port -`9998` and see the web interface for fabio. The routing table will be empty -since we have not yet deployed anything that fabio can route to. -Accordingly, if you visit any of the client nodes at port `9999` at this -point, you will get a `404` HTTP response. That will change soon. - -### Step 3: Create a Job for Apache Web Server - -Create a job for Apache and name it `webserver.nomad` - -```hcl -job "webserver" { - datacenters = ["dc1"] - type = "service" - - group "webserver" { - count = 3 - restart { - attempts = 2 - interval = "30m" - delay = "15s" - mode = "fail" - } - ephemeral_disk { - size = 300 - } - - task "apache" { - driver = "docker" - config { - image = "httpd:latest" - port_map { - http = 80 - } - } - - resources { - network { - mbits = 10 - port "http" {} - } - } - - service { - name = "apache-webserver" - tags = ["urlprefix-/"] - port = "http" - check { - name = "alive" - type = "http" - path = "/" - interval = "10s" - timeout = "2s" - } - } - } - } -} -``` - -Notice the tag in the service stanza begins with `urlprefix-`. This is how a -path is registered with fabio. In this case, we are registering the path '/' -with fabio (which will route us to the default page for Apache web server). - -### Step 4: Run the Job for Apache Web Server - -We can now register our job for Apache: - -```shell -$ nomad job run webserver.nomad -==> Monitoring evaluation "c7bcaf40" - Evaluation triggered by job "webserver" - Evaluation within deployment: "e3603b50" - Allocation "20951ad4" created: node "510898b6", group "webserver" - Allocation "43807686" created: node "28d7f859", group "webserver" - Allocation "7b60eb24" created: node "f3739267", group "webserver" - Evaluation status changed: "pending" -> "complete" -==> Evaluation "c7bcaf40" finished with status "complete" -``` -You have now deployed and registered your web servers with fabio! At this point, -you should be able to visit any of the Nomad clients at port `9999` and -see the default web page for Apache web server. If you visit fabio's web -interface by going to any of the client nodes at port `9998`, you will see that -the routing table has been populated as shown below (**Note:** your destination IP -addresses will be different). - -[![Routing Table][routing-table]][routing-table] - -Feel free to reduce the `count` in `webserver.nomad` for testing purposes. You -will see that you still get routed to the Apache home page by accessing -any client node on port `9999`. Accordingly, the routing table -in the web interface on port `9999` will reflect the changes. - -### Step 5: Place Nomad Client Nodes Behind AWS Load Balancer - -At this point, you are ready to place your Nomad client nodes behind an AWS load -balancer. Your Nomad client nodes may change over time, and it is important -to provide your end users with a single endpoint to access your services. This guide will use the [Classic Load Balancer][classic-lb]. - -The AWS [documentation][classic-lb-doc] provides instruction on how to create a -load balancer. The basic steps involve creating a load balancer, registering -instances behind the load balancer (in our case these will be the Nomad client -nodes), creating listeners, and configuring health checks. - -Once you are done -with this, you should be able to hit the DNS name of your load balancer at port -80 (or whichever port you configured in your listener) and see the home page of -Apache web server. If you configured your listener to also forward traffic to -the web interface at port `9998`, you should be able to access that as well. - -[![Home Page][lb-homepage]][lb-homepage] - -[![Routing Table][lb-routing-table]][lb-routing-table] - -[apache]: https://httpd.apache.org/ -[classic-lb]: https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/introduction.html -[classic-lb-doc]: https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/elb-getting-started.html -[elb]: https://aws.amazon.com/elasticloadbalancing/ -[fabio]: https://fabiolb.net/ -[lb-homepage]: /assets/images/lb-homepage.png -[lb-routing-table]: /assets/images/lb-routing-table.png -[routing-table]: /assets/images/routing-table.png -[system]: /docs/schedulers.html#system diff --git a/website/source/guides/load-balancing/haproxy.html.md b/website/source/guides/load-balancing/haproxy.html.md deleted file mode 100644 index 2dd62409f..000000000 --- a/website/source/guides/load-balancing/haproxy.html.md +++ /dev/null @@ -1,279 +0,0 @@ ---- -layout: "guides" -page_title: "Load Balancing with HAProxy" -sidebar_current: "guides-load-balancing-haproxy" -description: |- - There are multiple approaches to load balancing within a Nomad cluster. - One approach involves using [HAProxy][haproxy] which natively integrates with - service discovery data from Consul. ---- - -# Load Balancing with HAProxy - -The main use case for HAProxy in this scenario is to distribute incoming HTTP(S) -and TCP requests from the internet to frontend services that can handle these -requests. This guide will show you one such example using a demo web -application. - -HAProxy version 1.8+ (LTS) includes the [server-template] directive, which lets -users specify placeholder backend servers to populate HAProxy’s load balancing -pools. Server-template can use Consul as one of these backend servers, -requesting SRV records from Consul DNS. - -## Reference Material - -- [HAProxy][haproxy] -- [Load Balancing Strategies for Consul][lb-strategies] - -## Estimated Time to Complete - -20 minutes - -## Prerequisites - -To perform the tasks described in this guide, you need to have a Nomad -environment with Consul installed. You can use this [repo][terraform-repo] to -easily provision a sandbox environment. This guide will assume a cluster with -one server node and three client nodes. - --> **Note:** This guide is for demo purposes and only assumes a single server -node. Please consult the [reference architecture][reference-arch] for production -configuration. - -## Steps - -### Step 1: Create a Job for Demo Web App - -Create a job for a demo web application and name the file `webapp.nomad`: - -```hcl -job "demo-webapp" { - datacenters = ["dc1"] - - group "demo" { - count = 3 - - task "server" { - env { - PORT = "${NOMAD_PORT_http}" - NODE_IP = "${NOMAD_IP_http}" - } - - driver = "docker" - - config { - image = "hashicorp/demo-webapp-lb-guide" - } - - resources { - network { - mbits = 10 - port "http" {} - } - } - - service { - name = "demo-webapp" - port = "http" - - check { - type = "http" - path = "/" - interval = "2s" - timeout = "2s" - } - } - } - } -} -``` - -Note that this job deploys 3 instances of our demo web application which we will -load balance with HAProxy in the next few steps. - -### Step 2: Deploy the Demo Web App - -We can now deploy our demo web application: - -```shell -$ nomad run webapp.nomad -==> Monitoring evaluation "8f3af425" - Evaluation triggered by job "demo-webapp" - Evaluation within deployment: "dc4c1925" - Allocation "bf9f850f" created: node "d16a11fb", group "demo" - Allocation "25e0496a" created: node "b78e27be", group "demo" - Allocation "a97e7d39" created: node "01d3eb32", group "demo" - Evaluation status changed: "pending" -> "complete" -==> Evaluation "8f3af425" finished with status "complete" -``` - -### Step 3: Create a Job for HAProxy - -Create a job for HAProxy and name it `haproxy.nomad`. This will be our load -balancer that will balance requests to the deployed instances of our web -application. - -```hcl -job "haproxy" { - region = "global" - datacenters = ["dc1"] - type = "service" - - group "haproxy" { - count = 1 - - task "haproxy" { - driver = "docker" - - config { - image = "haproxy:2.0" - network_mode = "host" - - volumes = [ - "local/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg", - ] - } - - template { - data = < Monitoring evaluation "937b1a2d" - Evaluation triggered by job "haproxy" - Evaluation within deployment: "e8214434" - Allocation "53145b8b" created: node "d16a11fb", group "haproxy" - Evaluation status changed: "pending" -> "complete" -==> Evaluation "937b1a2d" finished with status "complete" -``` - -### Step 5: Check the HAProxy Statistics Page - -You can visit the statistics and monitoring page for HAProxy at -`http://:1936`. You can use this page to verify your -settings and for basic monitoring. - -[![Home Page][haproxy_ui]][haproxy_ui] - -Notice there are 10 pre-provisioned load balancer backend slots for your service -but that only three of them are being used, corresponding to the three allocations in the current job. - -### Step 6: Make a Request to the Load Balancer - -If you query the HAProxy load balancer, you should be able to see a response -similar to the one shown below (this command should be run from a -node inside your cluster): - -```shell -$ curl haproxy.service.consul:8080 -Welcome! You are on node 172.31.54.242:20124 -``` - -Note that your request has been forwarded to one of the several deployed -instances of the demo web application (which is spread across 3 Nomad clients). -The output shows the IP address of the host it is deployed on. If you repeat -your requests, you will see that the IP address changes. - -* Note: if you would like to access HAProxy from outside your cluster, you - can set up a load balancer in your environment that maps to an active port - `8080` on your clients (or whichever port you have configured for HAProxy to - listen on). You can then send your requests directly to your external load - balancer. - -[consul-template]: https://github.com/hashicorp/consul-template#consul-template -[consul-temp-syntax]: https://github.com/hashicorp/consul-template#service -[haproxy]: http://www.haproxy.org/ -[haproxy_ui]: /assets/images/haproxy_ui.png -[inline]: /docs/job-specification/template.html#inline-template -[lb-strategies]: https://www.hashicorp.com/blog/configuring-third-party-loadbalancers-with-consul-nginx-haproxy-f5/ -[reference-arch]: /guides/install/production/reference-architecture.html#high-availability -[remote-template]: /docs/job-specification/template.html#remote-template -[server-template]: https://www.haproxy.com/blog/whats-new-haproxy-1-8/#server-template-configuration-directive -[template-stanza]: /docs/job-specification/template.html -[terraform-repo]: https://github.com/hashicorp/nomad/tree/master/terraform#provision-a-nomad-cluster-in-the-cloud - diff --git a/website/source/guides/load-balancing/load-balancing.html.md b/website/source/guides/load-balancing/load-balancing.html.md index f09334a90..025865aeb 100644 --- a/website/source/guides/load-balancing/load-balancing.html.md +++ b/website/source/guides/load-balancing/load-balancing.html.md @@ -9,16 +9,13 @@ description: |- # Load Balancing -There are multiple approaches to set up load balancing across a Nomad cluster. +These guides have been migrated to [HashiCorp's Learn website]. -Most of these methods assume Consul is installed alongside Nomad (see [Load -Balancing Strategies for -Consul](https://www.hashicorp.com/blog/load-balancing-strategies-for-consul)). +You can follow these links to specific guides at Learn: -- [Fabio](/guides/load-balancing/fabio.html) -- [NGINX](/guides/load-balancing/nginx.html) -- [HAProxy](/guides/load-balancing/haproxy.html) -- [Traefik](/guides/load-balancing/traefik.html) +- [Fabio](https://learn.hashicorp.com/nomad/load-balancing/fabio) +- [NGINX](https://learn.hashicorp.com/nomad/load-balancing/nginx) +- [HAProxy](https://learn.hashicorp.com/nomad/load-balancing/haproxy) +- [Traefik](https://learn.hashicorp.com/nomad/load-balancing/traefik) -Please refer to the specific documentation above or in the sidebar for more -detailed information about each strategy. +[HashiCorp's Learn website]: https://learn.hashicorp.com/nomad?track=load-balancing#load-balancing \ No newline at end of file diff --git a/website/source/guides/load-balancing/nginx.html.md b/website/source/guides/load-balancing/nginx.html.md deleted file mode 100644 index c4b63f128..000000000 --- a/website/source/guides/load-balancing/nginx.html.md +++ /dev/null @@ -1,293 +0,0 @@ ---- -layout: "guides" -page_title: "Load Balancing with NGINX" -sidebar_current: "guides-load-balancing-nginx" -description: |- - There are multiple approaches to load balancing within a Nomad cluster. - One approach involves using [NGINX][nginx]. NGINX works well with Nomad's - template stanza to allow for dynamic updates to its load balancing - configuration. ---- - -# Load Balancing with NGINX - -You can use Nomad's [template stanza][template-stanza] to configure -[NGINX][nginx] so that it can dynamically update its load balancer configuration -to scale along with your services. - -The main use case for NGINX in this scenario is to distribute incoming HTTP(S) -and TCP requests from the internet to frontend services that can handle these -requests. This guide will show you one such example using a demo web -application. - -## Reference Material - -- [NGINX][nginx] -- [Load Balancing Strategies for Consul][lb-strategies] - -## Estimated Time to Complete - -20 minutes - -## Prerequisites - -To perform the tasks described in this guide, you need to have a Nomad -environment with Consul installed. You can use this [repo][terraform-repo] to -easily provision a sandbox environment. This guide will assume a cluster with -one server node and three client nodes. - --> **Note:** This guide is for demo purposes and only assumes a single server -node. Please consult the [reference architecture][reference-arch] for production -configuration. - -## Steps - -### Step 1: Create a Job for Demo Web App - -Create a job for a demo web application and name the file `webapp.nomad`: - -```hcl -job "demo-webapp" { - datacenters = ["dc1"] - - group "demo" { - count = 3 - - task "server" { - env { - PORT = "${NOMAD_PORT_http}" - NODE_IP = "${NOMAD_IP_http}" - } - - driver = "docker" - - config { - image = "hashicorp/demo-webapp-lb-guide" - } - - resources { - network { - mbits = 10 - port "http"{} - } - } - - service { - name = "demo-webapp" - port = "http" - - check { - type = "http" - path = "/" - interval = "2s" - timeout = "2s" - } - } - } - } -} -``` - -Note that this job deploys 3 instances of our demo web application which we will -load balance with NGINX in the next few steps. - -### Step 2: Deploy the Demo Web App - -We can now deploy our demo web application: - -```shell -$ nomad run webapp.nomad -==> Monitoring evaluation "ea1e8528" - Evaluation triggered by job "demo-webapp" - Allocation "9b4bac9f" created: node "e4637e03", group "demo" - Allocation "c386de2d" created: node "983a64df", group "demo" - Allocation "082653f0" created: node "f5fdf017", group "demo" - Evaluation status changed: "pending" -> "complete" -==> Evaluation "ea1e8528" finished with status "complete" -``` - -### Step 3: Create a Job for NGINX - -Create a job for NGINX and name it `nginx.nomad`. This will be our load balancer -that will balance requests to the deployed instances of our web application. - -```hcl -job "nginx" { - datacenters = ["dc1"] - - group "nginx" { - count = 1 - - task "nginx" { - driver = "docker" - - config { - image = "nginx" - - port_map { - http = 80 - } - - volumes = [ - "local:/etc/nginx/conf.d", - ] - } - - template { - data = < Monitoring evaluation "45da5a89" - Evaluation triggered by job "nginx" - Allocation "c7f8af51" created: node "983a64df", group "nginx" - Evaluation status changed: "pending" -> "complete" -==> Evaluation "45da5a89" finished with status "complete" -``` - -### Step 5: Verify Load Balancer Configuration - -Consul Template supports [blocking queries][ct-blocking-queries]. This means -your NGINX deployment (which is using the [template][template-stanza] stanza) -will be notified immediately when a change in the health of one of the service -endpoints occurs and will re-render a new load balancer configuration file that -only includes healthy service instances. - -You can use the [alloc fs][alloc-fs] command on your NGINX allocation to read -the rendered load balancer configuration file. - -First, obtain the allocation ID of your NGINX deployment (output below is -abbreviated): - -```shell -$ nomad status nginx -ID = nginx -Name = nginx -... -Summary -Task Group Queued Starting Running Failed Complete Lost -nginx 0 0 1 0 0 0 - -Allocations -ID Node ID Task Group Version Desired Status Created Modified -76692834 f5fdf017 nginx 0 run running 17m40s ago 17m25s ago -``` - -* Keep in mind your allocation ID will be different. - -Next, use the `alloc fs` command to read the load balancer config: - -```shell -$ nomad alloc fs 766 nginx/local/load-balancer.conf -upstream backend { - - server 172.31.48.118:21354; - - server 172.31.52.52:25958; - - server 172.31.52.7:29728; - -} - -server { - listen 80; - - location / { - proxy_pass http://backend; - } -} -``` - -At this point, you can change the count of your `demo-webapp` job and repeat the -previous command to verify the load balancer config is dynamically changing. - -### Step 6: Make a Request to the Load Balancer - -If you query the NGINX load balancer, you should be able to see a response -similar to the one shown below (this command should be run from a node inside -your cluster): - -```shell -$ curl nginx.service.consul:8080 -Welcome! You are on node 172.31.48.118:21354 -``` - -Note that your request has been forwarded to one of the several deployed -instances of the demo web application (which is spread across 3 Nomad clients). -The output shows the IP address of the host it is deployed on. If you repeat -your requests, you will see that the IP address changes. - -* Note: if you would like to access NGINX from outside your cluster, you can set - up a load balancer in your environment that maps to an active port `8080` on - your clients (or whichever port you have configured for NGINX to listen on). - You can then send your requests directly to your external load balancer. - -[alloc-fs]: /docs/commands/alloc/fs.html -[consul-template]: https://github.com/hashicorp/consul-template#consul-template -[consul-temp-syntax]: https://github.com/hashicorp/consul-template#service -[ct-blocking-queries]: https://github.com/hashicorp/consul-template#key -[inline]: /docs/job-specification/template.html#inline-template -[lb-strategies]: https://www.hashicorp.com/blog/configuring-third-party-loadbalancers-with-consul-nginx-haproxy-f5/ -[nginx]: https://www.nginx.com/ -[reference-arch]: /guides/install/production/reference-architecture.html#high-availability -[remote-template]: /docs/job-specification/template.html#remote-template -[template-stanza]: /docs/job-specification/template.html -[terraform-repo]: https://github.com/hashicorp/nomad/tree/master/terraform#provision-a-nomad-cluster-in-the-cloud - diff --git a/website/source/guides/load-balancing/traefik.html.md b/website/source/guides/load-balancing/traefik.html.md deleted file mode 100644 index b541a8541..000000000 --- a/website/source/guides/load-balancing/traefik.html.md +++ /dev/null @@ -1,265 +0,0 @@ ---- -layout: "guides" -page_title: "Load Balancing with Traefik" -sidebar_current: "guides-load-balancing-traefik" -description: |- - There are multiple approaches to load balancing within a Nomad cluster. - One approach involves using [Traefik][traefik] which natively integrates - with service discovery data from Consul. ---- - -# Load Balancing with Traefik - -The main use case for Traefik in this scenario is to distribute incoming HTTP(S) -and TCP requests from the internet to frontend services that can handle these -requests. This guide will show you one such example using a demo web -application. - -Traefik can natively integrate with Consul using the [Consul Catalog -Provider][traefik-consul-provider] and can use [tags][traefik-tags] to route -traffic. - -## Reference Material - -- [Traefik][traefik] -- [Traefik Consul Catalog Provider Documentation][traefik-consul-provider] - -## Estimated Time to Complete - -20 minutes - -## Prerequisites - -To perform the tasks described in this guide, you need to have a Nomad -environment with Consul installed. You can use this [repo][terraform-repo] to -easily provision a sandbox environment. This guide will assume a cluster with -one server node and three client nodes. - --> **Note:** This guide is for demo purposes and only assumes a single server -node. Please consult the [reference architecture][reference-arch] for production -configuration. - -## Steps - -### Step 1: Create a Job for Demo Web App - -Create a job for a demo web application and name the file `webapp.nomad`: - -```hcl -job "demo-webapp" { - datacenters = ["dc1"] - - group "demo" { - count = 3 - - task "server" { - env { - PORT = "${NOMAD_PORT_http}" - NODE_IP = "${NOMAD_IP_http}" - } - - driver = "docker" - - config { - image = "hashicorp/demo-webapp-lb-guide" - } - - resources { - network { - mbits = 10 - port "http" {} - } - } - - service { - name = "demo-webapp" - port = "http" - tags = [ - "traefik.tags=service", - "traefik.frontend.rule=PathPrefixStrip:/myapp", - ] - - check { - type = "http" - path = "/" - interval = "2s" - timeout = "2s" - } - } - } - } -} -``` - -- Note that this job deploys 3 instances of our demo web application which we - will load balance with Traefik in the next few steps. -- We are using tags to configure routing to our web app. Even though our - application listens on `/`, it is possible to define `/myapp` as the route - because of the [`PathPrefixStrip`][matchers] option. - -### Step 2: Deploy the Demo Web App - -We can now deploy our demo web application: - -```shell -$ nomad run webapp.nomad -==> Monitoring evaluation "a2061ab7" - Evaluation triggered by job "demo-webapp" - Evaluation within deployment: "8ca6d358" - Allocation "1d14babe" created: node "2d6eea6e", group "demo" - Allocation "3abb950d" created: node "a62fa99d", group "demo" - Allocation "c65e14bf" created: node "a209a662", group "demo" - Evaluation status changed: "pending" -> "complete" -==> Evaluation "a2061ab7" finished with status "complete" -``` - -### Step 3: Create a Job for Traefik - -Create a job for Traefik and name it `traefik.nomad`. This will be our load -balancer that will balance requests to the deployed instances of our web -application. - -```hcl -job "traefik" { - region = "global" - datacenters = ["dc1"] - type = "service" - - group "traefik" { - count = 1 - - task "traefik" { - driver = "docker" - - config { - image = "traefik:1.7" - network_mode = "host" - - volumes = [ - "local/traefik.toml:/etc/traefik/traefik.toml", - ] - } - - template { - data = < Monitoring evaluation "e22ce276" - Evaluation triggered by job "traefik" - Evaluation within deployment: "c6466497" - Allocation "695c5632" created: node "a62fa99d", group "traefik" - Evaluation status changed: "pending" -> "complete" -==> Evaluation "e22ce276" finished with status "complete" -``` - -### Step 5: Check the Traefik Dashboard - -You can visit the dashboard for Traefik at -`http://:8081`. You can use this page to verify your -settings and for basic monitoring. - -[![Home Page][traefik_ui]][traefik_ui] - -### Step 6: Make a Request to the Load Balancer - -If you query the Traefik load balancer, you should be able to see a response -similar to the one shown below (this command should be run from a -node inside your cluster): - -```shell -$ curl http://traefik.service.consul:8080/myapp -Welcome! You are on node 172.31.28.103:28893 -``` - -Note that your request has been forwarded to one of the several deployed -instances of the demo web application (which is spread across 3 Nomad clients). -The output shows the IP address of the host it is deployed on. If you repeat -your requests, you will see that the IP address changes. - -* Note: if you would like to access Traefik from outside your cluster, you - can set up a load balancer in your environment that maps to an active port - `8080` on your clients (or whichever port you have configured for Traefik to - listen on). You can then send your requests directly to your external load - balancer. - -[inline]: /docs/job-specification/template.html#inline-template -[matchers]: https://docs.traefik.io/v1.4/basics/#matchers -[reference-arch]: /guides/install/production/reference-architecture.html#high-availability -[remote-template]: /docs/job-specification/template.html#remote-template -[template-stanza]: /docs/job-specification/template.html -[terraform-repo]: https://github.com/hashicorp/nomad/tree/master/terraform#provision-a-nomad-cluster-in-the-cloud -[traefik]: https://traefik.io/ -[traefik_ui]: /assets/images/traefik_ui.png -[traefik-consul-provider]: https://docs.traefik.io/v1.7/configuration/backends/consulcatalog/ -[traefik-tags]: https://docs.traefik.io/v1.5/configuration/backends/consulcatalog/#tags diff --git a/website/source/redirects.txt b/website/source/redirects.txt index de6c00e91..29239653c 100644 --- a/website/source/redirects.txt +++ b/website/source/redirects.txt @@ -44,6 +44,11 @@ /intro/getting-started/ui.html https://learn.hashicorp.com/nomad/getting-started/ui /intro/getting-started/next-steps.html https://learn.hashicorp.com/nomad/getting-started/next-steps +/guides/load-balancing/fabio.html https://learn.hashicorp.com/nomad/load-balancing/fabio +/guides/load-balancing/nginx.html https://learn.hashicorp.com/nomad/load-balancing/nginx +/guides/load-balancing/haproxy.html https://learn.hashicorp.com/nomad/load-balancing/haproxy +/guides/load-balancing/traefik.html https://learn.hashicorp.com/nomad/load-balancing/traefik + # Website /community.html /resources.html From 5b44f334c15d41e203cc9efa265a6aaa7e3905c0 Mon Sep 17 00:00:00 2001 From: Charlie Voiselle <464492+angrycub@users.noreply.github.com> Date: Fri, 22 Nov 2019 14:46:45 -0500 Subject: [PATCH 35/67] Removed sidenav links for LB guides --- website/source/layouts/guides.erb | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/website/source/layouts/guides.erb b/website/source/layouts/guides.erb index 70ed6a336..bc7966e64 100644 --- a/website/source/layouts/guides.erb +++ b/website/source/layouts/guides.erb @@ -258,26 +258,6 @@ > Load Balancing - - - - > From 909519c4ca2ed21cec97a4d1859649a5f7f1ebf2 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 14:50:49 -0500 Subject: [PATCH 36/67] ci: verify .circleci/config.yml is up to date --- .circleci/config.yml | 6 ++++ .../commands/install-circleci-local-cli.yml | 31 +++++++++++++++++++ .circleci/config/jobs/lint-go.yml | 4 +++ 3 files changed, 41 insertions(+) create mode 100644 .circleci/config/commands/install-circleci-local-cli.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 02c78ce4e..5e28eec14 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -424,12 +424,18 @@ jobs: sudo rm -rf /usr/bin/protoc sudo ./scripts/vagrant-linux-priv-protoc.sh name: install protoc + - run: + command: "CCI_VERSION=\"0.1.5879\"\nCCI_SHA256=\"f178ea62c781aec06267017404f87983c87f171fd0e66ef3737916246ae66dd6\"\n\nURL=\"https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CCI_VERSION}/circleci-cli_${CCI_VERSION}_linux_amd64.tar.gz\"\n\nmkdir -p /tmp/circleci-cli/\ncurl --fail --show-error --location \\\n -o /tmp/circleci-cli/cli.tar.gz \"${URL}\"\n\necho \"$CCI_SHA256 /tmp/circleci-cli/cli.tar.gz\" | sha256sum -c\n\ntar -xz --strip-components=1 \\\n -C /tmp/circleci-cli \\\n -f /tmp/circleci-cli/cli.tar.gz \\\n \"circleci-cli_${CCI_VERSION}_linux_amd64/circleci\" \n\nsudo cp /tmp/circleci-cli/circleci /usr/bin/circleci-local-cli\n\ncircleci-local-cli version\n" + name: Install CircleCI CLI 0.1.5879 - run: command: make deps lint-deps - run: command: make check - run: command: make checkscripts + - run: + command: make -C .circleci CIRCLECI="circleci-local-cli --skip-update-check" ci-verify + name: check .circleci/config.yml is up-to-date environment: - GIT_PAGER: cat - GOMAXPROCS: 1 diff --git a/.circleci/config/commands/install-circleci-local-cli.yml b/.circleci/config/commands/install-circleci-local-cli.yml new file mode 100644 index 000000000..5b89a3f69 --- /dev/null +++ b/.circleci/config/commands/install-circleci-local-cli.yml @@ -0,0 +1,31 @@ +parameters: + version: + type: string + default: 0.1.5879 + + sha256: + type: string + default: f178ea62c781aec06267017404f87983c87f171fd0e66ef3737916246ae66dd6 +steps: + - run: + name: Install CircleCI CLI << parameters.version >> + command: | + CCI_VERSION="<< parameters.version >>" + CCI_SHA256="<< parameters.sha256 >>" + + URL="https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CCI_VERSION}/circleci-cli_${CCI_VERSION}_linux_amd64.tar.gz" + + mkdir -p /tmp/circleci-cli/ + curl --fail --show-error --location \ + -o /tmp/circleci-cli/cli.tar.gz "${URL}" + + echo "$CCI_SHA256 /tmp/circleci-cli/cli.tar.gz" | sha256sum -c + + tar -xz --strip-components=1 \ + -C /tmp/circleci-cli \ + -f /tmp/circleci-cli/cli.tar.gz \ + "circleci-cli_${CCI_VERSION}_linux_amd64/circleci" + + sudo cp /tmp/circleci-cli/circleci /usr/bin/circleci-local-cli + + circleci-local-cli version diff --git a/.circleci/config/jobs/lint-go.yml b/.circleci/config/jobs/lint-go.yml index 90a9abcab..72e955010 100644 --- a/.circleci/config/jobs/lint-go.yml +++ b/.circleci/config/jobs/lint-go.yml @@ -3,6 +3,10 @@ steps: - checkout - run: apt-get update; apt-get install -y shellcheck sudo unzip - install-protoc + - install-circleci-local-cli - run: make deps lint-deps - run: make check - run: make checkscripts + - run: + name: check .circleci/config.yml is up-to-date + command: make -C .circleci CIRCLECI="circleci-local-cli --skip-update-check" ci-verify From 1730e1f53aec3ab22f5d83324b998d73381e86c1 Mon Sep 17 00:00:00 2001 From: Charlie Voiselle <464492+angrycub@users.noreply.github.com> Date: Fri, 22 Nov 2019 15:30:32 -0500 Subject: [PATCH 37/67] Removed 404 links; pointed to learn.hashicorp.com --- README.md | 2 +- .../source/guides/integrations/vault-integration/index.html.md | 3 +-- .../monitoring-and-alerting/prometheus-metrics.html.md | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 59f3dc4d5..9470f4198 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Documentation & Guides * [Installing Nomad for Production](https://www.nomadproject.io/guides/operations/deployment-guide.html) * [Advanced Job Scheduling on Nomad with Affinities](https://www.nomadproject.io/guides/operating-a-job/advanced-scheduling/affinity.html) * [Increasing Nomad Fault Tolerance with Spread](https://www.nomadproject.io/guides/operating-a-job/advanced-scheduling/spread.html) -* [Load Balancing on Nomad with Fabio & Consul](https://www.nomadproject.io/guides/load-balancing/fabio.html) +* [Load Balancing on Nomad with Fabio & Consul](https://learn.hashicorp.com/guides/load-balancing/fabio) * [Deploying Stateful Workloads via Portworx](https://www.nomadproject.io/guides/stateful-workloads/portworx.html) * [Running Apache Spark on Nomad](https://www.nomadproject.io/guides/spark/spark.html) * [Integrating Vault with Nomad for Secrets Management](https://www.nomadproject.io/guides/operations/vault-integration/index.html) diff --git a/website/source/guides/integrations/vault-integration/index.html.md b/website/source/guides/integrations/vault-integration/index.html.md index 07b5c1c8f..a15445cc5 100644 --- a/website/source/guides/integrations/vault-integration/index.html.md +++ b/website/source/guides/integrations/vault-integration/index.html.md @@ -665,8 +665,7 @@ below [creation-statements]: https://www.vaultproject.io/api/secret/databases/index.html#creation_statements [destination]: /docs/job-specification/template.html#destination [fabio]: https://github.com/fabiolb/fabio -[fabio-job]: /guides/load-balancing/fabio.html#step-1-create-a-job-for-fabio -[fabio-lb]: /guides/load-balancing/fabio.html +[fabio-lb]: https://learn.hashicorp.com/guides/load-balancing/fabio [inline]: /docs/job-specification/template.html#inline-template [login]: https://www.vaultproject.io/docs/commands/login.html [nomad-alloc-fs]: /docs/commands/alloc/fs.html diff --git a/website/source/guides/operations/monitoring-and-alerting/prometheus-metrics.html.md b/website/source/guides/operations/monitoring-and-alerting/prometheus-metrics.html.md index b49af5d28..4501602af 100644 --- a/website/source/guides/operations/monitoring-and-alerting/prometheus-metrics.html.md +++ b/website/source/guides/operations/monitoring-and-alerting/prometheus-metrics.html.md @@ -554,7 +554,7 @@ to send out notifications to a [receiver][receivers] of your choice. [consul_sd_config]: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#%3Cconsul_sd_config%3E [env]: /docs/runtime/environment.html [fabio]: https://fabiolb.net/ -[fabio-lb]: /guides/load-balancing/fabio.html +[fabio-lb]: https://learn.hashicorp.com/guides/load-balancing/fabio [new-targets]: /assets/images/new-targets.png [prometheus-targets]: /assets/images/prometheus-targets.png [running-jobs]: /assets/images/running-jobs.png From 0e157cf648a7e3dd86f962b355b12ac843281841 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 15:32:48 -0500 Subject: [PATCH 38/67] ci: avoid building binaries in stable-website This speeds up building and pushing releases --- .circleci/config.yml | 6 +++++- .circleci/config/workflows/build-test.yml | 8 +++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5e28eec14..73ae04eed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -831,7 +831,11 @@ jobs: workflows: build-test: jobs: - - build-binaries + - build-binaries: + filters: + branches: + ignore: + - stable-website - lint-go: filters: branches: diff --git a/.circleci/config/workflows/build-test.yml b/.circleci/config/workflows/build-test.yml index 3cbc270d7..f3d7f7b76 100644 --- a/.circleci/config/workflows/build-test.yml +++ b/.circleci/config/workflows/build-test.yml @@ -1,5 +1,11 @@ jobs: -- build-binaries +- build-binaries: + # almost always build binaries as they may be needed + # for e2e tests + filters: + branches: + ignore: + - stable-website - lint-go: filters: &backend_branches_filter branches: From 2afc9395884cfd746f0b1fe1cadadfb96d9b8c82 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 18:41:21 -0500 Subject: [PATCH 39/67] tests: deflake TestHTTP_FreshClientAllocMetrics The test asserts that alloc counts get reported accurately in metrics by inspecting the metrics endpoint directly. Sadly, the metrics as collected by `armon/go-metrics` seem to be stateful and may contain info from other tests. This means that the test can fail depending on the order of returned metrics. Inspecting the metrics output of one failing run, you can see the duplicate guage entries but for different node_ids: ``` { "Name": "service-name.default-0a3ba4b6-2109-485e-be74-6864228aed3d.client.allocations.terminal", "Value": 10, "Labels": { "datacenter": "dc1", "node_class": "none", "node_id": "67402bf4-00f3-bd8d-9fa8-f4d1924a892a" } }, { "Name": "service-name.default-0a3ba4b6-2109-485e-be74-6864228aed3d.client.allocations.terminal", "Value": 0, "Labels": { "datacenter": "dc1", "node_class": "none", "node_id": "a2945b48-7e66-68e2-c922-49b20dd4e20c" } }, ``` --- command/agent/metrics_endpoint_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/command/agent/metrics_endpoint_test.go b/command/agent/metrics_endpoint_test.go index d97fe2e69..b10e41708 100644 --- a/command/agent/metrics_endpoint_test.go +++ b/command/agent/metrics_endpoint_test.go @@ -91,6 +91,8 @@ func TestHTTP_FreshClientAllocMetrics(t *testing.T) { require.Fail("timed-out waiting for job to complete") }) + nodeID := s.client.NodeID() + // wait for metrics to converge var pending, running, terminal float32 = -1.0, -1.0, -1.0 testutil.WaitForResultRetries(100, func() (bool, error) { @@ -106,6 +108,13 @@ func TestHTTP_FreshClientAllocMetrics(t *testing.T) { metrics := obj.(metrics.MetricsSummary) for _, g := range metrics.Gauges { + + // ignore client metrics belonging to other test nodes + // from other tests that contaminate go-metrics reporting + if g.DisplayLabels["node_id"] != nodeID { + continue + } + if strings.HasSuffix(g.Name, "client.allocations.pending") { pending = g.Value } From d0788837260a70de469469c4c70888b3ba718cb8 Mon Sep 17 00:00:00 2001 From: Ruslan Stelmachenko Date: Sun, 24 Nov 2019 02:35:09 +0200 Subject: [PATCH 40/67] Fix demo vagrant provision hung when libssl asks for confirmation The provision shell script tries to install libssl1.1 package as dependency of ca-certificates package. The installing of libssl requires to restart some services, and it asks for confirmation of this. But there are no interactive session at this stage, so Vagrant provisioning just hungs. This commit add a `libraries/restart-without-asking boolean true` setting before installing libssl, so it doesn't ask confirmation anymore and the provisioning works again. --- demo/vagrant/Vagrantfile | 1 + 1 file changed, 1 insertion(+) diff --git a/demo/vagrant/Vagrantfile b/demo/vagrant/Vagrantfile index 17289a4eb..1dedf96d8 100644 --- a/demo/vagrant/Vagrantfile +++ b/demo/vagrant/Vagrantfile @@ -5,6 +5,7 @@ $script = <