From bbf1eaa833edc0ff2447702b8df4e096ccf183ab Mon Sep 17 00:00:00 2001 From: Kris Hicks Date: Tue, 8 Dec 2020 13:18:11 -0800 Subject: [PATCH] Remove multi-file CircleCI config (#9571) --- .circleci/.gitattributes | 1 - .circleci/.gitignore | 1 - .circleci/Makefile | 57 +- .circleci/README.md | 3 - .circleci/config.yml | 2011 +++++------------ .circleci/config/commands/install-buf.yml | 5 - .../commands/install-circleci-local-cli.yml | 31 - .circleci/config/commands/install-consul.yml | 11 - .circleci/config/commands/install-golang.yml | 33 - .circleci/config/commands/install-vault.yml | 27 - .circleci/config/commands/run-tests.yml | 24 - .circleci/config/config.yml | 70 - .circleci/config/jobs/algolia_index.yml | 15 - .circleci/config/jobs/build-binaries.yml | 17 - .../config/jobs/build-darwin-binaries.yml | 17 - .circleci/config/jobs/lint-go.yml | 13 - .circleci/config/jobs/test-container.yml | 27 - .circleci/config/jobs/test-e2e.yml | 18 - .circleci/config/jobs/test-machine.yml | 43 - .circleci/config/jobs/test-ui.yml | 31 - .circleci/config/jobs/test-windows.yml | 45 - .../config/jobs/website-docker-image.yml | 26 - .circleci/config/workflows/build-test.yml | 96 - .circleci/config/workflows/website.yml | 14 - 24 files changed, 557 insertions(+), 2079 deletions(-) delete mode 100644 .circleci/.gitattributes delete mode 100644 .circleci/.gitignore delete mode 100644 .circleci/README.md delete mode 100644 .circleci/config/commands/install-buf.yml delete mode 100644 .circleci/config/commands/install-circleci-local-cli.yml delete mode 100644 .circleci/config/commands/install-consul.yml delete mode 100644 .circleci/config/commands/install-golang.yml delete mode 100644 .circleci/config/commands/install-vault.yml delete mode 100644 .circleci/config/commands/run-tests.yml delete mode 100644 .circleci/config/config.yml delete mode 100644 .circleci/config/jobs/algolia_index.yml delete mode 100644 .circleci/config/jobs/build-binaries.yml delete mode 100644 .circleci/config/jobs/build-darwin-binaries.yml delete mode 100644 .circleci/config/jobs/lint-go.yml delete mode 100644 .circleci/config/jobs/test-container.yml delete mode 100644 .circleci/config/jobs/test-e2e.yml delete mode 100644 .circleci/config/jobs/test-machine.yml delete mode 100644 .circleci/config/jobs/test-ui.yml delete mode 100644 .circleci/config/jobs/test-windows.yml delete mode 100644 .circleci/config/jobs/website-docker-image.yml delete mode 100644 .circleci/config/workflows/build-test.yml delete mode 100644 .circleci/config/workflows/website.yml diff --git a/.circleci/.gitattributes b/.circleci/.gitattributes deleted file mode 100644 index f7c6b31eb..000000000 --- a/.circleci/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -/config.yml linguist-generated diff --git a/.circleci/.gitignore b/.circleci/.gitignore deleted file mode 100644 index 3018b3a68..000000000 --- a/.circleci/.gitignore +++ /dev/null @@ -1 +0,0 @@ -.tmp/ diff --git a/.circleci/Makefile b/.circleci/Makefile index 3852d19f7..c6b40b067 100644 --- a/.circleci/Makefile +++ b/.circleci/Makefile @@ -14,67 +14,16 @@ ifeq ($(CCI_VERSION),) CIRCLECI := echo '$(CCI_INSTALL_MSG)'; exit 1; \# endif -SOURCE_DIR := config -SOURCE_YML := $(shell [ ! -d $(SOURCE_DIR) ] || find $(SOURCE_DIR) -name '*.yml') -CONFIG_SOURCE := Makefile $(SOURCE_YML) | $(SOURCE_DIR) -OUT := config.yml -TMP := .tmp/config-processed -CONFIG_PACKED := .tmp/config-packed - default: help help: @echo "Usage:" - @echo " make ci-config: recompile config.yml from $(SOURCE_DIR)/" - @echo " make ci-verify: verify that config.yml is a true mapping from $(SOURCE_DIR)/" + @echo " make ci-verify: validate that config.yml for correctness" @echo @echo "Diagnostics:" @[ -z "$(CCI_VERSION)" ] || echo " circleci-cli version $(CCI_VERSION)" @[ -n "$(CCI_VERSION)" ] || echo " $(CCI_INSTALL_MSG)" -$(SOURCE_DIR): - @echo No source directory $(SOURCE_DIR) found.; exit 1 - -# Make sure our .tmp dir exists. -$(shell [ -d .tmp ] || mkdir .tmp) - -.PHONY: ci-config -ci-config: $(OUT) - .PHONY: ci-verify -ci-verify: config-up-to-date - @$(CIRCLECI) config validate $(OUT) - -define GENERATED_FILE_HEADER -### *** -### WARNING: DO NOT manually EDIT or MERGE this file, it is generated by 'make ci-config'. -### INSTEAD: Edit or merge the source in $(SOURCE_DIR)/ then run 'make ci-config'. -### *** -endef -export GENERATED_FILE_HEADER - -# GEN_CONFIG writes the config to a temporary file. If the whole process succeeds, -# it then moves that file to $@. This makes is an atomic operation, so if it fails -# make doesn't consider a half-baked file up to date. -define GEN_CONFIG - @$(CIRCLECI) config pack $(SOURCE_DIR) > $(CONFIG_PACKED) - @echo "$$GENERATED_FILE_HEADER" > $@.tmp || { rm -f $@; exit 1; } - @$(CIRCLECI) config process $(CONFIG_PACKED) >> $@.tmp || { rm -f $@.tmp; exit 1; } - @mv -f $@.tmp $@ -endef - -$(OUT): $(CONFIG_SOURCE) - $(GEN_CONFIG) - @echo "$@ updated" - -$(TMP): $(CONFIG_SOURCE) - $(GEN_CONFIG) - -.PHONY: config-up-to-date -config-up-to-date: $(TMP) # Note this must not depend on $(OUT)! - @if diff config.yml $<; then \ - echo "Generated $(OUT) is up to date!"; \ - else \ - echo "Generated $(OUT) is out of date, run make $(CONFIG) to update."; \ - exit 1; \ - fi +ci-verify: + @$(CIRCLECI) config validate config.yml diff --git a/.circleci/README.md b/.circleci/README.md deleted file mode 100644 index c21070097..000000000 --- a/.circleci/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# How to use CircleCI multi-file config - -Refer to https://github.com/hashicorp/vault/blob/master/.circleci/README.md . diff --git a/.circleci/config.yml b/.circleci/config.yml index d77ae4f2c..7554724a9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,1487 +1,584 @@ -### *** -### WARNING: DO NOT manually EDIT or MERGE this file, it is generated by 'make ci-config'. -### INSTEAD: Edit or merge the source in config/ then run 'make ci-config'. -### *** -version: 2 +version: 2.1 + +references: + # environment specific references - aim to avoid conflicts + go-machine-image: &go_machine_image + circleci/classic:201808-01 + go-machine-recent-image: &go_machine_recent_image + ubuntu-1604:201903-01 + go-windows-image: &go_windows_image + windows-server-2019-vs2019:stable + + # common references + common_envs: &common_envs + GOMAXPROCS: 1 + NOMAD_SLOW_TEST: 1 + GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml + GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json + # disable implicit git paging. CircleCI runs commands with in a tty + # making git assume it's an interactive session. + PAGER: cat + GIT_PAGER: cat + +commands: + install-buf: + steps: + - run: + name: install buf + command: | + sudo ./scripts/vagrant-linux-priv-buf.sh + install-vault: + parameters: + version: + type: string + default: 1.2.4 + steps: + - run: + name: Install Vault + command: | + set -x + + case "${OSTYPE}" in + linux*) os="linux" ;; + darwin*) os="darwin" ;; + msys*) os="windows" ;; + *) echo "unknown os: ${OSTYPE}"; exit 1 ;; + esac + + curl -SL --fail -o /tmp/vault.zip https://releases.hashicorp.com/vault/"<< parameters.version >>"/vault_"<< parameters.version >>"_"${os}"_amd64.zip + + if [[ "${os}" != "windows" ]]; then + sudo unzip -d /usr/local/bin /tmp/vault.zip + rm -rf /tmp/vault* + else + rm -rf c:\Windows\vault.exe + unzip /tmp/vault.zip -d "/c/Windows" + rm -rf /tmp/vault* + fi + install-consul: + parameters: + version: + type: string + default: 1.8.3 + steps: + - run: + name: Install Consul << parameters.version >> + command: | + curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/<< parameters.version >>/consul_<< parameters.version >>_linux_amd64.zip + sudo unzip -d /usr/local/bin /tmp/consul.zip + rm -rf /tmp/consul* + install-golang: + parameters: + target_directory: + type: string + default: /usr/local + steps: + - run: + name: Install golang + command: | + set -x + + echo installing golang ${GOLANG_VERSION} + + case "${OSTYPE}" in + linux*) os="linux" ;; + darwin*) os="darwin" ;; + msys*) os="windows" ;; + *) echo "unknown os: ${OSTYPE}"; exit 1 ;; + esac + + if [[ "${os}" != "windows" ]] + then + sudo rm -rf << parameters.target_directory >>/go + sudo mkdir -p << parameters.target_directory >> + curl -SL --fail -o /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.${os}-amd64.tar.gz + sudo tar -C << parameters.target_directory >> -xzf /tmp/golang.tar.gz + rm -rf /tmp/golang.tar.gz + else + rm -rf << parameters.target_directory >>/go + mkdir -p <> + curl -SL --fail -o /tmp/go.zip https://dl.google.com/go/go${GOLANG_VERSION}.windows-amd64.zip + unzip -q -o /tmp/go.zip -d << parameters.target_directory >> + rm -rf /tmp/go.zip + fi + + install-circleci-local-cli: + parameters: + version: + type: string + default: 0.1.5879 + + sha256: + type: string + default: f178ea62c781aec06267017404f87983c87f171fd0e66ef3737916246ae66dd6 + steps: + - run: + name: Install CircleCI CLI << parameters.version >> + command: | + CCI_VERSION="<< parameters.version >>" + CCI_SHA256="<< parameters.sha256 >>" + + URL="https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CCI_VERSION}/circleci-cli_${CCI_VERSION}_linux_amd64.tar.gz" + + mkdir -p /tmp/circleci-cli/ + curl --fail --show-error --location \ + -o /tmp/circleci-cli/cli.tar.gz "${URL}" + + echo "$CCI_SHA256 /tmp/circleci-cli/cli.tar.gz" | sha256sum -c + + tar -xz --strip-components=1 \ + -C /tmp/circleci-cli \ + -f /tmp/circleci-cli/cli.tar.gz \ + "circleci-cli_${CCI_VERSION}_linux_amd64/circleci" + + sudo cp /tmp/circleci-cli/circleci /usr/bin/circleci-local-cli + + circleci-local-cli version + run-tests: + steps: + - run: + name: Running Nomad Tests + no_output_timeout: 20m + command: | + if [ -z $GOTEST_PKGS_EXCLUDE ]; + then + unset GOTEST_PKGS_EXCLUDE + else + unset GOTEST_PKGS + fi + + if [ ! -z $GOTESTARCH ]; then + export GOARCH="$GOTESTARCH"; + fi + + mkdir -p /tmp/test-reports + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs + + if [ ! -z $GOTEST_MOD ]; then + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad-module + else + sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad + fi + jobs: - test-nomad: - machine: - image: ubuntu-1604:201903-01 - working_directory: ~/go/src/github.com/hashicorp/nomad - environment: - - GIT_PAGER: cat - - GOLANG_VERSION: 1.15.5 - - GOMAXPROCS: 1 - - GOPATH: /home/circleci/go - - GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json - - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - - NOMAD_SLOW_TEST: 1 - - PAGER: cat - - GOTEST_MOD: '' - - GOTEST_PKGS: ./nomad/... - - GOTEST_PKGS_EXCLUDE: '' - - GOTESTARCH: amd64 - steps: - - checkout - - run: - command: | - set -x - - echo installing golang ${GOLANG_VERSION} - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - if [[ "${os}" != "windows" ]] - then - sudo rm -rf /usr/local/go - sudo mkdir -p /usr/local - curl -SL --fail -o /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.${os}-amd64.tar.gz - sudo tar -C /usr/local -xzf /tmp/golang.tar.gz - rm -rf /tmp/golang.tar.gz - else - rm -rf /usr/local/go - mkdir -p /usr/local - curl -SL --fail -o /tmp/go.zip https://dl.google.com/go/go${GOLANG_VERSION}.windows-amd64.zip - unzip -q -o /tmp/go.zip -d /usr/local - rm -rf /tmp/go.zip - fi - name: Install golang - - run: - command: | - sudo ./scripts/vagrant-linux-priv-buf.sh - name: install buf - - run: - command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip - sudo unzip -d /usr/local/bin /tmp/consul.zip - rm -rf /tmp/consul* - name: Install Consul 1.8.3 - - run: - command: | - set -x - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - curl -SL --fail -o /tmp/vault.zip https://releases.hashicorp.com/vault/"1.2.4"/vault_"1.2.4"_"${os}"_amd64.zip - - if [[ "${os}" != "windows" ]]; then - sudo unzip -d /usr/local/bin /tmp/vault.zip - rm -rf /tmp/vault* - else - rm -rf c:\Windows\vault.exe - unzip /tmp/vault.zip -d "/c/Windows" - rm -rf /tmp/vault* - fi - name: Install Vault - - run: - command: | - if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then - sudo apt-get update - sudo apt-get install -y gcc-multilib - else - echo "Skipping 32bit lib installation while building for not 386" - fi - name: Install 32bit gcc libs - - run: - command: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap - - run: - command: | - if [ -z $GOTEST_PKGS_EXCLUDE ]; - then - unset GOTEST_PKGS_EXCLUDE - else - unset GOTEST_PKGS - fi - - if [ ! -z $GOTESTARCH ]; then - export GOARCH="$GOTESTARCH"; - fi - - mkdir -p /tmp/test-reports - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs - - if [ ! -z $GOTEST_MOD ]; then - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad-module - else - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad - fi - name: Running Nomad Tests - no_output_timeout: 20m - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-reports - test-api: - machine: - image: ubuntu-1604:201903-01 - working_directory: ~/go/src/github.com/hashicorp/nomad - environment: - - GIT_PAGER: cat - - GOLANG_VERSION: 1.15.5 - - GOMAXPROCS: 1 - - GOPATH: /home/circleci/go - - GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json - - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - - NOMAD_SLOW_TEST: 1 - - PAGER: cat - - GOTEST_MOD: api - - GOTEST_PKGS: '' - - GOTEST_PKGS_EXCLUDE: '' - - GOTESTARCH: amd64 - steps: - - checkout - - run: - command: | - set -x - - echo installing golang ${GOLANG_VERSION} - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - if [[ "${os}" != "windows" ]] - then - sudo rm -rf /usr/local/go - sudo mkdir -p /usr/local - curl -SL --fail -o /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.${os}-amd64.tar.gz - sudo tar -C /usr/local -xzf /tmp/golang.tar.gz - rm -rf /tmp/golang.tar.gz - else - rm -rf /usr/local/go - mkdir -p /usr/local - curl -SL --fail -o /tmp/go.zip https://dl.google.com/go/go${GOLANG_VERSION}.windows-amd64.zip - unzip -q -o /tmp/go.zip -d /usr/local - rm -rf /tmp/go.zip - fi - name: Install golang - - run: - command: | - sudo ./scripts/vagrant-linux-priv-buf.sh - name: install buf - - run: - command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip - sudo unzip -d /usr/local/bin /tmp/consul.zip - rm -rf /tmp/consul* - name: Install Consul 1.8.3 - - run: - command: | - set -x - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - curl -SL --fail -o /tmp/vault.zip https://releases.hashicorp.com/vault/"1.2.4"/vault_"1.2.4"_"${os}"_amd64.zip - - if [[ "${os}" != "windows" ]]; then - sudo unzip -d /usr/local/bin /tmp/vault.zip - rm -rf /tmp/vault* - else - rm -rf c:\Windows\vault.exe - unzip /tmp/vault.zip -d "/c/Windows" - rm -rf /tmp/vault* - fi - name: Install Vault - - run: - command: | - if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then - sudo apt-get update - sudo apt-get install -y gcc-multilib - else - echo "Skipping 32bit lib installation while building for not 386" - fi - name: Install 32bit gcc libs - - run: - command: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap - - run: - command: | - if [ -z $GOTEST_PKGS_EXCLUDE ]; - then - unset GOTEST_PKGS_EXCLUDE - else - unset GOTEST_PKGS - fi - - if [ ! -z $GOTESTARCH ]; then - export GOARCH="$GOTESTARCH"; - fi - - mkdir -p /tmp/test-reports - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs - - if [ ! -z $GOTEST_MOD ]; then - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad-module - else - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad - fi - name: Running Nomad Tests - no_output_timeout: 20m - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-reports - test-exec: - machine: - image: ubuntu-1604:201903-01 - working_directory: ~/go/src/github.com/hashicorp/nomad - environment: - - GIT_PAGER: cat - - GOLANG_VERSION: 1.15.5 - - GOMAXPROCS: 1 - - GOPATH: /home/circleci/go - - GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json - - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - - NOMAD_SLOW_TEST: 1 - - PAGER: cat - - GOTEST_MOD: '' - - GOTEST_PKGS: ./drivers/exec - - GOTEST_PKGS_EXCLUDE: '' - - GOTESTARCH: amd64 - steps: - - checkout - - run: - command: | - set -x - - echo installing golang ${GOLANG_VERSION} - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - if [[ "${os}" != "windows" ]] - then - sudo rm -rf /usr/local/go - sudo mkdir -p /usr/local - curl -SL --fail -o /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.${os}-amd64.tar.gz - sudo tar -C /usr/local -xzf /tmp/golang.tar.gz - rm -rf /tmp/golang.tar.gz - else - rm -rf /usr/local/go - mkdir -p /usr/local - curl -SL --fail -o /tmp/go.zip https://dl.google.com/go/go${GOLANG_VERSION}.windows-amd64.zip - unzip -q -o /tmp/go.zip -d /usr/local - rm -rf /tmp/go.zip - fi - name: Install golang - - run: - command: | - sudo ./scripts/vagrant-linux-priv-buf.sh - name: install buf - - run: - command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip - sudo unzip -d /usr/local/bin /tmp/consul.zip - rm -rf /tmp/consul* - name: Install Consul 1.8.3 - - run: - command: | - set -x - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - curl -SL --fail -o /tmp/vault.zip https://releases.hashicorp.com/vault/"1.2.4"/vault_"1.2.4"_"${os}"_amd64.zip - - if [[ "${os}" != "windows" ]]; then - sudo unzip -d /usr/local/bin /tmp/vault.zip - rm -rf /tmp/vault* - else - rm -rf c:\Windows\vault.exe - unzip /tmp/vault.zip -d "/c/Windows" - rm -rf /tmp/vault* - fi - name: Install Vault - - run: - command: | - if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then - sudo apt-get update - sudo apt-get install -y gcc-multilib - else - echo "Skipping 32bit lib installation while building for not 386" - fi - name: Install 32bit gcc libs - - run: - command: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap - - run: - command: | - if [ -z $GOTEST_PKGS_EXCLUDE ]; - then - unset GOTEST_PKGS_EXCLUDE - else - unset GOTEST_PKGS - fi - - if [ ! -z $GOTESTARCH ]; then - export GOARCH="$GOTESTARCH"; - fi - - mkdir -p /tmp/test-reports - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs - - if [ ! -z $GOTEST_MOD ]; then - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad-module - else - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad - fi - name: Running Nomad Tests - no_output_timeout: 20m - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-reports - test-client: - machine: - image: ubuntu-1604:201903-01 - working_directory: ~/go/src/github.com/hashicorp/nomad - environment: - - GIT_PAGER: cat - - GOLANG_VERSION: 1.15.5 - - GOMAXPROCS: 1 - - GOPATH: /home/circleci/go - - GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json - - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - - NOMAD_SLOW_TEST: 1 - - PAGER: cat - - GOTEST_MOD: '' - - GOTEST_PKGS: ./client/... - - GOTEST_PKGS_EXCLUDE: '' - - GOTESTARCH: amd64 - steps: - - checkout - - run: - command: | - set -x - - echo installing golang ${GOLANG_VERSION} - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - if [[ "${os}" != "windows" ]] - then - sudo rm -rf /usr/local/go - sudo mkdir -p /usr/local - curl -SL --fail -o /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.${os}-amd64.tar.gz - sudo tar -C /usr/local -xzf /tmp/golang.tar.gz - rm -rf /tmp/golang.tar.gz - else - rm -rf /usr/local/go - mkdir -p /usr/local - curl -SL --fail -o /tmp/go.zip https://dl.google.com/go/go${GOLANG_VERSION}.windows-amd64.zip - unzip -q -o /tmp/go.zip -d /usr/local - rm -rf /tmp/go.zip - fi - name: Install golang - - run: - command: | - sudo ./scripts/vagrant-linux-priv-buf.sh - name: install buf - - run: - command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip - sudo unzip -d /usr/local/bin /tmp/consul.zip - rm -rf /tmp/consul* - name: Install Consul 1.8.3 - - run: - command: | - set -x - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - curl -SL --fail -o /tmp/vault.zip https://releases.hashicorp.com/vault/"1.2.4"/vault_"1.2.4"_"${os}"_amd64.zip - - if [[ "${os}" != "windows" ]]; then - sudo unzip -d /usr/local/bin /tmp/vault.zip - rm -rf /tmp/vault* - else - rm -rf c:\Windows\vault.exe - unzip /tmp/vault.zip -d "/c/Windows" - rm -rf /tmp/vault* - fi - name: Install Vault - - run: - command: | - if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then - sudo apt-get update - sudo apt-get install -y gcc-multilib - else - echo "Skipping 32bit lib installation while building for not 386" - fi - name: Install 32bit gcc libs - - run: - command: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap - - run: - command: | - if [ -z $GOTEST_PKGS_EXCLUDE ]; - then - unset GOTEST_PKGS_EXCLUDE - else - unset GOTEST_PKGS - fi - - if [ ! -z $GOTESTARCH ]; then - export GOARCH="$GOTESTARCH"; - fi - - mkdir -p /tmp/test-reports - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs - - if [ ! -z $GOTEST_MOD ]; then - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad-module - else - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad - fi - name: Running Nomad Tests - no_output_timeout: 20m - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-reports - test-windows: - machine: - image: windows-server-2019-vs2019:stable - working_directory: c:\gopath\src\github.com\hashicorp\nomad - steps: - - run: - command: git config --global core.autocrlf false - - checkout - - run: - command: | - mkdir -p $GOBIN - mkdir -p $GOTESTSUM_PATH - name: Setup - - run: - command: | - set -x - - echo installing golang ${GOLANG_VERSION} - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - if [[ "${os}" != "windows" ]] - then - sudo rm -rf c:/go - sudo mkdir -p c: - curl -SL --fail -o /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.${os}-amd64.tar.gz - sudo tar -C c: -xzf /tmp/golang.tar.gz - rm -rf /tmp/golang.tar.gz - else - rm -rf c:/go - mkdir -p c: - curl -SL --fail -o /tmp/go.zip https://dl.google.com/go/go${GOLANG_VERSION}.windows-amd64.zip - unzip -q -o /tmp/go.zip -d c: - rm -rf /tmp/go.zip - fi - name: Install golang - - run: - command: go version - - run: - command: | - set -x - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - curl -SL --fail -o /tmp/vault.zip https://releases.hashicorp.com/vault/"$VAULT_VERSION"/vault_"$VAULT_VERSION"_"${os}"_amd64.zip - - if [[ "${os}" != "windows" ]]; then - sudo unzip -d /usr/local/bin /tmp/vault.zip - rm -rf /tmp/vault* - else - rm -rf c:\Windows\vault.exe - unzip /tmp/vault.zip -d "/c/Windows" - rm -rf /tmp/vault* - fi - name: Install Vault - - run: - command: vault version - - run: - command: choco install make - - run: - command: | - export PATH=$PATH:/c/go/bin:/c/gopath/bin - make deps - name: Install golang dependencies - - run: - command: docker pull docker.mirror.hashicorp.services/hashicorpnomad/busybox-windows:server2016-0.1 - name: Pre-download docker test image - - run: - command: | - export PATH=$PATH:/c/go/bin:/c/gopath/bin - go build -o $GOBIN\nomad.exe - name: Build nomad - - run: - command: | - # Only test docker driver tests for now - export PATH=$PATH:/c/go/bin:/c/gopath/bin - gotestsum --format=short-verbose \ - --junitfile $GOTESTSUM_PATH/results.xml \ - github.com/hashicorp/nomad/drivers/docker \ - github.com/hashicorp/nomad/client/lib/fifo \ - github.com/hashicorp/nomad/client/logmon - name: Run tests with gotestsum - - store_test_results: - path: c:\tmp\test-reports - - store_artifacts: - path: c:\tmp\test-reports - environment: - - GOBIN: c:\gopath\bin - - GOLANG_VERSION: 1.15.5 - - GOPATH: c:\gopath - - GOTESTSUM_PATH: c:\tmp\test-reports - - GOTESTSUM_VERSION: 0.4.2 - - VAULT_VERSION: 1.4.1 - resource_class: windows.medium - shell: bash --login -eo pipefail - test-shared-exec: - machine: - image: ubuntu-1604:201903-01 - working_directory: ~/go/src/github.com/hashicorp/nomad - environment: - - GIT_PAGER: cat - - GOLANG_VERSION: 1.15.5 - - GOMAXPROCS: 1 - - GOPATH: /home/circleci/go - - GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json - - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - - NOMAD_SLOW_TEST: 1 - - PAGER: cat - - GOTEST_MOD: '' - - GOTEST_PKGS: ./drivers/shared/executor - - GOTEST_PKGS_EXCLUDE: '' - - GOTESTARCH: amd64 - steps: - - checkout - - run: - command: | - set -x - - echo installing golang ${GOLANG_VERSION} - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - if [[ "${os}" != "windows" ]] - then - sudo rm -rf /usr/local/go - sudo mkdir -p /usr/local - curl -SL --fail -o /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.${os}-amd64.tar.gz - sudo tar -C /usr/local -xzf /tmp/golang.tar.gz - rm -rf /tmp/golang.tar.gz - else - rm -rf /usr/local/go - mkdir -p /usr/local - curl -SL --fail -o /tmp/go.zip https://dl.google.com/go/go${GOLANG_VERSION}.windows-amd64.zip - unzip -q -o /tmp/go.zip -d /usr/local - rm -rf /tmp/go.zip - fi - name: Install golang - - run: - command: | - sudo ./scripts/vagrant-linux-priv-buf.sh - name: install buf - - run: - command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip - sudo unzip -d /usr/local/bin /tmp/consul.zip - rm -rf /tmp/consul* - name: Install Consul 1.8.3 - - run: - command: | - set -x - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - curl -SL --fail -o /tmp/vault.zip https://releases.hashicorp.com/vault/"1.2.4"/vault_"1.2.4"_"${os}"_amd64.zip - - if [[ "${os}" != "windows" ]]; then - sudo unzip -d /usr/local/bin /tmp/vault.zip - rm -rf /tmp/vault* - else - rm -rf c:\Windows\vault.exe - unzip /tmp/vault.zip -d "/c/Windows" - rm -rf /tmp/vault* - fi - name: Install Vault - - run: - command: | - if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then - sudo apt-get update - sudo apt-get install -y gcc-multilib - else - echo "Skipping 32bit lib installation while building for not 386" - fi - name: Install 32bit gcc libs - - run: - command: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap - - run: - command: | - if [ -z $GOTEST_PKGS_EXCLUDE ]; - then - unset GOTEST_PKGS_EXCLUDE - else - unset GOTEST_PKGS - fi - - if [ ! -z $GOTESTARCH ]; then - export GOARCH="$GOTESTARCH"; - fi - - mkdir -p /tmp/test-reports - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs - - if [ ! -z $GOTEST_MOD ]; then - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad-module - else - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad - fi - name: Running Nomad Tests - no_output_timeout: 20m - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-reports - test-ui: - docker: - - environment: - JOBS: 2 - image: docker.mirror.hashicorp.services/circleci/node:12-browsers - steps: - - checkout - - restore_cache: - keys: - - v2-deps-{{ checksum "ui/yarn.lock" }} - - v2-deps- - - run: - command: cd ui && yarn install - name: yarn install - - save_cache: - key: v2-deps-{{ checksum "ui/yarn.lock" }} - paths: - - ./ui/node_modules - - run: - command: cd ui && yarn run lint:js - name: lint:js - - run: - command: cd ui && yarn run lint:hbs - name: lint:hbs - - run: - command: mkdir -p /tmp/test-reports && cd ui && yarn test - name: Ember tests - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-reports - lint-go: - docker: - - image: docker.mirror.hashicorp.services/golang:1.15.5 - working_directory: /go/src/github.com/hashicorp/nomad - steps: - - checkout - - run: - command: apt-get update; apt-get install -y shellcheck sudo unzip - - run: - command: | - sudo ./scripts/vagrant-linux-priv-buf.sh - name: install buf - - run: - command: "CCI_VERSION=\"0.1.5879\"\nCCI_SHA256=\"f178ea62c781aec06267017404f87983c87f171fd0e66ef3737916246ae66dd6\"\n\nURL=\"https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CCI_VERSION}/circleci-cli_${CCI_VERSION}_linux_amd64.tar.gz\"\n\nmkdir -p /tmp/circleci-cli/\ncurl --fail --show-error --location \\\n -o /tmp/circleci-cli/cli.tar.gz \"${URL}\"\n\necho \"$CCI_SHA256 /tmp/circleci-cli/cli.tar.gz\" | sha256sum -c\n\ntar -xz --strip-components=1 \\\n -C /tmp/circleci-cli \\\n -f /tmp/circleci-cli/cli.tar.gz \\\n \"circleci-cli_${CCI_VERSION}_linux_amd64/circleci\" \n\nsudo cp /tmp/circleci-cli/circleci /usr/bin/circleci-local-cli\n\ncircleci-local-cli version\n" - name: Install CircleCI CLI 0.1.5879 - - run: - command: make deps lint-deps - - run: - command: make check - - run: - command: make checkscripts - - run: - command: mkdir -p ui/dist && make generate-all static-assets - - run: - command: make -C .circleci CIRCLECI="circleci-local-cli --skip-update-check" ci-verify - name: check .circleci/config.yml is up-to-date - environment: - - GIT_PAGER: cat - - GOMAXPROCS: 1 - - GOPATH: /go - - GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json - - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - - NOMAD_SLOW_TEST: 1 - - PAGER: cat website-docker-image: docker: - - image: docker.mirror.hashicorp.services/circleci/buildpack-deps + - image: docker.mirror.hashicorp.services/circleci/buildpack-deps shell: /usr/bin/env bash -euo pipefail -c steps: - - checkout - - setup_remote_docker - - run: - command: | - # There is an edge case that would cause an issue here - if dependencies are updated to an exact copy - # of a previous version, for example if packge-lock.json is reverted, we need to manually push the new - # image to the "latest" tag - # Ignore job if running an enterprise build - IMAGE_TAG=$(cat website/Dockerfile website/package-lock.json | sha256sum | awk '{print $1;}') - echo "Using $IMAGE_TAG" - if [ "$CIRCLE_REPOSITORY_URL" != "git@github.com:hashicorp/nomad.git" ]; then - echo "Not Nomad OSS Repo, not building website docker image" - elif curl https://hub.docker.com/v2/repositories/hashicorp/nomad-website/tags/$IMAGE_TAG -fsL > /dev/null; then - echo "Dependencies have not changed, not building a new website docker image." - else - cd website/ - docker build -t hashicorp/nomad-website:$IMAGE_TAG . - docker tag hashicorp/nomad-website:$IMAGE_TAG hashicorp/nomad-website:latest - docker login -u $WEBSITE_DOCKER_USER -p $WEBSITE_DOCKER_PASS - docker push hashicorp/nomad-website - fi - name: Build Docker Image if Necessary - test-other: - machine: - image: ubuntu-1604:201903-01 - working_directory: ~/go/src/github.com/hashicorp/nomad - environment: - - GIT_PAGER: cat - - GOLANG_VERSION: 1.15.5 - - GOMAXPROCS: 1 - - GOPATH: /home/circleci/go - - GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json - - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - - NOMAD_SLOW_TEST: 1 - - PAGER: cat - - GOTEST_MOD: '' - - GOTEST_PKGS: '' - - GOTEST_PKGS_EXCLUDE: ./api|./client|./drivers/docker|./drivers/exec|./drivers/shared/executor|./nomad|./devices|./e2e - - GOTESTARCH: amd64 + - checkout + - setup_remote_docker + - run: + name: Build Docker Image if Necessary + command: | + # There is an edge case that would cause an issue here - if dependencies are updated to an exact copy + # of a previous version, for example if packge-lock.json is reverted, we need to manually push the new + # image to the "latest" tag + # Ignore job if running an enterprise build + IMAGE_TAG=$(cat website/Dockerfile website/package-lock.json | sha256sum | awk '{print $1;}') + echo "Using $IMAGE_TAG" + if [ "$CIRCLE_REPOSITORY_URL" != "git@github.com:hashicorp/nomad.git" ]; then + echo "Not Nomad OSS Repo, not building website docker image" + elif curl https://hub.docker.com/v2/repositories/hashicorp/nomad-website/tags/$IMAGE_TAG -fsL > /dev/null; then + echo "Dependencies have not changed, not building a new website docker image." + else + cd website/ + docker build -t hashicorp/nomad-website:$IMAGE_TAG . + docker tag hashicorp/nomad-website:$IMAGE_TAG hashicorp/nomad-website:latest + docker login -u $WEBSITE_DOCKER_USER -p $WEBSITE_DOCKER_PASS + docker push hashicorp/nomad-website + fi + test-windows: + executor: go-windows + steps: - - checkout - - run: - command: | - set -x + - run: git config --global core.autocrlf false + - checkout + - run: + name: Setup + command: | + mkdir -p $GOBIN + mkdir -p $GOTESTSUM_PATH + - install-golang: + target_directory: "c:" + - run: go version + - install-vault: + version: $VAULT_VERSION + - run: vault version + - run: choco install make + - run: + name: Install golang dependencies + command: | + export PATH=$PATH:/c/go/bin:/c/gopath/bin + make deps + - run: + name: Pre-download docker test image + command: docker pull docker.mirror.hashicorp.services/hashicorpnomad/busybox-windows:server2016-0.1 + - run: + name: Build nomad + command: | + export PATH=$PATH:/c/go/bin:/c/gopath/bin + go build -o $GOBIN\nomad.exe + - run: + name: Run tests with gotestsum + command: | + # Only test docker driver tests for now + export PATH=$PATH:/c/go/bin:/c/gopath/bin + gotestsum --format=short-verbose \ + --junitfile $GOTESTSUM_PATH/results.xml \ + github.com/hashicorp/nomad/drivers/docker \ + github.com/hashicorp/nomad/client/lib/fifo \ + github.com/hashicorp/nomad/client/logmon - echo installing golang ${GOLANG_VERSION} - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - if [[ "${os}" != "windows" ]] - then - sudo rm -rf /usr/local/go - sudo mkdir -p /usr/local - curl -SL --fail -o /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.${os}-amd64.tar.gz - sudo tar -C /usr/local -xzf /tmp/golang.tar.gz - rm -rf /tmp/golang.tar.gz - else - rm -rf /usr/local/go - mkdir -p /usr/local - curl -SL --fail -o /tmp/go.zip https://dl.google.com/go/go${GOLANG_VERSION}.windows-amd64.zip - unzip -q -o /tmp/go.zip -d /usr/local - rm -rf /tmp/go.zip - fi - name: Install golang - - run: - command: | - sudo ./scripts/vagrant-linux-priv-buf.sh - name: install buf - - run: - command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip - sudo unzip -d /usr/local/bin /tmp/consul.zip - rm -rf /tmp/consul* - name: Install Consul 1.8.3 - - run: - command: | - set -x - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - curl -SL --fail -o /tmp/vault.zip https://releases.hashicorp.com/vault/"1.2.4"/vault_"1.2.4"_"${os}"_amd64.zip - - if [[ "${os}" != "windows" ]]; then - sudo unzip -d /usr/local/bin /tmp/vault.zip - rm -rf /tmp/vault* - else - rm -rf c:\Windows\vault.exe - unzip /tmp/vault.zip -d "/c/Windows" - rm -rf /tmp/vault* - fi - name: Install Vault - - run: - command: | - if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then - sudo apt-get update - sudo apt-get install -y gcc-multilib - else - echo "Skipping 32bit lib installation while building for not 386" - fi - name: Install 32bit gcc libs - - run: - command: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap - - run: - command: | - if [ -z $GOTEST_PKGS_EXCLUDE ]; - then - unset GOTEST_PKGS_EXCLUDE - else - unset GOTEST_PKGS - fi - - if [ ! -z $GOTESTARCH ]; then - export GOARCH="$GOTESTARCH"; - fi - - mkdir -p /tmp/test-reports - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs - - if [ ! -z $GOTEST_MOD ]; then - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad-module - else - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad - fi - name: Running Nomad Tests - no_output_timeout: 20m - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-reports - test-devices: + - store_test_results: + path: c:\tmp\test-reports + - store_artifacts: + path: c:\tmp\test-reports + test-ui: docker: - - image: docker.mirror.hashicorp.services/golang:1.15.5 - working_directory: /go/src/github.com/hashicorp/nomad - environment: - - GIT_PAGER: cat - - GOMAXPROCS: 1 - - GOPATH: /go - - GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json - - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - - NOMAD_SLOW_TEST: 1 - - PAGER: cat - - GOTEST_PKGS: ./devices/... - - GOTEST_PKGS_EXCLUDE: '' - - GOTESTARCH: amd64 + - image: docker.mirror.hashicorp.services/circleci/node:12-browsers + environment: + # See https://git.io/vdao3 for details. + JOBS: 2 steps: - - checkout - - run: - command: apt-get update; apt-get install -y shellcheck sudo unzip - - run: - command: make deps - - run: - command: | - sudo ./scripts/vagrant-linux-priv-buf.sh - name: install buf - - run: - command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip - sudo unzip -d /usr/local/bin /tmp/consul.zip - rm -rf /tmp/consul* - name: Install Consul 1.8.3 - - run: - command: | - set -x + - checkout + - restore_cache: + keys: + - v2-deps-{{ checksum "ui/yarn.lock" }} + - v2-deps- + - run: + name: yarn install + command: cd ui && yarn install + - save_cache: + key: v2-deps-{{ checksum "ui/yarn.lock" }} + paths: + - ./ui/node_modules + - run: + name: lint:js + command: cd ui && yarn run lint:js + - run: + name: lint:hbs + command: cd ui && yarn run lint:hbs + - run: + name: Ember tests + command: mkdir -p /tmp/test-reports && cd ui && yarn test + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-reports + test-machine: + executor: "<< parameters.executor >>" + parameters: + test_packages: + type: string + default: "" + test_module: + type: string + default: "" + exclude_packages: + type: string + default: "" + executor: + type: string + default: "go-machine-recent" + goarch: + type: string + default: "amd64" + environment: + GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>" + GOTEST_PKGS: "<< parameters.test_packages >>" + GOTEST_MOD: "<< parameters.test_module >>" + GOTESTARCH: "<< parameters.goarch >>" + steps: + - checkout + - install-golang + - install-buf + - install-consul + - install-vault + - run: + name: Install 32bit gcc libs + command: | + if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then + sudo apt-get update + sudo apt-get install -y gcc-multilib + else + echo "Skipping 32bit lib installation while building for not 386" + fi + - run: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap + - run-tests + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-reports + test-e2e: + executor: go + steps: + - checkout + - run: apt-get update; apt-get install -y sudo unzip + # e2e tests require privileged mount/umount permissions when running as root + # TODO: switch to using machine executor and run as root to test e2e path + - run: + name: prepare non-root user + command: | + groupadd --gid 3434 circleci + useradd --uid 3434 --gid circleci --shell /bin/bash --create-home circleci + echo 'circleci ALL=NOPASSWD: ALL' >> /etc/sudoers.d/50-circleci + echo 'Defaults env_keep += "DEBIAN_FRONTEND"' >> /etc/sudoers.d/env_keep + chown -R circleci:circleci /go - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - curl -SL --fail -o /tmp/vault.zip https://releases.hashicorp.com/vault/"1.2.4"/vault_"1.2.4"_"${os}"_amd64.zip - - if [[ "${os}" != "windows" ]]; then - sudo unzip -d /usr/local/bin /tmp/vault.zip - rm -rf /tmp/vault* - else - rm -rf c:\Windows\vault.exe - unzip /tmp/vault.zip -d "/c/Windows" - rm -rf /tmp/vault* - fi - name: Install Vault - - run: - command: | - if [ -z $GOTEST_PKGS_EXCLUDE ]; - then - unset GOTEST_PKGS_EXCLUDE - else - unset GOTEST_PKGS - fi - - if [ ! -z $GOTESTARCH ]; then - export GOARCH="$GOTESTARCH"; - fi - - mkdir -p /tmp/test-reports - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs - - if [ ! -z $GOTEST_MOD ]; then - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad-module - else - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad - fi - name: Running Nomad Tests - no_output_timeout: 20m - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-reports + - run: sudo -E -H -u circleci PATH=${PATH} make deps + - run: sudo -E -H -u circleci PATH=${PATH} make integration-test + - run: sudo -E -H -u circleci PATH=${PATH} make e2e-test + test-container: + executor: go + parameters: + test_packages: + type: string + default: "" + exclude_packages: + type: string + default: "" + goarch: + type: string + default: "amd64" + environment: + GOTEST_PKGS: "<< parameters.test_packages >>" + GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>" + GOTESTARCH: "<< parameters.goarch >>" + steps: + - checkout + - run: apt-get update; apt-get install -y shellcheck sudo unzip + - run: make deps + - install-buf + - install-consul + - install-vault + - run-tests + - store_test_results: + path: /tmp/test-reports + - store_artifacts: + path: /tmp/test-reports + lint-go: + executor: go + steps: + - checkout + - run: apt-get update; apt-get install -y shellcheck sudo unzip + - install-buf + - install-circleci-local-cli + - run: make deps lint-deps + - run: make check + - run: make checkscripts + - run: mkdir -p ui/dist && make generate-all static-assets build-darwin-binaries: - macos: - xcode: 11.3.1 - working_directory: ~/go/src/github.com/hashicorp/nomad + executor: go-macos steps: - - checkout - - run: - command: echo 'export PATH="${GOPATH}/bin:${HOME}/goinstall/go/bin:$PATH"' >> ${BASH_ENV} - name: configure PATH - - run: - command: | - set -x + - checkout + - run: + name: configure PATH + command: + echo 'export PATH="${GOPATH}/bin:${HOME}/goinstall/go/bin:$PATH"' >> ${BASH_ENV} - echo installing golang ${GOLANG_VERSION} - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - if [[ "${os}" != "windows" ]] - then - sudo rm -rf ~/goinstall/go - sudo mkdir -p ~/goinstall - curl -SL --fail -o /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.${os}-amd64.tar.gz - sudo tar -C ~/goinstall -xzf /tmp/golang.tar.gz - rm -rf /tmp/golang.tar.gz - else - rm -rf ~/goinstall/go - mkdir -p ~/goinstall - curl -SL --fail -o /tmp/go.zip https://dl.google.com/go/go${GOLANG_VERSION}.windows-amd64.zip - unzip -q -o /tmp/go.zip -d ~/goinstall - rm -rf /tmp/go.zip - fi - name: Install golang - - run: - command: source ${BASH_ENV} && make deps - - run: - command: brew install protobuf - - run: - command: PATH="$GOPATH/bin:${HOME}/goinstall/go/bin:$PATH" make generate-structs - - run: - command: source ${BASH_ENV} && make pkg/darwin_amd64.zip - - store_artifacts: - destination: /builds/nomad_darwin_amd64.zip - path: pkg/darwin_amd64.zip + - install-golang: + target_directory: ~/goinstall + - run: source ${BASH_ENV} && make deps + - run: brew install protobuf + - run: PATH="$GOPATH/bin:${HOME}/goinstall/go/bin:$PATH" make generate-structs + - run: source ${BASH_ENV} && make pkg/darwin_amd64.zip + - store_artifacts: + path: pkg/darwin_amd64.zip + destination: /builds/nomad_darwin_amd64.zip + build-binaries: + executor: go environment: - - GIT_PAGER: cat - - GOLANG_VERSION: 1.15.5 - - GOMAXPROCS: 1 - - GOPATH: /Users/distiller/go - - GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json - - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - - NOMAD_SLOW_TEST: 1 - - PAGER: cat + # TODO: add ui tag here + GO_TAGS: "codegen_generated" + steps: + - checkout + - run: apt-get update; apt-get install -y sudo unzip zip + - run: make deps + - install-buf + - run: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs + - run: make pkg/windows_amd64.zip pkg/linux_amd64.zip + - store_artifacts: + path: pkg/windows_amd64.zip + destination: /builds/nomad_windows_amd64.zip + - store_artifacts: + path: pkg/linux_amd64.zip + destination: /builds/nomad_linux_amd64.zip algolia_index: docker: - - image: docker.mirror.hashicorp.services/node:12 + - image: docker.mirror.hashicorp.services/node:12 steps: - - checkout - - run: - command: | - if [ "$CIRCLE_REPOSITORY_URL" != "git@github.com:hashicorp/nomad.git" ]; then - echo "Not Nomad OSS Repo, not indexing Algolia" - exit 0 - fi + - checkout + - run: + name: Push content to Algolia Index + command: | + if [ "$CIRCLE_REPOSITORY_URL" != "git@github.com:hashicorp/nomad.git" ]; then + echo "Not Nomad OSS Repo, not indexing Algolia" + exit 0 + fi - cd website/ - npm install - node scripts/index_search_content.js - name: Push content to Algolia Index - test-docker: - machine: - image: circleci/classic:201808-01 - working_directory: ~/go/src/github.com/hashicorp/nomad - environment: - - GIT_PAGER: cat - - GOLANG_VERSION: 1.15.5 - - GOMAXPROCS: 1 - - GOPATH: /home/circleci/go - - GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json - - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - - NOMAD_SLOW_TEST: 1 - - PAGER: cat - - GOTEST_MOD: '' - - GOTEST_PKGS: ./drivers/docker - - GOTEST_PKGS_EXCLUDE: '' - - GOTESTARCH: amd64 - steps: - - checkout - - run: - command: | - set -x - - echo installing golang ${GOLANG_VERSION} - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - if [[ "${os}" != "windows" ]] - then - sudo rm -rf /usr/local/go - sudo mkdir -p /usr/local - curl -SL --fail -o /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.${os}-amd64.tar.gz - sudo tar -C /usr/local -xzf /tmp/golang.tar.gz - rm -rf /tmp/golang.tar.gz - else - rm -rf /usr/local/go - mkdir -p /usr/local - curl -SL --fail -o /tmp/go.zip https://dl.google.com/go/go${GOLANG_VERSION}.windows-amd64.zip - unzip -q -o /tmp/go.zip -d /usr/local - rm -rf /tmp/go.zip - fi - name: Install golang - - run: - command: | - sudo ./scripts/vagrant-linux-priv-buf.sh - name: install buf - - run: - command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip - sudo unzip -d /usr/local/bin /tmp/consul.zip - rm -rf /tmp/consul* - name: Install Consul 1.8.3 - - run: - command: | - set -x - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - curl -SL --fail -o /tmp/vault.zip https://releases.hashicorp.com/vault/"1.2.4"/vault_"1.2.4"_"${os}"_amd64.zip - - if [[ "${os}" != "windows" ]]; then - sudo unzip -d /usr/local/bin /tmp/vault.zip - rm -rf /tmp/vault* - else - rm -rf c:\Windows\vault.exe - unzip /tmp/vault.zip -d "/c/Windows" - rm -rf /tmp/vault* - fi - name: Install Vault - - run: - command: | - if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then - sudo apt-get update - sudo apt-get install -y gcc-multilib - else - echo "Skipping 32bit lib installation while building for not 386" - fi - name: Install 32bit gcc libs - - run: - command: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap - - run: - command: | - if [ -z $GOTEST_PKGS_EXCLUDE ]; - then - unset GOTEST_PKGS_EXCLUDE - else - unset GOTEST_PKGS - fi - - if [ ! -z $GOTESTARCH ]; then - export GOARCH="$GOTESTARCH"; - fi - - mkdir -p /tmp/test-reports - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs - - if [ ! -z $GOTEST_MOD ]; then - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad-module - else - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad - fi - name: Running Nomad Tests - no_output_timeout: 20m - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-reports - build-binaries: - docker: - - image: docker.mirror.hashicorp.services/golang:1.15.5 + cd website/ + npm install + node scripts/index_search_content.js +executors: + go: working_directory: /go/src/github.com/hashicorp/nomad - environment: - - GIT_PAGER: cat - - GOMAXPROCS: 1 - - GOPATH: /go - - GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json - - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - - NOMAD_SLOW_TEST: 1 - - PAGER: cat - - GO_TAGS: codegen_generated - steps: - - checkout - - run: - command: apt-get update; apt-get install -y sudo unzip zip - - run: - command: make deps - - run: - command: | - sudo ./scripts/vagrant-linux-priv-buf.sh - name: install buf - - run: - command: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs - - run: - command: make pkg/windows_amd64.zip pkg/linux_amd64.zip - - store_artifacts: - destination: /builds/nomad_windows_amd64.zip - path: pkg/windows_amd64.zip - - store_artifacts: - destination: /builds/nomad_linux_amd64.zip - path: pkg/linux_amd64.zip - test-e2e: docker: - - image: docker.mirror.hashicorp.services/golang:1.15.5 - working_directory: /go/src/github.com/hashicorp/nomad - steps: - - checkout - - run: - command: apt-get update; apt-get install -y sudo unzip - - run: - command: | - groupadd --gid 3434 circleci - useradd --uid 3434 --gid circleci --shell /bin/bash --create-home circleci - echo 'circleci ALL=NOPASSWD: ALL' >> /etc/sudoers.d/50-circleci - echo 'Defaults env_keep += "DEBIAN_FRONTEND"' >> /etc/sudoers.d/env_keep - chown -R circleci:circleci /go - name: prepare non-root user - - run: - command: sudo -E -H -u circleci PATH=${PATH} make deps - - run: - command: sudo -E -H -u circleci PATH=${PATH} make integration-test - - run: - command: sudo -E -H -u circleci PATH=${PATH} make e2e-test + - image: docker.mirror.hashicorp.services/golang:1.15.5 environment: - - GIT_PAGER: cat - - GOMAXPROCS: 1 - - GOPATH: /go - - GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json - - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - - NOMAD_SLOW_TEST: 1 - - PAGER: cat - test-32bit: - machine: - image: ubuntu-1604:201903-01 + <<: *common_envs + GOPATH: /go + + go-machine: working_directory: ~/go/src/github.com/hashicorp/nomad + machine: + image: *go_machine_image + environment: &machine_env + <<: *common_envs + GOPATH: /home/circleci/go + GOLANG_VERSION: 1.15.5 + + # uses a more recent image with unattended upgrades disabled properly + # but seems to break docker builds + go-machine-recent: + working_directory: ~/go/src/github.com/hashicorp/nomad + machine: + image: *go_machine_recent_image + environment: *machine_env + + go-macos: + working_directory: ~/go/src/github.com/hashicorp/nomad + macos: + xcode: 11.3.1 environment: - - GIT_PAGER: cat - - GOLANG_VERSION: 1.15.5 - - GOMAXPROCS: 1 - - GOPATH: /home/circleci/go - - GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json - - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - - NOMAD_SLOW_TEST: 1 - - PAGER: cat - - GOTEST_MOD: '' - - GOTEST_PKGS: ./client/fingerprint - - GOTEST_PKGS_EXCLUDE: '' - - GOTESTARCH: '386' - steps: - - checkout - - run: - command: | - set -x + <<: *common_envs + GOPATH: /Users/distiller/go + GOLANG_VERSION: 1.15.5 - echo installing golang ${GOLANG_VERSION} + go-windows: + machine: + image: *go_windows_image + resource_class: windows.medium + shell: bash --login -eo pipefail + working_directory: c:\gopath\src\github.com\hashicorp\nomad + environment: + GOPATH: c:\gopath + GOBIN: c:\gopath\bin + GOTESTSUM_PATH: c:\tmp\test-reports + GOLANG_VERSION: 1.15.5 + GOTESTSUM_VERSION: 0.4.2 + VAULT_VERSION: 1.4.1 - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - if [[ "${os}" != "windows" ]] - then - sudo rm -rf /usr/local/go - sudo mkdir -p /usr/local - curl -SL --fail -o /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.${os}-amd64.tar.gz - sudo tar -C /usr/local -xzf /tmp/golang.tar.gz - rm -rf /tmp/golang.tar.gz - else - rm -rf /usr/local/go - mkdir -p /usr/local - curl -SL --fail -o /tmp/go.zip https://dl.google.com/go/go${GOLANG_VERSION}.windows-amd64.zip - unzip -q -o /tmp/go.zip -d /usr/local - rm -rf /tmp/go.zip - fi - name: Install golang - - run: - command: | - sudo ./scripts/vagrant-linux-priv-buf.sh - name: install buf - - run: - command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/1.8.3/consul_1.8.3_linux_amd64.zip - sudo unzip -d /usr/local/bin /tmp/consul.zip - rm -rf /tmp/consul* - name: Install Consul 1.8.3 - - run: - command: | - set -x - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - curl -SL --fail -o /tmp/vault.zip https://releases.hashicorp.com/vault/"1.2.4"/vault_"1.2.4"_"${os}"_amd64.zip - - if [[ "${os}" != "windows" ]]; then - sudo unzip -d /usr/local/bin /tmp/vault.zip - rm -rf /tmp/vault* - else - rm -rf c:\Windows\vault.exe - unzip /tmp/vault.zip -d "/c/Windows" - rm -rf /tmp/vault* - fi - name: Install Vault - - run: - command: | - if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then - sudo apt-get update - sudo apt-get install -y gcc-multilib - else - echo "Skipping 32bit lib installation while building for not 386" - fi - name: Install 32bit gcc libs - - run: - command: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap - - run: - command: | - if [ -z $GOTEST_PKGS_EXCLUDE ]; - then - unset GOTEST_PKGS_EXCLUDE - else - unset GOTEST_PKGS - fi - - if [ ! -z $GOTESTARCH ]; then - export GOARCH="$GOTESTARCH"; - fi - - mkdir -p /tmp/test-reports - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs - - if [ ! -z $GOTEST_MOD ]; then - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad-module - else - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad - fi - name: Running Nomad Tests - no_output_timeout: 20m - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-reports workflows: build-test: jobs: - - build-binaries: - filters: - branches: - ignore: - - stable-website - - /^.-ui\b.*/ - - lint-go: - filters: - branches: - ignore: - - /^.-ui\b.*/ - - /^docs-.*/ - - stable-website - - build-darwin-binaries: - filters: - branches: - ignore: - - /^.-ui\b.*/ - - /^docs-.*/ - - stable-website - - test-e2e: - filters: - branches: - ignore: - - /^.-ui\b.*/ - - /^docs-.*/ - - stable-website - - test-ui: - filters: - branches: - ignore: - - stable-website - - /^docs-.*/ - - /^e2e-.*/ - - test-windows: - filters: - branches: - ignore: - - /^.-ui\b.*/ - - /^docs-.*/ - - /^e2e-.*/ - - stable-website - - test-client: - filters: - branches: - ignore: - - /^.-ui\b.*/ - - /^docs-.*/ - - /^e2e-.*/ - - stable-website - - test-nomad: - filters: - branches: - ignore: - - /^.-ui\b.*/ - - /^docs-.*/ - - /^e2e-.*/ - - stable-website - - test-api: - filters: - branches: - ignore: - - /^.-ui\b.*/ - - /^docs-.*/ - - /^e2e-.*/ - - stable-website - - test-devices: - filters: - branches: - ignore: - - /^.-ui\b.*/ - - /^docs-.*/ - - /^e2e-.*/ - - stable-website - - test-other: - filters: - branches: - ignore: - - /^.-ui\b.*/ - - /^docs-.*/ - - /^e2e-.*/ - - stable-website - - test-docker: - filters: - branches: - ignore: - - /^.-ui\b.*/ - - /^docs-.*/ - - /^e2e-.*/ - - stable-website - - test-exec: - filters: - branches: - ignore: - - /^.-ui\b.*/ - - /^docs-.*/ - - /^e2e-.*/ - - stable-website - - test-shared-exec: - filters: - branches: - ignore: - - /^.-ui\b.*/ - - /^docs-.*/ - - /^e2e-.*/ - - stable-website - - test-32bit: - filters: - branches: - ignore: - - /^.-ui\b.*/ - - /^docs-.*/ - - /^e2e-.*/ - - stable-website + - build-binaries: + # almost always build binaries as they may be needed + # for e2e tests + filters: + branches: + ignore: + - stable-website + - /^.-ui\b.*/ + + - lint-go: + # check branches are almost all the backend branches + filters: &backend_check_branches_filter + branches: + ignore: + - /^.-ui\b.*/ + - /^docs-.*/ + - stable-website + + # Note: comment-out this job in ENT + - build-darwin-binaries: + filters: *backend_check_branches_filter + + - test-e2e: + filters: *backend_check_branches_filter + + - test-ui: + filters: + branches: + ignore: + - stable-website + - /^docs-.*/ + - /^e2e-.*/ + + # Note: comment-out this job in ENT + - test-windows: + filters: + branches: + ignore: + - /^.-ui\b.*/ + - /^docs-.*/ + - /^e2e-.*/ + - stable-website + + - test-machine: + name: "test-client" + test_packages: "./client/..." + # test branches are the branches that can impact unit tests + filters: &backend_test_branches_filter + branches: + ignore: + - /^.-ui\b.*/ + - /^docs-.*/ + - /^e2e-.*/ + - stable-website + - test-machine: + name: "test-nomad" + test_packages: "./nomad/..." + filters: *backend_test_branches_filter + - test-machine: + # API Tests run in a VM rather than container due to the FS tests + # requiring `mount` priviliges. + name: "test-api" + test_module: "api" + filters: *backend_test_branches_filter + - test-container: + name: "test-devices" + test_packages: "./devices/..." + filters: *backend_test_branches_filter + - test-machine: + name: "test-other" + exclude_packages: "./api|./client|./drivers/docker|./drivers/exec|./drivers/shared/executor|./nomad|./devices|./e2e" + filters: *backend_test_branches_filter + - test-machine: + name: "test-docker" + test_packages: "./drivers/docker" + # docker is misbehaving in docker-machine-recent image + # and we get unexpected failures + # e.g. https://circleci.com/gh/hashicorp/nomad/3854 + executor: go-machine + filters: *backend_test_branches_filter + - test-machine: + name: "test-exec" + test_packages: "./drivers/exec" + filters: *backend_test_branches_filter + - test-machine: + name: "test-shared-exec" + test_packages: "./drivers/shared/executor" + filters: *backend_test_branches_filter + - test-machine: + name: "test-32bit" + # Currently we only explicitly test fingerprinting on 32bit + # architectures. + test_packages: "./client/fingerprint" + goarch: "386" + filters: *backend_test_branches_filter website: jobs: - - website-docker-image: - filters: - branches: - only: - - master - context: static-sites - - algolia_index: - filters: - branches: - only: - - stable-website - context: static-sites - version: 2 + - website-docker-image: + context: static-sites + filters: + branches: + only: + - master + + - algolia_index: + context: static-sites + filters: + branches: + only: + - stable-website diff --git a/.circleci/config/commands/install-buf.yml b/.circleci/config/commands/install-buf.yml deleted file mode 100644 index 794c3a319..000000000 --- a/.circleci/config/commands/install-buf.yml +++ /dev/null @@ -1,5 +0,0 @@ -steps: - - run: - name: install buf - command: | - sudo ./scripts/vagrant-linux-priv-buf.sh diff --git a/.circleci/config/commands/install-circleci-local-cli.yml b/.circleci/config/commands/install-circleci-local-cli.yml deleted file mode 100644 index 5b89a3f69..000000000 --- a/.circleci/config/commands/install-circleci-local-cli.yml +++ /dev/null @@ -1,31 +0,0 @@ -parameters: - version: - type: string - default: 0.1.5879 - - sha256: - type: string - default: f178ea62c781aec06267017404f87983c87f171fd0e66ef3737916246ae66dd6 -steps: - - run: - name: Install CircleCI CLI << parameters.version >> - command: | - CCI_VERSION="<< parameters.version >>" - CCI_SHA256="<< parameters.sha256 >>" - - URL="https://github.com/CircleCI-Public/circleci-cli/releases/download/v${CCI_VERSION}/circleci-cli_${CCI_VERSION}_linux_amd64.tar.gz" - - mkdir -p /tmp/circleci-cli/ - curl --fail --show-error --location \ - -o /tmp/circleci-cli/cli.tar.gz "${URL}" - - echo "$CCI_SHA256 /tmp/circleci-cli/cli.tar.gz" | sha256sum -c - - tar -xz --strip-components=1 \ - -C /tmp/circleci-cli \ - -f /tmp/circleci-cli/cli.tar.gz \ - "circleci-cli_${CCI_VERSION}_linux_amd64/circleci" - - sudo cp /tmp/circleci-cli/circleci /usr/bin/circleci-local-cli - - circleci-local-cli version diff --git a/.circleci/config/commands/install-consul.yml b/.circleci/config/commands/install-consul.yml deleted file mode 100644 index c3eaf3b49..000000000 --- a/.circleci/config/commands/install-consul.yml +++ /dev/null @@ -1,11 +0,0 @@ -parameters: - version: - type: string - default: 1.8.3 -steps: - - run: - name: Install Consul << parameters.version >> - command: | - curl -SL --fail -o /tmp/consul.zip https://releases.hashicorp.com/consul/<< parameters.version >>/consul_<< parameters.version >>_linux_amd64.zip - sudo unzip -d /usr/local/bin /tmp/consul.zip - rm -rf /tmp/consul* diff --git a/.circleci/config/commands/install-golang.yml b/.circleci/config/commands/install-golang.yml deleted file mode 100644 index 00de89328..000000000 --- a/.circleci/config/commands/install-golang.yml +++ /dev/null @@ -1,33 +0,0 @@ -parameters: - target_directory: - type: string - default: /usr/local -steps: - - run: - name: Install golang - command: | - set -x - - echo installing golang ${GOLANG_VERSION} - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - if [[ "${os}" != "windows" ]] - then - sudo rm -rf << parameters.target_directory >>/go - sudo mkdir -p << parameters.target_directory >> - curl -SL --fail -o /tmp/golang.tar.gz https://dl.google.com/go/go${GOLANG_VERSION}.${os}-amd64.tar.gz - sudo tar -C << parameters.target_directory >> -xzf /tmp/golang.tar.gz - rm -rf /tmp/golang.tar.gz - else - rm -rf << parameters.target_directory >>/go - mkdir -p <> - curl -SL --fail -o /tmp/go.zip https://dl.google.com/go/go${GOLANG_VERSION}.windows-amd64.zip - unzip -q -o /tmp/go.zip -d << parameters.target_directory >> - rm -rf /tmp/go.zip - fi diff --git a/.circleci/config/commands/install-vault.yml b/.circleci/config/commands/install-vault.yml deleted file mode 100644 index ba325b4c6..000000000 --- a/.circleci/config/commands/install-vault.yml +++ /dev/null @@ -1,27 +0,0 @@ -parameters: - version: - type: string - default: 1.2.4 -steps: - - run: - name: Install Vault - command: | - set -x - - case "${OSTYPE}" in - linux*) os="linux" ;; - darwin*) os="darwin" ;; - msys*) os="windows" ;; - *) echo "unknown os: ${OSTYPE}"; exit 1 ;; - esac - - curl -SL --fail -o /tmp/vault.zip https://releases.hashicorp.com/vault/"<< parameters.version >>"/vault_"<< parameters.version >>"_"${os}"_amd64.zip - - if [[ "${os}" != "windows" ]]; then - sudo unzip -d /usr/local/bin /tmp/vault.zip - rm -rf /tmp/vault* - else - rm -rf c:\Windows\vault.exe - unzip /tmp/vault.zip -d "/c/Windows" - rm -rf /tmp/vault* - fi diff --git a/.circleci/config/commands/run-tests.yml b/.circleci/config/commands/run-tests.yml deleted file mode 100644 index a67ee28bf..000000000 --- a/.circleci/config/commands/run-tests.yml +++ /dev/null @@ -1,24 +0,0 @@ -steps: - - run: - name: Running Nomad Tests - no_output_timeout: 20m - command: | - if [ -z $GOTEST_PKGS_EXCLUDE ]; - then - unset GOTEST_PKGS_EXCLUDE - else - unset GOTEST_PKGS - fi - - if [ ! -z $GOTESTARCH ]; then - export GOARCH="$GOTESTARCH"; - fi - - mkdir -p /tmp/test-reports - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs - - if [ ! -z $GOTEST_MOD ]; then - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad-module - else - sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make test-nomad - fi diff --git a/.circleci/config/config.yml b/.circleci/config/config.yml deleted file mode 100644 index ba9da2443..000000000 --- a/.circleci/config/config.yml +++ /dev/null @@ -1,70 +0,0 @@ -version: 2.1 - -references: - # environment specific references - aim to avoid conflicts - go-machine-image: &go_machine_image - circleci/classic:201808-01 - go-machine-recent-image: &go_machine_recent_image - ubuntu-1604:201903-01 - go-windows-image: &go_windows_image - windows-server-2019-vs2019:stable - - # common references - common_envs: &common_envs - GOMAXPROCS: 1 - NOMAD_SLOW_TEST: 1 - GOTESTSUM_JUNITFILE: /tmp/test-reports/results.xml - GOTESTSUM_JSONFILE: /tmp/test-reports/testjsonfile.json - # disable implicit git paging. CircleCI runs commands with in a tty - # making git assume it's an interactive session. - PAGER: cat - GIT_PAGER: cat - -executors: - go: - working_directory: /go/src/github.com/hashicorp/nomad - docker: - - image: docker.mirror.hashicorp.services/golang:1.15.5 - environment: - <<: *common_envs - GOPATH: /go - - go-machine: - working_directory: ~/go/src/github.com/hashicorp/nomad - machine: - image: *go_machine_image - environment: &machine_env - <<: *common_envs - GOPATH: /home/circleci/go - GOLANG_VERSION: 1.15.5 - - # uses a more recent image with unattended upgrades disabled properly - # but seems to break docker builds - go-machine-recent: - working_directory: ~/go/src/github.com/hashicorp/nomad - machine: - image: *go_machine_recent_image - environment: *machine_env - - go-macos: - working_directory: ~/go/src/github.com/hashicorp/nomad - macos: - xcode: 11.3.1 - environment: - <<: *common_envs - GOPATH: /Users/distiller/go - GOLANG_VERSION: 1.15.5 - - go-windows: - machine: - image: *go_windows_image - resource_class: windows.medium - shell: bash --login -eo pipefail - working_directory: c:\gopath\src\github.com\hashicorp\nomad - environment: - GOPATH: c:\gopath - GOBIN: c:\gopath\bin - GOTESTSUM_PATH: c:\tmp\test-reports - GOLANG_VERSION: 1.15.5 - GOTESTSUM_VERSION: 0.4.2 - VAULT_VERSION: 1.4.1 diff --git a/.circleci/config/jobs/algolia_index.yml b/.circleci/config/jobs/algolia_index.yml deleted file mode 100644 index d9b5c8fb7..000000000 --- a/.circleci/config/jobs/algolia_index.yml +++ /dev/null @@ -1,15 +0,0 @@ -docker: - - image: docker.mirror.hashicorp.services/node:12 -steps: - - checkout - - run: - name: Push content to Algolia Index - command: | - if [ "$CIRCLE_REPOSITORY_URL" != "git@github.com:hashicorp/nomad.git" ]; then - echo "Not Nomad OSS Repo, not indexing Algolia" - exit 0 - fi - - cd website/ - npm install - node scripts/index_search_content.js diff --git a/.circleci/config/jobs/build-binaries.yml b/.circleci/config/jobs/build-binaries.yml deleted file mode 100644 index d6d603343..000000000 --- a/.circleci/config/jobs/build-binaries.yml +++ /dev/null @@ -1,17 +0,0 @@ -executor: go -environment: - # TODO: add ui tag here - GO_TAGS: "codegen_generated" -steps: - - checkout - - run: apt-get update; apt-get install -y sudo unzip zip - - run: make deps - - install-buf - - run: sudo -E PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make generate-structs - - run: make pkg/windows_amd64.zip pkg/linux_amd64.zip - - store_artifacts: - path: pkg/windows_amd64.zip - destination: /builds/nomad_windows_amd64.zip - - store_artifacts: - path: pkg/linux_amd64.zip - destination: /builds/nomad_linux_amd64.zip diff --git a/.circleci/config/jobs/build-darwin-binaries.yml b/.circleci/config/jobs/build-darwin-binaries.yml deleted file mode 100644 index e4a744baa..000000000 --- a/.circleci/config/jobs/build-darwin-binaries.yml +++ /dev/null @@ -1,17 +0,0 @@ -executor: go-macos -steps: - - checkout - - run: - name: configure PATH - command: - echo 'export PATH="${GOPATH}/bin:${HOME}/goinstall/go/bin:$PATH"' >> ${BASH_ENV} - - - install-golang: - target_directory: ~/goinstall - - run: source ${BASH_ENV} && make deps - - run: brew install protobuf - - run: PATH="$GOPATH/bin:${HOME}/goinstall/go/bin:$PATH" make generate-structs - - run: source ${BASH_ENV} && make pkg/darwin_amd64.zip - - store_artifacts: - path: pkg/darwin_amd64.zip - destination: /builds/nomad_darwin_amd64.zip diff --git a/.circleci/config/jobs/lint-go.yml b/.circleci/config/jobs/lint-go.yml deleted file mode 100644 index 7a52d8690..000000000 --- a/.circleci/config/jobs/lint-go.yml +++ /dev/null @@ -1,13 +0,0 @@ -executor: go -steps: - - checkout - - run: apt-get update; apt-get install -y shellcheck sudo unzip - - install-buf - - install-circleci-local-cli - - run: make deps lint-deps - - run: make check - - run: make checkscripts - - run: mkdir -p ui/dist && make generate-all static-assets - - run: - name: check .circleci/config.yml is up-to-date - command: make -C .circleci CIRCLECI="circleci-local-cli --skip-update-check" ci-verify diff --git a/.circleci/config/jobs/test-container.yml b/.circleci/config/jobs/test-container.yml deleted file mode 100644 index 933e2afb9..000000000 --- a/.circleci/config/jobs/test-container.yml +++ /dev/null @@ -1,27 +0,0 @@ -executor: go -parameters: - test_packages: - type: string - default: "" - exclude_packages: - type: string - default: "" - goarch: - type: string - default: "amd64" -environment: - GOTEST_PKGS: "<< parameters.test_packages >>" - GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>" - GOTESTARCH: "<< parameters.goarch >>" -steps: - - checkout - - run: apt-get update; apt-get install -y shellcheck sudo unzip - - run: make deps - - install-buf - - install-consul - - install-vault - - run-tests - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-reports diff --git a/.circleci/config/jobs/test-e2e.yml b/.circleci/config/jobs/test-e2e.yml deleted file mode 100644 index 0cde200c1..000000000 --- a/.circleci/config/jobs/test-e2e.yml +++ /dev/null @@ -1,18 +0,0 @@ -executor: go -steps: - - checkout - - run: apt-get update; apt-get install -y sudo unzip - # e2e tests require privileged mount/umount permissions when running as root - # TODO: switch to using machine executor and run as root to test e2e path - - run: - name: prepare non-root user - command: | - groupadd --gid 3434 circleci - useradd --uid 3434 --gid circleci --shell /bin/bash --create-home circleci - echo 'circleci ALL=NOPASSWD: ALL' >> /etc/sudoers.d/50-circleci - echo 'Defaults env_keep += "DEBIAN_FRONTEND"' >> /etc/sudoers.d/env_keep - chown -R circleci:circleci /go - - - run: sudo -E -H -u circleci PATH=${PATH} make deps - - run: sudo -E -H -u circleci PATH=${PATH} make integration-test - - run: sudo -E -H -u circleci PATH=${PATH} make e2e-test diff --git a/.circleci/config/jobs/test-machine.yml b/.circleci/config/jobs/test-machine.yml deleted file mode 100644 index 179b85a17..000000000 --- a/.circleci/config/jobs/test-machine.yml +++ /dev/null @@ -1,43 +0,0 @@ -executor: "<< parameters.executor >>" -parameters: - test_packages: - type: string - default: "" - test_module: - type: string - default: "" - exclude_packages: - type: string - default: "" - executor: - type: string - default: "go-machine-recent" - goarch: - type: string - default: "amd64" -environment: - GOTEST_PKGS_EXCLUDE: "<< parameters.exclude_packages >>" - GOTEST_PKGS: "<< parameters.test_packages >>" - GOTEST_MOD: "<< parameters.test_module >>" - GOTESTARCH: "<< parameters.goarch >>" -steps: - - checkout - - install-golang - - install-buf - - install-consul - - install-vault - - run: - name: Install 32bit gcc libs - command: | - if [ ! -z $GOTESTARCH ] && [ $GOTESTARCH == "386" ]; then - sudo apt-get update - sudo apt-get install -y gcc-multilib - else - echo "Skipping 32bit lib installation while building for not 386" - fi - - run: PATH="$GOPATH/bin:/usr/local/go/bin:$PATH" make bootstrap - - run-tests - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-reports diff --git a/.circleci/config/jobs/test-ui.yml b/.circleci/config/jobs/test-ui.yml deleted file mode 100644 index 8e68a48ef..000000000 --- a/.circleci/config/jobs/test-ui.yml +++ /dev/null @@ -1,31 +0,0 @@ -docker: - - image: docker.mirror.hashicorp.services/circleci/node:12-browsers - environment: - # See https://git.io/vdao3 for details. - JOBS: 2 -steps: - - checkout - - restore_cache: - keys: - - v2-deps-{{ checksum "ui/yarn.lock" }} - - v2-deps- - - run: - name: yarn install - command: cd ui && yarn install - - save_cache: - key: v2-deps-{{ checksum "ui/yarn.lock" }} - paths: - - ./ui/node_modules - - run: - name: lint:js - command: cd ui && yarn run lint:js - - run: - name: lint:hbs - command: cd ui && yarn run lint:hbs - - run: - name: Ember tests - command: mkdir -p /tmp/test-reports && cd ui && yarn test - - store_test_results: - path: /tmp/test-reports - - store_artifacts: - path: /tmp/test-reports diff --git a/.circleci/config/jobs/test-windows.yml b/.circleci/config/jobs/test-windows.yml deleted file mode 100644 index 8e2570a8b..000000000 --- a/.circleci/config/jobs/test-windows.yml +++ /dev/null @@ -1,45 +0,0 @@ -executor: go-windows - -steps: - - run: git config --global core.autocrlf false - - checkout - - run: - name: Setup - command: | - mkdir -p $GOBIN - mkdir -p $GOTESTSUM_PATH - - install-golang: - target_directory: "c:" - - run: go version - - install-vault: - version: $VAULT_VERSION - - run: vault version - - run: choco install make - - run: - name: Install golang dependencies - command: | - export PATH=$PATH:/c/go/bin:/c/gopath/bin - make deps - - run: - name: Pre-download docker test image - command: docker pull docker.mirror.hashicorp.services/hashicorpnomad/busybox-windows:server2016-0.1 - - run: - name: Build nomad - command: | - export PATH=$PATH:/c/go/bin:/c/gopath/bin - go build -o $GOBIN\nomad.exe - - run: - name: Run tests with gotestsum - command: | - # Only test docker driver tests for now - export PATH=$PATH:/c/go/bin:/c/gopath/bin - gotestsum --format=short-verbose \ - --junitfile $GOTESTSUM_PATH/results.xml \ - github.com/hashicorp/nomad/drivers/docker \ - github.com/hashicorp/nomad/client/lib/fifo \ - github.com/hashicorp/nomad/client/logmon - - - store_test_results: - path: c:\tmp\test-reports - - store_artifacts: - path: c:\tmp\test-reports diff --git a/.circleci/config/jobs/website-docker-image.yml b/.circleci/config/jobs/website-docker-image.yml deleted file mode 100644 index b8f3934fc..000000000 --- a/.circleci/config/jobs/website-docker-image.yml +++ /dev/null @@ -1,26 +0,0 @@ -docker: - - image: docker.mirror.hashicorp.services/circleci/buildpack-deps -shell: /usr/bin/env bash -euo pipefail -c -steps: - - checkout - - setup_remote_docker - - run: - name: Build Docker Image if Necessary - command: | - # There is an edge case that would cause an issue here - if dependencies are updated to an exact copy - # of a previous version, for example if packge-lock.json is reverted, we need to manually push the new - # image to the "latest" tag - # Ignore job if running an enterprise build - IMAGE_TAG=$(cat website/Dockerfile website/package-lock.json | sha256sum | awk '{print $1;}') - echo "Using $IMAGE_TAG" - if [ "$CIRCLE_REPOSITORY_URL" != "git@github.com:hashicorp/nomad.git" ]; then - echo "Not Nomad OSS Repo, not building website docker image" - elif curl https://hub.docker.com/v2/repositories/hashicorp/nomad-website/tags/$IMAGE_TAG -fsL > /dev/null; then - echo "Dependencies have not changed, not building a new website docker image." - else - cd website/ - docker build -t hashicorp/nomad-website:$IMAGE_TAG . - docker tag hashicorp/nomad-website:$IMAGE_TAG hashicorp/nomad-website:latest - docker login -u $WEBSITE_DOCKER_USER -p $WEBSITE_DOCKER_PASS - docker push hashicorp/nomad-website - fi diff --git a/.circleci/config/workflows/build-test.yml b/.circleci/config/workflows/build-test.yml deleted file mode 100644 index f04abd931..000000000 --- a/.circleci/config/workflows/build-test.yml +++ /dev/null @@ -1,96 +0,0 @@ -jobs: - - build-binaries: - # almost always build binaries as they may be needed - # for e2e tests - filters: - branches: - ignore: - - stable-website - - /^.-ui\b.*/ - - - lint-go: - # check branches are almost all the backend branches - filters: &backend_check_branches_filter - branches: - ignore: - - /^.-ui\b.*/ - - /^docs-.*/ - - stable-website - - # Note: comment-out this job in ENT - - build-darwin-binaries: - filters: *backend_check_branches_filter - - - test-e2e: - filters: *backend_check_branches_filter - - - test-ui: - filters: - branches: - ignore: - - stable-website - - /^docs-.*/ - - /^e2e-.*/ - - # Note: comment-out this job in ENT - - test-windows: - filters: - branches: - ignore: - - /^.-ui\b.*/ - - /^docs-.*/ - - /^e2e-.*/ - - stable-website - - - test-machine: - name: "test-client" - test_packages: "./client/..." - # test branches are the branches that can impact unit tests - filters: &backend_test_branches_filter - branches: - ignore: - - /^.-ui\b.*/ - - /^docs-.*/ - - /^e2e-.*/ - - stable-website - - test-machine: - name: "test-nomad" - test_packages: "./nomad/..." - filters: *backend_test_branches_filter - - test-machine: - # API Tests run in a VM rather than container due to the FS tests - # requiring `mount` priviliges. - name: "test-api" - test_module: "api" - filters: *backend_test_branches_filter - - test-container: - name: "test-devices" - test_packages: "./devices/..." - filters: *backend_test_branches_filter - - test-machine: - name: "test-other" - exclude_packages: "./api|./client|./drivers/docker|./drivers/exec|./drivers/shared/executor|./nomad|./devices|./e2e" - filters: *backend_test_branches_filter - - test-machine: - name: "test-docker" - test_packages: "./drivers/docker" - # docker is misbehaving in docker-machine-recent image - # and we get unexpected failures - # e.g. https://circleci.com/gh/hashicorp/nomad/3854 - executor: go-machine - filters: *backend_test_branches_filter - - test-machine: - name: "test-exec" - test_packages: "./drivers/exec" - filters: *backend_test_branches_filter - - test-machine: - name: "test-shared-exec" - test_packages: "./drivers/shared/executor" - filters: *backend_test_branches_filter - - test-machine: - name: "test-32bit" - # Currently we only explicitly test fingerprinting on 32bit - # architectures. - test_packages: "./client/fingerprint" - goarch: "386" - filters: *backend_test_branches_filter diff --git a/.circleci/config/workflows/website.yml b/.circleci/config/workflows/website.yml deleted file mode 100644 index 49b602beb..000000000 --- a/.circleci/config/workflows/website.yml +++ /dev/null @@ -1,14 +0,0 @@ -jobs: - - website-docker-image: - context: static-sites - filters: - branches: - only: - - master - - - algolia_index: - context: static-sites - filters: - branches: - only: - - stable-website