From c85f477d97ff9ef81f909f946a34872e6a07b6d6 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Tue, 14 Jan 2020 11:02:02 -0800 Subject: [PATCH 1/2] test: download Vault binaries for e2e test Modernize Vault integration/e2e test a bit: - Download from releases.hashicorp.com instead of using a hardcoded list - Remove old unused make target e2e-test - Use NOMAD_E2E env var instead of -integration flag - Add a README On my machine with ~250 Mbps internet it takes ~400s to download all Vault binaries. --- GNUmakefile | 14 --- e2e/vault/README.md | 12 +++ e2e/vault/matrix_test.go | 42 -------- e2e/vault/vault_test.go | 208 ++++++++++++++++++++++++++------------- 4 files changed, 152 insertions(+), 124 deletions(-) create mode 100644 e2e/vault/README.md delete mode 100644 e2e/vault/matrix_test.go diff --git a/GNUmakefile b/GNUmakefile index a56d32d5d..d48d82f9a 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -263,9 +263,6 @@ test: ## Run the Nomad test suite and/or the Nomad UI test suite @if [ $(RUN_UI_TESTS) ]; then \ make test-ui; \ fi - @if [ $(RUN_E2E_TESTS) ]; then \ - make e2e-test; \ - fi .PHONY: test-nomad test-nomad: dev ## Run Nomad test suites @@ -280,17 +277,6 @@ test-nomad: dev ## Run Nomad test suites bash -C "$(PROJECT_ROOT)/scripts/test_check.sh" ; \ fi -.PHONY: e2e-test -e2e-test: dev ## Run the Nomad e2e test suite - @echo "==> Running Nomad E2E test suites:" - go test \ - $(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \ - -cover \ - -timeout=900s \ - -tags "$(GO_TAGS)" \ - github.com/hashicorp/nomad/e2e/vault/ \ - -integration - .PHONY: clean clean: GOPATH=$(shell go env GOPATH) clean: ## Remove build artifacts diff --git a/e2e/vault/README.md b/e2e/vault/README.md new file mode 100644 index 000000000..79d56bd2c --- /dev/null +++ b/e2e/vault/README.md @@ -0,0 +1,12 @@ +# Vault Integration Test + +Downloads, caches, and tests Nomad against open source Vault binaries. Runs +only when `NOMAD_E2E` is set. + +Run with: + +``` +NOMAD_E2E=1 go test +``` + +**Warning: Downloads a lot of Vault versions!** diff --git a/e2e/vault/matrix_test.go b/e2e/vault/matrix_test.go deleted file mode 100644 index 86dcf96f6..000000000 --- a/e2e/vault/matrix_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package vault - -var ( - // versions is the set of Vault versions we test for backwards compatibility - versions = []string{ - "1.1.0", - "1.0.3", - "1.0.2", - "1.0.1", - "1.0.0", - "0.11.5", - "0.11.4", - "0.11.3", - "0.11.2", - "0.11.1", - "0.11.0", - "0.10.4", - "0.10.3", - "0.10.2", - "0.10.1", - "0.10.0", - "0.9.6", - "0.9.5", - "0.9.4", - "0.9.3", - "0.9.2", - "0.9.1", - "0.9.0", - "0.8.3", - "0.8.2", - "0.8.1", - "0.8.0", - "0.7.3", - "0.7.2", - "0.7.1", - "0.7.0", - "0.6.5", - "0.6.4", - "0.6.3", - "0.6.2", - } -) diff --git a/e2e/vault/vault_test.go b/e2e/vault/vault_test.go index e985bcfb3..fe008e6a4 100644 --- a/e2e/vault/vault_test.go +++ b/e2e/vault/vault_test.go @@ -4,7 +4,7 @@ import ( "archive/zip" "bytes" "context" - "flag" + "encoding/json" "fmt" "io" "io/ioutil" @@ -27,114 +27,185 @@ import ( vapi "github.com/hashicorp/vault/api" ) -var integration = flag.Bool("integration", false, "run integration tests") +var ( + minVaultVer = version.Must(version.NewVersion("0.6.2")) +) -// harness is used to retrieve the required Vault test binaries -type harness struct { - t *testing.T - binDir string - os string - arch string -} +// syncVault discovers available versions of Vault, downloads the binaries, +// returns a map of version to binary path. +func syncVault(t *testing.T) map[string]string { -// newHarness returns a new Vault test harness. -func newHarness(t *testing.T) *harness { - return &harness{ - t: t, - binDir: filepath.Join(os.TempDir(), "vault-bins/"), - os: runtime.GOOS, - arch: runtime.GOARCH, - } -} + binDir := filepath.Join(os.TempDir(), "vault-bins/") + + versions := vaultVersions(t) -// reconcile retrieves the desired binaries, returning a map of version to -// binary path -func (h *harness) reconcile() map[string]string { // Get the binaries we need to download - missing := h.diff() + missing, err := missingVault(binDir, versions) + require.NoError(t, err) // Create the directory for the binaries - h.createBinDir() + require.NoError(t, createBinDir(binDir)) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() + // Limit to N concurrent downloads + sema := make(chan int, 5) + + // Download in parallel + start := time.Now() g, _ := errgroup.WithContext(ctx) - for _, v := range missing { - version := v + for ver, url := range missing { + dst := filepath.Join(binDir, ver) g.Go(func() error { - return h.get(version) + sema <- 1 + defer func() { + <-sema + }() + return getVault(dst, url) }) } - if err := g.Wait(); err != nil { - h.t.Fatalf("failed getting versions: %v", err) + require.NoError(t, g.Wait()) + if n := len(missing); n > 0 { + t.Logf("Downloaded %d versions of Vault in %s", n, time.Now().Sub(start)) } binaries := make(map[string]string, len(versions)) - for _, v := range versions { - binaries[v] = filepath.Join(h.binDir, v) + for ver, _ := range versions { + binaries[ver] = filepath.Join(binDir, ver) } return binaries } +// vaultVersions discovers available Vault versions from releases.hashicorp.com +// and returns a map of version to url. +func vaultVersions(t *testing.T) map[string]string { + resp, err := http.Get("https://releases.hashicorp.com/vault/index.json") + require.NoError(t, err) + + respJson := struct { + Versions map[string]struct { + Builds []struct { + Version string `json:"version"` + Os string `json:"os"` + Arch string `json:"arch"` + URL string `json:"url"` + } `json:"builds"` + } + }{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&respJson)) + require.NoError(t, resp.Body.Close()) + + versions := map[string]string{} + for vk, vv := range respJson.Versions { + gover, err := version.NewVersion(vk) + if err != nil { + t.Logf("error parsing Vault version %q -> %v", vk, err) + continue + } + + // Skip ancient versions + if gover.LessThan(minVaultVer) { + continue + } + + // Skip prerelease and enterprise versions + if gover.Prerelease() != "" || gover.Metadata() != "" { + continue + } + + url := "" + for _, b := range vv.Builds { + buildver, err := version.NewVersion(b.Version) + if err != nil { + t.Logf("error parsing Vault build version %q -> %v", b.Version, err) + continue + } + + if buildver.Prerelease() != "" { + continue + } + + if buildver.Metadata() != "" { + continue + } + + if b.Os != runtime.GOOS { + continue + } + + if b.Arch != runtime.GOARCH { + continue + } + + // Match! + url = b.URL + break + } + + if url != "" { + versions[vk] = url + } + } + + return versions +} + // createBinDir creates the binary directory -func (h *harness) createBinDir() { +func createBinDir(binDir string) error { // Check if the directory exists, otherwise create it - f, err := os.Stat(h.binDir) + f, err := os.Stat(binDir) if err != nil && !os.IsNotExist(err) { - h.t.Fatalf("failed to stat directory: %v", err) + return fmt.Errorf("failed to stat directory: %v", err) } if f != nil && f.IsDir() { - return + return nil } else if f != nil { - if err := os.RemoveAll(h.binDir); err != nil { - h.t.Fatalf("failed to remove file at directory path: %v", err) + if err := os.RemoveAll(binDir); err != nil { + return fmt.Errorf("failed to remove file at directory path: %v", err) } } // Create the directory - if err := os.Mkdir(h.binDir, 0700); err != nil { - h.t.Fatalf("failed to make directory: %v", err) + if err := os.Mkdir(binDir, 075); err != nil { + return fmt.Errorf("failed to make directory: %v", err) } - if err := os.Chmod(h.binDir, 0700); err != nil { - h.t.Fatalf("failed to chmod: %v", err) + if err := os.Chmod(binDir, 0755); err != nil { + return fmt.Errorf("failed to chmod: %v", err) } + + return nil } -// diff returns the binaries that must be downloaded -func (h *harness) diff() (missing []string) { - files, err := ioutil.ReadDir(h.binDir) +// missingVault returns the binaries that must be downloaded. versions key must +// be the Vault version. +func missingVault(binDir string, versions map[string]string) (map[string]string, error) { + files, err := ioutil.ReadDir(binDir) if err != nil { if os.IsNotExist(err) { - return versions + return versions, nil } - h.t.Fatalf("failed to stat directory: %v", err) + return nil, fmt.Errorf("failed to stat directory: %v", err) } - // Build the set we need - missingSet := make(map[string]struct{}, len(versions)) - for _, v := range versions { - missingSet[v] = struct{}{} + // Copy versions so we don't mutate it + missingSet := make(map[string]string, len(versions)) + for k, v := range versions { + missingSet[k] = v } for _, f := range files { delete(missingSet, f.Name()) } - for k := range missingSet { - missing = append(missing, k) - } - - return missing + return missingSet, nil } -// get retrieves the given Vault binary -func (h *harness) get(version string) error { - resp, err := http.Get( - fmt.Sprintf("https://releases.hashicorp.com/vault/%s/vault_%s_%s_%s.zip", - version, version, h.os, h.arch)) +// getVault downloads the given Vault binary +func getVault(dst, url string) error { + resp, err := http.Get(url) if err != nil { return err } @@ -142,7 +213,9 @@ func (h *harness) get(version string) error { // Wrap in an in-mem buffer b := bytes.NewBuffer(nil) - io.Copy(b, resp.Body) + if _, err := io.Copy(b, resp.Body); err != nil { + return fmt.Errorf("error reading response body: %v", err) + } resp.Body.Close() zreader, err := zip.NewReader(bytes.NewReader(b.Bytes()), resp.ContentLength) @@ -155,8 +228,7 @@ func (h *harness) get(version string) error { } // Copy the file to its destination - file := filepath.Join(h.binDir, version) - out, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0777) + out, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0777) if err != nil { return err } @@ -176,17 +248,17 @@ func (h *harness) get(version string) error { // TestVaultCompatibility tests compatibility across Vault versions func TestVaultCompatibility(t *testing.T) { - if !*integration { - t.Skip("skipping test in non-integration mode.") + if os.Getenv("NOMAD_E2E") == "" { + t.Skip("Skipping e2e tests, NOMAD_E2E not set") } - h := newHarness(t) - vaultBinaries := h.reconcile() + vaultBinaries := syncVault(t) for version, vaultBin := range vaultBinaries { - vbin := vaultBin + ver := version + bin := vaultBin t.Run(version, func(t *testing.T) { - testVaultCompatibility(t, vbin, version) + testVaultCompatibility(t, bin, ver) }) } } From 96f4e9a47d527e0054a031acc93236a739d707d4 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Tue, 14 Jan 2020 13:47:51 -0800 Subject: [PATCH 2/2] test: restore e2e-test target and use -integration --- GNUmakefile | 14 ++++++++++++++ e2e/vault/README.md | 7 +++++-- e2e/vault/vault_test.go | 6 ++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index d48d82f9a..a56d32d5d 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -263,6 +263,9 @@ test: ## Run the Nomad test suite and/or the Nomad UI test suite @if [ $(RUN_UI_TESTS) ]; then \ make test-ui; \ fi + @if [ $(RUN_E2E_TESTS) ]; then \ + make e2e-test; \ + fi .PHONY: test-nomad test-nomad: dev ## Run Nomad test suites @@ -277,6 +280,17 @@ test-nomad: dev ## Run Nomad test suites bash -C "$(PROJECT_ROOT)/scripts/test_check.sh" ; \ fi +.PHONY: e2e-test +e2e-test: dev ## Run the Nomad e2e test suite + @echo "==> Running Nomad E2E test suites:" + go test \ + $(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \ + -cover \ + -timeout=900s \ + -tags "$(GO_TAGS)" \ + github.com/hashicorp/nomad/e2e/vault/ \ + -integration + .PHONY: clean clean: GOPATH=$(shell go env GOPATH) clean: ## Remove build artifacts diff --git a/e2e/vault/README.md b/e2e/vault/README.md index 79d56bd2c..189840bad 100644 --- a/e2e/vault/README.md +++ b/e2e/vault/README.md @@ -1,12 +1,15 @@ # Vault Integration Test +Not run as part of nightly e2e suite at this point. + Downloads, caches, and tests Nomad against open source Vault binaries. Runs -only when `NOMAD_E2E` is set. +only when `-integration` is set. Run with: ``` -NOMAD_E2E=1 go test +cd e2e/vault/ +go test -integration ``` **Warning: Downloads a lot of Vault versions!** diff --git a/e2e/vault/vault_test.go b/e2e/vault/vault_test.go index fe008e6a4..fed40e717 100644 --- a/e2e/vault/vault_test.go +++ b/e2e/vault/vault_test.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "encoding/json" + "flag" "fmt" "io" "io/ioutil" @@ -28,6 +29,7 @@ import ( ) var ( + integration = flag.Bool("integration", false, "run integration tests") minVaultVer = version.Must(version.NewVersion("0.6.2")) ) @@ -248,8 +250,8 @@ func getVault(dst, url string) error { // TestVaultCompatibility tests compatibility across Vault versions func TestVaultCompatibility(t *testing.T) { - if os.Getenv("NOMAD_E2E") == "" { - t.Skip("Skipping e2e tests, NOMAD_E2E not set") + if !*integration { + t.Skip("skipping test in non-integration mode: add -integration flag to run") } vaultBinaries := syncVault(t)