diff --git a/client/allocrunner/alloc_runner.go b/client/allocrunner/alloc_runner.go index f628ef501..bbee6a022 100644 --- a/client/allocrunner/alloc_runner.go +++ b/client/allocrunner/alloc_runner.go @@ -259,7 +259,7 @@ func (ar *allocRunner) Run() { ar.logger.Error("prerun failed", "error", err) for _, tr := range ar.tasks { - tr.MarkFailedDead(fmt.Sprintf("failed to setup runner: %v", err)) + tr.MarkFailedDead(fmt.Sprintf("failed to setup alloc: %v", err)) } goto POST diff --git a/e2e/connect/connect.go b/e2e/connect/connect.go new file mode 100644 index 000000000..5a113ede5 --- /dev/null +++ b/e2e/connect/connect.go @@ -0,0 +1,174 @@ +package connect + +import ( + "os" + "strings" + "time" + + consulapi "github.com/hashicorp/consul/api" + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/e2e/e2eutil" + "github.com/hashicorp/nomad/e2e/framework" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/jobspec" + "github.com/kr/pretty" + "github.com/stretchr/testify/require" +) + +type ConnectE2ETest struct { + framework.TC + jobIds []string +} + +func init() { + framework.AddSuites(&framework.TestSuite{ + Component: "Connect", + CanRunLocal: true, + Consul: true, + Cases: []framework.TestCase{ + new(ConnectE2ETest), + }, + }) +} + +func (tc *ConnectE2ETest) BeforeAll(f *framework.F) { + e2eutil.WaitForLeader(f.T(), tc.Nomad()) + e2eutil.WaitForNodesReady(f.T(), tc.Nomad(), 2) +} + +func (tc *ConnectE2ETest) AfterEach(f *framework.F) { + if os.Getenv("NOMAD_TEST_SKIPCLEANUP") == "1" { + return + } + + for _, id := range tc.jobIds { + tc.Nomad().Jobs().Deregister(id, true, nil) + } + tc.jobIds = []string{} + tc.Nomad().System().GarbageCollect() +} + +// TestConnectDemo tests the demo job file from the Consul Connect Technology +// Preview. +// +// https://github.com/hashicorp/nomad/blob/v0.9.5/website/source/guides/integrations/consul-connect/index.html.md#run-the-connect-enabled-services +// +func (tc *ConnectE2ETest) TestConnectDemo(f *framework.F) { + t := f.T() + uuid := uuid.Generate() + jobID := "connect" + uuid[0:8] + tc.jobIds = append(tc.jobIds, jobID) + jobapi := tc.Nomad().Jobs() + + job, err := jobspec.ParseFile("connect/input/demo.nomad") + require.NoError(t, err) + job.ID = &jobID + + resp, _, err := jobapi.Register(job, nil) + require.NoError(t, err) + require.NotNil(t, resp) + require.Zero(t, resp.Warnings) + +EVAL: + qopts := &api.QueryOptions{ + WaitIndex: resp.EvalCreateIndex, + } + evalapi := tc.Nomad().Evaluations() + eval, qmeta, err := evalapi.Info(resp.EvalID, qopts) + require.NoError(t, err) + qopts.WaitIndex = qmeta.LastIndex + + switch eval.Status { + case "pending": + goto EVAL + case "complete": + // Ok! + case "failed", "canceled", "blocked": + t.Fatalf("eval %s\n%s\n", eval.Status, pretty.Sprint(eval)) + default: + t.Fatalf("unknown eval status: %s\n%s\n", eval.Status, pretty.Sprint(eval)) + } + + // Assert there were 0 placement failures + require.Zero(t, eval.FailedTGAllocs, pretty.Sprint(eval.FailedTGAllocs)) + require.Len(t, eval.QueuedAllocations, 2, pretty.Sprint(eval.QueuedAllocations)) + + // Assert allocs are running + for i := 0; i < 20; i++ { + allocs, qmeta, err := evalapi.Allocations(eval.ID, qopts) + require.NoError(t, err) + require.Len(t, allocs, 2) + qopts.WaitIndex = qmeta.LastIndex + + running := 0 + for _, alloc := range allocs { + switch alloc.ClientStatus { + case "running": + running++ + case "pending": + // keep trying + default: + t.Fatalf("alloc failed: %s", pretty.Sprint(alloc)) + } + } + + if running == len(allocs) { + break + } + + time.Sleep(500 * time.Millisecond) + } + + allocs, _, err := evalapi.Allocations(eval.ID, qopts) + require.NoError(t, err) + allocIDs := make(map[string]bool, 2) + for _, a := range allocs { + if a.ClientStatus != "running" || a.DesiredStatus != "run" { + t.Fatalf("alloc %s (%s) terminal; client=%s desired=%s", a.TaskGroup, a.ID, a.ClientStatus, a.DesiredStatus) + } + allocIDs[a.ID] = true + } + + // Check Consul service health + agentapi := tc.Consul().Agent() + + failing := map[string]*consulapi.AgentCheck{} + for i := 0; i < 60; i++ { + checks, err := agentapi.Checks() + require.NoError(t, err) + + // Filter out checks for other services + for cid, check := range checks { + found := false + for allocID := range allocIDs { + if strings.Contains(check.ServiceID, allocID) { + found = true + break + } + } + + if !found { + delete(checks, cid) + } + } + + // Ensure checks are all passing + failing = map[string]*consulapi.AgentCheck{} + for _, check := range checks { + if check.Status != "passing" { + failing[check.CheckID] = check + break + } + } + + if len(failing) == 0 { + break + } + + t.Logf("still %d checks not passing", len(failing)) + + time.Sleep(time.Second) + } + + require.Len(t, failing, 0, pretty.Sprint(failing)) +} diff --git a/e2e/connect/input/demo.nomad b/e2e/connect/input/demo.nomad new file mode 100644 index 000000000..0264b77dd --- /dev/null +++ b/e2e/connect/input/demo.nomad @@ -0,0 +1,61 @@ + job "countdash" { + datacenters = ["dc1"] + + group "api" { + network { + mode = "bridge" + } + + service { + name = "count-api" + port = "9001" + + connect { + sidecar_service {} + } + } + + task "web" { + driver = "docker" + config { + image = "hashicorpnomad/counter-api:v1" + } + } + } + + group "dashboard" { + network { + mode ="bridge" + port "http" { + static = 9002 + to = 9002 + } + } + + service { + name = "count-dashboard" + port = "9002" + + connect { + sidecar_service { + proxy { + upstreams { + destination_name = "count-api" + local_bind_port = 8080 + } + } + } + } + } + + task "dashboard" { + driver = "docker" + env { + COUNTING_SERVICE_URL = "http://${NOMAD_UPSTREAM_ADDR_count_api}" + } + config { + image = "hashicorpnomad/counter-dashboard:v1" + } + } + } + } diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 10fde5f5a..3c861728c 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -6,6 +6,7 @@ import ( _ "github.com/hashicorp/nomad/e2e/affinities" _ "github.com/hashicorp/nomad/e2e/allocstats" _ "github.com/hashicorp/nomad/e2e/clientstate" + _ "github.com/hashicorp/nomad/e2e/connect" _ "github.com/hashicorp/nomad/e2e/consul" _ "github.com/hashicorp/nomad/e2e/consultemplate" _ "github.com/hashicorp/nomad/e2e/deployment" diff --git a/e2e/terraform/compute.tf b/e2e/terraform/compute.tf index 33da7ce52..230ac8b36 100644 --- a/e2e/terraform/compute.tf +++ b/e2e/terraform/compute.tf @@ -114,7 +114,10 @@ resource "aws_instance" "client" { "sudo chmod 0755 /usr/local/bin/nomad", "sudo chown root:root /usr/local/bin/nomad", "sudo systemctl enable nomad.service", - "sudo systemctl start nomad.service" + "sudo systemctl start nomad.service", + # Install CNI plugins + "sudo mkdir -p /opt/cni/bin", + "wget -q -O - https://github.com/containernetworking/plugins/releases/download/v0.8.2/cni-plugins-linux-amd64-v0.8.2.tgz | sudo tar -C /opt/cni/bin -xz", ] connection { diff --git a/e2e/terraform/shared/config/consul.json b/e2e/terraform/shared/config/consul.json index 5d6cd3ea8..7edf0b153 100644 --- a/e2e/terraform/shared/config/consul.json +++ b/e2e/terraform/shared/config/consul.json @@ -10,5 +10,11 @@ "service": { "name": "consul" }, - "retry_join": ["RETRY_JOIN"] + "retry_join": ["RETRY_JOIN"], + "connect": { + "enabled": true + }, + "ports": { + "grpc": 8502 + } } diff --git a/e2e/terraform/shared/config/consul_client.json b/e2e/terraform/shared/config/consul_client.json index 94026fa1e..a261b5a5d 100644 --- a/e2e/terraform/shared/config/consul_client.json +++ b/e2e/terraform/shared/config/consul_client.json @@ -5,5 +5,11 @@ "bind_addr": "0.0.0.0", "client_addr": "0.0.0.0", "advertise_addr": "IP_ADDRESS", - "retry_join": ["RETRY_JOIN"] + "retry_join": ["RETRY_JOIN"], + "connect": { + "enabled": true + }, + "ports": { + "grpc": 8502 + } } diff --git a/e2e/terraform/shared/scripts/setup.sh b/e2e/terraform/shared/scripts/setup.sh index c83cdc001..3e4071383 100644 --- a/e2e/terraform/shared/scripts/setup.sh +++ b/e2e/terraform/shared/scripts/setup.sh @@ -9,7 +9,7 @@ cd /ops CONFIGDIR=/ops/shared/config -CONSULVERSION=1.4.4 +CONSULVERSION=1.6.0 CONSULDOWNLOAD=https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip CONSULCONFIGDIR=/etc/consul.d CONSULDIR=/opt/consul @@ -25,7 +25,7 @@ NOMADDOWNLOAD=https://releases.hashicorp.com/nomad/${NOMADVERSION}/nomad_${NOMAD NOMADCONFIGDIR=/etc/nomad.d NOMADDIR=/opt/nomad -HADOOP_VERSION=2.7.6 +HADOOP_VERSION=2.7.7 # Dependencies sudo apt-get install -y software-properties-common diff --git a/terraform/shared/scripts/setup.sh b/terraform/shared/scripts/setup.sh index d9a3ef14c..c3cca9c72 100644 --- a/terraform/shared/scripts/setup.sh +++ b/terraform/shared/scripts/setup.sh @@ -9,7 +9,7 @@ cd /ops CONFIGDIR=/ops/shared/config -CONSULVERSION=1.4.3 +CONSULVERSION=1.6.0 CONSULDOWNLOAD=https://releases.hashicorp.com/consul/${CONSULVERSION}/consul_${CONSULVERSION}_linux_amd64.zip CONSULCONFIGDIR=/etc/consul.d CONSULDIR=/opt/consul