From 04a1623b853e2ce1471777490cd28fc8b5a1172d Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Fri, 6 Dec 2019 01:49:01 -0500 Subject: [PATCH 1/4] connect: configure envoy such that multiple sidecars can run in the same alloc --- .../taskrunner/envoybootstrap_hook.go | 21 ++++++++++++++++++- nomad/job_endpoint_hook_connect.go | 1 + 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/client/allocrunner/taskrunner/envoybootstrap_hook.go b/client/allocrunner/taskrunner/envoybootstrap_hook.go index 35beda5f6..137285315 100644 --- a/client/allocrunner/taskrunner/envoybootstrap_hook.go +++ b/client/allocrunner/taskrunner/envoybootstrap_hook.go @@ -18,6 +18,8 @@ import ( var _ interfaces.TaskPrestartHook = &envoyBootstrapHook{} +const envoyBaseAdminPort = 19000 + // envoyBootstrapHook writes the bootstrap config for the Connect Envoy proxy // sidecar. type envoyBootstrapHook struct { @@ -76,12 +78,17 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskP // the host netns. grpcAddr := "unix://" + allocdir.AllocGRPCSocket + // Envoy runs an administrative API on the loopback interface. If multiple sidecars + // are running, the bind addresses need to have unique ports. + // TODO: support running in host netns, using freeport to find available port + envoyAdminBind := buildEnvoyAdminBind(h.alloc, req.Task.Name) + // Envoy bootstrap configuration may contain a Consul token, so write // it to the secrets directory like Vault tokens. fn := filepath.Join(req.TaskDir.SecretsDir, "envoy_bootstrap.json") id := agentconsul.MakeAllocServiceID(h.alloc.ID, "group-"+tg.Name, service) - h.logger.Debug("bootstrapping envoy", "sidecar_for", service.Name, "boostrap_file", fn, "sidecar_for_id", id, "grpc_addr", grpcAddr) + h.logger.Debug("bootstrapping envoy", "sidecar_for", service.Name, "boostrap_file", fn, "sidecar_for_id", id, "grpc_addr", grpcAddr, "admin_bind", envoyAdminBind) // Since Consul services are registered asynchronously with this task // hook running, retry a small number of times with backoff. @@ -89,6 +96,7 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskP cmd := exec.CommandContext(ctx, "consul", "connect", "envoy", "-grpc-addr", grpcAddr, "-http-addr", h.consulHTTPAddr, + "-admin-bind", envoyAdminBind, "-bootstrap", "-sidecar-for", id, ) @@ -148,3 +156,14 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskP resp.Done = true return nil } + +func buildEnvoyAdminBind(alloc *structs.Allocation, taskName string) string { + port := envoyBaseAdminPort + for idx, task := range alloc.Job.LookupTaskGroup(alloc.TaskGroup).Tasks { + if task.Name == taskName { + port += idx + break + } + } + return fmt.Sprintf("localhost:%d", port) +} diff --git a/nomad/job_endpoint_hook_connect.go b/nomad/job_endpoint_hook_connect.go index b3392f415..80eec7e92 100644 --- a/nomad/job_endpoint_hook_connect.go +++ b/nomad/job_endpoint_hook_connect.go @@ -25,6 +25,7 @@ var ( "args": []interface{}{ "-c", structs.EnvoyBootstrapPath, "-l", "${meta.connect.log_level}", + "--disable-hot-restart", }, } From 8bca0e0431bd597314d472fd8a571114629506d6 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Mon, 6 Jan 2020 12:48:35 -0500 Subject: [PATCH 2/4] e2e: add test for multiple sevice sidecars in the same alloc --- e2e/connect/input/multi-service.nomad | 50 ++++++++++ e2e/connect/multi_service.go | 133 ++++++++++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 e2e/connect/input/multi-service.nomad create mode 100644 e2e/connect/multi_service.go diff --git a/e2e/connect/input/multi-service.nomad b/e2e/connect/input/multi-service.nomad new file mode 100644 index 000000000..1699dbf85 --- /dev/null +++ b/e2e/connect/input/multi-service.nomad @@ -0,0 +1,50 @@ +job "multi-service" { + datacenters = ["dc1"] + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "test" { + network { + mode = "bridge" + } + + service { + name = "echo1" + port = "9001" + + connect { + sidecar_service {} + } + } + + task "echo1" { + driver = "docker" + + config { + image = "hashicorp/http-echo" + args = ["-listen=:9001", "-text=echo1"] + } + } + + service { + name = "echo2" + port = "9002" + + connect { + sidecar_service {} + } + } + + task "echo2" { + driver = "docker" + + config { + image = "hashicorp/http-echo" + args = ["-listen=:9002", "-text=echo2"] + } + } + } +} diff --git a/e2e/connect/multi_service.go b/e2e/connect/multi_service.go new file mode 100644 index 000000000..0aa4eb8c0 --- /dev/null +++ b/e2e/connect/multi_service.go @@ -0,0 +1,133 @@ +package connect + +import ( + "strings" + "time" + + consulapi "github.com/hashicorp/consul/api" + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/e2e/framework" + "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/jobspec" + "github.com/kr/pretty" + "github.com/stretchr/testify/require" +) + +// TestMultiServiceConnect tests running multiple envoy sidecars in the same allocation. +func (tc *ConnectE2ETest) TestMultiServiceConnect(f *framework.F) { + t := f.T() + uuid := uuid.Generate() + jobID := "connect" + uuid[0:8] + tc.jobIds = append(tc.jobIds, jobID) + jobapi := tc.Nomad().Jobs() + + job, err := jobspec.ParseFile("connect/input/multi-service.nomad") + require.NoError(t, err) + job.ID = &jobID + + resp, _, err := jobapi.Register(job, nil) + require.NoError(t, err) + require.NotNil(t, resp) + require.Zero(t, resp.Warnings) + +EVAL: + qopts := &api.QueryOptions{ + WaitIndex: resp.EvalCreateIndex, + } + evalapi := tc.Nomad().Evaluations() + eval, qmeta, err := evalapi.Info(resp.EvalID, qopts) + require.NoError(t, err) + qopts.WaitIndex = qmeta.LastIndex + + switch eval.Status { + case "pending": + goto EVAL + case "complete": + // Ok! + case "failed", "canceled", "blocked": + t.Fatalf("eval %s\n%s\n", eval.Status, pretty.Sprint(eval)) + default: + t.Fatalf("unknown eval status: %s\n%s\n", eval.Status, pretty.Sprint(eval)) + } + + // Assert there were 0 placement failures + require.Zero(t, eval.FailedTGAllocs, pretty.Sprint(eval.FailedTGAllocs)) + require.Len(t, eval.QueuedAllocations, 1, pretty.Sprint(eval.QueuedAllocations)) + + // Assert allocs are running + require.Eventually(t, func() bool { + allocs, qmeta, err := evalapi.Allocations(eval.ID, qopts) + require.NoError(t, err) + require.Len(t, allocs, 1) + qopts.WaitIndex = qmeta.LastIndex + + running := 0 + for _, alloc := range allocs { + switch alloc.ClientStatus { + case "running": + running++ + case "pending": + // keep trying + default: + t.Fatalf("alloc failed: %s", pretty.Sprint(alloc)) + } + } + + if running == len(allocs) { + return true + } + return false + }, 10*time.Second, 500*time.Millisecond) + + allocs, _, err := evalapi.Allocations(eval.ID, qopts) + require.NoError(t, err) + allocIDs := make(map[string]bool, 1) + for _, a := range allocs { + if a.ClientStatus != "running" || a.DesiredStatus != "run" { + t.Fatalf("alloc %s (%s) terminal; client=%s desired=%s", a.TaskGroup, a.ID, a.ClientStatus, a.DesiredStatus) + } + allocIDs[a.ID] = true + } + + // Check Consul service health + agentapi := tc.Consul().Agent() + + failing := map[string]*consulapi.AgentCheck{} + require.Eventually(t, func() bool { + checks, err := agentapi.Checks() + require.NoError(t, err) + + // Filter out checks for other services + for cid, check := range checks { + found := false + for allocID := range allocIDs { + if strings.Contains(check.ServiceID, allocID) { + found = true + break + } + } + + if !found { + delete(checks, cid) + } + } + + // Ensure checks are all passing + failing = map[string]*consulapi.AgentCheck{} + for _, check := range checks { + if check.Status != "passing" { + failing[check.CheckID] = check + break + } + } + + if len(failing) == 0 { + return true + } + + t.Logf("still %d checks not passing", len(failing)) + return false + }, time.Minute, time.Second) + + require.Len(t, failing, 0, pretty.Sprint(failing)) +} From a44490182bf55043a6282586e8530e4864a47ba3 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Mon, 6 Jan 2020 21:53:45 -0500 Subject: [PATCH 3/4] tr: expose envoy sidecar admin port as environment variable --- client/allocrunner/taskrunner/envoybootstrap_hook.go | 7 ++++++- website/source/docs/runtime/_envvars.html.md.erb | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/client/allocrunner/taskrunner/envoybootstrap_hook.go b/client/allocrunner/taskrunner/envoybootstrap_hook.go index 137285315..11bb1d93f 100644 --- a/client/allocrunner/taskrunner/envoybootstrap_hook.go +++ b/client/allocrunner/taskrunner/envoybootstrap_hook.go @@ -13,12 +13,16 @@ import ( "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/allocrunner/interfaces" agentconsul "github.com/hashicorp/nomad/command/agent/consul" + "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/structs" ) var _ interfaces.TaskPrestartHook = &envoyBootstrapHook{} -const envoyBaseAdminPort = 19000 +const ( + envoyBaseAdminPort = 19000 + envoyAdminBindEnvPrefix = "NOMAD_ENVOY_ADMIN_ADDR_" +) // envoyBootstrapHook writes the bootstrap config for the Connect Envoy proxy // sidecar. @@ -82,6 +86,7 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskP // are running, the bind addresses need to have unique ports. // TODO: support running in host netns, using freeport to find available port envoyAdminBind := buildEnvoyAdminBind(h.alloc, req.Task.Name) + resp.Env[helper.CleanEnvVar(envoyAdminBindEnvPrefix+serviceName, '_')] = envoyAdminBind // Envoy bootstrap configuration may contain a Consul token, so write // it to the secrets directory like Vault tokens. diff --git a/website/source/docs/runtime/_envvars.html.md.erb b/website/source/docs/runtime/_envvars.html.md.erb index 54fd3f588..7378c5bde 100644 --- a/website/source/docs/runtime/_envvars.html.md.erb +++ b/website/source/docs/runtime/_envvars.html.md.erb @@ -161,4 +161,11 @@ Consul Connect upstream. + + NOMAD_ENVOY_ADMIN_ADDR_<service> + + Local address localhost:Port for the admin port of the envoy sidecar for the + given service when defined as a Consul Connect enabled service. + + From 55217423c79e1477c96481390877c435b59a961f Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Wed, 8 Jan 2020 13:41:38 -0500 Subject: [PATCH 4/4] tr: initialize envoybootstrap prestart hook response.Env field --- client/allocrunner/taskrunner/envoybootstrap_hook.go | 4 +++- client/allocrunner/taskrunner/envoybootstrap_hook_test.go | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/client/allocrunner/taskrunner/envoybootstrap_hook.go b/client/allocrunner/taskrunner/envoybootstrap_hook.go index 11bb1d93f..b24f17642 100644 --- a/client/allocrunner/taskrunner/envoybootstrap_hook.go +++ b/client/allocrunner/taskrunner/envoybootstrap_hook.go @@ -86,7 +86,9 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskP // are running, the bind addresses need to have unique ports. // TODO: support running in host netns, using freeport to find available port envoyAdminBind := buildEnvoyAdminBind(h.alloc, req.Task.Name) - resp.Env[helper.CleanEnvVar(envoyAdminBindEnvPrefix+serviceName, '_')] = envoyAdminBind + resp.Env = map[string]string{ + helper.CleanEnvVar(envoyAdminBindEnvPrefix+serviceName, '_'): envoyAdminBind, + } // Envoy bootstrap configuration may contain a Consul token, so write // it to the secrets directory like Vault tokens. diff --git a/client/allocrunner/taskrunner/envoybootstrap_hook_test.go b/client/allocrunner/taskrunner/envoybootstrap_hook_test.go index 1bbfaf1b9..94e1c0162 100644 --- a/client/allocrunner/taskrunner/envoybootstrap_hook_test.go +++ b/client/allocrunner/taskrunner/envoybootstrap_hook_test.go @@ -104,6 +104,9 @@ func TestTaskRunner_EnvoyBootstrapHook_Ok(t *testing.T) { // Assert it is Done require.True(t, resp.Done) + require.NotNil(t, resp.Env) + require.Equal(t, "localhost:19001", resp.Env[envoyAdminBindEnvPrefix+"foo"]) + // Ensure the default path matches env := map[string]string{ taskenv.SecretsDir: req.TaskDir.SecretsDir,