Merge pull request #6816 from hashicorp/b-multiple-envoy

connect: configure envoy to support multiple sidecars in the same alloc

client/allocrunner/taskrunner/envoybootstrap_hook.go

@@ -13,11 +13,17 @@ import (
 	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
 	agentconsul "github.com/hashicorp/nomad/command/agent/consul"
+	"github.com/hashicorp/nomad/helper"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
 var _ interfaces.TaskPrestartHook = &envoyBootstrapHook{}
 
+const (
+	envoyBaseAdminPort      = 19000
+	envoyAdminBindEnvPrefix = "NOMAD_ENVOY_ADMIN_ADDR_"
+)
+
 // envoyBootstrapHook writes the bootstrap config for the Connect Envoy proxy
 // sidecar.
 type envoyBootstrapHook struct {
@@ -76,12 +82,20 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskP
 	//     the host netns.
 	grpcAddr := "unix://" + allocdir.AllocGRPCSocket
 
+	// Envoy runs an administrative API on the loopback interface. If multiple sidecars
+	// are running, the bind addresses need to have unique ports.
+	// TODO: support running in host netns, using freeport to find available port
+	envoyAdminBind := buildEnvoyAdminBind(h.alloc, req.Task.Name)
+	resp.Env = map[string]string{
+		helper.CleanEnvVar(envoyAdminBindEnvPrefix+serviceName, '_'): envoyAdminBind,
+	}
+
 	// Envoy bootstrap configuration may contain a Consul token, so write
 	// it to the secrets directory like Vault tokens.
 	fn := filepath.Join(req.TaskDir.SecretsDir, "envoy_bootstrap.json")
 
 	id := agentconsul.MakeAllocServiceID(h.alloc.ID, "group-"+tg.Name, service)
-	h.logger.Debug("bootstrapping envoy", "sidecar_for", service.Name, "boostrap_file", fn, "sidecar_for_id", id, "grpc_addr", grpcAddr)
+	h.logger.Debug("bootstrapping envoy", "sidecar_for", service.Name, "boostrap_file", fn, "sidecar_for_id", id, "grpc_addr", grpcAddr, "admin_bind", envoyAdminBind)
 
 	// Since Consul services are registered asynchronously with this task
 	// hook running, retry a small number of times with backoff.
@@ -89,6 +103,7 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskP
 		cmd := exec.CommandContext(ctx, "consul", "connect", "envoy",
 			"-grpc-addr", grpcAddr,
 			"-http-addr", h.consulHTTPAddr,
+			"-admin-bind", envoyAdminBind,
 			"-bootstrap",
 			"-sidecar-for", id,
 		)
@@ -148,3 +163,14 @@ func (h *envoyBootstrapHook) Prestart(ctx context.Context, req *interfaces.TaskP
 	resp.Done = true
 	return nil
 }
+
+func buildEnvoyAdminBind(alloc *structs.Allocation, taskName string) string {
+	port := envoyBaseAdminPort
+	for idx, task := range alloc.Job.LookupTaskGroup(alloc.TaskGroup).Tasks {
+		if task.Name == taskName {
+			port += idx
+			break
+		}
+	}
+	return fmt.Sprintf("localhost:%d", port)
+}

client/allocrunner/taskrunner/envoybootstrap_hook_test.go

@@ -104,6 +104,9 @@ func TestTaskRunner_EnvoyBootstrapHook_Ok(t *testing.T) {
 	// Assert it is Done
 	require.True(t, resp.Done)
 
+	require.NotNil(t, resp.Env)
+	require.Equal(t, "localhost:19001", resp.Env[envoyAdminBindEnvPrefix+"foo"])
+
 	// Ensure the default path matches
 	env := map[string]string{
 		taskenv.SecretsDir: req.TaskDir.SecretsDir,