Merge pull request #9639 from hashicorp/f-gateway-task

consul/connect: enable configuring custom gateway task
2026-01-07 19:05:42 +03:00 · 2020-12-17 11:10:16 -06:00
parent cee67869b9 ac24722ead
commit dde3780194
5 changed files with 261 additions and 68 deletions
--- a/nomad/job_endpoint_hook_connect.go
+++ b/nomad/job_endpoint_hook_connect.go
@@ -263,11 +263,16 @@ func groupConnectHook(job *structs.Job, g *structs.TaskGroup) error {

 			// inject the gateway task only if it does not yet already exist
 			if !hasGatewayTaskForService(g, service.Name) {
-				// use the default envoy image, for now there is no support for a custom task
 				task := newConnectGatewayTask(service.Name, netHost)

 				g.Tasks = append(g.Tasks, task)

+				// the connect.sidecar_task stanza can also be used to configure
+				// a custom task to use as a gateway proxy
+				if service.Connect.SidecarTask != nil {
+					service.Connect.SidecarTask.MergeIntoTask(task)
+				}
+
 				task.Canonicalize(job, g)
 			}
 		}
--- a/nomad/job_endpoint_hook_connect_test.go
+++ b/nomad/job_endpoint_hook_connect_test.go
@@ -92,7 +92,7 @@ func TestJobEndpointConnect_groupConnectHook(t *testing.T) {
 	tgExp.Services[0].Name = "backend"
 	tgExp.Services[1].Name = "admin"

-	// Expect sidecar tasks to be properly canonicalized
+	// Expect sidecar tasks to be in canonical form.
 	tgExp.Tasks[0].Canonicalize(job, tgExp)
 	tgExp.Tasks[1].Canonicalize(job, tgExp)
 	tgExp.Networks[0].DynamicPorts = []structs.Port{{
@@ -146,6 +146,75 @@ func TestJobEndpointConnect_groupConnectHook_IngressGateway(t *testing.T) {
 	require.Exactly(t, expTG, job.TaskGroups[0])
 }

+func TestJobEndpointConnect_groupConnectHook_IngressGateway_CustomTask(t *testing.T) {
+	t.Parallel()
+
+	// Test that the connect gateway task is inserted if a gateway service exists
+	// and since this is a bridge network, will rewrite the default gateway proxy
+	// block with correct configuration.
+	job := mock.ConnectIngressGatewayJob("bridge", false)
+
+	job.Meta = map[string]string{
+		"gateway_name": "my-gateway",
+	}
+
+	job.TaskGroups[0].Services[0].Name = "${NOMAD_META_gateway_name}"
+	job.TaskGroups[0].Services[0].Connect.SidecarTask = &structs.SidecarTask{
+		Driver: "raw_exec",
+		User:   "sidecars",
+		Config: map[string]interface{}{
+			"command": "/bin/sidecar",
+			"args":    []string{"a", "b"},
+		},
+		Resources: &structs.Resources{
+			CPU: 400,
+			// Memory: inherit 128
+		},
+		KillSignal: "SIGHUP",
+	}
+
+	expTG := job.TaskGroups[0].Copy()
+	expTG.Tasks = []*structs.Task{
+		// inject merged gateway task
+		{
+			Name:   "connect-ingress-my-gateway",
+			Kind:   structs.NewTaskKind(structs.ConnectIngressPrefix, "my-gateway"),
+			Driver: "raw_exec",
+			User:   "sidecars",
+			Config: map[string]interface{}{
+				"command": "/bin/sidecar",
+				"args":    []string{"a", "b"},
+			},
+			Resources: &structs.Resources{
+				CPU:      400,
+				MemoryMB: 128,
+			},
+			LogConfig: &structs.LogConfig{
+				MaxFiles:      2,
+				MaxFileSizeMB: 2,
+			},
+			ShutdownDelay: 5 * time.Second,
+			KillSignal:    "SIGHUP",
+			Constraints: structs.Constraints{
+				connectGatewayVersionConstraint(),
+			},
+		},
+	}
+	expTG.Services[0].Name = "my-gateway"
+	expTG.Tasks[0].Canonicalize(job, expTG)
+	expTG.Networks[0].Canonicalize()
+
+	// rewrite the service gateway proxy configuration
+	expTG.Services[0].Connect.Gateway.Proxy = gatewayProxyForBridge(expTG.Services[0].Connect.Gateway)
+
+	require.NoError(t, groupConnectHook(job, job.TaskGroups[0]))
+	require.Exactly(t, expTG, job.TaskGroups[0])
+
+	// Test that the hook is idempotent
+	require.NoError(t, groupConnectHook(job, job.TaskGroups[0]))
+	require.Exactly(t, expTG, job.TaskGroups[0])
+}
+
 // TestJobEndpoint_ConnectInterpolation asserts that when a Connect sidecar
 // proxy task is being created for a group service with an interpolated name,
 // the service name is interpolated *before the task is created.
@@ -330,7 +399,7 @@ func TestJobEndpointConnect_gatewayProxyIsDefault(t *testing.T) {
 	t.Run("bind-addresses set", func(t *testing.T) {
 		result := gatewayProxyIsDefault(&structs.ConsulGatewayProxy{
 			EnvoyGatewayBindAddresses: map[string]*structs.ConsulGatewayBindAddress{
-				"listener1": &structs.ConsulGatewayBindAddress{
+				"listener1": {
 					Address: "1.1.1.1",
 					Port:    9000,
 				},
@@ -362,7 +431,7 @@ func TestJobEndpointConnect_gatewayBindAddresses(t *testing.T) {
 			}},
 		})
 		require.Equal(t, map[string]*structs.ConsulGatewayBindAddress{
-			"service1": &structs.ConsulGatewayBindAddress{
+			"service1": {
 				Address: "0.0.0.0",
 				Port:    3000,
 			},
@@ -388,15 +457,15 @@ func TestJobEndpointConnect_gatewayBindAddresses(t *testing.T) {
 			}},
 		})
 		require.Equal(t, map[string]*structs.ConsulGatewayBindAddress{
-			"service1": &structs.ConsulGatewayBindAddress{
+			"service1": {
 				Address: "0.0.0.0",
 				Port:    3000,
 			},
-			"service2": &structs.ConsulGatewayBindAddress{
+			"service2": {
 				Address: "0.0.0.0",
 				Port:    3000,
 			},
-			"service3": &structs.ConsulGatewayBindAddress{
+			"service3": {
 				Address: "0.0.0.0",
 				Port:    3001,
 			},
--- a/website/pages/docs/job-specification/connect.mdx
+++ b/website/pages/docs/job-specification/connect.mdx
@@ -47,20 +47,41 @@ job "countdash" {

 ## `connect` Parameters

+Used to configure a connect service. Only one of `native`, `sidecar_service`,
+or `gateway` may be realized per `connect` block.
+
 - `native` - `(bool: false)` - This is used to configure the service as supporting
-  [Connect Native](https://www.consul.io/docs/connect/native) applications. If set,
-  the service definition must provide the name of the implementing task in the
-  [task][service_task] field.
-  Incompatible with `sidecar_service` and `sidecar_task`.
+  [Connect Native](https://www.consul.io/docs/connect/native) applications.

- `sidecar_service` - <code>([sidecar_service][]: nil)</code> - This is used to configure the sidecar
-  service injected by Nomad for Consul Connect. Incompatible with `native`.
+- `sidecar_service` - <code>([sidecar_service][]: nil)</code> - This is used to
+  configure the sidecar service created by Nomad for Consul Connect.

- `sidecar_task` - <code>([sidecar_task][]:nil)</code> - This modifies the configuration of the Envoy
-  proxy task. Incompatible with `native`.
+- `sidecar_task` - <code>([sidecar_task][]:nil)</code> - This modifies the
+  task configuration of the Envoy proxy created as a sidecar or gateway.
+
+- `gateway` - <code>([gateway][]:nil)</code> - This is used to configure the
+  gateway service created by Nomad for Consul Connect.

 ## `connect` Examples

+### Using Connect Native
+
+The following example is a minimal service stanza for a
+[Consul Connect Native](https://www.consul.io/docs/connect/native)
+application implemented by a task named `generate`.
+
+```hcl
+service {
+  name = "uuid-api"
+  port = "${NOMAD_PORT_api}"
+  task = "generate"
+
+  connect {
+    native = true
+  }
+}
+```
+
 ### Using Sidecar Service

 The following example is a minimal connect stanza with defaults and is
@@ -169,35 +190,64 @@ job "countdash" {
 }
 ```

-### Using Connect Native
+### Using a Gateway

-The following example is a minimal service stanza for a
-[Consul Connect Native](https://www.consul.io/docs/connect/native)
-application implemented by a task named `generate`.
+The following is an example service stanza for creating and using a connect ingress
+gateway. It includes a gateway service definition and an api service fronted by
+the gateway. Once running, the gateway can be used to reach the api service by first
+looking up the gateway Consul DNS address, e.g.
+
+```
+curl $(dig +short @127.0.0.1 -p 8600 uuid-api.ingress.dc1.consul. ANY):8080
+```

 ```hcl
-service {
-  name = "uuid-api"
-  port = "${NOMAD_PORT_api}"
-  task = "generate"
+job "ingress-demo" {

-  connect {
-    native = true
+  datacenters = ["dc1"]
+
+  group "ingress-group" {
+
+    network {
+      mode = "bridge"
+      port "inbound" {
+        static = 8080
+        to     = 8080
+      }
+    }
+
+    service {
+      name = "my-ingress-service"
+      port = "8080"
+
+      connect {
+        gateway {
+          ingress {
+            listener {
+              port     = 8080
+              protocol = "tcp"
+              service {
+                name = "uuid-api"
+              }
+            }
+          }
+        }
+      }
+    }
  }
 }
 ```

 ### Limitations

-[Nomad variable interpolation][interpolation] is _not_ yet supported ([gh-7221]).
-
+[gateway]: /docs/job-specification/gateway
 [gh-7221]: https://github.com/hashicorp/nomad/issues/7221
-[job]: /docs/job-specification/job 'Nomad job Job Specification'
 [group]: /docs/job-specification/group 'Nomad group Job Specification'
-[task]: /docs/job-specification/task 'Nomad task Job Specification'
 [interpolation]: /docs/runtime/interpolation 'Nomad interpolation'
-[sidecar_service]: /docs/job-specification/sidecar_service 'Nomad sidecar service Specification'
-[sidecar_task]: /docs/job-specification/sidecar_task 'Nomad sidecar task config Specification'
-[upstreams]: /docs/job-specification/upstreams 'Nomad sidecar service upstreams Specification'
+[job]: /docs/job-specification/job 'Nomad job Job Specification'
 [native]: https://www.consul.io/docs/connect/native
 [service_task]: /docs/job-specification/service#task-1 'Nomad service task'
+[sidecar_service]: /docs/job-specification/sidecar_service 'Nomad sidecar service Specification'
+[sidecar_task]: /docs/job-specification/sidecar_task 'Nomad sidecar task config Specification'
+[task]: /docs/job-specification/task 'Nomad task Job Specification'
+[upstreams]: /docs/job-specification/upstreams 'Nomad sidecar service upstreams Specification'
--- a/website/pages/docs/job-specification/gateway.mdx
+++ b/website/pages/docs/job-specification/gateway.mdx
@@ -25,52 +25,60 @@ same network. For public ingress products like [NGINX](https://learn.hashicorp.c
 provide more suitable features.

 ```hcl
-job "ingress-example" {
+job "ingress-demo" {

  datacenters = ["dc1"]

+  # This group will have a task providing the ingress gateway automatically
+  # created by Nomad. The ingress gateway is based on the Envoy proxy being
+  # managed by the docker driver.
  group "ingress-group" {

    network {
      mode = "bridge"

+      # This example will enable plain HTTP traffic to access the uuid-api connect
+      # native example service on port 8080.
      port "inbound" {
        static = 8080
+        to     = 8080
      }
    }

    service {
-      name = "ingress-service"
+      name = "my-ingress-service"
      port = "8080"

      connect {
        gateway {
+
+          # Consul gateway [envoy] proxy options.
          proxy {
-            // Consul Gateway Proxy configuration options
-            connect_timeout = "500ms"
+            # The following options are automatically set by Nomad if not
+            # explicitly configured when using bridge networking.
+            #
+            # envoy_gateway_no_default_bind = true
+            # envoy_gateway_bind_addresses "uuid-api" {
+            #   address = "0.0.0.0"
+            #   port    = <associated listener.port>
+            # }
+            #
+            # Additional options are documented at
+            # https://www.nomadproject.io/docs/job-specification/gateway#proxy-parameters
          }

+          # Consul Ingress Gateway Configuration Entry.
          ingress {
-            // Consul Ingress Gateway Configuration Entry
-
-            tls {
-              enabled = false
-            }
-
+            # Nomad will automatically manage the Configuration Entry in Consul
+            # given the parameters in the ingress block.
+            #
+            # Additional options are documented at
+            # https://www.nomadproject.io/docs/job-specification/gateway#ingress-parameters
            listener {
              port     = 8080
-              protocol = "http"
-              service {
-                name  = "web"
-                hosts = ["example.com", "example.com:8080"]
-              }
-            }
-
-            listener {
-              port = 3306
              protocol = "tcp"
              service {
-                name = "database"
+                name = "uuid-api"
              }
            }
          }
@@ -78,6 +86,41 @@ job "ingress-example" {
      }
    }
  }
+
+  # The UUID generator from the connect-native demo is used as an example service.
+  # The ingress gateway above makes access to the service possible over normal HTTP.
+  # For example,
+  #
+  # $ curl $(dig +short @127.0.0.1 -p 8600 uuid-api.ingress.dc1.consul. ANY):8080
+  group "generator" {
+    network {
+      mode = "host"
+      port "api" {}
+    }
+
+    service {
+      name = "uuid-api"
+      port = "${NOMAD_PORT_api}"
+
+      connect {
+        native = true
+      }
+    }
+
+    task "generate" {
+      driver = "docker"
+
+      config {
+        image        = "hashicorpnomad/uuid-api:v3"
+        network_mode = "host"
+      }
+
+      env {
+        BIND = "0.0.0.0"
+        PORT = "${NOMAD_PORT_api}"
+      }
+    }
+  }
 }
 ```

@@ -199,14 +242,31 @@ make use of the envoy version interpolation, e.g.
 meta.connect.gateway_image = custom/envoy-${NOMAD_envoy_version}:latest
 ```

+### Custom gateway task
+
+The task created for the gateway can be configured manually using the
+[`sidecar_task`][sidecar_task] stanza.
+
+```
+connect {
+  gateway {
+    # ...
+  }
+
+  sidecar_task {
+    # see /docs/job-specification/sidecar_task for more details
+  }
+}
+```
+
 [proxy]: /docs/job-specification/gateway#proxy-parameters
 [ingress]: /docs/job-specification/gateway#ingress-parameters
 [tls]: /docs/job-specification/gateway#tls-parameters
 [listener]: /docs/job-specification/gateway#listener-parameters
 [service]: /docs/job-specification/gateway#service-parameters
 [service-default]: https://www.consul.io/docs/agent/config-entries/service-defaults
+[sidecar_task]: /docs/job-specification/sidecar_task
 [connect_timeout_ms]: https://www.consul.io/docs/agent/config-entries/service-resolver#connecttimeout
 [address]: /docs/job-specification/gateway#address-parameters
 [Advanced Configuration]: https://www.consul.io/docs/connect/proxies/envoy#advanced-configuration
 [Envoy Docker]: https://hub.docker.com/r/envoyproxy/envoy/tags
-
--- a/website/pages/docs/job-specification/sidecar_task.mdx
+++ b/website/pages/docs/job-specification/sidecar_task.mdx
@@ -12,7 +12,7 @@ description: |-
 <Placement groups={['job', 'group', 'service', 'connect', 'sidecar_task']} />

 The `sidecar_task` stanza allows configuring various options for the proxy
-sidecar managed by Nomad for [Consul
+sidecar or Connect gateway managed by Nomad for the [Consul
 Connect](/docs/integrations/consul-connect) integration such as
 resource requirements, kill timeouts and more as defined below. It is valid
 only within the context of a [`connect`][connect] stanza.
@@ -53,25 +53,30 @@ job "countdash" {
 }
 ```

-## Default Envoy proxy sidecar
+## Default Envoy configuration

-Nomad automatically includes a default Envoy proxy sidecar task whenever a
-group service has a [`sidecar_service`][sidecar_service] stanza.
+Nomad automatically launches and manages an Envoy task for use as a proxy sidecar
+or connect gateway, when [`sidecar_service`][sidecar_service] or [`gateway`][gateway]
+are configured.

-The default sidecar task is equivalent to:
+The default Envoy task is equivalent to:

 ```hcl
 sidecar_task {
  name = "connect-proxy-<service>"
+  #      "connect-gateway-<service>" when used as a gateway

-  lifecycle {
+  lifecycle { # absent when used as a gateway
    hook    = "prestart"
    sidecar = true
  }

  driver = "docker"
+
  config {
    image = "${meta.connect.sidecar_image}"
+    #       "${meta.connect.gateway_image}" when used as a gateway
+
    args = [
      "-c",
      "${NOMAD_SECRETS_DIR}/envoy_bootstrap.json",
@@ -97,13 +102,16 @@ sidecar_task {
 }
 ```

-The `meta.connect.sidecar_image`, `meta.connect.log_level`, and
-`meta.connect.proxy_concurrency` variables are [_client_
-configurable][nodemeta] variables with the following defaults:
+The `meta.connect.sidecar_image`, `meta.connect.gateway_image`, `meta.connect.log_level`,
+and `meta.connect.proxy_concurrency` variables are [client configurable][nodemeta]
+variables with the following defaults:

 - `sidecar_image` - `(string: "envoyproxy/envoy:v${NOMAD_envoy_version}")` - The official
   upstream Envoy Docker image, where `${NOMAD_envoy_version}` is resolved automatically
   by a query to Consul.
+- `gateway_image` - `(string: "envoyproxy/envoy:v${NOMAD_envoy_version}")` - The official
+   upstream Envoy Docker image, where `${NOMAD_envoy_version}` is resolved automatically
+   by a query to Consul.
 - `log_level` - `(string: "info")` - Envoy sidecar log level. "`debug`" is useful for
  debugging Connect related issues.
 - `proxy_concurrency` - `(string: "1")` - The number of [worker threads][worker_threads] the Envoy
@@ -118,8 +126,8 @@ meta.connect.sidecar_image = custom/envoy-${NOMAD_envoy_version}:latest

 ## `sidecar_task` Parameters

- `name` `(string: "connect-proxy-<service>")` - Name of the task. Defaults to
-  including the name of the service it is a proxy for.
+- `name` `(string: "connect-[proxy|gateway]-<service>")` - Name of the task. Defaults to
+  including the name of the service the proxy or gateway is providing.

 - `driver` `(string: "docker")` - Driver used for the sidecar task.

@@ -166,12 +174,13 @@ The following example configures resources for the sidecar task and other config
 ```

 [connect]: /docs/job-specification/connect 'Nomad connect Job Specification'
-[job]: /docs/job-specification/job 'Nomad job Job Specification'
+[gateway]: /docs/job-specification/gateway
 [group]: /docs/job-specification/group 'Nomad group Job Specification'
-[task]: /docs/job-specification/task 'Nomad task Job Specification'
 [interpolation]: /docs/runtime/interpolation 'Nomad interpolation'
-[sidecar_service]: /docs/job-specification/sidecar_service 'Nomad sidecar service Specification'
-[resources]: /docs/job-specification/resources 'Nomad resources Job Specification'
+[job]: /docs/job-specification/job 'Nomad job Job Specification'
 [logs]: /docs/job-specification/logs 'Nomad logs Job Specification'
+[resources]: /docs/job-specification/resources 'Nomad resources Job Specification'
+[sidecar_service]: /docs/job-specification/sidecar_service 'Nomad sidecar service Specification'
+[task]: /docs/job-specification/task 'Nomad task Job Specification'
 [nodemeta]: /docs/configuration/client#meta
 [worker_threads]: https://www.envoyproxy.io/docs/envoy/latest/operations/cli#cmdoption-concurrency