diff --git a/nomad/job_endpoint_hook_connect.go b/nomad/job_endpoint_hook_connect.go index 1ac2b6cb5..d429360df 100644 --- a/nomad/job_endpoint_hook_connect.go +++ b/nomad/job_endpoint_hook_connect.go @@ -263,11 +263,16 @@ func groupConnectHook(job *structs.Job, g *structs.TaskGroup) error { // inject the gateway task only if it does not yet already exist if !hasGatewayTaskForService(g, service.Name) { - // use the default envoy image, for now there is no support for a custom task task := newConnectGatewayTask(service.Name, netHost) g.Tasks = append(g.Tasks, task) + // the connect.sidecar_task stanza can also be used to configure + // a custom task to use as a gateway proxy + if service.Connect.SidecarTask != nil { + service.Connect.SidecarTask.MergeIntoTask(task) + } + task.Canonicalize(job, g) } } diff --git a/nomad/job_endpoint_hook_connect_test.go b/nomad/job_endpoint_hook_connect_test.go index 894c1b1ff..09c12f8c1 100644 --- a/nomad/job_endpoint_hook_connect_test.go +++ b/nomad/job_endpoint_hook_connect_test.go @@ -92,7 +92,7 @@ func TestJobEndpointConnect_groupConnectHook(t *testing.T) { tgExp.Services[0].Name = "backend" tgExp.Services[1].Name = "admin" - // Expect sidecar tasks to be properly canonicalized + // Expect sidecar tasks to be in canonical form. tgExp.Tasks[0].Canonicalize(job, tgExp) tgExp.Tasks[1].Canonicalize(job, tgExp) tgExp.Networks[0].DynamicPorts = []structs.Port{{ @@ -146,6 +146,75 @@ func TestJobEndpointConnect_groupConnectHook_IngressGateway(t *testing.T) { require.Exactly(t, expTG, job.TaskGroups[0]) } +func TestJobEndpointConnect_groupConnectHook_IngressGateway_CustomTask(t *testing.T) { + t.Parallel() + + // Test that the connect gateway task is inserted if a gateway service exists + // and since this is a bridge network, will rewrite the default gateway proxy + // block with correct configuration. + job := mock.ConnectIngressGatewayJob("bridge", false) + + job.Meta = map[string]string{ + "gateway_name": "my-gateway", + } + + job.TaskGroups[0].Services[0].Name = "${NOMAD_META_gateway_name}" + job.TaskGroups[0].Services[0].Connect.SidecarTask = &structs.SidecarTask{ + Driver: "raw_exec", + User: "sidecars", + Config: map[string]interface{}{ + "command": "/bin/sidecar", + "args": []string{"a", "b"}, + }, + Resources: &structs.Resources{ + CPU: 400, + // Memory: inherit 128 + }, + KillSignal: "SIGHUP", + } + + expTG := job.TaskGroups[0].Copy() + expTG.Tasks = []*structs.Task{ + // inject merged gateway task + { + Name: "connect-ingress-my-gateway", + Kind: structs.NewTaskKind(structs.ConnectIngressPrefix, "my-gateway"), + Driver: "raw_exec", + User: "sidecars", + Config: map[string]interface{}{ + "command": "/bin/sidecar", + "args": []string{"a", "b"}, + }, + Resources: &structs.Resources{ + CPU: 400, + MemoryMB: 128, + }, + LogConfig: &structs.LogConfig{ + MaxFiles: 2, + MaxFileSizeMB: 2, + }, + ShutdownDelay: 5 * time.Second, + KillSignal: "SIGHUP", + Constraints: structs.Constraints{ + connectGatewayVersionConstraint(), + }, + }, + } + expTG.Services[0].Name = "my-gateway" + expTG.Tasks[0].Canonicalize(job, expTG) + expTG.Networks[0].Canonicalize() + + // rewrite the service gateway proxy configuration + expTG.Services[0].Connect.Gateway.Proxy = gatewayProxyForBridge(expTG.Services[0].Connect.Gateway) + + require.NoError(t, groupConnectHook(job, job.TaskGroups[0])) + require.Exactly(t, expTG, job.TaskGroups[0]) + + // Test that the hook is idempotent + require.NoError(t, groupConnectHook(job, job.TaskGroups[0])) + require.Exactly(t, expTG, job.TaskGroups[0]) +} + // TestJobEndpoint_ConnectInterpolation asserts that when a Connect sidecar // proxy task is being created for a group service with an interpolated name, // the service name is interpolated *before the task is created. @@ -330,7 +399,7 @@ func TestJobEndpointConnect_gatewayProxyIsDefault(t *testing.T) { t.Run("bind-addresses set", func(t *testing.T) { result := gatewayProxyIsDefault(&structs.ConsulGatewayProxy{ EnvoyGatewayBindAddresses: map[string]*structs.ConsulGatewayBindAddress{ - "listener1": &structs.ConsulGatewayBindAddress{ + "listener1": { Address: "1.1.1.1", Port: 9000, }, @@ -362,7 +431,7 @@ func TestJobEndpointConnect_gatewayBindAddresses(t *testing.T) { }}, }) require.Equal(t, map[string]*structs.ConsulGatewayBindAddress{ - "service1": &structs.ConsulGatewayBindAddress{ + "service1": { Address: "0.0.0.0", Port: 3000, }, @@ -388,15 +457,15 @@ func TestJobEndpointConnect_gatewayBindAddresses(t *testing.T) { }}, }) require.Equal(t, map[string]*structs.ConsulGatewayBindAddress{ - "service1": &structs.ConsulGatewayBindAddress{ + "service1": { Address: "0.0.0.0", Port: 3000, }, - "service2": &structs.ConsulGatewayBindAddress{ + "service2": { Address: "0.0.0.0", Port: 3000, }, - "service3": &structs.ConsulGatewayBindAddress{ + "service3": { Address: "0.0.0.0", Port: 3001, }, diff --git a/website/pages/docs/job-specification/connect.mdx b/website/pages/docs/job-specification/connect.mdx index f0aced1db..0c91d27e1 100644 --- a/website/pages/docs/job-specification/connect.mdx +++ b/website/pages/docs/job-specification/connect.mdx @@ -47,20 +47,41 @@ job "countdash" { ## `connect` Parameters +Used to configure a connect service. Only one of `native`, `sidecar_service`, +or `gateway` may be realized per `connect` block. + - `native` - `(bool: false)` - This is used to configure the service as supporting - [Connect Native](https://www.consul.io/docs/connect/native) applications. If set, - the service definition must provide the name of the implementing task in the - [task][service_task] field. - Incompatible with `sidecar_service` and `sidecar_task`. + [Connect Native](https://www.consul.io/docs/connect/native) applications. -- `sidecar_service` - ([sidecar_service][]: nil) - This is used to configure the sidecar - service injected by Nomad for Consul Connect. Incompatible with `native`. +- `sidecar_service` - ([sidecar_service][]: nil) - This is used to + configure the sidecar service created by Nomad for Consul Connect. -- `sidecar_task` - ([sidecar_task][]:nil) - This modifies the configuration of the Envoy - proxy task. Incompatible with `native`. +- `sidecar_task` - ([sidecar_task][]:nil) - This modifies the + task configuration of the Envoy proxy created as a sidecar or gateway. + +- `gateway` - ([gateway][]:nil) - This is used to configure the + gateway service created by Nomad for Consul Connect. ## `connect` Examples +### Using Connect Native + +The following example is a minimal service stanza for a +[Consul Connect Native](https://www.consul.io/docs/connect/native) +application implemented by a task named `generate`. + +```hcl +service { + name = "uuid-api" + port = "${NOMAD_PORT_api}" + task = "generate" + + connect { + native = true + } +} +``` + ### Using Sidecar Service The following example is a minimal connect stanza with defaults and is @@ -169,35 +190,64 @@ job "countdash" { } ``` -### Using Connect Native +### Using a Gateway -The following example is a minimal service stanza for a -[Consul Connect Native](https://www.consul.io/docs/connect/native) -application implemented by a task named `generate`. +The following is an example service stanza for creating and using a connect ingress +gateway. It includes a gateway service definition and an api service fronted by +the gateway. Once running, the gateway can be used to reach the api service by first +looking up the gateway Consul DNS address, e.g. + +``` +curl $(dig +short @127.0.0.1 -p 8600 uuid-api.ingress.dc1.consul. ANY):8080 +``` ```hcl -service { - name = "uuid-api" - port = "${NOMAD_PORT_api}" - task = "generate" +job "ingress-demo" { - connect { - native = true + datacenters = ["dc1"] + + group "ingress-group" { + + network { + mode = "bridge" + port "inbound" { + static = 8080 + to = 8080 + } + } + + service { + name = "my-ingress-service" + port = "8080" + + connect { + gateway { + ingress { + listener { + port = 8080 + protocol = "tcp" + service { + name = "uuid-api" + } + } + } + } + } + } } } ``` ### Limitations -[Nomad variable interpolation][interpolation] is _not_ yet supported ([gh-7221]). - +[gateway]: /docs/job-specification/gateway [gh-7221]: https://github.com/hashicorp/nomad/issues/7221 -[job]: /docs/job-specification/job 'Nomad job Job Specification' [group]: /docs/job-specification/group 'Nomad group Job Specification' -[task]: /docs/job-specification/task 'Nomad task Job Specification' [interpolation]: /docs/runtime/interpolation 'Nomad interpolation' -[sidecar_service]: /docs/job-specification/sidecar_service 'Nomad sidecar service Specification' -[sidecar_task]: /docs/job-specification/sidecar_task 'Nomad sidecar task config Specification' -[upstreams]: /docs/job-specification/upstreams 'Nomad sidecar service upstreams Specification' +[job]: /docs/job-specification/job 'Nomad job Job Specification' [native]: https://www.consul.io/docs/connect/native [service_task]: /docs/job-specification/service#task-1 'Nomad service task' +[sidecar_service]: /docs/job-specification/sidecar_service 'Nomad sidecar service Specification' +[sidecar_task]: /docs/job-specification/sidecar_task 'Nomad sidecar task config Specification' +[task]: /docs/job-specification/task 'Nomad task Job Specification' +[upstreams]: /docs/job-specification/upstreams 'Nomad sidecar service upstreams Specification' diff --git a/website/pages/docs/job-specification/gateway.mdx b/website/pages/docs/job-specification/gateway.mdx index c109cc06d..14dd1720a 100644 --- a/website/pages/docs/job-specification/gateway.mdx +++ b/website/pages/docs/job-specification/gateway.mdx @@ -25,52 +25,60 @@ same network. For public ingress products like [NGINX](https://learn.hashicorp.c provide more suitable features. ```hcl -job "ingress-example" { +job "ingress-demo" { datacenters = ["dc1"] + # This group will have a task providing the ingress gateway automatically + # created by Nomad. The ingress gateway is based on the Envoy proxy being + # managed by the docker driver. group "ingress-group" { network { mode = "bridge" + # This example will enable plain HTTP traffic to access the uuid-api connect + # native example service on port 8080. port "inbound" { static = 8080 + to = 8080 } } service { - name = "ingress-service" + name = "my-ingress-service" port = "8080" connect { gateway { + + # Consul gateway [envoy] proxy options. proxy { - // Consul Gateway Proxy configuration options - connect_timeout = "500ms" + # The following options are automatically set by Nomad if not + # explicitly configured when using bridge networking. + # + # envoy_gateway_no_default_bind = true + # envoy_gateway_bind_addresses "uuid-api" { + # address = "0.0.0.0" + # port = + # } + # + # Additional options are documented at + # https://www.nomadproject.io/docs/job-specification/gateway#proxy-parameters } + # Consul Ingress Gateway Configuration Entry. ingress { - // Consul Ingress Gateway Configuration Entry - - tls { - enabled = false - } - + # Nomad will automatically manage the Configuration Entry in Consul + # given the parameters in the ingress block. + # + # Additional options are documented at + # https://www.nomadproject.io/docs/job-specification/gateway#ingress-parameters listener { port = 8080 - protocol = "http" - service { - name = "web" - hosts = ["example.com", "example.com:8080"] - } - } - - listener { - port = 3306 protocol = "tcp" service { - name = "database" + name = "uuid-api" } } } @@ -78,6 +86,41 @@ job "ingress-example" { } } } + + # The UUID generator from the connect-native demo is used as an example service. + # The ingress gateway above makes access to the service possible over normal HTTP. + # For example, + # + # $ curl $(dig +short @127.0.0.1 -p 8600 uuid-api.ingress.dc1.consul. ANY):8080 + group "generator" { + network { + mode = "host" + port "api" {} + } + + service { + name = "uuid-api" + port = "${NOMAD_PORT_api}" + + connect { + native = true + } + } + + task "generate" { + driver = "docker" + + config { + image = "hashicorpnomad/uuid-api:v3" + network_mode = "host" + } + + env { + BIND = "0.0.0.0" + PORT = "${NOMAD_PORT_api}" + } + } + } } ``` @@ -199,14 +242,31 @@ make use of the envoy version interpolation, e.g. meta.connect.gateway_image = custom/envoy-${NOMAD_envoy_version}:latest ``` +### Custom gateway task + +The task created for the gateway can be configured manually using the +[`sidecar_task`][sidecar_task] stanza. + +``` +connect { + gateway { + # ... + } + + sidecar_task { + # see /docs/job-specification/sidecar_task for more details + } +} +``` + [proxy]: /docs/job-specification/gateway#proxy-parameters [ingress]: /docs/job-specification/gateway#ingress-parameters [tls]: /docs/job-specification/gateway#tls-parameters [listener]: /docs/job-specification/gateway#listener-parameters [service]: /docs/job-specification/gateway#service-parameters [service-default]: https://www.consul.io/docs/agent/config-entries/service-defaults +[sidecar_task]: /docs/job-specification/sidecar_task [connect_timeout_ms]: https://www.consul.io/docs/agent/config-entries/service-resolver#connecttimeout [address]: /docs/job-specification/gateway#address-parameters [Advanced Configuration]: https://www.consul.io/docs/connect/proxies/envoy#advanced-configuration [Envoy Docker]: https://hub.docker.com/r/envoyproxy/envoy/tags - diff --git a/website/pages/docs/job-specification/sidecar_task.mdx b/website/pages/docs/job-specification/sidecar_task.mdx index 64c64ce0c..7d8986e42 100644 --- a/website/pages/docs/job-specification/sidecar_task.mdx +++ b/website/pages/docs/job-specification/sidecar_task.mdx @@ -12,7 +12,7 @@ description: |- The `sidecar_task` stanza allows configuring various options for the proxy -sidecar managed by Nomad for [Consul +sidecar or Connect gateway managed by Nomad for the [Consul Connect](/docs/integrations/consul-connect) integration such as resource requirements, kill timeouts and more as defined below. It is valid only within the context of a [`connect`][connect] stanza. @@ -53,25 +53,30 @@ job "countdash" { } ``` -## Default Envoy proxy sidecar +## Default Envoy configuration -Nomad automatically includes a default Envoy proxy sidecar task whenever a -group service has a [`sidecar_service`][sidecar_service] stanza. +Nomad automatically launches and manages an Envoy task for use as a proxy sidecar +or connect gateway, when [`sidecar_service`][sidecar_service] or [`gateway`][gateway] +are configured. -The default sidecar task is equivalent to: +The default Envoy task is equivalent to: ```hcl sidecar_task { name = "connect-proxy-" + # "connect-gateway-" when used as a gateway - lifecycle { + lifecycle { # absent when used as a gateway hook = "prestart" sidecar = true } driver = "docker" + config { image = "${meta.connect.sidecar_image}" + # "${meta.connect.gateway_image}" when used as a gateway + args = [ "-c", "${NOMAD_SECRETS_DIR}/envoy_bootstrap.json", @@ -97,13 +102,16 @@ sidecar_task { } ``` -The `meta.connect.sidecar_image`, `meta.connect.log_level`, and -`meta.connect.proxy_concurrency` variables are [_client_ -configurable][nodemeta] variables with the following defaults: +The `meta.connect.sidecar_image`, `meta.connect.gateway_image`, `meta.connect.log_level`, +and `meta.connect.proxy_concurrency` variables are [client configurable][nodemeta] +variables with the following defaults: - `sidecar_image` - `(string: "envoyproxy/envoy:v${NOMAD_envoy_version}")` - The official upstream Envoy Docker image, where `${NOMAD_envoy_version}` is resolved automatically by a query to Consul. +- `gateway_image` - `(string: "envoyproxy/envoy:v${NOMAD_envoy_version}")` - The official + upstream Envoy Docker image, where `${NOMAD_envoy_version}` is resolved automatically + by a query to Consul. - `log_level` - `(string: "info")` - Envoy sidecar log level. "`debug`" is useful for debugging Connect related issues. - `proxy_concurrency` - `(string: "1")` - The number of [worker threads][worker_threads] the Envoy @@ -118,8 +126,8 @@ meta.connect.sidecar_image = custom/envoy-${NOMAD_envoy_version}:latest ## `sidecar_task` Parameters -- `name` `(string: "connect-proxy-")` - Name of the task. Defaults to - including the name of the service it is a proxy for. +- `name` `(string: "connect-[proxy|gateway]-")` - Name of the task. Defaults to + including the name of the service the proxy or gateway is providing. - `driver` `(string: "docker")` - Driver used for the sidecar task. @@ -166,12 +174,13 @@ The following example configures resources for the sidecar task and other config ``` [connect]: /docs/job-specification/connect 'Nomad connect Job Specification' -[job]: /docs/job-specification/job 'Nomad job Job Specification' +[gateway]: /docs/job-specification/gateway [group]: /docs/job-specification/group 'Nomad group Job Specification' -[task]: /docs/job-specification/task 'Nomad task Job Specification' [interpolation]: /docs/runtime/interpolation 'Nomad interpolation' -[sidecar_service]: /docs/job-specification/sidecar_service 'Nomad sidecar service Specification' -[resources]: /docs/job-specification/resources 'Nomad resources Job Specification' +[job]: /docs/job-specification/job 'Nomad job Job Specification' [logs]: /docs/job-specification/logs 'Nomad logs Job Specification' +[resources]: /docs/job-specification/resources 'Nomad resources Job Specification' +[sidecar_service]: /docs/job-specification/sidecar_service 'Nomad sidecar service Specification' +[task]: /docs/job-specification/task 'Nomad task Job Specification' [nodemeta]: /docs/configuration/client#meta [worker_threads]: https://www.envoyproxy.io/docs/envoy/latest/operations/cli#cmdoption-concurrency