connect: enable proxy.passthrough configuration

Enable configuration of HTTP and gRPC endpoints which should be exposed by
the Connect sidecar proxy. This changeset is the first "non-magical" pass
that lays the groundwork for enabling Consul service checks for tasks
running in a network namespace because they are Connect-enabled. The changes
here provide for full configuration of the

  connect {
    sidecar_service {
      proxy {
        expose {
          paths = [{
		path = <exposed endpoint>
                protocol = <http or grpc>
                local_path_port = <local endpoint port>
                listener_port = <inbound mesh port>
	  }, ... ]
       }
    }
  }

stanza. Everything from `expose` and below is new, and partially implements
the precedent set by Consul:
  https://www.consul.io/docs/connect/registration/service-registration.html#expose-paths-configuration-reference

Combined with a task-group level network port-mapping in the form:

  port "exposeExample" { to = -1 }

it is now possible to "punch a hole" through the network namespace
to a specific HTTP or gRPC path, with the anticipated use case of creating
Consul checks on Connect enabled services.

A future PR may introduce more automagic behavior, where we can do things like

1) auto-fill the 'expose.path.local_path_port' with the default value of the
   'service.port' value for task-group level connect-enabled services.

2) automatically generate a port-mapping

3) enable an 'expose.checks' flag which automatically creates exposed endpoints
   for every compatible consul service check (http/grpc checks on connect
   enabled services).

command/agent/agent.go

@@ -587,7 +587,7 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) {
 	conf.ACLTokenTTL = agentConfig.ACL.TokenTTL
 	conf.ACLPolicyTTL = agentConfig.ACL.PolicyTTL
 
-	// Setup networking configration
+	// Setup networking configuration
 	conf.CNIPath = agentConfig.Client.CNIPath
 	conf.BridgeNetworkName = agentConfig.Client.BridgeNetworkName
 	conf.BridgeNetworkAllocSubnet = agentConfig.Client.BridgeNetworkSubnet

command/agent/consul/client.go

@@ -1280,7 +1280,7 @@ func MakeCheckID(serviceID string, check *structs.ServiceCheck) string {
 // createCheckReg creates a Check that can be registered with Consul.
 //
 // Script checks simply have a TTL set and the caller is responsible for
-// running the script and heartbeating.
+// running the script and heart-beating.
 func createCheckReg(serviceID, checkID string, check *structs.ServiceCheck, host string, port int) (*api.AgentCheckRegistration, error) {
 	chkReg := api.AgentCheckRegistration{
 		ID:        checkID,
@@ -1313,8 +1313,8 @@ func createCheckReg(serviceID, checkID string, check *structs.ServiceCheck, host
 		if err != nil {
 			return nil, err
 		}
-		url := base.ResolveReference(relative)
-		chkReg.HTTP = url.String()
+		checkURL := base.ResolveReference(relative)
+		chkReg.HTTP = checkURL.String()
 		chkReg.Method = check.Method
 		chkReg.Header = check.Header
 
@@ -1471,90 +1471,3 @@ func getAddress(addrMode, portLabel string, networks structs.Networks, driverNet
 		return "", 0, fmt.Errorf("invalid address mode %q", addrMode)
 	}
 }
-
-// newConnect creates a new Consul AgentServiceConnect struct based on a Nomad
-// Connect struct. If the nomad Connect struct is nil, nil will be returned to
-// disable Connect for this service.
-func newConnect(serviceName string, nc *structs.ConsulConnect, networks structs.Networks) (*api.AgentServiceConnect, error) {
-	if nc == nil {
-		// No Connect stanza, returning nil is fine
-		return nil, nil
-	}
-
-	cc := &api.AgentServiceConnect{
-		Native: nc.Native,
-	}
-
-	if nc.SidecarService == nil {
-		return cc, nil
-	}
-
-	net, port, err := getConnectPort(serviceName, networks)
-	if err != nil {
-		return nil, err
-	}
-
-	// Bind to netns IP(s):port
-	proxyConfig := map[string]interface{}{}
-	localServiceAddress := ""
-	localServicePort := 0
-	if nc.SidecarService.Proxy != nil {
-		localServiceAddress = nc.SidecarService.Proxy.LocalServiceAddress
-		localServicePort = nc.SidecarService.Proxy.LocalServicePort
-		if nc.SidecarService.Proxy.Config != nil {
-			proxyConfig = nc.SidecarService.Proxy.Config
-		}
-	}
-	proxyConfig["bind_address"] = "0.0.0.0"
-	proxyConfig["bind_port"] = port.To
-
-	// Advertise host IP:port
-	cc.SidecarService = &api.AgentServiceRegistration{
-		Tags:    helper.CopySliceString(nc.SidecarService.Tags),
-		Address: net.IP,
-		Port:    port.Value,
-
-		// Automatically configure the proxy to bind to all addresses
-		// within the netns.
-		Proxy: &api.AgentServiceConnectProxyConfig{
-			LocalServiceAddress: localServiceAddress,
-			LocalServicePort:    localServicePort,
-			Config:              proxyConfig,
-		},
-	}
-
-	// If no further proxy settings were explicitly configured, exit early
-	if nc.SidecarService.Proxy == nil {
-		return cc, nil
-	}
-
-	numUpstreams := len(nc.SidecarService.Proxy.Upstreams)
-	if numUpstreams == 0 {
-		return cc, nil
-	}
-
-	upstreams := make([]api.Upstream, numUpstreams)
-	for i, nu := range nc.SidecarService.Proxy.Upstreams {
-		upstreams[i].DestinationName = nu.DestinationName
-		upstreams[i].LocalBindPort = nu.LocalBindPort
-	}
-	cc.SidecarService.Proxy.Upstreams = upstreams
-
-	return cc, nil
-}
-
-// getConnectPort returns the network and port for the Connect proxy sidecar
-// defined for this service. An error is returned if the network and port
-// cannot be determined.
-func getConnectPort(serviceName string, networks structs.Networks) (*structs.NetworkResource, structs.Port, error) {
-	if n := len(networks); n != 1 {
-		return nil, structs.Port{}, fmt.Errorf("Connect only supported with exactly 1 network (found %d)", n)
-	}
-
-	port, ok := networks[0].PortForService(serviceName)
-	if !ok {
-		return nil, structs.Port{}, fmt.Errorf("No Connect port defined for service %q", serviceName)
-	}
-
-	return networks[0], port, nil
-}

command/agent/consul/connect.go

@@ -0,0 +1,176 @@
+package consul
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/hashicorp/nomad/helper"
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+// newConnect creates a new Consul AgentServiceConnect struct based on a Nomad
+// Connect struct. If the nomad Connect struct is nil, nil will be returned to
+// disable Connect for this service.
+func newConnect(serviceName string, nc *structs.ConsulConnect, networks structs.Networks) (*api.AgentServiceConnect, error) {
+	if nc == nil {
+		// no connect stanza means there is no connect service to register
+		return nil, nil
+	}
+
+	if nc.Native {
+		return &api.AgentServiceConnect{Native: true}, nil
+	}
+
+	sidecarReg, err := connectSidecarRegistration(serviceName, nc.SidecarService, networks)
+	if err != nil {
+		return nil, err
+	}
+
+	return &api.AgentServiceConnect{
+		Native:         false,
+		SidecarService: sidecarReg,
+	}, nil
+}
+
+func connectSidecarRegistration(serviceName string, css *structs.ConsulSidecarService, networks structs.Networks) (*api.AgentServiceRegistration, error) {
+	if css == nil {
+		// no sidecar stanza means there is no sidecar service to register
+		return nil, nil
+	}
+
+	cNet, cPort, err := connectPort(serviceName, networks)
+	if err != nil {
+		return nil, err
+	}
+
+	proxy, err := connectProxy(css.Proxy, cPort.To, networks)
+	if err != nil {
+		return nil, err
+	}
+
+	return &api.AgentServiceRegistration{
+		Tags:    helper.CopySliceString(css.Tags),
+		Port:    cPort.Value,
+		Address: cNet.IP,
+		Proxy:   proxy,
+	}, nil
+}
+
+func connectProxy(proxy *structs.ConsulProxy, cPort int, networks structs.Networks) (*api.AgentServiceConnectProxyConfig, error) {
+	if proxy == nil {
+		return nil, nil
+	}
+
+	expose, err := connectProxyExpose(proxy.Expose, networks)
+	if err != nil {
+		return nil, err
+	}
+
+	return &api.AgentServiceConnectProxyConfig{
+		LocalServiceAddress: proxy.LocalServiceAddress,
+		LocalServicePort:    proxy.LocalServicePort,
+		Config:              connectProxyConfig(proxy.Config, cPort),
+		Upstreams:           connectUpstreams(proxy.Upstreams),
+		Expose:              expose,
+	}, nil
+}
+
+func connectProxyExpose(expose *structs.ConsulExposeConfig, networks structs.Networks) (api.ExposeConfig, error) {
+	if expose == nil {
+		return api.ExposeConfig{}, nil
+	}
+
+	paths, err := connectProxyExposePaths(expose.Paths, networks)
+	if err != nil {
+		return api.ExposeConfig{}, err
+	}
+
+	return api.ExposeConfig{
+		Checks: false,
+		Paths:  paths,
+	}, nil
+}
+
+func connectProxyExposePaths(in []structs.ConsulExposePath, networks structs.Networks) ([]api.ExposePath, error) {
+	if len(in) == 0 {
+		return nil, nil
+	}
+
+	paths := make([]api.ExposePath, len(in))
+	for i, path := range in {
+		if _, exposedPort, err := connectExposePathPort(path.ListenerPort, networks); err != nil {
+			return nil, err
+		} else {
+			paths[i] = api.ExposePath{
+				ListenerPort:    exposedPort,
+				Path:            path.Path,
+				LocalPathPort:   path.LocalPathPort,
+				Protocol:        path.Protocol,
+				ParsedFromCheck: false,
+			}
+		}
+	}
+	return paths, nil
+}
+
+func connectUpstreams(in []structs.ConsulUpstream) []api.Upstream {
+	if len(in) == 0 {
+		return nil
+	}
+
+	upstreams := make([]api.Upstream, len(in))
+	for i, upstream := range in {
+		upstreams[i] = api.Upstream{
+			DestinationName: upstream.DestinationName,
+			LocalBindPort:   upstream.LocalBindPort,
+		}
+	}
+	return upstreams
+}
+
+func connectProxyConfig(cfg map[string]interface{}, port int) map[string]interface{} {
+	if cfg == nil {
+		cfg = make(map[string]interface{})
+	}
+	cfg["bind_address"] = "0.0.0.0"
+	cfg["bind_port"] = port
+	return cfg
+}
+
+func connectNetworkInvariants(networks structs.Networks) error {
+	if n := len(networks); n != 1 {
+		return fmt.Errorf("Connect only supported with exactly 1 network (found %d)", n)
+	}
+	return nil
+}
+
+// connectPort returns the network and port for the Connect proxy sidecar
+// defined for this service. An error is returned if the network and port
+// cannot be determined.
+func connectPort(serviceName string, networks structs.Networks) (*structs.NetworkResource, structs.Port, error) {
+	if err := connectNetworkInvariants(networks); err != nil {
+		return nil, structs.Port{}, err
+	}
+
+	port, ok := networks[0].PortForService(serviceName)
+	if !ok {
+		return nil, structs.Port{}, fmt.Errorf("No Connect port defined for service %q", serviceName)
+	}
+
+	return networks[0], port, nil
+}
+
+// connectExposePathPort returns the port for the exposed path for the exposed
+// proxy path.
+func connectExposePathPort(portLabel string, networks structs.Networks) (string, int, error) {
+	if err := connectNetworkInvariants(networks); err != nil {
+		return "", 0, err
+	}
+
+	ip, port := networks.Port(portLabel)
+	if port == 0 {
+		return "", 0, fmt.Errorf("No port of label %q defined", portLabel)
+	}
+
+	return ip, port, nil
+}

command/agent/consul/connect_test.go

@@ -0,0 +1,376 @@
+package consul
+
+import (
+	"testing"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	testConnectNetwork = structs.Networks{{
+		Mode:   "bridge",
+		Device: "eth0",
+		IP:     "192.168.30.1",
+		DynamicPorts: []structs.Port{
+			{Label: "healthPort", Value: 23100, To: 23100},
+			{Label: "metricsPort", Value: 23200, To: 23200},
+			{Label: "connect-proxy-redis", Value: 3000, To: 3000},
+		},
+	}}
+)
+
+func TestConnect_newConnect(t *testing.T) {
+	t.Parallel()
+
+	t.Run("nil", func(t *testing.T) {
+		asr, err := newConnect("", nil, nil)
+		require.NoError(t, err)
+		require.Nil(t, asr)
+	})
+
+	t.Run("native", func(t *testing.T) {
+		asr, err := newConnect("", &structs.ConsulConnect{
+			Native: true,
+		}, nil)
+		require.NoError(t, err)
+		require.True(t, asr.Native)
+		require.Nil(t, asr.SidecarService)
+	})
+
+	t.Run("with sidecar", func(t *testing.T) {
+		asr, err := newConnect("redis", &structs.ConsulConnect{
+			Native: false,
+			SidecarService: &structs.ConsulSidecarService{
+				Tags: []string{"foo", "bar"},
+				Port: "sidecarPort",
+			},
+		}, testConnectNetwork)
+		require.NoError(t, err)
+		require.Equal(t, &api.AgentServiceRegistration{
+			Tags:    []string{"foo", "bar"},
+			Port:    3000,
+			Address: "192.168.30.1",
+		}, asr.SidecarService)
+	})
+}
+
+func TestConnect_connectSidecarRegistration(t *testing.T) {
+	t.Parallel()
+
+	t.Run("nil", func(t *testing.T) {
+		sidecarReg, err := connectSidecarRegistration("", nil, testConnectNetwork)
+		require.NoError(t, err)
+		require.Nil(t, sidecarReg)
+	})
+
+	t.Run("no service port", func(t *testing.T) {
+		_, err := connectSidecarRegistration("unknown", &structs.ConsulSidecarService{
+			// irrelevant
+		}, testConnectNetwork)
+		require.EqualError(t, err, `No Connect port defined for service "unknown"`)
+	})
+
+	t.Run("bad proxy", func(t *testing.T) {
+		_, err := connectSidecarRegistration("redis", &structs.ConsulSidecarService{
+			Proxy: &structs.ConsulProxy{
+				Expose: &structs.ConsulExposeConfig{
+					Paths: []structs.ConsulExposePath{{
+						ListenerPort: "badPort",
+					}},
+				},
+			},
+		}, testConnectNetwork)
+		require.EqualError(t, err, `No port of label "badPort" defined`)
+	})
+
+	t.Run("normal", func(t *testing.T) {
+		proxy, err := connectSidecarRegistration("redis", &structs.ConsulSidecarService{
+			Tags: []string{"foo", "bar"},
+			Port: "sidecarPort",
+		}, testConnectNetwork)
+		require.NoError(t, err)
+		require.Equal(t, &api.AgentServiceRegistration{
+			Tags:    []string{"foo", "bar"},
+			Port:    3000,
+			Address: "192.168.30.1",
+		}, proxy)
+	})
+}
+
+func TestConnect_connectProxy(t *testing.T) {
+	t.Parallel()
+
+	t.Run("nil", func(t *testing.T) {
+		proxy, err := connectProxy(nil, 2000, nil)
+		require.NoError(t, err)
+		require.Nil(t, proxy)
+	})
+
+	t.Run("bad proxy", func(t *testing.T) {
+		_, err := connectProxy(&structs.ConsulProxy{
+			LocalServiceAddress: "0.0.0.0",
+			LocalServicePort:    2000,
+			Upstreams:           nil,
+			Expose: &structs.ConsulExposeConfig{
+				Paths: []structs.ConsulExposePath{{
+					ListenerPort: "badPort",
+				}},
+			},
+			Config: nil,
+		}, 2000, testConnectNetwork)
+		require.EqualError(t, err, `No port of label "badPort" defined`)
+	})
+
+	t.Run("normal", func(t *testing.T) {
+		proxy, err := connectProxy(&structs.ConsulProxy{
+			LocalServiceAddress: "0.0.0.0",
+			LocalServicePort:    2000,
+			Upstreams:           nil,
+			Expose: &structs.ConsulExposeConfig{
+				Paths: []structs.ConsulExposePath{{
+					Path:          "/health",
+					Protocol:      "http",
+					LocalPathPort: 8000,
+					ListenerPort:  "healthPort",
+				}},
+			},
+			Config: nil,
+		}, 2000, testConnectNetwork)
+		require.NoError(t, err)
+		require.Equal(t, &api.AgentServiceConnectProxyConfig{
+			LocalServiceAddress: "0.0.0.0",
+			LocalServicePort:    2000,
+			Upstreams:           nil,
+			Expose: api.ExposeConfig{
+				Paths: []api.ExposePath{{
+					Path:          "/health",
+					Protocol:      "http",
+					LocalPathPort: 8000,
+					ListenerPort:  23100,
+				}},
+			},
+			Config: map[string]interface{}{
+				"bind_address": "0.0.0.0",
+				"bind_port":    2000,
+			},
+		}, proxy)
+	})
+}
+
+func TestConnect_connectProxyExpose(t *testing.T) {
+	t.Parallel()
+
+	t.Run("nil", func(t *testing.T) {
+		exposeConfig, err := connectProxyExpose(nil, nil)
+		require.NoError(t, err)
+		require.Equal(t, api.ExposeConfig{}, exposeConfig)
+	})
+
+	t.Run("bad port", func(t *testing.T) {
+		_, err := connectProxyExpose(&structs.ConsulExposeConfig{
+			Paths: []structs.ConsulExposePath{{
+				ListenerPort: "badPort",
+			}},
+		}, testConnectNetwork)
+		require.EqualError(t, err, `No port of label "badPort" defined`)
+	})
+
+	t.Run("normal", func(t *testing.T) {
+		expose, err := connectProxyExpose(&structs.ConsulExposeConfig{
+			Paths: []structs.ConsulExposePath{{
+				Path:          "/health",
+				Protocol:      "http",
+				LocalPathPort: 8000,
+				ListenerPort:  "healthPort",
+			}},
+		}, testConnectNetwork)
+		require.NoError(t, err)
+		require.Equal(t, api.ExposeConfig{
+			Checks: false,
+			Paths: []api.ExposePath{{
+				Path:            "/health",
+				ListenerPort:    23100,
+				LocalPathPort:   8000,
+				Protocol:        "http",
+				ParsedFromCheck: false,
+			}},
+		}, expose)
+	})
+}
+
+func TestConnect_connectProxyExposePaths(t *testing.T) {
+	t.Parallel()
+
+	t.Run("nil", func(t *testing.T) {
+		upstreams, err := connectProxyExposePaths(nil, nil)
+		require.NoError(t, err)
+		require.Empty(t, upstreams)
+	})
+
+	t.Run("no network", func(t *testing.T) {
+		original := []structs.ConsulExposePath{{Path: "/path"}}
+		_, err := connectProxyExposePaths(original, nil)
+		require.EqualError(t, err, `Connect only supported with exactly 1 network (found 0)`)
+	})
+
+	t.Run("normal", func(t *testing.T) {
+		original := []structs.ConsulExposePath{{
+			Path:          "/health",
+			Protocol:      "http",
+			LocalPathPort: 8000,
+			ListenerPort:  "healthPort",
+		}, {
+			Path:          "/metrics",
+			Protocol:      "grpc",
+			LocalPathPort: 9500,
+			ListenerPort:  "metricsPort",
+		}}
+		exposePaths, err := connectProxyExposePaths(original, testConnectNetwork)
+		require.NoError(t, err)
+		require.Equal(t, []api.ExposePath{
+			{
+				Path:            "/health",
+				Protocol:        "http",
+				LocalPathPort:   8000,
+				ListenerPort:    23100,
+				ParsedFromCheck: false,
+			},
+			{
+				Path:            "/metrics",
+				Protocol:        "grpc",
+				LocalPathPort:   9500,
+				ListenerPort:    23200,
+				ParsedFromCheck: false,
+			},
+		}, exposePaths)
+	})
+}
+
+func TestConnect_connectUpstreams(t *testing.T) {
+	t.Parallel()
+
+	t.Run("nil", func(t *testing.T) {
+		require.Nil(t, connectUpstreams(nil))
+	})
+
+	t.Run("not empty", func(t *testing.T) {
+		require.Equal(t,
+			[]api.Upstream{{
+				DestinationName: "foo",
+				LocalBindPort:   8000,
+			}, {
+				DestinationName: "bar",
+				LocalBindPort:   9000,
+			}},
+			connectUpstreams([]structs.ConsulUpstream{{
+				DestinationName: "foo",
+				LocalBindPort:   8000,
+			}, {
+				DestinationName: "bar",
+				LocalBindPort:   9000,
+			}}),
+		)
+	})
+}
+
+func TestConnect_connectProxyConfig(t *testing.T) {
+	t.Parallel()
+
+	t.Run("nil map", func(t *testing.T) {
+		require.Equal(t, map[string]interface{}{
+			"bind_address": "0.0.0.0",
+			"bind_port":    42,
+		}, connectProxyConfig(nil, 42))
+	})
+
+	t.Run("pre-existing map", func(t *testing.T) {
+		require.Equal(t, map[string]interface{}{
+			"bind_address": "0.0.0.0",
+			"bind_port":    42,
+			"foo":          "bar",
+		}, connectProxyConfig(map[string]interface{}{
+			"foo": "bar",
+		}, 42))
+	})
+}
+
+func TestConnect_getConnectPort(t *testing.T) {
+	t.Parallel()
+
+	networks := structs.Networks{{
+		IP: "192.168.30.1",
+		DynamicPorts: []structs.Port{{
+			Label: "connect-proxy-foo",
+			Value: 23456,
+			To:    23456,
+		}}}}
+
+	t.Run("normal", func(t *testing.T) {
+		nr, port, err := connectPort("foo", networks)
+		require.NoError(t, err)
+		require.Equal(t, structs.Port{
+			Label: "connect-proxy-foo",
+			Value: 23456,
+			To:    23456,
+		}, port)
+		require.Equal(t, "192.168.30.1", nr.IP)
+	})
+
+	t.Run("no such service", func(t *testing.T) {
+		_, _, err := connectPort("other", networks)
+		require.EqualError(t, err, `No Connect port defined for service "other"`)
+	})
+
+	t.Run("no network", func(t *testing.T) {
+		_, _, err := connectPort("foo", nil)
+		require.EqualError(t, err, "Connect only supported with exactly 1 network (found 0)")
+	})
+
+	t.Run("multi network", func(t *testing.T) {
+		_, _, err := connectPort("foo", append(networks, &structs.NetworkResource{
+			Device: "eth1",
+			IP:     "10.0.10.0",
+		}))
+		require.EqualError(t, err, "Connect only supported with exactly 1 network (found 2)")
+	})
+}
+
+func TestConnect_getExposePathPort(t *testing.T) {
+	t.Parallel()
+
+	networks := structs.Networks{{
+		Device: "eth0",
+		IP:     "192.168.30.1",
+		DynamicPorts: []structs.Port{{
+			Label: "myPort",
+			Value: 23456,
+			To:    23456,
+		}}}}
+
+	t.Run("normal", func(t *testing.T) {
+		ip, port, err := connectExposePathPort("myPort", networks)
+		require.NoError(t, err)
+		require.Equal(t, ip, "192.168.30.1")
+		require.Equal(t, 23456, port)
+	})
+
+	t.Run("no such port label", func(t *testing.T) {
+		_, _, err := connectExposePathPort("otherPort", networks)
+		require.EqualError(t, err, `No port of label "otherPort" defined`)
+	})
+
+	t.Run("no network", func(t *testing.T) {
+		_, _, err := connectExposePathPort("myPort", nil)
+		require.EqualError(t, err, "Connect only supported with exactly 1 network (found 0)")
+	})
+
+	t.Run("multi network", func(t *testing.T) {
+		_, _, err := connectExposePathPort("myPort", append(networks, &structs.NetworkResource{
+			Device: "eth1",
+			IP:     "10.0.10.0",
+		}))
+		require.EqualError(t, err, "Connect only supported with exactly 1 network (found 2)")
+	})
+}

command/agent/job_endpoint.go

@@ -1185,67 +1185,110 @@ func ApiConsulConnectToStructs(in *api.ConsulConnect) *structs.ConsulConnect {
 	if in == nil {
 		return nil
 	}
-
-	out := &structs.ConsulConnect{
-		Native: in.Native,
+	return &structs.ConsulConnect{
+		Native:         in.Native,
+		SidecarService: apiConnectSidecarServiceToStructs(in.SidecarService),
+		SidecarTask:    apiConnectSidecarTaskToStructs(in.SidecarTask),
 	}
+}
 
-	if in.SidecarService != nil {
+func apiConnectSidecarServiceToStructs(in *api.ConsulSidecarService) *structs.ConsulSidecarService {
+	if in == nil {
+		return nil
+	}
+	return &structs.ConsulSidecarService{
+		Port:  in.Port,
+		Tags:  helper.CopySliceString(in.Tags),
+		Proxy: apiConnectSidecarServiceProxyToStructs(in.Proxy),
+	}
+}
 
-		out.SidecarService = &structs.ConsulSidecarService{
-			Tags: helper.CopySliceString(in.SidecarService.Tags),
-			Port: in.SidecarService.Port,
-		}
+func apiConnectSidecarServiceProxyToStructs(in *api.ConsulProxy) *structs.ConsulProxy {
+	if in == nil {
+		return nil
+	}
+	return &structs.ConsulProxy{
+		LocalServiceAddress: in.LocalServiceAddress,
+		LocalServicePort:    in.LocalServicePort,
+		Upstreams:           apiUpstreamsToStructs(in.Upstreams),
+		Expose:              apiConsulExposeConfigToStructs(in.ExposeConfig),
+		Config:              in.Config,
+	}
+}
 
-		if in.SidecarService.Proxy != nil {
-
-			out.SidecarService.Proxy = &structs.ConsulProxy{
-				LocalServiceAddress: in.SidecarService.Proxy.LocalServiceAddress,
-				LocalServicePort:    in.SidecarService.Proxy.LocalServicePort,
-				Config:              in.SidecarService.Proxy.Config,
-			}
-
-			upstreams := make([]structs.ConsulUpstream, len(in.SidecarService.Proxy.Upstreams))
-			for i, p := range in.SidecarService.Proxy.Upstreams {
-				upstreams[i] = structs.ConsulUpstream{
-					DestinationName: p.DestinationName,
-					LocalBindPort:   p.LocalBindPort,
-				}
-			}
-
-			out.SidecarService.Proxy.Upstreams = upstreams
+func apiUpstreamsToStructs(in []*api.ConsulUpstream) []structs.ConsulUpstream {
+	if len(in) == 0 {
+		return nil
+	}
+	upstreams := make([]structs.ConsulUpstream, len(in))
+	for i, upstream := range in {
+		upstreams[i] = structs.ConsulUpstream{
+			DestinationName: upstream.DestinationName,
+			LocalBindPort:   upstream.LocalBindPort,
 		}
 	}
+	return upstreams
+}
 
-	if in.SidecarTask != nil {
-		out.SidecarTask = &structs.SidecarTask{
-			Name:          in.SidecarTask.Name,
-			Driver:        in.SidecarTask.Driver,
-			Config:        in.SidecarTask.Config,
-			User:          in.SidecarTask.User,
-			Env:           in.SidecarTask.Env,
-			Resources:     ApiResourcesToStructs(in.SidecarTask.Resources),
-			Meta:          in.SidecarTask.Meta,
-			LogConfig:     &structs.LogConfig{},
-			ShutdownDelay: in.SidecarTask.ShutdownDelay,
-			KillSignal:    in.SidecarTask.KillSignal,
-		}
+func apiConsulExposeConfigToStructs(in *api.ConsulExposeConfig) *structs.ConsulExposeConfig {
+	if in == nil {
+		return nil
+	}
+	return &structs.ConsulExposeConfig{
+		Paths: apiConsulExposePathsToStructs(in.Paths),
+	}
+}
 
-		if in.SidecarTask.KillTimeout != nil {
-			out.SidecarTask.KillTimeout = in.SidecarTask.KillTimeout
-		}
-		if in.SidecarTask.LogConfig != nil {
-			out.SidecarTask.LogConfig = &structs.LogConfig{}
-			if in.SidecarTask.LogConfig.MaxFiles != nil {
-				out.SidecarTask.LogConfig.MaxFiles = *in.SidecarTask.LogConfig.MaxFiles
-			}
-			if in.SidecarTask.LogConfig.MaxFileSizeMB != nil {
-				out.SidecarTask.LogConfig.MaxFileSizeMB = *in.SidecarTask.LogConfig.MaxFileSizeMB
-			}
+func apiConsulExposePathsToStructs(in []*api.ConsulExposePath) []structs.ConsulExposePath {
+	if len(in) == 0 {
+		return nil
+	}
+	paths := make([]structs.ConsulExposePath, len(in))
+	for i, path := range in {
+		paths[i] = structs.ConsulExposePath{
+			Path:          path.Path,
+			Protocol:      path.Protocol,
+			LocalPathPort: path.LocalPathPort,
+			ListenerPort:  path.ListenerPort,
 		}
 	}
+	return paths
+}
 
-	return out
+func apiConnectSidecarTaskToStructs(in *api.SidecarTask) *structs.SidecarTask {
+	if in == nil {
+		return nil
+	}
+	return &structs.SidecarTask{
+		Name:          in.Name,
+		Driver:        in.Driver,
+		User:          in.User,
+		Config:        in.Config,
+		Env:           in.Env,
+		Resources:     ApiResourcesToStructs(in.Resources),
+		Meta:          in.Meta,
+		ShutdownDelay: in.ShutdownDelay,
+		KillSignal:    in.KillSignal,
+		KillTimeout:   in.KillTimeout,
+		LogConfig:     apiLogConfigToStructs(in.LogConfig),
+	}
+}
+
+func apiLogConfigToStructs(in *api.LogConfig) *structs.LogConfig {
+	if in == nil {
+		return nil
+	}
+	return &structs.LogConfig{
+		MaxFiles:      dereferenceInt(in.MaxFiles),
+		MaxFileSizeMB: dereferenceInt(in.MaxFileSizeMB),
+	}
+}
+
+func dereferenceInt(in *int) int {
+	if in == nil {
+		return 0
+	}
+	return *in
 }
 
 func ApiConstraintsToStructs(in []*api.Constraint) []*structs.Constraint {

command/agent/job_endpoint_test.go

@@ -2542,3 +2542,169 @@ func TestHTTP_JobValidate_SystemMigrate(t *testing.T) {
 		require.Contains(t, resp.Error, `Job type "system" does not allow migrate block`)
 	})
 }
+
+func TestConversion_dereferenceInt(t *testing.T) {
+	t.Parallel()
+	require.Equal(t, 0, dereferenceInt(nil))
+	require.Equal(t, 42, dereferenceInt(helper.IntToPtr(42)))
+}
+
+func TestConversion_apiLogConfigToStructs(t *testing.T) {
+	t.Parallel()
+	require.Nil(t, apiLogConfigToStructs(nil))
+	require.Equal(t, &structs.LogConfig{
+		MaxFiles:      2,
+		MaxFileSizeMB: 8,
+	}, apiLogConfigToStructs(&api.LogConfig{
+		MaxFiles:      helper.IntToPtr(2),
+		MaxFileSizeMB: helper.IntToPtr(8),
+	}))
+}
+
+func TestConversion_apiConnectSidecarTaskToStructs(t *testing.T) {
+	t.Parallel()
+	require.Nil(t, apiConnectSidecarTaskToStructs(nil))
+	delay := time.Duration(200)
+	timeout := time.Duration(1000)
+	config := make(map[string]interface{})
+	env := make(map[string]string)
+	meta := make(map[string]string)
+	require.Equal(t, &structs.SidecarTask{
+		Name:   "name",
+		Driver: "driver",
+		User:   "user",
+		Config: config,
+		Env:    env,
+		Resources: &structs.Resources{
+			CPU:      1,
+			MemoryMB: 128,
+		},
+		Meta:        meta,
+		KillTimeout: &timeout,
+		LogConfig: &structs.LogConfig{
+			MaxFiles:      2,
+			MaxFileSizeMB: 8,
+		},
+		ShutdownDelay: &delay,
+		KillSignal:    "SIGTERM",
+	}, apiConnectSidecarTaskToStructs(&api.SidecarTask{
+		Name:   "name",
+		Driver: "driver",
+		User:   "user",
+		Config: config,
+		Env:    env,
+		Resources: &api.Resources{
+			CPU:      helper.IntToPtr(1),
+			MemoryMB: helper.IntToPtr(128),
+		},
+		Meta:        meta,
+		KillTimeout: &timeout,
+		LogConfig: &api.LogConfig{
+			MaxFiles:      helper.IntToPtr(2),
+			MaxFileSizeMB: helper.IntToPtr(8),
+		},
+		ShutdownDelay: &delay,
+		KillSignal:    "SIGTERM",
+	}))
+}
+
+func TestConversion_apiConsulExposePathsToStructs(t *testing.T) {
+	t.Parallel()
+	require.Nil(t, apiConsulExposePathsToStructs(nil))
+	require.Nil(t, apiConsulExposePathsToStructs(make([]*api.ConsulExposePath, 0)))
+	require.Equal(t, []structs.ConsulExposePath{{
+		Path:          "/health",
+		Protocol:      "http",
+		LocalPathPort: 8080,
+		ListenerPort:  "hcPort",
+	}}, apiConsulExposePathsToStructs([]*api.ConsulExposePath{{
+		Path:          "/health",
+		Protocol:      "http",
+		LocalPathPort: 8080,
+		ListenerPort:  "hcPort",
+	}}))
+}
+
+func TestConversion_apiConsulExposeConfigToStructs(t *testing.T) {
+	t.Parallel()
+	require.Nil(t, apiConsulExposeConfigToStructs(nil))
+	require.Equal(t, &structs.ConsulExposeConfig{
+		Paths: []structs.ConsulExposePath{{Path: "/health"}},
+	}, apiConsulExposeConfigToStructs(&api.ConsulExposeConfig{
+		Paths: []*api.ConsulExposePath{{Path: "/health"}},
+	}))
+}
+
+func TestConversion_apiUpstreamsToStructs(t *testing.T) {
+	t.Parallel()
+	require.Nil(t, apiUpstreamsToStructs(nil))
+	require.Nil(t, apiUpstreamsToStructs(make([]*api.ConsulUpstream, 0)))
+	require.Equal(t, []structs.ConsulUpstream{{
+		DestinationName: "upstream",
+		LocalBindPort:   8000,
+	}}, apiUpstreamsToStructs([]*api.ConsulUpstream{{
+		DestinationName: "upstream",
+		LocalBindPort:   8000,
+	}}))
+}
+
+func TestConversion_apiConnectSidecarServiceProxyToStructs(t *testing.T) {
+	t.Parallel()
+	require.Nil(t, apiConnectSidecarServiceProxyToStructs(nil))
+	config := make(map[string]interface{})
+	require.Equal(t, &structs.ConsulProxy{
+		LocalServiceAddress: "192.168.30.1",
+		LocalServicePort:    9000,
+		Config:              config,
+		Upstreams: []structs.ConsulUpstream{{
+			DestinationName: "upstream",
+		}},
+		Expose: &structs.ConsulExposeConfig{
+			Paths: []structs.ConsulExposePath{{Path: "/health"}},
+		},
+	}, apiConnectSidecarServiceProxyToStructs(&api.ConsulProxy{
+		LocalServiceAddress: "192.168.30.1",
+		LocalServicePort:    9000,
+		Config:              config,
+		Upstreams: []*api.ConsulUpstream{{
+			DestinationName: "upstream",
+		}},
+		ExposeConfig: &api.ConsulExposeConfig{
+			Paths: []*api.ConsulExposePath{{
+				Path: "/health",
+			}},
+		},
+	}))
+}
+
+func TestConversion_apiConnectSidecarServiceToStructs(t *testing.T) {
+	t.Parallel()
+	require.Nil(t, apiConnectSidecarTaskToStructs(nil))
+	require.Equal(t, &structs.ConsulSidecarService{
+		Tags: []string{"foo"},
+		Port: "myPort",
+		Proxy: &structs.ConsulProxy{
+			LocalServiceAddress: "192.168.30.1",
+		},
+	}, apiConnectSidecarServiceToStructs(&api.ConsulSidecarService{
+		Tags: []string{"foo"},
+		Port: "myPort",
+		Proxy: &api.ConsulProxy{
+			LocalServiceAddress: "192.168.30.1",
+		},
+	}))
+}
+
+func TestConversion_ApiConsulConnectToStructs(t *testing.T) {
+	t.Parallel()
+	require.Nil(t, ApiConsulConnectToStructs(nil))
+	require.Equal(t, &structs.ConsulConnect{
+		Native:         false,
+		SidecarService: &structs.ConsulSidecarService{Port: "myPort"},
+		SidecarTask:    &structs.SidecarTask{Name: "task"},
+	}, ApiConsulConnectToStructs(&api.ConsulConnect{
+		Native:         false,
+		SidecarService: &api.ConsulSidecarService{Port: "myPort"},
+		SidecarTask:    &api.SidecarTask{Name: "task"},
+	}))
+}