transparent proxy: add jobspec support (#20144)

Add a transparent proxy block to the existing Connect sidecar service proxy
block. This changeset is plumbing required to support transparent proxy
configuration on the client.

Ref: https://github.com/hashicorp/nomad/issues/10628
This commit is contained in:
Tim Gross
2024-03-21 09:22:24 -04:00
parent 648daceca1
commit e8d203e7ce
14 changed files with 705 additions and 16 deletions

View File

@@ -158,12 +158,17 @@ func (st *SidecarTask) Canonicalize() {
// ConsulProxy represents a Consul Connect sidecar proxy jobspec block.
type ConsulProxy struct {
LocalServiceAddress string `mapstructure:"local_service_address" hcl:"local_service_address,optional"`
LocalServicePort int `mapstructure:"local_service_port" hcl:"local_service_port,optional"`
Expose *ConsulExposeConfig `mapstructure:"expose" hcl:"expose,block"`
ExposeConfig *ConsulExposeConfig // Deprecated: only to maintain backwards compatibility. Use Expose instead.
Upstreams []*ConsulUpstream `hcl:"upstreams,block"`
Config map[string]interface{} `hcl:"config,block"`
LocalServiceAddress string `mapstructure:"local_service_address" hcl:"local_service_address,optional"`
LocalServicePort int `mapstructure:"local_service_port" hcl:"local_service_port,optional"`
Expose *ConsulExposeConfig `mapstructure:"expose" hcl:"expose,block"`
ExposeConfig *ConsulExposeConfig // Deprecated: only to maintain backwards compatibility. Use Expose instead.
Upstreams []*ConsulUpstream `hcl:"upstreams,block"`
// TransparentProxy configures the Envoy sidecar to use "transparent
// proxying", which creates IP tables rules inside the network namespace to
// ensure traffic flows thru the Envoy proxy
TransparentProxy *ConsulTransparentProxy `mapstructure:"transparent_proxy" hcl:"transparent_proxy,block"`
Config map[string]interface{} `hcl:"config,block"`
}
func (cp *ConsulProxy) Canonicalize() {
@@ -177,6 +182,8 @@ func (cp *ConsulProxy) Canonicalize() {
cp.Upstreams = nil
}
cp.TransparentProxy.Canonicalize()
for _, upstream := range cp.Upstreams {
upstream.Canonicalize()
}
@@ -257,6 +264,61 @@ func (cu *ConsulUpstream) Canonicalize() {
}
}
// ConsulTransparentProxy is used to configure the Envoy sidecar for
// "transparent proxying", which creates IP tables rules inside the network
// namespace to ensure traffic flows thru the Envoy proxy
type ConsulTransparentProxy struct {
// UID of the Envoy proxy. Defaults to the default Envoy proxy container
// image user.
UID string `mapstructure:"uid" hcl:"uid,optional"`
// OutboundPort is the Envoy proxy's outbound listener port. Inbound TCP
// traffic hitting the PROXY_IN_REDIRECT chain will be redirected here.
// Defaults to 15001.
OutboundPort uint16 `mapstructure:"outbound_port" hcl:"outbound_port,optional"`
// ExcludeInboundPorts is an additional set of ports will be excluded from
// redirection to the Envoy proxy. Can be Port.Label or Port.Value. This set
// will be added to the ports automatically excluded for the Expose.Port and
// Check.Expose fields.
ExcludeInboundPorts []string `mapstructure:"exclude_inbound_ports" hcl:"exclude_inbound_ports,optional"`
// ExcludeOutboundPorts is a set of outbound ports that will not be
// redirected to the Envoy proxy, specified as port numbers.
ExcludeOutboundPorts []uint16 `mapstructure:"exclude_outbound_ports" hcl:"exclude_outbound_ports,optional"`
// ExcludeOutboundCIDRs is a set of outbound CIDR blocks that will not be
// redirected to the Envoy proxy.
ExcludeOutboundCIDRs []string `mapstructure:"exclude_outbound_cidrs" hcl:"exclude_outbound_cidrs,optional"`
// ExcludeUIDs is a set of user IDs whose network traffic will not be
// redirected through the Envoy proxy.
ExcludeUIDs []string `mapstructure:"exclude_uids" hcl:"exclude_uids,optional"`
// NoDNS disables redirection of DNS traffic to Consul DNS. By default NoDNS
// is false and transparent proxy will direct DNS traffic to Consul DNS if
// available on the client.
NoDNS bool `mapstructure:"no_dns" hcl:"no_dns,optional"`
}
func (tp *ConsulTransparentProxy) Canonicalize() {
if tp == nil {
return
}
if len(tp.ExcludeInboundPorts) == 0 {
tp.ExcludeInboundPorts = nil
}
if len(tp.ExcludeOutboundCIDRs) == 0 {
tp.ExcludeOutboundCIDRs = nil
}
if len(tp.ExcludeOutboundPorts) == 0 {
tp.ExcludeOutboundPorts = nil
}
if len(tp.ExcludeUIDs) == 0 {
tp.ExcludeUIDs = nil
}
}
type ConsulExposeConfig struct {
Paths []*ConsulExposePath `mapstructure:"path" hcl:"path,block"`
Path []*ConsulExposePath // Deprecated: only to maintain backwards compatibility. Use Paths instead.

View File

@@ -145,8 +145,13 @@ func connectSidecarProxy(info structs.AllocInfo, proxy *structs.ConsulProxy, cPo
if err != nil {
return nil, err
}
mode := api.ProxyModeDefault
if proxy.TransparentProxy != nil {
mode = api.ProxyModeTransparent
}
return &api.AgentServiceConnectProxyConfig{
Mode: mode,
LocalServiceAddress: proxy.LocalServiceAddress,
LocalServicePort: proxy.LocalServicePort,
Config: connectProxyConfig(proxy.Config, cPort, info),

View File

@@ -1897,6 +1897,7 @@ func apiConnectSidecarServiceProxyToStructs(in *api.ConsulProxy) *structs.Consul
LocalServicePort: in.LocalServicePort,
Upstreams: apiUpstreamsToStructs(in.Upstreams),
Expose: apiConsulExposeConfigToStructs(expose),
TransparentProxy: apiConnectTransparentProxyToStructs(in.TransparentProxy),
Config: maps.Clone(in.Config),
}
}
@@ -1949,6 +1950,21 @@ func apiConsulExposeConfigToStructs(in *api.ConsulExposeConfig) *structs.ConsulE
}
}
func apiConnectTransparentProxyToStructs(in *api.ConsulTransparentProxy) *structs.ConsulTransparentProxy {
if in == nil {
return nil
}
return &structs.ConsulTransparentProxy{
UID: in.UID,
OutboundPort: in.OutboundPort,
ExcludeInboundPorts: in.ExcludeInboundPorts,
ExcludeOutboundPorts: in.ExcludeOutboundPorts,
ExcludeOutboundCIDRs: in.ExcludeOutboundCIDRs,
ExcludeUIDs: in.ExcludeUIDs,
NoDNS: in.NoDNS,
}
}
func apiConsulExposePathsToStructs(in []*api.ConsulExposePath) []structs.ConsulExposePath {
if len(in) == 0 {
return nil

View File

@@ -927,6 +927,7 @@ func parseProxy(o *ast.ObjectItem) (*api.ConsulProxy, error) {
"local_service_port",
"upstreams",
"expose",
"transparent_proxy",
"config",
}
@@ -942,6 +943,7 @@ func parseProxy(o *ast.ObjectItem) (*api.ConsulProxy, error) {
delete(m, "upstreams")
delete(m, "expose")
delete(m, "transparent_proxy")
delete(m, "config")
dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
@@ -985,6 +987,16 @@ func parseProxy(o *ast.ObjectItem) (*api.ConsulProxy, error) {
}
}
if tpo := listVal.Filter("transparent_proxy"); len(tpo.Items) > 1 {
return nil, fmt.Errorf("only 1 transparent_proxy object supported")
} else if len(tpo.Items) == 1 {
if tp, err := parseTproxy(tpo.Items[0]); err != nil {
return nil, err
} else {
proxy.TransparentProxy = tp
}
}
// If we have config, then parse that
if o := listVal.Filter("config"); len(o.Items) > 1 {
return nil, fmt.Errorf("only 1 meta object supported")
@@ -1077,6 +1089,41 @@ func parseExposePath(epo *ast.ObjectItem) (*api.ConsulExposePath, error) {
return &path, nil
}
func parseTproxy(epo *ast.ObjectItem) (*api.ConsulTransparentProxy, error) {
valid := []string{
"uid",
"outbound_port",
"exclude_inbound_ports",
"exclude_outbound_ports",
"exclude_outbound_cidrs",
"exclude_uids",
"no_dns",
}
if err := checkHCLKeys(epo.Val, valid); err != nil {
return nil, multierror.Prefix(err, "tproxy ->")
}
var tproxy api.ConsulTransparentProxy
var m map[string]interface{}
if err := hcl.DecodeObject(&m, epo.Val); err != nil {
return nil, err
}
dec, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
Result: &tproxy,
})
if err != nil {
return nil, err
}
if err := dec.Decode(m); err != nil {
return nil, err
}
return &tproxy, nil
}
func parseUpstream(uo *ast.ObjectItem) (*api.ConsulUpstream, error) {
valid := []string{
"destination_name",

View File

@@ -1470,6 +1470,15 @@ func TestParse(t *testing.T) {
DestinationName: "upstream2",
LocalBindPort: 2002,
}},
TransparentProxy: &api.ConsulTransparentProxy{
UID: "101",
OutboundPort: 15001,
ExcludeInboundPorts: []string{"www", "9000"},
ExcludeOutboundPorts: []uint16{443, 80},
ExcludeOutboundCIDRs: []string{"10.0.0.0/8"},
ExcludeUIDs: []string{"10", "1001"},
NoDNS: true,
},
Config: map[string]interface{}{
"foo": "bar",
},

View File

@@ -40,6 +40,16 @@ job "service-connect-proxy" {
}
}
transparent_proxy {
uid = "101"
outbound_port = 15001
exclude_inbound_ports = ["www", "9000"]
exclude_outbound_ports = [443, 80]
exclude_outbound_cidrs = ["10.0.0.0/8"]
exclude_uids = ["10", "1001"]
no_dns = true
}
config {
foo = "bar"
}

View File

@@ -561,33 +561,68 @@ func groupConnectValidate(g *structs.TaskGroup) error {
}
}
if err := groupConnectUpstreamsValidate(g.Name, g.Services); err != nil {
if err := groupConnectUpstreamsValidate(g, g.Services); err != nil {
return err
}
return nil
}
func groupConnectUpstreamsValidate(group string, services []*structs.Service) error {
func groupConnectUpstreamsValidate(g *structs.TaskGroup, services []*structs.Service) error {
listeners := make(map[string]string) // address -> service
var connectBlockCount int
var hasTproxy bool
for _, service := range services {
if service.Connect != nil {
connectBlockCount++
}
if service.Connect.HasSidecar() && service.Connect.SidecarService.Proxy != nil {
for _, up := range service.Connect.SidecarService.Proxy.Upstreams {
listener := net.JoinHostPort(up.LocalBindAddress, strconv.Itoa(up.LocalBindPort))
if s, exists := listeners[listener]; exists {
return fmt.Errorf(
"Consul Connect services %q and %q in group %q using same address for upstreams (%s)",
service.Name, s, group, listener,
service.Name, s, g.Name, listener,
)
}
listeners[listener] = service.Name
}
if tp := service.Connect.SidecarService.Proxy.TransparentProxy; tp != nil {
hasTproxy = true
for _, portLabel := range tp.ExcludeInboundPorts {
if !transparentProxyPortLabelValidate(g, portLabel) {
return fmt.Errorf(
"Consul Connect transparent proxy port %q must be numeric or one of network.port labels", portLabel)
}
}
}
}
}
if hasTproxy && connectBlockCount > 1 {
return fmt.Errorf("Consul Connect transparent proxy requires there is only one connect block")
}
return nil
}
func transparentProxyPortLabelValidate(g *structs.TaskGroup, portLabel string) bool {
if _, err := strconv.ParseUint(portLabel, 10, 64); err == nil {
return true
}
for _, network := range g.Networks {
for _, reservedPort := range network.ReservedPorts {
if reservedPort.Label == portLabel {
return true
}
}
}
return false
}
func groupConnectSidecarValidate(g *structs.TaskGroup, s *structs.Service) error {
if n := len(g.Networks); n != 1 {
return fmt.Errorf("Consul Connect sidecars require exactly 1 network, found %d in group %q", n, g.Name)

View File

@@ -548,13 +548,15 @@ func TestJobEndpointConnect_groupConnectUpstreamsValidate(t *testing.T) {
ci.Parallel(t)
t.Run("no connect services", func(t *testing.T) {
err := groupConnectUpstreamsValidate("group",
tg := &structs.TaskGroup{Name: "group"}
err := groupConnectUpstreamsValidate(tg,
[]*structs.Service{{Name: "s1"}, {Name: "s2"}})
require.NoError(t, err)
must.NoError(t, err)
})
t.Run("connect services no overlap", func(t *testing.T) {
err := groupConnectUpstreamsValidate("group",
tg := &structs.TaskGroup{Name: "group"}
err := groupConnectUpstreamsValidate(tg,
[]*structs.Service{
{
Name: "s1",
@@ -589,11 +591,12 @@ func TestJobEndpointConnect_groupConnectUpstreamsValidate(t *testing.T) {
},
},
})
require.NoError(t, err)
must.NoError(t, err)
})
t.Run("connect services overlap port", func(t *testing.T) {
err := groupConnectUpstreamsValidate("group",
tg := &structs.TaskGroup{Name: "group"}
err := groupConnectUpstreamsValidate(tg,
[]*structs.Service{
{
Name: "s1",
@@ -628,7 +631,55 @@ func TestJobEndpointConnect_groupConnectUpstreamsValidate(t *testing.T) {
},
},
})
require.EqualError(t, err, `Consul Connect services "s2" and "s1" in group "group" using same address for upstreams (127.0.0.1:9002)`)
must.EqError(t, err, `Consul Connect services "s2" and "s1" in group "group" using same address for upstreams (127.0.0.1:9002)`)
})
t.Run("connect tproxy excludes invalid port", func(t *testing.T) {
tg := &structs.TaskGroup{Name: "group", Networks: structs.Networks{
{
ReservedPorts: []structs.Port{{
Label: "www",
}},
},
}}
err := groupConnectUpstreamsValidate(tg,
[]*structs.Service{
{
Name: "s1",
Connect: &structs.ConsulConnect{
SidecarService: &structs.ConsulSidecarService{
Proxy: &structs.ConsulProxy{
TransparentProxy: &structs.ConsulTransparentProxy{
ExcludeInboundPorts: []string{"www", "9000", "no-such-label"},
},
},
},
},
},
})
must.EqError(t, err, `Consul Connect transparent proxy port "no-such-label" must be numeric or one of network.port labels`)
})
t.Run("Consul Connect transparent proxy allows only one Connect block", func(t *testing.T) {
tg := &structs.TaskGroup{Name: "group"}
err := groupConnectUpstreamsValidate(tg,
[]*structs.Service{
{
Name: "s1",
Connect: &structs.ConsulConnect{},
},
{
Name: "s2",
Connect: &structs.ConsulConnect{
SidecarService: &structs.ConsulSidecarService{
Proxy: &structs.ConsulProxy{
TransparentProxy: &structs.ConsulTransparentProxy{},
},
},
},
},
})
must.EqError(t, err, `Consul Connect transparent proxy requires there is only one connect block`)
})
}

View File

@@ -3,6 +3,16 @@
package structs
import (
"fmt"
"net/netip"
"slices"
"strconv"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/nomad/helper"
)
// ConsulConfigEntries represents Consul ConfigEntry definitions from a job for
// a single Consul namespace.
type ConsulConfigEntries struct {
@@ -43,3 +53,131 @@ func (j *Job) ConfigEntries() map[string]*ConsulConfigEntries {
return collection
}
// ConsulTransparentProxy is used to configure the Envoy sidecar for
// "transparent proxying", which creates IP tables rules inside the network
// namespace to ensure traffic flows thru the Envoy proxy
type ConsulTransparentProxy struct {
// UID of the Envoy proxy. Defaults to the default Envoy proxy container
// image user.
UID string
// OutboundPort is the Envoy proxy's outbound listener port. Inbound TCP
// traffic hitting the PROXY_IN_REDIRECT chain will be redirected here.
// Defaults to 15001.
OutboundPort uint16
// ExcludeInboundPorts is an additional set of ports will be excluded from
// redirection to the Envoy proxy. Can be Port.Label or Port.Value. This set
// will be added to the ports automatically excluded for the Expose.Port and
// Check.Expose fields.
ExcludeInboundPorts []string
// ExcludeOutboundPorts is a set of outbound ports that will not be
// redirected to the Envoy proxy, specified as port numbers.
ExcludeOutboundPorts []uint16
// ExcludeOutboundCIDRs is a set of outbound CIDR blocks that will not be
// redirected to the Envoy proxy.
ExcludeOutboundCIDRs []string
// ExcludeUIDs is a set of user IDs whose network traffic will not be
// redirected through the Envoy proxy.
ExcludeUIDs []string
// NoDNS disables redirection of DNS traffic to Consul DNS. By default NoDNS
// is false and transparent proxy will direct DNS traffic to Consul DNS if
// available on the client.
NoDNS bool
}
func (tp *ConsulTransparentProxy) Copy() *ConsulTransparentProxy {
if tp == nil {
return nil
}
ntp := new(ConsulTransparentProxy)
*ntp = *tp
ntp.ExcludeInboundPorts = slices.Clone(tp.ExcludeInboundPorts)
ntp.ExcludeOutboundPorts = slices.Clone(tp.ExcludeOutboundPorts)
ntp.ExcludeOutboundCIDRs = slices.Clone(tp.ExcludeOutboundCIDRs)
ntp.ExcludeUIDs = slices.Clone(tp.ExcludeUIDs)
return ntp
}
func (tp *ConsulTransparentProxy) Validate() error {
var mErr multierror.Error
for _, rawCidr := range tp.ExcludeOutboundCIDRs {
_, err := netip.ParsePrefix(rawCidr)
if err != nil {
// note: error returned always include parsed string
mErr.Errors = append(mErr.Errors,
fmt.Errorf("could not parse transparent proxy excluded outbound CIDR as network prefix: %w", err))
}
}
requireUIDisUint := func(uidRaw string) error {
_, err := strconv.ParseUint(uidRaw, 10, 16)
if err != nil {
e, ok := err.(*strconv.NumError)
if !ok {
return fmt.Errorf("invalid user ID %q: %w", uidRaw, err)
}
return fmt.Errorf("invalid user ID %q: %w", uidRaw, e.Err)
}
return nil
}
if tp.UID != "" {
if err := requireUIDisUint(tp.UID); err != nil {
mErr.Errors = append(mErr.Errors,
fmt.Errorf("transparent proxy block has invalid UID field: %w", err))
}
}
for _, uid := range tp.ExcludeUIDs {
if err := requireUIDisUint(uid); err != nil {
mErr.Errors = append(mErr.Errors,
fmt.Errorf("transparent proxy block has invalid ExcludeUIDs field: %w", err))
}
}
// note: ExcludeInboundPorts are validated in connect validation hook
// because we need information from the network block
if mErr.Len() == 1 {
return mErr.Errors[0]
}
return mErr.ErrorOrNil()
}
func (tp *ConsulTransparentProxy) Equal(o *ConsulTransparentProxy) bool {
if tp == nil || o == nil {
return tp == o
}
if tp.UID != o.UID {
return false
}
if tp.OutboundPort != o.OutboundPort {
return false
}
if !helper.SliceSetEq(tp.ExcludeInboundPorts, o.ExcludeInboundPorts) {
return false
}
if !helper.SliceSetEq(tp.ExcludeOutboundPorts, o.ExcludeOutboundPorts) {
return false
}
if !helper.SliceSetEq(tp.ExcludeOutboundCIDRs, o.ExcludeOutboundCIDRs) {
return false
}
if !helper.SliceSetEq(tp.ExcludeUIDs, o.ExcludeUIDs) {
return false
}
if tp.NoDNS != o.NoDNS {
return false
}
return true
}

View File

@@ -7,6 +7,7 @@ import (
"testing"
"github.com/hashicorp/nomad/ci"
"github.com/shoenig/test/must"
"github.com/stretchr/testify/require"
)
@@ -25,3 +26,65 @@ func TestTaskKind_IsAnyConnectGateway(t *testing.T) {
require.False(t, NewTaskKind("", "foo").IsAnyConnectGateway())
})
}
func TestConnectTransparentProxy_Validate(t *testing.T) {
testCases := []struct {
name string
tp *ConsulTransparentProxy
expectErr string
}{
{
name: "empty is valid",
tp: &ConsulTransparentProxy{},
},
{
name: "invalid CIDR",
tp: &ConsulTransparentProxy{ExcludeOutboundCIDRs: []string{"192.168.1.1"}},
expectErr: `could not parse transparent proxy excluded outbound CIDR as network prefix: netip.ParsePrefix("192.168.1.1"): no '/'`,
},
{
name: "invalid UID",
tp: &ConsulTransparentProxy{UID: "foo"},
expectErr: `transparent proxy block has invalid UID field: invalid user ID "foo": invalid syntax`,
},
{
name: "invalid ExcludeUIDs",
tp: &ConsulTransparentProxy{ExcludeUIDs: []string{"500000"}},
expectErr: `transparent proxy block has invalid ExcludeUIDs field: invalid user ID "500000": value out of range`,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
err := tc.tp.Validate()
if tc.expectErr != "" {
must.EqError(t, err, tc.expectErr)
} else {
must.NoError(t, err)
}
})
}
}
func TestConnectTransparentProxy_Equal(t *testing.T) {
tp1 := &ConsulTransparentProxy{
UID: "101",
OutboundPort: 1001,
ExcludeInboundPorts: []string{"9000", "443"},
ExcludeOutboundPorts: []uint16{443, 80},
ExcludeOutboundCIDRs: []string{"10.0.0.0/8", "192.168.1.1"},
ExcludeUIDs: []string{"1001", "10"},
NoDNS: true,
}
tp2 := &ConsulTransparentProxy{
UID: "101",
OutboundPort: 1001,
ExcludeInboundPorts: []string{"443", "9000"},
ExcludeOutboundPorts: []uint16{80, 443},
ExcludeOutboundCIDRs: []string{"192.168.1.1", "10.0.0.0/8"},
ExcludeUIDs: []string{"10", "1001"},
NoDNS: true,
}
must.Equal(t, tp1, tp2)
}

View File

@@ -11,6 +11,7 @@ import (
"strconv"
"strings"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/helper/flatmap"
"github.com/mitchellh/hashstructure"
)
@@ -1734,6 +1735,10 @@ func consulProxyDiff(old, new *ConsulProxy, contextual bool) *ObjectDiff {
diff.Objects = append(diff.Objects, exposeDiff)
}
if tproxyDiff := consulTProxyDiff(old.TransparentProxy, new.TransparentProxy, contextual); tproxyDiff != nil {
diff.Objects = append(diff.Objects, tproxyDiff)
}
// diff the config blob
if cDiff := configDiff(old.Config, new.Config, contextual); cDiff != nil {
diff.Objects = append(diff.Objects, cDiff)
@@ -1844,6 +1849,57 @@ func consulProxyExposeDiff(prev, next *ConsulExposeConfig, contextual bool) *Obj
return diff
}
func consulTProxyDiff(prev, next *ConsulTransparentProxy, contextual bool) *ObjectDiff {
diff := &ObjectDiff{Type: DiffTypeNone, Name: "TransparentProxy"}
var oldPrimFlat, newPrimFlat map[string]string
if prev.Equal(next) {
return diff
} else if prev == nil {
prev = &ConsulTransparentProxy{}
diff.Type = DiffTypeAdded
newPrimFlat = flatmap.Flatten(next, nil, true)
} else if next == nil {
next = &ConsulTransparentProxy{}
diff.Type = DiffTypeDeleted
oldPrimFlat = flatmap.Flatten(prev, nil, true)
} else {
diff.Type = DiffTypeEdited
oldPrimFlat = flatmap.Flatten(prev, nil, true)
newPrimFlat = flatmap.Flatten(next, nil, true)
}
// diff the primitive fields
diff.Fields = fieldDiffs(oldPrimFlat, newPrimFlat, contextual)
if setDiff := stringSetDiff(prev.ExcludeInboundPorts, next.ExcludeInboundPorts,
"ExcludeInboundPorts", contextual); setDiff != nil && setDiff.Type != DiffTypeNone {
diff.Objects = append(diff.Objects, setDiff)
}
if setDiff := stringSetDiff(
helper.ConvertSlice(prev.ExcludeOutboundPorts, func(a uint16) string { return fmt.Sprint(a) }),
helper.ConvertSlice(next.ExcludeOutboundPorts, func(a uint16) string { return fmt.Sprint(a) }),
"ExcludeOutboundPorts",
contextual,
); setDiff != nil && setDiff.Type != DiffTypeNone {
diff.Objects = append(diff.Objects, setDiff)
}
if setDiff := stringSetDiff(prev.ExcludeOutboundCIDRs, next.ExcludeOutboundCIDRs,
"ExcludeOutboundCIDRs", contextual); setDiff != nil && setDiff.Type != DiffTypeNone {
diff.Objects = append(diff.Objects, setDiff)
}
if setDiff := stringSetDiff(prev.ExcludeUIDs, next.ExcludeUIDs,
"ExcludeUIDs", contextual); setDiff != nil && setDiff.Type != DiffTypeNone {
diff.Objects = append(diff.Objects, setDiff)
}
return diff
}
// serviceCheckDiffs diffs a set of service checks. If contextual diff is
// enabled, unchanged fields within objects nested in the tasks will be
// returned.

View File

@@ -3490,6 +3490,15 @@ func TestTaskGroupDiff(t *testing.T) {
Config: map[string]interface{}{
"foo": "qux",
},
TransparentProxy: &ConsulTransparentProxy{
UID: "101",
OutboundPort: 15001,
ExcludeInboundPorts: []string{"www", "9000"},
ExcludeOutboundPorts: []uint16{4443},
ExcludeOutboundCIDRs: []string{"10.0.0.0/8"},
ExcludeUIDs: []string{"1", "10"},
NoDNS: true,
},
},
},
Gateway: &ConsulGateway{
@@ -3924,6 +3933,92 @@ func TestTaskGroupDiff(t *testing.T) {
},
},
},
{
Type: DiffTypeAdded,
Name: "TransparentProxy",
Objects: []*ObjectDiff{
{
Type: DiffTypeAdded,
Name: "ExcludeInboundPorts",
Fields: []*FieldDiff{
{
Type: DiffTypeAdded,
Name: "ExcludeInboundPorts",
Old: "",
New: "9000",
},
{
Type: DiffTypeAdded,
Name: "ExcludeInboundPorts",
Old: "",
New: "www",
},
},
},
{
Type: DiffTypeAdded,
Name: "ExcludeOutboundPorts",
Fields: []*FieldDiff{
{
Type: DiffTypeAdded,
Name: "ExcludeOutboundPorts",
Old: "",
New: "4443",
},
},
},
{
Type: DiffTypeAdded,
Name: "ExcludeOutboundCIDRs",
Fields: []*FieldDiff{
{
Type: DiffTypeAdded,
Name: "ExcludeOutboundCIDRs",
Old: "",
New: "10.0.0.0/8",
},
},
},
{
Type: DiffTypeAdded,
Name: "ExcludeUIDs",
Fields: []*FieldDiff{
{
Type: DiffTypeAdded,
Name: "ExcludeUIDs",
Old: "",
New: "1",
},
{
Type: DiffTypeAdded,
Name: "ExcludeUIDs",
Old: "",
New: "10",
},
},
},
},
Fields: []*FieldDiff{
{
Type: DiffTypeAdded,
Name: "NoDNS",
Old: "",
New: "true",
},
{
Type: DiffTypeAdded,
Name: "OutboundPort",
Old: "",
New: "15001",
},
{
Type: DiffTypeAdded,
Name: "UID",
Old: "",
New: "101",
},
},
},
{
Type: DiffTypeAdded,
Name: "Config",
@@ -10024,6 +10119,10 @@ func TestServicesDiff(t *testing.T) {
},
Objects: nil,
},
{
Type: DiffTypeNone,
Name: "TransparentProxy",
},
},
},
},

View File

@@ -958,6 +958,7 @@ func hashConnect(h hash.Hash, connect *ConsulConnect) {
hashString(h, p.LocalServiceAddress)
hashString(h, strconv.Itoa(p.LocalServicePort))
hashConfig(h, p.Config)
hashTProxy(h, p.TransparentProxy)
for _, upstream := range p.Upstreams {
hashString(h, upstream.DestinationName)
hashString(h, upstream.DestinationNamespace)
@@ -1015,6 +1016,22 @@ func hashConfig(h hash.Hash, c map[string]interface{}) {
_, _ = fmt.Fprintf(h, "%v", c)
}
func hashTProxy(h hash.Hash, tp *ConsulTransparentProxy) {
if tp == nil {
return
}
hashStringIfNonEmpty(h, tp.UID)
hashIntIfNonZero(h, "OutboundPort", int(tp.OutboundPort))
hashTags(h, tp.ExcludeInboundPorts)
for _, port := range tp.ExcludeOutboundPorts {
hashIntIfNonZero(h, "ExcludeOutboundPorts", int(port))
}
hashTags(h, tp.ExcludeOutboundCIDRs)
hashTags(h, tp.ExcludeUIDs)
hashBool(h, tp.NoDNS, "NoDNS")
}
// Equal returns true if the structs are recursively equal.
func (s *Service) Equal(o *Service) bool {
if s == nil || o == nil {
@@ -1187,6 +1204,14 @@ func (c *ConsulConnect) IsMesh() bool {
return c.IsGateway() && c.Gateway.Mesh != nil
}
// HasTransparentProxy checks if a service with a Connect sidecar has a
// transparent proxy configuration
func (c *ConsulConnect) HasTransparentProxy() bool {
return c.HasSidecar() &&
c.SidecarService.Proxy != nil &&
c.SidecarService.Proxy.TransparentProxy != nil
}
// Validate that the Connect block represents exactly one of:
// - Connect non-native service sidecar proxy
// - Connect native service
@@ -1201,6 +1226,11 @@ func (c *ConsulConnect) Validate() error {
count := 0
if c.HasSidecar() {
if c.HasTransparentProxy() {
if err := c.SidecarService.Proxy.TransparentProxy.Validate(); err != nil {
return err
}
}
count++
}
@@ -1222,7 +1252,8 @@ func (c *ConsulConnect) Validate() error {
}
}
// The Native and Sidecar cases are validated up at the service level.
// Checking against the surrounding task group is validated up at the
// service level or job endpint connect validation hook
return nil
}
@@ -1509,6 +1540,11 @@ type ConsulProxy struct {
// used by task-group level service checks using HTTP or gRPC protocols.
Expose *ConsulExposeConfig
// TransparentProxy configures the Envoy sidecar to use "transparent
// proxying", which creates IP tables rules inside the network namespace to
// ensure traffic flows thru the Envoy proxy
TransparentProxy *ConsulTransparentProxy
// Config is a proxy configuration. It is opaque to Nomad and passed
// directly to Consul.
Config map[string]interface{}
@@ -1525,6 +1561,7 @@ func (p *ConsulProxy) Copy() *ConsulProxy {
LocalServicePort: p.LocalServicePort,
Expose: p.Expose.Copy(),
Upstreams: slices.Clone(p.Upstreams),
TransparentProxy: p.TransparentProxy.Copy(),
Config: maps.Clone(p.Config),
}
}
@@ -1551,6 +1588,10 @@ func (p *ConsulProxy) Equal(o *ConsulProxy) bool {
return false
}
if !p.TransparentProxy.Equal(o.TransparentProxy) {
return false
}
// envoy config, use reflect
if !reflect.DeepEqual(p.Config, o.Config) {
return false

View File

@@ -432,6 +432,15 @@ func TestService_Hash(t *testing.T) {
LocalBindPort: 29000,
Config: map[string]any{"foo": "bar"},
}},
TransparentProxy: &ConsulTransparentProxy{
UID: "101",
OutboundPort: 15001,
ExcludeInboundPorts: []string{"www", "9000"},
ExcludeOutboundPorts: []uint16{4443},
ExcludeOutboundCIDRs: []string{"10.0.0.0/8"},
ExcludeUIDs: []string{"1", "10"},
NoDNS: true,
},
},
Meta: map[string]string{
"test-key": "test-value",
@@ -529,6 +538,54 @@ func TestService_Hash(t *testing.T) {
t.Run("mod connect sidecar proxy upstream config", func(t *testing.T) {
try(t, func(s *svc) { s.Connect.SidecarService.Proxy.Upstreams[0].Config = map[string]any{"foo": "baz"} })
})
t.Run("mod connect transparent proxy removed", func(t *testing.T) {
try(t, func(s *svc) {
s.Connect.SidecarService.Proxy.TransparentProxy = nil
})
})
t.Run("mod connect transparent proxy uid", func(t *testing.T) {
try(t, func(s *svc) {
s.Connect.SidecarService.Proxy.TransparentProxy.UID = "42"
})
})
t.Run("mod connect transparent proxy outbound port", func(t *testing.T) {
try(t, func(s *svc) {
s.Connect.SidecarService.Proxy.TransparentProxy.OutboundPort = 42
})
})
t.Run("mod connect transparent proxy inbound ports", func(t *testing.T) {
try(t, func(s *svc) {
s.Connect.SidecarService.Proxy.TransparentProxy.ExcludeInboundPorts = []string{"443"}
})
})
t.Run("mod connect transparent proxy outbound ports", func(t *testing.T) {
try(t, func(s *svc) {
s.Connect.SidecarService.Proxy.TransparentProxy.ExcludeOutboundPorts = []uint16{42}
})
})
t.Run("mod connect transparent proxy outbound cidr", func(t *testing.T) {
try(t, func(s *svc) {
s.Connect.SidecarService.Proxy.TransparentProxy.ExcludeOutboundCIDRs = []string{"192.168.1.0/24"}
})
})
t.Run("mod connect transparent proxy exclude uids", func(t *testing.T) {
try(t, func(s *svc) {
s.Connect.SidecarService.Proxy.TransparentProxy.ExcludeUIDs = []string{"42"}
})
})
t.Run("mod connect transparent proxy no dns", func(t *testing.T) {
try(t, func(s *svc) {
s.Connect.SidecarService.Proxy.TransparentProxy.NoDNS = false
})
})
}
func TestConsulConnect_Validate(t *testing.T) {