networking: option to enable ipv6 on bridge network (#23882)

by setting bridge_network_subnet_ipv6 in client config

Co-authored-by: Martina Santangelo <martina.santangelo@hashicorp.com>
This commit is contained in:
Daniel Bennett
2024-09-04 10:17:10 -05:00
committed by GitHub
parent ce3e159ee8
commit 2f5cf8efae
16 changed files with 385 additions and 37 deletions

3
.changelog/23882.txt Normal file
View File

@@ -0,0 +1,3 @@
```release-note:improvement
networking: IPv6 can now be enabled on the Nomad bridge network mode
```

View File

@@ -22,6 +22,7 @@ type NomadBridgeConfig struct {
BridgeName string
AdminChainName string
IPv4Subnet string
IPv6Subnet string
HairpinMode bool
ConsulCNI bool
}
@@ -40,6 +41,10 @@ func NewNomadBridgeConflist(conf NomadBridgeConfig) Conflist {
ipRoutes := []Route{
{Dst: "0.0.0.0/0"},
}
if conf.IPv6Subnet != "" {
ipRanges = append(ipRanges, []Range{{Subnet: conf.IPv6Subnet}})
ipRoutes = append(ipRoutes, Route{Dst: "::/0"})
}
plugins := []any{
Generic{

View File

@@ -190,7 +190,10 @@ func newNetworkConfigurator(log hclog.Logger, alloc *structs.Allocation, config
switch {
case netMode == "bridge":
c, err := newBridgeNetworkConfigurator(log, alloc, config.BridgeNetworkName, config.BridgeNetworkAllocSubnet, config.BridgeNetworkHairpinMode, config.CNIPath, ignorePortMappingHostIP, config.Node)
c, err := newBridgeNetworkConfigurator(log, alloc,
config.BridgeNetworkName, config.BridgeNetworkAllocSubnet, config.BridgeNetworkAllocSubnetIPv6, config.CNIPath,
config.BridgeNetworkHairpinMode, ignorePortMappingHostIP,
config.Node)
if err != nil {
return nil, err
}

View File

@@ -31,31 +31,33 @@ const (
// shared bridge, configures masquerading for egress traffic and port mapping
// for ingress
type bridgeNetworkConfigurator struct {
cni *cniNetworkConfigurator
allocSubnet string
bridgeName string
hairpinMode bool
cni *cniNetworkConfigurator
allocSubnetIPv6 string
allocSubnetIPv4 string
bridgeName string
hairpinMode bool
newIPTables func(structs.NodeNetworkAF) (IPTablesChain, error)
logger hclog.Logger
}
func newBridgeNetworkConfigurator(log hclog.Logger, alloc *structs.Allocation, bridgeName, ipRange string, hairpinMode bool, cniPath string, ignorePortMappingHostIP bool, node *structs.Node) (*bridgeNetworkConfigurator, error) {
func newBridgeNetworkConfigurator(log hclog.Logger, alloc *structs.Allocation, bridgeName, ipv4Range, ipv6Range, cniPath string, hairpinMode, ignorePortMappingHostIP bool, node *structs.Node) (*bridgeNetworkConfigurator, error) {
b := &bridgeNetworkConfigurator{
bridgeName: bridgeName,
allocSubnet: ipRange,
hairpinMode: hairpinMode,
newIPTables: newIPTablesChain,
logger: log,
bridgeName: bridgeName,
hairpinMode: hairpinMode,
allocSubnetIPv4: ipv4Range,
allocSubnetIPv6: ipv6Range,
newIPTables: newIPTablesChain,
logger: log,
}
if b.bridgeName == "" {
b.bridgeName = defaultNomadBridgeName
}
if b.allocSubnet == "" {
b.allocSubnet = defaultNomadAllocSubnet
if b.allocSubnetIPv4 == "" {
b.allocSubnetIPv4 = defaultNomadAllocSubnet
}
var netCfg []byte
@@ -95,12 +97,22 @@ func newBridgeNetworkConfigurator(log hclog.Logger, alloc *structs.Allocation, b
// ensureForwardingRules ensures that a forwarding rule is added to iptables
// to allow traffic inbound to the bridge network
func (b *bridgeNetworkConfigurator) ensureForwardingRules() error {
if b.allocSubnetIPv6 != "" {
ip6t, err := b.newIPTables(structs.NodeNetworkAF_IPv6)
if err != nil {
return err
}
if err = ensureChainRule(ip6t, b.bridgeName, b.allocSubnetIPv6); err != nil {
return err
}
}
ipt, err := b.newIPTables(structs.NodeNetworkAF_IPv4)
if err != nil {
return err
}
if err = ensureChainRule(ipt, b.bridgeName, b.allocSubnet); err != nil {
if err = ensureChainRule(ipt, b.bridgeName, b.allocSubnetIPv4); err != nil {
return err
}
@@ -125,7 +137,8 @@ func buildNomadBridgeNetConfig(b bridgeNetworkConfigurator, withConsulCNI bool)
conf := cni.NewNomadBridgeConflist(cni.NomadBridgeConfig{
BridgeName: b.bridgeName,
AdminChainName: cniAdminChainName,
IPv4Subnet: b.allocSubnet,
IPv4Subnet: b.allocSubnetIPv4,
IPv6Subnet: b.allocSubnetIPv6,
HairpinMode: b.hairpinMode,
ConsulCNI: withConsulCNI,
})

View File

@@ -5,11 +5,17 @@ package allocrunner
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"testing"
"github.com/coreos/go-iptables/iptables"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/nomad/ci"
"github.com/hashicorp/nomad/nomad/mock"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/shoenig/test/must"
)
@@ -25,29 +31,37 @@ func Test_buildNomadBridgeNetConfig(t *testing.T) {
b: &bridgeNetworkConfigurator{},
},
{
name: "ipv6",
b: &bridgeNetworkConfigurator{
bridgeName: defaultNomadBridgeName,
allocSubnetIPv6: "3fff:cab0:0d13::/120",
allocSubnetIPv4: defaultNomadAllocSubnet,
},
},
{
name: "hairpin",
b: &bridgeNetworkConfigurator{
bridgeName: defaultNomadBridgeName,
allocSubnet: defaultNomadAllocSubnet,
hairpinMode: true,
bridgeName: defaultNomadBridgeName,
allocSubnetIPv4: defaultNomadAllocSubnet,
hairpinMode: true,
},
},
{
name: "bad_input",
b: &bridgeNetworkConfigurator{
bridgeName: `bad"`,
allocSubnet: defaultNomadAllocSubnet,
hairpinMode: true,
bridgeName: `bad"`,
allocSubnetIPv4: defaultNomadAllocSubnet,
hairpinMode: true,
},
},
{
name: "consul-cni",
withConsulCNI: true,
b: &bridgeNetworkConfigurator{
bridgeName: defaultNomadBridgeName,
allocSubnet: defaultNomadAllocSubnet,
hairpinMode: true,
bridgeName: defaultNomadBridgeName,
allocSubnetIPv4: defaultNomadAllocSubnet,
hairpinMode: true,
},
},
}
@@ -67,3 +81,118 @@ func Test_buildNomadBridgeNetConfig(t *testing.T) {
})
}
}
func TestBridgeNetworkConfigurator_newIPTables_default(t *testing.T) {
t.Parallel()
b, err := newBridgeNetworkConfigurator(hclog.Default(),
mock.MinAlloc(),
"", "", "", "",
false, false,
mock.Node())
must.NoError(t, err)
for family, expect := range map[structs.NodeNetworkAF]iptables.Protocol{
"ipv6": iptables.ProtocolIPv6,
"ipv4": iptables.ProtocolIPv4,
"other": iptables.ProtocolIPv4,
} {
t.Run(string(family), func(t *testing.T) {
mgr, err := b.newIPTables(family)
must.NoError(t, err)
ipt := mgr.(*iptables.IPTables)
must.Eq(t, expect, ipt.Proto(), must.Sprint("unexpected ip family"))
})
}
}
func TestBridgeNetworkConfigurator_ensureForwardingRules(t *testing.T) {
t.Parallel()
newMockIPTables := func(b *bridgeNetworkConfigurator) (*mockIPTablesChain, *mockIPTablesChain) {
ipt := &mockIPTablesChain{}
ip6t := &mockIPTablesChain{}
b.newIPTables = func(fam structs.NodeNetworkAF) (IPTablesChain, error) {
switch fam {
case "ipv6":
return ip6t, nil
case "ipv4":
return ipt, nil
}
return nil, fmt.Errorf("unknown fam %q in newMockIPTables", fam)
}
return ipt, ip6t
}
cases := []struct {
name string
bridgeName, ip4, ip6 string
expectIP4Rules []string
expectIP6Rules []string
ip4Err, ip6Err error
}{
{
name: "defaults",
expectIP4Rules: []string{"-o", defaultNomadBridgeName, "-d", defaultNomadAllocSubnet, "-j", "ACCEPT"},
},
{
name: "configured",
bridgeName: "golden-gate",
ip4: "a.b.c.d/z",
ip6: "aa:bb:cc:dd/z",
expectIP4Rules: []string{"-o", "golden-gate", "-d", "a.b.c.d/z", "-j", "ACCEPT"},
expectIP6Rules: []string{"-o", "golden-gate", "-d", "aa:bb:cc:dd/z", "-j", "ACCEPT"},
},
{
name: "ip4error",
ip4Err: errors.New("test ip4error"),
},
{
name: "ip6error",
ip6: "aa:bb:cc:dd/z",
ip6Err: errors.New("test ip6error"),
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
b, err := newBridgeNetworkConfigurator(hclog.Default(),
mock.MinAlloc(),
tc.bridgeName, tc.ip4, tc.ip6, "",
false, false,
mock.Node())
must.NoError(t, err)
ipt, ip6t := newMockIPTables(b)
ipt.newChainErr = tc.ip4Err
ip6t.newChainErr = tc.ip6Err
// method under test
err = b.ensureForwardingRules()
if tc.ip6Err != nil {
must.ErrorIs(t, err, tc.ip6Err)
return
}
if tc.ip4Err != nil {
must.ErrorIs(t, err, tc.ip4Err)
return
}
must.NoError(t, err)
must.Eq(t, ipt.chain, cniAdminChainName)
must.Eq(t, ipt.table, "filter")
must.Eq(t, ipt.rules, tc.expectIP4Rules)
if tc.expectIP6Rules != nil {
must.Eq(t, ip6t.chain, cniAdminChainName)
must.Eq(t, ip6t.table, "filter")
must.Eq(t, ip6t.rules, tc.expectIP6Rules)
} else {
must.Eq(t, "", ip6t.chain, must.Sprint("expect empty ip6tables chain"))
must.Eq(t, "", ip6t.table, must.Sprint("expect empty ip6tables table"))
must.Len(t, 0, ip6t.rules, must.Sprint("expect empty ip6tables rules"))
}
})
}
}

View File

@@ -513,8 +513,20 @@ func (c *cniNetworkConfigurator) Teardown(ctx context.Context, alloc *structs.Al
portMap := getPortMapping(alloc, c.ignorePortMappingHostIP)
if err := c.cni.Remove(ctx, alloc.ID, spec.Path, cni.WithCapabilityPortMap(portMap.ports)); err != nil {
c.logger.Warn("error from cni.Remove; attempting manual iptables cleanup", "err", err)
// best effort cleanup ipv6
ipt, iptErr := c.newIPTables(structs.NodeNetworkAF_IPv6)
if iptErr != nil {
c.logger.Debug("failed to detect ip6tables: %v", iptErr)
} else {
if err := c.forceCleanup(ipt, alloc.ID); err != nil {
c.logger.Warn("ip6tables: %v", err)
}
}
// create a real handle to iptables
ipt, iptErr := c.newIPTables(structs.NodeNetworkAF_IPv4)
ipt, iptErr = c.newIPTables(structs.NodeNetworkAF_IPv4)
if iptErr != nil {
return fmt.Errorf("failed to detect iptables: %w", iptErr)
}
@@ -560,7 +572,8 @@ func (c *cniNetworkConfigurator) forceCleanup(ipt IPTablesCleanup, allocID strin
// no rule found for our allocation, just give up
if ruleToPurge == "" {
return fmt.Errorf("failed to find postrouting rule for alloc %s", allocID)
c.logger.Info("iptables cleanup: did not find postrouting rule for alloc", "alloc_id", allocID)
return nil
}
// re-create the rule we need to delete, as tokens

View File

@@ -318,7 +318,7 @@ func TestCNI_forceCleanup(t *testing.T) {
},
}
err := c.forceCleanup(ipt, "2dd71cac-2b1e-ff08-167c-735f7f9f4964")
must.EqError(t, err, "failed to find postrouting rule for alloc 2dd71cac-2b1e-ff08-167c-735f7f9f4964")
must.NoError(t, err, must.Sprint("absent rule should not error"))
})
t.Run("list error", func(t *testing.T) {

View File

@@ -0,0 +1,52 @@
{
"cniVersion": "0.4.0",
"name": "nomad",
"plugins": [
{
"type": "loopback"
},
{
"type": "bridge",
"bridge": "nomad",
"ipMasq": true,
"isGateway": true,
"forceAddress": true,
"hairpinMode": false,
"ipam": {
"type": "host-local",
"ranges": [
[
{
"subnet": "172.26.64.0/20"
}
],
[
{
"subnet": "3fff:cab0:0d13::/120"
}
]
],
"routes": [
{
"dst": "0.0.0.0/0"
},
{
"dst": "::/0"
}
]
}
},
{
"type": "firewall",
"backend": "iptables",
"iptablesAdminChainName": "NOMAD-ADMIN"
},
{
"type": "portmap",
"capabilities": {
"portMappings": true
},
"snat": true
}
]
}

View File

@@ -291,9 +291,14 @@ type Config struct {
// BridgeNetworkAllocSubnet is the IP subnet to use for address allocation
// for allocations in bridge networking mode. Subnet must be in CIDR
// notation
// notation and must be an IPv4 address.
BridgeNetworkAllocSubnet string
// BridgeNetworkAllocSubnetIPv6 is the IP subnet to use for address allocation
// for allocations in bridge networking mode. Subnet must be in CIDR
// notation and must be an IPv6 address.
BridgeNetworkAllocSubnetIPv6 string
// HostVolumes is a map of the configured host volumes by name.
HostVolumes map[string]*structs.ClientHostVolumeConfig

View File

@@ -889,7 +889,30 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) {
conf.CNIPath = agentConfig.Client.CNIPath
conf.CNIConfigDir = agentConfig.Client.CNIConfigDir
conf.BridgeNetworkName = agentConfig.Client.BridgeNetworkName
conf.BridgeNetworkAllocSubnet = agentConfig.Client.BridgeNetworkSubnet
ipv4Subnet := agentConfig.Client.BridgeNetworkSubnet
if ipv4Subnet != "" {
ip, _, err := net.ParseCIDR(ipv4Subnet)
if err != nil {
return nil, fmt.Errorf("invalid bridge_network_subnet: %w", err)
}
// it's a valid IP, so now make sure it is ipv4
if ip.To4() == nil {
return nil, fmt.Errorf("invalid bridge_network_subnet: not an IPv4 address: %s", ipv4Subnet)
}
conf.BridgeNetworkAllocSubnet = ipv4Subnet
}
ipv6Subnet := agentConfig.Client.BridgeNetworkSubnetIPv6
if ipv6Subnet != "" {
ip, _, err := net.ParseCIDR(ipv6Subnet)
if err != nil {
return nil, fmt.Errorf("invalid bridge_network_subnet_ipv6: %w", err)
}
// it's valid, so now make sure it's *not* ipv4
if ip.To4() != nil {
return nil, fmt.Errorf("invalid bridge_network_subnet_ipv6: not an IPv6 address: %s", ipv6Subnet)
}
conf.BridgeNetworkAllocSubnetIPv6 = ipv6Subnet
}
conf.BridgeNetworkHairpinMode = agentConfig.Client.BridgeNetworkHairpinMode
for _, hn := range agentConfig.Client.HostNetworks {

View File

@@ -13,6 +13,7 @@ import (
"time"
"github.com/hashicorp/nomad/ci"
clientconfig "github.com/hashicorp/nomad/client/config"
cstructs "github.com/hashicorp/nomad/client/structs"
"github.com/hashicorp/nomad/helper/pointer"
"github.com/hashicorp/nomad/helper/testlog"
@@ -677,6 +678,93 @@ func TestAgent_ServerConfig_RaftProtocol_3(t *testing.T) {
}
}
func TestConvertClientConfig(t *testing.T) {
ci.Parallel(t)
cases := []struct {
name string
// modConfig modifies the agent config before passing to convertClientConfig()
modConfig func(*Config)
// assert makes assertions about the resulting client config
assert func(*testing.T, *clientconfig.Config)
expectErr string
}{
{
name: "default",
assert: func(t *testing.T, cc *clientconfig.Config) {
must.Eq(t, "global", cc.Region)
},
},
{
name: "ipv4 bridge subnet",
modConfig: func(c *Config) {
c.Client.BridgeNetworkSubnet = "10.0.0.0/24"
},
assert: func(t *testing.T, cc *clientconfig.Config) {
must.Eq(t, "10.0.0.0/24", cc.BridgeNetworkAllocSubnet)
},
},
{
name: "invalid ipv4 bridge subnet",
modConfig: func(c *Config) {
c.Client.BridgeNetworkSubnet = "invalid-ip4"
},
expectErr: "invalid bridge_network_subnet: invalid CIDR address: invalid-ip4",
},
{
name: "invalid ipv4 bridge subnet is ipv6",
modConfig: func(c *Config) {
c.Client.BridgeNetworkSubnet = "fd00:a110:c8::/120"
},
expectErr: "invalid bridge_network_subnet: not an IPv4 address: fd00:a110:c8::/120",
},
{
name: "ipv6 bridge subnet",
modConfig: func(c *Config) {
c.Client.BridgeNetworkSubnetIPv6 = "fd00:a110:c8::/120"
},
assert: func(t *testing.T, cc *clientconfig.Config) {
must.Eq(t, "fd00:a110:c8::/120", cc.BridgeNetworkAllocSubnetIPv6)
},
},
{
name: "invalid ipv6 bridge subnet",
modConfig: func(c *Config) {
c.Client.BridgeNetworkSubnetIPv6 = "invalid-ip6"
},
expectErr: "invalid bridge_network_subnet_ipv6: invalid CIDR address: invalid-ip6",
},
{
name: "invalid ipv6 bridge subnet is ipv4",
modConfig: func(c *Config) {
c.Client.BridgeNetworkSubnetIPv6 = "10.0.0.1/24"
},
expectErr: "invalid bridge_network_subnet_ipv6: not an IPv6 address: 10.0.0.1/24",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
c := DefaultConfig()
if tc.modConfig != nil {
tc.modConfig(c)
}
// method under test
cc, err := convertClientConfig(c)
if tc.expectErr != "" {
must.ErrorContains(t, err, tc.expectErr)
} else {
must.NoError(t, err)
}
if tc.assert != nil {
tc.assert(t, cc)
}
})
}
}
func TestAgent_ClientConfig_discovery(t *testing.T) {
ci.Parallel(t)
conf := DefaultConfig()

View File

@@ -357,11 +357,16 @@ type ClientConfig struct {
// bridge network mode
BridgeNetworkName string `hcl:"bridge_network_name"`
// BridgeNetworkSubnet is the subnet to allocate IP addresses from when
// BridgeNetworkSubnet is the subnet to allocate IPv4 addresses from when
// creating allocations with bridge networking mode. This range is local to
// the host
BridgeNetworkSubnet string `hcl:"bridge_network_subnet"`
// BridgeNetworkSubnetIPv6 is the subnet to allocate IPv6 addresses when
// creating allocations with bridge networking mode. This range is local to
// the host
BridgeNetworkSubnetIPv6 string `hcl:"bridge_network_subnet_ipv6"`
// BridgeNetworkHairpinMode is whether or not to enable hairpin mode on the
// internal bridge network
BridgeNetworkHairpinMode bool `hcl:"bridge_network_hairpin_mode"`
@@ -2435,7 +2440,9 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig {
if b.BridgeNetworkSubnet != "" {
result.BridgeNetworkSubnet = b.BridgeNetworkSubnet
}
if b.BridgeNetworkSubnetIPv6 != "" {
result.BridgeNetworkSubnetIPv6 = b.BridgeNetworkSubnetIPv6
}
if b.BridgeNetworkHairpinMode {
result.BridgeNetworkHairpinMode = true
}

View File

@@ -92,9 +92,10 @@ var basicConfig = &Config{
HostVolumes: []*structs.ClientHostVolumeConfig{
{Name: "tmp", Path: "/tmp"},
},
CNIPath: "/tmp/cni_path",
BridgeNetworkName: "custom_bridge_name",
BridgeNetworkSubnet: "custom_bridge_subnet",
CNIPath: "/tmp/cni_path",
BridgeNetworkName: "custom_bridge_name",
BridgeNetworkSubnet: "custom_bridge_subnet",
BridgeNetworkSubnetIPv6: "custom_bridge_subnet_ipv6",
},
Server: &ServerConfig{
Enabled: true,

View File

@@ -102,9 +102,10 @@ client {
path = "/tmp"
}
cni_path = "/tmp/cni_path"
bridge_network_name = "custom_bridge_name"
bridge_network_subnet = "custom_bridge_subnet"
cni_path = "/tmp/cni_path"
bridge_network_name = "custom_bridge_name"
bridge_network_subnet = "custom_bridge_subnet"
bridge_network_subnet_ipv6 = "custom_bridge_subnet_ipv6"
}
server {

View File

@@ -76,6 +76,7 @@
"alloc_mounts_dir": "/tmp/mounts",
"bridge_network_name": "custom_bridge_name",
"bridge_network_subnet": "custom_bridge_subnet",
"bridge_network_subnet_ipv6": "custom_bridge_subnet_ipv6",
"chroot_env": [
{
"/opt/myapp/bin": "/bin",

View File

@@ -178,6 +178,10 @@ client {
- `bridge_network_subnet` `(string: "172.26.64.0/20")` - Specifies the subnet
which the client will use to allocate IP addresses from.
- `bridge_network_subnet_ipv6` `(string: "")` - Enables IPv6 on Nomad's bridge
network by specifying the subnet which the client will use to allocate IPv6
addresses.
- `bridge_network_hairpin_mode` `(bool: false)` - Specifies if hairpin mode
is enabled on the network bridge created by Nomad for allocations running
with bridge networking mode on this client. You may use the corresponding