diff --git a/client/allocrunner/alloc_runner.go b/client/allocrunner/alloc_runner.go index 21d97fdfa..39fbe01e6 100644 --- a/client/allocrunner/alloc_runner.go +++ b/client/allocrunner/alloc_runner.go @@ -185,7 +185,7 @@ func NewAllocRunner(config *Config) (*allocRunner, error) { ar.allocDir = allocdir.NewAllocDir(ar.logger, filepath.Join(config.ClientConfig.AllocDir, alloc.ID)) // Initialize the runners hooks. - if err := ar.initRunnerHooks(); err != nil { + if err := ar.initRunnerHooks(config.ClientConfig); err != nil { return nil, err } diff --git a/client/allocrunner/alloc_runner_hooks.go b/client/allocrunner/alloc_runner_hooks.go index 6e1cc8501..2517ee3ae 100644 --- a/client/allocrunner/alloc_runner_hooks.go +++ b/client/allocrunner/alloc_runner_hooks.go @@ -6,6 +6,7 @@ import ( multierror "github.com/hashicorp/go-multierror" "github.com/hashicorp/nomad/client/allocrunner/interfaces" + clientconfig "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/drivers" ) @@ -94,7 +95,7 @@ func (a *allocHealthSetter) SetHealth(healthy, isDeploy bool, trackerTaskEvents } // initRunnerHooks intializes the runners hooks. -func (ar *allocRunner) initRunnerHooks() error { +func (ar *allocRunner) initRunnerHooks(config *clientconfig.Config) error { hookLogger := ar.logger.Named("runner_hook") // create health setting shim @@ -109,6 +110,9 @@ func (ar *allocRunner) initRunnerHooks() error { return fmt.Errorf("failed to configure network manager: %v", err) } + // create network configurator + nc := newNetworkConfigurator(ar.Alloc(), config) + // Create the alloc directory hook. This is run first to ensure the // directory path exists for other hooks. ar.runnerHooks = []interfaces.RunnerHook{ @@ -116,7 +120,7 @@ func (ar *allocRunner) initRunnerHooks() error { newUpstreamAllocsHook(hookLogger, ar.prevAllocWatcher), newDiskMigrationHook(hookLogger, ar.prevAllocMigrator, ar.allocDir), newAllocHealthWatcherHook(hookLogger, ar.Alloc(), hs, ar.Listener(), ar.consulClient), - newNetworkHook(hookLogger, ns, ar.Alloc(), nm), + newNetworkHook(hookLogger, ns, ar.Alloc(), nm, nc), } return nil diff --git a/client/allocrunner/network_hook.go b/client/allocrunner/network_hook.go index bda1efc24..7ee885ec2 100644 --- a/client/allocrunner/network_hook.go +++ b/client/allocrunner/network_hook.go @@ -26,15 +26,22 @@ type networkHook struct { // spec described the network namespace and is syncronized by specLock spec *drivers.NetworkIsolationSpec + // networkConfigurator configures the network interfaces, routes, etc once + // the alloc network has been created + networkConfigurator NetworkConfigurator + logger hclog.Logger } -func newNetworkHook(logger hclog.Logger, ns networkIsolationSetter, alloc *structs.Allocation, netManager drivers.DriverNetworkManager) *networkHook { +func newNetworkHook(logger hclog.Logger, ns networkIsolationSetter, + alloc *structs.Allocation, netManager drivers.DriverNetworkManager, + netConfigurator NetworkConfigurator) *networkHook { return &networkHook{ - setter: ns, - alloc: alloc, - manager: netManager, - logger: logger, + setter: ns, + alloc: alloc, + manager: netManager, + networkConfigurator: netConfigurator, + logger: logger, } } @@ -43,15 +50,16 @@ func (h *networkHook) Name() string { } func (h *networkHook) Prerun() error { - if h.manager == nil { - h.logger.Trace("shared network namespaces are not supported on this platform, skipping network hook") - return nil - } tg := h.alloc.Job.LookupTaskGroup(h.alloc.TaskGroup) if len(tg.Networks) == 0 || tg.Networks[0].Mode == "host" || tg.Networks[0].Mode == "" { return nil } + if h.manager == nil || h.networkConfigurator == nil { + h.logger.Trace("shared network namespaces are not supported on this platform, skipping network hook") + return nil + } + spec, err := h.manager.CreateNetwork(h.alloc.ID) if err != nil { return fmt.Errorf("failed to create network for alloc: %v", err) @@ -62,7 +70,7 @@ func (h *networkHook) Prerun() error { h.setter.SetNetworkIsolation(spec) } - if err := ConfigureNetworking(h.alloc, spec); err != nil { + if err := h.networkConfigurator.Setup(h.alloc, spec); err != nil { return fmt.Errorf("failed to configure networking for alloc: %v", err) } return nil @@ -73,7 +81,7 @@ func (h *networkHook) Postrun() error { return nil } - if err := CleanupNetworking(h.alloc, h.spec); err != nil { + if err := h.networkConfigurator.Teardown(h.alloc, h.spec); err != nil { h.logger.Error("failed to cleanup network for allocation, resources may have leaked", "alloc", h.alloc.ID, "error", err) } return h.manager.DestroyNetwork(h.alloc.ID, h.spec) diff --git a/client/allocrunner/network_hook_linux.go b/client/allocrunner/network_hook_linux.go new file mode 100644 index 000000000..cc0b24938 --- /dev/null +++ b/client/allocrunner/network_hook_linux.go @@ -0,0 +1 @@ +package allocrunner diff --git a/client/allocrunner/network_hook_test.go b/client/allocrunner/network_hook_test.go index 611fac5a8..e54f81943 100644 --- a/client/allocrunner/network_hook_test.go +++ b/client/allocrunner/network_hook_test.go @@ -65,7 +65,7 @@ func TestNetworkHook_Prerun_Postrun(t *testing.T) { require := require.New(t) logger := testlog.HCLogger(t) - hook := newNetworkHook(logger, setter, alloc, nm) + hook := newNetworkHook(logger, setter, alloc, nm, &hostNetworkConfigurator{}) require.NoError(hook.Prerun()) require.True(setter.called) require.False(destroyCalled) @@ -76,7 +76,7 @@ func TestNetworkHook_Prerun_Postrun(t *testing.T) { setter.called = false destroyCalled = false alloc.Job.TaskGroups[0].Networks[0].Mode = "host" - hook = newNetworkHook(logger, setter, alloc, nm) + hook = newNetworkHook(logger, setter, alloc, nm, &hostNetworkConfigurator{}) require.NoError(hook.Prerun()) require.False(setter.called) require.False(destroyCalled) diff --git a/client/allocrunner/network_manager_linux.go b/client/allocrunner/network_manager_linux.go index 9d9bd62b6..cc212aa4a 100644 --- a/client/allocrunner/network_manager_linux.go +++ b/client/allocrunner/network_manager_linux.go @@ -1,9 +1,11 @@ package allocrunner import ( + "context" "fmt" "strings" + clientconfig "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/lib/nsutil" "github.com/hashicorp/nomad/client/pluginmanager/drivermanager" "github.com/hashicorp/nomad/nomad/structs" @@ -119,37 +121,12 @@ func netModeToIsolationMode(netMode string) drivers.NetIsolationMode { } } -func getPortMapping(alloc *structs.Allocation) []*nsutil.PortMapping { - ports := []*nsutil.PortMapping{} - for _, network := range alloc.AllocatedResources.Shared.Networks { - for _, port := range append(network.DynamicPorts, network.ReservedPorts...) { - for _, proto := range []string{"tcp", "udp"} { - ports = append(ports, &nsutil.PortMapping{ - Host: port.Value, - Container: port.To, - Proto: proto, - }) - } - } +func newNetworkConfigurator(alloc *structs.Allocation, config *clientconfig.Config) NetworkConfigurator { + tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup) + switch strings.ToLower(tg.Networks[0].Mode) { + case "bridge": + return newBridgeNetworkConfigurator(context.Background(), config.BridgeNetworkName, config.BridgeNetworkAllocSubnet, config.CNIPath) + default: + return &hostNetworkConfigurator{} } - return ports -} - -func ConfigureNetworking(alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error { - - // TODO: CNI support - if err := nsutil.SetupBridgeNetworking(alloc.ID, spec.Path, getPortMapping(alloc)); err != nil { - return err - } - - return nil -} - -func CleanupNetworking(alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error { - if err := nsutil.TeardownBridgeNetworking(alloc.ID, spec.Path, getPortMapping(alloc)); err != nil { - return err - } - - return nil - } diff --git a/client/allocrunner/network_manager_nonlinux.go b/client/allocrunner/network_manager_nonlinux.go index 96a8fde1a..f4cecb6c2 100644 --- a/client/allocrunner/network_manager_nonlinux.go +++ b/client/allocrunner/network_manager_nonlinux.go @@ -3,6 +3,7 @@ package allocrunner import ( + clientconfig "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/pluginmanager/drivermanager" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/drivers" @@ -12,3 +13,7 @@ import ( func newNetworkManager(alloc *structs.Allocation, driverManager drivermanager.Manager) (nm drivers.DriverNetworkManager, err error) { return nil, nil } + +func newNetworkConfigurator(alloc *structs.Allocation, config *clientconfig.Config) NetworkConfigurator { + return &hostNetworkConfigurator{} +} diff --git a/client/allocrunner/networking.go b/client/allocrunner/networking.go new file mode 100644 index 000000000..e4532aef9 --- /dev/null +++ b/client/allocrunner/networking.go @@ -0,0 +1,25 @@ +package allocrunner + +import ( + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/drivers" +) + +// NetworkConfigurator sets up and tears down the interfaces, routes, firewall +// rules, etc for the configured networking mode of the allocation. +type NetworkConfigurator interface { + Setup(*structs.Allocation, *drivers.NetworkIsolationSpec) error + Teardown(*structs.Allocation, *drivers.NetworkIsolationSpec) error +} + +// hostNetworkConfigurator is a noop implementation of a NetworkConfigurator for +// when the alloc join's a client host's network namespace and thus does not +// require further configuration +type hostNetworkConfigurator struct{} + +func (h *hostNetworkConfigurator) Setup(*structs.Allocation, *drivers.NetworkIsolationSpec) error { + return nil +} +func (h *hostNetworkConfigurator) Teardown(*structs.Allocation, *drivers.NetworkIsolationSpec) error { + return nil +} diff --git a/client/allocrunner/networking_bridge_linux.go b/client/allocrunner/networking_bridge_linux.go new file mode 100644 index 000000000..ea3e69c1c --- /dev/null +++ b/client/allocrunner/networking_bridge_linux.go @@ -0,0 +1,172 @@ +package allocrunner + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/containernetworking/cni/libcni" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/drivers" +) + +const ( + // envCNIPath is the environment variable name to use to derive the CNI path + // when it is not explicitly set by the client + envCNIPath = "CNI_PATH" + + // defaultCNIPath is the CNI path to use when it is not set by the client + // and is not set by environment variable + defaultCNIPath = "/opt/cni/bin" + + // defaultNomadBridgeName is the name of the bridge to use when not set by + // the client + defaultNomadBridgeName = "nomad" + + // bridgeNetworkAllocIfName is the name that is set for the interface created + // inside of the alloc network which is connected to the bridge + bridgeNetworkContainerIfName = "eth0" + + // defaultNomadAllocSubnet is the subnet to use for host local ip address + // allocation when not specified by the client + defaultNomadAllocSubnet = "172.26.66.0/23" +) + +// bridgeNetworkConfigurator is a NetworkConfigurator which adds the alloc to a +// shared bridge, configures masquerading for egress traffic and port mapping +// for ingress +type bridgeNetworkConfigurator struct { + ctx context.Context + cniConfig *libcni.CNIConfig + allocSubnet string + bridgeName string +} + +func newBridgeNetworkConfigurator(ctx context.Context, bridgeName, ipRange, cniPath string) *bridgeNetworkConfigurator { + b := &bridgeNetworkConfigurator{ + ctx: ctx, + bridgeName: bridgeName, + allocSubnet: ipRange, + } + if cniPath == "" { + if cniPath = os.Getenv(envCNIPath); cniPath == "" { + cniPath = defaultCNIPath + } + } + b.cniConfig = libcni.NewCNIConfig(filepath.SplitList(cniPath), nil) + + if b.bridgeName == "" { + b.bridgeName = defaultNomadBridgeName + } + + if b.allocSubnet == "" { + b.allocSubnet = defaultNomadAllocSubnet + } + + return b +} + +// Setup calls the CNI plugins with the add action +func (b *bridgeNetworkConfigurator) Setup(alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error { + netconf, err := b.buildNomadNetConfig() + if err != nil { + return err + } + + result, err := b.cniConfig.AddNetworkList(b.ctx, netconf, b.runtimeConf(alloc, spec)) + if result != nil { + result.Print() + } + + return err + +} + +// Teardown calls the CNI plugins with the delete action +func (b *bridgeNetworkConfigurator) Teardown(alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error { + netconf, err := b.buildNomadNetConfig() + if err != nil { + return err + } + + err = b.cniConfig.DelNetworkList(b.ctx, netconf, b.runtimeConf(alloc, spec)) + return err + +} + +// getPortMapping builds a list of portMapping structs that are used as the +// portmapping capability arguments for the portmap CNI plugin +func getPortMapping(alloc *structs.Allocation) []*portMapping { + ports := []*portMapping{} + for _, network := range alloc.AllocatedResources.Shared.Networks { + for _, port := range append(network.DynamicPorts, network.ReservedPorts...) { + for _, proto := range []string{"tcp", "udp"} { + ports = append(ports, &portMapping{ + Host: port.Value, + Container: port.To, + Proto: proto, + }) + } + } + } + return ports +} + +// portMapping is the json representation of the portmapping capability arguments +// for the portmap CNI plugin +type portMapping struct { + Host int `json:"hostPort"` + Container int `json:"containerPort"` + Proto string `json:"protocol"` +} + +// runtimeConf builds the configuration needed by CNI to locate the target netns +func (b *bridgeNetworkConfigurator) runtimeConf(alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) *libcni.RuntimeConf { + return &libcni.RuntimeConf{ + ContainerID: fmt.Sprintf("nomad-%s", alloc.ID[:8]), + NetNS: spec.Path, + IfName: bridgeNetworkContainerIfName, + CapabilityArgs: map[string]interface{}{ + "portMappings": getPortMapping(alloc), + }, + } +} + +// buildNomadNetConfig generates the CNI network configuration for the bridge +// networking mode +func (b *bridgeNetworkConfigurator) buildNomadNetConfig() (*libcni.NetworkConfigList, error) { + rendered := fmt.Sprintf(nomadCNIConfigTemplate, b.bridgeName, b.allocSubnet) + return libcni.ConfListFromBytes([]byte(rendered)) +} + +const nomadCNIConfigTemplate = `{ + "cniVersion": "0.4.0", + "name": "nomad", + "plugins": [ + { + "type": "bridge", + "bridge": "%s", + "isDefaultGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "ranges": [ + [ + { + "subnet": "%s" + } + ] + ] + } + }, + { + "type": "firewall" + }, + { + "type": "portmap", + "capabilities": {"portMappings": true} + } + ] +} +` diff --git a/client/config/config.go b/client/config/config.go index da1980e4e..9bac1803a 100644 --- a/client/config/config.go +++ b/client/config/config.go @@ -221,6 +221,19 @@ type Config struct { // StateDBFactory is used to override stateDB implementations, StateDBFactory state.NewStateDBFunc + + // CNIPath is the path used to search for CNI plugins. Multiple paths can + // be specified with colon delimited + CNIPath string + + // BridgeNetworkName is the name to use for the bridge created in bridge + // networking mode. This defaults to 'nomad' if not set + BridgeNetworkName string + + // BridgeNetworkAllocSubnet is the IP subnet to use for address allocation + // for allocations in bridge networking mode. Subnet must be in CIDR + // notation + BridgeNetworkAllocSubnet string } func (c *Config) Copy() *Config { diff --git a/client/lib/nsutil/bridge_linux.go b/client/lib/nsutil/bridge_linux.go deleted file mode 100644 index 33e6d5f41..000000000 --- a/client/lib/nsutil/bridge_linux.go +++ /dev/null @@ -1,97 +0,0 @@ -package nsutil - -import ( - "context" - "fmt" - "os" - "path/filepath" - - "github.com/containernetworking/cni/libcni" -) - -const ( - EnvCNIPath = "CNI_PATH" -) - -type PortMapping struct { - Host int `json:"hostPort"` - Container int `json:"containerPort"` - Proto string `json:"protocol"` -} - -func SetupBridgeNetworking(allocID string, nsPath string, portMappings []*PortMapping) error { - netconf, err := libcni.ConfListFromBytes([]byte(nomadCNIConfig)) - if err != nil { - return err - } - containerID := fmt.Sprintf("nomad-%s", allocID[:8]) - cninet := libcni.NewCNIConfig(filepath.SplitList(os.Getenv(EnvCNIPath)), nil) - - rt := &libcni.RuntimeConf{ - ContainerID: containerID, - NetNS: nsPath, - IfName: "eth0", - CapabilityArgs: map[string]interface{}{ - "portMappings": portMappings, - }, - } - - result, err := cninet.AddNetworkList(context.TODO(), netconf, rt) - if result != nil { - result.Print() - } - - return err -} - -func TeardownBridgeNetworking(allocID, nsPath string, portMappings []*PortMapping) error { - netconf, err := libcni.ConfListFromBytes([]byte(nomadCNIConfig)) - if err != nil { - return err - } - - containerID := fmt.Sprintf("nomad-%s", allocID[:8]) - cninet := libcni.NewCNIConfig(filepath.SplitList(os.Getenv(EnvCNIPath)), nil) - rt := &libcni.RuntimeConf{ - ContainerID: containerID, - NetNS: nsPath, - IfName: "eth0", - CapabilityArgs: map[string]interface{}{ - "portMappings": portMappings, - }, - } - err = cninet.DelNetworkList(context.TODO(), netconf, rt) - - return err -} - -const nomadCNIConfig = `{ - "cniVersion": "0.4.0", - "name": "nomad", - "plugins": [ - { - "type": "bridge", - "bridge": "nomad", - "isDefaultGateway": true, - "ipMasq": true, - "ipam": { - "type": "host-local", - "ranges": [ - [ - { - "subnet": "172.26.66.0/23" - } - ] - ] - } - }, - { - "type": "firewall" - }, - { - "type": "portmap", - "capabilities": {"portMappings": true} - } - ] -} -` diff --git a/command/agent/agent.go b/command/agent/agent.go index 2a5780234..f0f2cebf5 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -538,6 +538,11 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) { conf.ACLTokenTTL = agentConfig.ACL.TokenTTL conf.ACLPolicyTTL = agentConfig.ACL.PolicyTTL + // Setup networking configration + conf.CNIPath = agentConfig.Client.CNIPath + conf.BridgeNetworkName = agentConfig.Client.BridgeNetworkName + conf.BridgeNetworkAllocSubnet = agentConfig.Client.BridgeNetworkSubnet + return conf, nil } diff --git a/command/agent/config.go b/command/agent/config.go index f06e368d9..fe3f74c4b 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -247,6 +247,19 @@ type ClientConfig struct { // ExtraKeysHCL is used by hcl to surface unexpected keys ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` + + // CNIPath is the path to search for CNI plugins, multiple paths can be + // specified colon delimited + CNIPath string `hcl:"cni_path"` + + // BridgeNetworkName is the name of the bridge to create when using the + // bridge network mode + BridgeNetworkName string `hcl:"bridge_network_name"` + + // BridgeNetworkSubnet is the subnet to allocate IP addresses from when + // creating allocations with bridge networking mode. This range is local to + // the host + BridgeNetworkSubnet string `hcl:"bridge_network_subnet"` } // ACLConfig is configuration specific to the ACL system