From 6ad648bec8e88cbeb860c96f3b175c55cec8289a Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Wed, 27 Mar 2024 15:11:39 -0500 Subject: [PATCH] networking: Inject implicit constraints on CNI plugins when using bridge mode (#15473) This PR adds a job mutator which injects constraints on the job taskgroups that make use of bridge networking. Creating a bridge network makes use of the CNI plugins: bridge, firewall, host-local, loopback, and portmap. Starting with Nomad 1.5 these plugins are fingerprinted on each node, and as such we can ensure jobs are correctly scheduled only on nodes where they are available, when needed. --- .changelog/15473.txt | 3 + client/allocrunner/networking_bridge_linux.go | 3 +- nomad/job_endpoint_hooks.go | 68 ++++++++++++++++++- nomad/job_endpoint_hooks_test.go | 34 ++++++++++ nomad/structs/job.go | 12 ++++ nomad/structs/structs.go | 7 ++ .../content/docs/upgrade/upgrade-specific.mdx | 18 ++++- 7 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 .changelog/15473.txt diff --git a/.changelog/15473.txt b/.changelog/15473.txt new file mode 100644 index 000000000..7e8a6535b --- /dev/null +++ b/.changelog/15473.txt @@ -0,0 +1,3 @@ +```release-note:improvement +networking: Inject constraints on CNI plugins when using bridge networking +``` diff --git a/client/allocrunner/networking_bridge_linux.go b/client/allocrunner/networking_bridge_linux.go index 89b7b2cb0..908bba96f 100644 --- a/client/allocrunner/networking_bridge_linux.go +++ b/client/allocrunner/networking_bridge_linux.go @@ -148,7 +148,8 @@ func buildNomadBridgeNetConfig(b bridgeNetworkConfigurator) []byte { } // Update website/content/docs/networking/cni.mdx when the bridge configuration -// is modified. +// is modified. If CNI plugins are added or versions need to be updated for new +// fields, add a new constraint to nomad/job_endpoint_hooks.go const nomadCNIConfigTemplate = `{ "cniVersion": "0.4.0", "name": "nomad", diff --git a/nomad/job_endpoint_hooks.go b/nomad/job_endpoint_hooks.go index c58a5a00a..809958c38 100644 --- a/nomad/job_endpoint_hooks.go +++ b/nomad/job_endpoint_hooks.go @@ -15,13 +15,24 @@ import ( "golang.org/x/exp/maps" ) +// Node attributes acquired via fingerprinting. const ( attrVaultVersion = `${attr.vault.version}` attrConsulVersion = `${attr.consul.version}` attrNomadVersion = `${attr.nomad.version}` attrNomadServiceDisco = `${attr.nomad.service_discovery}` + attrBridgeCNI = `${attr.plugins.cni.version.bridge}` + attrFirewallCNI = `${attr.plugins.cni.version.firewall}` + attrHostLocalCNI = `${attr.plugins.cni.version.host-local}` + attrLoopbackCNI = `${attr.plugins.cni.version.loopback}` + attrPortMapCNI = `${attr.plugins.cni.version.portmap}` ) +// cniMinVersion is the version expression for the minimum CNI version supported +// for the CNI container-networking plugins. Support was added at v0.4.0, so +// we set the minimum to that. +const cniMinVersion = ">= 0.4.0" + var ( // vaultConstraint is the implicit constraint added to jobs requesting a // Vault token @@ -78,6 +89,51 @@ var ( RTarget: "linux", Operand: "=", } + + // cniBridgeConstraint is an implicit constraint added to jobs making use + // of bridge networking mode. This is one of the CNI plugins used to support + // bridge networking. + cniBridgeConstraint = &structs.Constraint{ + LTarget: attrBridgeCNI, + RTarget: cniMinVersion, + Operand: structs.ConstraintSemver, + } + + // cniFirewallConstraint is an implicit constraint added to jobs making use + // of bridge networking mode. This is one of the CNI plugins used to support + // bridge networking. + cniFirewallConstraint = &structs.Constraint{ + LTarget: attrFirewallCNI, + RTarget: cniMinVersion, + Operand: structs.ConstraintSemver, + } + + // cniHostLocalConstraint is an implicit constraint added to jobs making use + // of bridge networking mode. This is one of the CNI plugins used to support + // bridge networking. + cniHostLocalConstraint = &structs.Constraint{ + LTarget: attrHostLocalCNI, + RTarget: cniMinVersion, + Operand: structs.ConstraintSemver, + } + + // cniLoopbackConstraint is an implicit constraint added to jobs making use + // of bridge networking mode. This is one of the CNI plugins used to support + // bridge networking. + cniLoopbackConstraint = &structs.Constraint{ + LTarget: attrLoopbackCNI, + RTarget: cniMinVersion, + Operand: structs.ConstraintSemver, + } + + // cniPortMapConstraint is an implicit constraint added to jobs making use + // of bridge networking mode. This is one of the CNI plugins used to support + // bridge networking. + cniPortMapConstraint = &structs.Constraint{ + LTarget: attrPortMapCNI, + RTarget: cniMinVersion, + Operand: structs.ConstraintSemver, + } ) type admissionController interface { @@ -192,12 +248,14 @@ func (jobImpliedConstraints) Mutate(j *structs.Job) (*structs.Job, []error, erro // Identify which task groups are utilizing NUMA resources. numaTaskGroups := j.RequiredNUMA() + bridgeNetworkingTaskGroups := j.RequiredBridgeNetwork() + // Hot path where none of our things require constraints. // // [UPDATE THIS] if you are adding a new constraint thing! if len(signals) == 0 && len(vaultBlocks) == 0 && nativeServiceDisco.Empty() && len(consulServiceDisco) == 0 && - numaTaskGroups.Empty() { + numaTaskGroups.Empty() && bridgeNetworkingTaskGroups.Empty() { return j, nil, nil } @@ -254,6 +312,14 @@ func (jobImpliedConstraints) Mutate(j *structs.Job) (*structs.Job, []error, erro } } } + + if bridgeNetworkingTaskGroups.Contains(tg.Name) { + mutateConstraint(constraintMatcherLeft, tg, cniBridgeConstraint) + mutateConstraint(constraintMatcherLeft, tg, cniFirewallConstraint) + mutateConstraint(constraintMatcherLeft, tg, cniHostLocalConstraint) + mutateConstraint(constraintMatcherLeft, tg, cniLoopbackConstraint) + mutateConstraint(constraintMatcherLeft, tg, cniPortMapConstraint) + } } return j, nil, nil diff --git a/nomad/job_endpoint_hooks_test.go b/nomad/job_endpoint_hooks_test.go index aaf8b7deb..8b73cc7f9 100644 --- a/nomad/job_endpoint_hooks_test.go +++ b/nomad/job_endpoint_hooks_test.go @@ -1160,6 +1160,40 @@ func Test_jobImpliedConstraints_Mutate(t *testing.T) { expectedOutputWarnings: nil, expectedOutputError: nil, }, + { + inputJob: &structs.Job{ + Name: "example", + TaskGroups: []*structs.TaskGroup{ + { + Name: "group-with-bridge", + Networks: []*structs.NetworkResource{ + {Mode: "bridge"}, + }, + }, + }, + }, + expectedOutputJob: &structs.Job{ + Name: "example", + TaskGroups: []*structs.TaskGroup{ + { + Name: "group-with-bridge", + Networks: []*structs.NetworkResource{ + {Mode: "bridge"}, + }, + Constraints: []*structs.Constraint{ + cniBridgeConstraint, + cniFirewallConstraint, + cniHostLocalConstraint, + cniLoopbackConstraint, + cniPortMapConstraint, + }, + }, + }, + }, + expectedOutputWarnings: nil, + expectedOutputError: nil, + name: "task group with bridge network", + }, } for _, tc := range testCases { diff --git a/nomad/structs/job.go b/nomad/structs/job.go index b356dade7..a064c5dc3 100644 --- a/nomad/structs/job.go +++ b/nomad/structs/job.go @@ -132,3 +132,15 @@ func (j *Job) RequiredNUMA() set.Collection[string] { } return result } + +// RequiredBridgeNetwork identifies which task groups, if any, within the job +// contain networks requesting bridge networking. +func (j *Job) RequiredBridgeNetwork() set.Collection[string] { + result := set.New[string](len(j.TaskGroups)) + for _, tg := range j.TaskGroups { + if tg.Networks.Modes().Contains("bridge") { + result.Insert(tg.Name) + } + } + return result +} diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 8e1cc49a7..f58b0bc19 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -3006,6 +3006,13 @@ func (ns Networks) NetIndex(n *NetworkResource) int { return -1 } +// Modes returns the set of network modes used by our NetworkResource blocks. +func (ns Networks) Modes() *set.Set[string] { + return set.FromFunc(ns, func(nr *NetworkResource) string { + return nr.Mode + }) +} + // RequestedDevice is used to request a device for a task. type RequestedDevice struct { // Name is the request name. The possible values are as follows: diff --git a/website/content/docs/upgrade/upgrade-specific.mdx b/website/content/docs/upgrade/upgrade-specific.mdx index 87a039d50..e58a20ae0 100644 --- a/website/content/docs/upgrade/upgrade-specific.mdx +++ b/website/content/docs/upgrade/upgrade-specific.mdx @@ -14,12 +14,26 @@ their upgrades as a result of new features or changed behavior. This page is used to document those details separately from the standard upgrade flow. ## Nomad 1.8.0 + +#### Deprecated Disconnect Fields + Nomad 1.8.0 introduces a `disconnect` block meant to group all the configuration -options related to disconnected client's and server's behavior, causing the -deprecation of the fileds `stop_after_client_disconnect`, `max_client_disconnect` +options related to disconnected client's and server's behavior, causing the +deprecation of the fields `stop_after_client_disconnect`, `max_client_disconnect` and `prevent_reschedule_on_lost`. This block also introduces new options for allocations reconciliation if the client regains connectivity. +#### CNI Constraints + +In Nomad 1.8.0, jobs with `bridge` networking will have constraints added during +job submit that require CNI plugins to be present on the node. Nodes have +fingerprinted the available CNI plugins starting in Nomad 1.5.0. + +If you are upgrading from Nomad 1.5.0 or later to 1.8.0 or later, there's +nothing additional for you to do. It's not recommended to skip more than 2 +versions of Nomad. But if you upgrade from earlier than 1.5.0 to 1.8.0 or later, +you will need to ensure that clients have been upgraded before submitting any +jobs that use `bridge` networking. #### Removal of `raw_exec` option `no_cgroups`