kemko/nomad
1
0
Fork 0
mirror of https://github.com/kemko/nomad.git synced 2026-01-01 16:05:42 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add NOMAD_* variables to CNI args. Fixes #23830 (#24319)

Browse Source 
Co-authored-by: Daniel Bennett <dbennett@hashicorp.com>
This commit is contained in:
Florian Apolloner
2024-11-19 21:48:48 +01:00
committed by GitHub
parent 4dfedf1aef
commit 0a343798b6
4 changed files with 78 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.changelog/24319.txt Normal file
View File

@@ -0,0 +1,3 @@
```release-note:improvement
cni: Add Nomad specific workload information to CNI_ARGS
```

24
client/allocrunner/networking_cni.go
View File

@@ -28,6 +28,7 @@ import (
consulIPTables "github.com/hashicorp/consul/sdk/iptables"
log "github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-set/v3"
"github.com/hashicorp/nomad/client/taskenv"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/helper/envoy"
"github.com/hashicorp/nomad/nomad/structs"
@@ -117,6 +118,26 @@ func addCustomCNIArgs(networks []*structs.NetworkResource, cniArgs map[string]st
}
}
func addNomadWorkloadCNIArgs(logger log.Logger, alloc *structs.Allocation, cniArgs map[string]string) {
for key, value := range map[string]string{
// these are the very same keys that are used to build task env vars
taskenv.Region: alloc.Job.Region, // NOMAD_REGION
taskenv.Namespace: alloc.Namespace, // NOMAD_NAMESPACE
taskenv.JobID: alloc.Job.ID, // NOMAD_JOB_ID
taskenv.GroupName: alloc.TaskGroup, // NOMAD_GROUP_NAME
taskenv.AllocID: alloc.ID, // NOMAD_ALLOC_ID
} {
// job ID and group name may contain ";" but CNI_ARGS are ";"-separated
// per the spec, so they may not be used in arg keys or values.
if strings.Contains(value, ";") {
logger.Warn("Skipping CNI arg because it contains a semicolon",
"key", key, "value", value)
} else {
cniArgs[key] = value
}
}
}
// Setup calls the CNI plugins with the add action
func (c *cniNetworkConfigurator) Setup(ctx context.Context, alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) (*structs.AllocNetworkStatus, error) {
if err := c.ensureCNIInitialized(); err != nil {
@@ -133,6 +154,9 @@ func (c *cniNetworkConfigurator) Setup(ctx context.Context, alloc *structs.Alloc
addCustomCNIArgs(tg.Networks, cniArgs)
// Add NOMAD_* after custom args so it cannot be overridden.
addNomadWorkloadCNIArgs(c.logger, alloc, cniArgs)
portMaps := getPortMapping(alloc, c.ignorePortMappingHostIP)
tproxyArgs, err := c.setupTransparentProxyArgs(alloc, spec, portMaps)

49
client/allocrunner/networking_cni_test.go
View File

@@ -137,7 +137,12 @@ func TestSetup(t *testing.T) {
Address: "99.99.99.99",
},
expectArgs: map[string]string{
"IgnoreUnknown": "true",
"IgnoreUnknown": "true",
"NOMAD_ALLOC_ID": "7cd08c6c-86c8-0bfa-f7ca-338466447711",
"NOMAD_GROUP_NAME": "web",
"NOMAD_JOB_ID": "mock-service",
"NOMAD_NAMESPACE": "default",
"NOMAD_REGION": "global",
},
},
{
@@ -148,7 +153,12 @@ func TestSetup(t *testing.T) {
Address: "99.99.99.99",
},
expectArgs: map[string]string{
"IgnoreUnknown": "true",
"IgnoreUnknown": "true",
"NOMAD_ALLOC_ID": "7cd08c6c-86c8-0bfa-f7ca-338466447711",
"NOMAD_GROUP_NAME": "web",
"NOMAD_JOB_ID": "mock-service",
"NOMAD_NAMESPACE": "default",
"NOMAD_REGION": "global",
},
},
{
@@ -174,9 +184,31 @@ func TestSetup(t *testing.T) {
Address: "99.99.99.99",
},
expectArgs: map[string]string{
"IgnoreUnknown": "true",
"first_arg": "example",
"new_arg": "example_2",
"IgnoreUnknown": "true",
"first_arg": "example",
"new_arg": "example_2",
"NOMAD_ALLOC_ID": "7cd08c6c-86c8-0bfa-f7ca-338466447711",
"NOMAD_GROUP_NAME": "web",
"NOMAD_JOB_ID": "mock-service",
"NOMAD_NAMESPACE": "default",
"NOMAD_REGION": "global",
},
},
{
name: "cni workload with invalid job id and namespace",
modAlloc: func(a *structs.Allocation) {
a.Job.ID = "this;does;not;work"
a.Namespace = "no;chance"
},
expectResult: &structs.AllocNetworkStatus{
InterfaceName: "eth0",
Address: "99.99.99.99",
},
expectArgs: map[string]string{
"IgnoreUnknown": "true",
"NOMAD_ALLOC_ID": "7cd08c6c-86c8-0bfa-f7ca-338466447711",
"NOMAD_GROUP_NAME": "web",
"NOMAD_REGION": "global",
},
},
{
@@ -215,6 +247,11 @@ func TestSetup(t *testing.T) {
"IgnoreUnknown": "true",
"extra_arg": "example",
"CONSUL_IPTABLES_CONFIG": `{"ConsulDNSIP":"192.168.1.117","ConsulDNSPort":8600,"ProxyUserID":"101","ProxyInboundPort":9999,"ProxyOutboundPort":15001,"ExcludeInboundPorts":["9002"],"ExcludeOutboundPorts":null,"ExcludeOutboundCIDRs":null,"ExcludeUIDs":null,"NetNS":"/var/run/docker/netns/nonsense-ns","IptablesProvider":null}`,
"NOMAD_ALLOC_ID": "7cd08c6c-86c8-0bfa-f7ca-338466447711",
"NOMAD_GROUP_NAME": "web",
"NOMAD_JOB_ID": "mock-service",
"NOMAD_NAMESPACE": "default",
"NOMAD_REGION": "global",
},
},
}
@@ -246,6 +283,8 @@ func TestSetup(t *testing.T) {
}
alloc := mock.ConnectAlloc()
alloc.ID = "7cd08c6c-86c8-0bfa-f7ca-338466447711" // Fix the ID for easier testing
alloc.Job.ID = "mock-service" // Fix the ID for easier testing
if tc.modAlloc != nil {
tc.modAlloc(alloc)
}

9
website/content/docs/networking/cni.mdx
View File

@@ -250,8 +250,13 @@ To specify that a job should use a CNI network, set the task group's network
that have fingerprinted a CNI configuration with the given name. For example, to
use the configuration named `mynet`, you should set the task group's network
mode to `cni/mynet`. Nodes that have a network configuration defining a network
named `mynet` in their `cni_config_dir` are eligible to run the workload.
named `mynet` in their `cni_config_dir` are eligible to run the workload. Nomad
additionally supplies the following arguments via `CNI_ARGS` to the CNI network:
`NOMAD_REGION`, `NOMAD_NAMESPACE`, `NOMAD_JOB_ID`, `NOMAD_GROUP_NAME`, and
`NOMAD_ALLOC_ID`. Since the `CNI_ARGS` do not allow values to contain a semicolon
Nomad will not set keys where the value contains a semicolon (this could happen
with the job ID). CNI plugins utilizing `NOMAD_*` CNI arguments are advised to
apply a defensive policy or simply error out.
[cni-spec]: https://www.cni.dev/docs/spec/
[cni-plugin-docs]: https://www.cni.dev/plugins/current/