From 7519df8d06e683426ac16b5ac09af0668c8c8aa3 Mon Sep 17 00:00:00 2001 From: Daniel Bennett Date: Wed, 11 Jun 2025 15:56:51 -0400 Subject: [PATCH] task env: add NOMAD_UNIX_ADDR var (#25598) for easier setup when using workload identity + task api --- .changelog/25598.txt | 3 ++ client/taskenv/env.go | 6 ++++ client/taskenv/env_test.go | 8 +++-- e2e/workload_id/input/api-nomad-cli.nomad.hcl | 32 +++++++++++++++++++ e2e/workload_id/taskapi_test.go | 17 ++++++++++ website/content/docs/commands/index.mdx | 7 ++-- website/content/partials/envvars.mdx | 2 ++ 7 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 .changelog/25598.txt create mode 100644 e2e/workload_id/input/api-nomad-cli.nomad.hcl diff --git a/.changelog/25598.txt b/.changelog/25598.txt new file mode 100644 index 000000000..dfc9c7c66 --- /dev/null +++ b/.changelog/25598.txt @@ -0,0 +1,3 @@ +```release-note:improvement +task environment: new NOMAD_UNIX_ADDR env var points to the task API unix socket, for use with workload identity +``` diff --git a/client/taskenv/env.go b/client/taskenv/env.go index 9a8ab9e97..05cb28480 100644 --- a/client/taskenv/env.go +++ b/client/taskenv/env.go @@ -98,6 +98,10 @@ const ( HostAddrPrefix = "NOMAD_HOST_ADDR_" + // UnixAddr is the task api unix socket, in the appropriate format + // for use in a NOMAD_ADDR (i.e. prefixed with "unix://") + UnixAddr = "NOMAD_UNIX_ADDR" + // IpPrefix is the prefix for passing the host IP of a port allocation // to a task. IpPrefix = "NOMAD_IP_" @@ -620,10 +624,12 @@ func (b *Builder) buildEnv(allocDir, localDir, secretsDir string, // Build the Nomad Workload Token if b.workloadTokenDefault != "" { envMap[WorkloadToken] = b.workloadTokenDefault + envMap[UnixAddr] = "unix://" + filepath.Join(secretsDir, "api.sock") } for name, token := range b.workloadTokens { envMap[WorkloadToken+"_"+name] = token + envMap[UnixAddr] = "unix://" + filepath.Join(secretsDir, "api.sock") } // Copy and interpolate task meta diff --git a/client/taskenv/env_test.go b/client/taskenv/env_test.go index 8df01fb72..540be233c 100644 --- a/client/taskenv/env_test.go +++ b/client/taskenv/env_test.go @@ -214,7 +214,7 @@ func TestEnvironment_AsList(t *testing.T) { } env := NewBuilder(n, a, task, "global").SetDriverNetwork( &drivers.DriverNetwork{PortMap: map[string]int{"https": 443}}, - ) + ).SetDefaultWorkloadToken("test-wi-token") act := env.Build().List() exp := []string{ @@ -263,6 +263,8 @@ func TestEnvironment_AsList(t *testing.T) { fmt.Sprintf("NOMAD_ALLOC_ID=%s", a.ID), fmt.Sprintf("NOMAD_SHORT_ALLOC_ID=%s", a.ID[:8]), "NOMAD_ALLOC_INDEX=0", + "NOMAD_TOKEN=test-wi-token", + "NOMAD_UNIX_ADDR=unix://api.sock", } sort.Strings(act) sort.Strings(exp) @@ -342,7 +344,7 @@ func TestEnvironment_AllValues(t *testing.T) { } env := NewBuilder(n, a, task, "global").SetDriverNetwork( &drivers.DriverNetwork{PortMap: map[string]int{"https": 443}}, - ) + ).SetDefaultWorkloadToken("test-wi-token") // Setting the network status ensures we trigger the addNomadAllocNetwork // for the test. @@ -453,6 +455,8 @@ func TestEnvironment_AllValues(t *testing.T) { "NOMAD_ALLOC_INTERFACE_admin": "eth0", "NOMAD_ALLOC_IP_admin": "172.26.64.19", "NOMAD_ALLOC_ADDR_admin": "172.26.64.19:9000", + "NOMAD_TOKEN": "test-wi-token", + "NOMAD_UNIX_ADDR": "unix://api.sock", // Env vars from the host. "LC_CTYPE": "C.UTF-8", diff --git a/e2e/workload_id/input/api-nomad-cli.nomad.hcl b/e2e/workload_id/input/api-nomad-cli.nomad.hcl new file mode 100644 index 000000000..6e0b34239 --- /dev/null +++ b/e2e/workload_id/input/api-nomad-cli.nomad.hcl @@ -0,0 +1,32 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +job "task-api-nomad-cli" { + type = "batch" + + group "grp" { + restart { attempts = 0 } + reschedule { attempts = 0 } + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + task "tsk" { + driver = "raw_exec" + config { + command = "bash" + // "|| true" because failure to get a var makes nomad cli exit 1, + // but for this test, "Variable not found" actually indicates successful + // API connection. + args = ["-xc", "echo $NOMAD_ADDR; nomad var get nothing || true"] + } + env { + NOMAD_ADDR = "${NOMAD_UNIX_ADDR}" + } + identity { # creates unix addr + env = true # provides NOMAD_TOKEN + } + } + } +} diff --git a/e2e/workload_id/taskapi_test.go b/e2e/workload_id/taskapi_test.go index e44dd5abe..189245904 100644 --- a/e2e/workload_id/taskapi_test.go +++ b/e2e/workload_id/taskapi_test.go @@ -11,6 +11,8 @@ import ( "testing" "github.com/hashicorp/nomad/e2e/e2eutil" + "github.com/hashicorp/nomad/e2e/v3/cluster3" + "github.com/hashicorp/nomad/e2e/v3/jobs3" "github.com/hashicorp/nomad/helper/uuid" "github.com/shoenig/test" "github.com/shoenig/test/must" @@ -26,6 +28,7 @@ func TestTaskAPI(t *testing.T) { t.Run("testTaskAPI_Auth", testTaskAPIAuth) t.Run("testTaskAPI_Windows", testTaskAPIWindows) + t.Run("testTaskAPI_NomadCLI", testTaskAPINomadCLI) } func testTaskAPIAuth(t *testing.T) { @@ -129,3 +132,17 @@ func testTaskAPIWindows(t *testing.T) { must.StrHasSuffix(t, `"ok":true}}`, logs) } + +func testTaskAPINomadCLI(t *testing.T) { + cluster3.Establish(t, + cluster3.LinuxClients(1), + ) + sub, _ := jobs3.Submit(t, + "./input/api-nomad-cli.nomad.hcl", + jobs3.WaitComplete("grp"), + ) + logs := sub.TaskLogs("grp", "tsk") + test.StrContains(t, logs.Stdout, "unix:/") // from `echo $NOMAD_ADDR` + test.StrContains(t, logs.Stdout, "secrets/api.sock") + test.StrContains(t, logs.Stderr, "Variable not found") // api success +} diff --git a/website/content/docs/commands/index.mdx b/website/content/docs/commands/index.mdx index 13bc8b105..94c200572 100644 --- a/website/content/docs/commands/index.mdx +++ b/website/content/docs/commands/index.mdx @@ -35,7 +35,8 @@ $ nomad -autocomplete-uninstall Nomad's CLI commands have implied contexts in their naming convention. Because the CLI is most commonly used to manipulate or query jobs, you can assume that any given command is working in that context unless the command name implies -otherwise. For example, the `nomad job run` command runs a new job and the `nomad status` command queries information about existing jobs. Conversely, +otherwise. For example, the `nomad job run` command runs a new job and the +`nomad status` command queries information about existing jobs. Conversely, commands with a prefix in their name likely operate in a different context. Examples include the `nomad agent-info` or `nomad node drain` commands, which operate in the agent or node contexts respectively. @@ -64,7 +65,8 @@ may override these environment variables with individual flags. #### Connection environment variables - `NOMAD_ADDR` - The address of the Nomad server. Defaults to - `http://127.0.0.1:4646`. + `http://127.0.0.1:4646`. For unix sockets, as with the [task API][], + the format is either `unix:/path/to/api.sock` or `unix:///path/to/api.sock`. - `NOMAD_REGION` - The region of the Nomad server to forward commands to. Defaults to the Agent's local region @@ -118,4 +120,5 @@ may override these environment variables with individual flags. - `NOMAD_LICENSE` - The Nomad Enterprise license file contents as a string. +[task API]: /nomad/api-docs/task-api [`tls` block in agent configuration]: /nomad/docs/configuration/tls diff --git a/website/content/partials/envvars.mdx b/website/content/partials/envvars.mdx index 1ed32107d..95109dd71 100644 --- a/website/content/partials/envvars.mdx +++ b/website/content/partials/envvars.mdx @@ -22,6 +22,7 @@ | `NOMAD_PARENT_CGROUP` | The parent cgroup used to contain task cgroups (Linux only) | | `NOMAD_NAMESPACE` | Namespace in which the allocation is running | | `NOMAD_REGION` | Region in which the allocation is running | +| `NOMAD_UNIX_ADDR` | Use this value as your `NOMAD_ADDR` to use `nomad` CLI with the [task API][]'s unix socket. The value is equivalent to `"unix://${NOMAD_SECRETS_DIR}/api.sock"` | `NOMAD_META_` | The metadata value given by `key` on the task's metadata. Any character in a key other than `[A-Za-z0-9_.]` will be converted to `_`.
**Note:** this is different from [`${meta.}`](/nomad/docs/runtime/interpolation#node-variables-) which are keys in the node's metadata. | | `CONSUL_HTTP_TOKEN` | The tasks' Consul token. See [Consul Integration][consul] documentation for more details. | | `CONSUL_TOKEN` | The tasks' Consul token. See [Consul Integration][consul] documentation for more details. This variable is deprecated and exists only for backwards compatibility. | @@ -68,6 +69,7 @@ names such as `NOMAD_ADDR__