kemko/nomad
1
0
Fork 0
mirror of https://github.com/kemko/nomad.git synced 2026-01-01 16:05:42 +03:00
Code Issues Packages Projects Releases Wiki Activity

e2e: remove legacy consul token tests (#25174)

Browse Source
This commit is contained in:
Michael Smithhisler
2025-02-28 11:31:33 -05:00
committed by GitHub
parent c52623d7d4
commit 7867957811
14 changed files with 120 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
e2e/consul/input/consul_wi.nomad.hcl
View File

@@ -1,62 +0,0 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
job "example" {
constraint {
attribute = "${attr.kernel.name}"
value = "linux"
}
group "example" {
network {
port "db" {
to = 5678
}
}
task "example" {
driver = "docker"
config {
image = "busybox:1"
command = "nc"
args = ["-ll", "-p", "5678", "-e", "/bin/cat"]
ports = ["db"]
}
identity {
name = "consul_default"
aud = ["consul.io"]
}
consul {}
template {
data = <<-EOT
CONSUL_TOKEN={{ env "CONSUL_TOKEN" }}
EOT
destination = "local/config.txt"
}
resources {
cpu = 100
memory = 100
}
service {
name = "consul-example"
tags = ["global", "cache"]
port = "db"
check {
name = "alive"
type = "tcp"
interval = "10s"
timeout = "2s"
}
}
}
}
}

1
e2e/consulcompat/consulcompat_test.go
View File

@@ -35,7 +35,6 @@ func TestConsulCompat(t *testing.T) {
for b := range versions.Items() {
downloadConsulBuild(t, b, baseDir)
testConsulBuildLegacy(t, b, baseDir)
testConsulBuild(t, b, baseDir)
}
})

48
e2e/consulcompat/input/consul-policy-for-nomad-legacy.hcl
View File

@@ -1,48 +0,0 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
# Policy for the Nomad agent. Note that this policy will work with Workload
# Identity for Connect jobs, but is more highly-privileged than we need.
# The operator:write permission is required for creating config entries for
# connect ingress gateways. operator ACLs are not namespaced, though the
# config entries they can generate are.
operator = "write"
agent_prefix "" {
policy = "read"
}
# The acl:write permission is required for minting Consul Service Identity
# tokens for Connect services with Consul CE (which has no namespaces)
acl = "write"
key_prefix "" {
policy = "read"
}
node_prefix "" {
policy = "read"
}
service_prefix "" {
policy = "write"
}
# for use with Consul ENT
namespace_prefix "prod" {
acl = "write"
key_prefix "" {
policy = "read"
}
node_prefix "" {
policy = "read"
}
service_prefix "" {
policy = "write"
}
}

32
e2e/consulcompat/run_ce_test.go
View File

@@ -25,38 +25,6 @@ func usable(v, minimum *version.Version) bool {
}
}
func testConsulBuildLegacy(t *testing.T, b build, baseDir string) {
t.Run("consul-legacy("+b.Version+")", func(t *testing.T) {
consulHTTPAddr, consulAPI := startConsul(t, b, baseDir, "")
// smoke test before we continue
verifyConsulVersion(t, consulAPI, b.Version)
// we need an ACL policy that allows the Nomad agent to fingerprint
// Consul, register services, render templates, and mint new SI tokens
consulToken := setupConsulACLsForServices(t, consulAPI,
"./input/consul-policy-for-nomad-legacy.hcl")
// we need service intentions so Connect apps can reach each other
setupConsulServiceIntentions(t, consulAPI)
// note: Nomad needs to be live before we can setupConsul because we
// need it up to serve the JWKS endpoint
consulCfg := &testutil.Consul{
Name: "default",
Address: consulHTTPAddr,
Auth: "",
Token: consulToken,
}
nc := startNomad(t, consulCfg)
verifyConsulFingerprint(t, nc, b.Version, "default")
runConnectJob(t, nc, "default", "./input/connect.nomad.hcl")
})
}
func testConsulBuild(t *testing.T, b build, baseDir string) {
t.Run("consul("+b.Version+")", func(t *testing.T) {
consulHTTPAddr, consulAPI := startConsul(t, b, baseDir, "")

3
e2e/consulcompat/shared_run_test.go
View File

@@ -61,6 +61,9 @@ func verifyConsulFingerprint(t *testing.T, nc *nomadapi.Client, expectVersion, c
// token that the Nomad agent can use
func setupConsulACLsForServices(t *testing.T, consulAPI *consulapi.Client, policyFilePath string) string {
d, err := os.Getwd()
must.NoError(t, err)
t.Log(d)
policyRules, err := os.ReadFile(policyFilePath)
must.NoError(t, err, must.Sprintf("could not open policy file %s", policyFilePath))

3
e2e/e2eutil/utils.go
View File

@@ -81,9 +81,6 @@ func RegisterAllocs(t *testing.T, nomadClient *api.Client, jobFile, jobID, cToke
// Set custom job ID (distinguish among tests)
job.ID = pointer.Of(jobID)
// Set a Consul "operator" token for the job, if provided.
job.ConsulToken = stringToPtrOrNil(cToken)
// Register job
var idx uint64
jobs := nomadClient.Jobs()

15
e2e/terraform/provision-infra/consul-servers.tf
View File

@@ -176,3 +176,18 @@ resource "null_resource" "bootstrap_consul_acls" {
}
}
}
resource "null_resource" "setup_consul_workload_identity" {
depends_on = [null_resource.bootstrap_consul_acls, null_resource.bootstrap_nomad_acls]
provisioner "local-exec" {
command = "${path.module}/scripts/setup-consul-wi.sh"
environment = {
CONSUL_HTTP_ADDR = "https://${aws_instance.consul_server.public_ip}:8501"
CONSUL_CACERT = "${local.keys_dir}/tls_ca.crt"
CONSUL_HTTP_TOKEN = "${random_uuid.consul_initial_management_token.result}"
CONSUL_AGENT_TOKEN = "${random_uuid.consul_agent_token.result}"
NOMAD_SERVER_ADDR = "https://${aws_instance.server[0].public_ip}:4646"
}
}
}

34
e2e/terraform/provision-infra/provision-nomad/etc/acls/consul/nomad-client-policy.hcl
View File

@@ -1,34 +0,0 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
// The Nomad Client will be registering things into its buddy Consul Client.
// Note: because we also test the use of Consul namespaces, this token must be
// able to register services, read the keystore, and read node data for any
// namespace.
// The operator=write permission is required for creating config entries for
// connect ingress gateways. operator ACLs are not namespaced, though the
// config entries they can generate are.
operator = "write"
agent_prefix "" {
policy = "read"
}
namespace_prefix "" {
// The acl=write permission is required for generating Consul Service Identity
// tokens for consul connect services. Those services could be configured for
// any Consul namespace the job-submitter has access to.
acl = "write"
key_prefix "" {
policy = "read"
}
node_prefix "" {
policy = "read"
}
service_prefix "" {
policy = "write"
}
}

30
e2e/terraform/provision-infra/provision-nomad/etc/acls/consul/nomad-server-policy.hcl
View File

@@ -1,30 +0,0 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
// The operator=write permission is required for creating config entries for
// connect ingress gateways. operator ACLs are not namespaced, though the
// config entries they can generate are.
operator = "write"
agent_prefix "" {
policy = "read"
}
namespace_prefix "" {
// The acl=write permission is required for generating Consul Service Identity
// tokens for consul connect services. Those services could be configured for
// any Consul namespace the job-submitter has access to.
acl = "write"
}
service_prefix "" {
policy = "write"
}
agent_prefix "" {
policy = "read"
}
node_prefix "" {
policy = "read"
}

5
e2e/terraform/provision-infra/provision-nomad/etc/nomad.d/client-consul.hcl
View File

@@ -1,10 +1,13 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
# TODO: add workload-identity configuration for servers
consul {
address = "127.0.0.1:8500"
token = "${token}"
client_service_name = "${client_service_name}"
server_service_name = "${server_service_name}"
// default auth-methods
service_auth_method = "nomad-workloads"
task_auth_method = "nomad-workloads"
}

11
e2e/terraform/provision-infra/provision-nomad/etc/nomad.d/server-consul.hcl
View File

@@ -1,10 +1,19 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
# TODO: add workload-identity configuration for servers
consul {
address = "127.0.0.1:8500"
token = "${token}"
client_service_name = "${client_service_name}"
server_service_name = "${server_service_name}"
service_identity {
aud = ["consul.io"]
ttl = "1h"
}
task_identity {
aud = ["consul.io"]
ttl = "1h"
}
}

2
e2e/terraform/provision-infra/scripts/bootstrap-consul.sh
View File

@@ -31,4 +31,4 @@ echo "writing Consul cluster policy and token"
consul acl policy create -name consul-agents -rules @${DIR}/consul-agents-policy.hcl
consul acl token create -policy-name=consul-agents -secret "$CONSUL_AGENT_TOKEN"
echo "Consul successfully bootstraped!"
echo "Consul successfully bootstraped!"

10
e2e/terraform/provision-infra/scripts/consul-workload-identity/nomad-task-policy.hcl Normal file
View File

@@ -0,0 +1,10 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
service_prefix "" {
policy = "read"
}
key_prefix "" {
policy = "read"
}

77
e2e/terraform/provision-infra/scripts/setup-consul-wi.sh Executable file
View File

@@ -0,0 +1,77 @@
#!/usr/bin/env bash
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
# The following ACL's are used so Nomad services and tasks can register
# via Workload Identity
echo "writing ACLs for Nomad Workload Identity integration..."
# replaces the newlines in the cert with escaped newlines so they are valid JSON
CERT=$(cat ${CONSUL_CACERT} | sed 's/$/\\n/g' | tr -d '\n')
AUTH=$(cat <<EOF
{
"JWKSURL": "${NOMAD_SERVER_ADDR}/.well-known/jwks.json",
"JWTSupportedAlgs": [
"RS256"
],
"JWKSCACert": "${CERT}",
"BoundAudiences": [
"consul.io"
],
"ClaimMappings": {
"consul_namespace": "consul_namespace",
"nomad_job_id": "nomad_job_id",
"nomad_namespace": "nomad_namespace",
"nomad_service": "nomad_service",
"nomad_task": "nomad_task"
}
}
EOF
)
echo "writing Consul auth-method"
if [ $(consul info | grep -q "version_metadata = ent") ]; then
consul acl auth-method create \
-name 'nomad-workloads' \
-type 'jwt' \
-description 'Login method for Nomad workloads using workload identities' \
-token-locality 'local' \
-config "${AUTH}" \
-namespace-rule-selector '"consul_namespace" in value' \
-namespace-rule-bind-namespace '${value.consul_namespace}'
else
consul acl auth-method create \
-name 'nomad-workloads' \
-type 'jwt' \
-description 'Login method for Nomad workloads using workload identities' \
-token-locality 'local' \
-config "${AUTH}"
fi
echo "writing binding-rule for Nomad services"
consul acl binding-rule create \
-method 'nomad-workloads' \
-description 'Binding rule for Nomad services authenticated using a workload identity' \
-bind-type 'service' \
-bind-name '${value.nomad_service}' \
-selector '"nomad_service" in value'
echo "writing binding-rule for Nomad tasks"
consul acl binding-rule create \
-method 'nomad-workloads' \
-description 'Binding rule for Nomad tasks authenticated using a workload identity' \
-bind-type 'role' \
-bind-name 'nomad-${value.nomad_namespace}-tasks' \
-selector '"nomad_service" not in value'
echo "writing policy for Nomad tasks"
consul acl policy create -name policy-nomad-tasks -rules @${DIR}/consul-workload-identity/nomad-task-policy.hcl
echo "creating role for Nomad tasks using previously created policy"
consul acl role create -name nomad-default-tasks -policy-name policy-nomad-tasks
echo "Consul successfully configured to use Nomad Workload Identity!"