diff --git a/e2e/consul/input/consul_wi.nomad.hcl b/e2e/consul/input/consul_wi.nomad.hcl deleted file mode 100644 index ff089aaac..000000000 --- a/e2e/consul/input/consul_wi.nomad.hcl +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: BUSL-1.1 - -job "example" { - - constraint { - attribute = "${attr.kernel.name}" - value = "linux" - } - - group "example" { - network { - port "db" { - to = 5678 - } - } - - task "example" { - driver = "docker" - - config { - image = "busybox:1" - command = "nc" - args = ["-ll", "-p", "5678", "-e", "/bin/cat"] - - ports = ["db"] - } - - identity { - name = "consul_default" - aud = ["consul.io"] - } - - consul {} - - template { - data = <<-EOT - CONSUL_TOKEN={{ env "CONSUL_TOKEN" }} - EOT - destination = "local/config.txt" - } - - resources { - cpu = 100 - memory = 100 - } - - service { - name = "consul-example" - tags = ["global", "cache"] - port = "db" - - check { - name = "alive" - type = "tcp" - interval = "10s" - timeout = "2s" - } - } - } - } -} diff --git a/e2e/consulcompat/consulcompat_test.go b/e2e/consulcompat/consulcompat_test.go index 3769e3748..8899d7571 100644 --- a/e2e/consulcompat/consulcompat_test.go +++ b/e2e/consulcompat/consulcompat_test.go @@ -35,7 +35,6 @@ func TestConsulCompat(t *testing.T) { for b := range versions.Items() { downloadConsulBuild(t, b, baseDir) - testConsulBuildLegacy(t, b, baseDir) testConsulBuild(t, b, baseDir) } }) diff --git a/e2e/consulcompat/input/consul-policy-for-nomad-legacy.hcl b/e2e/consulcompat/input/consul-policy-for-nomad-legacy.hcl deleted file mode 100644 index c2184e4d6..000000000 --- a/e2e/consulcompat/input/consul-policy-for-nomad-legacy.hcl +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: BUSL-1.1 - -# Policy for the Nomad agent. Note that this policy will work with Workload -# Identity for Connect jobs, but is more highly-privileged than we need. - -# The operator:write permission is required for creating config entries for -# connect ingress gateways. operator ACLs are not namespaced, though the -# config entries they can generate are. -operator = "write" - -agent_prefix "" { - policy = "read" -} - -# The acl:write permission is required for minting Consul Service Identity -# tokens for Connect services with Consul CE (which has no namespaces) -acl = "write" - -key_prefix "" { - policy = "read" -} - -node_prefix "" { - policy = "read" -} - -service_prefix "" { - policy = "write" -} - -# for use with Consul ENT -namespace_prefix "prod" { - - acl = "write" - - key_prefix "" { - policy = "read" - } - - node_prefix "" { - policy = "read" - } - - service_prefix "" { - policy = "write" - } -} diff --git a/e2e/consulcompat/run_ce_test.go b/e2e/consulcompat/run_ce_test.go index 86749380b..042778268 100644 --- a/e2e/consulcompat/run_ce_test.go +++ b/e2e/consulcompat/run_ce_test.go @@ -25,38 +25,6 @@ func usable(v, minimum *version.Version) bool { } } -func testConsulBuildLegacy(t *testing.T, b build, baseDir string) { - t.Run("consul-legacy("+b.Version+")", func(t *testing.T) { - consulHTTPAddr, consulAPI := startConsul(t, b, baseDir, "") - - // smoke test before we continue - verifyConsulVersion(t, consulAPI, b.Version) - - // we need an ACL policy that allows the Nomad agent to fingerprint - // Consul, register services, render templates, and mint new SI tokens - consulToken := setupConsulACLsForServices(t, consulAPI, - "./input/consul-policy-for-nomad-legacy.hcl") - - // we need service intentions so Connect apps can reach each other - setupConsulServiceIntentions(t, consulAPI) - - // note: Nomad needs to be live before we can setupConsul because we - // need it up to serve the JWKS endpoint - - consulCfg := &testutil.Consul{ - Name: "default", - Address: consulHTTPAddr, - Auth: "", - Token: consulToken, - } - - nc := startNomad(t, consulCfg) - - verifyConsulFingerprint(t, nc, b.Version, "default") - runConnectJob(t, nc, "default", "./input/connect.nomad.hcl") - }) -} - func testConsulBuild(t *testing.T, b build, baseDir string) { t.Run("consul("+b.Version+")", func(t *testing.T) { consulHTTPAddr, consulAPI := startConsul(t, b, baseDir, "") diff --git a/e2e/consulcompat/shared_run_test.go b/e2e/consulcompat/shared_run_test.go index 239edcbf1..95eac0c6e 100644 --- a/e2e/consulcompat/shared_run_test.go +++ b/e2e/consulcompat/shared_run_test.go @@ -61,6 +61,9 @@ func verifyConsulFingerprint(t *testing.T, nc *nomadapi.Client, expectVersion, c // token that the Nomad agent can use func setupConsulACLsForServices(t *testing.T, consulAPI *consulapi.Client, policyFilePath string) string { + d, err := os.Getwd() + must.NoError(t, err) + t.Log(d) policyRules, err := os.ReadFile(policyFilePath) must.NoError(t, err, must.Sprintf("could not open policy file %s", policyFilePath)) diff --git a/e2e/e2eutil/utils.go b/e2e/e2eutil/utils.go index 5e24d658e..dd1e116d2 100644 --- a/e2e/e2eutil/utils.go +++ b/e2e/e2eutil/utils.go @@ -81,9 +81,6 @@ func RegisterAllocs(t *testing.T, nomadClient *api.Client, jobFile, jobID, cToke // Set custom job ID (distinguish among tests) job.ID = pointer.Of(jobID) - // Set a Consul "operator" token for the job, if provided. - job.ConsulToken = stringToPtrOrNil(cToken) - // Register job var idx uint64 jobs := nomadClient.Jobs() diff --git a/e2e/terraform/provision-infra/consul-servers.tf b/e2e/terraform/provision-infra/consul-servers.tf index 35d96534d..520f40539 100644 --- a/e2e/terraform/provision-infra/consul-servers.tf +++ b/e2e/terraform/provision-infra/consul-servers.tf @@ -176,3 +176,18 @@ resource "null_resource" "bootstrap_consul_acls" { } } } + +resource "null_resource" "setup_consul_workload_identity" { + depends_on = [null_resource.bootstrap_consul_acls, null_resource.bootstrap_nomad_acls] + + provisioner "local-exec" { + command = "${path.module}/scripts/setup-consul-wi.sh" + environment = { + CONSUL_HTTP_ADDR = "https://${aws_instance.consul_server.public_ip}:8501" + CONSUL_CACERT = "${local.keys_dir}/tls_ca.crt" + CONSUL_HTTP_TOKEN = "${random_uuid.consul_initial_management_token.result}" + CONSUL_AGENT_TOKEN = "${random_uuid.consul_agent_token.result}" + NOMAD_SERVER_ADDR = "https://${aws_instance.server[0].public_ip}:4646" + } + } +} diff --git a/e2e/terraform/provision-infra/provision-nomad/etc/acls/consul/nomad-client-policy.hcl b/e2e/terraform/provision-infra/provision-nomad/etc/acls/consul/nomad-client-policy.hcl deleted file mode 100644 index c07dc09b0..000000000 --- a/e2e/terraform/provision-infra/provision-nomad/etc/acls/consul/nomad-client-policy.hcl +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: BUSL-1.1 - -// The Nomad Client will be registering things into its buddy Consul Client. -// Note: because we also test the use of Consul namespaces, this token must be -// able to register services, read the keystore, and read node data for any -// namespace. -// The operator=write permission is required for creating config entries for -// connect ingress gateways. operator ACLs are not namespaced, though the -// config entries they can generate are. -operator = "write" - -agent_prefix "" { - policy = "read" -} - -namespace_prefix "" { - // The acl=write permission is required for generating Consul Service Identity - // tokens for consul connect services. Those services could be configured for - // any Consul namespace the job-submitter has access to. - acl = "write" - - key_prefix "" { - policy = "read" - } - - node_prefix "" { - policy = "read" - } - - service_prefix "" { - policy = "write" - } -} diff --git a/e2e/terraform/provision-infra/provision-nomad/etc/acls/consul/nomad-server-policy.hcl b/e2e/terraform/provision-infra/provision-nomad/etc/acls/consul/nomad-server-policy.hcl deleted file mode 100644 index 5df422466..000000000 --- a/e2e/terraform/provision-infra/provision-nomad/etc/acls/consul/nomad-server-policy.hcl +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) HashiCorp, Inc. -# SPDX-License-Identifier: BUSL-1.1 - -// The operator=write permission is required for creating config entries for -// connect ingress gateways. operator ACLs are not namespaced, though the -// config entries they can generate are. -operator = "write" - -agent_prefix "" { - policy = "read" -} - -namespace_prefix "" { - // The acl=write permission is required for generating Consul Service Identity - // tokens for consul connect services. Those services could be configured for - // any Consul namespace the job-submitter has access to. - acl = "write" -} - -service_prefix "" { - policy = "write" -} - -agent_prefix "" { - policy = "read" -} - -node_prefix "" { - policy = "read" -} diff --git a/e2e/terraform/provision-infra/provision-nomad/etc/nomad.d/client-consul.hcl b/e2e/terraform/provision-infra/provision-nomad/etc/nomad.d/client-consul.hcl index cb097207e..172526011 100644 --- a/e2e/terraform/provision-infra/provision-nomad/etc/nomad.d/client-consul.hcl +++ b/e2e/terraform/provision-infra/provision-nomad/etc/nomad.d/client-consul.hcl @@ -1,10 +1,13 @@ # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 -# TODO: add workload-identity configuration for servers consul { address = "127.0.0.1:8500" token = "${token}" client_service_name = "${client_service_name}" server_service_name = "${server_service_name}" + + // default auth-methods + service_auth_method = "nomad-workloads" + task_auth_method = "nomad-workloads" } diff --git a/e2e/terraform/provision-infra/provision-nomad/etc/nomad.d/server-consul.hcl b/e2e/terraform/provision-infra/provision-nomad/etc/nomad.d/server-consul.hcl index cb097207e..17665fa60 100644 --- a/e2e/terraform/provision-infra/provision-nomad/etc/nomad.d/server-consul.hcl +++ b/e2e/terraform/provision-infra/provision-nomad/etc/nomad.d/server-consul.hcl @@ -1,10 +1,19 @@ # Copyright (c) HashiCorp, Inc. # SPDX-License-Identifier: BUSL-1.1 -# TODO: add workload-identity configuration for servers consul { address = "127.0.0.1:8500" token = "${token}" client_service_name = "${client_service_name}" server_service_name = "${server_service_name}" + + service_identity { + aud = ["consul.io"] + ttl = "1h" + } + + task_identity { + aud = ["consul.io"] + ttl = "1h" + } } diff --git a/e2e/terraform/provision-infra/scripts/bootstrap-consul.sh b/e2e/terraform/provision-infra/scripts/bootstrap-consul.sh index 3c688a7a7..59d2cd136 100755 --- a/e2e/terraform/provision-infra/scripts/bootstrap-consul.sh +++ b/e2e/terraform/provision-infra/scripts/bootstrap-consul.sh @@ -31,4 +31,4 @@ echo "writing Consul cluster policy and token" consul acl policy create -name consul-agents -rules @${DIR}/consul-agents-policy.hcl consul acl token create -policy-name=consul-agents -secret "$CONSUL_AGENT_TOKEN" -echo "Consul successfully bootstraped!" \ No newline at end of file +echo "Consul successfully bootstraped!" diff --git a/e2e/terraform/provision-infra/scripts/consul-workload-identity/nomad-task-policy.hcl b/e2e/terraform/provision-infra/scripts/consul-workload-identity/nomad-task-policy.hcl new file mode 100644 index 000000000..1a0c691a4 --- /dev/null +++ b/e2e/terraform/provision-infra/scripts/consul-workload-identity/nomad-task-policy.hcl @@ -0,0 +1,10 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +service_prefix "" { + policy = "read" +} + +key_prefix "" { + policy = "read" +} diff --git a/e2e/terraform/provision-infra/scripts/setup-consul-wi.sh b/e2e/terraform/provision-infra/scripts/setup-consul-wi.sh new file mode 100755 index 000000000..704306fb6 --- /dev/null +++ b/e2e/terraform/provision-infra/scripts/setup-consul-wi.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# The following ACL's are used so Nomad services and tasks can register +# via Workload Identity +echo "writing ACLs for Nomad Workload Identity integration..." + +# replaces the newlines in the cert with escaped newlines so they are valid JSON +CERT=$(cat ${CONSUL_CACERT} | sed 's/$/\\n/g' | tr -d '\n') + +AUTH=$(cat <