From 1c9c75cc832493a3af2e6ca9b9c595de61fc73c1 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Fri, 10 Nov 2023 15:05:51 -0500 Subject: [PATCH] E2E: refactor `consulcompat` to allow for ENT tests (#19068) We want to run the Consul compatibility E2E test with Consul Enterprise binaries and use Consul namespaces. Refactor the `consulcompat` test so as to parameterize most of the test setup logic with the namespace, and add the appropriate build tag for the CE version of the test. Ref: https://github.com/hashicorp/nomad-enterprise/pull/1305 --- .../input/consul-policy-for-nomad-legacy.hcl | 2 +- .../input/consul-policy-for-nomad.hcl | 10 +- e2e/consulcompat/run_ce_test.go | 192 +++--------------- e2e/consulcompat/shared_download_test.go | 13 -- e2e/consulcompat/shared_run_test.go | 165 ++++++++++++++- e2e/consulcompat/shared_setup_test.go | 2 +- 6 files changed, 192 insertions(+), 192 deletions(-) diff --git a/e2e/consulcompat/input/consul-policy-for-nomad-legacy.hcl b/e2e/consulcompat/input/consul-policy-for-nomad-legacy.hcl index 3015582ba..c2184e4d6 100644 --- a/e2e/consulcompat/input/consul-policy-for-nomad-legacy.hcl +++ b/e2e/consulcompat/input/consul-policy-for-nomad-legacy.hcl @@ -30,7 +30,7 @@ service_prefix "" { } # for use with Consul ENT -namespace_prefix "" { +namespace_prefix "prod" { acl = "write" diff --git a/e2e/consulcompat/input/consul-policy-for-nomad.hcl b/e2e/consulcompat/input/consul-policy-for-nomad.hcl index 5308f4c46..ddf9de81f 100644 --- a/e2e/consulcompat/input/consul-policy-for-nomad.hcl +++ b/e2e/consulcompat/input/consul-policy-for-nomad.hcl @@ -27,10 +27,7 @@ service_prefix "" { } # for use with Consul ENT -namespace_prefix "" { - key_prefix "" { - policy = "read" - } +namespace_prefix "prod" { node_prefix "" { policy = "read" @@ -39,4 +36,9 @@ namespace_prefix "" { service_prefix "nomad" { policy = "write" } + + service_prefix "" { + policy = "read" + } + } diff --git a/e2e/consulcompat/run_ce_test.go b/e2e/consulcompat/run_ce_test.go index f300f3694..686939c47 100644 --- a/e2e/consulcompat/run_ce_test.go +++ b/e2e/consulcompat/run_ce_test.go @@ -1,20 +1,32 @@ // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: BUSL-1.1 +//go:build !ent + package consulcompat import ( - "fmt" - "os" "testing" - "time" - consulapi "github.com/hashicorp/consul/api" - "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/go-version" "github.com/hashicorp/nomad/testutil" - "github.com/shoenig/test/must" ) +// usable is used by the downloader to verify that we're getting the right +// versions of Consul CE +func usable(v, minimum *version.Version) bool { + switch { + case v.Prerelease() != "": + return false + case v.Metadata() != "": + return false + case v.LessThan(minimum): + return false + default: + return true + } +} + func testConsulBuildLegacy(t *testing.T, b build, baseDir string) { t.Run("consul-legacy("+b.Version+")", func(t *testing.T) { consulHTTPAddr, consulAPI := startConsul(t, b, baseDir, "") @@ -43,7 +55,7 @@ func testConsulBuildLegacy(t *testing.T, b build, baseDir string) { nc := startNomad(t, consulCfg) verifyConsulFingerprint(t, nc, b.Version, "default") - runConnectJob(t, nc) + runConnectJob(t, nc, "default", "./input/connect.nomad.hcl") }) } @@ -56,6 +68,10 @@ func testConsulBuild(t *testing.T, b build, baseDir string) { // we need an ACL policy that only allows the Nomad agent to fingerprint // Consul and register itself, and set up service intentions + // + // Note that with this policy we must use Workload Identity for Connect + // jobs, or we'll get "failed to derive SI token" errors from the client + // because the Nomad agent's token doesn't have "acl:write" consulToken := setupConsulACLsForServices(t, consulAPI, "./input/consul-policy-for-nomad.hcl") @@ -63,7 +79,8 @@ func testConsulBuild(t *testing.T, b build, baseDir string) { // an ACL role and policy that tasks will be able to use to render // templates setupConsulServiceIntentions(t, consulAPI) - setupConsulACLsForTasks(t, consulAPI, "./input/consul-policy-for-tasks.hcl") + setupConsulACLsForTasks(t, consulAPI, + "nomad-default", "./input/consul-policy-for-tasks.hcl") // note: Nomad needs to be live before we can setup Consul auth methods // because we need it up to serve the JWKS endpoint @@ -88,165 +105,10 @@ func testConsulBuild(t *testing.T, b build, baseDir string) { nc := startNomad(t, consulCfg) // configure authentication for WI to Consul - setupConsulJWTAuthForServices(t, consulAPI, nc.Address()) + setupConsulJWTAuthForServices(t, consulAPI, nc.Address(), nil) setupConsulJWTAuthForTasks(t, consulAPI, nc.Address()) verifyConsulFingerprint(t, nc, b.Version, "default") - runConnectJob(t, nc) + runConnectJob(t, nc, "default", "./input/connect.nomad.hcl") }) } - -// setupConsulACLsForServices installs a base set of ACL policies and returns a -// token that the Nomad agent can use -func setupConsulACLsForServices(t *testing.T, consulAPI *consulapi.Client, policyFilePath string) string { - - policyRules, err := os.ReadFile(policyFilePath) - must.NoError(t, err, must.Sprintf("could not open policy file %s", policyFilePath)) - - // policy without namespaces, for Consul CE. Note that with this policy we - // must use Workload Identity for Connect jobs, or we'll get "failed to - // derive SI token" errors from the client because the Nomad agent's token - // doesn't have "acl:write" - policy := &consulapi.ACLPolicy{ - Name: "nomad-cluster-" + uuid.Short(), - Description: "policy for nomad agent", - Rules: string(policyRules), - } - - policy, _, err = consulAPI.ACL().PolicyCreate(policy, nil) - must.NoError(t, err, must.Sprint("could not write policy to Consul")) - - token := &consulapi.ACLToken{ - Description: "token for Nomad agent", - Policies: []*consulapi.ACLLink{{ - ID: policy.ID, - Name: policy.Name, - }}, - } - token, _, err = consulAPI.ACL().TokenCreate(token, nil) - must.NoError(t, err, must.Sprint("could not create token in Consul")) - - return token.SecretID -} - -func setupConsulServiceIntentions(t *testing.T, consulAPI *consulapi.Client) { - ixn := &consulapi.Intention{ - SourceName: "count-dashboard", - DestinationName: "count-api", - Action: "allow", - } - _, err := consulAPI.Connect().IntentionUpsert(ixn, nil) - must.NoError(t, err, must.Sprint("could not create intention")) -} - -// setupConsulACLsForTasks installs a base set of ACL policies and returns a -// token that the Nomad agent can use -func setupConsulACLsForTasks(t *testing.T, consulAPI *consulapi.Client, policyFilePath string) { - - policyRules, err := os.ReadFile(policyFilePath) - must.NoError(t, err, must.Sprintf("could not open policy file %s", policyFilePath)) - - // policy without namespaces, for Consul CE. - policy := &consulapi.ACLPolicy{ - Name: "nomad-tasks-" + uuid.Short(), - Description: "policy for nomad tasks", - Rules: string(policyRules), - } - - policy, _, err = consulAPI.ACL().PolicyCreate(policy, nil) - must.NoError(t, err, must.Sprint("could not write policy to Consul")) - - role := &consulapi.ACLRole{ - Name: "nomad-default", // must match Nomad namespace - Description: "role for nomad tasks", - Policies: []*consulapi.ACLLink{{ - ID: policy.ID, - Name: policy.Name, - }}, - } - _, _, err = consulAPI.ACL().RoleCreate(role, nil) - must.NoError(t, err, must.Sprint("could not create token in Consul")) -} - -func setupConsulJWTAuthForServices(t *testing.T, consulAPI *consulapi.Client, address string) { - - authConfig := map[string]any{ - "JWKSURL": fmt.Sprintf("%s/.well-known/jwks.json", address), - "JWTSupportedAlgs": []string{"RS256"}, - "BoundAudiences": "consul.io", - "ClaimMappings": map[string]string{ - "nomad_namespace": "nomad_namespace", - "nomad_job_id": "nomad_job_id", - "nomad_task": "nomad_task", - "nomad_service": "nomad_service", - }, - } - - // note: we can't include NamespaceRules here because Consul CE doesn't - // support namespaces - _, _, err := consulAPI.ACL().AuthMethodCreate(&consulapi.ACLAuthMethod{ - Name: "nomad-services", - Type: "jwt", - DisplayName: "nomad-services", - Description: "login method for Nomad workload identities (WI)", - MaxTokenTTL: time.Hour, - TokenLocality: "local", - Config: authConfig, - }, nil) - - must.NoError(t, err, must.Sprint("could not create Consul auth method for services")) - - // note: we can't include Namespace here because Consul CE doesn't support - // namespaces - rule := &consulapi.ACLBindingRule{ - ID: "", - Description: "binding rule for Nomad workload identities (WI) for services", - AuthMethod: "nomad-services", - Selector: "", - BindType: "service", - BindName: "${value.nomad_service}", - } - _, _, err = consulAPI.ACL().BindingRuleCreate(rule, nil) - must.NoError(t, err, must.Sprint("could not create Consul binding rule")) -} - -func setupConsulJWTAuthForTasks(t *testing.T, consulAPI *consulapi.Client, address string) { - - authConfig := map[string]any{ - "JWKSURL": fmt.Sprintf("%s/.well-known/jwks.json", address), - "JWTSupportedAlgs": []string{"RS256"}, - "BoundAudiences": "consul.io", - "ClaimMappings": map[string]string{ - "nomad_namespace": "nomad_namespace", - "nomad_job_id": "nomad_job_id", - "nomad_task": "nomad_task", - "nomad_service": "nomad_service", - }, - } - - // note: we can't include NamespaceRules here because Consul CE doesn't - // support namespaces - _, _, err := consulAPI.ACL().AuthMethodCreate(&consulapi.ACLAuthMethod{ - Name: "nomad-tasks", - Type: "jwt", - DisplayName: "nomad-tasks", - Description: "login method for Nomad tasks with workload identity (WI)", - MaxTokenTTL: time.Hour, - TokenLocality: "local", - Config: authConfig, - }, nil) - must.NoError(t, err, must.Sprint("could not create Consul auth method for tasks")) - - // note: we can't include Namespace here because Consul CE doesn't support - // namespaces - rule := &consulapi.ACLBindingRule{ - ID: "", - Description: "binding rule for Nomad workload identities (WI) for tasks", - AuthMethod: "nomad-tasks", - Selector: "", - BindType: "role", - BindName: "nomad-${value.nomad_namespace}", - } - _, _, err = consulAPI.ACL().BindingRuleCreate(rule, nil) - must.NoError(t, err, must.Sprint("could not create Consul binding rule")) -} diff --git a/e2e/consulcompat/shared_download_test.go b/e2e/consulcompat/shared_download_test.go index 91c5b5aa0..53805cf16 100644 --- a/e2e/consulcompat/shared_download_test.go +++ b/e2e/consulcompat/shared_download_test.go @@ -75,19 +75,6 @@ type consulJSON struct { } `json:"versions"` } -func usable(v, minimum *version.Version) bool { - switch { - case v.Prerelease() != "": - return false - case v.Metadata() != "": - return false - case v.LessThan(minimum): - return false - default: - return true - } -} - func keep(b build) bool { exactVersion := os.Getenv(exactConsulVersionEnv) if exactVersion != "" { diff --git a/e2e/consulcompat/shared_run_test.go b/e2e/consulcompat/shared_run_test.go index 0c1081b86..f48a45d69 100644 --- a/e2e/consulcompat/shared_run_test.go +++ b/e2e/consulcompat/shared_run_test.go @@ -13,7 +13,9 @@ import ( "time" consulapi "github.com/hashicorp/consul/api" + "github.com/hashicorp/nomad/api" nomadapi "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/helper/uuid" "github.com/shoenig/test/must" "github.com/shoenig/test/wait" ) @@ -42,23 +44,170 @@ func verifyConsulFingerprint(t *testing.T, nc *nomadapi.Client, version, cluster } } -func runConnectJob(t *testing.T, nc *nomadapi.Client) { +// setupConsulACLsForServices installs a base set of ACL policies and returns a +// token that the Nomad agent can use +func setupConsulACLsForServices(t *testing.T, consulAPI *consulapi.Client, policyFilePath string) string { - b, err := os.ReadFile("./input/connect.nomad.hcl") + policyRules, err := os.ReadFile(policyFilePath) + must.NoError(t, err, must.Sprintf("could not open policy file %s", policyFilePath)) + + policy := &consulapi.ACLPolicy{ + Name: "nomad-cluster-" + uuid.Short(), + Description: "policy for nomad agent", + Rules: string(policyRules), + } + + policy, _, err = consulAPI.ACL().PolicyCreate(policy, nil) + must.NoError(t, err, must.Sprint("could not write policy to Consul")) + + token := &consulapi.ACLToken{ + Description: "token for Nomad agent", + Policies: []*consulapi.ACLLink{{ + ID: policy.ID, + Name: policy.Name, + }}, + } + token, _, err = consulAPI.ACL().TokenCreate(token, nil) + must.NoError(t, err, must.Sprint("could not create token in Consul")) + + return token.SecretID +} + +func setupConsulServiceIntentions(t *testing.T, consulAPI *consulapi.Client) { + ixn := &consulapi.Intention{ + SourceName: "count-dashboard", + DestinationName: "count-api", + Action: "allow", + } + _, err := consulAPI.Connect().IntentionUpsert(ixn, nil) + must.NoError(t, err, must.Sprint("could not create intention")) +} + +// setupConsulACLsForTasks installs a base set of ACL policies and returns a +// token that the Nomad agent can use +func setupConsulACLsForTasks(t *testing.T, consulAPI *consulapi.Client, roleName, policyFilePath string) { + + policyRules, err := os.ReadFile(policyFilePath) + must.NoError(t, err, must.Sprintf("could not open policy file %s", policyFilePath)) + + policy := &consulapi.ACLPolicy{ + Name: "nomad-tasks-" + uuid.Short(), + Description: "policy for nomad tasks", + Rules: string(policyRules), + } + + policy, _, err = consulAPI.ACL().PolicyCreate(policy, nil) + must.NoError(t, err, must.Sprint("could not write policy to Consul")) + + role := &consulapi.ACLRole{ + Name: roleName, // note: must match "prod-${nomad_namespace}" + Description: "role for nomad tasks", + Policies: []*consulapi.ACLLink{{ + ID: policy.ID, + Name: policy.Name, + }}, + } + _, _, err = consulAPI.ACL().RoleCreate(role, nil) + must.NoError(t, err, must.Sprint("could not create token in Consul")) +} + +func setupConsulJWTAuthForTasks(t *testing.T, consulAPI *consulapi.Client, address string) { + + authConfig := map[string]any{ + "JWKSURL": fmt.Sprintf("%s/.well-known/jwks.json", address), + "JWTSupportedAlgs": []string{"RS256"}, + "BoundAudiences": "consul.io", + "ClaimMappings": map[string]string{ + "nomad_namespace": "nomad_namespace", + "nomad_job_id": "nomad_job_id", + "nomad_task": "nomad_task", + "nomad_service": "nomad_service", + }, + } + + _, _, err := consulAPI.ACL().AuthMethodCreate(&consulapi.ACLAuthMethod{ + Name: "nomad-tasks", + Type: "jwt", + DisplayName: "nomad-tasks", + Description: "login method for Nomad tasks with workload identity (WI)", + MaxTokenTTL: time.Hour, + TokenLocality: "local", + Config: authConfig, + }, nil) + must.NoError(t, err, must.Sprint("could not create Consul auth method for tasks")) + + rule := &consulapi.ACLBindingRule{ + ID: "", + Description: "binding rule for Nomad workload identities (WI) for tasks", + AuthMethod: "nomad-tasks", + Selector: "", + BindType: "role", + BindName: "nomad-${value.nomad_namespace}", + } + _, _, err = consulAPI.ACL().BindingRuleCreate(rule, nil) + must.NoError(t, err, must.Sprint("could not create Consul binding rule")) +} + +func setupConsulJWTAuthForServices(t *testing.T, consulAPI *consulapi.Client, address string, namespaceRules []*consulapi.ACLAuthMethodNamespaceRule) { + + authConfig := map[string]any{ + "JWKSURL": fmt.Sprintf("%s/.well-known/jwks.json", address), + "JWTSupportedAlgs": []string{"RS256"}, + "BoundAudiences": "consul.io", + "ClaimMappings": map[string]string{ + "nomad_namespace": "nomad_namespace", + "nomad_job_id": "nomad_job_id", + "nomad_task": "nomad_task", + "nomad_service": "nomad_service", + }, + } + + // create an auth method with namespace rule for Consul ENT + _, _, err := consulAPI.ACL().AuthMethodCreate(&consulapi.ACLAuthMethod{ + Name: "nomad-services", + Type: "jwt", + DisplayName: "nomad-services", + Description: "login method for Nomad workload identities (WI)", + MaxTokenTTL: time.Hour, + TokenLocality: "local", + Config: authConfig, + NamespaceRules: namespaceRules, + }, nil) + + must.NoError(t, err, must.Sprint("could not create Consul auth method for services")) + + rule := &consulapi.ACLBindingRule{ + ID: "", + Description: "binding rule for Nomad workload identities (WI) for services", + AuthMethod: "nomad-services", + Selector: "", + BindType: "service", + BindName: "${value.nomad_service}", + } + _, _, err = consulAPI.ACL().BindingRuleCreate(rule, nil) + must.NoError(t, err, must.Sprint("could not create Consul binding rule")) +} + +func runConnectJob(t *testing.T, nc *nomadapi.Client, ns, filePath string) { + + b, err := os.ReadFile(filePath) must.NoError(t, err) jobs := nc.Jobs() job, err := jobs.ParseHCL(string(b), true) must.NoError(t, err, must.Sprint("failed to parse job HCL")) - resp, _, err := jobs.Register(job, nil) + qOpts := &api.QueryOptions{Namespace: ns} + wOpts := &api.WriteOptions{Namespace: ns} + + resp, _, err := jobs.Register(job, wOpts) must.NoError(t, err, must.Sprint("failed to register job")) evalID := resp.EvalID t.Logf("eval: %s", evalID) must.Wait(t, wait.InitialSuccess( wait.ErrorFunc(func() error { - eval, _, err := nc.Evaluations().Info(evalID, nil) + eval, _, err := nc.Evaluations().Info(evalID, qOpts) must.NoError(t, err) if eval.Status == "complete" { // if we have failed allocations it can be difficult to debug in @@ -77,12 +226,12 @@ func runConnectJob(t *testing.T, nc *nomadapi.Client) { )) t.Cleanup(func() { - _, _, err = jobs.Deregister(*job.Name, true, nil) + _, _, err = jobs.Deregister(*job.Name, true, wOpts) must.NoError(t, err, must.Sprint("failed to deregister job")) must.Wait(t, wait.InitialSuccess( wait.ErrorFunc(func() error { - allocs, _, err := jobs.Allocations(*job.ID, false, nil) + allocs, _, err := jobs.Allocations(*job.ID, false, qOpts) if err != nil { return err } @@ -105,7 +254,7 @@ func runConnectJob(t *testing.T, nc *nomadapi.Client) { must.Wait(t, wait.InitialSuccess( wait.ErrorFunc(func() error { - allocs, _, err := jobs.Allocations(*job.ID, false, nil) + allocs, _, err := jobs.Allocations(*job.ID, false, qOpts) if err != nil { return err } @@ -129,7 +278,7 @@ func runConnectJob(t *testing.T, nc *nomadapi.Client) { )) // Ensure that the dashboard is reachable and can connect to the API - alloc, _, err := nc.Allocations().Info(dashboardAllocID, nil) + alloc, _, err := nc.Allocations().Info(dashboardAllocID, qOpts) must.NoError(t, err) network := alloc.AllocatedResources.Shared.Networks[0] diff --git a/e2e/consulcompat/shared_setup_test.go b/e2e/consulcompat/shared_setup_test.go index d3f3dd886..c7aee7deb 100644 --- a/e2e/consulcompat/shared_setup_test.go +++ b/e2e/consulcompat/shared_setup_test.go @@ -50,7 +50,7 @@ func startConsul(t *testing.T, b build, baseDir, ns string) (string, *consulapi. } c.Datacenter = consulDC1 c.DataDir = t.TempDir() - c.LogLevel = "debug" + c.LogLevel = testlog.HCLoggerTestLevel().String() c.Connect = map[string]any{"enabled": true} c.Server = true