From 9e912ab938803eeaea120c24b8bdd337bc13707b Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Wed, 25 Nov 2020 14:30:22 -0600 Subject: [PATCH] e2e: add e2e test for consul connect ingress gateway demo Add the ingress gateway example from the noamd connect examples to the e2e Connect suite. Includes the ACLs enabled version, which means the nomad server consul acl policy will require operator=write permission. --- e2e/connect/acls.go | 35 +++++++++- e2e/connect/connect.go | 11 ++++ e2e/connect/input/ingress-gateway.nomad | 88 +++++++++++++++++++++++++ e2e/consulacls/nomad-server-policy.hcl | 11 ++-- 4 files changed, 140 insertions(+), 5 deletions(-) create mode 100644 e2e/connect/input/ingress-gateway.nomad diff --git a/e2e/connect/acls.go b/e2e/connect/acls.go index 3a89119aa..42d382ddc 100644 --- a/e2e/connect/acls.go +++ b/e2e/connect/acls.go @@ -25,6 +25,9 @@ const ( // demoConnectNativeJob is the example connect native enabled job useful for testing demoConnectNativeJob = "connect/input/native-demo.nomad" + + // demoConnectIngressGateway is the example ingress gateway job useful for testing + demoConnectIngressGateway = "connect/input/ingress-gateway.nomad" ) type ConnectACLsE2ETest struct { @@ -312,7 +315,37 @@ func (tc *ConnectACLsE2ETest) TestConnectACLsConnectNativeDemo(f *framework.F) { r.Equal(1, foundSITokens["frontend"], "expected 1 SI token for frontend: %v", foundSITokens) r.Equal(1, foundSITokens["generate"], "expected 1 SI token for generate: %v", foundSITokens) - t.Log("connect native job with ACLs enable finished") + t.Log("connect native job with ACLs enabled finished") +} + +func (tc *ConnectACLsE2ETest) TestConnectACLsConnectIngressGatewayDemo(f *framework.F) { + t := f.T() + r := require.New(t) + + t.Log("test register Connect Ingress Gateway job w/ ACLs enabled") + + // setup ACL policy and mint operator token + + policyID := tc.createConsulPolicy(consulPolicy{ + Name: "nomad-operator-policy", + Rules: `service "my-ingress-service" { policy = "write" } service "uuid-api" { policy = "write" }`, + }, f) + operatorToken := tc.createOperatorToken(policyID, f) + t.Log("created operator token:", operatorToken) + + jobID := connectJobID() + tc.jobIDs = append(tc.jobIDs, jobID) + + allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), demoConnectIngressGateway, jobID, operatorToken) + allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocs) + e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs) + + foundSITokens := tc.countSITokens(t) + r.Equal(2, len(foundSITokens), "expected 2 SI tokens total: %v", foundSITokens) + r.Equal(1, foundSITokens["connect-ingress-my-ingress-service"], "expected 1 SI token for connect-ingress-my-ingress-service: %v", foundSITokens) + r.Equal(1, foundSITokens["generate"], "expected 1 SI token for generate: %v", foundSITokens) + + t.Log("connect ingress gateway job with ACLs enabled finished") } var ( diff --git a/e2e/connect/connect.go b/e2e/connect/connect.go index 561ff3b50..66d582dc4 100644 --- a/e2e/connect/connect.go +++ b/e2e/connect/connect.go @@ -81,3 +81,14 @@ func (tc *ConnectE2ETest) TestConnectNativeDemo(f *framework.F) { allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocs) e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs) } + +func (tc *ConnectE2ETest) TestConnectIngressGatewayDemo(f *framework.F) { + t := f.T() + + jobID := connectJobID() + tc.jobIds = append(tc.jobIds, jobID) + + allocs := e2eutil.RegisterAndWaitForAllocs(t, tc.Nomad(), demoConnectIngressGateway, jobID, "") + allocIDs := e2eutil.AllocIDsFromAllocationListStubs(allocs) + e2eutil.WaitForAllocsRunning(t, tc.Nomad(), allocIDs) +} diff --git a/e2e/connect/input/ingress-gateway.nomad b/e2e/connect/input/ingress-gateway.nomad new file mode 100644 index 000000000..9efbb53fb --- /dev/null +++ b/e2e/connect/input/ingress-gateway.nomad @@ -0,0 +1,88 @@ +job "ig-bridge-demo" { + + datacenters = ["dc1"] + + constraint { + attribute = "${attr.kernel.name}" + value = "linux" + } + + group "ingress-group" { + + network { + mode = "bridge" + port "inbound" { + static = 8080 + to = 8080 + } + } + + service { + name = "my-ingress-service" + port = "8080" + + connect { + gateway { + proxy { + # The following options are automatically set by Nomad if not + # explicitly configured when using bridge networking. + # + # envoy_gateway_no_default_bind = true + # envoy_gateway_bind_addresses "uuid-api" { + # address = "0.0.0.0" + # port = + # } + # + # Additional options are documented at + # https://www.nomadproject.io/docs/job-specification/gateway#proxy-parameters + } + + ingress { + # Nomad will automatically manage the Configuration Entry in Consul + # given the parameters in the ingress block. + # + # Additional options are documented at + # https://www.nomadproject.io/docs/job-specification/gateway#ingress-parameters + listener { + port = 8080 + protocol = "tcp" + service { + name = "uuid-api" + } + } + } + } + } + } + } + + group "generator" { + network { + mode = "host" + port "api" {} + } + + service { + name = "uuid-api" + port = "${NOMAD_PORT_api}" + + connect { + native = true + } + } + + task "generate" { + driver = "docker" + + config { + image = "hashicorpnomad/uuid-api:v3" + network_mode = "host" + } + + env { + BIND = "0.0.0.0" + PORT = "${NOMAD_PORT_api}" + } + } + } +} diff --git a/e2e/consulacls/nomad-server-policy.hcl b/e2e/consulacls/nomad-server-policy.hcl index 160d1a966..4753838a2 100644 --- a/e2e/consulacls/nomad-server-policy.hcl +++ b/e2e/consulacls/nomad-server-policy.hcl @@ -1,8 +1,11 @@ -// The Nomad Server requires total access to Consul ACLs, because the Server -// will be requesting new SI tokens from Consul. - +// The acl=write permission is required for generating Consul Service Identity +// tokens for consul connect services. acl = "write" +// The operator=write permission is required for creating config entries for +// connect ingress gateways. +operator = "write" + service_prefix "" { policy = "write" } @@ -13,4 +16,4 @@ agent_prefix "" { node_prefix "" { policy = "read" -} +} \ No newline at end of file