consul connect: allow "cni/*" network mode (#26449)

don't require "bridge" network mode when using connect{}

we document this as "at your own risk" because CNI configuration
is so flexible that we can't guarantee a user's network will work,
but Nomad's "bridge" CNI config may be used as a reference.

e2e/connect/connect_test.go

@@ -25,7 +25,8 @@ func TestConnect(t *testing.T) {
 		test.NoError(t, err)
 	})
 
-	t.Run("ConnectDemo", testConnectDemo)
+	t.Run("ConnectDemo", testConnectDemo("bridge"))
+	t.Run("ConnectDemoCNI", testConnectDemo("cni/nomad-bridge-copy"))
 	t.Run("ConnectCustomSidecarExposed", testConnectCustomSidecarExposed)
 	t.Run("ConnectNativeDemo", testConnectNativeDemo)
 	t.Run("ConnectIngressGatewayDemo", testConnectIngressGatewayDemo)
@@ -36,30 +37,34 @@ func TestConnect(t *testing.T) {
 }
 
 // testConnectDemo tests the demo job file used in Connect Integration examples.
-func testConnectDemo(t *testing.T) {
-	sub, _ := jobs3.Submit(t, "./input/demo.nomad", jobs3.Timeout(time.Second*60))
+func testConnectDemo(networkMode string) func(t *testing.T) {
+	return func(t *testing.T) {
+		sub, _ := jobs3.Submit(t, "./input/demo.nomad", jobs3.Timeout(time.Second*60),
+			jobs3.Var("network_mode", networkMode),
+		)
 
-	cc := e2eutil.ConsulClient(t)
+		cc := e2eutil.ConsulClient(t)
 
-	ixn := &capi.Intention{
-		SourceName:      "count-dashboard",
-		DestinationName: "count-api",
-		Action:          "allow",
+		ixn := &capi.Intention{
+			SourceName:      "count-dashboard",
+			DestinationName: "count-api",
+			Action:          "allow",
+		}
+		_, err := cc.Connect().IntentionUpsert(ixn, nil)
+		must.NoError(t, err, must.Sprint("could not create intention"))
+
+		t.Cleanup(func() {
+			_, err := cc.Connect().IntentionDeleteExact("count-dashboard", "count-api", nil)
+			test.NoError(t, err)
+		})
+
+		assertServiceOk(t, cc, "count-api-sidecar-proxy")
+		assertServiceOk(t, cc, "count-dashboard-sidecar-proxy")
+
+		logs := sub.Exec("dashboard", "dashboard",
+			[]string{"/bin/sh", "-c", "wget -O /dev/null http://${NOMAD_UPSTREAM_ADDR_count_api}"})
+		must.StrContains(t, logs.Stderr, "saving to")
 	}
-	_, err := cc.Connect().IntentionUpsert(ixn, nil)
-	must.NoError(t, err, must.Sprint("could not create intention"))
-
-	t.Cleanup(func() {
-		_, err := cc.Connect().IntentionDeleteExact("count-dashboard", "count-api", nil)
-		test.NoError(t, err)
-	})
-
-	assertServiceOk(t, cc, "count-api-sidecar-proxy")
-	assertServiceOk(t, cc, "count-dashboard-sidecar-proxy")
-
-	logs := sub.Exec("dashboard", "dashboard",
-		[]string{"/bin/sh", "-c", "wget -O /dev/null http://${NOMAD_UPSTREAM_ADDR_count_api}"})
-	must.StrContains(t, logs.Stderr, "saving to")
 }
 
 // testConnectCustomSidecarExposed tests that a connect sidecar with custom task

e2e/connect/input/demo.nomad

@@ -1,6 +1,10 @@
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 
+variable "network_mode" {
+  default = "bridge"
+}
+
 job "countdash" {
   datacenters = ["dc1"]
 
@@ -11,7 +15,7 @@ job "countdash" {
 
   group "api" {
     network {
-      mode = "bridge"
+      mode = var.network_mode
     }
 
     service {
@@ -43,7 +47,7 @@ job "countdash" {
 
   group "dashboard" {
     network {
-      mode = "bridge"
+      mode = var.network_mode
 
       port "http" {
         static = 9002

e2e/terraform/packer/ubuntu-jammy-amd64/cni/nomad_bridge_copy.conflist

@@ -0,0 +1,41 @@
+{
+  "cniVersion": "1.0.0",
+  "name": "nomad-bridge-copy",
+  "plugins": [
+    {
+      "type": "loopback"
+    },
+    {
+      "type": "bridge",
+      "bridge": "nomad",
+      "ipMasq": true,
+      "isGateway": true,
+      "forceAddress": true,
+      "hairpinMode": false,
+      "ipam": {
+        "type": "host-local",
+        "ranges": [
+          [{"subnet": "172.26.64.0/20"}],
+          [{"subnet": "a110:c8::/112"}]
+        ],
+        "routes": [
+          {"dst": "0.0.0.0/0"},
+          {"dst": "::/0"}
+        ],
+        "dataDir": "/var/run/cni"
+      }
+    },
+    {
+      "type": "firewall",
+      "backend": "iptables",
+      "iptablesAdminChainName": "NOMAD-ADMIN"
+    },
+    {
+      "type": "portmap",
+      "capabilities": {
+        "portMappings": true
+      },
+      "snat": true
+    }
+  ]
+}

e2e/terraform/packer/ubuntu-jammy-amd64/setup.sh

@@ -109,6 +109,10 @@ sudo mv /tmp/linux/cni/loopback.* /opt/cni/config/
 sudo mv /tmp/linux/cni/cni_args.conflist /opt/cni/config/
 sudo mv /tmp/linux/cni/cni_args.sh /opt/cni/bin/
 
+echo "Installing additional CNI network configs"
+# copy of nomad's "bridge" for connect+cni test (e2e/connect/)
+sudo mv /tmp/linux/cni/nomad_bridge_copy.conflist /opt/cni/config/
+
 # Podman
 echo "Installing Podman"
 sudo apt-get -y install podman catatonit