tls enforcement flaky tests (#16543)

* tests: add WaitForLeaders helpers using must/wait timings

* tests: start servers for mtls tests together

Fixes #16253 (hopefully)

testutil/wait.go

@@ -11,6 +11,7 @@ import (
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/kr/pretty"
 	"github.com/shoenig/test/must"
+	"github.com/shoenig/test/wait"
 )
 
 type testFn func() (bool, error)
@@ -142,6 +143,25 @@ func WaitForLeader(t testing.TB, rpc rpcFn) {
 	})
 }
 
+// WaitForLeaders blocks until each serverRPC knows the leader.
+func WaitForLeaders(t testing.TB, serverRPCs ...rpcFn) {
+	t.Helper()
+
+	for i := 0; i < len(serverRPCs); i++ {
+		ok := func() (bool, error) {
+			args := &structs.GenericRequest{}
+			var leader string
+			err := serverRPCs[i]("Status.Leader", args, &leader)
+			return leader != "", err
+		}
+		must.Wait(t, wait.InitialSuccess(
+			wait.TestFunc(ok),
+			wait.Timeout(10*time.Second),
+			wait.Gap(1*time.Second),
+		))
+	}
+}
+
 // WaitForClient blocks until the client can be found
 func WaitForClient(t testing.TB, rpc rpcFn, nodeID string, region string) {
 	t.Helper()