keyring: filter by region before checking version (#14901)

In #14821 we fixed a panic that can happen if a leadership election happens in
the middle of an upgrade. That fix checks that all servers are at the minimum
version before initializing the keyring (which blocks evaluation processing
during trhe upgrade). But the check we implemented is over the serf membership,
which includes servers in any federated regions, which don't necessarily have
the same upgrade cycle.

Filter the version check by the leader's region.

Also bump up log levels of major keyring operations

.changelog/14901.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+keyring: Fixed a bug where keyring initialization is blocked by un-upgraded federated regions
+```

nomad/encrypter.go

@@ -474,7 +474,7 @@ START:
 					// new leader has not yet replicated the key from
 					// the old leader before the transition. Ask all
 					// the other servers if they have it.
-					krr.logger.Debug("failed to fetch key from current leader",
+					krr.logger.Warn("failed to fetch key from current leader, trying peers",
 						"key", keyID, "error", err)
 					getReq.AllowStale = true
 					for _, peer := range krr.getAllPeers() {
@@ -494,7 +494,7 @@ START:
 					krr.logger.Error("failed to add key", "key", keyID, "error", err)
 					goto ERR_WAIT
 				}
-				krr.logger.Trace("added key", "key", keyID)
+				krr.logger.Info("added key", "key", keyID)
 			}
 		}
 	}

nomad/leader.go

@@ -1990,7 +1990,15 @@ func (s *Server) initializeKeyring(stopCh <-chan struct{}) {
 			return
 		default:
 		}
-		if ServersMeetMinimumVersion(s.serf.Members(), minVersionKeyring, true) {
+
+		members := s.serf.Members()
+		regionMembers := []serf.Member{}
+		for _, member := range members {
+			if member.Tags["region"] == s.Region() {
+				regionMembers = append(regionMembers, member)
+			}
+		}
+		if ServersMeetMinimumVersion(regionMembers, minVersionKeyring, true) {
 			break
 		}
 	}