consul: allow stale namespace results (#12953)

Nomad reconciles services it expects to be registered in Consul with
what is actually registered in the local Consul agent. This is necessary
to prevent leaking service registrations if Nomad crashes at certain
points (or if there are bugs).

When Consul has namespaces enabled, we must iterate over each available
namespace to be sure no services were leaked into non-default
namespaces.

Since this reconciliation happens often, there's no need to require
results from the Consul leader server. In large clusters this creates
far more load than the "freshness" of the response is worth.

Therefore this patch switches the request to AllowStale=true

.changelog/12953.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+consul: Reduce load on Consul leader server by allowing stale results when listing namespaces.
+```

command/agent/consul/namespaces_client.go

@@ -4,6 +4,8 @@ import (
 	"sort"
 	"sync"
 	"time"
+
+	"github.com/hashicorp/consul/api"
 )
 
 const (
@@ -66,8 +68,6 @@ func (ns *NamespacesClient) allowable(now time.Time) bool {
 }
 
 // List returns a list of Consul Namespaces.
-//
-// TODO(shoenig): return empty string instead of "default" when namespaces are not enabled. (Coming in followup PR).
 func (ns *NamespacesClient) List() ([]string, error) {
 	if !ns.allowable(time.Now()) {
 		// TODO(shoenig): lets return the empty string instead, that way we do not
@@ -75,7 +75,10 @@ func (ns *NamespacesClient) List() ([]string, error) {
 		return []string{"default"}, nil
 	}
 
-	namespaces, _, err := ns.namespacesAPI.List(nil)
+	qo := &api.QueryOptions{
+		AllowStale: true,
+	}
+	namespaces, _, err := ns.namespacesAPI.List(qo)
 	if err != nil {
 		return nil, err
 	}