consul: plubming for specifying consul namespace in job/group

This PR adds the common OSS changes for adding support for Consul Namespaces,
which is going to be a Nomad Enterprise feature. There is no new functionality
provided by this changeset and hopefully no new bugs.

vendor/github.com/hashicorp/consul/api/acl.go

@@ -270,16 +270,61 @@ type ACLAuthMethodNamespaceRule struct {
 type ACLAuthMethodListEntry struct {
 	Name        string
 	Type        string
-	DisplayName string `json:",omitempty"`
-	Description string `json:",omitempty"`
-	CreateIndex uint64
-	ModifyIndex uint64
+	DisplayName string        `json:",omitempty"`
+	Description string        `json:",omitempty"`
+	MaxTokenTTL time.Duration `json:",omitempty"`
+
+	// TokenLocality defines the kind of token that this auth method produces.
+	// This can be either 'local' or 'global'. If empty 'local' is assumed.
+	TokenLocality string `json:",omitempty"`
+	CreateIndex   uint64
+	ModifyIndex   uint64
 
 	// Namespace is the namespace the ACLAuthMethodListEntry is associated with.
 	// Namespacing is a Consul Enterprise feature.
 	Namespace string `json:",omitempty"`
 }
 
+// This is nearly identical to the ACLAuthMethod MarshalJSON
+func (m *ACLAuthMethodListEntry) MarshalJSON() ([]byte, error) {
+	type Alias ACLAuthMethodListEntry
+	exported := &struct {
+		MaxTokenTTL string `json:",omitempty"`
+		*Alias
+	}{
+		MaxTokenTTL: m.MaxTokenTTL.String(),
+		Alias:       (*Alias)(m),
+	}
+	if m.MaxTokenTTL == 0 {
+		exported.MaxTokenTTL = ""
+	}
+
+	return json.Marshal(exported)
+}
+
+// This is nearly identical to the ACLAuthMethod UnmarshalJSON
+func (m *ACLAuthMethodListEntry) UnmarshalJSON(data []byte) error {
+	type Alias ACLAuthMethodListEntry
+	aux := &struct {
+		MaxTokenTTL string
+		*Alias
+	}{
+		Alias: (*Alias)(m),
+	}
+
+	if err := json.Unmarshal(data, &aux); err != nil {
+		return err
+	}
+	var err error
+	if aux.MaxTokenTTL != "" {
+		if m.MaxTokenTTL, err = time.ParseDuration(aux.MaxTokenTTL); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // ParseKubernetesAuthMethodConfig takes a raw config map and returns a parsed
 // KubernetesAuthMethodConfig.
 func ParseKubernetesAuthMethodConfig(raw map[string]interface{}) (*KubernetesAuthMethodConfig, error) {

vendor/github.com/hashicorp/consul/api/agent.go

@@ -3,6 +3,7 @@ package api
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"net/http"
@@ -121,6 +122,7 @@ type AgentServiceConnectProxyConfig struct {
 	Upstreams              []Upstream             `json:",omitempty"`
 	MeshGateway            MeshGatewayConfig      `json:",omitempty"`
 	Expose                 ExposeConfig           `json:",omitempty"`
+	TransparentProxy       bool                   `json:",omitempty"`
 }
 
 const (
@@ -266,12 +268,23 @@ type AgentServiceRegistration struct {
 	Namespace         string                          `json:",omitempty" bexpr:"-" hash:"ignore"`
 }
 
-//ServiceRegisterOpts is used to pass extra options to the service register.
+// ServiceRegisterOpts is used to pass extra options to the service register.
 type ServiceRegisterOpts struct {
 	//Missing healthchecks will be deleted from the agent.
 	//Using this parameter allows to idempotently register a service and its checks without
 	//having to manually deregister checks.
 	ReplaceExistingChecks bool
+
+	// ctx is an optional context pass through to the underlying HTTP
+	// request layer. Use WithContext() to set the context.
+	ctx context.Context
+}
+
+// WithContext sets the context to be used for the request on a new ServiceRegisterOpts,
+// and returns the opts.
+func (o ServiceRegisterOpts) WithContext(ctx context.Context) ServiceRegisterOpts {
+	o.ctx = ctx
+	return o
 }
 
 // AgentCheckRegistration is used to register a new check
@@ -301,6 +314,7 @@ type AgentServiceCheck struct {
 	TCP                    string              `json:",omitempty"`
 	Status                 string              `json:",omitempty"`
 	Notes                  string              `json:",omitempty"`
+	TLSServerName          string              `json:",omitempty"`
 	TLSSkipVerify          bool                `json:",omitempty"`
 	GRPC                   string              `json:",omitempty"`
 	GRPCUseTLS             bool                `json:",omitempty"`
@@ -394,6 +408,7 @@ type Upstream struct {
 	LocalBindPort        int                    `json:",omitempty"`
 	Config               map[string]interface{} `json:",omitempty" bexpr:"-"`
 	MeshGateway          MeshGatewayConfig      `json:",omitempty"`
+	CentrallyConfigured  bool                   `json:",omitempty" bexpr:"-"`
 }
 
 // Agent can be used to query the Agent endpoints
@@ -494,7 +509,14 @@ func (a *Agent) Checks() (map[string]*AgentCheck, error) {
 // ChecksWithFilter returns a subset of the locally registered checks that match
 // the given filter expression
 func (a *Agent) ChecksWithFilter(filter string) (map[string]*AgentCheck, error) {
+	return a.ChecksWithFilterOpts(filter, nil)
+}
+
+// ChecksWithFilterOpts returns a subset of the locally registered checks that match
+// the given filter expression and QueryOptions.
+func (a *Agent) ChecksWithFilterOpts(filter string, q *QueryOptions) (map[string]*AgentCheck, error) {
 	r := a.c.newRequest("GET", "/v1/agent/checks")
+	r.setQueryOptions(q)
 	r.filterQuery(filter)
 	_, resp, err := requireOK(a.c.doRequest(r))
 	if err != nil {
@@ -517,7 +539,14 @@ func (a *Agent) Services() (map[string]*AgentService, error) {
 // ServicesWithFilter returns a subset of the locally registered services that match
 // the given filter expression
 func (a *Agent) ServicesWithFilter(filter string) (map[string]*AgentService, error) {
+	return a.ServicesWithFilterOpts(filter, nil)
+}
+
+// ServicesWithFilterOpts returns a subset of the locally registered services that match
+// the given filter expression and QueryOptions.
+func (a *Agent) ServicesWithFilterOpts(filter string, q *QueryOptions) (map[string]*AgentService, error) {
 	r := a.c.newRequest("GET", "/v1/agent/services")
+	r.setQueryOptions(q)
 	r.filterQuery(filter)
 	_, resp, err := requireOK(a.c.doRequest(r))
 	if err != nil {
@@ -688,6 +717,7 @@ func (a *Agent) ServiceRegisterOpts(service *AgentServiceRegistration, opts Serv
 func (a *Agent) serviceRegister(service *AgentServiceRegistration, opts ServiceRegisterOpts) error {
 	r := a.c.newRequest("PUT", "/v1/agent/service/register")
 	r.obj = service
+	r.ctx = opts.ctx
 	if opts.ReplaceExistingChecks {
 		r.params.Set("replace-existing-checks", "true")
 	}
@@ -711,6 +741,19 @@ func (a *Agent) ServiceDeregister(serviceID string) error {
 	return nil
 }
 
+// ServiceDeregisterOpts is used to deregister a service with
+// the local agent with QueryOptions.
+func (a *Agent) ServiceDeregisterOpts(serviceID string, q *QueryOptions) error {
+	r := a.c.newRequest("PUT", "/v1/agent/service/deregister/"+serviceID)
+	r.setQueryOptions(q)
+	_, resp, err := requireOK(a.c.doRequest(r))
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	return nil
+}
+
 // PassTTL is used to set a TTL check to the passing state.
 //
 // DEPRECATION NOTICE: This interface is deprecated in favor of UpdateTTL().
@@ -785,6 +828,10 @@ type checkUpdate struct {
 // strings for compatibility (though a newer version of Consul will still be
 // required to use this API).
 func (a *Agent) UpdateTTL(checkID, output, status string) error {
+	return a.UpdateTTLOpts(checkID, output, status, nil)
+}
+
+func (a *Agent) UpdateTTLOpts(checkID, output, status string, q *QueryOptions) error {
 	switch status {
 	case "pass", HealthPassing:
 		status = HealthPassing
@@ -798,6 +845,7 @@ func (a *Agent) UpdateTTL(checkID, output, status string) error {
 
 	endpoint := fmt.Sprintf("/v1/agent/check/update/%s", checkID)
 	r := a.c.newRequest("PUT", endpoint)
+	r.setQueryOptions(q)
 	r.obj = &checkUpdate{
 		Status: status,
 		Output: output,
@@ -827,7 +875,14 @@ func (a *Agent) CheckRegister(check *AgentCheckRegistration) error {
 // CheckDeregister is used to deregister a check with
 // the local agent
 func (a *Agent) CheckDeregister(checkID string) error {
+	return a.CheckDeregisterOpts(checkID, nil)
+}
+
+// CheckDeregisterOpts is used to deregister a check with
+// the local agent using query options
+func (a *Agent) CheckDeregisterOpts(checkID string, q *QueryOptions) error {
 	r := a.c.newRequest("PUT", "/v1/agent/check/deregister/"+checkID)
+	r.setQueryOptions(q)
 	_, resp, err := requireOK(a.c.doRequest(r))
 	if err != nil {
 		return err

vendor/github.com/hashicorp/consul/api/api.go

@@ -14,6 +14,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/hashicorp/go-cleanhttp"
@@ -548,9 +549,48 @@ func (c *Config) GenerateEnv() []string {
 
 // Client provides a client to the Consul API
 type Client struct {
+	modifyLock sync.RWMutex
+	headers    http.Header
+
 	config Config
 }
 
+// Headers gets the current set of headers used for requests. This returns a
+// copy; to modify it call AddHeader or SetHeaders.
+func (c *Client) Headers() http.Header {
+	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
+
+	if c.headers == nil {
+		return nil
+	}
+
+	ret := make(http.Header)
+	for k, v := range c.headers {
+		for _, val := range v {
+			ret[k] = append(ret[k], val)
+		}
+	}
+
+	return ret
+}
+
+// AddHeader allows a single header key/value pair to be added
+// in a race-safe fashion.
+func (c *Client) AddHeader(key, value string) {
+	c.modifyLock.Lock()
+	defer c.modifyLock.Unlock()
+	c.headers.Add(key, value)
+}
+
+// SetHeaders clears all previous headers and uses only the given
+// ones going forward.
+func (c *Client) SetHeaders(headers http.Header) {
+	c.modifyLock.Lock()
+	defer c.modifyLock.Unlock()
+	c.headers = headers
+}
+
 // NewClient returns a new client
 func NewClient(config *Config) (*Client, error) {
 	// bootstrap the config
@@ -640,7 +680,7 @@ func NewClient(config *Config) (*Client, error) {
 		config.Token = defConfig.Token
 	}
 
-	return &Client{config: *config}, nil
+	return &Client{config: *config, headers: make(http.Header)}, nil
 }
 
 // NewHttpClient returns an http client configured with the given Transport and TLS
@@ -853,8 +893,9 @@ func (c *Client) newRequest(method, path string) *request {
 			Path:   path,
 		},
 		params: make(map[string][]string),
-		header: make(http.Header),
+		header: c.Headers(),
 	}
+
 	if c.config.Datacenter != "" {
 		r.params.Set("dc", c.config.Datacenter)
 	}

vendor/github.com/hashicorp/consul/api/config_entry.go

@@ -91,17 +91,94 @@ type ExposePath struct {
 	ParsedFromCheck bool
 }
 
+type ConnectConfiguration struct {
+	// UpstreamConfigs is a map of <namespace/>service to per-upstream configuration
+	UpstreamConfigs map[string]UpstreamConfig `json:",omitempty" alias:"upstream_configs"`
+
+	// UpstreamDefaults contains default configuration for all upstreams of a given service
+	UpstreamDefaults UpstreamConfig `json:",omitempty" alias:"upstream_defaults"`
+}
+
+type UpstreamConfig struct {
+	// EnvoyListenerJSON is a complete override ("escape hatch") for the upstream's
+	// listener.
+	//
+	// Note: This escape hatch is NOT compatible with the discovery chain and
+	// will be ignored if a discovery chain is active.
+	EnvoyListenerJSON string `json:",omitempty" alias:"envoy_listener_json"`
+
+	// EnvoyClusterJSON is a complete override ("escape hatch") for the upstream's
+	// cluster. The Connect client TLS certificate and context will be injected
+	// overriding any TLS settings present.
+	//
+	// Note: This escape hatch is NOT compatible with the discovery chain and
+	// will be ignored if a discovery chain is active.
+	EnvoyClusterJSON string `json:",omitempty" alias:"envoy_cluster_json"`
+
+	// Protocol describes the upstream's service protocol. Valid values are "tcp",
+	// "http" and "grpc". Anything else is treated as tcp. The enables protocol
+	// aware features like per-request metrics and connection pooling, tracing,
+	// routing etc.
+	Protocol string `json:",omitempty"`
+
+	// ConnectTimeoutMs is the number of milliseconds to timeout making a new
+	// connection to this upstream. Defaults to 5000 (5 seconds) if not set.
+	ConnectTimeoutMs int `json:",omitempty" alias:"connect_timeout_ms"`
+
+	// Limits are the set of limits that are applied to the proxy for a specific upstream of a
+	// service instance.
+	Limits *UpstreamLimits `json:",omitempty"`
+
+	// PassiveHealthCheck configuration determines how upstream proxy instances will
+	// be monitored for removal from the load balancing pool.
+	PassiveHealthCheck *PassiveHealthCheck `json:",omitempty" alias:"passive_health_check"`
+
+	// MeshGatewayConfig controls how Mesh Gateways are configured and used
+	MeshGateway MeshGatewayConfig `json:",omitempty" alias:"mesh_gateway" `
+}
+
+type PassiveHealthCheck struct {
+	// Interval between health check analysis sweeps. Each sweep may remove
+	// hosts or return hosts to the pool.
+	Interval time.Duration `json:",omitempty"`
+
+	// MaxFailures is the count of consecutive failures that results in a host
+	// being removed from the pool.
+	MaxFailures uint32 `alias:"max_failures"`
+}
+
+// UpstreamLimits describes the limits that are associated with a specific
+// upstream of a service instance.
+type UpstreamLimits struct {
+	// MaxConnections is the maximum number of connections the local proxy can
+	// make to the upstream service.
+	MaxConnections int `alias:"max_connections"`
+
+	// MaxPendingRequests is the maximum number of requests that will be queued
+	// waiting for an available connection. This is mostly applicable to HTTP/1.1
+	// clusters since all HTTP/2 requests are streamed over a single
+	// connection.
+	MaxPendingRequests int `alias:"max_pending_requests"`
+
+	// MaxConcurrentRequests is the maximum number of in-flight requests that will be allowed
+	// to the upstream cluster at a point in time. This is mostly applicable to HTTP/2
+	// clusters since all HTTP/1.1 requests are limited by MaxConnections.
+	MaxConcurrentRequests int `alias:"max_concurrent_requests"`
+}
+
 type ServiceConfigEntry struct {
-	Kind        string
-	Name        string
-	Namespace   string            `json:",omitempty"`
-	Protocol    string            `json:",omitempty"`
-	MeshGateway MeshGatewayConfig `json:",omitempty" alias:"mesh_gateway"`
-	Expose      ExposeConfig      `json:",omitempty"`
-	ExternalSNI string            `json:",omitempty" alias:"external_sni"`
-	Meta        map[string]string `json:",omitempty"`
-	CreateIndex uint64
-	ModifyIndex uint64
+	Kind             string
+	Name             string
+	Namespace        string               `json:",omitempty"`
+	Protocol         string               `json:",omitempty"`
+	MeshGateway      MeshGatewayConfig    `json:",omitempty" alias:"mesh_gateway"`
+	Connect          ConnectConfiguration `json:",omitempty"`
+	Expose           ExposeConfig         `json:",omitempty"`
+	TransparentProxy bool                 `json:",omitempty" alias:"transparent_proxy"`
+	ExternalSNI      string               `json:",omitempty" alias:"external_sni"`
+	Meta             map[string]string    `json:",omitempty"`
+	CreateIndex      uint64
+	ModifyIndex      uint64
 }
 
 func (s *ServiceConfigEntry) GetKind() string {
@@ -129,15 +206,16 @@ func (s *ServiceConfigEntry) GetModifyIndex() uint64 {
 }
 
 type ProxyConfigEntry struct {
-	Kind        string
-	Name        string
-	Namespace   string                 `json:",omitempty"`
-	Config      map[string]interface{} `json:",omitempty"`
-	MeshGateway MeshGatewayConfig      `json:",omitempty" alias:"mesh_gateway"`
-	Expose      ExposeConfig           `json:",omitempty"`
-	Meta        map[string]string      `json:",omitempty"`
-	CreateIndex uint64
-	ModifyIndex uint64
+	Kind             string
+	Name             string
+	Namespace        string                 `json:",omitempty"`
+	Config           map[string]interface{} `json:",omitempty"`
+	MeshGateway      MeshGatewayConfig      `json:",omitempty" alias:"mesh_gateway"`
+	Expose           ExposeConfig           `json:",omitempty"`
+	TransparentProxy bool                   `json:",omitempty" alias:"transparent_proxy"`
+	Meta             map[string]string      `json:",omitempty"`
+	CreateIndex      uint64
+	ModifyIndex      uint64
 }
 
 func (p *ProxyConfigEntry) GetKind() string {

vendor/github.com/hashicorp/consul/api/connect_ca.go

@@ -23,6 +23,14 @@ type CAConfig struct {
 	// configuration is an error.
 	State map[string]string
 
+	// ForceWithoutCrossSigning indicates that the CA reconfiguration should go
+	// ahead even if the current CA is unable to cross sign certificates. This
+	// risks temporary connection failures during the rollout as new leafs will be
+	// rejected by proxies that have not yet observed the new root cert but is the
+	// only option if a CA that doesn't support cross signing needs to be
+	// reconfigured or mirated away from.
+	ForceWithoutCrossSigning bool
+
 	CreateIndex uint64
 	ModifyIndex uint64
 }

vendor/github.com/hashicorp/consul/api/go.sum

@@ -83,6 +83,7 @@ github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSg
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=

vendor/github.com/hashicorp/consul/api/health.go

@@ -58,6 +58,7 @@ type HealthCheckDefinition struct {
 	Header                                 map[string][]string
 	Method                                 string
 	Body                                   string
+	TLSServerName                          string
 	TLSSkipVerify                          bool
 	TCP                                    string
 	IntervalDuration                       time.Duration `json:"-"`

vendor/github.com/hashicorp/consul/api/operator_autopilot.go

@@ -334,10 +334,23 @@ func (op *Operator) AutopilotCASConfiguration(conf *AutopilotConfiguration, q *W
 func (op *Operator) AutopilotServerHealth(q *QueryOptions) (*OperatorHealthReply, error) {
 	r := op.c.newRequest("GET", "/v1/operator/autopilot/health")
 	r.setQueryOptions(q)
-	_, resp, err := requireOK(op.c.doRequest(r))
+
+	// we cannot just use requireOK because this endpoint might use a 429 status to indicate
+	// that unhealthiness
+	_, resp, err := op.c.doRequest(r)
 	if err != nil {
+		if resp != nil {
+			resp.Body.Close()
+		}
 		return nil, err
 	}
+
+	// these are the only 2 status codes that would indicate that we should
+	// expect the body to contain the right format.
+	if resp.StatusCode != 200 && resp.StatusCode != 429 {
+		return nil, generateUnexpectedResponseCodeError(resp)
+	}
+
 	defer resp.Body.Close()
 
 	var out OperatorHealthReply

vendor/github.com/hashicorp/nomad/api/consul.go

@@ -0,0 +1,35 @@
+package api
+
+// Consul represents configuration related to consul.
+type Consul struct {
+	// (Enterprise-only) Namespace represents a Consul namespace.
+	Namespace string `mapstructure:"namespace" hcl:"namespace,optional"`
+}
+
+// Canonicalize Consul into a canonical form. The Canonicalize structs containing
+// a Consul should ensure it is not nil.
+func (c *Consul) Canonicalize() {
+	// Nothing to do here.
+	//
+	// If Namespace is nil, that is a choice of the job submitter that
+	// we should inherit from higher up (i.e. job<-group). Likewise, if
+	// Namespace is set but empty, that is a choice to use the default consul
+	// namespace.
+}
+
+// Copy creates a deep copy of c.
+func (c *Consul) Copy() *Consul {
+	return &Consul{
+		Namespace: c.Namespace,
+	}
+}
+
+// MergeNamespace sets Namespace to namespace if not already configured.
+// This is used to inherit the job-level consul_namespace if the group-level
+// namespace is not explicitly configured.
+func (c *Consul) MergeNamespace(namespace *string) {
+	// only inherit namespace from above if not already set
+	if c.Namespace == "" && namespace != nil {
+		c.Namespace = *namespace
+	}
+}

vendor/github.com/hashicorp/nomad/api/jobs.go

@@ -817,6 +817,7 @@ type Job struct {
 	ParentID          *string
 	Dispatched        bool
 	Payload           []byte
+	ConsulNamespace   *string `mapstructure:"consul_namespace"`
 	VaultNamespace    *string `mapstructure:"vault_namespace"`
 	NomadTokenID      *string `mapstructure:"nomad_token_id"`
 	Status            *string
@@ -878,6 +879,9 @@ func (j *Job) Canonicalize() {
 	if j.ConsulToken == nil {
 		j.ConsulToken = stringToPtr("")
 	}
+	if j.ConsulNamespace == nil {
+		j.ConsulNamespace = stringToPtr("")
+	}
 	if j.VaultToken == nil {
 		j.VaultToken = stringToPtr("")
 	}

vendor/github.com/hashicorp/nomad/api/tasks.go

@@ -430,6 +430,7 @@ type TaskGroup struct {
 	ShutdownDelay             *time.Duration            `mapstructure:"shutdown_delay" hcl:"shutdown_delay,optional"`
 	StopAfterClientDisconnect *time.Duration            `mapstructure:"stop_after_client_disconnect" hcl:"stop_after_client_disconnect,optional"`
 	Scaling                   *ScalingPolicy            `hcl:"scaling,block"`
+	Consul                    *Consul                   `hcl:"consul,block"`
 }
 
 // NewTaskGroup creates a new TaskGroup.
@@ -462,6 +463,13 @@ func (g *TaskGroup) Canonicalize(job *Job) {
 		g.EphemeralDisk.Canonicalize()
 	}
 
+	// Merge job.consul onto group.consul
+	if g.Consul == nil {
+		g.Consul = new(Consul)
+	}
+	g.Consul.MergeNamespace(job.ConsulNamespace)
+	g.Consul.Canonicalize()
+
 	// Merge the update policy from the job
 	if ju, tu := job.Update != nil, g.Update != nil; ju && tu {
 		// Merge the jobs and task groups definition of the update strategy