mirror of
https://github.com/kemko/nomad.git
synced 2026-01-01 16:05:42 +03:00
fingerprint: convert consul and vault fingerprinters to be reloadable (#24526)
This PR changes the Consul and Vault fingerprint implementations to be reloadable rather than periodic. Reasons described in the issue.
This commit is contained in:
3
.changelog/24526.txt
Normal file
3
.changelog/24526.txt
Normal file
@@ -0,0 +1,3 @@
|
||||
```release-note:breaking-change
|
||||
fingerprint: Consul and Vault fingerprints no longer reload periodically
|
||||
```
|
||||
@@ -12,12 +12,10 @@ import (
|
||||
|
||||
consulapi "github.com/hashicorp/consul/api"
|
||||
"github.com/hashicorp/go-hclog"
|
||||
log "github.com/hashicorp/go-hclog"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/hashicorp/go-sockaddr"
|
||||
"github.com/hashicorp/go-version"
|
||||
agentconsul "github.com/hashicorp/nomad/command/agent/consul"
|
||||
"github.com/hashicorp/nomad/helper"
|
||||
"github.com/hashicorp/nomad/nomad/structs"
|
||||
"github.com/hashicorp/nomad/nomad/structs/config"
|
||||
)
|
||||
@@ -28,34 +26,34 @@ var (
|
||||
// perform different fingerprinting depending on which version of Consul it
|
||||
// is communicating with.
|
||||
consulGRPCPortChangeVersion = version.Must(version.NewVersion("1.14.0"))
|
||||
|
||||
// consulBaseFingerprintInterval is the initial interval for periodic
|
||||
// fingerprinting
|
||||
consulBaseFingerprintInterval = 15 * time.Second
|
||||
)
|
||||
|
||||
// ConsulFingerprint is used to fingerprint for Consul
|
||||
type ConsulFingerprint struct {
|
||||
logger log.Logger
|
||||
states map[string]*consulFingerprintState
|
||||
logger hclog.Logger
|
||||
|
||||
// clusters maintains the latest fingerprinted state for each cluster
|
||||
// defined in nomad consul client configuration(s).
|
||||
clusters map[string]*consulState
|
||||
}
|
||||
|
||||
type consulFingerprintState struct {
|
||||
client *consulapi.Client
|
||||
isAvailable bool
|
||||
extractors map[string]consulExtractor
|
||||
nextCheck time.Time
|
||||
type consulState struct {
|
||||
client *consulapi.Client
|
||||
|
||||
// readers associates a function used to parse the value associated
|
||||
// with the given key from a consul api response
|
||||
readers map[string]valueReader
|
||||
}
|
||||
|
||||
// consulExtractor is used to parse out one attribute from consulInfo. Returns
|
||||
// valueReader is used to parse out one attribute from consulInfo. Returns
|
||||
// the value of the attribute, and whether the attribute exists.
|
||||
type consulExtractor func(agentconsul.Self) (string, bool)
|
||||
type valueReader func(agentconsul.Self) (string, bool)
|
||||
|
||||
// NewConsulFingerprint is used to create a Consul fingerprint
|
||||
func NewConsulFingerprint(logger log.Logger) Fingerprint {
|
||||
func NewConsulFingerprint(logger hclog.Logger) Fingerprint {
|
||||
return &ConsulFingerprint{
|
||||
logger: logger.Named("consul"),
|
||||
states: map[string]*consulFingerprintState{},
|
||||
logger: logger.Named("consul"),
|
||||
clusters: map[string]*consulState{},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,16 +71,12 @@ func (f *ConsulFingerprint) Fingerprint(req *FingerprintRequest, resp *Fingerpri
|
||||
}
|
||||
|
||||
func (f *ConsulFingerprint) fingerprintImpl(cfg *config.ConsulConfig, resp *FingerprintResponse) error {
|
||||
|
||||
logger := f.logger.With("cluster", cfg.Name)
|
||||
|
||||
state, ok := f.states[cfg.Name]
|
||||
state, ok := f.clusters[cfg.Name]
|
||||
if !ok {
|
||||
state = &consulFingerprintState{}
|
||||
f.states[cfg.Name] = state
|
||||
}
|
||||
if state.nextCheck.After(time.Now()) {
|
||||
return nil
|
||||
state = &consulState{}
|
||||
f.clusters[cfg.Name] = state
|
||||
}
|
||||
|
||||
if err := state.initialize(cfg, logger); err != nil {
|
||||
@@ -92,12 +86,13 @@ func (f *ConsulFingerprint) fingerprintImpl(cfg *config.ConsulConfig, resp *Fing
|
||||
// query consul for agent self api
|
||||
info := state.query(logger)
|
||||
if len(info) == 0 {
|
||||
// unable to reach consul, nothing to do this time
|
||||
// unable to reach consul, clear out existing attributes
|
||||
resp.Detected = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// apply the extractor for each attribute
|
||||
for attr, extractor := range state.extractors {
|
||||
for attr, extractor := range state.readers {
|
||||
if s, ok := extractor(info); !ok {
|
||||
logger.Warn("unable to fingerprint consul", "attribute", attr)
|
||||
} else if s != "" {
|
||||
@@ -108,38 +103,18 @@ func (f *ConsulFingerprint) fingerprintImpl(cfg *config.ConsulConfig, resp *Fing
|
||||
// create link for consul
|
||||
f.link(resp)
|
||||
|
||||
// indicate Consul is now available
|
||||
if !state.isAvailable {
|
||||
logger.Info("consul agent is available")
|
||||
}
|
||||
|
||||
// Widen the minimum window to the next check so that if one out of a set of
|
||||
// Consuls is unhealthy we don't greatly increase requests to the healthy
|
||||
// ones. This is less than the minimum window if all Consuls are healthy so
|
||||
// that we don't desync from the larger window provided by Periodic
|
||||
state.nextCheck = time.Now().Add(29 * time.Second)
|
||||
state.isAvailable = true
|
||||
resp.Detected = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *ConsulFingerprint) Periodic() (bool, time.Duration) {
|
||||
if len(f.states) == 0 {
|
||||
return true, consulBaseFingerprintInterval
|
||||
}
|
||||
for _, state := range f.states {
|
||||
if !state.isAvailable {
|
||||
return true, consulBaseFingerprintInterval
|
||||
}
|
||||
}
|
||||
|
||||
// Once all Consuls are initially discovered and healthy we fingerprint with
|
||||
// a wide jitter to avoid thundering herds of fingerprints against central
|
||||
// Consul servers.
|
||||
return true, (30 * time.Second) + helper.RandomStagger(90*time.Second)
|
||||
return false, 0
|
||||
}
|
||||
|
||||
func (cfs *consulFingerprintState) initialize(cfg *config.ConsulConfig, logger hclog.Logger) error {
|
||||
// Reload satisfies ReloadableFingerprint.
|
||||
func (f *ConsulFingerprint) Reload() {}
|
||||
|
||||
func (cfs *consulState) initialize(cfg *config.ConsulConfig, logger hclog.Logger) error {
|
||||
if cfs.client != nil {
|
||||
return nil // already initialized!
|
||||
}
|
||||
@@ -155,7 +130,7 @@ func (cfs *consulFingerprintState) initialize(cfg *config.ConsulConfig, logger h
|
||||
}
|
||||
|
||||
if cfg.Name == structs.ConsulDefaultCluster {
|
||||
cfs.extractors = map[string]consulExtractor{
|
||||
cfs.readers = map[string]valueReader{
|
||||
"consul.server": cfs.server,
|
||||
"consul.version": cfs.version,
|
||||
"consul.sku": cfs.sku,
|
||||
@@ -171,7 +146,7 @@ func (cfs *consulFingerprintState) initialize(cfg *config.ConsulConfig, logger h
|
||||
"consul.dns.addr": cfs.dnsAddr(logger),
|
||||
}
|
||||
} else {
|
||||
cfs.extractors = map[string]consulExtractor{
|
||||
cfs.readers = map[string]valueReader{
|
||||
fmt.Sprintf("consul.%s.server", cfg.Name): cfs.server,
|
||||
fmt.Sprintf("consul.%s.version", cfg.Name): cfs.version,
|
||||
fmt.Sprintf("consul.%s.sku", cfg.Name): cfs.sku,
|
||||
@@ -190,17 +165,12 @@ func (cfs *consulFingerprintState) initialize(cfg *config.ConsulConfig, logger h
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cfs *consulFingerprintState) query(logger hclog.Logger) agentconsul.Self {
|
||||
func (cfs *consulState) query(logger hclog.Logger) agentconsul.Self {
|
||||
// We'll try to detect consul by making a query to to the agent's self API.
|
||||
// If we can't hit this URL consul is probably not running on this machine.
|
||||
info, err := cfs.client.Agent().Self()
|
||||
if err != nil {
|
||||
// indicate consul no longer available
|
||||
if cfs.isAvailable {
|
||||
logger.Info("consul agent is unavailable", "error", err)
|
||||
}
|
||||
cfs.isAvailable = false
|
||||
cfs.nextCheck = time.Time{} // force check on next interval
|
||||
logger.Warn("failed to acquire consul self endpoint", "error", err)
|
||||
return nil
|
||||
}
|
||||
return info
|
||||
@@ -216,36 +186,36 @@ func (f *ConsulFingerprint) link(resp *FingerprintResponse) {
|
||||
}
|
||||
}
|
||||
|
||||
func (cfs *consulFingerprintState) server(info agentconsul.Self) (string, bool) {
|
||||
func (cfs *consulState) server(info agentconsul.Self) (string, bool) {
|
||||
s, ok := info["Config"]["Server"].(bool)
|
||||
return strconv.FormatBool(s), ok
|
||||
}
|
||||
|
||||
func (cfs *consulFingerprintState) version(info agentconsul.Self) (string, bool) {
|
||||
func (cfs *consulState) version(info agentconsul.Self) (string, bool) {
|
||||
v, ok := info["Config"]["Version"].(string)
|
||||
return v, ok
|
||||
}
|
||||
|
||||
func (cfs *consulFingerprintState) sku(info agentconsul.Self) (string, bool) {
|
||||
func (cfs *consulState) sku(info agentconsul.Self) (string, bool) {
|
||||
return agentconsul.SKU(info)
|
||||
}
|
||||
|
||||
func (cfs *consulFingerprintState) revision(info agentconsul.Self) (string, bool) {
|
||||
func (cfs *consulState) revision(info agentconsul.Self) (string, bool) {
|
||||
r, ok := info["Config"]["Revision"].(string)
|
||||
return r, ok
|
||||
}
|
||||
|
||||
func (cfs *consulFingerprintState) name(info agentconsul.Self) (string, bool) {
|
||||
func (cfs *consulState) name(info agentconsul.Self) (string, bool) {
|
||||
n, ok := info["Config"]["NodeName"].(string)
|
||||
return n, ok
|
||||
}
|
||||
|
||||
func (cfs *consulFingerprintState) dc(info agentconsul.Self) (string, bool) {
|
||||
func (cfs *consulState) dc(info agentconsul.Self) (string, bool) {
|
||||
d, ok := info["Config"]["Datacenter"].(string)
|
||||
return d, ok
|
||||
}
|
||||
|
||||
func (cfs *consulFingerprintState) segment(info agentconsul.Self) (string, bool) {
|
||||
func (cfs *consulState) segment(info agentconsul.Self) (string, bool) {
|
||||
tags, tagsOK := info["Member"]["Tags"].(map[string]interface{})
|
||||
if !tagsOK {
|
||||
return "", false
|
||||
@@ -254,12 +224,12 @@ func (cfs *consulFingerprintState) segment(info agentconsul.Self) (string, bool)
|
||||
return s, ok
|
||||
}
|
||||
|
||||
func (cfs *consulFingerprintState) connect(info agentconsul.Self) (string, bool) {
|
||||
func (cfs *consulState) connect(info agentconsul.Self) (string, bool) {
|
||||
c, ok := info["DebugConfig"]["ConnectEnabled"].(bool)
|
||||
return strconv.FormatBool(c), ok
|
||||
}
|
||||
|
||||
func (cfs *consulFingerprintState) grpc(scheme string, logger hclog.Logger) func(info agentconsul.Self) (string, bool) {
|
||||
func (cfs *consulState) grpc(scheme string, logger hclog.Logger) func(info agentconsul.Self) (string, bool) {
|
||||
return func(info agentconsul.Self) (string, bool) {
|
||||
|
||||
// The version is needed in order to understand which config object to
|
||||
@@ -294,24 +264,24 @@ func (cfs *consulFingerprintState) grpc(scheme string, logger hclog.Logger) func
|
||||
}
|
||||
}
|
||||
|
||||
func (cfs *consulFingerprintState) grpcPort(info agentconsul.Self) (string, bool) {
|
||||
func (cfs *consulState) grpcPort(info agentconsul.Self) (string, bool) {
|
||||
p, ok := info["DebugConfig"]["GRPCPort"].(float64)
|
||||
return fmt.Sprintf("%d", int(p)), ok
|
||||
}
|
||||
|
||||
func (cfs *consulFingerprintState) grpcTLSPort(info agentconsul.Self) (string, bool) {
|
||||
func (cfs *consulState) grpcTLSPort(info agentconsul.Self) (string, bool) {
|
||||
p, ok := info["DebugConfig"]["GRPCTLSPort"].(float64)
|
||||
return fmt.Sprintf("%d", int(p)), ok
|
||||
}
|
||||
|
||||
func (cfs *consulFingerprintState) dnsPort(info agentconsul.Self) (string, bool) {
|
||||
func (cfs *consulState) dnsPort(info agentconsul.Self) (string, bool) {
|
||||
p, ok := info["DebugConfig"]["DNSPort"].(float64)
|
||||
return fmt.Sprintf("%d", int(p)), ok
|
||||
}
|
||||
|
||||
// dnsAddr fingerprints the Consul DNS address, but only if Nomad can use it
|
||||
// usefully to provide an iptables rule to a task
|
||||
func (cfs *consulFingerprintState) dnsAddr(logger hclog.Logger) func(info agentconsul.Self) (string, bool) {
|
||||
func (cfs *consulState) dnsAddr(logger hclog.Logger) func(info agentconsul.Self) (string, bool) {
|
||||
return func(info agentconsul.Self) (string, bool) {
|
||||
|
||||
var listenOnEveryIP bool
|
||||
@@ -382,11 +352,11 @@ func (cfs *consulFingerprintState) dnsAddr(logger hclog.Logger) func(info agentc
|
||||
}
|
||||
}
|
||||
|
||||
func (cfs *consulFingerprintState) namespaces(info agentconsul.Self) (string, bool) {
|
||||
func (cfs *consulState) namespaces(info agentconsul.Self) (string, bool) {
|
||||
return strconv.FormatBool(agentconsul.Namespaces(info)), true
|
||||
}
|
||||
|
||||
func (cfs *consulFingerprintState) partition(info agentconsul.Self) (string, bool) {
|
||||
func (cfs *consulState) partition(info agentconsul.Self) (string, bool) {
|
||||
sku, ok := agentconsul.SKU(info)
|
||||
if ok && sku == "ent" {
|
||||
p, ok := info["Config"]["Partition"].(string)
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/nomad/ci"
|
||||
"github.com/hashicorp/nomad/client/config"
|
||||
@@ -54,7 +53,7 @@ func newConsulFingerPrint(t *testing.T) *ConsulFingerprint {
|
||||
func TestConsulFingerprint_server(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
cfs := consulFingerprintState{}
|
||||
cfs := new(consulState)
|
||||
|
||||
t.Run("is server", func(t *testing.T) {
|
||||
s, ok := cfs.server(agentconsul.Self{
|
||||
@@ -90,7 +89,7 @@ func TestConsulFingerprint_server(t *testing.T) {
|
||||
func TestConsulFingerprint_version(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
cfs := consulFingerprintState{}
|
||||
cfs := new(consulState)
|
||||
|
||||
t.Run("oss", func(t *testing.T) {
|
||||
v, ok := cfs.version(agentconsul.Self{
|
||||
@@ -126,7 +125,7 @@ func TestConsulFingerprint_version(t *testing.T) {
|
||||
func TestConsulFingerprint_sku(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
cfs := consulFingerprintState{}
|
||||
cfs := new(consulState)
|
||||
|
||||
t.Run("oss", func(t *testing.T) {
|
||||
s, ok := cfs.sku(agentconsul.Self{
|
||||
@@ -186,7 +185,7 @@ func TestConsulFingerprint_sku(t *testing.T) {
|
||||
func TestConsulFingerprint_revision(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
cfs := consulFingerprintState{}
|
||||
cfs := new(consulState)
|
||||
|
||||
t.Run("ok", func(t *testing.T) {
|
||||
r, ok := cfs.revision(agentconsul.Self{
|
||||
@@ -214,7 +213,7 @@ func TestConsulFingerprint_revision(t *testing.T) {
|
||||
func TestConsulFingerprint_dc(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
cfs := consulFingerprintState{}
|
||||
cfs := new(consulState)
|
||||
|
||||
t.Run("ok", func(t *testing.T) {
|
||||
dc, ok := cfs.dc(agentconsul.Self{
|
||||
@@ -242,7 +241,7 @@ func TestConsulFingerprint_dc(t *testing.T) {
|
||||
func TestConsulFingerprint_segment(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
cfs := consulFingerprintState{}
|
||||
cfs := new(consulState)
|
||||
|
||||
t.Run("ok", func(t *testing.T) {
|
||||
s, ok := cfs.segment(agentconsul.Self{
|
||||
@@ -277,7 +276,7 @@ func TestConsulFingerprint_segment(t *testing.T) {
|
||||
func TestConsulFingerprint_connect(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
cfs := consulFingerprintState{}
|
||||
cfs := new(consulState)
|
||||
|
||||
t.Run("connect enabled", func(t *testing.T) {
|
||||
s, ok := cfs.connect(agentconsul.Self{
|
||||
@@ -306,7 +305,7 @@ func TestConsulFingerprint_connect(t *testing.T) {
|
||||
func TestConsulFingerprint_grpc(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
cfs := consulFingerprintState{}
|
||||
cfs := new(consulState)
|
||||
|
||||
t.Run("grpc set pre-1.14 http", func(t *testing.T) {
|
||||
s, ok := cfs.grpc("http", testlog.HCLogger(t))(agentconsul.Self{
|
||||
@@ -407,7 +406,7 @@ func TestConsulFingerprint_grpc(t *testing.T) {
|
||||
func TestConsulFingerprint_namespaces(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
cfs := consulFingerprintState{}
|
||||
cfs := new(consulState)
|
||||
|
||||
t.Run("supports namespaces", func(t *testing.T) {
|
||||
value, ok := cfs.namespaces(agentconsul.Self{
|
||||
@@ -448,7 +447,7 @@ func TestConsulFingerprint_namespaces(t *testing.T) {
|
||||
func TestConsulFingerprint_partition(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
cfs := consulFingerprintState{}
|
||||
cfs := new(consulState)
|
||||
|
||||
t.Run("oss", func(t *testing.T) {
|
||||
p, ok := cfs.partition(agentconsul.Self{
|
||||
@@ -494,7 +493,7 @@ func TestConsulFingerprint_partition(t *testing.T) {
|
||||
func TestConsulFingerprint_dns(t *testing.T) {
|
||||
ci.Parallel(t)
|
||||
|
||||
cfs := consulFingerprintState{}
|
||||
cfs := new(consulState)
|
||||
|
||||
t.Run("dns port not enabled", func(t *testing.T) {
|
||||
port, ok := cfs.dnsPort(agentconsul.Self{
|
||||
@@ -601,7 +600,7 @@ func TestConsulFingerprint_Fingerprint_oss(t *testing.T) {
|
||||
node := &structs.Node{Attributes: make(map[string]string)}
|
||||
|
||||
// consul not available before first run
|
||||
must.Nil(t, cf.states[structs.ConsulDefaultCluster])
|
||||
must.Nil(t, cf.clusters[structs.ConsulDefaultCluster])
|
||||
|
||||
// execute first query with good response
|
||||
var resp FingerprintResponse
|
||||
@@ -623,7 +622,7 @@ func TestConsulFingerprint_Fingerprint_oss(t *testing.T) {
|
||||
must.True(t, resp.Detected)
|
||||
|
||||
// consul now available
|
||||
must.True(t, cf.states[structs.ConsulDefaultCluster].isAvailable)
|
||||
must.NotNil(t, cf.clusters[structs.ConsulDefaultCluster])
|
||||
|
||||
var resp2 FingerprintResponse
|
||||
|
||||
@@ -638,18 +637,15 @@ func TestConsulFingerprint_Fingerprint_oss(t *testing.T) {
|
||||
node.Attributes["connect.grpc"] = "foo"
|
||||
node.Attributes["unique.consul.name"] = "foo"
|
||||
|
||||
// Reset the nextCheck time for testing purposes, or we won't pick up the
|
||||
// change until the next period, up to 2min from now
|
||||
cf.states[structs.ConsulDefaultCluster].nextCheck = time.Now()
|
||||
|
||||
// execute second query with error
|
||||
err2 := cf.Fingerprint(&FingerprintRequest{Config: cfg, Node: node}, &resp2)
|
||||
must.NoError(t, err2) // does not return error
|
||||
must.Nil(t, resp2.Attributes) // attributes unset so they don't change
|
||||
must.True(t, resp.Detected) // never downgrade
|
||||
|
||||
// consul no longer available
|
||||
must.False(t, cf.states[structs.ConsulDefaultCluster].isAvailable)
|
||||
// consul no longer available; an agent restart is required to clear an
|
||||
// existing fingerprint
|
||||
must.NotNil(t, cf.clusters[structs.ConsulDefaultCluster])
|
||||
|
||||
// execute third query no error
|
||||
var resp3 FingerprintResponse
|
||||
@@ -670,7 +666,7 @@ func TestConsulFingerprint_Fingerprint_oss(t *testing.T) {
|
||||
}, resp3.Attributes)
|
||||
|
||||
// consul now available again
|
||||
must.True(t, cf.states[structs.ConsulDefaultCluster].isAvailable)
|
||||
must.NotNil(t, cf.clusters[structs.ConsulDefaultCluster])
|
||||
must.True(t, resp.Detected)
|
||||
}
|
||||
|
||||
@@ -685,7 +681,7 @@ func TestConsulFingerprint_Fingerprint_ent(t *testing.T) {
|
||||
node := &structs.Node{Attributes: make(map[string]string)}
|
||||
|
||||
// consul not available before first run
|
||||
must.Nil(t, cf.states[structs.ConsulDefaultCluster])
|
||||
must.Nil(t, cf.clusters[structs.ConsulDefaultCluster])
|
||||
|
||||
// execute first query with good response
|
||||
var resp FingerprintResponse
|
||||
@@ -709,7 +705,7 @@ func TestConsulFingerprint_Fingerprint_ent(t *testing.T) {
|
||||
must.True(t, resp.Detected)
|
||||
|
||||
// consul now available
|
||||
must.True(t, cf.states[structs.ConsulDefaultCluster].isAvailable)
|
||||
must.NotNil(t, cf.clusters[structs.ConsulDefaultCluster])
|
||||
|
||||
var resp2 FingerprintResponse
|
||||
|
||||
@@ -725,18 +721,15 @@ func TestConsulFingerprint_Fingerprint_ent(t *testing.T) {
|
||||
node.Attributes["connect.grpc"] = "foo"
|
||||
node.Attributes["unique.consul.name"] = "foo"
|
||||
|
||||
// Reset the nextCheck time for testing purposes, or we won't pick up the
|
||||
// change until the next period, up to 2min from now
|
||||
cf.states[structs.ConsulDefaultCluster].nextCheck = time.Now()
|
||||
|
||||
// execute second query with error
|
||||
err2 := cf.Fingerprint(&FingerprintRequest{Config: cfg, Node: node}, &resp2)
|
||||
must.NoError(t, err2) // does not return error
|
||||
must.Nil(t, resp2.Attributes) // attributes unset so they don't change
|
||||
must.True(t, resp.Detected) // never downgrade
|
||||
|
||||
// consul no longer available
|
||||
must.False(t, cf.states[structs.ConsulDefaultCluster].isAvailable)
|
||||
// consul no longer available; an agent restart is required to clear
|
||||
// a detected cluster
|
||||
must.NotNil(t, cf.clusters[structs.ConsulDefaultCluster])
|
||||
|
||||
// execute third query no error
|
||||
var resp3 FingerprintResponse
|
||||
@@ -759,6 +752,6 @@ func TestConsulFingerprint_Fingerprint_ent(t *testing.T) {
|
||||
}, resp3.Attributes)
|
||||
|
||||
// consul now available again
|
||||
must.True(t, cf.states[structs.ConsulDefaultCluster].isAvailable)
|
||||
must.NotNil(t, cf.clusters[structs.ConsulDefaultCluster])
|
||||
must.True(t, resp.Detected)
|
||||
}
|
||||
|
||||
@@ -11,15 +11,12 @@ import (
|
||||
|
||||
log "github.com/hashicorp/go-hclog"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/hashicorp/nomad/helper"
|
||||
"github.com/hashicorp/nomad/helper/useragent"
|
||||
"github.com/hashicorp/nomad/nomad/structs"
|
||||
"github.com/hashicorp/nomad/nomad/structs/config"
|
||||
vapi "github.com/hashicorp/vault/api"
|
||||
)
|
||||
|
||||
var vaultBaseFingerprintInterval = 15 * time.Second
|
||||
|
||||
// VaultFingerprint is used to fingerprint for Vault
|
||||
type VaultFingerprint struct {
|
||||
logger log.Logger
|
||||
@@ -29,7 +26,6 @@ type VaultFingerprint struct {
|
||||
type vaultFingerprintState struct {
|
||||
client *vapi.Client
|
||||
isAvailable bool
|
||||
nextCheck time.Time
|
||||
}
|
||||
|
||||
// NewVaultFingerprint is used to create a Vault fingerprint
|
||||
@@ -56,7 +52,6 @@ func (f *VaultFingerprint) Fingerprint(req *FingerprintRequest, resp *Fingerprin
|
||||
|
||||
// fingerprintImpl fingerprints for a single Vault cluster
|
||||
func (f *VaultFingerprint) fingerprintImpl(cfg *config.VaultConfig, resp *FingerprintResponse) error {
|
||||
|
||||
logger := f.logger.With("cluster", cfg.Name)
|
||||
|
||||
state, ok := f.states[cfg.Name]
|
||||
@@ -64,9 +59,6 @@ func (f *VaultFingerprint) fingerprintImpl(cfg *config.VaultConfig, resp *Finger
|
||||
state = &vaultFingerprintState{}
|
||||
f.states[cfg.Name] = state
|
||||
}
|
||||
if state.nextCheck.After(time.Now()) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Only create the client once to avoid creating too many connections to Vault
|
||||
if state.client == nil {
|
||||
@@ -89,7 +81,6 @@ func (f *VaultFingerprint) fingerprintImpl(cfg *config.VaultConfig, resp *Finger
|
||||
logger.Info("Vault is unavailable")
|
||||
}
|
||||
state.isAvailable = false
|
||||
state.nextCheck = time.Time{} // always check on next interval
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -111,30 +102,15 @@ func (f *VaultFingerprint) fingerprintImpl(cfg *config.VaultConfig, resp *Finger
|
||||
logger.Info("Vault is available")
|
||||
}
|
||||
|
||||
// Widen the minimum window to the next check so that if one out of a set of
|
||||
// Vaults is unhealthy we don't greatly increase requests to the healthy
|
||||
// ones. This is less than the minimum window if all Vaults are healthy so
|
||||
// that we don't desync from the larger window provided by Periodic
|
||||
state.nextCheck = time.Now().Add(29 * time.Second)
|
||||
state.isAvailable = true
|
||||
|
||||
resp.Detected = true
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *VaultFingerprint) Periodic() (bool, time.Duration) {
|
||||
if len(f.states) == 0 {
|
||||
return true, vaultBaseFingerprintInterval
|
||||
}
|
||||
for _, state := range f.states {
|
||||
if !state.isAvailable {
|
||||
return true, vaultBaseFingerprintInterval
|
||||
}
|
||||
}
|
||||
|
||||
// Once all Vaults are initially discovered and healthy we fingerprint with
|
||||
// a wide jitter to avoid thundering herds of fingerprints against central
|
||||
// Vault servers.
|
||||
return true, (30 * time.Second) + helper.RandomStagger(90*time.Second)
|
||||
return false, 0
|
||||
}
|
||||
|
||||
// Reload satisfies ReloadableFingerprint.
|
||||
func (f *VaultFingerprint) Reload() {}
|
||||
|
||||
@@ -5,13 +5,13 @@ package fingerprint
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/nomad/ci"
|
||||
"github.com/hashicorp/nomad/client/config"
|
||||
"github.com/hashicorp/nomad/helper/testlog"
|
||||
"github.com/hashicorp/nomad/nomad/structs"
|
||||
"github.com/hashicorp/nomad/testutil"
|
||||
"github.com/shoenig/test/must"
|
||||
)
|
||||
|
||||
func TestVaultFingerprint(t *testing.T) {
|
||||
@@ -26,69 +26,29 @@ func TestVaultFingerprint(t *testing.T) {
|
||||
}
|
||||
|
||||
p, period := fp.Periodic()
|
||||
if !p {
|
||||
t.Fatalf("expected fingerprint to be periodic")
|
||||
}
|
||||
if period != (15 * time.Second) {
|
||||
t.Fatalf("expected period to be 15s but found: %s", period)
|
||||
}
|
||||
must.False(t, p)
|
||||
must.Zero(t, period)
|
||||
|
||||
conf := config.DefaultConfig()
|
||||
conf.VaultConfigs[structs.VaultDefaultCluster] = tv.Config
|
||||
|
||||
request := &FingerprintRequest{Config: conf, Node: node}
|
||||
var response FingerprintResponse
|
||||
err := fp.Fingerprint(request, &response)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to fingerprint: %s", err)
|
||||
}
|
||||
var response1 FingerprintResponse
|
||||
err := fp.Fingerprint(request, &response1)
|
||||
must.NoError(t, err)
|
||||
must.True(t, response1.Detected)
|
||||
|
||||
if !response.Detected {
|
||||
t.Fatalf("expected response to be applicable")
|
||||
}
|
||||
|
||||
assertNodeAttributeContains(t, response.Attributes, "vault.accessible")
|
||||
assertNodeAttributeContains(t, response.Attributes, "vault.version")
|
||||
assertNodeAttributeContains(t, response.Attributes, "vault.cluster_id")
|
||||
assertNodeAttributeContains(t, response.Attributes, "vault.cluster_name")
|
||||
|
||||
// Period should be longer after initial discovery
|
||||
p, period = fp.Periodic()
|
||||
if !p {
|
||||
t.Fatalf("expected fingerprint to be periodic")
|
||||
}
|
||||
if period < (30*time.Second) || period > (2*time.Minute) {
|
||||
t.Fatalf("expected period to be between 30s and 2m but found: %s", period)
|
||||
}
|
||||
assertNodeAttributeEquals(t, response1.Attributes, "vault.accessible", "true")
|
||||
assertNodeAttributeContains(t, response1.Attributes, "vault.version")
|
||||
assertNodeAttributeContains(t, response1.Attributes, "vault.cluster_id")
|
||||
assertNodeAttributeContains(t, response1.Attributes, "vault.cluster_name")
|
||||
|
||||
// Stop Vault to simulate it being unavailable
|
||||
tv.Stop()
|
||||
|
||||
// Reset the nextCheck time for testing purposes, or we won't pick up the
|
||||
// change until the next period, up to 2min from now
|
||||
vfp := fp.(*VaultFingerprint)
|
||||
vfp.states[structs.VaultDefaultCluster].nextCheck = time.Now()
|
||||
|
||||
err = fp.Fingerprint(request, &response)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to fingerprint: %s", err)
|
||||
}
|
||||
|
||||
if !response.Detected {
|
||||
t.Fatalf("should still show as detected")
|
||||
}
|
||||
|
||||
assertNodeAttributeContains(t, response.Attributes, "vault.accessible")
|
||||
assertNodeAttributeContains(t, response.Attributes, "vault.version")
|
||||
assertNodeAttributeContains(t, response.Attributes, "vault.cluster_id")
|
||||
assertNodeAttributeContains(t, response.Attributes, "vault.cluster_name")
|
||||
|
||||
// Period should be original once trying to discover Vault is available again
|
||||
p, period = fp.Periodic()
|
||||
if !p {
|
||||
t.Fatalf("expected fingerprint to be periodic")
|
||||
}
|
||||
if period != (15 * time.Second) {
|
||||
t.Fatalf("expected period to be 15s but found: %s", period)
|
||||
}
|
||||
// Not detected this time
|
||||
var response2 FingerprintResponse
|
||||
err = fp.Fingerprint(request, &response2)
|
||||
must.NoError(t, err)
|
||||
must.False(t, response2.Detected)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user