From 08845cef04ebde008ab19874d8a53287116c539b Mon Sep 17 00:00:00 2001 From: James Rasell Date: Tue, 12 Jul 2022 13:43:25 +0200 Subject: [PATCH] server: add ACL token expiration config parameters. (#13667) This commit adds configuration parameters to control ACL token expirations. This includes both limits on the min and max TTL expiration values, as well as a GC threshold for expired tokens. --- command/agent/agent.go | 13 +++++++++++ command/agent/config.go | 35 ++++++++++++++++++++++++++++-- command/agent/config_parse.go | 2 ++ command/agent/config_parse_test.go | 17 ++++++++++----- command/agent/config_test.go | 20 ++++++++++------- command/agent/testdata/basic.hcl | 11 ++++++---- command/agent/testdata/basic.json | 5 ++++- nomad/config.go | 20 +++++++++++++++++ 8 files changed, 102 insertions(+), 21 deletions(-) diff --git a/command/agent/agent.go b/command/agent/agent.go index 0c8217d35..c1213bff9 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -241,6 +241,12 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) { if agentConfig.ACL.ReplicationToken != "" { conf.ReplicationToken = agentConfig.ACL.ReplicationToken } + if agentConfig.ACL.TokenMinExpirationTTL != 0 { + conf.ACLTokenMinExpirationTTL = agentConfig.ACL.TokenMinExpirationTTL + } + if agentConfig.ACL.TokenMaxExpirationTTL != 0 { + conf.ACLTokenMaxExpirationTTL = agentConfig.ACL.TokenMaxExpirationTTL + } if agentConfig.Sentinel != nil { conf.SentinelConfig = agentConfig.Sentinel } @@ -377,6 +383,13 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) { } conf.CSIPluginGCThreshold = dur } + if gcThreshold := agentConfig.Server.ACLTokenGCThreshold; gcThreshold != "" { + dur, err := time.ParseDuration(gcThreshold) + if err != nil { + return nil, err + } + conf.ACLTokenExpirationGCThreshold = dur + } if heartbeatGrace := agentConfig.Server.HeartbeatGrace; heartbeatGrace != 0 { conf.HeartbeatGrace = heartbeatGrace diff --git a/command/agent/config.go b/command/agent/config.go index 50e547592..649ca8829 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -355,6 +355,18 @@ type ACLConfig struct { // within the authoritative region. ReplicationToken string `hcl:"replication_token"` + // TokenMinExpirationTTL is used to enforce the lowest acceptable value for + // ACL token expiration. This is used by the Nomad servers to validate ACL + // tokens with an expiration value set upon creation. + TokenMinExpirationTTL time.Duration + TokenMinExpirationTTLHCL string `hcl:"token_min_expiration_ttl" json:"-"` + + // TokenMaxExpirationTTL is used to enforce the highest acceptable value + // for ACL token expiration. This is used by the Nomad servers to validate + // ACL tokens with an expiration value set upon creation. + TokenMaxExpirationTTL time.Duration + TokenMaxExpirationTTLHCL string `hcl:"token_max_expiration_ttl" json:"-"` + // ExtraKeysHCL is used by hcl to surface unexpected keys ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } @@ -417,7 +429,7 @@ type ServerConfig struct { EvalGCThreshold string `hcl:"eval_gc_threshold"` // DeploymentGCThreshold controls how "old" a deployment must be to be - // collected by GC. Age is not the only requirement for a deployment to be + // collected by GC. Age is not the only requirement for a deployment to be // GCed but the threshold can be used to filter by age. DeploymentGCThreshold string `hcl:"deployment_gc_threshold"` @@ -431,6 +443,10 @@ type ServerConfig struct { // GCed but the threshold can be used to filter by age. CSIPluginGCThreshold string `hcl:"csi_plugin_gc_threshold"` + // ACLTokenGCThreshold controls how "old" an expired ACL token must be to + // be collected by GC. + ACLTokenGCThreshold string `hcl:"acl_token_gc_threshold"` + // HeartbeatGrace is the grace period beyond the TTL to account for network, // processing delays and clock skew before marking a node as "down". HeartbeatGrace time.Duration @@ -932,7 +948,7 @@ func DevConfig(mode *devModeConfig) *Config { return conf } -// DefaultConfig is a the baseline configuration for Nomad +// DefaultConfig is the baseline configuration for Nomad. func DefaultConfig() *Config { return &Config{ LogLevel: "INFO", @@ -1470,6 +1486,18 @@ func (a *ACLConfig) Merge(b *ACLConfig) *ACLConfig { if b.PolicyTTLHCL != "" { result.PolicyTTLHCL = b.PolicyTTLHCL } + if b.TokenMinExpirationTTL != 0 { + result.TokenMinExpirationTTL = b.TokenMinExpirationTTL + } + if b.TokenMinExpirationTTLHCL != "" { + result.TokenMinExpirationTTLHCL = b.TokenMinExpirationTTLHCL + } + if b.TokenMaxExpirationTTL != 0 { + result.TokenMaxExpirationTTL = b.TokenMaxExpirationTTL + } + if b.TokenMaxExpirationTTLHCL != "" { + result.TokenMaxExpirationTTLHCL = b.TokenMaxExpirationTTLHCL + } if b.ReplicationToken != "" { result.ReplicationToken = b.ReplicationToken } @@ -1526,6 +1554,9 @@ func (s *ServerConfig) Merge(b *ServerConfig) *ServerConfig { if b.CSIPluginGCThreshold != "" { result.CSIPluginGCThreshold = b.CSIPluginGCThreshold } + if b.ACLTokenGCThreshold != "" { + result.ACLTokenGCThreshold = b.ACLTokenGCThreshold + } if b.HeartbeatGrace != 0 { result.HeartbeatGrace = b.HeartbeatGrace } diff --git a/command/agent/config_parse.go b/command/agent/config_parse.go index 363db9e1e..ea99cb36d 100644 --- a/command/agent/config_parse.go +++ b/command/agent/config_parse.go @@ -62,6 +62,8 @@ func ParseConfigFile(path string) (*Config, error) { {"gc_interval", &c.Client.GCInterval, &c.Client.GCIntervalHCL, nil}, {"acl.token_ttl", &c.ACL.TokenTTL, &c.ACL.TokenTTLHCL, nil}, {"acl.policy_ttl", &c.ACL.PolicyTTL, &c.ACL.PolicyTTLHCL, nil}, + {"acl.token_min_expiration_ttl", &c.ACL.TokenMinExpirationTTL, &c.ACL.TokenMinExpirationTTLHCL, nil}, + {"acl.token_max_expiration_ttl", &c.ACL.TokenMaxExpirationTTL, &c.ACL.TokenMaxExpirationTTLHCL, nil}, {"client.server_join.retry_interval", &c.Client.ServerJoin.RetryInterval, &c.Client.ServerJoin.RetryIntervalHCL, nil}, {"server.heartbeat_grace", &c.Server.HeartbeatGrace, &c.Server.HeartbeatGraceHCL, nil}, {"server.min_heartbeat_ttl", &c.Server.MinHeartbeatTTL, &c.Server.MinHeartbeatTTLHCL, nil}, diff --git a/command/agent/config_parse_test.go b/command/agent/config_parse_test.go index d2567c7fc..a1cbc0208 100644 --- a/command/agent/config_parse_test.go +++ b/command/agent/config_parse_test.go @@ -107,6 +107,7 @@ var basicConfig = &Config{ DeploymentGCThreshold: "12h", CSIVolumeClaimGCThreshold: "12h", CSIPluginGCThreshold: "12h", + ACLTokenGCThreshold: "12h", HeartbeatGrace: 30 * time.Second, HeartbeatGraceHCL: "30s", MinHeartbeatTTL: 33 * time.Second, @@ -143,12 +144,16 @@ var basicConfig = &Config{ LicensePath: "/tmp/nomad.hclic", }, ACL: &ACLConfig{ - Enabled: true, - TokenTTL: 60 * time.Second, - TokenTTLHCL: "60s", - PolicyTTL: 60 * time.Second, - PolicyTTLHCL: "60s", - ReplicationToken: "foobar", + Enabled: true, + TokenTTL: 60 * time.Second, + TokenTTLHCL: "60s", + PolicyTTL: 60 * time.Second, + PolicyTTLHCL: "60s", + TokenMinExpirationTTLHCL: "1h", + TokenMinExpirationTTL: 1 * time.Hour, + TokenMaxExpirationTTLHCL: "100h", + TokenMaxExpirationTTL: 100 * time.Hour, + ReplicationToken: "foobar", }, Audit: &config.AuditConfig{ Enabled: helper.BoolToPtr(true), diff --git a/command/agent/config_test.go b/command/agent/config_test.go index 36e263a31..be29ada38 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -150,10 +150,12 @@ func TestConfig_Merge(t *testing.T) { EventBufferSize: helper.IntToPtr(0), }, ACL: &ACLConfig{ - Enabled: true, - TokenTTL: 60 * time.Second, - PolicyTTL: 60 * time.Second, - ReplicationToken: "foo", + Enabled: true, + TokenTTL: 60 * time.Second, + PolicyTTL: 60 * time.Second, + TokenMinExpirationTTL: 60 * time.Second, + TokenMaxExpirationTTL: 60 * time.Second, + ReplicationToken: "foo", }, Ports: &Ports{ HTTP: 4646, @@ -345,10 +347,12 @@ func TestConfig_Merge(t *testing.T) { EventBufferSize: helper.IntToPtr(100), }, ACL: &ACLConfig{ - Enabled: true, - TokenTTL: 20 * time.Second, - PolicyTTL: 20 * time.Second, - ReplicationToken: "foobar", + Enabled: true, + TokenTTL: 20 * time.Second, + PolicyTTL: 20 * time.Second, + TokenMinExpirationTTL: 20 * time.Second, + TokenMaxExpirationTTL: 20 * time.Second, + ReplicationToken: "foobar", }, Ports: &Ports{ HTTP: 20000, diff --git a/command/agent/testdata/basic.hcl b/command/agent/testdata/basic.hcl index 27754b957..700959c48 100644 --- a/command/agent/testdata/basic.hcl +++ b/command/agent/testdata/basic.hcl @@ -116,6 +116,7 @@ server { deployment_gc_threshold = "12h" csi_volume_claim_gc_threshold = "12h" csi_plugin_gc_threshold = "12h" + acl_token_gc_threshold = "12h" heartbeat_grace = "30s" min_heartbeat_ttl = "33s" max_heartbeats_per_second = 11.0 @@ -153,10 +154,12 @@ server { } acl { - enabled = true - token_ttl = "60s" - policy_ttl = "60s" - replication_token = "foobar" + enabled = true + token_ttl = "60s" + policy_ttl = "60s" + token_min_expiration_ttl = "1h" + token_max_expiration_ttl = "100h" + replication_token = "foobar" } audit { diff --git a/command/agent/testdata/basic.json b/command/agent/testdata/basic.json index 406e314a9..c60fc40db 100644 --- a/command/agent/testdata/basic.json +++ b/command/agent/testdata/basic.json @@ -4,7 +4,9 @@ "enabled": true, "policy_ttl": "60s", "replication_token": "foobar", - "token_ttl": "60s" + "token_ttl": "60s", + "token_min_expiration_ttl": "1h", + "token_max_expiration_ttl": "100h" } ], "audit": { @@ -254,6 +256,7 @@ ], "server": [ { + "acl_token_gc_threshold": "12h", "authoritative_region": "foobar", "bootstrap_expect": 5, "csi_plugin_gc_threshold": "12h", diff --git a/nomad/config.go b/nomad/config.go index 6ea8cfd09..8ca99723d 100644 --- a/nomad/config.go +++ b/nomad/config.go @@ -196,6 +196,14 @@ type Config struct { // one-time tokens. OneTimeTokenGCInterval time.Duration + // ACLTokenExpirationGCInterval is how often we dispatch a job to GC + // expired ACL tokens. + ACLTokenExpirationGCInterval time.Duration + + // ACLTokenExpirationGCThreshold controls how "old" an expired ACL token + // must be to be collected by GC. + ACLTokenExpirationGCThreshold time.Duration + // EvalNackTimeout controls how long we allow a sub-scheduler to // work on an evaluation before we consider it failed and Nack it. // This allows that evaluation to be handed to another sub-scheduler @@ -278,6 +286,14 @@ type Config struct { // the Authoritative Region. ReplicationToken string + // TokenMinExpirationTTL is used to enforce the lowest acceptable value for + // ACL token expiration. + ACLTokenMinExpirationTTL time.Duration + + // TokenMaxExpirationTTL is used to enforce the highest acceptable value + // for ACL token expiration. + ACLTokenMaxExpirationTTL time.Duration + // SentinelGCInterval is the interval that we GC unused policies. SentinelGCInterval time.Duration @@ -385,6 +401,8 @@ func DefaultConfig() *Config { CSIVolumeClaimGCInterval: 5 * time.Minute, CSIVolumeClaimGCThreshold: 5 * time.Minute, OneTimeTokenGCInterval: 10 * time.Minute, + ACLTokenExpirationGCInterval: 5 * time.Minute, + ACLTokenExpirationGCThreshold: 1 * time.Hour, EvalNackTimeout: 60 * time.Second, EvalDeliveryLimit: 3, EvalNackInitialReenqueueDelay: 1 * time.Second, @@ -405,6 +423,8 @@ func DefaultConfig() *Config { LicenseConfig: &LicenseConfig{}, EnableEventBroker: true, EventBufferSize: 100, + ACLTokenMinExpirationTTL: 1 * time.Minute, + ACLTokenMaxExpirationTTL: 24 * time.Hour, AutopilotConfig: &structs.AutopilotConfig{ CleanupDeadServers: true, LastContactThreshold: 200 * time.Millisecond,