From c970d22164626bb674396031c83c1a495975894e Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Thu, 18 Jul 2024 09:42:28 -0400 Subject: [PATCH] keyring: support external KMS for key encryption key (KEK) (#23580) In Nomad 1.4.0, we shipped support for encrypted Variables and signed Workload Identities, but the key material is protected only by a AEAD encrypting the KEK. Add support for Vault transit encryption and external KMS from major cloud providers. The servers call out to the external service to decrypt each key in the on-disk keystore. Ref: https://hashicorp.atlassian.net/browse/NET-10334 Fixes: https://github.com/hashicorp/nomad/issues/14852 --- .changelog/23580.txt | 3 + command/agent/agent.go | 2 + command/agent/config.go | 41 +++ command/agent/config_parse.go | 56 +++++ command/agent/config_parse_test.go | 40 +++ command/agent/config_test.go | 71 ++++++ command/agent/testdata/basic.hcl | 10 + command/agent/testdata/basic.json | 8 + command/agent/testdata/sample0.json | 7 + command/agent/testdata/sample1/sample0.json | 7 + command/agent/testdata/sample1/sample1.json | 5 +- go.mod | 22 +- go.sum | 57 ++++- nomad/config.go | 4 + nomad/encrypter.go | 264 +++++++++++++++----- nomad/encrypter_ce.go | 42 ++++ nomad/encrypter_test.go | 107 +++++++- nomad/keyring_endpoint.go | 7 +- nomad/structs/keyring.go | 80 +++++- 19 files changed, 733 insertions(+), 100 deletions(-) create mode 100644 .changelog/23580.txt create mode 100644 nomad/encrypter_ce.go diff --git a/.changelog/23580.txt b/.changelog/23580.txt new file mode 100644 index 000000000..7ebd2ef3b --- /dev/null +++ b/.changelog/23580.txt @@ -0,0 +1,3 @@ +```release-note:improvement +keyring: Added support for encrypting the keyring via Vault transit or external KMS +``` diff --git a/command/agent/agent.go b/command/agent/agent.go index c8f3fc31b..6c645d7df 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -614,6 +614,8 @@ func convertServerConfig(agentConfig *Config) (*nomad.Config, error) { conf.Reporting = agentConfig.Reporting + conf.KEKProviderConfigs = agentConfig.KEKProviders + return conf, nil } diff --git a/command/agent/config.go b/command/agent/config.go index ef80e7718..98cf47870 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -190,6 +190,9 @@ type Config struct { // Reporting is used to enable go census reporting Reporting *config.ReportingConfig `hcl:"reporting,block"` + // KEKProviders are used to wrap the Nomad keyring + KEKProviders []*structs.KEKProviderConfig `hcl:"keyring"` + // ExtraKeysHCL is used by hcl to surface unexpected keys ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` } @@ -1453,6 +1456,7 @@ func DefaultConfig() *Config { DisableUpdateCheck: pointer.Of(false), Limits: config.DefaultLimits(), Reporting: config.DefaultReporting(), + KEKProviders: []*structs.KEKProviderConfig{}, } return cfg @@ -1678,6 +1682,8 @@ func (c *Config) Merge(b *Config) *Config { result.Limits = c.Limits.Merge(b.Limits) + result.KEKProviders = mergeKEKProviderConfigs(result.KEKProviders, b.KEKProviders) + return &result } @@ -1749,6 +1755,40 @@ func mergeConsulConfigs(left, right []*config.ConsulConfig) []*config.ConsulConf return results } +func mergeKEKProviderConfigs(left, right []*structs.KEKProviderConfig) []*structs.KEKProviderConfig { + if len(left) == 0 { + return right + } + if len(right) == 0 { + return left + } + results := []*structs.KEKProviderConfig{} + doMerge := func(dstConfigs, srcConfigs []*structs.KEKProviderConfig) []*structs.KEKProviderConfig { + for _, src := range srcConfigs { + var found bool + for i, dst := range dstConfigs { + if dst.Provider == src.Provider && dst.Name == src.Name { + dstConfigs[i] = dst.Merge(src) + found = true + break + } + } + if !found { + dstConfigs = append(dstConfigs, src) + } + } + return dstConfigs + } + + results = doMerge(results, left) + results = doMerge(results, right) + sort.Slice(results, func(i, j int) bool { + return results[i].ID() < results[j].ID() + }) + + return results +} + // Copy returns a deep copy safe for mutation. func (c *Config) Copy() *Config { if c == nil { @@ -1782,6 +1822,7 @@ func (c *Config) Copy() *Config { nc.Limits = c.Limits.Copy() nc.Audit = c.Audit.Copy() nc.Reporting = c.Reporting.Copy() + nc.KEKProviders = helper.CopySlice(c.KEKProviders) nc.ExtraKeysHCL = slices.Clone(c.ExtraKeysHCL) return &nc } diff --git a/command/agent/config_parse.go b/command/agent/config_parse.go index 0ca361704..6aab5d2e0 100644 --- a/command/agent/config_parse.go +++ b/command/agent/config_parse.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "slices" + "sort" "time" "github.com/hashicorp/hcl" @@ -92,6 +93,13 @@ func ParseConfigFile(path string) (*Config, error) { } } + matches = list.Filter("keyring") + if len(matches.Items) > 0 { + if err := parseKeyringConfigs(c, matches); err != nil { + return nil, fmt.Errorf("error parsing 'keyring': %w", err) + } + } + // convert strings to time.Durations tds := []durationConversionMap{ {"gc_interval", &c.Client.GCInterval, &c.Client.GCIntervalHCL, nil}, @@ -330,6 +338,11 @@ func extraKeys(c *Config) error { helper.RemoveEqualFold(&c.ExtraKeysHCL, "telemetry") } + helper.RemoveEqualFold(&c.ExtraKeysHCL, "keyring") + for _, provider := range c.KEKProviders { + helper.RemoveEqualFold(&c.ExtraKeysHCL, provider.Provider) + } + // Remove reporting extra keys c.ExtraKeysHCL = slices.DeleteFunc(c.ExtraKeysHCL, func(s string) bool { return s == "license" }) @@ -522,3 +535,46 @@ func parseConsuls(c *Config, list *ast.ObjectList) error { return nil } + +// parseKeyringConfigs parses the keyring blocks. At this point we have a list +// of ast.Nodes and a KEKProviderConfig for each one. The KEKProviderConfig has +// the unknown fields (provider-specific config) but not their values. So we +// decode the ast.Node into a map and then read out the values for the unknown +// fields. The results get added to the KEKProviderConfig's Config field +func parseKeyringConfigs(c *Config, keyringBlocks *ast.ObjectList) error { + if len(keyringBlocks.Items) == 0 { + return nil + } + + for idx, obj := range keyringBlocks.Items { + provider := c.KEKProviders[idx] + if len(provider.ExtraKeysHCL) == 0 { + continue + } + + provider.Config = map[string]string{} + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, obj.Val); err != nil { + return err + } + + for _, extraKey := range provider.ExtraKeysHCL { + val, ok := m[extraKey].(string) + if !ok { + return fmt.Errorf("failed to decode key %q to string", extraKey) + } + provider.Config[extraKey] = val + } + + // clear the extra keys for these blocks because we've already handled + // them and don't want them to bubble up to the caller + provider.ExtraKeysHCL = nil + } + + sort.Slice(c.KEKProviders, func(i, j int) bool { + return c.KEKProviders[i].ID() < c.KEKProviders[j].ID() + }) + + return nil +} diff --git a/command/agent/config_parse_test.go b/command/agent/config_parse_test.go index 0a128c0ce..7b8808751 100644 --- a/command/agent/config_parse_test.go +++ b/command/agent/config_parse_test.go @@ -348,6 +348,20 @@ var basicConfig = &Config{ Enabled: pointer.Of(true), }, }, + KEKProviders: []*structs.KEKProviderConfig{ + { + Provider: "aead", + Active: false, + }, + { + Provider: "awskms", + Active: true, + Config: map[string]string{ + "region": "us-east-1", + "kms_key_id": "alias/kms-nomad-keyring", + }, + }, + }, } var pluginConfig = &Config{ @@ -481,6 +495,7 @@ func TestConfig_ParseMerge(t *testing.T) { must.NoError(t, err) actual, err := ParseConfigFile(path) + must.NoError(t, err) // The Vault connection retry interval is an internal only configuration // option, and therefore needs to be added here to ensure the test passes. @@ -548,6 +563,7 @@ func TestConfig_Parse(t *testing.T) { } actual = oldDefault.Merge(actual) + must.Eq(t, tc.Result.KEKProviders, actual.KEKProviders) must.Eq(t, tc.Result, removeHelperAttributes(actual)) }) } @@ -751,6 +767,16 @@ var sample0 = &Config{ CleanupDeadServers: pointer.Of(true), }, Reporting: config.DefaultReporting(), + KEKProviders: []*structs.KEKProviderConfig{ + { + Provider: "awskms", + Active: true, + Config: map[string]string{ + "region": "us-east-1", + "kms_key_id": "alias/kms-nomad-keyring", + }, + }, + }, } func TestConfig_ParseSample0(t *testing.T) { @@ -869,6 +895,20 @@ var sample1 = &Config{ Reporting: &config.ReportingConfig{ &config.LicenseReportingConfig{}, }, + KEKProviders: []*structs.KEKProviderConfig{ + { + Provider: "aead", + Active: false, + }, + { + Provider: "awskms", + Active: true, + Config: map[string]string{ + "region": "us-east-1", + "kms_key_id": "alias/kms-nomad-keyring", + }, + }, + }, } func TestConfig_ParseDir(t *testing.T) { diff --git a/command/agent/config_test.go b/command/agent/config_test.go index 48dec05eb..9debdc3a3 100644 --- a/command/agent/config_test.go +++ b/command/agent/config_test.go @@ -1779,3 +1779,74 @@ func Test_mergeConsulConfigs(t *testing.T) { must.Eq(t, c0.Consuls[0].Token, result.Consuls[0].Token) must.Eq(t, c0.Consuls[0].AllowUnauthenticated, result.Consuls[0].AllowUnauthenticated) } + +func Test_mergeKEKProviderConfigs(t *testing.T) { + ci.Parallel(t) + + left := []*structs.KEKProviderConfig{ + { + // incomplete config with name + Provider: "awskms", + Name: "foo", + Active: true, + Config: map[string]string{ + "region": "us-east-1", + "access_key": "AKIAIOSFODNN7EXAMPLE", + }, + }, + { + // empty config + Provider: "aead", + }, + } + right := []*structs.KEKProviderConfig{ + { + // same awskms.foo provider with fields to merge + Provider: "awskms", + Name: "foo", + Active: false, + Config: map[string]string{ + "access_key": "AKIAIOSXABCD7EXAMPLE", + "secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "kms_key_id": "19ec80b0-dfdd-4d97-8164-c6examplekey", + }, + }, + { + // same awskms provider, different name + Provider: "awskms", + Name: "bar", + Active: false, + Config: map[string]string{ + "region": "us-east-1", + "access_key": "AKIAIOSFODNN7EXAMPLE", + }, + }, + } + + result := mergeKEKProviderConfigs(left, right) + must.Eq(t, []*structs.KEKProviderConfig{ + { + Provider: "aead", + }, + { + Provider: "awskms", + Name: "bar", + Active: false, + Config: map[string]string{ + "region": "us-east-1", + "access_key": "AKIAIOSFODNN7EXAMPLE", + }, + }, + { + Provider: "awskms", + Name: "foo", + Active: false, // should be flipped + Config: map[string]string{ + "region": "us-east-1", + "access_key": "AKIAIOSXABCD7EXAMPLE", // override + "secret_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", // added + "kms_key_id": "19ec80b0-dfdd-4d97-8164-c6examplekey", // added + }, + }, + }, result) +} diff --git a/command/agent/testdata/basic.hcl b/command/agent/testdata/basic.hcl index 4ee1f70d7..2ca7ab1de 100644 --- a/command/agent/testdata/basic.hcl +++ b/command/agent/testdata/basic.hcl @@ -349,3 +349,13 @@ reporting { enabled = true } } + +keyring "awskms" { + active = true + region = "us-east-1" + kms_key_id = "alias/kms-nomad-keyring" +} + +keyring "aead" { + active = false +} diff --git a/command/agent/testdata/basic.json b/command/agent/testdata/basic.json index bc29b1897..2727e9244 100644 --- a/command/agent/testdata/basic.json +++ b/command/agent/testdata/basic.json @@ -204,6 +204,14 @@ "Access-Control-Allow-Origin": "*" } ], + "keyring": { + "awskms": { + "active": true, + "region": "us-east-1", + "kms_key_id": "alias/kms-nomad-keyring" + }, + "aead": {} + }, "leave_on_interrupt": true, "leave_on_terminate": true, "log_file": "/var/log/nomad.log", diff --git a/command/agent/testdata/sample0.json b/command/agent/testdata/sample0.json index cc7736893..a836f93c0 100644 --- a/command/agent/testdata/sample0.json +++ b/command/agent/testdata/sample0.json @@ -48,6 +48,13 @@ "data_dir": "/opt/data/nomad/data", "datacenter": "dc1", "enable_syslog": true, + "keyring": { + "awskms": { + "active": true, + "region": "us-east-1", + "kms_key_id": "alias/kms-nomad-keyring" + } + }, "leave_on_interrupt": true, "leave_on_terminate": true, "log_level": "INFO", diff --git a/command/agent/testdata/sample1/sample0.json b/command/agent/testdata/sample1/sample0.json index a806ea909..1a23c7378 100644 --- a/command/agent/testdata/sample1/sample0.json +++ b/command/agent/testdata/sample1/sample0.json @@ -12,6 +12,13 @@ "data_dir": "/opt/data/nomad/data", "datacenter": "dc1", "enable_syslog": true, + "keyring": { + "awskms": { + "active": true, + "region": "us-east-1", + "kms_key_id": "alias/kms-nomad-keyring" + } + }, "leave_on_interrupt": true, "leave_on_terminate": true, "log_level": "INFO", diff --git a/command/agent/testdata/sample1/sample1.json b/command/agent/testdata/sample1/sample1.json index 8f83354d4..f147acfb6 100644 --- a/command/agent/testdata/sample1/sample1.json +++ b/command/agent/testdata/sample1/sample1.json @@ -11,13 +11,16 @@ "data_dir": "/opt/data/nomad/data", "datacenter": "dc1", "enable_syslog": true, + "keyring": { + "aead": {} + }, "leave_on_interrupt": true, "leave_on_terminate": true, "log_level": "INFO", "region": "global", "server": { "bootstrap_expect": 3, - "enabled": true, + "enabled": true }, "syslog_facility": "LOCAL0", "telemetry": { diff --git a/go.mod b/go.mod index 4b2bd4fe5..d001d2dee 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/Microsoft/go-winio v0.6.1 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e github.com/armon/go-metrics v0.5.3 - github.com/aws/aws-sdk-go v1.44.184 + github.com/aws/aws-sdk-go v1.44.210 github.com/brianvoe/gofakeit/v6 v6.20.1 github.com/container-storage-interface/spec v1.10.0 github.com/containerd/go-cni v1.1.9 @@ -58,6 +58,10 @@ require ( github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-immutable-radix/v2 v2.1.0 github.com/hashicorp/go-kms-wrapping/v2 v2.0.15 + github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2 v2.0.9 + github.com/hashicorp/go-kms-wrapping/wrappers/azurekeyvault/v2 v2.0.11 + github.com/hashicorp/go-kms-wrapping/wrappers/gcpckms/v2 v2.0.12 + github.com/hashicorp/go-kms-wrapping/wrappers/transit/v2 v2.0.11 github.com/hashicorp/go-memdb v1.3.4 github.com/hashicorp/go-msgpack/v2 v2.1.2 github.com/hashicorp/go-multierror v1.1.1 @@ -139,13 +143,19 @@ require ( cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.1 // indirect + cloud.google.com/go/kms v1.15.0 // indirect cloud.google.com/go/storage v1.30.1 // indirect dario.cat/mergo v1.0.0 // indirect github.com/Azure/azure-sdk-for-go v56.3.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest v0.11.20 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.15 // indirect + github.com/Azure/go-autorest/autorest v0.11.29 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.22 // indirect github.com/Azure/go-autorest/autorest/azure/auth v0.5.1 // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.0 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect @@ -153,6 +163,7 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.3.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect github.com/BurntSushi/toml v1.3.2 // indirect github.com/DataDog/datadog-go v3.2.0+incompatible // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -196,7 +207,7 @@ require ( github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/gojuno/minimock/v3 v3.0.6 // indirect - github.com/golang-jwt/jwt/v4 v4.4.3 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/btree v1.1.2 // indirect github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect @@ -213,6 +224,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect + github.com/hashicorp/go-secure-stdlib/awsutil v0.1.6 // indirect github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 // indirect github.com/hashicorp/go-secure-stdlib/reloadutil v0.1.1 // indirect github.com/hashicorp/go-secure-stdlib/tlsutil v0.1.2 // indirect @@ -228,6 +240,7 @@ require ( github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/linode/linodego v0.7.1 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -250,6 +263,7 @@ require ( github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/selinux v1.11.0 // indirect github.com/packethost/packngo v0.1.1-0.20180711074735-b9cb5096f54c // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect diff --git a/go.sum b/go.sum index b3d92beba..1db11b3ef 100644 --- a/go.sum +++ b/go.sum @@ -111,6 +111,8 @@ cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/kms v1.15.0 h1:xYl5WEaSekKYN5gGRyhjvZKM22GVBBCzegGNVPy+aIs= +cloud.google.com/go/kms v1.15.0/go.mod h1:c9J991h5DTl+kg7gi3MYomh12YEENGrf48ee/N/2CDM= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= @@ -192,6 +194,16 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h github.com/Azure/azure-sdk-for-go v44.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible h1:DmhwMrUIvpeoTDiWRDtNHqelNUd3Og8JCkrLHQK795c= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 h1:fb8kj/Dh4CSwgsOzHeZY4Xh68cFVbzXx+ONXGMY//4w= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0/go.mod h1:uReU2sSxZExRPBAg3qKzmAucSi51+SP1OhohieR821Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0 h1:d81/ng9rET2YqdVkVwkb6EXeRrLJIwyGnJcAlAWKwhs= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 h1:m/sWOGCREuSBqg2htVQTBY8nOZpyajYztF0vUvSZTuM= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0/go.mod h1:Pu5Zksi2KrU7LPbZbNINx6fuVrUp/ffvpxdDj+i8LeE= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -199,14 +211,14 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.11.0/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest v0.11.20 h1:s8H1PbCZSqg/DH7JMlOz6YMig6htWLNPsjDdlLqCx3M= -github.com/Azure/go-autorest/autorest v0.11.20/go.mod h1:o3tqFY+QR40VOlk+pV4d77mORO64jOXSgEnPQgLK6JY= +github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= +github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/adal v0.9.15 h1:X+p2GF0GWyOiSmqohIaEeuNFNDY4I4EOlVuUQvFdWMk= -github.com/Azure/go-autorest/autorest/adal v0.9.15/go.mod h1:tGMin8I49Yij6AQ+rvV+Xa/zwxYQB5hmsd6DkfAx2+A= +github.com/Azure/go-autorest/autorest/adal v0.9.22 h1:/GblQdIudfEM3AWWZ0mrYJQSd7JS4S/Mbzh6F0ov0Xc= +github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= github.com/Azure/go-autorest/autorest/azure/auth v0.5.0/go.mod h1:QRTvSZQpxqm8mSErhnbI+tANIBAKP7B+UIE2z4ypUO0= github.com/Azure/go-autorest/autorest/azure/auth v0.5.1 h1:bvUhZciHydpBxBmCheUgxxbSwJy7xcfjkUsjUcqSojc= github.com/Azure/go-autorest/autorest/azure/auth v0.5.1/go.mod h1:ea90/jvmnAwDrSooLH4sRIehEPtG/EPUXavDh31MnA4= @@ -218,8 +230,9 @@ github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSY github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.3.0 h1:3I9AAI63HfcLtphd9g39ruUwRI+Ca+z/f36KHPFRUss= @@ -231,6 +244,8 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= @@ -280,9 +295,10 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.25.41/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.30.27/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.44.122/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.44.184 h1:/MggyE66rOImXJKl1HqhLQITvWvqIV7w1Q4MaG6FHUo= -github.com/aws/aws-sdk-go v1.44.184/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.210 h1:/cqRMHSSgzLEKILIDGwhaX2hiIpyRurw7MRy6aaSufg= +github.com/aws/aws-sdk-go v1.44.210/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -372,8 +388,9 @@ github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TR github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= @@ -450,6 +467,7 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+ github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= @@ -466,8 +484,8 @@ github.com/gojuno/minimock/v3 v3.0.4/go.mod h1:HqeqnwV8mAABn3pO5hqF+RE7gjA0jsN8c github.com/gojuno/minimock/v3 v3.0.6 h1:YqHcVR10x2ZvswPK8Ix5yk+hMpspdQ3ckSpkOzyF85I= github.com/gojuno/minimock/v3 v3.0.6/go.mod h1:v61ZjAKHr+WnEkND63nQPCZ/DTfQgJdvbCi3IuoMblY= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= -github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -650,6 +668,14 @@ github.com/hashicorp/go-immutable-radix/v2 v2.1.0 h1:CUW5RYIcysz+D3B+l1mDeXrQ7fU github.com/hashicorp/go-immutable-radix/v2 v2.1.0/go.mod h1:hgdqLXA4f6NIjRVisM1TJ9aOJVNRqKZj+xDGF6m7PBw= github.com/hashicorp/go-kms-wrapping/v2 v2.0.15 h1:f3+/VbanXOmVAaDBKwRiVmeL7EX340a4YmaTItMF4Xs= github.com/hashicorp/go-kms-wrapping/v2 v2.0.15/go.mod h1:0dWtzl2ilqKpavgM3id/kFK9L3tjo6fS4OhbVPSYpnQ= +github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2 v2.0.9 h1:qdxeZvDMRGZ3YSE4Oz0Pp7WUSUn5S6cWZguEOkEVL50= +github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2 v2.0.9/go.mod h1:DcXbvVpgNWbxGmxgmu3QN64bEydMu14Cpe34RRR30HY= +github.com/hashicorp/go-kms-wrapping/wrappers/azurekeyvault/v2 v2.0.11 h1:/7SKkYIhA8cr3l8m1EKT6Q90bPoSVqqVBuQ6HgoMIkw= +github.com/hashicorp/go-kms-wrapping/wrappers/azurekeyvault/v2 v2.0.11/go.mod h1:LepS5s6ESGE0qQMpYaui5lX+mQYeiYiy06VzwWRioO8= +github.com/hashicorp/go-kms-wrapping/wrappers/gcpckms/v2 v2.0.12 h1:PCqWzT/Hii0KL07JsBZ3lJbv/wx02IAHYlhWQq8rxRY= +github.com/hashicorp/go-kms-wrapping/wrappers/gcpckms/v2 v2.0.12/go.mod h1:HSaOaX/lv3ShCdilUYbOTPnSvmoZ9xtQhgw+8hYcZkg= +github.com/hashicorp/go-kms-wrapping/wrappers/transit/v2 v2.0.11 h1:hdzSrDJ0CgHgGFx+1toaf7Z5bmQ2EYaFQ/dtWNXxu1I= +github.com/hashicorp/go-kms-wrapping/wrappers/transit/v2 v2.0.11/go.mod h1:ywjP17x2t88pT3GA8gCc2vEH1vhvU1R9d5XwRQ0d7PQ= github.com/hashicorp/go-memdb v1.3.4 h1:XSL3NR682X/cVk2IeV0d70N4DZ9ljI885xAEU8IoK3c= github.com/hashicorp/go-memdb v1.3.4/go.mod h1:uBTr1oQbtuMgd1SSGoR8YV27eT3sBHbYiNm53bMpgSg= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= @@ -672,6 +698,8 @@ github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5O github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= +github.com/hashicorp/go-secure-stdlib/awsutil v0.1.6 h1:W9WN8p6moV1fjKLkeqEgkAMu5rauy9QeYDAmIaPuuiA= +github.com/hashicorp/go-secure-stdlib/awsutil v0.1.6/go.mod h1:MpCPSPGLDILGb4JMm94/mMi3YysIqsXzGCzkEZjcjXg= github.com/hashicorp/go-secure-stdlib/listenerutil v0.1.4 h1:6ajbq64FhrIJZ6prrff3upVVDil4yfCrnSKwTH0HIPE= github.com/hashicorp/go-secure-stdlib/listenerutil v0.1.4/go.mod h1:myX7XYMJRIP4PLHtYJiKMTJcKOX0M5ZJNwP0iw+l3uw= github.com/hashicorp/go-secure-stdlib/parseutil v0.1.1/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= @@ -765,6 +793,7 @@ github.com/jefferai/isbadcipher v0.0.0-20190226160619-51d2077c035f/go.mod h1:3J2 github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -795,12 +824,15 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/linode/linodego v0.7.1 h1:4WZmMpSA2NRwlPZcc0+4Gyn7rr99Evk9bnr0B3gXRKE= github.com/linode/linodego v0.7.1/go.mod h1:ga11n3ivecUrPCHN0rANxKmfWBJVkOXfLMZinAbj2sY= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= @@ -937,6 +969,8 @@ github.com/packethost/packngo v0.1.1-0.20180711074735-b9cb5096f54c/go.mod h1:otz github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -980,6 +1014,7 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= @@ -1116,6 +1151,7 @@ golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= @@ -1337,6 +1373,7 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/nomad/config.go b/nomad/config.go index 149d13759..551bfd23a 100644 --- a/nomad/config.go +++ b/nomad/config.go @@ -434,6 +434,9 @@ type Config struct { // If this is not configured the /.well-known/openid-configuration endpoint // will not be available. OIDCIssuer string + + // KEKProviders are used to wrap the Nomad keyring + KEKProviderConfigs []*structs.KEKProviderConfig } func (c *Config) Copy() *Config { @@ -462,6 +465,7 @@ func (c *Config) Copy() *Config { nc.AutopilotConfig = c.AutopilotConfig.Copy() nc.LicenseConfig = c.LicenseConfig.Copy() nc.SearchConfig = c.SearchConfig.Copy() + nc.KEKProviderConfigs = helper.CopySlice(c.KEKProviderConfigs) return &nc } diff --git a/nomad/encrypter.go b/nomad/encrypter.go index 9264612f7..f39a2855b 100644 --- a/nomad/encrypter.go +++ b/nomad/encrypter.go @@ -11,6 +11,7 @@ import ( "crypto/rsa" "crypto/x509" "encoding/json" + "errors" "fmt" "io/fs" "os" @@ -21,15 +22,21 @@ import ( "github.com/go-jose/go-jose/v3" "github.com/go-jose/go-jose/v3/jwt" + "github.com/hashicorp/go-hclog" log "github.com/hashicorp/go-hclog" kms "github.com/hashicorp/go-kms-wrapping/v2" + wrapping "github.com/hashicorp/go-kms-wrapping/v2" "github.com/hashicorp/go-kms-wrapping/v2/aead" - "golang.org/x/time/rate" - + "github.com/hashicorp/go-kms-wrapping/wrappers/awskms/v2" + "github.com/hashicorp/go-kms-wrapping/wrappers/azurekeyvault/v2" + "github.com/hashicorp/go-kms-wrapping/wrappers/gcpckms/v2" + "github.com/hashicorp/go-kms-wrapping/wrappers/transit/v2" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/crypto" "github.com/hashicorp/nomad/helper/joseutil" "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/nomad/structs/config" + "golang.org/x/time/rate" ) const nomadKeystoreExtension = ".nks.json" @@ -43,8 +50,10 @@ var _ claimSigner = &Encrypter{} // Encrypter is the keyring for encrypting variables and signing workload // identities. type Encrypter struct { - srv *Server - keystorePath string + srv *Server + log hclog.Logger + providerConfigs map[string]*structs.KEKProviderConfig + keystorePath string // issuer is the OIDC Issuer to use for workload identities if configured issuer string @@ -70,25 +79,67 @@ type keyset struct { func NewEncrypter(srv *Server, keystorePath string) (*Encrypter, error) { encrypter := &Encrypter{ - srv: srv, - keystorePath: keystorePath, - keyring: make(map[string]*keyset), - issuer: srv.GetConfig().OIDCIssuer, + srv: srv, + log: srv.logger.With("keyring"), + keystorePath: keystorePath, + keyring: make(map[string]*keyset), + issuer: srv.GetConfig().OIDCIssuer, + providerConfigs: map[string]*structs.KEKProviderConfig{}, } - err := encrypter.loadKeystore() + providerConfigs, err := getProviderConfigs(srv) + if err != nil { + return nil, err + } + encrypter.providerConfigs = providerConfigs + + err = encrypter.loadKeystore() if err != nil { return nil, err } return encrypter, nil } +// fallbackVaultConfig allows the transit provider to fallback to using the +// default Vault cluster's configuration block, instead of repeating those +// fields +func fallbackVaultConfig(provider *structs.KEKProviderConfig, vaultcfg *config.VaultConfig) { + + setFallback := func(key, fallback, env string) { + if provider.Config == nil { + provider.Config = map[string]string{} + } + if _, ok := provider.Config[key]; !ok { + if fallback != "" { + provider.Config[key] = fallback + } else { + provider.Config[key] = os.Getenv(env) + } + } + } + + setFallback("address", vaultcfg.Addr, "VAULT_ADDR") + setFallback("token", vaultcfg.Token, "VAULT_TOKEN") + setFallback("tls_ca_cert", vaultcfg.TLSCaPath, "VAULT_CACERT") + setFallback("tls_client_cert", vaultcfg.TLSCertFile, "VAULT_CLIENT_CERT") + setFallback("tls_client_key", vaultcfg.TLSKeyFile, "VAULT_CLIENT_KEY") + setFallback("tls_server_name", vaultcfg.TLSServerName, "VAULT_TLS_SERVER_NAME") + + skipVerify := "" + if vaultcfg.TLSSkipVerify != nil { + skipVerify = fmt.Sprintf("%v", *vaultcfg.TLSSkipVerify) + } + setFallback("tls_skip_verify", skipVerify, "VAULT_SKIP_VERIFY") +} + func (e *Encrypter) loadKeystore() error { if err := os.MkdirAll(e.keystorePath, 0o700); err != nil { return err } + keyErrors := map[string]error{} + return filepath.Walk(e.keystorePath, func(path string, info fs.FileInfo, err error) error { if err != nil { return fmt.Errorf("could not read path %s from keystore: %v", path, err) @@ -103,13 +154,22 @@ func (e *Encrypter) loadKeystore() error { if !strings.HasSuffix(path, nomadKeystoreExtension) { return nil } - id := strings.TrimSuffix(filepath.Base(path), nomadKeystoreExtension) + idWithIndex := strings.TrimSuffix(filepath.Base(path), nomadKeystoreExtension) + id, _, _ := strings.Cut(idWithIndex, ".") if !helper.IsUUID(id) { return nil } + e.lock.RLock() + _, ok := e.keyring[id] + e.lock.RUnlock() + if ok { + return nil // already loaded this key from another file + } + key, err := e.loadKeyFromStore(path) if err != nil { + keyErrors[id] = err return fmt.Errorf("could not load key file %s from keystore: %w", path, err) } if key.Meta.KeyID != id { @@ -120,6 +180,10 @@ func (e *Encrypter) loadKeystore() error { if err != nil { return fmt.Errorf("could not add key file %s to keystore: %w", path, err) } + + // we loaded this key from at least one KEK configuration, so clear any + // error from a previous file that we couldn't read from + delete(keyErrors, id) return nil }) } @@ -337,16 +401,16 @@ func (e *Encrypter) addCipher(rootKey *structs.RootKey) error { return nil } -// GetKey retrieves the key material by ID from the keyring -func (e *Encrypter) GetKey(keyID string) ([]byte, []byte, error) { +// GetKey retrieves the key material by ID from the keyring. +func (e *Encrypter) GetKey(keyID string) (*structs.RootKey, error) { e.lock.RLock() defer e.lock.RUnlock() keyset, err := e.keysetByIDLocked(keyID) if err != nil { - return nil, nil, err + return nil, err } - return keyset.rootKey.Key, keyset.rootKey.RSAKey, nil + return keyset.rootKey, nil } // activeKeySetLocked returns the keyset that belongs to the key marked as @@ -384,47 +448,73 @@ func (e *Encrypter) RemoveKey(keyID string) error { return nil } -// saveKeyToStore serializes a root key to the on-disk keystore. -func (e *Encrypter) saveKeyToStore(rootKey *structs.RootKey) error { +func (e *Encrypter) encryptDEK(rootKey *structs.RootKey, provider *structs.KEKProviderConfig) (*structs.KeyEncryptionKeyWrapper, error) { + if provider == nil { + panic("can't encrypt DEK without a provider") + } + var kek []byte + var err error + if provider.Provider == string(structs.KEKProviderAEAD) || provider.Provider == "" { + kek, err = crypto.Bytes(32) + if err != nil { + return nil, fmt.Errorf("failed to generate key wrapper key: %w", err) + } + } + wrapper, err := e.newKMSWrapper(provider, rootKey.Meta.KeyID, kek) + if err != nil { + return nil, fmt.Errorf("unable to create key wrapper: %w", err) + } - kek, err := crypto.Bytes(32) - if err != nil { - return fmt.Errorf("failed to generate key wrapper key: %w", err) - } - wrapper, err := e.newKMSWrapper(rootKey.Meta.KeyID, kek) - if err != nil { - return fmt.Errorf("failed to create encryption wrapper: %w", err) - } rootBlob, err := wrapper.Encrypt(e.srv.shutdownCtx, rootKey.Key) if err != nil { - return fmt.Errorf("failed to encrypt root key: %w", err) + return nil, fmt.Errorf("failed to encrypt root key: %w", err) } - kekWrapper := &structs.KeyEncryptionKeyWrapper{ - Meta: rootKey.Meta, - EncryptedDataEncryptionKey: rootBlob.Ciphertext, - KeyEncryptionKey: kek, + Meta: rootKey.Meta, + KeyEncryptionKey: kek, + Provider: provider.Provider, + ProviderID: provider.ID(), + WrappedDataEncryptionKey: rootBlob, } // Only keysets created after 1.7.0 will contain an RSA key. if len(rootKey.RSAKey) > 0 { rsaBlob, err := wrapper.Encrypt(e.srv.shutdownCtx, rootKey.RSAKey) if err != nil { - return fmt.Errorf("failed to encrypt rsa key: %w", err) + return nil, fmt.Errorf("failed to encrypt rsa key: %w", err) } - kekWrapper.EncryptedRSAKey = rsaBlob.Ciphertext + kekWrapper.WrappedRSAKey = rsaBlob } - buf, err := json.Marshal(kekWrapper) - if err != nil { - return err + return kekWrapper, nil +} + +// saveKeyToStore serializes a root key to the on-disk keystore. +func (e *Encrypter) saveKeyToStore(rootKey *structs.RootKey) error { + + for _, provider := range e.providerConfigs { + if !provider.Active { + continue + } + kekWrapper, err := e.encryptDEK(rootKey, provider) + if err != nil { + return err + } + + buf, err := json.Marshal(kekWrapper) + if err != nil { + return err + } + + filename := fmt.Sprintf("%s.%s%s", + rootKey.Meta.KeyID, provider.ID(), nomadKeystoreExtension) + path := filepath.Join(e.keystorePath, filename) + err = os.WriteFile(path, buf, 0o600) + if err != nil { + return err + } } - path := filepath.Join(e.keystorePath, rootKey.Meta.KeyID+nomadKeystoreExtension) - err = os.WriteFile(path, buf, 0o600) - if err != nil { - return err - } return nil } @@ -446,29 +536,47 @@ func (e *Encrypter) loadKeyFromStore(path string) (*structs.RootKey, error) { return nil, err } + if kekWrapper.ProviderID == "" { + kekWrapper.ProviderID = string(structs.KEKProviderAEAD) + } + provider, ok := e.providerConfigs[kekWrapper.ProviderID] + if !ok { + return nil, fmt.Errorf("no such provider %q configured", kekWrapper.ProviderID) + } + // the errors that bubble up from this library can be a bit opaque, so make // sure we wrap them with as much context as possible - wrapper, err := e.newKMSWrapper(meta.KeyID, kekWrapper.KeyEncryptionKey) + wrapper, err := e.newKMSWrapper(provider, meta.KeyID, kekWrapper.KeyEncryptionKey) if err != nil { - return nil, fmt.Errorf("unable to create key wrapper cipher: %w", err) + return nil, fmt.Errorf("unable to create key wrapper: %w", err) } - key, err := wrapper.Decrypt(e.srv.shutdownCtx, &kms.BlobInfo{ - Ciphertext: kekWrapper.EncryptedDataEncryptionKey, - }) + wrappedDEK := kekWrapper.WrappedDataEncryptionKey + if wrappedDEK == nil { + // older KEK wrapper versions with AEAD-only have the key material in a + // different field + wrappedDEK = &wrapping.BlobInfo{Ciphertext: kekWrapper.EncryptedDataEncryptionKey} + } + key, err := wrapper.Decrypt(e.srv.shutdownCtx, wrappedDEK) if err != nil { - return nil, fmt.Errorf("unable to decrypt wrapped root key: %w", err) + return nil, fmt.Errorf("%w (root key): %w", ErrDecryptFailed, err) } // Decrypt RSAKey for Workload Identity JWT signing if one exists. Prior to // 1.7 an ed25519 key derived from the root key was used instead of an RSA // key. var rsaKey []byte - if len(kekWrapper.EncryptedRSAKey) > 0 { - rsaKey, err = wrapper.Decrypt(e.srv.shutdownCtx, &kms.BlobInfo{ - Ciphertext: kekWrapper.EncryptedRSAKey, - }) + if kekWrapper.WrappedRSAKey != nil { + rsaKey, err = wrapper.Decrypt(e.srv.shutdownCtx, kekWrapper.WrappedRSAKey) if err != nil { - return nil, fmt.Errorf("unable to decrypt wrapped rsa key: %w", err) + return nil, fmt.Errorf("%w (rsa key): %w", ErrDecryptFailed, err) + } + } else if len(kekWrapper.EncryptedRSAKey) > 0 { + // older KEK wrapper versions with AEAD-only have the key material in a + // different field + rsaKey, err = wrapper.Decrypt(e.srv.shutdownCtx, &wrapping.BlobInfo{ + Ciphertext: kekWrapper.EncryptedRSAKey}) + if err != nil { + return nil, fmt.Errorf("%w (rsa key): %w", ErrDecryptFailed, err) } } @@ -479,6 +587,8 @@ func (e *Encrypter) loadKeyFromStore(path string) (*structs.RootKey, error) { }, nil } +var ErrDecryptFailed = errors.New("unable to decrypt wrapped key") + // GetPublicKey returns the public signing key for the requested key id or an // error if the key could not be found. func (e *Encrypter) GetPublicKey(keyID string) (*structs.KeyringPublicKey, error) { @@ -508,19 +618,44 @@ func (e *Encrypter) GetPublicKey(keyID string) (*structs.KeyringPublicKey, error } // newKMSWrapper returns a go-kms-wrapping interface the caller can use to -// encrypt the RootKey with a key encryption key (KEK). This is a bit of -// security theatre for local on-disk key material, but gives us a shim for -// external KMS providers in the future. -func (e *Encrypter) newKMSWrapper(keyID string, kek []byte) (kms.Wrapper, error) { - wrapper := aead.NewWrapper() - wrapper.SetConfig(context.Background(), - aead.WithAeadType(kms.AeadTypeAesGcm), - aead.WithHashType(kms.HashTypeSha256), - kms.WithKeyId(keyID), - ) - err := wrapper.SetAesGcmKeyBytes(kek) - if err != nil { - return nil, err +// encrypt the RootKey with a key encryption key (KEK). +func (e *Encrypter) newKMSWrapper(provider *structs.KEKProviderConfig, keyID string, kek []byte) (kms.Wrapper, error) { + var wrapper kms.Wrapper + + // note: adding support for another provider from go-kms-wrapping is a + // matter of adding the dependency and another case here, but the remaining + // third-party providers add significantly to binary size + + switch provider.Provider { + case structs.KEKProviderAWSKMS: + wrapper = awskms.NewWrapper() + case structs.KEKProviderAzureKeyVault: + wrapper = azurekeyvault.NewWrapper() + case structs.KEKProviderGCPCloudKMS: + wrapper = gcpckms.NewWrapper() + case structs.KEKProviderVaultTransit: + wrapper = transit.NewWrapper() + + default: // "aead" + wrapper := aead.NewWrapper() + wrapper.SetConfig(context.Background(), + aead.WithAeadType(kms.AeadTypeAesGcm), + aead.WithHashType(kms.HashTypeSha256), + kms.WithKeyId(keyID), + ) + err := wrapper.SetAesGcmKeyBytes(kek) + if err != nil { + return nil, err + } + return wrapper, nil + } + + config, ok := e.providerConfigs[provider.ID()] + if ok { + _, err := wrapper.SetConfig(context.Background(), wrapping.WithConfigMap(config.Config)) + if err != nil { + return nil, err + } } return wrapper, nil } @@ -582,7 +717,7 @@ func (krr *KeyringReplicator) run(ctx context.Context) { } keyMeta := raw.(*structs.RootKeyMeta) - if key, _, err := krr.encrypter.GetKey(keyMeta.KeyID); err == nil && len(key) > 0 { + if key, err := krr.encrypter.GetKey(keyMeta.KeyID); err == nil && len(key.Key) > 0 { // the key material is immutable so if we've already got it // we can move on to the next key continue @@ -595,6 +730,7 @@ func (krr *KeyringReplicator) run(ctx context.Context) { // prevent this case from sending excessive RPCs krr.logger.Error(err.Error(), "key", keyMeta.KeyID) } + } } } diff --git a/nomad/encrypter_ce.go b/nomad/encrypter_ce.go new file mode 100644 index 000000000..211f960f1 --- /dev/null +++ b/nomad/encrypter_ce.go @@ -0,0 +1,42 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +//go:build !ent +// +build !ent + +package nomad + +import ( + "fmt" + + "github.com/hashicorp/nomad/nomad/structs" +) + +func getProviderConfigs(srv *Server) (map[string]*structs.KEKProviderConfig, error) { + providerConfigs := map[string]*structs.KEKProviderConfig{} + config := srv.GetConfig() + var active int + for _, provider := range config.KEKProviderConfigs { + if provider.Active { + active++ + } + if provider.Provider == structs.KEKProviderVaultTransit { + fallbackVaultConfig(provider, config.GetDefaultVault()) + } + + providerConfigs[provider.ID()] = provider + } + if active > 1 { + return nil, fmt.Errorf( + "only one server.keyring can be active in Nomad Community Edition") + } + + if len(srv.config.KEKProviderConfigs) == 0 { + providerConfigs[string(structs.KEKProviderAEAD)] = &structs.KEKProviderConfig{ + Provider: string(structs.KEKProviderAEAD), + Active: true, + } + } + + return providerConfigs, nil +} diff --git a/nomad/encrypter_test.go b/nomad/encrypter_test.go index a126f4683..e3227daf3 100644 --- a/nomad/encrypter_test.go +++ b/nomad/encrypter_test.go @@ -9,6 +9,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "maps" "os" "path/filepath" "testing" @@ -22,9 +23,12 @@ import ( "github.com/stretchr/testify/require" "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/helper/pointer" + "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/nomad/structs/config" "github.com/hashicorp/nomad/testutil" ) @@ -51,10 +55,10 @@ func (s *mockSigner) SignClaims(c *structs.IdentityClaims) (token, keyID string, func TestEncrypter_LoadSave(t *testing.T) { ci.Parallel(t) - srv, cleanupSrv := TestServer(t, func(c *Config) { - c.NumSchedulers = 0 - }) - t.Cleanup(cleanupSrv) + srv := &Server{ + logger: testlog.HCLogger(t), + config: &Config{}, + } tmpDir := t.TempDir() encrypter, err := NewEncrypter(srv, tmpDir) @@ -73,7 +77,7 @@ func TestEncrypter_LoadSave(t *testing.T) { // startup code path gotKey, err := encrypter.loadKeyFromStore( - filepath.Join(tmpDir, key.Meta.KeyID+".nks.json")) + filepath.Join(tmpDir, key.Meta.KeyID+".aead.nks.json")) must.NoError(t, err) must.NoError(t, encrypter.addCipher(gotKey)) must.Greater(t, 0, len(gotKey.RSAKey)) @@ -84,6 +88,33 @@ func TestEncrypter_LoadSave(t *testing.T) { must.Greater(t, 0, len(active.rootKey.RSAKey)) }) } + + t.Run("legacy aead wrapper", func(t *testing.T) { + key, err := structs.NewRootKey(structs.EncryptionAlgorithmAES256GCM) + must.NoError(t, err) + + // create a wrapper file identical to those before we had external KMS + kekWrapper, err := encrypter.encryptDEK(key, &structs.KEKProviderConfig{}) + kekWrapper.Provider = "" + kekWrapper.ProviderID = "" + kekWrapper.EncryptedDataEncryptionKey = kekWrapper.WrappedDataEncryptionKey.Ciphertext + kekWrapper.EncryptedRSAKey = kekWrapper.WrappedRSAKey.Ciphertext + kekWrapper.WrappedDataEncryptionKey = nil + kekWrapper.WrappedRSAKey = nil + + buf, err := json.Marshal(kekWrapper) + must.NoError(t, err) + + path := filepath.Join(tmpDir, key.Meta.KeyID+".nks.json") + err = os.WriteFile(path, buf, 0o600) + must.NoError(t, err) + + gotKey, err := encrypter.loadKeyFromStore(path) + must.NoError(t, err) + must.NoError(t, encrypter.addCipher(gotKey)) + must.Greater(t, 0, len(gotKey.RSAKey)) + }) + } // TestEncrypter_Restore exercises the entire reload of a keystore, @@ -253,7 +284,7 @@ func TestEncrypter_KeyringReplication(t *testing.T) { keyID1 := listResp.Keys[0].KeyID keyPath := filepath.Join(leader.GetConfig().DataDir, "keystore", - keyID1+nomadKeystoreExtension) + keyID1+".aead.nks.json") _, err := os.Stat(keyPath) must.NoError(t, err, must.Sprint("expected key to be found in leader keystore")) @@ -264,7 +295,7 @@ func TestEncrypter_KeyringReplication(t *testing.T) { return func() bool { for _, srv := range servers { keyPath := filepath.Join(srv.GetConfig().DataDir, "keystore", - keyID+nomadKeystoreExtension) + keyID+".aead.nks.json") if _, err := os.Stat(keyPath); err != nil { return false } @@ -302,7 +333,7 @@ func TestEncrypter_KeyringReplication(t *testing.T) { must.NotNil(t, getResp.Key, must.Sprint("expected key to be found on leader")) keyPath = filepath.Join(leader.GetConfig().DataDir, "keystore", - keyID2+nomadKeystoreExtension) + keyID2+".aead.nks.json") _, err = os.Stat(keyPath) must.NoError(t, err, must.Sprint("expected key to be found in leader keystore")) @@ -639,3 +670,63 @@ func TestEncrypter_Upgrade17(t *testing.T) { _, err = srv.encrypter.VerifyClaim(oldRawJWT) must.NoError(t, err) } + +func TestEncrypter_TransitConfigFallback(t *testing.T) { + srv := &Server{ + logger: testlog.HCLogger(t), + config: &Config{ + VaultConfigs: map[string]*config.VaultConfig{structs.VaultDefaultCluster: { + Addr: "https://localhost:8203", + TLSCaPath: "/etc/certs/ca", + TLSCertFile: "/var/certs/vault.crt", + TLSKeyFile: "/var/certs/vault.key", + TLSSkipVerify: pointer.Of(true), + TLSServerName: "foo", + Token: "vault-token", + }}, + KEKProviderConfigs: []*structs.KEKProviderConfig{ + { + Provider: "transit", + Name: "no-fallback", + Config: map[string]string{ + "address": "https://localhost:8203", + "token": "vault-token", + "tls_ca_cert": "/etc/certs/ca", + "tls_client_cert": "/var/certs/vault.crt", + "tls_client_key": "/var/certs/vault.key", + "tls_server_name": "foo", + "tls_skip_verify": "true", + }, + }, + { + Provider: "transit", + Name: "fallback-to-vault-block", + }, + { + Provider: "transit", + Name: "fallback-to-env", + }, + }, + }, + } + + providers := srv.config.KEKProviderConfigs + expect := maps.Clone(providers[0].Config) + + fallbackVaultConfig(providers[0], srv.config.GetDefaultVault()) + must.Eq(t, expect, providers[0].Config, must.Sprint("expected no change")) + + fallbackVaultConfig(providers[1], srv.config.GetDefaultVault()) + must.Eq(t, expect, providers[1].Config, must.Sprint("expected fallback to vault block")) + + t.Setenv("VAULT_ADDR", "https://localhost:8203") + t.Setenv("VAULT_TOKEN", "vault-token") + t.Setenv("VAULT_CACERT", "/etc/certs/ca") + t.Setenv("VAULT_CLIENT_CERT", "/var/certs/vault.crt") + t.Setenv("VAULT_CLIENT_KEY", "/var/certs/vault.key") + t.Setenv("VAULT_TLS_SERVER_NAME", "foo") + t.Setenv("VAULT_SKIP_VERIFY", "true") + + fallbackVaultConfig(providers[2], &config.VaultConfig{}) + must.Eq(t, expect, providers[2].Config, must.Sprint("expected fallback to env")) +} diff --git a/nomad/keyring_endpoint.go b/nomad/keyring_endpoint.go index 65a8d02b3..95302b639 100644 --- a/nomad/keyring_endpoint.go +++ b/nomad/keyring_endpoint.go @@ -268,15 +268,10 @@ func (k *Keyring) Get(args *structs.KeyringGetRootKeyRequest, reply *structs.Key } // retrieve the key material from the keyring - key, rsaKey, err := k.encrypter.GetKey(keyMeta.KeyID) + rootKey, err := k.encrypter.GetKey(keyMeta.KeyID) if err != nil { return err } - rootKey := &structs.RootKey{ - Meta: keyMeta, - Key: key, - RSAKey: rsaKey, - } reply.Key = rootKey // Use the last index that affected the policy table diff --git a/nomad/structs/keyring.go b/nomad/structs/keyring.go index 103068716..67379d89a 100644 --- a/nomad/structs/keyring.go +++ b/nomad/structs/keyring.go @@ -9,10 +9,12 @@ import ( "crypto/rsa" "crypto/x509" "fmt" + "maps" "net/url" "time" "github.com/go-jose/go-jose/v3" + wrapping "github.com/hashicorp/go-kms-wrapping/v2" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/crypto" "github.com/hashicorp/nomad/helper/uuid" @@ -86,6 +88,59 @@ type RootKeyMeta struct { State RootKeyState } +// KEKProviderName enum are the built-in KEK providers. +type KEKProviderName string + +const ( + KEKProviderAEAD KEKProviderName = "aead" + KEKProviderAWSKMS = "awskms" + KEKProviderAzureKeyVault = "azurekeyvault" + KEKProviderGCPCloudKMS = "gcpckms" + KEKProviderVaultTransit = "transit" +) + +// KEKProviderConfig is the server configuration for an external KMS provider +// the server will use as a Key Encryption Key (KEK) for encrypting/decrypting +// the DEK. +type KEKProviderConfig struct { + Provider string `hcl:",key"` + Name string `hcl:"name"` + Active bool `hcl:"active"` + Config map[string]string `hcl:"-" json:"-"` + + // ExtraKeysHCL gets used by HCL to surface unknown keys. The parser will + // then read these keys to create the Config map, so that we don't need a + // nested "config" block/map in the config file + ExtraKeysHCL []string `hcl:",unusedKeys" json:"-"` +} + +func (c *KEKProviderConfig) Copy() *KEKProviderConfig { + return &KEKProviderConfig{ + Provider: c.Provider, + Active: c.Active, + Name: c.Name, + Config: maps.Clone(c.Config), + } +} + +// Merge is used to merge two configurations. Note that Provider and Name should +// always be identical before we merge. +func (c *KEKProviderConfig) Merge(o *KEKProviderConfig) *KEKProviderConfig { + result := c.Copy() + result.Active = o.Active + for k, v := range o.Config { + result.Config[k] = v + } + return result +} + +func (c *KEKProviderConfig) ID() string { + if c.Name == "" { + return c.Provider + } + return c.Provider + "." + c.Name +} + // RootKeyState enum describes the lifecycle of a root key. type RootKeyState string @@ -192,13 +247,24 @@ func (rkm *RootKeyMeta) Validate() error { } // KeyEncryptionKeyWrapper is the struct that gets serialized for the on-disk -// KMS wrapper. This struct includes the server-specific key-wrapping key and -// should never be sent over RPC. +// KMS wrapper. When using the AEAD provider, this struct includes the +// server-specific key-wrapping key. This struct should never be sent over RPC +// or written to Raft. type KeyEncryptionKeyWrapper struct { - Meta *RootKeyMeta - EncryptedDataEncryptionKey []byte `json:"DEK"` - EncryptedRSAKey []byte `json:"RSAKey"` - KeyEncryptionKey []byte `json:"KEK"` + Meta *RootKeyMeta + + Provider string `json:"Provider,omitempty"` + ProviderID string `json:"ProviderID,omitempty"` + WrappedDataEncryptionKey *wrapping.BlobInfo `json:"WrappedDEK,omitempty"` + WrappedRSAKey *wrapping.BlobInfo `json:"WrappedRSAKey,omitempty"` + KeyEncryptionKey []byte `json:"KEK,omitempty"` + + // These fields were used for AEAD before we added support for external + // KMS. The wrapped key returned from the go-kms-wrapper library includes + // the ciphertext but we need all the fields in order to decrypt. We'll + // leave these fields so we can load keys from older servers. + EncryptedDataEncryptionKey []byte `json:"DEK,omitempty"` + EncryptedRSAKey []byte `json:"RSAKey,omitempty"` } // EncryptionAlgorithm chooses which algorithm is used for @@ -261,7 +327,7 @@ type KeyringGetRootKeyResponse struct { // KeyringUpdateRootKeyMetaRequest is used internally for key // replication so that we have a request wrapper for writing the -// metadata to the FSM without including the key material +// metadata to the FSM without including the key material. type KeyringUpdateRootKeyMetaRequest struct { RootKeyMeta *RootKeyMeta Rekey bool