diff --git a/go.mod b/go.mod index 952342216..e6404376e 100644 --- a/go.mod +++ b/go.mod @@ -58,6 +58,7 @@ require ( github.com/hashicorp/go-getter v1.6.1 github.com/hashicorp/go-hclog v1.2.2 github.com/hashicorp/go-immutable-radix v1.3.1 + github.com/hashicorp/go-kms-wrapping/v2 v2.0.5 github.com/hashicorp/go-memdb v1.3.3 github.com/hashicorp/go-msgpack v1.1.5 github.com/hashicorp/go-multierror v1.1.1 diff --git a/go.sum b/go.sum index cdb023554..896f75964 100644 --- a/go.sum +++ b/go.sum @@ -717,6 +717,8 @@ github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-kms-wrapping/entropy v0.1.0/go.mod h1:d1g9WGtAunDNpek8jUIEJnBlbgKS1N2Q61QkHiZyR1g= +github.com/hashicorp/go-kms-wrapping/v2 v2.0.5 h1:rOFDv+3k05mnW0oaDLffhVUwg03Csn0mvfO98Wdd2bE= +github.com/hashicorp/go-kms-wrapping/v2 v2.0.5/go.mod h1:sDQAfwJGv25uGPZA04x87ERglCG6avnRcBT9wYoMII8= github.com/hashicorp/go-memdb v1.0.3/go.mod h1:LWQ8R70vPrS4OEY9k28D2z8/Zzyu34NVzeRibGAzHO0= github.com/hashicorp/go-memdb v1.3.3 h1:oGfEWrFuxtIUF3W2q/Jzt6G85TrMk9ey6XfYLvVe1Wo= github.com/hashicorp/go-memdb v1.3.3/go.mod h1:uBTr1oQbtuMgd1SSGoR8YV27eT3sBHbYiNm53bMpgSg= diff --git a/helper/crypto/crypto.go b/helper/crypto/crypto.go new file mode 100644 index 000000000..03af45928 --- /dev/null +++ b/helper/crypto/crypto.go @@ -0,0 +1,24 @@ +package crypto + +import ( + "errors" + "fmt" + + // note: this is aliased so that it's more noticeable if someone + // accidentally swaps it out for math/rand via running goimports + cryptorand "crypto/rand" +) + +// Bytes gets a slice of cryptographically random bytes of the given length and +// enforces that we check for short reads to avoid entropy exhaustion. +func Bytes(length int) ([]byte, error) { + key := make([]byte, length) + n, err := cryptorand.Read(key) + if err != nil { + return nil, fmt.Errorf("could not read from random source: %v", err) + } + if n < length { + return nil, errors.New("entropy exhausted") + } + return key, nil +} diff --git a/helper/uuid/uuid.go b/helper/uuid/uuid.go index 22b5d5b9e..7a2226ee0 100644 --- a/helper/uuid/uuid.go +++ b/helper/uuid/uuid.go @@ -1,14 +1,15 @@ package uuid import ( - "crypto/rand" "fmt" + + "github.com/hashicorp/nomad/helper/crypto" ) // Generate is used to generate a random UUID. func Generate() string { - buf := make([]byte, 16) - if _, err := rand.Read(buf); err != nil { + buf, err := crypto.Bytes(16) + if err != nil { panic(fmt.Errorf("failed to read random bytes: %v", err)) } diff --git a/nomad/encrypter.go b/nomad/encrypter.go index 984196b46..778715414 100644 --- a/nomad/encrypter.go +++ b/nomad/encrypter.go @@ -1,12 +1,10 @@ package nomad import ( - "bytes" "context" "crypto/aes" "crypto/cipher" "crypto/ed25519" - "encoding/base64" "encoding/json" "fmt" "io/fs" @@ -16,16 +14,14 @@ import ( "sync" "time" - // note: this is aliased so that it's more noticeable if someone - // accidentally swaps it out for math/rand via running goimports - cryptorand "crypto/rand" - jwt "github.com/golang-jwt/jwt/v4" log "github.com/hashicorp/go-hclog" - "github.com/hashicorp/go-msgpack/codec" + kms "github.com/hashicorp/go-kms-wrapping/v2" + "github.com/hashicorp/go-kms-wrapping/v2/aead" "golang.org/x/time/rate" "github.com/hashicorp/nomad/helper" + "github.com/hashicorp/nomad/helper/crypto" "github.com/hashicorp/nomad/nomad/structs" ) @@ -50,26 +46,27 @@ type keyset struct { // NewEncrypter loads or creates a new local keystore and returns an // encryption keyring with the keys it finds. func NewEncrypter(srv *Server, keystorePath string) (*Encrypter, error) { - err := os.MkdirAll(keystorePath, 0700) + + encrypter := &Encrypter{ + srv: srv, + keystorePath: keystorePath, + keyring: make(map[string]*keyset), + } + + err := encrypter.loadKeystore() if err != nil { return nil, err } - encrypter, err := encrypterFromKeystore(keystorePath) - if err != nil { - return nil, err - } - encrypter.srv = srv return encrypter, nil } -func encrypterFromKeystore(keystoreDirectory string) (*Encrypter, error) { +func (e *Encrypter) loadKeystore() error { - encrypter := &Encrypter{ - keyring: make(map[string]*keyset), - keystorePath: keystoreDirectory, + if err := os.MkdirAll(e.keystorePath, 0o700); err != nil { + return err } - err := filepath.Walk(keystoreDirectory, func(path string, info fs.FileInfo, err error) error { + return filepath.Walk(e.keystorePath, func(path string, info fs.FileInfo, err error) error { if err != nil { return fmt.Errorf("could not read path %s from keystore: %v", path, err) } @@ -77,7 +74,7 @@ func encrypterFromKeystore(keystoreDirectory string) (*Encrypter, error) { // skip over subdirectories and non-key files; they shouldn't // be here but there's no reason to fail startup for it if the // administrator has left something there - if path != keystoreDirectory && info.IsDir() { + if path != e.keystorePath && info.IsDir() { return filepath.SkipDir } if !strings.HasSuffix(path, nomadKeystoreExtension) { @@ -88,7 +85,7 @@ func encrypterFromKeystore(keystoreDirectory string) (*Encrypter, error) { return nil } - key, err := encrypter.loadKeyFromStore(path) + key, err := e.loadKeyFromStore(path) if err != nil { return fmt.Errorf("could not load key file %s from keystore: %v", path, err) } @@ -96,17 +93,12 @@ func encrypterFromKeystore(keystoreDirectory string) (*Encrypter, error) { return fmt.Errorf("root key ID %s must match key file %s", key.Meta.KeyID, path) } - err = encrypter.AddKey(key) + err = e.AddKey(key) if err != nil { return fmt.Errorf("could not add key file %s to keystore: %v", path, err) } return nil }) - if err != nil { - return nil, err - } - - return encrypter, nil } // Encrypt encrypts the clear data with the cipher for the current @@ -121,14 +113,9 @@ func (e *Encrypter) Encrypt(cleartext []byte) ([]byte, string, error) { return nil, "", err } - nonceSize := keyset.cipher.NonceSize() - nonce := make([]byte, nonceSize) - n, err := cryptorand.Read(nonce) + nonce, err := crypto.Bytes(keyset.cipher.NonceSize()) if err != nil { - return nil, "", err - } - if n < nonceSize { - return nil, "", fmt.Errorf("failed to encrypt: entropy exhausted") + return nil, "", fmt.Errorf("failed to generate key wrapper nonce: %v", err) } keyID := keyset.rootKey.Meta.KeyID @@ -306,9 +293,6 @@ func (e *Encrypter) keysetByIDLocked(keyID string) (*keyset, error) { // RemoveKey removes a key by ID from the keyring func (e *Encrypter) RemoveKey(keyID string) error { - // TODO: should the server remove the serialized file here? - // TODO: given that it's irreversible, should the server *ever* - // remove the serialized file? e.lock.Lock() defer e.lock.Unlock() delete(e.keyring, keyID) @@ -317,14 +301,33 @@ func (e *Encrypter) RemoveKey(keyID string) error { // saveKeyToStore serializes a root key to the on-disk keystore. func (e *Encrypter) saveKeyToStore(rootKey *structs.RootKey) error { - var buf bytes.Buffer - enc := codec.NewEncoder(&buf, structs.JsonHandleWithExtensions) - err := enc.Encode(rootKey) + + kek, err := crypto.Bytes(32) + if err != nil { + return fmt.Errorf("failed to generate key wrapper key: %v", err) + } + wrapper, err := e.newKMSWrapper(rootKey.Meta.KeyID, kek) + if err != nil { + return fmt.Errorf("failed to create encryption wrapper: %v", err) + } + blob, err := wrapper.Encrypt(e.srv.shutdownCtx, rootKey.Key) + if err != nil { + return fmt.Errorf("failed to encrypt root key: %v", err) + } + + kekWrapper := &structs.KeyEncryptionKeyWrapper{ + Meta: rootKey.Meta, + EncryptedDataEncryptionKey: blob.Ciphertext, + KeyEncryptionKey: kek, + } + + buf, err := json.Marshal(kekWrapper) if err != nil { return err } + path := filepath.Join(e.keystorePath, rootKey.Meta.KeyID+nomadKeystoreExtension) - err = os.WriteFile(path, buf.Bytes(), 0600) + err = os.WriteFile(path, buf, 0o600) if err != nil { return err } @@ -339,28 +342,27 @@ func (e *Encrypter) loadKeyFromStore(path string) (*structs.RootKey, error) { return nil, err } - storedKey := &struct { - Meta *structs.RootKeyMetaStub - Key string - }{} - - if err := json.Unmarshal(raw, storedKey); err != nil { + kekWrapper := &structs.KeyEncryptionKeyWrapper{} + if err := json.Unmarshal(raw, kekWrapper); err != nil { return nil, err } - meta := &structs.RootKeyMeta{ - State: storedKey.Meta.State, - KeyID: storedKey.Meta.KeyID, - Algorithm: storedKey.Meta.Algorithm, - CreateTime: storedKey.Meta.CreateTime, - } + meta := kekWrapper.Meta if err = meta.Validate(); err != nil { return nil, err } - key, err := base64.StdEncoding.DecodeString(storedKey.Key) + // the errors that bubble up from this library can be a bit opaque, so make + // sure we wrap them with as much context as possible + wrapper, err := e.newKMSWrapper(meta.KeyID, kekWrapper.KeyEncryptionKey) if err != nil { - return nil, fmt.Errorf("could not decode key: %v", err) + return nil, fmt.Errorf("unable to create key wrapper cipher: %v", err) + } + key, err := wrapper.Decrypt(e.srv.shutdownCtx, &kms.BlobInfo{ + Ciphertext: kekWrapper.EncryptedDataEncryptionKey, + }) + if err != nil { + return nil, fmt.Errorf("unable to decrypt wrapped root key: %v", err) } return &structs.RootKey{ @@ -369,6 +371,24 @@ func (e *Encrypter) loadKeyFromStore(path string) (*structs.RootKey, error) { }, nil } +// newKMSWrapper returns a go-kms-wrapping interface the caller can use to +// encrypt the RootKey with a key encryption key (KEK). This is a bit of +// security theatre for local on-disk key material, but gives us a shim for +// external KMS providers in the future. +func (e *Encrypter) newKMSWrapper(keyID string, kek []byte) (kms.Wrapper, error) { + wrapper := aead.NewWrapper() + wrapper.SetConfig(context.Background(), + aead.WithAeadType(kms.AeadTypeAesGcm), + aead.WithHashType(kms.HashTypeSha256), + kms.WithKeyId(keyID), + ) + err := wrapper.SetAesGcmKeyBytes(kek) + if err != nil { + return nil, err + } + return wrapper, nil +} + type KeyringReplicator struct { srv *Server encrypter *Encrypter diff --git a/nomad/encrypter_test.go b/nomad/encrypter_test.go index f1fa95faf..b006b503d 100644 --- a/nomad/encrypter_test.go +++ b/nomad/encrypter_test.go @@ -21,7 +21,7 @@ func TestEncrypter_LoadSave(t *testing.T) { ci.Parallel(t) tmpDir := t.TempDir() - encrypter, err := NewEncrypter(nil, tmpDir) + encrypter, err := NewEncrypter(&Server{shutdownCtx: context.Background()}, tmpDir) require.NoError(t, err) algos := []structs.EncryptionAlgorithm{ diff --git a/nomad/structs/extensions.go b/nomad/structs/extensions.go index fcda97927..19d287aa2 100644 --- a/nomad/structs/extensions.go +++ b/nomad/structs/extensions.go @@ -1,7 +1,6 @@ package structs import ( - "encoding/base64" "reflect" ) @@ -13,8 +12,6 @@ var ( reflect.TypeOf(&Node{}): nodeExt, reflect.TypeOf(CSIVolume{}): csiVolumeExt, reflect.TypeOf(&CSIVolume{}): csiVolumeExt, - reflect.TypeOf(&RootKey{}): rootKeyExt, - reflect.TypeOf(RootKey{}): rootKeyExt, } ) @@ -79,21 +76,3 @@ func csiVolumeExt(v interface{}) interface{} { return apiVol } - -// rootKeyExt safely serializes a RootKey by base64 encoding the key -// material and extracting the metadata stub. We only store the root -// key in the keystore and never in raft or return it via the API, so -// by having this extension as the default we make it slightly harder -// to misuse. -func rootKeyExt(v interface{}) interface{} { - key := v.(*RootKey) - encodedKey := base64.StdEncoding.EncodeToString(key.Key) - - return &struct { - Meta *RootKeyMetaStub - Key string - }{ - Meta: key.Meta.Stub(), - Key: encodedKey, - } -} diff --git a/nomad/structs/keyring.go b/nomad/structs/keyring.go index 24f48f59b..6ba03c462 100644 --- a/nomad/structs/keyring.go +++ b/nomad/structs/keyring.go @@ -4,11 +4,8 @@ import ( "fmt" "time" - // note: this is aliased so that it's more noticeable if someone - // accidentally swaps it out for math/rand via running goimports - cryptorand "crypto/rand" - "github.com/hashicorp/nomad/helper" + "github.com/hashicorp/nomad/helper/crypto" "github.com/hashicorp/nomad/helper/uuid" ) @@ -29,14 +26,9 @@ func NewRootKey(algorithm EncryptionAlgorithm) (*RootKey, error) { switch algorithm { case EncryptionAlgorithmAES256GCM: - const keyBytes = 32 - key := make([]byte, keyBytes) - n, err := cryptorand.Read(key) + key, err := crypto.Bytes(32) if err != nil { - return nil, err - } - if n < keyBytes { - return nil, fmt.Errorf("failed to generate key: entropy exhausted") + return nil, fmt.Errorf("failed to generate key: %v", err) } rootKey.Key = key } @@ -160,6 +152,15 @@ func (rkm *RootKeyMeta) Validate() error { return nil } +// KeyEncryptionKeyWrapper is the struct that gets serialized for the on-disk +// KMS wrapper. This struct includes the server-specific key-wrapping key and +// should never be sent over RPC. +type KeyEncryptionKeyWrapper struct { + Meta *RootKeyMeta + EncryptedDataEncryptionKey []byte `json:"DEK"` + KeyEncryptionKey []byte `json:"KEK"` +} + // EncryptionAlgorithm chooses which algorithm is used for // encrypting / decrypting entries with this key type EncryptionAlgorithm string @@ -168,6 +169,7 @@ const ( EncryptionAlgorithmAES256GCM EncryptionAlgorithm = "aes256-gcm" ) +// KeyringRotateRootKeyRequest is the argument to the Keyring.Rotate RPC type KeyringRotateRootKeyRequest struct { Algorithm EncryptionAlgorithm Full bool @@ -180,11 +182,12 @@ type KeyringRotateRootKeyResponse struct { WriteMeta } +// KeyringListRootKeyMetaRequest is the argument to the Keyring.List RPC type KeyringListRootKeyMetaRequest struct { - // TODO: do we need any fields here? QueryOptions } +// KeyringListRootKeyMetaRequest is the response value of the List RPC type KeyringListRootKeyMetaResponse struct { Keys []*RootKeyMeta QueryMeta diff --git a/website/content/docs/operations/key-management.mdx b/website/content/docs/operations/key-management.mdx index 04d138bc8..3f243c5bb 100644 --- a/website/content/docs/operations/key-management.mdx +++ b/website/content/docs/operations/key-management.mdx @@ -10,7 +10,8 @@ Nomad servers maintain an encryption keyring used to encrypt [Variables][] and sign task [workload identities][]. The servers store key metadata in raft, but the encryption key material is stored in a separate file in the `keystore` subdirectory of the Nomad [data directory][]. These files have the extension -`.nks.json`. +`.nks.json`. The key material in each file is wrapped in a unique key encryption +key (KEK) that is not shared between servers. Under normal operations the keyring is entirely managed by Nomad, but this section provides administrators additional context around key replication and @@ -49,8 +50,10 @@ snapshots you might provide without exposing any of your keys or variables. However, this means that to restore a cluster from snapshot you need to also provide the keystore directory with the `.nks.json` key files on at least one -server. Operators should include these files as part of your organization's -backup and recovery strategy for the cluster. +server. The `.nks.json` key files are unique per server, but only one server's +key files are needed to recover the cluster. Operators should include these +files as part of your organization's backup and recovery strategy for the +cluster. [Variables]: /docs/concepts/variables [workload identities]: /docs/concepts/workload-identity