keyring: wrap root key in key encryption key (#14388)

Update the on-disk format for the root key so that it's wrapped with a unique
per-key/per-server key encryption key. This is a bit of security theatre for the
current implementation, but it uses `go-kms-wrapping` as the interface for
wrapping the key. This provides a shim for future support of external KMS such
as cloud provider APIs or Vault transit encryption.

* Removes the JSON serialization extension we had on the `RootKey` struct; this
  struct is now only used for key replication and not for disk serialization, so
  we don't need this helper.

* Creates a helper for generating cryptographically random slices of bytes that
  properly accounts for short reads from the source.

* No observable functional changes outside of the on-disk format, so there are
  no test updates.

go.mod

@@ -58,6 +58,7 @@ require (
 	github.com/hashicorp/go-getter v1.6.1
 	github.com/hashicorp/go-hclog v1.2.2
 	github.com/hashicorp/go-immutable-radix v1.3.1
+	github.com/hashicorp/go-kms-wrapping/v2 v2.0.5
 	github.com/hashicorp/go-memdb v1.3.3
 	github.com/hashicorp/go-msgpack v1.1.5
 	github.com/hashicorp/go-multierror v1.1.1

go.sum

@@ -717,6 +717,8 @@ github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh
 github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
 github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-kms-wrapping/entropy v0.1.0/go.mod h1:d1g9WGtAunDNpek8jUIEJnBlbgKS1N2Q61QkHiZyR1g=
+github.com/hashicorp/go-kms-wrapping/v2 v2.0.5 h1:rOFDv+3k05mnW0oaDLffhVUwg03Csn0mvfO98Wdd2bE=
+github.com/hashicorp/go-kms-wrapping/v2 v2.0.5/go.mod h1:sDQAfwJGv25uGPZA04x87ERglCG6avnRcBT9wYoMII8=
 github.com/hashicorp/go-memdb v1.0.3/go.mod h1:LWQ8R70vPrS4OEY9k28D2z8/Zzyu34NVzeRibGAzHO0=
 github.com/hashicorp/go-memdb v1.3.3 h1:oGfEWrFuxtIUF3W2q/Jzt6G85TrMk9ey6XfYLvVe1Wo=
 github.com/hashicorp/go-memdb v1.3.3/go.mod h1:uBTr1oQbtuMgd1SSGoR8YV27eT3sBHbYiNm53bMpgSg=

helper/crypto/crypto.go

@@ -0,0 +1,24 @@
+package crypto
+
+import (
+	"errors"
+	"fmt"
+
+	// note: this is aliased so that it's more noticeable if someone
+	// accidentally swaps it out for math/rand via running goimports
+	cryptorand "crypto/rand"
+)
+
+// Bytes gets a slice of cryptographically random bytes of the given length and
+// enforces that we check for short reads to avoid entropy exhaustion.
+func Bytes(length int) ([]byte, error) {
+	key := make([]byte, length)
+	n, err := cryptorand.Read(key)
+	if err != nil {
+		return nil, fmt.Errorf("could not read from random source: %v", err)
+	}
+	if n < length {
+		return nil, errors.New("entropy exhausted")
+	}
+	return key, nil
+}

helper/uuid/uuid.go

@@ -1,14 +1,15 @@
 package uuid
 
 import (
-	"crypto/rand"
 	"fmt"
+
+	"github.com/hashicorp/nomad/helper/crypto"
 )
 
 // Generate is used to generate a random UUID.
 func Generate() string {
-	buf := make([]byte, 16)
-	if _, err := rand.Read(buf); err != nil {
+	buf, err := crypto.Bytes(16)
+	if err != nil {
 		panic(fmt.Errorf("failed to read random bytes: %v", err))
 	}
 

nomad/encrypter.go

@@ -1,12 +1,10 @@
 package nomad
 
 import (
-	"bytes"
 	"context"
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/ed25519"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io/fs"
@@ -16,16 +14,14 @@ import (
 	"sync"
 	"time"
 
-	// note: this is aliased so that it's more noticeable if someone
-	// accidentally swaps it out for math/rand via running goimports
-	cryptorand "crypto/rand"
-
 	jwt "github.com/golang-jwt/jwt/v4"
 	log "github.com/hashicorp/go-hclog"
-	"github.com/hashicorp/go-msgpack/codec"
+	kms "github.com/hashicorp/go-kms-wrapping/v2"
+	"github.com/hashicorp/go-kms-wrapping/v2/aead"
 	"golang.org/x/time/rate"
 
 	"github.com/hashicorp/nomad/helper"
+	"github.com/hashicorp/nomad/helper/crypto"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
@@ -50,26 +46,27 @@ type keyset struct {
 // NewEncrypter loads or creates a new local keystore and returns an
 // encryption keyring with the keys it finds.
 func NewEncrypter(srv *Server, keystorePath string) (*Encrypter, error) {
-	err := os.MkdirAll(keystorePath, 0700)
+
+	encrypter := &Encrypter{
+		srv:          srv,
+		keystorePath: keystorePath,
+		keyring:      make(map[string]*keyset),
+	}
+
+	err := encrypter.loadKeystore()
 	if err != nil {
 		return nil, err
 	}
-	encrypter, err := encrypterFromKeystore(keystorePath)
-	if err != nil {
-		return nil, err
-	}
-	encrypter.srv = srv
 	return encrypter, nil
 }
 
-func encrypterFromKeystore(keystoreDirectory string) (*Encrypter, error) {
+func (e *Encrypter) loadKeystore() error {
 
-	encrypter := &Encrypter{
-		keyring:      make(map[string]*keyset),
-		keystorePath: keystoreDirectory,
+	if err := os.MkdirAll(e.keystorePath, 0o700); err != nil {
+		return err
 	}
 
-	err := filepath.Walk(keystoreDirectory, func(path string, info fs.FileInfo, err error) error {
+	return filepath.Walk(e.keystorePath, func(path string, info fs.FileInfo, err error) error {
 		if err != nil {
 			return fmt.Errorf("could not read path %s from keystore: %v", path, err)
 		}
@@ -77,7 +74,7 @@ func encrypterFromKeystore(keystoreDirectory string) (*Encrypter, error) {
 		// skip over subdirectories and non-key files; they shouldn't
 		// be here but there's no reason to fail startup for it if the
 		// administrator has left something there
-		if path != keystoreDirectory && info.IsDir() {
+		if path != e.keystorePath && info.IsDir() {
 			return filepath.SkipDir
 		}
 		if !strings.HasSuffix(path, nomadKeystoreExtension) {
@@ -88,7 +85,7 @@ func encrypterFromKeystore(keystoreDirectory string) (*Encrypter, error) {
 			return nil
 		}
 
-		key, err := encrypter.loadKeyFromStore(path)
+		key, err := e.loadKeyFromStore(path)
 		if err != nil {
 			return fmt.Errorf("could not load key file %s from keystore: %v", path, err)
 		}
@@ -96,17 +93,12 @@ func encrypterFromKeystore(keystoreDirectory string) (*Encrypter, error) {
 			return fmt.Errorf("root key ID %s must match key file %s", key.Meta.KeyID, path)
 		}
 
-		err = encrypter.AddKey(key)
+		err = e.AddKey(key)
 		if err != nil {
 			return fmt.Errorf("could not add key file %s to keystore: %v", path, err)
 		}
 		return nil
 	})
-	if err != nil {
-		return nil, err
-	}
-
-	return encrypter, nil
 }
 
 // Encrypt encrypts the clear data with the cipher for the current
@@ -121,14 +113,9 @@ func (e *Encrypter) Encrypt(cleartext []byte) ([]byte, string, error) {
 		return nil, "", err
 	}
 
-	nonceSize := keyset.cipher.NonceSize()
-	nonce := make([]byte, nonceSize)
-	n, err := cryptorand.Read(nonce)
+	nonce, err := crypto.Bytes(keyset.cipher.NonceSize())
 	if err != nil {
-		return nil, "", err
-	}
-	if n < nonceSize {
-		return nil, "", fmt.Errorf("failed to encrypt: entropy exhausted")
+		return nil, "", fmt.Errorf("failed to generate key wrapper nonce: %v", err)
 	}
 
 	keyID := keyset.rootKey.Meta.KeyID
@@ -306,9 +293,6 @@ func (e *Encrypter) keysetByIDLocked(keyID string) (*keyset, error) {
 
 // RemoveKey removes a key by ID from the keyring
 func (e *Encrypter) RemoveKey(keyID string) error {
-	// TODO: should the server remove the serialized file here?
-	// TODO: given that it's irreversible, should the server *ever*
-	// remove the serialized file?
 	e.lock.Lock()
 	defer e.lock.Unlock()
 	delete(e.keyring, keyID)
@@ -317,14 +301,33 @@ func (e *Encrypter) RemoveKey(keyID string) error {
 
 // saveKeyToStore serializes a root key to the on-disk keystore.
 func (e *Encrypter) saveKeyToStore(rootKey *structs.RootKey) error {
-	var buf bytes.Buffer
-	enc := codec.NewEncoder(&buf, structs.JsonHandleWithExtensions)
-	err := enc.Encode(rootKey)
+
+	kek, err := crypto.Bytes(32)
+	if err != nil {
+		return fmt.Errorf("failed to generate key wrapper key: %v", err)
+	}
+	wrapper, err := e.newKMSWrapper(rootKey.Meta.KeyID, kek)
+	if err != nil {
+		return fmt.Errorf("failed to create encryption wrapper: %v", err)
+	}
+	blob, err := wrapper.Encrypt(e.srv.shutdownCtx, rootKey.Key)
+	if err != nil {
+		return fmt.Errorf("failed to encrypt root key: %v", err)
+	}
+
+	kekWrapper := &structs.KeyEncryptionKeyWrapper{
+		Meta:                       rootKey.Meta,
+		EncryptedDataEncryptionKey: blob.Ciphertext,
+		KeyEncryptionKey:           kek,
+	}
+
+	buf, err := json.Marshal(kekWrapper)
 	if err != nil {
 		return err
 	}
+
 	path := filepath.Join(e.keystorePath, rootKey.Meta.KeyID+nomadKeystoreExtension)
-	err = os.WriteFile(path, buf.Bytes(), 0600)
+	err = os.WriteFile(path, buf, 0o600)
 	if err != nil {
 		return err
 	}
@@ -339,28 +342,27 @@ func (e *Encrypter) loadKeyFromStore(path string) (*structs.RootKey, error) {
 		return nil, err
 	}
 
-	storedKey := &struct {
-		Meta *structs.RootKeyMetaStub
-		Key  string
-	}{}
-
-	if err := json.Unmarshal(raw, storedKey); err != nil {
+	kekWrapper := &structs.KeyEncryptionKeyWrapper{}
+	if err := json.Unmarshal(raw, kekWrapper); err != nil {
 		return nil, err
 	}
 
-	meta := &structs.RootKeyMeta{
-		State:      storedKey.Meta.State,
-		KeyID:      storedKey.Meta.KeyID,
-		Algorithm:  storedKey.Meta.Algorithm,
-		CreateTime: storedKey.Meta.CreateTime,
-	}
+	meta := kekWrapper.Meta
 	if err = meta.Validate(); err != nil {
 		return nil, err
 	}
 
-	key, err := base64.StdEncoding.DecodeString(storedKey.Key)
+	// the errors that bubble up from this library can be a bit opaque, so make
+	// sure we wrap them with as much context as possible
+	wrapper, err := e.newKMSWrapper(meta.KeyID, kekWrapper.KeyEncryptionKey)
 	if err != nil {
-		return nil, fmt.Errorf("could not decode key: %v", err)
+		return nil, fmt.Errorf("unable to create key wrapper cipher: %v", err)
+	}
+	key, err := wrapper.Decrypt(e.srv.shutdownCtx, &kms.BlobInfo{
+		Ciphertext: kekWrapper.EncryptedDataEncryptionKey,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("unable to decrypt wrapped root key: %v", err)
 	}
 
 	return &structs.RootKey{
@@ -369,6 +371,24 @@ func (e *Encrypter) loadKeyFromStore(path string) (*structs.RootKey, error) {
 	}, nil
 }
 
+// newKMSWrapper returns a go-kms-wrapping interface the caller can use to
+// encrypt the RootKey with a key encryption key (KEK). This is a bit of
+// security theatre for local on-disk key material, but gives us a shim for
+// external KMS providers in the future.
+func (e *Encrypter) newKMSWrapper(keyID string, kek []byte) (kms.Wrapper, error) {
+	wrapper := aead.NewWrapper()
+	wrapper.SetConfig(context.Background(),
+		aead.WithAeadType(kms.AeadTypeAesGcm),
+		aead.WithHashType(kms.HashTypeSha256),
+		kms.WithKeyId(keyID),
+	)
+	err := wrapper.SetAesGcmKeyBytes(kek)
+	if err != nil {
+		return nil, err
+	}
+	return wrapper, nil
+}
+
 type KeyringReplicator struct {
 	srv       *Server
 	encrypter *Encrypter

nomad/encrypter_test.go

@@ -21,7 +21,7 @@ func TestEncrypter_LoadSave(t *testing.T) {
 	ci.Parallel(t)
 
 	tmpDir := t.TempDir()
-	encrypter, err := NewEncrypter(nil, tmpDir)
+	encrypter, err := NewEncrypter(&Server{shutdownCtx: context.Background()}, tmpDir)
 	require.NoError(t, err)
 
 	algos := []structs.EncryptionAlgorithm{

nomad/structs/extensions.go

@@ -1,7 +1,6 @@
 package structs
 
 import (
-	"encoding/base64"
 	"reflect"
 )
 
@@ -13,8 +12,6 @@ var (
 		reflect.TypeOf(&Node{}):      nodeExt,
 		reflect.TypeOf(CSIVolume{}):  csiVolumeExt,
 		reflect.TypeOf(&CSIVolume{}): csiVolumeExt,
-		reflect.TypeOf(&RootKey{}):   rootKeyExt,
-		reflect.TypeOf(RootKey{}):    rootKeyExt,
 	}
 )
 
@@ -79,21 +76,3 @@ func csiVolumeExt(v interface{}) interface{} {
 
 	return apiVol
 }
-
-// rootKeyExt safely serializes a RootKey by base64 encoding the key
-// material and extracting the metadata stub. We only store the root
-// key in the keystore and never in raft or return it via the API, so
-// by having this extension as the default we make it slightly harder
-// to misuse.
-func rootKeyExt(v interface{}) interface{} {
-	key := v.(*RootKey)
-	encodedKey := base64.StdEncoding.EncodeToString(key.Key)
-
-	return &struct {
-		Meta *RootKeyMetaStub
-		Key  string
-	}{
-		Meta: key.Meta.Stub(),
-		Key:  encodedKey,
-	}
-}

nomad/structs/keyring.go

@@ -4,11 +4,8 @@ import (
 	"fmt"
 	"time"
 
-	// note: this is aliased so that it's more noticeable if someone
-	// accidentally swaps it out for math/rand via running goimports
-	cryptorand "crypto/rand"
-
 	"github.com/hashicorp/nomad/helper"
+	"github.com/hashicorp/nomad/helper/crypto"
 	"github.com/hashicorp/nomad/helper/uuid"
 )
 
@@ -29,14 +26,9 @@ func NewRootKey(algorithm EncryptionAlgorithm) (*RootKey, error) {
 
 	switch algorithm {
 	case EncryptionAlgorithmAES256GCM:
-		const keyBytes = 32
-		key := make([]byte, keyBytes)
-		n, err := cryptorand.Read(key)
+		key, err := crypto.Bytes(32)
 		if err != nil {
-			return nil, err
-		}
-		if n < keyBytes {
-			return nil, fmt.Errorf("failed to generate key: entropy exhausted")
+			return nil, fmt.Errorf("failed to generate key: %v", err)
 		}
 		rootKey.Key = key
 	}
@@ -160,6 +152,15 @@ func (rkm *RootKeyMeta) Validate() error {
 	return nil
 }
 
+// KeyEncryptionKeyWrapper is the struct that gets serialized for the on-disk
+// KMS wrapper. This struct includes the server-specific key-wrapping key and
+// should never be sent over RPC.
+type KeyEncryptionKeyWrapper struct {
+	Meta                       *RootKeyMeta
+	EncryptedDataEncryptionKey []byte `json:"DEK"`
+	KeyEncryptionKey           []byte `json:"KEK"`
+}
+
 // EncryptionAlgorithm chooses which algorithm is used for
 // encrypting / decrypting entries with this key
 type EncryptionAlgorithm string
@@ -168,6 +169,7 @@ const (
 	EncryptionAlgorithmAES256GCM EncryptionAlgorithm = "aes256-gcm"
 )
 
+// KeyringRotateRootKeyRequest is the argument to the Keyring.Rotate RPC
 type KeyringRotateRootKeyRequest struct {
 	Algorithm EncryptionAlgorithm
 	Full      bool
@@ -180,11 +182,12 @@ type KeyringRotateRootKeyResponse struct {
 	WriteMeta
 }
 
+// KeyringListRootKeyMetaRequest is the argument to the Keyring.List RPC
 type KeyringListRootKeyMetaRequest struct {
-	// TODO: do we need any fields here?
 	QueryOptions
 }
 
+// KeyringListRootKeyMetaRequest is the response value of the List RPC
 type KeyringListRootKeyMetaResponse struct {
 	Keys []*RootKeyMeta
 	QueryMeta

website/content/docs/operations/key-management.mdx

@@ -10,7 +10,8 @@ Nomad servers maintain an encryption keyring used to encrypt [Variables][] and
 sign task [workload identities][]. The servers store key metadata in raft, but
 the encryption key material is stored in a separate file in the `keystore`
 subdirectory of the Nomad [data directory][]. These files have the extension
-`.nks.json`.
+`.nks.json`. The key material in each file is wrapped in a unique key encryption
+key (KEK) that is not shared between servers.
 
 Under normal operations the keyring is entirely managed by Nomad, but this
 section provides administrators additional context around key replication and
@@ -49,8 +50,10 @@ snapshots you might provide without exposing any of your keys or variables.
 
 However, this means that to restore a cluster from snapshot you need to also
 provide the keystore directory with the `.nks.json` key files on at least one
-server. Operators should include these files as part of your organization's
-backup and recovery strategy for the cluster.
+server. The `.nks.json` key files are unique per server, but only one server's
+key files are needed to recover the cluster. Operators should include these
+files as part of your organization's backup and recovery strategy for the
+cluster.
 
 [Variables]: /docs/concepts/variables
 [workload identities]: /docs/concepts/workload-identity