keyring: E2E testing for KMS/rotation (#23601)

In #23580 we're implementing support for encrypting Nomad's key material with
external KMS providers or Vault Transit. This changeset breaks out the E2E
infrastructure and testing from that PR to keep the review manageable.

Ref: https://hashicorp.atlassian.net/browse/NET-10334
Ref: https://github.com/hashicorp/nomad/issues/14852
Ref: https://github.com/hashicorp/nomad/pull/23580

e2e/keyring/doc.go

@@ -0,0 +1,7 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package keyring
+
+// This package contains only tests, so this is a placeholder file to
+// make sure builds don't fail with "no non-test Go files in" errors

e2e/keyring/keyring_test.go

@@ -0,0 +1,80 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package keyring
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/go-jose/go-jose/v3"
+	"github.com/hashicorp/nomad/api"
+	"github.com/hashicorp/nomad/e2e/e2eutil"
+	"github.com/shoenig/test/must"
+)
+
+func TestKeyringRotation(t *testing.T) {
+
+	nc := e2eutil.NomadClient(t)
+
+	currentKeys, activeKeyID := getKeyMeta(t, nc)
+	must.NotEq(t, "", activeKeyID, must.Sprint("expected an active key"))
+
+	keyset := getJWKS(t)
+
+	must.Len(t, len(currentKeys), keyset.Keys)
+	for _, key := range keyset.Keys {
+		must.MapContainsKey(t, currentKeys, key.KeyID)
+	}
+
+	out, err := e2eutil.Commandf("nomad operator root keyring rotate -verbose -prepublish 1h")
+	must.NoError(t, err)
+	cols, err := e2eutil.ParseColumns(out)
+	must.NoError(t, err)
+	must.Greater(t, 0, len(cols))
+	newKeyID := cols[0]["Key"]
+	must.Eq(t, "prepublished", cols[0]["State"], must.Sprint("expected new key to be prepublished"))
+
+	newCurrentKeys, newActiveKeyID := getKeyMeta(t, nc)
+	must.NotEq(t, "", newActiveKeyID, must.Sprint("expected an active key"))
+	must.Eq(t, activeKeyID, newActiveKeyID, must.Sprint("active key should not have rotated yet"))
+	must.Greater(t, len(currentKeys), len(newCurrentKeys), must.Sprint("expected more keys after prepublishing"))
+
+	keyset = getJWKS(t)
+	must.Len(t, len(newCurrentKeys), keyset.Keys, must.Sprint("number of keys in jwks keyset should match keyring"))
+	for _, key := range keyset.Keys {
+		must.MapContainsKey(t, newCurrentKeys, key.KeyID, must.Sprint("jwks keyset contains unexpected key"))
+	}
+	must.SliceContainsFunc(t, keyset.Keys, newKeyID, func(a jose.JSONWebKey, b string) bool {
+		return a.KeyID == b
+	}, must.Sprint("expected prepublished key to appear in JWKS endpoint"))
+}
+
+func getKeyMeta(t *testing.T, nc *api.Client) (map[string]*api.RootKeyMeta, string) {
+	t.Helper()
+	keyMetas, _, err := nc.Keyring().List(nil)
+	must.NoError(t, err)
+
+	currentKeys := map[string]*api.RootKeyMeta{}
+	var activeKeyID string
+	for _, keyMeta := range keyMetas {
+		currentKeys[keyMeta.KeyID] = keyMeta
+		if keyMeta.State == api.RootKeyStateActive {
+			activeKeyID = keyMeta.KeyID
+		}
+	}
+	must.NotEq(t, "", activeKeyID, must.Sprint("expected an active key"))
+	return currentKeys, activeKeyID
+}
+
+func getJWKS(t *testing.T) *jose.JSONWebKeySet {
+	t.Helper()
+	out, err := e2eutil.Commandf("nomad operator api /.well-known/jwks.json")
+	must.NoError(t, err)
+
+	keyset := &jose.JSONWebKeySet{}
+	err = json.Unmarshal([]byte(out), keyset)
+	must.NoError(t, err)
+
+	return keyset
+}

e2e/terraform/etc/nomad.d/server-linux.hcl

@@ -5,3 +5,13 @@ server {
   enabled          = true
   bootstrap_expect = 3
 }
+
+keyring "awskms" {
+  active     = true
+  region     = "${aws_region}"
+  kms_key_id = "${aws_kms_key_id}"
+}
+
+keyring "aead" {
+  active = false
+}

e2e/terraform/main.tf

@@ -28,3 +28,7 @@ module "keys" {
   source  = "mitchellh/dynamic-keys/aws"
   version = "v2.0.0"
 }
+
+data "aws_kms_alias" "e2e" {
+  name = "alias/${var.aws_kms_alias}"
+}

e2e/terraform/nomad.tf

@@ -18,6 +18,9 @@ module "nomad_server" {
   tls_ca_key    = tls_private_key.ca.private_key_pem
   tls_ca_cert   = tls_self_signed_cert.ca.cert_pem
 
+  aws_region     = var.region
+  aws_kms_key_id = data.aws_kms_alias.e2e.target_key_id
+
   connection = {
     type        = "ssh"
     user        = "ubuntu"

e2e/terraform/provision-nomad/main.tf

@@ -26,7 +26,10 @@ resource "local_sensitive_file" "nomad_base_config" {
 }
 
 resource "local_sensitive_file" "nomad_role_config" {
-  content         = templatefile("etc/nomad.d/${var.role}-${var.platform}.hcl", {})
+  content = templatefile("etc/nomad.d/${var.role}-${var.platform}.hcl", {
+    aws_region     = var.aws_region
+    aws_kms_key_id = var.aws_kms_key_id
+  })
   filename        = "${local.upload_dir}/nomad.d/${var.role}.hcl"
   file_permission = "0600"
 }

e2e/terraform/provision-nomad/variables.tf

@@ -73,3 +73,14 @@ variable "connection" {
   })
   description = "ssh connection information for remote target"
 }
+
+variable "aws_region" {
+  type    = string
+  default = "us-east-1"
+}
+
+variable "aws_kms_key_id" {
+  type        = string
+  description = "AWS KMS key ID for encrypting and decrypting the Nomad keyring"
+  default     = ""
+}

e2e/terraform/variables.tf

@@ -87,6 +87,12 @@ variable "hcp_vault_namespace" {
   default     = "admin"
 }
 
+variable "aws_kms_alias" {
+  description = "The alias for the AWS KMS key ID"
+  type        = string
+  default     = "kms-nomad-keyring"
+}
+
 # ----------------------------------------
 # If you want to deploy multiple versions you can use these variables to
 # provide a list of builds to override the values of nomad_sha, nomad_version,