From a29f9b6fc0aa937ed24f33e6a1ef10a7ed2b6cd4 Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Fri, 19 Jul 2024 13:49:48 -0400 Subject: [PATCH] keyring: E2E testing for KMS/rotation (#23601) In #23580 we're implementing support for encrypting Nomad's key material with external KMS providers or Vault Transit. This changeset breaks out the E2E infrastructure and testing from that PR to keep the review manageable. Ref: https://hashicorp.atlassian.net/browse/NET-10334 Ref: https://github.com/hashicorp/nomad/issues/14852 Ref: https://github.com/hashicorp/nomad/pull/23580 --- e2e/keyring/doc.go | 7 ++ e2e/keyring/keyring_test.go | 80 ++++++++++++++++++++++ e2e/terraform/etc/nomad.d/server-linux.hcl | 10 +++ e2e/terraform/main.tf | 4 ++ e2e/terraform/nomad.tf | 3 + e2e/terraform/provision-nomad/main.tf | 5 +- e2e/terraform/provision-nomad/variables.tf | 11 +++ e2e/terraform/variables.tf | 6 ++ 8 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 e2e/keyring/doc.go create mode 100644 e2e/keyring/keyring_test.go diff --git a/e2e/keyring/doc.go b/e2e/keyring/doc.go new file mode 100644 index 000000000..1f194ed88 --- /dev/null +++ b/e2e/keyring/doc.go @@ -0,0 +1,7 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package keyring + +// This package contains only tests, so this is a placeholder file to +// make sure builds don't fail with "no non-test Go files in" errors diff --git a/e2e/keyring/keyring_test.go b/e2e/keyring/keyring_test.go new file mode 100644 index 000000000..91d3ec1c2 --- /dev/null +++ b/e2e/keyring/keyring_test.go @@ -0,0 +1,80 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package keyring + +import ( + "encoding/json" + "testing" + + "github.com/go-jose/go-jose/v3" + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/e2e/e2eutil" + "github.com/shoenig/test/must" +) + +func TestKeyringRotation(t *testing.T) { + + nc := e2eutil.NomadClient(t) + + currentKeys, activeKeyID := getKeyMeta(t, nc) + must.NotEq(t, "", activeKeyID, must.Sprint("expected an active key")) + + keyset := getJWKS(t) + + must.Len(t, len(currentKeys), keyset.Keys) + for _, key := range keyset.Keys { + must.MapContainsKey(t, currentKeys, key.KeyID) + } + + out, err := e2eutil.Commandf("nomad operator root keyring rotate -verbose -prepublish 1h") + must.NoError(t, err) + cols, err := e2eutil.ParseColumns(out) + must.NoError(t, err) + must.Greater(t, 0, len(cols)) + newKeyID := cols[0]["Key"] + must.Eq(t, "prepublished", cols[0]["State"], must.Sprint("expected new key to be prepublished")) + + newCurrentKeys, newActiveKeyID := getKeyMeta(t, nc) + must.NotEq(t, "", newActiveKeyID, must.Sprint("expected an active key")) + must.Eq(t, activeKeyID, newActiveKeyID, must.Sprint("active key should not have rotated yet")) + must.Greater(t, len(currentKeys), len(newCurrentKeys), must.Sprint("expected more keys after prepublishing")) + + keyset = getJWKS(t) + must.Len(t, len(newCurrentKeys), keyset.Keys, must.Sprint("number of keys in jwks keyset should match keyring")) + for _, key := range keyset.Keys { + must.MapContainsKey(t, newCurrentKeys, key.KeyID, must.Sprint("jwks keyset contains unexpected key")) + } + must.SliceContainsFunc(t, keyset.Keys, newKeyID, func(a jose.JSONWebKey, b string) bool { + return a.KeyID == b + }, must.Sprint("expected prepublished key to appear in JWKS endpoint")) +} + +func getKeyMeta(t *testing.T, nc *api.Client) (map[string]*api.RootKeyMeta, string) { + t.Helper() + keyMetas, _, err := nc.Keyring().List(nil) + must.NoError(t, err) + + currentKeys := map[string]*api.RootKeyMeta{} + var activeKeyID string + for _, keyMeta := range keyMetas { + currentKeys[keyMeta.KeyID] = keyMeta + if keyMeta.State == api.RootKeyStateActive { + activeKeyID = keyMeta.KeyID + } + } + must.NotEq(t, "", activeKeyID, must.Sprint("expected an active key")) + return currentKeys, activeKeyID +} + +func getJWKS(t *testing.T) *jose.JSONWebKeySet { + t.Helper() + out, err := e2eutil.Commandf("nomad operator api /.well-known/jwks.json") + must.NoError(t, err) + + keyset := &jose.JSONWebKeySet{} + err = json.Unmarshal([]byte(out), keyset) + must.NoError(t, err) + + return keyset +} diff --git a/e2e/terraform/etc/nomad.d/server-linux.hcl b/e2e/terraform/etc/nomad.d/server-linux.hcl index 9e09df10c..9db84855b 100644 --- a/e2e/terraform/etc/nomad.d/server-linux.hcl +++ b/e2e/terraform/etc/nomad.d/server-linux.hcl @@ -5,3 +5,13 @@ server { enabled = true bootstrap_expect = 3 } + +keyring "awskms" { + active = true + region = "${aws_region}" + kms_key_id = "${aws_kms_key_id}" +} + +keyring "aead" { + active = false +} diff --git a/e2e/terraform/main.tf b/e2e/terraform/main.tf index c82005136..f6e84ef5e 100644 --- a/e2e/terraform/main.tf +++ b/e2e/terraform/main.tf @@ -28,3 +28,7 @@ module "keys" { source = "mitchellh/dynamic-keys/aws" version = "v2.0.0" } + +data "aws_kms_alias" "e2e" { + name = "alias/${var.aws_kms_alias}" +} diff --git a/e2e/terraform/nomad.tf b/e2e/terraform/nomad.tf index 2cf0170a9..0c7719b93 100644 --- a/e2e/terraform/nomad.tf +++ b/e2e/terraform/nomad.tf @@ -18,6 +18,9 @@ module "nomad_server" { tls_ca_key = tls_private_key.ca.private_key_pem tls_ca_cert = tls_self_signed_cert.ca.cert_pem + aws_region = var.region + aws_kms_key_id = data.aws_kms_alias.e2e.target_key_id + connection = { type = "ssh" user = "ubuntu" diff --git a/e2e/terraform/provision-nomad/main.tf b/e2e/terraform/provision-nomad/main.tf index 002a623c1..80f974a6a 100644 --- a/e2e/terraform/provision-nomad/main.tf +++ b/e2e/terraform/provision-nomad/main.tf @@ -26,7 +26,10 @@ resource "local_sensitive_file" "nomad_base_config" { } resource "local_sensitive_file" "nomad_role_config" { - content = templatefile("etc/nomad.d/${var.role}-${var.platform}.hcl", {}) + content = templatefile("etc/nomad.d/${var.role}-${var.platform}.hcl", { + aws_region = var.aws_region + aws_kms_key_id = var.aws_kms_key_id + }) filename = "${local.upload_dir}/nomad.d/${var.role}.hcl" file_permission = "0600" } diff --git a/e2e/terraform/provision-nomad/variables.tf b/e2e/terraform/provision-nomad/variables.tf index 60c9b77d4..472db13f8 100644 --- a/e2e/terraform/provision-nomad/variables.tf +++ b/e2e/terraform/provision-nomad/variables.tf @@ -73,3 +73,14 @@ variable "connection" { }) description = "ssh connection information for remote target" } + +variable "aws_region" { + type = string + default = "us-east-1" +} + +variable "aws_kms_key_id" { + type = string + description = "AWS KMS key ID for encrypting and decrypting the Nomad keyring" + default = "" +} diff --git a/e2e/terraform/variables.tf b/e2e/terraform/variables.tf index 81c612c9b..ca2a64fd6 100644 --- a/e2e/terraform/variables.tf +++ b/e2e/terraform/variables.tf @@ -87,6 +87,12 @@ variable "hcp_vault_namespace" { default = "admin" } +variable "aws_kms_alias" { + description = "The alias for the AWS KMS key ID" + type = string + default = "kms-nomad-keyring" +} + # ---------------------------------------- # If you want to deploy multiple versions you can use these variables to # provide a list of builds to override the values of nomad_sha, nomad_version,