diff --git a/e2e/keyring/doc.go b/e2e/keyring/doc.go new file mode 100644 index 000000000..1f194ed88 --- /dev/null +++ b/e2e/keyring/doc.go @@ -0,0 +1,7 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package keyring + +// This package contains only tests, so this is a placeholder file to +// make sure builds don't fail with "no non-test Go files in" errors diff --git a/e2e/keyring/keyring_test.go b/e2e/keyring/keyring_test.go new file mode 100644 index 000000000..91d3ec1c2 --- /dev/null +++ b/e2e/keyring/keyring_test.go @@ -0,0 +1,80 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package keyring + +import ( + "encoding/json" + "testing" + + "github.com/go-jose/go-jose/v3" + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/e2e/e2eutil" + "github.com/shoenig/test/must" +) + +func TestKeyringRotation(t *testing.T) { + + nc := e2eutil.NomadClient(t) + + currentKeys, activeKeyID := getKeyMeta(t, nc) + must.NotEq(t, "", activeKeyID, must.Sprint("expected an active key")) + + keyset := getJWKS(t) + + must.Len(t, len(currentKeys), keyset.Keys) + for _, key := range keyset.Keys { + must.MapContainsKey(t, currentKeys, key.KeyID) + } + + out, err := e2eutil.Commandf("nomad operator root keyring rotate -verbose -prepublish 1h") + must.NoError(t, err) + cols, err := e2eutil.ParseColumns(out) + must.NoError(t, err) + must.Greater(t, 0, len(cols)) + newKeyID := cols[0]["Key"] + must.Eq(t, "prepublished", cols[0]["State"], must.Sprint("expected new key to be prepublished")) + + newCurrentKeys, newActiveKeyID := getKeyMeta(t, nc) + must.NotEq(t, "", newActiveKeyID, must.Sprint("expected an active key")) + must.Eq(t, activeKeyID, newActiveKeyID, must.Sprint("active key should not have rotated yet")) + must.Greater(t, len(currentKeys), len(newCurrentKeys), must.Sprint("expected more keys after prepublishing")) + + keyset = getJWKS(t) + must.Len(t, len(newCurrentKeys), keyset.Keys, must.Sprint("number of keys in jwks keyset should match keyring")) + for _, key := range keyset.Keys { + must.MapContainsKey(t, newCurrentKeys, key.KeyID, must.Sprint("jwks keyset contains unexpected key")) + } + must.SliceContainsFunc(t, keyset.Keys, newKeyID, func(a jose.JSONWebKey, b string) bool { + return a.KeyID == b + }, must.Sprint("expected prepublished key to appear in JWKS endpoint")) +} + +func getKeyMeta(t *testing.T, nc *api.Client) (map[string]*api.RootKeyMeta, string) { + t.Helper() + keyMetas, _, err := nc.Keyring().List(nil) + must.NoError(t, err) + + currentKeys := map[string]*api.RootKeyMeta{} + var activeKeyID string + for _, keyMeta := range keyMetas { + currentKeys[keyMeta.KeyID] = keyMeta + if keyMeta.State == api.RootKeyStateActive { + activeKeyID = keyMeta.KeyID + } + } + must.NotEq(t, "", activeKeyID, must.Sprint("expected an active key")) + return currentKeys, activeKeyID +} + +func getJWKS(t *testing.T) *jose.JSONWebKeySet { + t.Helper() + out, err := e2eutil.Commandf("nomad operator api /.well-known/jwks.json") + must.NoError(t, err) + + keyset := &jose.JSONWebKeySet{} + err = json.Unmarshal([]byte(out), keyset) + must.NoError(t, err) + + return keyset +} diff --git a/e2e/terraform/etc/nomad.d/server-linux.hcl b/e2e/terraform/etc/nomad.d/server-linux.hcl index 9e09df10c..9db84855b 100644 --- a/e2e/terraform/etc/nomad.d/server-linux.hcl +++ b/e2e/terraform/etc/nomad.d/server-linux.hcl @@ -5,3 +5,13 @@ server { enabled = true bootstrap_expect = 3 } + +keyring "awskms" { + active = true + region = "${aws_region}" + kms_key_id = "${aws_kms_key_id}" +} + +keyring "aead" { + active = false +} diff --git a/e2e/terraform/main.tf b/e2e/terraform/main.tf index c82005136..f6e84ef5e 100644 --- a/e2e/terraform/main.tf +++ b/e2e/terraform/main.tf @@ -28,3 +28,7 @@ module "keys" { source = "mitchellh/dynamic-keys/aws" version = "v2.0.0" } + +data "aws_kms_alias" "e2e" { + name = "alias/${var.aws_kms_alias}" +} diff --git a/e2e/terraform/nomad.tf b/e2e/terraform/nomad.tf index 2cf0170a9..0c7719b93 100644 --- a/e2e/terraform/nomad.tf +++ b/e2e/terraform/nomad.tf @@ -18,6 +18,9 @@ module "nomad_server" { tls_ca_key = tls_private_key.ca.private_key_pem tls_ca_cert = tls_self_signed_cert.ca.cert_pem + aws_region = var.region + aws_kms_key_id = data.aws_kms_alias.e2e.target_key_id + connection = { type = "ssh" user = "ubuntu" diff --git a/e2e/terraform/provision-nomad/main.tf b/e2e/terraform/provision-nomad/main.tf index 002a623c1..80f974a6a 100644 --- a/e2e/terraform/provision-nomad/main.tf +++ b/e2e/terraform/provision-nomad/main.tf @@ -26,7 +26,10 @@ resource "local_sensitive_file" "nomad_base_config" { } resource "local_sensitive_file" "nomad_role_config" { - content = templatefile("etc/nomad.d/${var.role}-${var.platform}.hcl", {}) + content = templatefile("etc/nomad.d/${var.role}-${var.platform}.hcl", { + aws_region = var.aws_region + aws_kms_key_id = var.aws_kms_key_id + }) filename = "${local.upload_dir}/nomad.d/${var.role}.hcl" file_permission = "0600" } diff --git a/e2e/terraform/provision-nomad/variables.tf b/e2e/terraform/provision-nomad/variables.tf index 60c9b77d4..472db13f8 100644 --- a/e2e/terraform/provision-nomad/variables.tf +++ b/e2e/terraform/provision-nomad/variables.tf @@ -73,3 +73,14 @@ variable "connection" { }) description = "ssh connection information for remote target" } + +variable "aws_region" { + type = string + default = "us-east-1" +} + +variable "aws_kms_key_id" { + type = string + description = "AWS KMS key ID for encrypting and decrypting the Nomad keyring" + default = "" +} diff --git a/e2e/terraform/variables.tf b/e2e/terraform/variables.tf index 81c612c9b..ca2a64fd6 100644 --- a/e2e/terraform/variables.tf +++ b/e2e/terraform/variables.tf @@ -87,6 +87,12 @@ variable "hcp_vault_namespace" { default = "admin" } +variable "aws_kms_alias" { + description = "The alias for the AWS KMS key ID" + type = string + default = "kms-nomad-keyring" +} + # ---------------------------------------- # If you want to deploy multiple versions you can use these variables to # provide a list of builds to override the values of nomad_sha, nomad_version,