core: add jwks rpc and http api (#18035)

Add JWKS endpoint to HTTP API for exposing the root public signing keys used for signing workload identity JWTs.

Part 1 of N components as part of making workload identities consumable by third party services such as Consul and Vault. Identity attenuation (audience) and expiration (+renewal) are necessary to securely use workload identities with 3rd parties, so this merge does not yet document this endpoint.

---------

Co-authored-by: Tim Gross <tgross@hashicorp.com>

.semgrep/rpc_endpoint.yml

@@ -108,6 +108,7 @@ rules:
             - pattern-not: '"Status.Leader"'
             - pattern-not: '"Status.Peers"'
             - pattern-not: '"Status.Version"'
+            - pattern-not: '"Keyring.ListPublic"'
     message: "RPC method $METHOD appears to be unauthenticated"
     languages:
       - "go"

command/agent/http.go

@@ -501,6 +501,9 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) {
 	s.mux.Handle("/v1/vars", wrapCORS(s.wrap(s.VariablesListRequest)))
 	s.mux.Handle("/v1/var/", wrapCORSWithAllowedMethods(s.wrap(s.VariableSpecificRequest), "HEAD", "GET", "PUT", "DELETE"))
 
+	// JWKS Handler
+	s.mux.HandleFunc("/.well-known/jwks.json", s.wrap(s.JWKSRequest))
+
 	agentConfig := s.agent.GetConfig()
 	uiConfigEnabled := agentConfig.UI != nil && agentConfig.UI.Enabled
 

command/agent/keyring_endpoint.go

@@ -4,12 +4,79 @@
 package agent
 
 import (
+	"crypto/ed25519"
+	"fmt"
 	"net/http"
 	"strings"
+	"time"
 
+	"github.com/go-jose/go-jose/v3"
+	"github.com/hashicorp/nomad/helper"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
+// jwksMinMaxAge is the minimum amount of time the JWKS endpoint will instruct
+// consumers to cache a response for.
+const jwksMinMaxAge = 15 * time.Minute
+
+// JWKSRequest is used to handle JWKS requests. JWKS stands for JSON Web Key
+// Sets and returns the public keys used for signing workload identities. Third
+// parties may use this endpoint to validate workload identities. Consumers
+// should cache this endpoint, preferably until an unknown kid is encountered.
+func (s *HTTPServer) JWKSRequest(resp http.ResponseWriter, req *http.Request) (any, error) {
+	if req.Method != http.MethodGet {
+		return nil, CodedError(405, ErrInvalidMethod)
+	}
+
+	args := structs.GenericRequest{}
+	if s.parse(resp, req, &args.Region, &args.QueryOptions) {
+		return nil, nil
+	}
+
+	var rpcReply structs.KeyringListPublicResponse
+	if err := s.agent.RPC("Keyring.ListPublic", &args, &rpcReply); err != nil {
+		return nil, err
+	}
+	setMeta(resp, &rpcReply.QueryMeta)
+
+	// Key set will change after max(CreateTime) + RotationThreshold.
+	var newestKey int64
+	jwks := make([]jose.JSONWebKey, 0, len(rpcReply.PublicKeys))
+	for _, pubKey := range rpcReply.PublicKeys {
+		if pubKey.CreateTime > newestKey {
+			newestKey = pubKey.CreateTime
+		}
+
+		jwk := jose.JSONWebKey{
+			KeyID:     pubKey.KeyID,
+			Algorithm: pubKey.Algorithm,
+			Use:       pubKey.Use,
+		}
+		switch alg := pubKey.Algorithm; alg {
+		case structs.PubKeyAlgEdDSA:
+			// Convert public key bytes to an ed25519 public key
+			jwk.Key = ed25519.PublicKey(pubKey.PublicKey)
+		default:
+			s.logger.Warn("unknown public key algorithm. server is likely newer than client", "alg", alg)
+		}
+
+		jwks = append(jwks, jwk)
+	}
+
+	// Have nonzero create times and threshold so set a reasonable cache time.
+	if newestKey > 0 && rpcReply.RotationThreshold > 0 {
+		exp := time.Unix(0, newestKey).Add(rpcReply.RotationThreshold)
+		maxAge := helper.ExpiryToRenewTime(exp, time.Now, jwksMinMaxAge)
+		resp.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d", int(maxAge.Seconds())))
+	}
+
+	out := &jose.JSONWebKeySet{
+		Keys: jwks,
+	}
+
+	return out, nil
+}
+
 // KeyringRequest is used route operator/raft API requests to the implementing
 // functions.
 func (s *HTTPServer) KeyringRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) {

go.mod

@@ -189,6 +189,7 @@ require (
 	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
 	github.com/fatih/color v1.15.0
 	github.com/felixge/httpsnoop v1.0.3 // indirect
+	github.com/go-jose/go-jose/v3 v3.0.0
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect

go.sum

@@ -599,6 +599,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo=
+github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -1464,6 +1466,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=

helper/retry.go

@@ -0,0 +1,27 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package helper
+
+import (
+	"time"
+)
+
+// ExpiryToRenewTime calculates how long until clients should try to renew
+// credentials based on their expiration time and now.
+//
+// Renewals will begin halfway between now and the expiry plus some jitter.
+//
+// If the expiration is in the past or less than the min wait, then the min
+// wait time will be used with jitter.
+func ExpiryToRenewTime(exp time.Time, now func() time.Time, minWait time.Duration) time.Duration {
+	left := exp.Sub(now())
+
+	renewAt := left / 2
+
+	if renewAt < minWait {
+		return minWait + RandomStagger(minWait/10)
+	}
+
+	return renewAt + RandomStagger(renewAt/10)
+}

helper/retry_test.go

@@ -0,0 +1,76 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package helper
+
+import (
+	"testing"
+	"time"
+
+	"github.com/shoenig/test/must"
+)
+
+// TestExpiryToRenewTime_0Min asserts that ExpiryToRenewTime with a 0 min wait
+// will cause an immediate renewal
+func TestExpiryToRenewTime_0Min(t *testing.T) {
+	exp := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC)
+	now := func() time.Time {
+		return time.Date(2023, 1, 1, 0, 0, 1, 0, time.UTC)
+	}
+
+	renew := ExpiryToRenewTime(exp, now, 0)
+
+	must.Zero(t, renew)
+}
+
+// TestExpiryToRenewTime_14Days asserts that ExpiryToRenewTime begins trying to
+// renew at or after 7 days of a 14 day expiration window.
+func TestExpiryToRenewTime_30Days(t *testing.T) {
+	exp := time.Date(2023, 1, 15, 0, 0, 0, 0, time.UTC)
+	now := func() time.Time {
+		return time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC)
+	}
+	min := 20 * time.Minute
+
+	renew := ExpiryToRenewTime(exp, now, min)
+
+	// Renew should be much greater than min wait
+	must.Greater(t, min, renew)
+
+	// Renew should be >= 7 days
+	must.GreaterEq(t, 7*24*time.Hour, renew)
+}
+
+// TestExpiryToRenewTime_UnderMin asserts that ExpiryToRenewTime uses the min
+// wait + jitter if it is greater than the time until expiry.
+func TestExpiryToRenewTime_UnderMin(t *testing.T) {
+	exp := time.Date(2023, 1, 1, 0, 0, 10, 0, time.UTC)
+	now := func() time.Time {
+		return time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC)
+	}
+	min := 20 * time.Second
+
+	renew := ExpiryToRenewTime(exp, now, min)
+
+	// Renew should be >= min wait (jitter can be 0)
+	must.GreaterEq(t, min, renew)
+
+	// When we fallback to the min wait it means we miss the expiration, but this
+	// is necessary to prevent stampedes after outages and partitions.
+	must.GreaterEq(t, exp.Sub(now()), renew)
+}
+
+// TestExpiryToRenewTime_Expired asserts that ExpiryToRenewTime defaults to
+// minWait (+jitter) if the renew time has already elapsed.
+func TestExpiryToRenewTime_Expired(t *testing.T) {
+	exp := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC)
+	now := func() time.Time {
+		return time.Date(2023, 2, 1, 0, 0, 0, 0, time.UTC)
+	}
+	min := time.Hour
+
+	renew := ExpiryToRenewTime(exp, now, min)
+
+	must.Greater(t, min, renew)
+	must.Less(t, min*2, renew)
+}

nomad/encrypter.go

@@ -392,6 +392,26 @@ func (e *Encrypter) loadKeyFromStore(path string) (*structs.RootKey, error) {
 	}, nil
 }
 
+// GetPublicKey returns the public signing key for the requested key id or an
+// error if the key could not be found.
+func (e *Encrypter) GetPublicKey(keyID string) (*structs.KeyringPublicKey, error) {
+	e.lock.Lock()
+	defer e.lock.Unlock()
+
+	ks, err := e.keysetByIDLocked(keyID)
+	if err != nil {
+		return nil, err
+	}
+
+	return &structs.KeyringPublicKey{
+		KeyID:      ks.rootKey.Meta.KeyID,
+		PublicKey:  ks.privateKey.Public().(ed25519.PublicKey),
+		Algorithm:  structs.PubKeyAlgEdDSA,
+		Use:        structs.PubKeyUseSig,
+		CreateTime: ks.rootKey.Meta.CreateTime,
+	}, nil
+}
+
 // newKMSWrapper returns a go-kms-wrapping interface the caller can use to
 // encrypt the RootKey with a key encryption key (KEK). This is a bit of
 // security theatre for local on-disk key material, but gives us a shim for

nomad/keyring_endpoint.go

@@ -360,3 +360,66 @@ func (k *Keyring) Delete(args *structs.KeyringDeleteRootKeyRequest, reply *struc
 	reply.Index = index
 	return nil
 }
+
+// ListPublic signing keys used for workload identities. This RPC is used to
+// back a JWKS endpoint.
+//
+// Unauthenticated because public keys are not sensitive.
+func (k *Keyring) ListPublic(args *structs.GenericRequest, reply *structs.KeyringListPublicResponse) error {
+
+	// JWKS is a public endpoint: intentionally ignore auth errors and only
+	// authenticate to measure rate metrics.
+	k.srv.Authenticate(k.ctx, args)
+	if done, err := k.srv.forward("Keyring.ListPublic", args, args, reply); done {
+		return err
+	}
+	k.srv.MeasureRPCRate("keyring", structs.RateMetricList, args)
+
+	defer metrics.MeasureSince([]string{"nomad", "keyring", "list_public"}, time.Now())
+
+	// Expose root_key_rotation_threshold so consumers can determine reasonable
+	// cache settings.
+	reply.RotationThreshold = k.srv.config.RootKeyRotationThreshold
+
+	// Setup the blocking query
+	opts := blockingOptions{
+		queryOpts: &args.QueryOptions,
+		queryMeta: &reply.QueryMeta,
+		run: func(ws memdb.WatchSet, s *state.StateStore) error {
+
+			// retrieve all the key metadata
+			snap, err := k.srv.fsm.State().Snapshot()
+			if err != nil {
+				return err
+			}
+			iter, err := snap.RootKeyMetas(ws)
+			if err != nil {
+				return err
+			}
+
+			pubKeys := []*structs.KeyringPublicKey{}
+			for {
+				raw := iter.Next()
+				if raw == nil {
+					break
+				}
+
+				keyMeta := raw.(*structs.RootKeyMeta)
+				if keyMeta.State == structs.RootKeyStateDeprecated {
+					// Only include valid keys
+					continue
+				}
+
+				pubKey, err := k.encrypter.GetPublicKey(keyMeta.KeyID)
+				if err != nil {
+					return err
+				}
+
+				pubKeys = append(pubKeys, pubKey)
+			}
+			reply.PublicKeys = pubKeys
+			return k.srv.replySetIndex(state.TableRootKeyMeta, &reply.QueryMeta)
+		},
+	}
+	return k.srv.blockingRPC(&opts)
+}

nomad/keyring_endpoint_test.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc"
+	"github.com/shoenig/test/must"
 	"github.com/stretchr/testify/require"
 
 	"github.com/hashicorp/nomad/ci"
@@ -295,3 +296,59 @@ func TestKeyringEndpoint_Rotate(t *testing.T) {
 	gotKey := getResp.Key
 	require.Len(t, gotKey.Key, 32)
 }
+
+// TestKeyringEndpoint_ListPublic asserts the Keyring.ListPublic RPC returns
+// all keys which may be in use for active crytpographic material (variables,
+// valid JWTs).
+func TestKeyringEndpoint_ListPublic(t *testing.T) {
+
+	ci.Parallel(t)
+	srv, rootToken, shutdown := TestACLServer(t, func(c *Config) {
+		c.NumSchedulers = 0 // Prevent automatic dequeue
+	})
+	defer shutdown()
+	testutil.WaitForLeader(t, srv.RPC)
+	codec := rpcClient(t, srv)
+
+	// Assert 1 key exists and normal fields are set
+	req := structs.GenericRequest{
+		QueryOptions: structs.QueryOptions{
+			Region:    "global",
+			AuthToken: "ignored!",
+		},
+	}
+	var resp structs.KeyringListPublicResponse
+	must.NoError(t, msgpackrpc.CallWithCodec(codec, "Keyring.ListPublic", &req, &resp))
+	must.Eq(t, srv.config.RootKeyRotationThreshold, resp.RotationThreshold)
+	must.Len(t, 1, resp.PublicKeys)
+	must.NonZero(t, resp.Index)
+
+	// Rotate keys and assert there are now 2 keys
+	rotateReq := &structs.KeyringRotateRootKeyRequest{
+		WriteRequest: structs.WriteRequest{
+			Region:    "global",
+			AuthToken: rootToken.SecretID,
+		},
+	}
+	var rotateResp structs.KeyringRotateRootKeyResponse
+	must.NoError(t, msgpackrpc.CallWithCodec(codec, "Keyring.Rotate", rotateReq, &rotateResp))
+	must.NotEq(t, resp.Index, rotateResp.Index)
+
+	// Verify we have a new key and the old one is inactive
+	var resp2 structs.KeyringListPublicResponse
+	must.NoError(t, msgpackrpc.CallWithCodec(codec, "Keyring.ListPublic", &req, &resp2))
+	must.Eq(t, srv.config.RootKeyRotationThreshold, resp2.RotationThreshold)
+	must.Len(t, 2, resp2.PublicKeys)
+	must.NonZero(t, resp2.Index)
+
+	found := false
+	for _, pk := range resp2.PublicKeys {
+		if pk.KeyID == resp.PublicKeys[0].KeyID {
+			must.False(t, found, must.Sprint("found the original public key twice"))
+			found = true
+			must.Eq(t, resp.PublicKeys[0], pk)
+			break
+		}
+	}
+	must.True(t, found, must.Sprint("original public key missing after rotation"))
+}

nomad/structs/keyring.go

@@ -12,6 +12,16 @@ import (
 	"github.com/hashicorp/nomad/helper/uuid"
 )
 
+const (
+	// PubKeyAlgEdDSA is the JWA (JSON Web Algorithm) for ed25519 public keys
+	// used for signatures.
+	PubKeyAlgEdDSA = "EdDSA"
+
+	// PubKeyUseSig is the JWK (JSON Web Key) "use" parameter value for
+	// signatures.
+	PubKeyUseSig = "sig"
+)
+
 // RootKey is used to encrypt and decrypt variables. It is never stored in raft.
 type RootKey struct {
 	Meta *RootKeyMeta
@@ -243,3 +253,38 @@ type KeyringDeleteRootKeyRequest struct {
 type KeyringDeleteRootKeyResponse struct {
 	WriteMeta
 }
+
+// KeyringListPublicResponse lists public key components of signing keys. Used
+// to build a JWKS endpoint.
+type KeyringListPublicResponse struct {
+	PublicKeys []*KeyringPublicKey
+
+	// RotationThreshold exposes root_key_rotation_threshold so that HTTP
+	// endpoints may set a reasonable cache control header informing consumers
+	// when to expect a new key.
+	RotationThreshold time.Duration
+
+	QueryMeta
+}
+
+// KeyringPublicKey is the public key component of a signing key. Used to build
+// a JWKS endpoint.
+type KeyringPublicKey struct {
+	KeyID     string
+	PublicKey []byte
+
+	// Algorithm should be the JWT "alg" parameter. So "EdDSA" for Ed25519 public
+	// keys used to validate signatures.
+	Algorithm string
+
+	// Use should be the JWK "use" parameter as defined in
+	// https://datatracker.ietf.org/doc/html/rfc7517#section-4.2.
+	//
+	// "sig" and "enc" being the two standard values with "sig" being the use for
+	// workload identity JWT signing.
+	Use string
+
+	// CreateTime + root_key_rotation_threshold = when consumers should look for
+	// a new key. Therefore this field can be used for cache control.
+	CreateTime int64
+}