intro: Add node introduction flow for Nomad client registration. (#26405)

This change implements the client -> server workflow for Nomad
node introduction. A Nomad node can optionally be started with an
introduction token, which is a signed JWT containing claims for
the node registration. The server handles this according to the
enforcement configuration.

The introduction token can be provided by env var, cli flag, or
by placing it within a default filesystem location. The latter
option does not override the CLI or env var.

The region claims has been removed from the initial claims set of
the intro identity. This boundary is guarded by mTLS and aligns
with the node identity.

client/client.go

@@ -2108,14 +2108,15 @@ func (c *Client) retryRegisterNode() {
 		}
 
 		retryIntv := registerRetryIntv
-		if err == noServersErr || structs.IsErrNoRegionPath(err) {
+		if errors.Is(err, noServersErr) || structs.IsErrNoRegionPath(err) {
 			c.logger.Debug("registration waiting on servers")
 			c.triggerDiscovery()
 			retryIntv = noServerRetryIntv
-		} else if structs.IsErrPermissionDenied(err) {
-			// any previous cluster state we have here is invalid (ex. client
+		} else if structs.IsErrPermissionDenied(err) && c.config.IntroToken == "" {
+			// Any previous cluster state we have here is invalid (ex. client
 			// has been assigned to a new region), so clear the token and local
-			// state for next pass.
+			// state for next pass. This is unless the operator has provided an
+			// intro token, in which case we will retry with that.
 			authToken = ""
 			c.stateDB.PutNodeRegistration(&cstructs.NodeRegistration{HasRegistered: false})
 			c.logger.Error("error registering", "error", err)
@@ -2131,10 +2132,14 @@ func (c *Client) retryRegisterNode() {
 	}
 }
 
-// getRegistrationToken gets the node secret to use for the Node.Register call.
-// Registration is trust-on-first-use so we can't send the auth token with the
-// initial request, but we want to add the auth token after that so that we can
-// capture metrics.
+// getRegistrationToken gets the appropriate authentication token to use for the
+// Node.Register call. When a client first register, it may optionally use an
+// intro token to bootstrap the registration. If this is not set, the existing
+// behavior of no auth token is used.
+//
+// If the client has already registered, it will use either the nodes secret ID
+// or its identity. This detail depends on whether the client is talking to
+// upgraded servers that support the new identity system or not.
 func (c *Client) getRegistrationToken() string {
 
 	select {
@@ -2149,12 +2154,34 @@ func (c *Client) getRegistrationToken() string {
 		if err != nil {
 			c.logger.Error("could not determine previous node registration", "error", err)
 		}
-		if registration != nil && registration.HasRegistered {
-			c.registeredOnce.Do(func() { close(c.registeredCh) })
-			return c.nodeAuthToken()
+
+		// If the state call indicates that we have not registered yet,
+		// fall-through to the end logic of this function to return any intro
+		// token.
+		if registration == nil || !registration.HasRegistered {
+			break
 		}
+
+		// Attempt to pull and use the node's identity from the state store. The
+		// state store restore happens asynchronously to this function, so we
+		// can't rely on it being populated in the client object at this time.
+		clientIdentity, err := c.stateDB.GetNodeIdentity()
+		if err != nil {
+			c.logger.Error("could not determine node identity", "error", err)
+		}
+		if clientIdentity != "" {
+			c.setNodeIdentityToken(clientIdentity)
+		}
+
+		c.registeredOnce.Do(func() { close(c.registeredCh) })
+		return c.nodeAuthToken()
 	}
-	return ""
+
+	// Reaching this point means we are registering for the first time. If the
+	// client configuration has a bootstrap token, we can use that to perform
+	// the initial registration. If this was not supplied, the parameter will be
+	// an empty string, which is fine and the backwards compatible behavior.
+	return c.GetConfig().IntroToken
 }
 
 // registerNode is used to register the node or update the registration

client/client_test.go

@@ -27,6 +27,7 @@ import (
 	regMock "github.com/hashicorp/nomad/client/serviceregistration/mock"
 	"github.com/hashicorp/nomad/client/state"
 	cstate "github.com/hashicorp/nomad/client/state"
+	cstructs "github.com/hashicorp/nomad/client/structs"
 	ctestutil "github.com/hashicorp/nomad/client/testutil"
 	"github.com/hashicorp/nomad/command/agent/consul"
 	"github.com/hashicorp/nomad/helper/pluginutils/catalog"
@@ -1453,6 +1454,79 @@ func TestClient_ServerList(t *testing.T) {
 	}
 }
 
+func TestClient_getRegistrationToken(t *testing.T) {
+	ci.Parallel(t)
+
+	t.Run("no intro initial register", func(t *testing.T) {
+		testClient, testClientCleanup := TestClient(t, func(c *config.Config) {})
+		t.Cleanup(func() { _ = testClientCleanup() })
+		must.Eq(t, "", testClient.getRegistrationToken())
+	})
+
+	t.Run("intro initial register", func(t *testing.T) {
+		testClient, testClientCleanup := TestClient(t, func(c *config.Config) {
+			c.IntroToken = "my-intro-token"
+		})
+		t.Cleanup(func() { _ = testClientCleanup() })
+		must.Eq(t, "my-intro-token", testClient.getRegistrationToken())
+	})
+
+	t.Run("secret id registered", func(t *testing.T) {
+		testClient, testClientCleanup := TestClient(t, func(c *config.Config) {})
+		t.Cleanup(func() { _ = testClientCleanup() })
+
+		close(testClient.registeredCh)
+
+		must.Eq(t, testClient.Node().SecretID, testClient.getRegistrationToken())
+	})
+
+	t.Run("node identity registered", func(t *testing.T) {
+		testClient, testClientCleanup := TestClient(t, func(c *config.Config) {})
+		t.Cleanup(func() { _ = testClientCleanup() })
+
+		testClient.identity.Store("mylovelylovelyidentity")
+		close(testClient.registeredCh)
+
+		must.Eq(t, testClient.identity.Load().(string), testClient.getRegistrationToken())
+	})
+
+	t.Run("secret id registered state", func(t *testing.T) {
+		testClient, testClientCleanup := TestClient(t, func(c *config.Config) {
+			c.StateDBFactory = func(logger hclog.Logger, stateDir string) (state.StateDB, error) {
+				return cstate.NewMemDB(logger), nil
+			}
+		})
+		t.Cleanup(func() { _ = testClientCleanup() })
+
+		must.NoError(t, testClient.stateDB.PutNodeRegistration(
+			&cstructs.NodeRegistration{
+				HasRegistered: true,
+			},
+		))
+
+		must.Eq(t, testClient.Node().SecretID, testClient.getRegistrationToken())
+	})
+
+	t.Run("node identity registered state", func(t *testing.T) {
+		testClient, testClientCleanup := TestClient(t, func(c *config.Config) {
+			c.StateDBFactory = func(logger hclog.Logger, stateDir string) (state.StateDB, error) {
+				return cstate.NewMemDB(logger), nil
+			}
+		})
+		t.Cleanup(func() { _ = testClientCleanup() })
+
+		must.NoError(t, testClient.stateDB.PutNodeRegistration(
+			&cstructs.NodeRegistration{
+				HasRegistered: true,
+			},
+		))
+
+		must.NoError(t, testClient.stateDB.PutNodeIdentity("my-identity-token"))
+
+		must.Eq(t, "my-identity-token", testClient.getRegistrationToken())
+	})
+}
+
 func TestClient_handleNodeUpdateResponse(t *testing.T) {
 	ci.Parallel(t)
 

client/config/config.go

@@ -108,6 +108,10 @@ type Config struct {
 	// should be owned  by root with file mode 0o755.
 	AllocMountsDir string
 
+	// IntroToken is the signed JWT token that should be used to introduce this
+	// client to the servers on first registration.
+	IntroToken string
+
 	// Logger provides a logger to the client
 	Logger log.InterceptLogger