From 953a1491808232b0c091a96ebf28600c509ac625 Mon Sep 17 00:00:00 2001 From: James Rasell Date: Wed, 16 Jul 2025 15:56:00 +0200 Subject: [PATCH] client: Allow operators to force a client to renew its identity. (#26277) The Nomad client will have its identity renewed according to the TTL which defaults to 24h. In certain situations such as root keyring rotation, operators may want to force clients to renew their identities before the TTL threshold is met. This change introduces a client HTTP and RPC endpoint which will instruct the node to request a new identity at its next heartbeat. This can be used via the API or a new command. While this is a manual intervention step on top of the any keyring rotation, it dramatically reduces the initial feature complexity as it provides an asynchronous and efficient method of renewal that utilises existing functionality. --- api/node_identity.go | 33 ++++++++ api/node_identity_test.go | 29 +++++++ client/client.go | 34 +++++++- client/node_identity_endpoint.go | 33 ++++++++ client/node_identity_endpoint_test.go | 103 ++++++++++++++++++++++++ client/rpc.go | 17 ++-- command/agent/http.go | 1 + command/agent/node_identity_endpoint.go | 43 ++++++++++ command/commands.go | 10 +++ command/node_identity.go | 34 ++++++++ command/node_identity_renew.go | 88 ++++++++++++++++++++ nomad/client_identity_endpoint.go | 47 +++++++++++ nomad/client_identity_endpoint_test.go | 85 +++++++++++++++++++ nomad/node_endpoint.go | 2 + nomad/server.go | 1 + nomad/structs/node.go | 22 +++++ 16 files changed, 574 insertions(+), 8 deletions(-) create mode 100644 api/node_identity.go create mode 100644 api/node_identity_test.go create mode 100644 client/node_identity_endpoint.go create mode 100644 client/node_identity_endpoint_test.go create mode 100644 command/agent/node_identity_endpoint.go create mode 100644 command/node_identity.go create mode 100644 command/node_identity_renew.go create mode 100644 nomad/client_identity_endpoint.go create mode 100644 nomad/client_identity_endpoint_test.go diff --git a/api/node_identity.go b/api/node_identity.go new file mode 100644 index 000000000..497ebcd23 --- /dev/null +++ b/api/node_identity.go @@ -0,0 +1,33 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package api + +type NodeIdentityRenewRequest struct { + NodeID string +} + +type NodeIdentityRenewResponse struct{} + +type NodeIdentity struct { + client *Client +} + +func (n *Nodes) Identity() *NodeIdentity { + return &NodeIdentity{client: n.client} +} + +// Renew instructs the node to request a new identity from the server at its +// next heartbeat. +// +// The request uses query options to control the forwarding behavior of the +// request only. Parameters such as Filter, WaitTime, and WaitIndex are not used +// and ignored. +func (n *NodeIdentity) Renew(req *NodeIdentityRenewRequest, qo *QueryOptions) (*NodeIdentityRenewResponse, error) { + var out NodeIdentityRenewResponse + _, err := n.client.postQuery("/v1/client/identity/renew", req, &out, qo) + if err != nil { + return nil, err + } + return &out, nil +} diff --git a/api/node_identity_test.go b/api/node_identity_test.go new file mode 100644 index 000000000..56f682f15 --- /dev/null +++ b/api/node_identity_test.go @@ -0,0 +1,29 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package api + +import ( + "testing" + + "github.com/hashicorp/nomad/api/internal/testutil" + "github.com/shoenig/test/must" +) + +func TestNodeIdentity_Renew(t *testing.T) { + testutil.Parallel(t) + + configCallback := func(c *testutil.TestServerConfig) { c.DevMode = true } + testClient, testServer := makeClient(t, nil, configCallback) + defer testServer.Stop() + + nodeID := oneNodeFromNodeList(t, testClient.Nodes()).ID + + req := NodeIdentityRenewRequest{ + NodeID: nodeID, + } + + resp, err := testClient.Nodes().Identity().Renew(&req, nil) + must.NoError(t, err) + must.NotNil(t, resp) +} diff --git a/client/client.go b/client/client.go index 74abd0c16..de9b465ec 100644 --- a/client/client.go +++ b/client/client.go @@ -339,6 +339,11 @@ type Client struct { // the servers. This is used to authenticate the client to the servers when // performing RPC calls. identity atomic.Value + + // identityForceRenewal is used to force the client to renew its identity + // at the next heartbeat. It is set by an operator calling the node identity + // renew RPC method. + identityForceRenewal atomic.Bool } var ( @@ -402,6 +407,7 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulProxie EnterpriseClient: newEnterpriseClient(logger), allocrunnerFactory: cfg.AllocRunnerFactory, identity: atomic.Value{}, + identityForceRenewal: atomic.Bool{}, } // we can't have this set in the default Config because of import cycles @@ -968,6 +974,10 @@ func (c *Client) nodeIdentityToken() string { // processes with a new node identity token. func (c *Client) setNodeIdentityToken(token string) { + // It's a bit of a simple log line, but it is useful to know when the client + // has renewed or set its node identity token. + c.logger.Info("setting node identity token") + // Store the token on the client as the first step, so it's available for // use by all RPCs immediately. c.identity.Store(token) @@ -2204,6 +2214,14 @@ func (c *Client) updateNodeStatus() error { AuthToken: c.nodeAuthToken(), }, } + + // Check if the client has been informed to force a renewal of its identity, + // and set the flag in the request if so. + if c.identityForceRenewal.Load() { + c.logger.Debug("forcing identity renewal") + req.ForceIdentityRenewal = true + } + var resp structs.NodeUpdateResponse if err := c.RPC("Node.UpdateStatus", &req, &resp); err != nil { c.triggerDiscovery() @@ -2226,7 +2244,17 @@ func (c *Client) updateNodeStatus() error { c.heartbeatLock.Unlock() c.logger.Trace("next heartbeat", "period", resp.HeartbeatTTL) - if resp.Index != 0 { + // The Nomad server will return an index of greater than zero when a Raft + // update has occurred, indicating a change in the state of the persisted + // node object. + // + // This can be due to a Nomad server invalidating the node's heartbeat timer + // and marking the node as down. In this case, we want to log a warning for + // the operator to see the client missed a heartbeat. If the server + // responded with a new identity, we assume the client did not miss a + // heartbeat. If we did, this line would appear each time the identity was + // renewed, which could confuse cluster operators. + if resp.Index != 0 && resp.SignedIdentity == nil { c.logger.Debug("state updated", "node_status", req.Status) // We have potentially missed our TTL log how delayed we were @@ -2276,6 +2304,10 @@ func (c *Client) handleNodeUpdateResponse(resp structs.NodeUpdateResponse) error return fmt.Errorf("error saving client identity: %w", err) } c.setNodeIdentityToken(*resp.SignedIdentity) + + // If the operator forced this renewal, reset the flag so that we don't + // keep renewing the identity on every heartbeat. + c.identityForceRenewal.Store(false) } // Convert []*NodeServerInfo to []*servers.Server diff --git a/client/node_identity_endpoint.go b/client/node_identity_endpoint.go new file mode 100644 index 000000000..8d3eb9289 --- /dev/null +++ b/client/node_identity_endpoint.go @@ -0,0 +1,33 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package client + +import ( + "github.com/hashicorp/nomad/nomad/structs" +) + +type NodeIdentity struct { + c *Client +} + +func newNodeIdentityEndpoint(c *Client) *NodeIdentity { + n := &NodeIdentity{c: c} + return n +} + +func (n *NodeIdentity) Renew(args *structs.NodeIdentityRenewReq, _ *structs.NodeIdentityRenewResp) error { + + // Check node write permissions. + if aclObj, err := n.c.ResolveToken(args.AuthToken); err != nil { + return err + } else if !aclObj.AllowNodeWrite() { + return structs.ErrPermissionDenied + } + + // Store the node identity renewal request on the client, so it can be + // picked up at the next heartbeat. + n.c.identityForceRenewal.Store(true) + + return nil +} diff --git a/client/node_identity_endpoint_test.go b/client/node_identity_endpoint_test.go new file mode 100644 index 000000000..cdbbd06e6 --- /dev/null +++ b/client/node_identity_endpoint_test.go @@ -0,0 +1,103 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package client + +import ( + "testing" + + "github.com/hashicorp/nomad/acl" + "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/nomad" + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/testutil" + "github.com/shoenig/test/must" +) + +func TestNodeIdentity_Renew(t *testing.T) { + ci.Parallel(t) + + // Create a test ACL server and client and perform our node identity renewal + // tests against it. + testACLServer, testServerToken, testACLServerCleanup := nomad.TestACLServer(t, nil) + t.Cleanup(func() { testACLServerCleanup() }) + testutil.WaitForLeader(t, testACLServer.RPC) + + testACLClient, testACLClientCleanup := TestClient(t, func(c *config.Config) { + c.ACLEnabled = true + c.Servers = []string{testACLServer.GetConfig().RPCAddr.String()} + }) + t.Cleanup(func() { _ = testACLClientCleanup() }) + testutil.WaitForClientStatusWithToken( + t, testACLServer.RPC, testACLClient.NodeID(), testACLClient.Region(), + structs.NodeStatusReady, testServerToken.SecretID, + ) + + t.Run("acl_denied", func(t *testing.T) { + must.ErrorContains( + t, + testACLClient.ClientRPC( + structs.NodeIdentityRenewRPCMethod, + &structs.NodeIdentityRenewReq{}, + &structs.NodeIdentityRenewResp{}, + ), + structs.ErrPermissionDenied.Error(), + ) + }) + + t.Run("acl_valid", func(t *testing.T) { + + aclPolicy := mock.NodePolicy(acl.PolicyWrite) + aclToken := mock.CreatePolicyAndToken(t, testACLServer.State(), 10, t.Name(), aclPolicy) + + req := structs.NodeIdentityRenewReq{ + NodeID: testACLClient.NodeID(), + QueryOptions: structs.QueryOptions{ + AuthToken: aclToken.SecretID, + }, + } + + must.NoError( + t, + testACLClient.ClientRPC( + structs.NodeIdentityRenewRPCMethod, + &req, + &structs.NodeIdentityRenewResp{}, + ), + ) + + renewalVal := testACLClient.identityForceRenewal.Load() + must.True(t, renewalVal) + }) + + // Create a test non-ACL server and client and perform our node identity + // renewal tests against it. + testServer, testServerCleanup := nomad.TestServer(t, nil) + t.Cleanup(func() { testServerCleanup() }) + testutil.WaitForLeader(t, testServer.RPC) + + testClient, testClientCleanup := TestClient(t, func(c *config.Config) { + c.Servers = []string{testServer.GetConfig().RPCAddr.String()} + }) + t.Cleanup(func() { _ = testClientCleanup() }) + testutil.WaitForClient(t, testServer.RPC, testClient.NodeID(), testClient.Region()) + + t.Run("non_acl_valid", func(t *testing.T) { + must.NoError( + t, + testClient.ClientRPC( + structs.NodeIdentityRenewRPCMethod, + &structs.NodeIdentityRenewReq{ + NodeID: testClient.NodeID(), + QueryOptions: structs.QueryOptions{}, + }, + &structs.NodeIdentityRenewResp{}, + ), + ) + + renewalVal := testClient.identityForceRenewal.Load() + must.True(t, renewalVal) + }) +} diff --git a/client/rpc.go b/client/rpc.go index 69db60c45..846589724 100644 --- a/client/rpc.go +++ b/client/rpc.go @@ -22,13 +22,14 @@ import ( // rpcEndpoints holds the RPC endpoints type rpcEndpoints struct { - ClientStats *ClientStats - CSI *CSI - FileSystem *FileSystem - Allocations *Allocations - Agent *Agent - NodeMeta *NodeMeta - HostVolume *HostVolume + ClientStats *ClientStats + CSI *CSI + FileSystem *FileSystem + Allocations *Allocations + Agent *Agent + NodeIdentity *NodeIdentity + NodeMeta *NodeMeta + HostVolume *HostVolume } // ClientRPC is used to make a local, client only RPC call @@ -301,6 +302,7 @@ func (c *Client) setupClientRpc(rpcs map[string]interface{}) { c.endpoints.FileSystem = NewFileSystemEndpoint(c) c.endpoints.Allocations = NewAllocationsEndpoint(c) c.endpoints.Agent = NewAgentEndpoint(c) + c.endpoints.NodeIdentity = newNodeIdentityEndpoint(c) c.endpoints.NodeMeta = newNodeMetaEndpoint(c) c.endpoints.HostVolume = newHostVolumesEndpoint(c) c.setupClientRpcServer(c.rpcServer) @@ -317,6 +319,7 @@ func (c *Client) setupClientRpcServer(server *rpc.Server) { server.Register(c.endpoints.FileSystem) server.Register(c.endpoints.Allocations) server.Register(c.endpoints.Agent) + _ = server.Register(c.endpoints.NodeIdentity) server.Register(c.endpoints.NodeMeta) server.Register(c.endpoints.HostVolume) } diff --git a/command/agent/http.go b/command/agent/http.go index 52c552677..67e88fa06 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -450,6 +450,7 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) { s.mux.Handle("/v1/client/stats", wrapCORS(s.wrap(s.ClientStatsRequest))) s.mux.Handle("/v1/client/allocation/", wrapCORS(s.wrap(s.ClientAllocRequest))) s.mux.Handle("/v1/client/metadata", wrapCORS(s.wrap(s.NodeMetaRequest))) + s.mux.Handle("/v1/client/identity/renew", wrapCORS(s.wrap(s.NodeIdentityRenewRequest))) s.mux.HandleFunc("/v1/agent/self", s.wrap(s.AgentSelfRequest)) s.mux.HandleFunc("/v1/agent/join", s.wrap(s.AgentJoinRequest)) diff --git a/command/agent/node_identity_endpoint.go b/command/agent/node_identity_endpoint.go new file mode 100644 index 000000000..4109c98e5 --- /dev/null +++ b/command/agent/node_identity_endpoint.go @@ -0,0 +1,43 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package agent + +import ( + "net/http" + + "github.com/hashicorp/nomad/nomad/structs" +) + +func (s *HTTPServer) NodeIdentityRenewRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + // Build the request by parsing all common parameters and node id + args := structs.NodeIdentityRenewReq{} + s.parse(resp, req, &args.QueryOptions.Region, &args.QueryOptions) + parseNode(req, &args.NodeID) + + // Determine the handler to use + useLocalClient, useClientRPC, useServerRPC := s.rpcHandlerForNode(args.NodeID) + + // Make the RPC + var reply structs.NodeIdentityRenewResp + var rpcErr error + if useLocalClient { + rpcErr = s.agent.Client().ClientRPC(structs.NodeIdentityRenewRPCMethod, &args, &reply) + } else if useClientRPC { + rpcErr = s.agent.Client().RPC(structs.NodeIdentityRenewRPCMethod, &args, &reply) + } else if useServerRPC { + rpcErr = s.agent.Server().RPC(structs.NodeIdentityRenewRPCMethod, &args, &reply) + } else { + rpcErr = CodedError(400, "no local Node and node_id not provided") + } + + if rpcErr != nil { + if structs.IsErrNoNodeConn(rpcErr) { + rpcErr = CodedError(404, rpcErr.Error()) + } + + return nil, rpcErr + } + + return reply, nil +} diff --git a/command/commands.go b/command/commands.go index 0766ea43f..dbcd37bd6 100644 --- a/command/commands.go +++ b/command/commands.go @@ -634,6 +634,16 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, + "node identity": func() (cli.Command, error) { + return &NodeIdentityCommand{ + Meta: meta, + }, nil + }, + "node identity renew": func() (cli.Command, error) { + return &NodeIdentityRenewCommand{ + Meta: meta, + }, nil + }, "node meta": func() (cli.Command, error) { return &NodeMetaCommand{ Meta: meta, diff --git a/command/node_identity.go b/command/node_identity.go new file mode 100644 index 000000000..095ec6e44 --- /dev/null +++ b/command/node_identity.go @@ -0,0 +1,34 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package command + +import ( + "strings" + + "github.com/hashicorp/cli" +) + +type NodeIdentityCommand struct { + Meta +} + +func (n *NodeIdentityCommand) Help() string { + helpText := ` +Usage: nomad node identity [subcommand] + + Interact with a node's identity. All commands interact directly with a client + and require setting the target node via its 36 character ID. + + Please see the individual subcommand help for detailed usage information. +` + return strings.TrimSpace(helpText) +} + +func (n *NodeIdentityCommand) Synopsis() string { return "Force renewal of a nodes identity" } + +func (n *NodeIdentityCommand) Name() string { return "node identity" } + +func (n *NodeIdentityCommand) Run(_ []string) int { + return cli.RunResultHelp +} diff --git a/command/node_identity_renew.go b/command/node_identity_renew.go new file mode 100644 index 000000000..991e9c5a4 --- /dev/null +++ b/command/node_identity_renew.go @@ -0,0 +1,88 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package command + +import ( + "fmt" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/posener/complete" +) + +type NodeIdentityRenewCommand struct { + Meta +} + +func (n *NodeIdentityRenewCommand) Help() string { + helpText := ` +Usage: nomad node identity renew [options] + + Instruct a node to renew its identity at the next heartbeat. This command only + applies to client agents. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault|usageOptsNoNamespace) + + return strings.TrimSpace(helpText) +} + +func (n *NodeIdentityRenewCommand) Synopsis() string { return "Force a node to renew its identity" } + +func (n *NodeIdentityRenewCommand) Name() string { return "node identity renew" } + +func (n *NodeIdentityRenewCommand) Run(args []string) int { + + flags := n.Meta.FlagSet(n.Name(), FlagSetClient) + flags.Usage = func() { n.Ui.Output(n.Help()) } + + if err := flags.Parse(args); err != nil { + return 1 + } + args = flags.Args() + + if len(args) != 1 { + n.Ui.Error("This command takes one argument: ") + n.Ui.Error(commandErrorText(n)) + return 1 + } + + // Get the HTTP client + client, err := n.Meta.Client() + if err != nil { + n.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + nodeID := args[0] + + // Lookup nodeID + if nodeID != "" { + nodeID, err = lookupNodeID(client.Nodes(), nodeID) + if err != nil { + n.Ui.Error(err.Error()) + return 1 + } + } + + req := api.NodeIdentityRenewRequest{ + NodeID: nodeID, + } + + if _, err := client.Nodes().Identity().Renew(&req, nil); err != nil { + n.Ui.Error(fmt.Sprintf("Error requesting node identity renewal: %s", err)) + return 1 + } + + return 0 +} + +func (n *NodeIdentityRenewCommand) AutocompleteFlags() complete.Flags { + return n.Meta.AutocompleteFlags(FlagSetClient) +} + +func (n *NodeIdentityRenewCommand) AutocompleteArgs() complete.Predictor { + return nodePredictor(n.Client, nil) +} diff --git a/nomad/client_identity_endpoint.go b/nomad/client_identity_endpoint.go new file mode 100644 index 000000000..78235d546 --- /dev/null +++ b/nomad/client_identity_endpoint.go @@ -0,0 +1,47 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package nomad + +import ( + "time" + + metrics "github.com/hashicorp/go-metrics/compat" + "github.com/hashicorp/nomad/nomad/structs" +) + +type NodeIdentity struct { + srv *Server +} + +func newNodeIdentityEndpoint(srv *Server) *NodeIdentity { + return &NodeIdentity{ + srv: srv, + } +} + +func (n *NodeIdentity) Renew(args *structs.NodeIdentityRenewReq, reply *structs.NodeIdentityRenewResp) error { + + // Prevent infinite loop between the leader and the follower with the target + // node connection. + args.QueryOptions.AllowStale = true + + authErr := n.srv.Authenticate(nil, args) + if done, err := n.srv.forward(structs.NodeIdentityRenewRPCMethod, args, args, reply); done { + return err + } + n.srv.MeasureRPCRate("client_identity", structs.RateMetricWrite, args) + if authErr != nil { + return structs.ErrPermissionDenied + } + defer metrics.MeasureSince([]string{"nomad", "client_identity", "renew"}, time.Now()) + + // Check node write permissions + if aclObj, err := n.srv.ResolveACL(args); err != nil { + return err + } else if !aclObj.AllowNodeWrite() { + return structs.ErrPermissionDenied + } + + return n.srv.forwardClientRPC(structs.NodeIdentityRenewRPCMethod, args.NodeID, args, reply) +} diff --git a/nomad/client_identity_endpoint_test.go b/nomad/client_identity_endpoint_test.go new file mode 100644 index 000000000..a40289b88 --- /dev/null +++ b/nomad/client_identity_endpoint_test.go @@ -0,0 +1,85 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package nomad + +import ( + "testing" + + "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/client" + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/testutil" + "github.com/shoenig/test/must" +) + +func TestNodeIdentity_Renew_Forward(t *testing.T) { + ci.Parallel(t) + + servers := []*Server{} + for i := 0; i < 3; i++ { + s, cleanup := TestServer(t, func(c *Config) { + c.BootstrapExpect = 3 + c.NumSchedulers = 0 + }) + t.Cleanup(cleanup) + servers = append(servers, s) + } + + TestJoin(t, servers...) + leader := testutil.WaitForLeaders(t, servers[0].RPC, servers[1].RPC, servers[2].RPC) + + followers := []string{} + for _, s := range servers { + if addr := s.config.RPCAddr.String(); addr != leader { + followers = append(followers, addr) + } + } + t.Logf("leader=%s followers=%q", leader, followers) + + clients := make([]*client.Client, 4) + + for i := 0; i < 4; i++ { + c, cleanup := client.TestClient(t, func(c *config.Config) { + c.Servers = followers + }) + t.Cleanup(func() { _ = cleanup() }) + clients[i] = c + } + for _, c := range clients { + testutil.WaitForClient(t, servers[0].RPC, c.NodeID(), c.Region()) + } + + agentRPCs := []func(string, any, any) error{} + nodeIDs := make([]string, 0, len(clients)) + + // Build list of agents and node IDs + for _, s := range servers { + agentRPCs = append(agentRPCs, s.RPC) + } + + for _, c := range clients { + agentRPCs = append(agentRPCs, c.RPC) + nodeIDs = append(nodeIDs, c.NodeID()) + } + + // Iterate through all the agent RPCs to ensure that the renew RPC will + // succeed, no matter which agent we connect to. + for _, agentRPC := range agentRPCs { + for _, nodeID := range nodeIDs { + args := &structs.NodeIdentityRenewReq{ + NodeID: nodeID, + QueryOptions: structs.QueryOptions{ + Region: clients[0].Region(), + }, + } + must.NoError(t, + agentRPC(structs.NodeIdentityRenewRPCMethod, + args, + &structs.NodeIdentityRenewResp{}, + ), + ) + } + } +} diff --git a/nomad/node_endpoint.go b/nomad/node_endpoint.go index c04b1db00..d752be090 100644 --- a/nomad/node_endpoint.go +++ b/nomad/node_endpoint.go @@ -262,6 +262,8 @@ func (n *Node) Register(args *structs.NodeRegisterRequest, reply *structs.NodeUp reply.SignedIdentity = &signedJWT args.Node.IdentitySigningKeyID = signingKeyID + } else if originalNode != nil { + args.Node.IdentitySigningKeyID = originalNode.IdentitySigningKeyID } _, index, err := n.srv.raftApply(structs.NodeRegisterRequestType, args) diff --git a/nomad/server.go b/nomad/server.go index c6f7b0611..649fecf02 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -1282,6 +1282,7 @@ func (s *Server) setupRpcServer(server *rpc.Server, ctx *RPCContext) { // These endpoints are client RPCs and don't include a connection context _ = server.Register(NewClientStatsEndpoint(s)) _ = server.Register(newNodeMetaEndpoint(s)) + _ = server.Register(newNodeIdentityEndpoint(s)) // These endpoints have their streaming component registered in // setupStreamingEndpoints, but their non-streaming RPCs are registered diff --git a/nomad/structs/node.go b/nomad/structs/node.go index a5a308e3f..8e62ddbf0 100644 --- a/nomad/structs/node.go +++ b/nomad/structs/node.go @@ -714,3 +714,25 @@ type NodeUpdateResponse struct { QueryMeta } + +const ( + // NodeIdentityRenewRPCMethod is the RPC method for instructing a client to + // forcibly request a renewal of its node identity at the next heartbeat. + // + // Args: NodeIdentityRenewReq + // Reply: NodeIdentityRenewResp + NodeIdentityRenewRPCMethod = "NodeIdentity.Renew" +) + +// NodeIdentityRenewReq is used to instruct the Nomad server to renew the client +// identity at its next heartbeat regardless of whether it is close to +// expiration. +type NodeIdentityRenewReq struct { + NodeID string + + // This is a client RPC, so we must use query options which allow us to set + // AllowStale=true. + QueryOptions +} + +type NodeIdentityRenewResp struct{}