diff --git a/api/node_identity.go b/api/node_identity.go new file mode 100644 index 000000000..497ebcd23 --- /dev/null +++ b/api/node_identity.go @@ -0,0 +1,33 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package api + +type NodeIdentityRenewRequest struct { + NodeID string +} + +type NodeIdentityRenewResponse struct{} + +type NodeIdentity struct { + client *Client +} + +func (n *Nodes) Identity() *NodeIdentity { + return &NodeIdentity{client: n.client} +} + +// Renew instructs the node to request a new identity from the server at its +// next heartbeat. +// +// The request uses query options to control the forwarding behavior of the +// request only. Parameters such as Filter, WaitTime, and WaitIndex are not used +// and ignored. +func (n *NodeIdentity) Renew(req *NodeIdentityRenewRequest, qo *QueryOptions) (*NodeIdentityRenewResponse, error) { + var out NodeIdentityRenewResponse + _, err := n.client.postQuery("/v1/client/identity/renew", req, &out, qo) + if err != nil { + return nil, err + } + return &out, nil +} diff --git a/api/node_identity_test.go b/api/node_identity_test.go new file mode 100644 index 000000000..56f682f15 --- /dev/null +++ b/api/node_identity_test.go @@ -0,0 +1,29 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package api + +import ( + "testing" + + "github.com/hashicorp/nomad/api/internal/testutil" + "github.com/shoenig/test/must" +) + +func TestNodeIdentity_Renew(t *testing.T) { + testutil.Parallel(t) + + configCallback := func(c *testutil.TestServerConfig) { c.DevMode = true } + testClient, testServer := makeClient(t, nil, configCallback) + defer testServer.Stop() + + nodeID := oneNodeFromNodeList(t, testClient.Nodes()).ID + + req := NodeIdentityRenewRequest{ + NodeID: nodeID, + } + + resp, err := testClient.Nodes().Identity().Renew(&req, nil) + must.NoError(t, err) + must.NotNil(t, resp) +} diff --git a/client/client.go b/client/client.go index 74abd0c16..de9b465ec 100644 --- a/client/client.go +++ b/client/client.go @@ -339,6 +339,11 @@ type Client struct { // the servers. This is used to authenticate the client to the servers when // performing RPC calls. identity atomic.Value + + // identityForceRenewal is used to force the client to renew its identity + // at the next heartbeat. It is set by an operator calling the node identity + // renew RPC method. + identityForceRenewal atomic.Bool } var ( @@ -402,6 +407,7 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulProxie EnterpriseClient: newEnterpriseClient(logger), allocrunnerFactory: cfg.AllocRunnerFactory, identity: atomic.Value{}, + identityForceRenewal: atomic.Bool{}, } // we can't have this set in the default Config because of import cycles @@ -968,6 +974,10 @@ func (c *Client) nodeIdentityToken() string { // processes with a new node identity token. func (c *Client) setNodeIdentityToken(token string) { + // It's a bit of a simple log line, but it is useful to know when the client + // has renewed or set its node identity token. + c.logger.Info("setting node identity token") + // Store the token on the client as the first step, so it's available for // use by all RPCs immediately. c.identity.Store(token) @@ -2204,6 +2214,14 @@ func (c *Client) updateNodeStatus() error { AuthToken: c.nodeAuthToken(), }, } + + // Check if the client has been informed to force a renewal of its identity, + // and set the flag in the request if so. + if c.identityForceRenewal.Load() { + c.logger.Debug("forcing identity renewal") + req.ForceIdentityRenewal = true + } + var resp structs.NodeUpdateResponse if err := c.RPC("Node.UpdateStatus", &req, &resp); err != nil { c.triggerDiscovery() @@ -2226,7 +2244,17 @@ func (c *Client) updateNodeStatus() error { c.heartbeatLock.Unlock() c.logger.Trace("next heartbeat", "period", resp.HeartbeatTTL) - if resp.Index != 0 { + // The Nomad server will return an index of greater than zero when a Raft + // update has occurred, indicating a change in the state of the persisted + // node object. + // + // This can be due to a Nomad server invalidating the node's heartbeat timer + // and marking the node as down. In this case, we want to log a warning for + // the operator to see the client missed a heartbeat. If the server + // responded with a new identity, we assume the client did not miss a + // heartbeat. If we did, this line would appear each time the identity was + // renewed, which could confuse cluster operators. + if resp.Index != 0 && resp.SignedIdentity == nil { c.logger.Debug("state updated", "node_status", req.Status) // We have potentially missed our TTL log how delayed we were @@ -2276,6 +2304,10 @@ func (c *Client) handleNodeUpdateResponse(resp structs.NodeUpdateResponse) error return fmt.Errorf("error saving client identity: %w", err) } c.setNodeIdentityToken(*resp.SignedIdentity) + + // If the operator forced this renewal, reset the flag so that we don't + // keep renewing the identity on every heartbeat. + c.identityForceRenewal.Store(false) } // Convert []*NodeServerInfo to []*servers.Server diff --git a/client/node_identity_endpoint.go b/client/node_identity_endpoint.go new file mode 100644 index 000000000..8d3eb9289 --- /dev/null +++ b/client/node_identity_endpoint.go @@ -0,0 +1,33 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package client + +import ( + "github.com/hashicorp/nomad/nomad/structs" +) + +type NodeIdentity struct { + c *Client +} + +func newNodeIdentityEndpoint(c *Client) *NodeIdentity { + n := &NodeIdentity{c: c} + return n +} + +func (n *NodeIdentity) Renew(args *structs.NodeIdentityRenewReq, _ *structs.NodeIdentityRenewResp) error { + + // Check node write permissions. + if aclObj, err := n.c.ResolveToken(args.AuthToken); err != nil { + return err + } else if !aclObj.AllowNodeWrite() { + return structs.ErrPermissionDenied + } + + // Store the node identity renewal request on the client, so it can be + // picked up at the next heartbeat. + n.c.identityForceRenewal.Store(true) + + return nil +} diff --git a/client/node_identity_endpoint_test.go b/client/node_identity_endpoint_test.go new file mode 100644 index 000000000..cdbbd06e6 --- /dev/null +++ b/client/node_identity_endpoint_test.go @@ -0,0 +1,103 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package client + +import ( + "testing" + + "github.com/hashicorp/nomad/acl" + "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/nomad" + "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/testutil" + "github.com/shoenig/test/must" +) + +func TestNodeIdentity_Renew(t *testing.T) { + ci.Parallel(t) + + // Create a test ACL server and client and perform our node identity renewal + // tests against it. + testACLServer, testServerToken, testACLServerCleanup := nomad.TestACLServer(t, nil) + t.Cleanup(func() { testACLServerCleanup() }) + testutil.WaitForLeader(t, testACLServer.RPC) + + testACLClient, testACLClientCleanup := TestClient(t, func(c *config.Config) { + c.ACLEnabled = true + c.Servers = []string{testACLServer.GetConfig().RPCAddr.String()} + }) + t.Cleanup(func() { _ = testACLClientCleanup() }) + testutil.WaitForClientStatusWithToken( + t, testACLServer.RPC, testACLClient.NodeID(), testACLClient.Region(), + structs.NodeStatusReady, testServerToken.SecretID, + ) + + t.Run("acl_denied", func(t *testing.T) { + must.ErrorContains( + t, + testACLClient.ClientRPC( + structs.NodeIdentityRenewRPCMethod, + &structs.NodeIdentityRenewReq{}, + &structs.NodeIdentityRenewResp{}, + ), + structs.ErrPermissionDenied.Error(), + ) + }) + + t.Run("acl_valid", func(t *testing.T) { + + aclPolicy := mock.NodePolicy(acl.PolicyWrite) + aclToken := mock.CreatePolicyAndToken(t, testACLServer.State(), 10, t.Name(), aclPolicy) + + req := structs.NodeIdentityRenewReq{ + NodeID: testACLClient.NodeID(), + QueryOptions: structs.QueryOptions{ + AuthToken: aclToken.SecretID, + }, + } + + must.NoError( + t, + testACLClient.ClientRPC( + structs.NodeIdentityRenewRPCMethod, + &req, + &structs.NodeIdentityRenewResp{}, + ), + ) + + renewalVal := testACLClient.identityForceRenewal.Load() + must.True(t, renewalVal) + }) + + // Create a test non-ACL server and client and perform our node identity + // renewal tests against it. + testServer, testServerCleanup := nomad.TestServer(t, nil) + t.Cleanup(func() { testServerCleanup() }) + testutil.WaitForLeader(t, testServer.RPC) + + testClient, testClientCleanup := TestClient(t, func(c *config.Config) { + c.Servers = []string{testServer.GetConfig().RPCAddr.String()} + }) + t.Cleanup(func() { _ = testClientCleanup() }) + testutil.WaitForClient(t, testServer.RPC, testClient.NodeID(), testClient.Region()) + + t.Run("non_acl_valid", func(t *testing.T) { + must.NoError( + t, + testClient.ClientRPC( + structs.NodeIdentityRenewRPCMethod, + &structs.NodeIdentityRenewReq{ + NodeID: testClient.NodeID(), + QueryOptions: structs.QueryOptions{}, + }, + &structs.NodeIdentityRenewResp{}, + ), + ) + + renewalVal := testClient.identityForceRenewal.Load() + must.True(t, renewalVal) + }) +} diff --git a/client/rpc.go b/client/rpc.go index 69db60c45..846589724 100644 --- a/client/rpc.go +++ b/client/rpc.go @@ -22,13 +22,14 @@ import ( // rpcEndpoints holds the RPC endpoints type rpcEndpoints struct { - ClientStats *ClientStats - CSI *CSI - FileSystem *FileSystem - Allocations *Allocations - Agent *Agent - NodeMeta *NodeMeta - HostVolume *HostVolume + ClientStats *ClientStats + CSI *CSI + FileSystem *FileSystem + Allocations *Allocations + Agent *Agent + NodeIdentity *NodeIdentity + NodeMeta *NodeMeta + HostVolume *HostVolume } // ClientRPC is used to make a local, client only RPC call @@ -301,6 +302,7 @@ func (c *Client) setupClientRpc(rpcs map[string]interface{}) { c.endpoints.FileSystem = NewFileSystemEndpoint(c) c.endpoints.Allocations = NewAllocationsEndpoint(c) c.endpoints.Agent = NewAgentEndpoint(c) + c.endpoints.NodeIdentity = newNodeIdentityEndpoint(c) c.endpoints.NodeMeta = newNodeMetaEndpoint(c) c.endpoints.HostVolume = newHostVolumesEndpoint(c) c.setupClientRpcServer(c.rpcServer) @@ -317,6 +319,7 @@ func (c *Client) setupClientRpcServer(server *rpc.Server) { server.Register(c.endpoints.FileSystem) server.Register(c.endpoints.Allocations) server.Register(c.endpoints.Agent) + _ = server.Register(c.endpoints.NodeIdentity) server.Register(c.endpoints.NodeMeta) server.Register(c.endpoints.HostVolume) } diff --git a/command/agent/http.go b/command/agent/http.go index 52c552677..67e88fa06 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -450,6 +450,7 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) { s.mux.Handle("/v1/client/stats", wrapCORS(s.wrap(s.ClientStatsRequest))) s.mux.Handle("/v1/client/allocation/", wrapCORS(s.wrap(s.ClientAllocRequest))) s.mux.Handle("/v1/client/metadata", wrapCORS(s.wrap(s.NodeMetaRequest))) + s.mux.Handle("/v1/client/identity/renew", wrapCORS(s.wrap(s.NodeIdentityRenewRequest))) s.mux.HandleFunc("/v1/agent/self", s.wrap(s.AgentSelfRequest)) s.mux.HandleFunc("/v1/agent/join", s.wrap(s.AgentJoinRequest)) diff --git a/command/agent/node_identity_endpoint.go b/command/agent/node_identity_endpoint.go new file mode 100644 index 000000000..4109c98e5 --- /dev/null +++ b/command/agent/node_identity_endpoint.go @@ -0,0 +1,43 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package agent + +import ( + "net/http" + + "github.com/hashicorp/nomad/nomad/structs" +) + +func (s *HTTPServer) NodeIdentityRenewRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + // Build the request by parsing all common parameters and node id + args := structs.NodeIdentityRenewReq{} + s.parse(resp, req, &args.QueryOptions.Region, &args.QueryOptions) + parseNode(req, &args.NodeID) + + // Determine the handler to use + useLocalClient, useClientRPC, useServerRPC := s.rpcHandlerForNode(args.NodeID) + + // Make the RPC + var reply structs.NodeIdentityRenewResp + var rpcErr error + if useLocalClient { + rpcErr = s.agent.Client().ClientRPC(structs.NodeIdentityRenewRPCMethod, &args, &reply) + } else if useClientRPC { + rpcErr = s.agent.Client().RPC(structs.NodeIdentityRenewRPCMethod, &args, &reply) + } else if useServerRPC { + rpcErr = s.agent.Server().RPC(structs.NodeIdentityRenewRPCMethod, &args, &reply) + } else { + rpcErr = CodedError(400, "no local Node and node_id not provided") + } + + if rpcErr != nil { + if structs.IsErrNoNodeConn(rpcErr) { + rpcErr = CodedError(404, rpcErr.Error()) + } + + return nil, rpcErr + } + + return reply, nil +} diff --git a/command/commands.go b/command/commands.go index 0766ea43f..dbcd37bd6 100644 --- a/command/commands.go +++ b/command/commands.go @@ -634,6 +634,16 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, + "node identity": func() (cli.Command, error) { + return &NodeIdentityCommand{ + Meta: meta, + }, nil + }, + "node identity renew": func() (cli.Command, error) { + return &NodeIdentityRenewCommand{ + Meta: meta, + }, nil + }, "node meta": func() (cli.Command, error) { return &NodeMetaCommand{ Meta: meta, diff --git a/command/node_identity.go b/command/node_identity.go new file mode 100644 index 000000000..095ec6e44 --- /dev/null +++ b/command/node_identity.go @@ -0,0 +1,34 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package command + +import ( + "strings" + + "github.com/hashicorp/cli" +) + +type NodeIdentityCommand struct { + Meta +} + +func (n *NodeIdentityCommand) Help() string { + helpText := ` +Usage: nomad node identity [subcommand] + + Interact with a node's identity. All commands interact directly with a client + and require setting the target node via its 36 character ID. + + Please see the individual subcommand help for detailed usage information. +` + return strings.TrimSpace(helpText) +} + +func (n *NodeIdentityCommand) Synopsis() string { return "Force renewal of a nodes identity" } + +func (n *NodeIdentityCommand) Name() string { return "node identity" } + +func (n *NodeIdentityCommand) Run(_ []string) int { + return cli.RunResultHelp +} diff --git a/command/node_identity_renew.go b/command/node_identity_renew.go new file mode 100644 index 000000000..991e9c5a4 --- /dev/null +++ b/command/node_identity_renew.go @@ -0,0 +1,88 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package command + +import ( + "fmt" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/posener/complete" +) + +type NodeIdentityRenewCommand struct { + Meta +} + +func (n *NodeIdentityRenewCommand) Help() string { + helpText := ` +Usage: nomad node identity renew [options] + + Instruct a node to renew its identity at the next heartbeat. This command only + applies to client agents. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault|usageOptsNoNamespace) + + return strings.TrimSpace(helpText) +} + +func (n *NodeIdentityRenewCommand) Synopsis() string { return "Force a node to renew its identity" } + +func (n *NodeIdentityRenewCommand) Name() string { return "node identity renew" } + +func (n *NodeIdentityRenewCommand) Run(args []string) int { + + flags := n.Meta.FlagSet(n.Name(), FlagSetClient) + flags.Usage = func() { n.Ui.Output(n.Help()) } + + if err := flags.Parse(args); err != nil { + return 1 + } + args = flags.Args() + + if len(args) != 1 { + n.Ui.Error("This command takes one argument: ") + n.Ui.Error(commandErrorText(n)) + return 1 + } + + // Get the HTTP client + client, err := n.Meta.Client() + if err != nil { + n.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + nodeID := args[0] + + // Lookup nodeID + if nodeID != "" { + nodeID, err = lookupNodeID(client.Nodes(), nodeID) + if err != nil { + n.Ui.Error(err.Error()) + return 1 + } + } + + req := api.NodeIdentityRenewRequest{ + NodeID: nodeID, + } + + if _, err := client.Nodes().Identity().Renew(&req, nil); err != nil { + n.Ui.Error(fmt.Sprintf("Error requesting node identity renewal: %s", err)) + return 1 + } + + return 0 +} + +func (n *NodeIdentityRenewCommand) AutocompleteFlags() complete.Flags { + return n.Meta.AutocompleteFlags(FlagSetClient) +} + +func (n *NodeIdentityRenewCommand) AutocompleteArgs() complete.Predictor { + return nodePredictor(n.Client, nil) +} diff --git a/nomad/client_identity_endpoint.go b/nomad/client_identity_endpoint.go new file mode 100644 index 000000000..78235d546 --- /dev/null +++ b/nomad/client_identity_endpoint.go @@ -0,0 +1,47 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package nomad + +import ( + "time" + + metrics "github.com/hashicorp/go-metrics/compat" + "github.com/hashicorp/nomad/nomad/structs" +) + +type NodeIdentity struct { + srv *Server +} + +func newNodeIdentityEndpoint(srv *Server) *NodeIdentity { + return &NodeIdentity{ + srv: srv, + } +} + +func (n *NodeIdentity) Renew(args *structs.NodeIdentityRenewReq, reply *structs.NodeIdentityRenewResp) error { + + // Prevent infinite loop between the leader and the follower with the target + // node connection. + args.QueryOptions.AllowStale = true + + authErr := n.srv.Authenticate(nil, args) + if done, err := n.srv.forward(structs.NodeIdentityRenewRPCMethod, args, args, reply); done { + return err + } + n.srv.MeasureRPCRate("client_identity", structs.RateMetricWrite, args) + if authErr != nil { + return structs.ErrPermissionDenied + } + defer metrics.MeasureSince([]string{"nomad", "client_identity", "renew"}, time.Now()) + + // Check node write permissions + if aclObj, err := n.srv.ResolveACL(args); err != nil { + return err + } else if !aclObj.AllowNodeWrite() { + return structs.ErrPermissionDenied + } + + return n.srv.forwardClientRPC(structs.NodeIdentityRenewRPCMethod, args.NodeID, args, reply) +} diff --git a/nomad/client_identity_endpoint_test.go b/nomad/client_identity_endpoint_test.go new file mode 100644 index 000000000..a40289b88 --- /dev/null +++ b/nomad/client_identity_endpoint_test.go @@ -0,0 +1,85 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package nomad + +import ( + "testing" + + "github.com/hashicorp/nomad/ci" + "github.com/hashicorp/nomad/client" + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/testutil" + "github.com/shoenig/test/must" +) + +func TestNodeIdentity_Renew_Forward(t *testing.T) { + ci.Parallel(t) + + servers := []*Server{} + for i := 0; i < 3; i++ { + s, cleanup := TestServer(t, func(c *Config) { + c.BootstrapExpect = 3 + c.NumSchedulers = 0 + }) + t.Cleanup(cleanup) + servers = append(servers, s) + } + + TestJoin(t, servers...) + leader := testutil.WaitForLeaders(t, servers[0].RPC, servers[1].RPC, servers[2].RPC) + + followers := []string{} + for _, s := range servers { + if addr := s.config.RPCAddr.String(); addr != leader { + followers = append(followers, addr) + } + } + t.Logf("leader=%s followers=%q", leader, followers) + + clients := make([]*client.Client, 4) + + for i := 0; i < 4; i++ { + c, cleanup := client.TestClient(t, func(c *config.Config) { + c.Servers = followers + }) + t.Cleanup(func() { _ = cleanup() }) + clients[i] = c + } + for _, c := range clients { + testutil.WaitForClient(t, servers[0].RPC, c.NodeID(), c.Region()) + } + + agentRPCs := []func(string, any, any) error{} + nodeIDs := make([]string, 0, len(clients)) + + // Build list of agents and node IDs + for _, s := range servers { + agentRPCs = append(agentRPCs, s.RPC) + } + + for _, c := range clients { + agentRPCs = append(agentRPCs, c.RPC) + nodeIDs = append(nodeIDs, c.NodeID()) + } + + // Iterate through all the agent RPCs to ensure that the renew RPC will + // succeed, no matter which agent we connect to. + for _, agentRPC := range agentRPCs { + for _, nodeID := range nodeIDs { + args := &structs.NodeIdentityRenewReq{ + NodeID: nodeID, + QueryOptions: structs.QueryOptions{ + Region: clients[0].Region(), + }, + } + must.NoError(t, + agentRPC(structs.NodeIdentityRenewRPCMethod, + args, + &structs.NodeIdentityRenewResp{}, + ), + ) + } + } +} diff --git a/nomad/node_endpoint.go b/nomad/node_endpoint.go index c04b1db00..d752be090 100644 --- a/nomad/node_endpoint.go +++ b/nomad/node_endpoint.go @@ -262,6 +262,8 @@ func (n *Node) Register(args *structs.NodeRegisterRequest, reply *structs.NodeUp reply.SignedIdentity = &signedJWT args.Node.IdentitySigningKeyID = signingKeyID + } else if originalNode != nil { + args.Node.IdentitySigningKeyID = originalNode.IdentitySigningKeyID } _, index, err := n.srv.raftApply(structs.NodeRegisterRequestType, args) diff --git a/nomad/server.go b/nomad/server.go index c6f7b0611..649fecf02 100644 --- a/nomad/server.go +++ b/nomad/server.go @@ -1282,6 +1282,7 @@ func (s *Server) setupRpcServer(server *rpc.Server, ctx *RPCContext) { // These endpoints are client RPCs and don't include a connection context _ = server.Register(NewClientStatsEndpoint(s)) _ = server.Register(newNodeMetaEndpoint(s)) + _ = server.Register(newNodeIdentityEndpoint(s)) // These endpoints have their streaming component registered in // setupStreamingEndpoints, but their non-streaming RPCs are registered diff --git a/nomad/structs/node.go b/nomad/structs/node.go index a5a308e3f..8e62ddbf0 100644 --- a/nomad/structs/node.go +++ b/nomad/structs/node.go @@ -714,3 +714,25 @@ type NodeUpdateResponse struct { QueryMeta } + +const ( + // NodeIdentityRenewRPCMethod is the RPC method for instructing a client to + // forcibly request a renewal of its node identity at the next heartbeat. + // + // Args: NodeIdentityRenewReq + // Reply: NodeIdentityRenewResp + NodeIdentityRenewRPCMethod = "NodeIdentity.Renew" +) + +// NodeIdentityRenewReq is used to instruct the Nomad server to renew the client +// identity at its next heartbeat regardless of whether it is close to +// expiration. +type NodeIdentityRenewReq struct { + NodeID string + + // This is a client RPC, so we must use query options which allow us to set + // AllowStale=true. + QueryOptions +} + +type NodeIdentityRenewResp struct{}