client: Add state functionality for set and get client identities. (#26184)

The Nomad client will persist its own identity within its state
store for restart persistence. The added benefit of using it over
the filesystem is that it supports transactions. This is useful
when considering the identity will be renewed periodically.

client/state/db_bolt.go

@@ -140,6 +140,12 @@ var (
 	nodeRegistrationKey = []byte("node_registration")
 
 	hostVolBucket = []byte("host_volumes_to_create")
+
+	// nodeIdentityBucket and nodeIdentityBucketStateKey are used to persist
+	// the client identity and its state. Each client will only have a single
+	// identity, so we use a single key value for the storage.
+	nodeIdentityBucket         = []byte("node_identity")
+	nodeIdentityBucketStateKey = []byte("node_identity_state")
 )
 
 // taskBucketName returns the bucket name for the given task name.
@@ -1089,6 +1095,42 @@ func (s *BoltStateDB) DeleteDynamicHostVolume(id string) error {
 	})
 }
 
+// clientIdentity wraps the signed client identity so we can safely add more
+// state in the future without needing a new entry type.
+type clientIdentity struct {
+	SignedIdentity string
+}
+
+func (s *BoltStateDB) PutNodeIdentity(identity string) error {
+	return s.db.Update(func(tx *boltdd.Tx) error {
+		b, err := tx.CreateBucketIfNotExists(nodeIdentityBucket)
+		if err != nil {
+			return err
+		}
+
+		identityWrapper := clientIdentity{SignedIdentity: identity}
+
+		return b.Put(nodeIdentityBucketStateKey, &identityWrapper)
+	})
+}
+
+func (s *BoltStateDB) GetNodeIdentity() (string, error) {
+	var identityWrapper clientIdentity
+	err := s.db.View(func(tx *boltdd.Tx) error {
+		b := tx.Bucket(nodeIdentityBucket)
+		if b == nil {
+			return nil
+		}
+		return b.Get(nodeIdentityBucketStateKey, &identityWrapper)
+	})
+
+	if boltdd.IsErrNotFound(err) {
+		return "", nil
+	}
+
+	return identityWrapper.SignedIdentity, err
+}
+
 // init initializes metadata entries in a newly created state database.
 func (s *BoltStateDB) init() error {
 	return s.db.Update(func(tx *boltdd.Tx) error {

client/state/db_error.go

@@ -172,3 +172,7 @@ func (m *ErrDB) DeleteDynamicHostVolume(_ string) error {
 func (m *ErrDB) Close() error {
 	return fmt.Errorf("Error!")
 }
+
+func (m *ErrDB) PutNodeIdentity(_ string) error { return ErrDBError }
+
+func (m *ErrDB) GetNodeIdentity() (string, error) { return "", ErrDBError }

client/state/db_mem.go

@@ -6,6 +6,7 @@ package state
 import (
 	"maps"
 	"sync"
+	"sync/atomic"
 
 	"github.com/hashicorp/go-hclog"
 	arstate "github.com/hashicorp/nomad/client/allocrunner/state"
@@ -62,6 +63,9 @@ type MemDB struct {
 
 	dynamicHostVolumes map[string]*cstructs.HostVolumeState
 
+	// clientIdentity is the persisted identity of the client.
+	clientIdentity atomic.Value
+
 	logger hclog.Logger
 
 	mu sync.RWMutex
@@ -79,6 +83,7 @@ func NewMemDB(logger hclog.Logger) *MemDB {
 		checks:             make(checks.ClientResults),
 		identities:         make(map[string][]*structs.SignedWorkloadIdentity),
 		dynamicHostVolumes: make(map[string]*cstructs.HostVolumeState),
+		clientIdentity:     atomic.Value{},
 		logger:             logger,
 	}
 }
@@ -379,6 +384,19 @@ func (m *MemDB) DeleteDynamicHostVolume(s string) error {
 	return nil
 }
 
+func (m *MemDB) PutNodeIdentity(identity string) error {
+	m.clientIdentity.Store(identity)
+	return nil
+}
+
+func (m *MemDB) GetNodeIdentity() (string, error) {
+	if obj := m.clientIdentity.Load(); obj == nil {
+		return "", nil
+	} else {
+		return obj.(string), nil
+	}
+}
+
 func (m *MemDB) Close() error {
 	m.mu.Lock()
 	defer m.mu.Unlock()

client/state/db_noop.go

@@ -157,6 +157,10 @@ func (n NoopDB) DeleteDynamicHostVolume(_ string) error {
 	return nil
 }
 
+func (n NoopDB) PutNodeIdentity(_ string) error { return nil }
+
+func (n NoopDB) GetNodeIdentity() (string, error) { return "", nil }
+
 func (n NoopDB) Close() error {
 	return nil
 }

client/state/db_test.go

@@ -493,6 +493,24 @@ func TestStateDB_CheckResult(t *testing.T) {
 
 }
 
+func TestStateDB_NodeIdentity(t *testing.T) {
+	ci.Parallel(t)
+
+	testDB(t, func(t *testing.T, db StateDB) {
+		identity, err := db.GetNodeIdentity()
+		must.NoError(t, err)
+		must.Eq(t, "", identity)
+
+		fakeIdentity := "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMn0.KMUFsIDTnFmyG3nMiGM6H9FNFUROf3wh7SmqJp-QV30"
+
+		must.NoError(t, db.PutNodeIdentity(fakeIdentity))
+
+		identity, err = db.GetNodeIdentity()
+		must.NoError(t, err)
+		must.Eq(t, fakeIdentity, identity)
+	})
+}
+
 // TestStateDB_Upgrade asserts calling Upgrade on new databases always
 // succeeds.
 func TestStateDB_Upgrade(t *testing.T) {

client/state/interface.go

@@ -141,6 +141,14 @@ type StateDB interface {
 	GetDynamicHostVolumes() ([]*cstructs.HostVolumeState, error)
 	DeleteDynamicHostVolume(string) error
 
+	// PutNodeIdentity stores the signed identity JWT for the client.
+	PutNodeIdentity(identity string) error
+
+	// GetNodeIdentity retrieves the signed identity JWT for the client. If the
+	// client has not generated an identity, this will return an empty string
+	// and no error.
+	GetNodeIdentity() (string, error)
+
 	// Close the database. Unsafe for further use after calling regardless
 	// of return value.
 	Close() error

command/operator_client_state.go

@@ -132,8 +132,18 @@ func (c *OperatorClientStateCommand) Run(args []string) int {
 			Tasks:        tasks,
 		}
 	}
+
+	// Get the node identity state, which is useful when debugging to see the
+	// real and current identity the node is using.
+	nodeIdentity, err := db.GetNodeIdentity()
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("failed to get node identity state: %v", err))
+		return 1
+	}
+
 	output := debugOutput{
-		Allocations: data,
+		Allocations:  data,
+		NodeIdentity: nodeIdentity,
 	}
 	bytes, err := json.Marshal(output)
 	if err != nil {
@@ -146,7 +156,8 @@ func (c *OperatorClientStateCommand) Run(args []string) int {
 }
 
 type debugOutput struct {
-	Allocations map[string]*clientStateAlloc
+	Allocations  map[string]*clientStateAlloc
+	NodeIdentity string
 }
 
 type clientStateAlloc struct {

command/operator_client_state_test.go

@@ -38,10 +38,16 @@ func TestOperatorClientStateCommand(t *testing.T) {
 	alloc := structs.MockAlloc()
 	err = db.PutAllocation(alloc)
 	must.NoError(t, err)
+
+	// Write a node identity to the DB, so we can test that the command reads
+	// this data.
+	must.NoError(t, db.PutNodeIdentity("mynodeidentity"))
+
 	must.NoError(t, db.Close())
 
 	// run against an incomplete client state directory
 	code = cmd.Run([]string{dir})
 	must.Eq(t, 0, code)
 	must.StrContains(t, ui.OutputWriter.String(), alloc.ID)
+	must.StrContains(t, ui.OutputWriter.String(), "NodeIdentity\":\"mynodeidentity")
 }