vault: remove revoked Vault accessors from state (#19706)

When using the no-op Vault client the Nomad server still needs to delete
the revoked Vault accessors from state to prevent them from lingering
forever after the cluster migrates to the workload identity flow.

nomad/leader_test.go

@@ -836,6 +836,11 @@ func TestLeader_revokeVaultAccessorsOnRestore_workloadIdentity(t *testing.T) {
 	// Do a restore
 	err = s1.revokeVaultAccessorsOnRestore()
 	must.NoError(t, err)
+
+	// Verify accessor was removed from state.
+	got, err := fsmState.VaultAccessor(nil, va.Accessor)
+	must.NoError(t, err)
+	must.Nil(t, got)
 }
 
 func TestLeader_revokeSITokenAccessorsOnRestore(t *testing.T) {

nomad/node_endpoint_test.go

@@ -717,6 +717,13 @@ func TestClientEndpoint_Deregister_Vault_WorkloadIdentity(t *testing.T) {
 	var resp2 structs.GenericResponse
 	err = msgpackrpc.CallWithCodec(codec, "Node.Deregister", dereg, &resp2)
 	must.NoError(t, err)
+
+	// Verify accessors are removed from state.
+	for _, va := range accessors {
+		got, err := state.VaultAccessor(nil, va.Accessor)
+		must.NoError(t, err)
+		must.Nil(t, got)
+	}
 }
 
 func TestClientEndpoint_UpdateStatus(t *testing.T) {
@@ -900,6 +907,13 @@ func TestClientEndpoint_UpdateStatus_Vault_WorkloadIdentity(t *testing.T) {
 	var resp2 structs.NodeUpdateResponse
 	err = msgpackrpc.CallWithCodec(codec, "Node.UpdateStatus", updateReq, &resp2)
 	must.NoError(t, err)
+
+	// Verify accessors are removed from state.
+	for _, va := range accessors {
+		got, err := state.VaultAccessor(nil, va.Accessor)
+		must.NoError(t, err)
+		must.Nil(t, got)
+	}
 }
 
 func TestClientEndpoint_UpdateStatus_Reconnect(t *testing.T) {
@@ -3371,6 +3385,13 @@ func TestClientEndpoint_UpdateAlloc_VaultWorkloadIdentity(t *testing.T) {
 	var resp2 structs.NodeAllocsResponse
 	err = msgpackrpc.CallWithCodec(codec, "Node.UpdateAlloc", update, &resp2)
 	must.NoError(t, err)
+
+	// Verify accessors are removed from state.
+	for _, va := range accessors {
+		got, err := state.VaultAccessor(nil, va.Accessor)
+		must.NoError(t, err)
+		must.Nil(t, got)
+	}
 }
 
 func TestClientEndpoint_CreateNodeEvals(t *testing.T) {

nomad/server.go

@@ -1210,7 +1210,7 @@ func (s *Server) setupConsul(consulConfigFunc consul.ConfigAPIFunc, consulACLs c
 func (s *Server) setupVaultClient() error {
 	vconfig := s.config.GetDefaultVault()
 	if vconfig != nil && vconfig.Token == "" {
-		s.vault = NewNoopVault(s.logger)
+		s.vault = NewNoopVault(vconfig, s.logger, s.purgeVaultAccessors)
 		return nil
 	}
 

nomad/vault_noop.go

@@ -16,14 +16,18 @@ import (
 )
 
 type NoopVault struct {
-	l      sync.Mutex
-	config *config.VaultConfig
-	logger log.Logger
+	l sync.Mutex
+
+	config  *config.VaultConfig
+	logger  log.Logger
+	purgeFn PurgeVaultAccessorFn
 }
 
-func NewNoopVault(logger log.Logger) *NoopVault {
+func NewNoopVault(c *config.VaultConfig, logger log.Logger, purgeFn PurgeVaultAccessorFn) *NoopVault {
 	return &NoopVault{
-		logger: logger.Named("vault-noop"),
+		config:  c,
+		logger:  logger.Named("vault-noop"),
+		purgeFn: purgeFn,
 	}
 }
 
@@ -56,6 +60,11 @@ func (v *NoopVault) RevokeTokens(_ context.Context, tokens []*structs.VaultAcces
 	for _, t := range tokens {
 		v.logger.Debug("Vault token is no longer used, but Nomad is not able to revoke it. The token may need to be revoked manually or will expire once its TTL reaches zero.", "accessor", t.Accessor, "ttl", t.CreationTTL)
 	}
+
+	if err := v.purgeFn(tokens); err != nil {
+		v.logger.Error("failed to purge Vault accessors", "error", err)
+	}
+
 	return nil
 }
 
@@ -63,6 +72,11 @@ func (v *NoopVault) MarkForRevocation(tokens []*structs.VaultAccessor) error {
 	for _, t := range tokens {
 		v.logger.Debug("Vault token is no longer used, but Nomad is not able to mark it for revocation. The token may need to be revoked manually or will expire once its TTL reaches zero.", "accessor", t.Accessor, "ttl", t.CreationTTL)
 	}
+
+	if err := v.purgeFn(tokens); err != nil {
+		v.logger.Error("failed to purge Vault accessors", "error", err)
+	}
+
 	return nil
 }