cli: extend coverage of operator client-state command (#18996)

The `operator client-state` command is mostly used for developer debugging of
the Nomad client state, but it hasn't been updated with several recent
additions. Add allocation identities, network status, and dynamic volumes to the
objects it outputs.

Also, fix a bug where reading the state for an allocation without task states
will crash the CLI. This can happen if the Nomad client stops after an alloc is
persisted to disk but before the task actually starts.

.changelog/18996.txt

@@ -0,0 +1,7 @@
+```release-note:improvement
+cli: Added identities, networks, and volumes to the output of the `operator client-state` command
+```
+
+```release-note:bug
+cli: Fixed a bug where the `operator client-state` command would crash if it reads an allocation without a task state
+```

command/operator_client_state.go

@@ -70,10 +70,37 @@ func (c *OperatorClientStateCommand) Run(args []string) int {
 			return 1
 		}
 
+		identities, err := db.GetAllocIdentities(allocID)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("failed to get identities for %s: %v", allocID, err))
+			return 1
+		}
+
+		networks, err := db.GetNetworkStatus(allocID)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("failed to get networks for %s: %v", allocID, err))
+			return 1
+		}
+
+		volumes, err := db.GetAllocVolumes(allocID)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("failed to get volumes for %s: %v", allocID, err))
+			return 1
+		}
+
 		tasks := map[string]*taskState{}
 		tg := alloc.Job.LookupTaskGroup(alloc.TaskGroup)
 		for _, jt := range tg.Tasks {
 			ls, rs, err := db.GetTaskRunnerState(allocID, jt.Name)
+			if ls == nil {
+				c.Ui.Warn(fmt.Sprintf("no task runner state for %s (%s)", allocID, jt.Name))
+				tasks[jt.Name] = &taskState{
+					LocalState:  ls,
+					RemoteState: rs,
+					DriverState: nil,
+				}
+				continue
+			}
 			if err != nil {
 				c.Ui.Error(fmt.Sprintf("failed to get task runner state %s: %v", allocID, err))
 				return 1
@@ -99,6 +126,9 @@ func (c *OperatorClientStateCommand) Run(args []string) int {
 		data[allocID] = &clientStateAlloc{
 			Alloc:        alloc,
 			DeployStatus: deployState,
+			Identities:   identities,
+			Networks:     networks,
+			Volumes:      volumes,
 			Tasks:        tasks,
 		}
 	}
@@ -122,6 +152,9 @@ type debugOutput struct {
 type clientStateAlloc struct {
 	Alloc        any
 	DeployStatus any
+	Identities   any
+	Networks     any
+	Volumes      any
 	Tasks        map[string]*taskState
 }
 

command/operator_client_state_test.go

@@ -4,12 +4,14 @@
 package command
 
 import (
-	"strings"
 	"testing"
 
 	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/client/state"
+	"github.com/hashicorp/nomad/helper/testlog"
+	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/mitchellh/cli"
-	"github.com/stretchr/testify/require"
+	"github.com/shoenig/test/must"
 )
 
 func TestOperatorClientStateCommand(t *testing.T) {
@@ -18,15 +20,28 @@ func TestOperatorClientStateCommand(t *testing.T) {
 	cmd := &OperatorClientStateCommand{Meta: Meta{Ui: ui}}
 
 	failedCode := cmd.Run([]string{"some", "bad", "args"})
-	require.Equal(t, 1, failedCode)
-	if out := ui.ErrorWriter.String(); !strings.Contains(out, commandErrorText(cmd)) {
-		t.Fatalf("expected help output, got: %s", out)
-	}
+	must.Eq(t, 1, failedCode)
+	out := ui.ErrorWriter.String()
+	must.StrContains(t, out, commandErrorText(cmd), must.Sprint("expected help output"))
 	ui.ErrorWriter.Reset()
 
 	dir := t.TempDir()
-	code := cmd.Run([]string{dir})
 
-	require.Equal(t, 0, code)
-	require.Contains(t, ui.OutputWriter.String(), "{}")
+	// run against an empty client state directory
+	code := cmd.Run([]string{dir})
+	must.Eq(t, 0, code)
+	must.StrContains(t, ui.OutputWriter.String(), "{}")
+
+	// create a minimal client state db
+	db, err := state.NewBoltStateDB(testlog.HCLogger(t), dir)
+	must.NoError(t, err)
+	alloc := structs.MockAlloc()
+	err = db.PutAllocation(alloc)
+	must.NoError(t, err)
+	must.NoError(t, db.Close())
+
+	// run against an incomplete client state directory
+	code = cmd.Run([]string{dir})
+	must.Eq(t, 0, code)
+	must.StrContains(t, ui.OutputWriter.String(), alloc.ID)
 }