autopilot: add operator autopilot health command (#20156)

Add a command line operation that reports Enterprise autopilot data from the
`/operator/autopilot/health` API. I've pulled this feature out of
@lindleywhite's PR in the Enterprise repo.

Ref: https://github.com/hashicorp/nomad-enterprise/pull/1394

Co-authored-by: Lindley <lindley@hashicorp.com>

.changelog/20156.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+autopilot: Added `operator autopilot health` command to review Autopilot health data
+```

command/commands.go

@@ -695,6 +695,11 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory {
 				Meta: meta,
 			}, nil
 		},
+		"operator autopilot health": func() (cli.Command, error) {
+			return &OperatorAutopilotHealthCommand{
+				Meta: meta,
+			}, nil
+		},
 
 		"operator client-state": func() (cli.Command, error) {
 			return &OperatorClientStateCommand{

command/operator_autopilot_health.go

@@ -0,0 +1,188 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package command
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/nomad/api"
+	"github.com/posener/complete"
+)
+
+type OperatorAutopilotHealthCommand struct {
+	Meta
+}
+
+func (c *OperatorAutopilotHealthCommand) AutocompleteFlags() complete.Flags {
+	return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient))
+}
+
+func (c *OperatorAutopilotHealthCommand) AutocompleteArgs() complete.Predictor {
+	return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient),
+		complete.Flags{
+			"-json": complete.PredictNothing,
+		})
+}
+
+func (c *OperatorAutopilotHealthCommand) Name() string { return "operator autopilot health" }
+func (c *OperatorAutopilotHealthCommand) Run(args []string) int {
+	var fJson bool
+	flags := c.Meta.FlagSet("autopilot", FlagSetClient)
+	flags.Usage = func() { c.Ui.Output(c.Help()) }
+	flags.BoolVar(&fJson, "json", false, "")
+
+	if err := flags.Parse(args); err != nil {
+		c.Ui.Error(fmt.Sprintf("Failed to parse args: %v", err))
+		return 1
+	}
+
+	// Set up a client.
+	client, err := c.Meta.Client()
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err))
+		return 1
+	}
+
+	// Fetch the current configuration.
+	state, _, err := client.Operator().AutopilotServerHealth(nil)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error querying Autopilot configuration: %s", err))
+		return 1
+	}
+	if fJson {
+		bytes, err := json.Marshal(state)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("failed to serialize client state: %v", err))
+			return 1
+		}
+		c.Ui.Output(string(bytes))
+	}
+
+	c.Ui.Output(formatAutopilotState(state))
+
+	return 0
+}
+
+func (c *OperatorAutopilotHealthCommand) Synopsis() string {
+	return "Display the current Autopilot health"
+}
+
+func (c *OperatorAutopilotHealthCommand) Help() string {
+	helpText := `
+Usage: nomad operator autopilot health [options]
+
+  Displays the current Autopilot state.
+
+  If ACLs are enabled, this command requires a token with the 'operator:read'
+  capability.
+
+General Options:
+
+Output Options:
+
+	-json
+	Output the autopilot health in JSON format.
+
+  ` + generalOptionsUsage(usageOptsDefault|usageOptsNoNamespace)
+
+	return strings.TrimSpace(helpText)
+}
+
+func formatAutopilotState(state *api.OperatorHealthReply) string {
+	var out string
+	out = fmt.Sprintf("Healthy: %t\n", state.Healthy)
+	out = out + fmt.Sprintf("FailureTolerance: %d\n", state.FailureTolerance)
+	out = out + fmt.Sprintf("Leader: %s\n", state.Leader)
+	out = out + fmt.Sprintf("Voters:  \n\t%s\n", renderServerIDList(state.Voters))
+	out = out + fmt.Sprintf("Servers: \n%s\n", formatServerHealth(state.Servers))
+
+	out = formatCommandToEnt(out, state)
+	return out
+}
+
+func formatVoters(voters []string) string {
+	out := make([]string, len(voters))
+	for i, p := range voters {
+		out[i] = fmt.Sprintf("\t%s", p)
+	}
+	return formatList(out)
+}
+
+func formatServerHealth(servers []api.ServerHealth) string {
+	out := make([]string, len(servers)+1)
+	out[0] = "ID|Name|Address|SerfStatus|Version|Leader|Voter|Healthy|LastContact|LastTerm|LastIndex|StableSince"
+	for i, p := range servers {
+		out[i+1] = fmt.Sprintf("%s|%s|%s|%s|%s|%t|%t|%t|%s|%d|%d|%s",
+			p.ID,
+			p.Name,
+			p.Address,
+			p.SerfStatus,
+			p.Version,
+			p.Leader,
+			p.Voter,
+			p.Healthy,
+			p.LastContact,
+			p.LastTerm,
+			p.LastIndex,
+			p.StableSince,
+		)
+	}
+	return formatList(out)
+}
+
+func renderServerIDList(ids []string) string {
+	rows := make([]string, len(ids))
+	for i, id := range ids {
+		rows[i] = fmt.Sprintf("\t%s", id)
+	}
+	return formatList(rows)
+}
+
+func formatCommandToEnt(out string, state *api.OperatorHealthReply) string {
+	if len(state.ReadReplicas) > 0 {
+		out = out + "\nReadReplicas:"
+		out = out + formatList(state.ReadReplicas)
+	}
+
+	if len(state.RedundancyZones) > 0 {
+		out = out + "\nRedundancyZones:"
+		for _, zone := range state.RedundancyZones {
+			out = out + fmt.Sprintf("  %v", zone)
+		}
+	}
+
+	if state.Upgrade != nil {
+		out = out + "Upgrade: \n"
+		out = out + fmt.Sprintf(" \tStatus: %v\n", state.Upgrade.Status)
+		out = out + fmt.Sprintf(" \tTargetVersion: %v\n", state.Upgrade.TargetVersion)
+		if len(state.Upgrade.TargetVersionVoters) > 0 {
+			out = out + fmt.Sprintf(" \tTargetVersionVoters: \n\t\t%s\n", renderServerIDList(state.Upgrade.TargetVersionVoters))
+		}
+		if len(state.Upgrade.TargetVersionNonVoters) > 0 {
+			out = out + fmt.Sprintf(" \tTargetVersionNonVoters: \n\t\t%s\n", renderServerIDList(state.Upgrade.TargetVersionNonVoters))
+		}
+		if len(state.Upgrade.TargetVersionReadReplicas) > 0 {
+			out = out + fmt.Sprintf(" \tTargetVersionReadReplicas: \n\t\t%s\n", renderServerIDList(state.Upgrade.TargetVersionReadReplicas))
+		}
+		if len(state.Upgrade.OtherVersionVoters) > 0 {
+			out = out + fmt.Sprintf(" \tOtherVersionVoters: \n\t\t%s\n", renderServerIDList(state.Upgrade.OtherVersionVoters))
+		}
+		if len(state.Upgrade.OtherVersionNonVoters) > 0 {
+			out = out + fmt.Sprintf(" \tOtherVersionNonVoters: \n\t\t%s\n", renderServerIDList(state.Upgrade.OtherVersionNonVoters))
+		}
+		if len(state.Upgrade.OtherVersionReadReplicas) > 0 {
+			out = out + fmt.Sprintf(" \tOtherVersionReadReplicas: \n\t\t%s\n", renderServerIDList(state.Upgrade.OtherVersionReadReplicas))
+		}
+		if len(state.Upgrade.RedundancyZones) > 0 {
+
+			out = out + " \tRedundancyZones:\n"
+			for _, zone := range state.Upgrade.RedundancyZones {
+				out = out + fmt.Sprintf("  \t\t%v", zone)
+			}
+		}
+	}
+	return out
+}

command/operator_autopilot_health_test.go

@@ -0,0 +1,33 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package command
+
+import (
+	"testing"
+
+	"github.com/hashicorp/nomad/ci"
+	"github.com/mitchellh/cli"
+	"github.com/shoenig/test/must"
+)
+
+func TestOperator_Autopilot_State_Implements(t *testing.T) {
+	ci.Parallel(t)
+	var _ cli.Command = &OperatorAutopilotHealthCommand{}
+}
+
+func TestOperatorAutopilotStateCommand(t *testing.T) {
+	ci.Parallel(t)
+	s, _, addr := testServer(t, false, nil)
+	defer s.Shutdown()
+
+	ui := cli.NewMockUi()
+	c := &OperatorAutopilotHealthCommand{Meta: Meta{Ui: ui}}
+	args := []string{"-address=" + addr}
+
+	code := c.Run(args)
+	must.Eq(t, 0, code, must.Sprintf("got error for exit code: %v", ui.ErrorWriter.String()))
+
+	out := ui.OutputWriter.String()
+	must.StrContains(t, out, "Healthy")
+}

website/content/docs/commands/operator/autopilot/health.mdx

@@ -0,0 +1,45 @@
+---
+layout: docs
+page_title: 'Commands: operator autopilot health'
+description: |
+  Display the current Autopilot internal health.
+---
+
+# Command: operator autopilot state
+
+The Autopilot operator command is used to view the current Autopilot
+state. See the [Autopilot Guide][] for more information about Autopilot.
+
+## Usage
+
+```plaintext
+nomad operator autopilot health [options]
+```
+
+If ACLs are enabled, this command requires a token with the `operator:read`
+capability.
+
+## General Options
+
+@include 'general_options_no_namespace.mdx'
+
+## Output Options
+
+- `-json`: Output the Autopilot health in unformatted JSON.
+
+The output will return like below, read about the output of the command in the [API docs][].
+
+```shell-session
+$ nomad operator autopilot health
+Healthy: true
+FailureTolerance: 0
+Leader: e349749b-3303-3ddf-959c-b5885a0e1f6e
+Voters:  
+        e349749b-3303-3ddf-959c-b5885a0e1f6e
+Servers: 
+ID                                    Name      Address         SerfStatus  Version   Leader  Voter  Healthy  LastContact  LastTerm  LastIndex  StableSince
+e349749b-3303-3ddf-959c-b5885a0e1f6e  node1     127.0.0.1:4647  alive       1.7.5     true    true   true     0s           2         14         2024-02-20 16:40:55 +0000 UTC
+```
+
+[autopilot guide]: /nomad/tutorials/manage-clusters/autopilot
+[api docs]: /nomad/api-docs/operator/autopilot#read-state

website/data/docs-nav-data.json

@@ -768,6 +768,10 @@
               {
                 "title": "set-config",
                 "path": "commands/operator/autopilot/set-config"
+              },
+              {
+                "title": "health",
+                "path": "commands/operator/autopilot/health"
               }
             ]
           },