Merge pull request #10808 from hashicorp/f-curl

cli: add operator api command

.changelog/10808.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+cli: Added `nomad operator api` command to ease querying Nomad's HTTP API.
+```

command/commands.go

@@ -496,6 +496,12 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory {
 			}, nil
 		},
 
+		"operator api": func() (cli.Command, error) {
+			return &OperatorAPICommand{
+				Meta: meta,
+			}, nil
+		},
+
 		"operator autopilot": func() (cli.Command, error) {
 			return &OperatorAutopilotCommand{
 				Meta: meta,

command/operator_api.go

@@ -0,0 +1,449 @@
+package command
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/hashicorp/go-cleanhttp"
+	"github.com/hashicorp/nomad/api"
+	"github.com/posener/complete"
+)
+
+type OperatorAPICommand struct {
+	Meta
+
+	verboseFlag bool
+	method      string
+	body        io.Reader
+}
+
+func (*OperatorAPICommand) Help() string {
+	helpText := `
+Usage: nomad operator api [options] <path>
+
+  api is a utility command for accessing Nomad's HTTP API and is inspired by
+  the popular curl command line tool. Nomad's operator api command populates
+  Nomad's standard environment variables into their appropriate HTTP headers.
+  If the 'path' does not begin with "http" then $NOMAD_ADDR will be used. 
+
+  The 'path' can be in one of the following forms:
+
+    /v1/allocations                       <- API Paths must start with a /
+    localhost:4646/v1/allocations         <- Scheme will be inferred
+    https://localhost:4646/v1/allocations <- Scheme will be https://
+
+  Note that this command does not always match the popular curl program's
+  behavior. Instead Nomad's operator api command is optimized for common Nomad
+  HTTP API operations.
+
+General Options:
+
+  ` + generalOptionsUsage(usageOptsDefault) + `
+
+Operator API Specific Options:
+
+  -dryrun
+    Output equivalent curl command to stdout and exit.
+    HTTP Basic Auth will never be output. If the $NOMAD_HTTP_AUTH environment
+    variable is set, it will be referenced in the appropriate curl flag in the
+    output.
+    ACL tokens set via the $NOMAD_TOKEN environment variable will only be
+    referenced by environment variable as with HTTP Basic Auth above. However
+    if the -token flag is explicitly used, the token will also be included in
+    the output.
+
+  -filter <query>
+    Specifies an expression used to filter query results.
+
+  -H <Header>
+    Adds an additional HTTP header to the request. May be specified more than
+    once. These headers take precedence over automatically set ones such as
+    X-Nomad-Token.
+
+  -verbose
+    Output extra information to stderr similar to curl's --verbose flag.
+
+  -X <HTTP Method>
+    HTTP method of request. If there is data piped to stdin, then the method
+    defaults to POST. Otherwise the method defaults to GET.
+`
+
+	return strings.TrimSpace(helpText)
+}
+
+func (*OperatorAPICommand) Synopsis() string {
+	return "Query Nomad's HTTP API"
+}
+
+func (c *OperatorAPICommand) AutocompleteFlags() complete.Flags {
+	return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient),
+		complete.Flags{
+			"-dryrun": complete.PredictNothing,
+		})
+}
+
+func (c *OperatorAPICommand) AutocompleteArgs() complete.Predictor {
+	//TODO(schmichael) wouldn't it be cool to build path autocompletion off
+	//                 of our http mux?
+	return complete.PredictNothing
+}
+
+func (*OperatorAPICommand) Name() string { return "operator api" }
+
+func (c *OperatorAPICommand) Run(args []string) int {
+	var dryrun bool
+	var filter string
+	headerFlags := newHeaderFlags()
+
+	flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
+	flags.Usage = func() { c.Ui.Output(c.Help()) }
+	flags.BoolVar(&dryrun, "dryrun", false, "")
+	flags.StringVar(&filter, "filter", "", "")
+	flags.BoolVar(&c.verboseFlag, "verbose", false, "")
+	flags.StringVar(&c.method, "X", "", "")
+	flags.Var(headerFlags, "H", "")
+
+	if err := flags.Parse(args); err != nil {
+		c.Ui.Error(fmt.Sprintf("Error parsing flags: %v", err))
+		return 1
+	}
+	args = flags.Args()
+
+	if len(args) < 1 {
+		c.Ui.Error("A path or URL is required")
+		c.Ui.Error(commandErrorText(c))
+		return 1
+	}
+
+	if n := len(args); n > 1 {
+		c.Ui.Error(fmt.Sprintf("operator api accepts exactly 1 argument, but %d arguments were found", n))
+		c.Ui.Error(commandErrorText(c))
+		return 1
+	}
+
+	// By default verbose func is a noop
+	verbose := func(string, ...interface{}) {}
+	if c.verboseFlag {
+		verbose = func(format string, a ...interface{}) {
+			// Use Warn instead of Info because Info goes to stdout
+			c.Ui.Warn(fmt.Sprintf(format, a...))
+		}
+	}
+
+	// Opportunistically read from stdin and POST unless method has been
+	// explicitly set.
+	stat, _ := os.Stdin.Stat()
+	if (stat.Mode() & os.ModeCharDevice) == 0 {
+		verbose("* Reading request body from stdin.")
+		c.body = os.Stdin
+		if c.method == "" {
+			c.method = "POST"
+		}
+	} else if c.method == "" {
+		c.method = "GET"
+	}
+
+	config := c.clientConfig()
+
+	// NewClient mutates or validates Config.Address, so call it to match
+	// the behavior of other commands.
+	_, err := api.NewClient(config)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error initializing client: %v", err))
+		return 1
+	}
+
+	path, err := pathToURL(config, args[0])
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error turning path into URL: %v", err))
+		return 1
+	}
+
+	// Set Filter query param
+	if filter != "" {
+		q := path.Query()
+		q.Set("filter", filter)
+		path.RawQuery = q.Encode()
+	}
+
+	if dryrun {
+		out, err := c.apiToCurl(config, headerFlags.headers, path)
+		if err != nil {
+			c.Ui.Error(fmt.Sprintf("Error creating curl command: %v", err))
+			return 1
+		}
+		c.Ui.Output(out)
+		return 0
+	}
+
+	// Re-implement a big chunk of api/api.go since we don't export it.
+	client := cleanhttp.DefaultClient()
+	transport := client.Transport.(*http.Transport)
+	transport.TLSHandshakeTimeout = 10 * time.Second
+	transport.TLSClientConfig = &tls.Config{
+		MinVersion: tls.VersionTLS12,
+	}
+
+	if err := api.ConfigureTLS(client, config.TLSConfig); err != nil {
+		c.Ui.Error(fmt.Sprintf("Error configuring TLS: %v", err))
+		return 1
+	}
+
+	setQueryParams(config, path)
+
+	verbose("> %s %s", c.method, path)
+
+	req, err := http.NewRequest(c.method, path.String(), c.body)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error making request: %v", err))
+		return 1
+	}
+
+	// Set headers from command line
+	req.Header = headerFlags.headers
+
+	// Add token header if it doesn't already exist and is set
+	if req.Header.Get("X-Nomad-Token") == "" && config.SecretID != "" {
+		req.Header.Set("X-Nomad-Token", config.SecretID)
+	}
+
+	// Configure HTTP basic authentication if set
+	if path.User != nil {
+		username := path.User.Username()
+		password, _ := path.User.Password()
+		req.SetBasicAuth(username, password)
+	} else if config.HttpAuth != nil {
+		req.SetBasicAuth(config.HttpAuth.Username, config.HttpAuth.Password)
+	}
+
+	for k, vals := range req.Header {
+		for _, v := range vals {
+			verbose("> %s: %s", k, v)
+		}
+	}
+
+	verbose("* Sending request and receiving response...")
+
+	// Do the request!
+	resp, err := client.Do(req)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error performing request: %v", err))
+		return 1
+	}
+	defer resp.Body.Close()
+
+	verbose("< %s %s", resp.Proto, resp.Status)
+	for k, vals := range resp.Header {
+		for _, v := range vals {
+			verbose("< %s: %s", k, v)
+		}
+	}
+
+	n, err := io.Copy(os.Stdout, resp.Body)
+	if err != nil {
+		c.Ui.Error(fmt.Sprintf("Error reading response after %d bytes: %v", n, err))
+		return 1
+	}
+
+	if len(resp.Trailer) > 0 {
+		verbose("* Trailer Headers")
+		for k, vals := range resp.Trailer {
+			for _, v := range vals {
+				verbose("< %s: %s", k, v)
+			}
+		}
+	}
+
+	return 0
+}
+
+// setQueryParams converts API configuration to query parameters. Updates path
+// parameter in place.
+func setQueryParams(config *api.Config, path *url.URL) {
+	queryParams := path.Query()
+
+	// Prefer region explicitly set in path, otherwise fallback to config
+	// if one is set.
+	if queryParams.Get("region") == "" && config.Region != "" {
+		queryParams["region"] = []string{config.Region}
+	}
+
+	// Prefer namespace explicitly set in path, otherwise fallback to
+	// config if one is set.
+	if queryParams.Get("namespace") == "" && config.Namespace != "" {
+		queryParams["namespace"] = []string{config.Namespace}
+	}
+
+	// Re-encode query parameters
+	path.RawQuery = queryParams.Encode()
+}
+
+// apiToCurl converts a Nomad HTTP API config and path to its corresponding
+// curl command or returns an error.
+func (c *OperatorAPICommand) apiToCurl(config *api.Config, headers http.Header, path *url.URL) (string, error) {
+	parts := []string{"curl"}
+
+	if c.verboseFlag {
+		parts = append(parts, "--verbose")
+	}
+
+	if c.method != "" {
+		parts = append(parts, "-X "+c.method)
+	}
+
+	if c.body != nil {
+		parts = append(parts, "--data-binary @-")
+	}
+
+	if config.TLSConfig != nil {
+		parts = tlsToCurl(parts, config.TLSConfig)
+
+		// If a TLS server name is set we must alter the URL and use
+		// curl's --connect-to flag.
+		if v := config.TLSConfig.TLSServerName; v != "" {
+			pathHost, port, err := net.SplitHostPort(path.Host)
+			if err != nil {
+				return "", fmt.Errorf("error determining port: %v", err)
+			}
+
+			// curl uses the url for SNI so override it with the
+			// configured server name
+			path.Host = net.JoinHostPort(v, port)
+
+			// curl uses --connect-to to allow specifying a
+			// different connection address for the hostname in the
+			// path. The format is:
+			//   logical-host:logical-port:actual-host:actual-port
+			// Ports will always match since only the hostname is
+			// overridden for SNI.
+			parts = append(parts, fmt.Sprintf(`--connect-to "%s:%s:%s:%s"`,
+				v, port, pathHost, port))
+		}
+	}
+
+	// Add headers
+	for k, vals := range headers {
+		for _, v := range vals {
+			parts = append(parts, fmt.Sprintf(`-H '%s: %s'`, k, v))
+		}
+	}
+
+	// Only write NOMAD_TOKEN to stdout if it was specified via -token.
+	// Otherwise output a static string that references the ACL token
+	// environment variable.
+	if headers.Get("X-Nomad-Token") == "" {
+		if c.Meta.token != "" {
+			parts = append(parts, fmt.Sprintf(`-H 'X-Nomad-Token: %s'`, c.Meta.token))
+		} else if v := os.Getenv("NOMAD_TOKEN"); v != "" {
+			parts = append(parts, `-H "X-Nomad-Token: ${NOMAD_TOKEN}"`)
+		}
+	}
+
+	// Never write http auth to stdout. Instead output a static string that
+	// references the HTTP auth environment variable.
+	if auth := os.Getenv("NOMAD_HTTP_AUTH"); auth != "" {
+		parts = append(parts, `-u "$NOMAD_HTTP_AUTH"`)
+	}
+
+	setQueryParams(config, path)
+
+	parts = append(parts, path.String())
+
+	return strings.Join(parts, " \\\n  "), nil
+}
+
+// tlsToCurl converts TLS configuration to their corresponding curl flags.
+func tlsToCurl(parts []string, tlsConfig *api.TLSConfig) []string {
+	if v := tlsConfig.CACert; v != "" {
+		parts = append(parts, fmt.Sprintf(`--cacert "%s"`, v))
+	}
+
+	if v := tlsConfig.CAPath; v != "" {
+		parts = append(parts, fmt.Sprintf(`--capath "%s"`, v))
+	}
+
+	if v := tlsConfig.ClientCert; v != "" {
+		parts = append(parts, fmt.Sprintf(`--cert "%s"`, v))
+	}
+
+	if v := tlsConfig.ClientKey; v != "" {
+		parts = append(parts, fmt.Sprintf(`--key "%s"`, v))
+	}
+
+	// TLSServerName has already been configured as it may change the path.
+
+	if tlsConfig.Insecure {
+		parts = append(parts, `--insecure`)
+	}
+
+	return parts
+}
+
+// pathToURL converts a curl path argumet to URL. Paths without a host are
+// prefixed with $NOMAD_ADDR or http://127.0.0.1:4646.
+func pathToURL(config *api.Config, path string) (*url.URL, error) {
+	// If the scheme is missing, add it
+	if !strings.HasPrefix(path, "http://") && !strings.HasPrefix(path, "https://") {
+		scheme := "http"
+		if config.TLSConfig != nil {
+			if config.TLSConfig.CACert != "" ||
+				config.TLSConfig.CAPath != "" ||
+				config.TLSConfig.ClientCert != "" ||
+				config.TLSConfig.TLSServerName != "" ||
+				config.TLSConfig.Insecure {
+
+				// TLS configured, but scheme not set. Assume
+				// https.
+				scheme = "https"
+			}
+		}
+
+		path = fmt.Sprintf("%s://%s", scheme, path)
+	}
+
+	u, err := url.Parse(path)
+	if err != nil {
+		return nil, err
+	}
+
+	// If URL.Scheme is empty, use defaults from client config
+	if u.Host == "" {
+		confURL, err := url.Parse(config.Address)
+		if err != nil {
+			return nil, fmt.Errorf("Unable to parse configured address: %v", err)
+		}
+		u.Host = confURL.Host
+	}
+
+	return u, nil
+}
+
+// headerFlags is a flag.Value implementation for collecting multiple -H flags.
+type headerFlags struct {
+	headers http.Header
+}
+
+func newHeaderFlags() *headerFlags {
+	return &headerFlags{
+		headers: make(http.Header),
+	}
+}
+
+func (*headerFlags) String() string { return "" }
+
+func (h *headerFlags) Set(v string) error {
+	parts := strings.SplitN(v, ":", 2)
+	if len(parts) != 2 {
+		return fmt.Errorf("Headers must be in the form 'Key: Value' but found: %q", v)
+	}
+
+	h.headers.Add(parts[0], strings.TrimSpace(parts[1]))
+	return nil
+}

command/operator_api_test.go

@@ -0,0 +1,103 @@
+package command
+
+import (
+	"bytes"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/mitchellh/cli"
+	"github.com/stretchr/testify/require"
+)
+
+// TestOperatorAPICommand_Paths asserts that the op api command normalizes
+// various path formats to the proper full address.
+func TestOperatorAPICommand_Paths(t *testing.T) {
+	hits := make(chan *url.URL, 1)
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits <- r.URL
+	}))
+	defer ts.Close()
+
+	// Always expect the same URL to be hit
+	expected := "/v1/jobs"
+
+	buf := bytes.NewBuffer(nil)
+	ui := &cli.BasicUi{
+		ErrorWriter: buf,
+		Writer:      buf,
+	}
+	cmd := &OperatorAPICommand{Meta: Meta{Ui: ui}}
+
+	// Assert that absolute paths are appended to the configured address
+	exitCode := cmd.Run([]string{"-address=" + ts.URL, "/v1/jobs"})
+	require.Zero(t, exitCode, buf.String())
+
+	select {
+	case hit := <-hits:
+		require.Equal(t, expected, hit.String())
+	case <-time.After(10 * time.Second):
+		t.Fatalf("timed out waiting for hit")
+	}
+
+	buf.Reset()
+
+	// Assert that full URLs are used as-is even if an invalid address is
+	// set.
+	exitCode = cmd.Run([]string{"-address=ftp://127.0.0.2:1", ts.URL + "/v1/jobs"})
+	require.Zero(t, exitCode, buf.String())
+
+	select {
+	case hit := <-hits:
+		require.Equal(t, expected, hit.String())
+	case <-time.After(10 * time.Second):
+		t.Fatalf("timed out waiting for hit")
+	}
+
+	buf.Reset()
+
+	// Assert that URLs lacking a scheme are used even if an invalid
+	// address is set.
+	exitCode = cmd.Run([]string{"-address=ftp://127.0.0.2:1", ts.Listener.Addr().String() + "/v1/jobs"})
+	require.Zero(t, exitCode, buf.String())
+
+	select {
+	case hit := <-hits:
+		require.Equal(t, expected, hit.String())
+	case <-time.After(10 * time.Second):
+		t.Fatalf("timed out waiting for hit")
+	}
+}
+
+// TestOperatorAPICommand_Curl asserts that -dryrun outputs a valid curl
+// command.
+func TestOperatorAPICommand_Curl(t *testing.T) {
+	buf := bytes.NewBuffer(nil)
+	ui := &cli.BasicUi{
+		ErrorWriter: buf,
+		Writer:      buf,
+	}
+	cmd := &OperatorAPICommand{Meta: Meta{Ui: ui}}
+
+	exitCode := cmd.Run([]string{
+		"-dryrun",
+		"-address=http://127.0.0.1:1",
+		"-region=not even a valid region",
+		`-filter=this == "that" or this != "foo"`,
+		"-X", "POST",
+		"-token=acl-token",
+		"-H", "Some-Other-Header: ok",
+		"/url",
+	})
+	require.Zero(t, exitCode, buf.String())
+
+	expected := `curl \
+  -X POST \
+  -H 'Some-Other-Header: ok' \
+  -H 'X-Nomad-Token: acl-token' \
+  http://127.0.0.1:1/url?filter=this+%3D%3D+%22that%22+or+this+%21%3D+%22foo%22&region=not+even+a+valid+region
+`
+	require.Equal(t, expected, buf.String())
+}

website/content/docs/commands/operator/api.mdx

@@ -0,0 +1,95 @@
+---
+layout: docs
+page_title: 'Commands: operator api'
+description: |
+  operator api is a utility command for accessing Nomad's HTTP API similar to
+  the popular open source program curl.
+---
+
+# Command: operator api
+
+The `operator api` command allows easy access to Nomad's HTTP API similar to
+the popular [curl] program. Nomad's `operator api` command reads [environment
+variables][envvars] to dramatically ease HTTP API access compared to trying to
+manually write the same command with the third party `curl` command.
+
+For example for the following environment:
+
+```
+NOMAD_TOKEN=d4434353-c797-19e4-a14d-4068241f86a4
+NOMAD_CACERT=$HOME/.nomad/ca.pem
+NOMAD_CLIENT_CERT=$HOME/.nomad/cli.pem
+NOMAD_CLIENT_KEY=$HOME/.nomad/client-key.pem
+NOMAD_TLS_SERVER_NAME=client.global.nomad
+NOMAD_ADDR=https://remote.client123.internal:4646
+```
+
+Accessing Nomad's [`/v1/jobs`][jobs] HTTP endpoint with `nomad operator
+api` would require:
+
+```
+nomad operator api /v1/jobs
+```
+
+Performing the same request using the external `curl` tool would require:
+
+```
+curl \
+  --cacert "$HOME/.nomad/ca.pem" \
+  --cert "$HOME/.nomad/client.pem" \
+  --key "$HOME/.nomad/client-key.pem" \
+  --connect-to "client.global.nomad:4646:remote.client123.internal:4646" \
+  -H "X-Nomad-Token: ${NOMAD_TOKEN}" \
+  https://client.global.nomad:4646/v1/jobs
+```
+
+## General Options
+
+@include 'general_options.mdx'
+
+## Operator API Options
+
+- `-dryrun`: output a curl command instead of performing the HTTP request
+  immediately. Note that you do *not* need the 3rd party `curl` command
+  installed to use `operator api`. The `curl` output from `-dryrun` is intended
+  for use in scripts or running in locations without a Nomad binary present.
+
+- `-filter`: Specifies an expression used to filter query results.
+
+- `-H`: Adds an additional HTTP header to the request. May be specified more
+  than once. These headers take precedence over automatically set ones such as
+  X-Nomad-Token.
+
+- `-verbose`: Output extra information to stderr similar to curl's --verbose
+  flag.
+
+- `-X`: HTTP method of request. If there is data piped to stdin, then the
+  method defaults to POST. Otherwise the method defaults to GET.
+
+## Examples
+
+```shell-session
+$ nomad operator api -verbose /v1/agent/members?pretty
+> GET http://127.0.0.1:4646/v1/agent/members?pretty=
+* Sending request and receiving response...
+< HTTP/1.1 200 OK
+< Date: Wed, 02 Mar 2022 01:10:59 GMT
+< Content-Type: application/json
+< Vary: Accept-Encoding
+{
+    "Members": [
+...
+
+
+$ nomad operator api -region eu-west -filter 'Status == "completed"' -dryrun /v1/evaluations
+curl \
+  -X GET \
+    http://127.0.0.1:4646/v1/evaluations?filter=.Status+%3D%3D+%22completed%22&region=eu-west
+```
+
+
+
+
+[curl]: https://curl.se/
+[envvars]: /docs/commands#environment-variables
+[jobs]: /api-docs/jobs

website/data/docs-nav-data.json

@@ -537,6 +537,10 @@
             "title": "Overview",
             "path": "commands/operator"
           },
+          {
+            "title": "api",
+            "path": "commands/operator/api"
+          },
           {
             "title": "autopilot get-config",
             "path": "commands/operator/autopilot-get-config"