From 39a3fd652c016b231d84be5c02f795db3c83888f Mon Sep 17 00:00:00 2001 From: Charlie Voiselle <464492+angrycub@users.noreply.github.com> Date: Fri, 9 Sep 2022 17:55:20 -0400 Subject: [PATCH] Vars: CLI commands for `var get`, `var put`, `var purge` (#14400) * Includes updates to `var init` --- api/variables.go | 26 +- command/commands.go | 19 +- command/var.go | 265 ++++++++++++++++++ command/var_get.go | 187 +++++++++++++ command/var_get_test.go | 201 ++++++++++++++ command/var_init.go | 91 ++++--- command/var_init_test.go | 120 +++++++++ command/var_list.go | 9 +- command/var_list_test.go | 9 +- command/var_purge.go | 121 +++++++++ command/var_purge_test.go | 133 +++++++++ command/var_put.go | 523 ++++++++++++++++++++++++++++++++++++ command/var_put_test.go | 123 +++++++++ command/var_test.go | 26 ++ nomad/variables_endpoint.go | 5 +- 15 files changed, 1791 insertions(+), 67 deletions(-) create mode 100644 command/var_get.go create mode 100644 command/var_get_test.go create mode 100644 command/var_init_test.go create mode 100644 command/var_purge.go create mode 100644 command/var_purge_test.go create mode 100644 command/var_put.go create mode 100644 command/var_put_test.go create mode 100644 command/var_test.go diff --git a/api/variables.go b/api/variables.go index 23fe2dfe3..a88c5325e 100644 --- a/api/variables.go +++ b/api/variables.go @@ -313,36 +313,36 @@ func (sv *Variables) writeChecked(endpoint string, in *Variable, out *Variable, // encrypted Nomad backend. type Variable struct { // Namespace is the Nomad namespace associated with the variable - Namespace string + Namespace string `hcl:"namespace"` // Path is the path to the variable - Path string + Path string `hcl:"path"` // Raft indexes to track creation and modification - CreateIndex uint64 - ModifyIndex uint64 + CreateIndex uint64 `hcl:"create_index"` + ModifyIndex uint64 `hcl:"modify_index"` // Times provided as a convenience for operators expressed time.UnixNanos - CreateTime int64 - ModifyTime int64 + CreateTime int64 `hcl:"create_time"` + ModifyTime int64 `hcl:"modify_time"` - Items VariableItems + Items VariableItems `hcl:"items"` } // VariableMetadata specifies the metadata for a variable and // is used as the list object type VariableMetadata struct { // Namespace is the Nomad namespace associated with the variable - Namespace string + Namespace string `hcl:"namespace"` // Path is the path to the variable - Path string + Path string `hcl:"path"` // Raft indexes to track creation and modification - CreateIndex uint64 - ModifyIndex uint64 + CreateIndex uint64 `hcl:"create_index"` + ModifyIndex uint64 `hcl:"modify_index"` // Times provided as a convenience for operators expressed time.UnixNanos - CreateTime int64 - ModifyTime int64 + CreateTime int64 `hcl:"create_time"` + ModifyTime int64 `hcl:"modify_time"` } type VariableItems map[string]string diff --git a/command/commands.go b/command/commands.go index 31d118c6a..34da7ad90 100644 --- a/command/commands.go +++ b/command/commands.go @@ -936,8 +936,8 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, - "var list": func() (cli.Command, error) { - return &VarListCommand{ + "var purge": func() (cli.Command, error) { + return &VarPurgeCommand{ Meta: meta, }, nil }, @@ -946,6 +946,21 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, + "var list": func() (cli.Command, error) { + return &VarListCommand{ + Meta: meta, + }, nil + }, + "var put": func() (cli.Command, error) { + return &VarPutCommand{ + Meta: meta, + }, nil + }, + "var get": func() (cli.Command, error) { + return &VarGetCommand{ + Meta: meta, + }, nil + }, "version": func() (cli.Command, error) { return &VersionCommand{ Version: version.GetVersion(), diff --git a/command/var.go b/command/var.go index dbb3fb20c..455827095 100644 --- a/command/var.go +++ b/command/var.go @@ -1,10 +1,23 @@ package command import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "sort" "strings" + "text/template" + "time" + "github.com/hashicorp/nomad/api" "github.com/hashicorp/nomad/api/contexts" "github.com/mitchellh/cli" + "github.com/mitchellh/colorstring" + "github.com/mitchellh/mapstructure" "github.com/posener/complete" ) @@ -41,6 +54,10 @@ Usage: nomad var [options] [args] $ nomad var list + Purge a variable: + + $ nomad var purge + Please see the individual subcommand help for detailed usage information. ` @@ -72,3 +89,251 @@ func VariablePathPredictor(factory ApiClientFactory) complete.Predictor { return resp.Matches[contexts.Variables] }) } + +type VarUI interface { + GetConcurrentUI() cli.ConcurrentUi + Colorize() *colorstring.Colorize +} + +// renderSVAsUiTable prints a variable as a table. It needs access to the +// command to get access to colorize and the UI itself. Commands that call it +// need to implement the VarUI interface. +func renderSVAsUiTable(sv *api.Variable, c VarUI) { + meta := []string{ + fmt.Sprintf("Namespace|%s", sv.Namespace), + fmt.Sprintf("Path|%s", sv.Path), + fmt.Sprintf("Create Time|%v", formatUnixNanoTime(sv.ModifyTime)), + } + if sv.CreateTime != sv.ModifyTime { + meta = append(meta, fmt.Sprintf("Modify Time|%v", time.Unix(0, sv.ModifyTime))) + } + meta = append(meta, fmt.Sprintf("Check Index|%v", sv.ModifyIndex)) + ui := c.GetConcurrentUI() + ui.Output(formatKV(meta)) + ui.Output(c.Colorize().Color("\n[bold]Items[reset]")) + items := make([]string, 0, len(sv.Items)) + + keys := make([]string, 0, len(sv.Items)) + for k := range sv.Items { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + items = append(items, fmt.Sprintf("%s|%s", k, sv.Items[k])) + } + ui.Output(formatKV(items)) +} + +func renderAsHCL(sv *api.Variable) string { + const tpl = ` +namespace = "{{.Namespace}}" +path = "{{.Path}}" +create_index = {{.CreateIndex}} # Set by server +modify_index = {{.ModifyIndex}} # Set by server; consulted for check-and-set +create_time = {{.CreateTime}} # Set by server +modify_time = {{.ModifyTime}} # Set by server + +items = { +{{- $PAD := 0 -}}{{- range $k,$v := .Items}}{{if gt (len $k) $PAD}}{{$PAD = (len $k)}}{{end}}{{end -}} +{{- $FMT := printf " %%%vs = %%q\n" $PAD}} +{{range $k,$v := .Items}}{{printf $FMT $k $v}}{{ end -}} +} +` + out, err := renderWithGoTemplate(sv, tpl) + if err != nil { + // Any errors in this should be caught as test panics. + // If we ship with one, the worst case is that it panics a single + // run of the CLI and only for output of variables in HCL. + panic(err) + } + return out +} + +func renderWithGoTemplate(sv *api.Variable, tpl string) (string, error) { + //TODO: Enhance this to take a template as an @-aliased filename too + t := template.Must(template.New("var").Parse(tpl)) + var out bytes.Buffer + if err := t.Execute(&out, sv); err != nil { + return "", err + } + + result := out.String() + return result, nil +} + +// KVBuilder is a struct to build a key/value mapping based on a list +// of "k=v" pairs, where the value might come from stdin, a file, etc. +type KVBuilder struct { + Stdin io.Reader + + result map[string]interface{} + stdin bool +} + +// Map returns the built map. +func (b *KVBuilder) Map() map[string]interface{} { + return b.result +} + +// Add adds to the mapping with the given args. +func (b *KVBuilder) Add(args ...string) error { + for _, a := range args { + if err := b.add(a); err != nil { + return fmt.Errorf("invalid key/value pair %q: %w", a, err) + } + } + + return nil +} + +func (b *KVBuilder) add(raw string) error { + // Regardless of validity, make sure we make our result + if b.result == nil { + b.result = make(map[string]interface{}) + } + + // Empty strings are fine, just ignored + if raw == "" { + return nil + } + + // Split into key/value + parts := strings.SplitN(raw, "=", 2) + + // If the arg is exactly "-", then we need to read from stdin + // and merge the results into the resulting structure. + if len(parts) == 1 { + if raw == "-" { + if b.Stdin == nil { + return fmt.Errorf("stdin is not supported") + } + if b.stdin { + return fmt.Errorf("stdin already consumed") + } + + b.stdin = true + return b.addReader(b.Stdin) + } + + // If the arg begins with "@" then we need to read a file directly + if raw[0] == '@' { + f, err := os.Open(raw[1:]) + if err != nil { + return err + } + defer f.Close() + + return b.addReader(f) + } + } + + if len(parts) != 2 { + return fmt.Errorf("format must be key=value") + } + key, value := parts[0], parts[1] + + if len(value) > 0 { + if value[0] == '@' { + contents, err := ioutil.ReadFile(value[1:]) + if err != nil { + return fmt.Errorf("error reading file: %w", err) + } + + value = string(contents) + } else if value[0] == '\\' && value[1] == '@' { + value = value[1:] + } else if value == "-" { + if b.Stdin == nil { + return fmt.Errorf("stdin is not supported") + } + if b.stdin { + return fmt.Errorf("stdin already consumed") + } + b.stdin = true + + var buf bytes.Buffer + if _, err := io.Copy(&buf, b.Stdin); err != nil { + return err + } + + value = buf.String() + } + } + + // Repeated keys will be converted into a slice + if existingValue, ok := b.result[key]; ok { + var sliceValue []interface{} + if err := mapstructure.WeakDecode(existingValue, &sliceValue); err != nil { + return err + } + sliceValue = append(sliceValue, value) + b.result[key] = sliceValue + return nil + } + + b.result[key] = value + return nil +} + +func (b *KVBuilder) addReader(r io.Reader) error { + if r == nil { + return fmt.Errorf("'io.Reader' being decoded is nil") + } + + dec := json.NewDecoder(r) + // While decoding JSON values, interpret the integer values as + // `json.Number`s instead of `float64`. + dec.UseNumber() + + return dec.Decode(&b.result) +} + +// handleCASError provides consistent output for operations that result in a +// check-and-set error +func handleCASError(err error, c VarUI) (handled bool) { + ui := c.GetConcurrentUI() + var cErr api.ErrCASConflict + if errors.As(err, &cErr) { + lastUpdate := "" + if cErr.Conflict.ModifyIndex > 0 { + lastUpdate = fmt.Sprintf( + tidyRawString(msgfmtCASConflictLastAccess), + formatUnixNanoTime(cErr.Conflict.ModifyTime)) + } + ui.Error(c.Colorize().Color("\n[bold][underline]Check-and-Set conflict[reset]\n")) + ui.Warn( + wrapAndPrepend( + c.Colorize().Color( + fmt.Sprintf( + tidyRawString(msgfmtCASMismatch), + cErr.CheckIndex, + cErr.Conflict.ModifyIndex, + lastUpdate), + ), + 80, " ") + "\n", + ) + handled = true + } + return +} + +const ( + errMissingTemplate = `A template must be supplied using '-template' when using go-template formatting` + errUnexpectedTemplate = `The '-template' flag is only valid when using 'go-template' formatting` + errVariableNotFound = `Variable not found` + errInvalidInFormat = `Invalid value for "-in"; valid values are [hcl, json]` + errInvalidOutFormat = `Invalid value for "-out"; valid values are [go-template, hcl, json, none, table]` + errWildcardNamespaceNotAllowed = `The wildcard namespace ("*") is not valid for this command.` + + msgfmtCASMismatch = ` + Your provided check-index [green](%v)[yellow] does not match the + server-side index [green](%v)[yellow]. + %s + If you are sure you want to perform this operation, add the [green]-force[yellow] or + [green]-check-index=%[2]v[yellow] flag before the positional arguments.` + + msgfmtCASConflictLastAccess = ` + The server-side item was last updated on [green]%s[yellow]. + ` +) diff --git a/command/var_get.go b/command/var_get.go new file mode 100644 index 000000000..e906b438e --- /dev/null +++ b/command/var_get.go @@ -0,0 +1,187 @@ +package command + +import ( + "errors" + "fmt" + "os" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/mitchellh/cli" + "github.com/posener/complete" +) + +type VarGetCommand struct { + Meta + outFmt string + tmpl string +} + +func (c *VarGetCommand) Help() string { + helpText := ` +Usage: nomad var get [options] + + The 'var get' command is used to get the contents of an existing variable. + + If ACLs are enabled, this command requires a token with the 'variables:read' + capability for the target variable's namespace. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault) + ` + +Read Options: + + -item + Print only the value of the given item. Specifying this option will + take precedence over other formatting directives. The result will not + have a trailing newline making it ideal for piping to other processes. + + -out ( go-template | hcl | json | none | table ) + Format to render the variable in. When using "go-template", you must + provide the template content with the "-template" option. Defaults + to "table" when stdout is a terminal and to "json" when stdout is + redirected. + + -template + Template to render output with. Required when output is "go-template". + +` + return strings.TrimSpace(helpText) +} + +func (c *VarGetCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{ + "-out": complete.PredictSet("go-template", "hcl", "json", "none", "table"), + "-template": complete.PredictAnything, + }, + ) +} + +func (c *VarGetCommand) AutocompleteArgs() complete.Predictor { + return VariablePathPredictor(c.Meta.Client) +} + +func (c *VarGetCommand) Synopsis() string { + return "Read a variable" +} + +func (c *VarGetCommand) Name() string { return "var read" } + +func (c *VarGetCommand) Run(args []string) int { + var out, item string + + flags := c.Meta.FlagSet(c.Name(), FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + + flags.StringVar(&item, "item", "", "") + flags.StringVar(&c.tmpl, "template", "", "") + + if fileInfo, _ := os.Stdout.Stat(); (fileInfo.Mode() & os.ModeCharDevice) != 0 { + flags.StringVar(&c.outFmt, "out", "table", "") + } else { + flags.StringVar(&c.outFmt, "out", "json", "") + } + + if err := flags.Parse(args); err != nil { + return 1 + } + + // Check that we got one argument + args = flags.Args() + if len(args) != 1 { + c.Ui.Error("This command takes one argument: ") + c.Ui.Error(commandErrorText(c)) + return 1 + } + + if err := c.validateOutputFlag(); err != nil { + c.Ui.Error(err.Error()) + c.Ui.Error(commandErrorText(c)) + return 1 + } + + if c.Meta.namespace == "*" { + c.Ui.Error(errWildcardNamespaceNotAllowed) + return 1 + } + + path := args[0] + + // Get the HTTP client + client, err := c.Meta.Client() + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + qo := &api.QueryOptions{ + Namespace: c.Meta.namespace, + } + + sv, _, err := client.Variables().Read(path, qo) + if err != nil { + if err.Error() == "variable not found" { + c.Ui.Warn(errVariableNotFound) + return 1 + } + c.Ui.Error(fmt.Sprintf("Error retrieving variable: %s", err)) + return 1 + } + // If the user provided an item key, return that value instead of the whole + // object + if item != "" { + if v, ok := sv.Items[item]; ok { + fmt.Print(v) + return 0 + } else { + c.Ui.Error(fmt.Sprintf("Variable does not contain %q item", args[1])) + return 1 + } + } + + // Output whole object + switch c.outFmt { + case "json": + out = sv.AsPrettyJSON() + case "hcl": + out = renderAsHCL(sv) + case "go-template": + if out, err = renderWithGoTemplate(sv, c.tmpl); err != nil { + c.Ui.Error(err.Error()) + return 1 + } + case "none": + // exit without more output + return 0 + default: + // the renderSVAsUiTable func writes directly to the ui and doesn't error. + renderSVAsUiTable(sv, c) + return 0 + } + + c.Ui.Output(out) + return 0 +} + +func (c *VarGetCommand) validateOutputFlag() error { + if c.outFmt != "go-template" && c.tmpl != "" { + return errors.New(errUnexpectedTemplate) + } + switch c.outFmt { + case "hcl", "json", "none", "table": + return nil + case "go-template": //noop - needs more validation + if c.tmpl == "" { + return errors.New(errMissingTemplate) + } + return nil + default: + return errors.New(errInvalidOutFormat) + } +} + +func (c *VarGetCommand) GetConcurrentUI() cli.ConcurrentUi { + return cli.ConcurrentUi{Ui: c.Ui} +} diff --git a/command/var_get_test.go b/command/var_get_test.go new file mode 100644 index 000000000..d13df47f3 --- /dev/null +++ b/command/var_get_test.go @@ -0,0 +1,201 @@ +package command + +import ( + "fmt" + "strings" + "testing" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/ci" + "github.com/mitchellh/cli" + "github.com/posener/complete" + "github.com/stretchr/testify/require" +) + +func TestVarGetCommand_Implements(t *testing.T) { + ci.Parallel(t) + var _ cli.Command = &VarGetCommand{} +} + +func TestVarGetCommand_Fails(t *testing.T) { + ci.Parallel(t) + t.Run("bad_args", func(t *testing.T) { + ci.Parallel(t) + ui := cli.NewMockUi() + cmd := &VarGetCommand{Meta: Meta{Ui: ui}} + code := cmd.Run([]string{"some", "bad", "args"}) + out := ui.ErrorWriter.String() + require.Equal(t, 1, code, "expected exit code 1, got: %d") + require.Contains(t, out, commandErrorText(cmd), "expected help output, got: %s", out) + }) + t.Run("bad_address", func(t *testing.T) { + ci.Parallel(t) + ui := cli.NewMockUi() + cmd := &VarGetCommand{Meta: Meta{Ui: ui}} + code := cmd.Run([]string{"-address=nope", "foo"}) + out := ui.ErrorWriter.String() + require.Equal(t, 1, code, "expected exit code 1, got: %d") + require.Contains(t, ui.ErrorWriter.String(), "retrieving variable", "connection error, got: %s", out) + require.Zero(t, ui.OutputWriter.String()) + }) + t.Run("missing_template", func(t *testing.T) { + ci.Parallel(t) + ui := cli.NewMockUi() + cmd := &VarGetCommand{Meta: Meta{Ui: ui}} + code := cmd.Run([]string{`-out=go-template`, "foo"}) + out := strings.TrimSpace(ui.ErrorWriter.String()) + require.Equal(t, 1, code, "expected exit code 1, got: %d", code) + require.Equal(t, errMissingTemplate+"\n"+commandErrorText(cmd), out) + require.Zero(t, ui.OutputWriter.String()) + }) + t.Run("unexpected_template", func(t *testing.T) { + ci.Parallel(t) + ui := cli.NewMockUi() + cmd := &VarGetCommand{Meta: Meta{Ui: ui}} + code := cmd.Run([]string{`-out=json`, `-template="bad"`, "foo"}) + out := strings.TrimSpace(ui.ErrorWriter.String()) + require.Equal(t, 1, code, "expected exit code 1, got: %d", code) + require.Equal(t, errUnexpectedTemplate+"\n"+commandErrorText(cmd), out) + require.Zero(t, ui.OutputWriter.String()) + }) +} + +func TestVarGetCommand(t *testing.T) { + ci.Parallel(t) + + // Create a server + srv, client, url := testServer(t, true, nil) + t.Cleanup(func() { + srv.Shutdown() + }) + + testCases := []struct { + name string + format string + template string + expected string + testPath string // defaulted to "test/var" in code; used for not-found + exitCode int + isError bool + }{ + { + name: "json", + format: "json", + }, + { + name: "table", + format: "table", + }, + { + name: "go-template", + format: "go-template", + template: `{{.Namespace}}.{{.Path}}`, + expected: "TestVarGetCommand-2-go-template.test/var", + }, + { + name: "not-found", + format: "json", + expected: errVariableNotFound, + testPath: "not-found", + isError: true, + exitCode: 1, + }, + } + for i, tc := range testCases { + t.Run(fmt.Sprintf("%v_%s", i, tc.name), func(t *testing.T) { + tc := tc + ci.Parallel(t) + var err error + // Create a namespace for the test case + testNS := strings.Map(validNS, t.Name()) + _, err = client.Namespaces().Register(&api.Namespace{Name: testNS}, nil) + require.NoError(t, err) + t.Cleanup(func() { + client.Namespaces().Delete(testNS, nil) + }) + + // Create a var to get + sv := testVariable() + sv.Namespace = testNS + sv, _, err = client.Variables().Create(sv, nil) + require.NoError(t, err) + t.Cleanup(func() { + _, _ = client.Variables().Delete(sv.Path, nil) + }) + + // Build and run the command + ui := cli.NewMockUi() + cmd := &VarGetCommand{Meta: Meta{Ui: ui}} + args := []string{ + "-address=" + url, + "-namespace=" + testNS, + "-out=" + tc.format, + } + if tc.template != "" { + args = append(args, "-template="+tc.template) + } + args = append(args, sv.Path) + if tc.testPath != "" { + // replace path with test case override + args[len(args)-1] = tc.testPath + } + code := cmd.Run(args) + + // Check the output + require.Equal(t, tc.exitCode, code, "expected exit %v, got: %d; %v", tc.exitCode, code, ui.ErrorWriter.String()) + if tc.isError { + require.Equal(t, tc.expected, strings.TrimSpace(ui.ErrorWriter.String())) + return + } + switch tc.format { + case "json": + require.Equal(t, sv.AsPrettyJSON(), strings.TrimSpace(ui.OutputWriter.String())) + case "table": + out := ui.OutputWriter.String() + outs := strings.Split(out, "\n") + require.Len(t, outs, 9) + require.Equal(t, "Namespace = "+testNS, outs[0]) + require.Equal(t, "Path = test/var", outs[1]) + case "go-template": + require.Equal(t, tc.expected, strings.TrimSpace(ui.OutputWriter.String())) + default: + t.Fatalf("invalid format: %q", tc.format) + } + }) + } + t.Run("Autocomplete", func(t *testing.T) { + ci.Parallel(t) + _, client, url, shutdownFn := testAPIClient(t) + defer shutdownFn() + + ui := cli.NewMockUi() + cmd := &VarGetCommand{Meta: Meta{Ui: ui, flagAddress: url}} + + // Create a var + testNS := strings.Map(validNS, t.Name()) + _, err := client.Namespaces().Register(&api.Namespace{Name: testNS}, nil) + require.NoError(t, err) + t.Cleanup(func() { client.Namespaces().Delete(testNS, nil) }) + + sv := testVariable() + sv.Path = "special/variable" + sv.Namespace = t.Name() + sv, _, err = client.Variables().Create(sv, nil) + require.NoError(t, err) + t.Cleanup(func() { client.Variables().Delete(sv.Path, nil) }) + + args := complete.Args{Last: "s"} + predictor := cmd.AutocompleteArgs() + + res := predictor.Predict(args) + require.Equal(t, 1, len(res)) + require.Equal(t, sv.Path, res[0]) + }) +} + +func validNS(r rune) rune { + if r == '/' || r == '_' { + return '-' + } + return r +} diff --git a/command/var_init.go b/command/var_init.go index df7a69a55..89106dd75 100644 --- a/command/var_init.go +++ b/command/var_init.go @@ -1,8 +1,9 @@ package command import ( + "errors" "fmt" - "io/ioutil" + "io/fs" "os" "regexp" "strings" @@ -30,17 +31,16 @@ func (c *VarInitCommand) Help() string { helpText := ` Usage: nomad var init - Creates an example variable specification file that can be used as a - starting point to customize further. If no filename is given, the default of - "spec.nsv.hcl" or "spec.nsv.json" will be used. + Creates an example variable specification file that can be used as a starting + point to customize further. When no filename is supplied, a default filename + of "spec.nsv.hcl" or "spec.nsv.json" will be used depending on the output + format. Init Options: - -json - Create an example JSON variable specification. + -out (hcl | json) + Format of generated variable specification. Defaults to "hcl". - -q - Suppress non-error output ` return strings.TrimSpace(helpText) } @@ -51,7 +51,7 @@ func (c *VarInitCommand) Synopsis() string { func (c *VarInitCommand) AutocompleteFlags() complete.Flags { return complete.Flags{ - "-json": complete.PredictNothing, + "-out": complete.PredictSet("hcl", "json"), } } @@ -62,13 +62,12 @@ func (c *VarInitCommand) AutocompleteArgs() complete.Predictor { func (c *VarInitCommand) Name() string { return "var init" } func (c *VarInitCommand) Run(args []string) int { - var jsonOutput bool + var outFmt string var quiet bool flags := c.Meta.FlagSet(c.Name(), FlagSetClient) flags.Usage = func() { c.Ui.Output(c.Help()) } - flags.BoolVar(&jsonOutput, "json", false, "") - flags.BoolVar(&quiet, "q", false, "") + flags.StringVar(&outFmt, "out", "hcl", "") if err := flags.Parse(args); err != nil { return 1 @@ -81,30 +80,33 @@ func (c *VarInitCommand) Run(args []string) int { c.Ui.Error(commandErrorText(c)) return 1 } - - fileName := DefaultHclVarInitName - fileContent := defaultHclVarSpec - if jsonOutput { + var fileName, fileContent string + switch outFmt { + case "hcl": + fileName = DefaultHclVarInitName + fileContent = defaultHclVarSpec + case "json": fileName = DefaultJsonVarInitName fileContent = defaultJsonVarSpec } + if len(args) == 1 { fileName = args[0] } // Check if the file already exists _, err := os.Stat(fileName) - if err != nil && !os.IsNotExist(err) { - c.Ui.Error(fmt.Sprintf("Failed to stat %q: %v", fileName, err)) + if err == nil { + c.Ui.Error(fmt.Sprintf("File %q already exists", fileName)) return 1 } - if !os.IsNotExist(err) { - c.Ui.Error(fmt.Sprintf("File %q already exists", fileName)) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + c.Ui.Error(fmt.Sprintf("Failed to stat %q: %v", fileName, err)) return 1 } // Write out the example - err = ioutil.WriteFile(fileName, []byte(fileContent), 0660) + err = os.WriteFile(fileName, []byte(fileContent), 0660) if err != nil { c.Ui.Error(fmt.Sprintf("Failed to write %q: %v", fileName, err)) return 1 @@ -112,7 +114,10 @@ func (c *VarInitCommand) Run(args []string) int { // Success if !quiet { - c.Ui.Warn(WrapAndPrepend(TidyRawString(msgWarnKeys), 70, "")) + if outFmt == "json" { + c.Ui.Info(wrapString(tidyRawString(strings.ReplaceAll(msgOnlyItemsRequired, "items", "Items")), 70)) + c.Ui.Warn(wrapString(tidyRawString(strings.ReplaceAll(msgWarnKeys, "items", "Items")), 70)) + } c.Ui.Output(fmt.Sprintf("Example variable specification written to %s", fileName)) } return 0 @@ -123,6 +128,11 @@ const ( REMINDER: While keys in the items map can contain dots, using them in templates is easier when they do not. As a best practice, avoid dotted keys when possible.` + msgOnlyItemsRequired = ` + The items map is the only strictly required part of a variable + specification, since path and namespace can be set via other means. It + contains the sensitive material to encrypt and store as a Nomad variable. + The entire items map is encrypted and decrypted as a single unit.` ) var defaultHclVarSpec = strings.TrimSpace(` @@ -132,16 +142,14 @@ var defaultHclVarSpec = strings.TrimSpace(` # HTTP API endpoint # path = "path/to/variable" -# The Namespace to write the variable can be included in the specification -# and is the highest precedence way to set the namespace value. +# The Namespace to write the variable can be included in the specification. This +# value can be overridden by specifying the "-namespace" flag on the "put" +# command. # namespace = "default" -# The items map is the only strictly required part of a variable -# specification, since path and namespace can be set via other means. It -# contains the sensitive material to encrypt and store as a Nomad secure -# variable. The entire items map is encrypted and decrypted as a single unit. +`+makeHCLComment(msgOnlyItemsRequired)+` -`+warnInHCLFile()+` +`+makeHCLComment(msgWarnKeys)+` items { key1 = "value 1" key2 = "value 2" @@ -150,6 +158,8 @@ items { var defaultJsonVarSpec = strings.TrimSpace(` { + "Namespace": "default", + "Path": "path/to/variable", "Items": { "key1": "value 1", "key2": "value 2" @@ -157,34 +167,39 @@ var defaultJsonVarSpec = strings.TrimSpace(` } `) + "\n" -func warnInHCLFile() string { - return WrapAndPrepend(TidyRawString(msgWarnKeys), 70, "# ") +// makeHCLComment is a helper function that will take the contents of a raw +// string, tidy them, wrap them to 68 characters and add a leading comment +// marker plus a space. +func makeHCLComment(in string) string { + return wrapAndPrepend(tidyRawString(in), 70, "# ") } -// WrapString is a convienience func to abstract away the word wrapping +// wrapString is a convenience func to abstract away the word wrapping // implementation -func WrapString(input string, lineLen int) string { +func wrapString(input string, lineLen int) string { return wordwrap.String(input, lineLen) } -// WrapAndPrepend will word wrap the input string to lineLen characters and +// wrapAndPrepend will word wrap the input string to lineLen characters and // prepend the provided prefix to every line. The total length of each returned // line will be at most len(input[line])+len(prefix) -func WrapAndPrepend(input string, lineLen int, prefix string) string { - ss := strings.Split(wordwrap.String(input, lineLen), "\n") +func wrapAndPrepend(input string, lineLen int, prefix string) string { + ss := strings.Split(wrapString(input, lineLen-len(prefix)), "\n") prefixStringList(ss, prefix) return strings.Join(ss, "\n") } -// TidyRawString will convert a wrapped and indented raw string into a single +// tidyRawString will convert a wrapped and indented raw string into a single // long string suitable for rewrapping with another tool. It trims leading and // trailing whitespace and then consume groups of tabs, newlines, and spaces // replacing them with a single space -func TidyRawString(raw string) string { +func tidyRawString(raw string) string { re := regexp.MustCompile("[\t\n ]+") return re.ReplaceAllString(strings.TrimSpace(raw), " ") } +// prefixStringList is a helper function that prepends each item in a slice of +// string with a provided prefix. func prefixStringList(ss []string, prefix string) []string { for i, s := range ss { ss[i] = prefix + s diff --git a/command/var_init_test.go b/command/var_init_test.go new file mode 100644 index 000000000..cdf762255 --- /dev/null +++ b/command/var_init_test.go @@ -0,0 +1,120 @@ +package command + +import ( + "os" + "path" + "testing" + + "github.com/hashicorp/nomad/ci" + "github.com/mitchellh/cli" + "github.com/stretchr/testify/require" +) + +func TestVarInitCommand_Implements(t *testing.T) { + ci.Parallel(t) + var _ cli.Command = &VarInitCommand{} +} + +func TestVarInitCommand_Run(t *testing.T) { + ci.Parallel(t) + dir := t.TempDir() + origDir, err := os.Getwd() + require.NoError(t, err) + err = os.Chdir(dir) + require.NoError(t, err) + t.Cleanup(func() { os.Chdir(origDir) }) + + t.Run("hcl", func(t *testing.T) { + ci.Parallel(t) + dir := dir + ui := cli.NewMockUi() + cmd := &VarInitCommand{Meta: Meta{Ui: ui}} + + // Fails on misuse + ec := cmd.Run([]string{"some", "bad", "args"}) + require.Equal(t, 1, ec) + require.Contains(t, ui.ErrorWriter.String(), commandErrorText(cmd)) + require.Empty(t, ui.OutputWriter.String()) + reset(ui) + + // Works if the file doesn't exist + ec = cmd.Run([]string{"-out", "hcl"}) + require.Empty(t, ui.ErrorWriter.String()) + require.Equal(t, "Example variable specification written to spec.nsv.hcl\n", ui.OutputWriter.String()) + require.Zero(t, ec) + reset(ui) + t.Cleanup(func() { os.Remove(path.Join(dir, "spec.nsv.hcl")) }) + + content, err := os.ReadFile(DefaultHclVarInitName) + require.NoError(t, err) + require.Equal(t, defaultHclVarSpec, string(content)) + + // Fails if the file exists + ec = cmd.Run([]string{"-out", "hcl"}) + require.Contains(t, ui.ErrorWriter.String(), "exists") + require.Empty(t, ui.OutputWriter.String()) + require.Equal(t, 1, ec) + reset(ui) + + // Works if file is passed + ec = cmd.Run([]string{"-out", "hcl", "myTest.hcl"}) + require.Empty(t, ui.ErrorWriter.String()) + require.Equal(t, "Example variable specification written to myTest.hcl\n", ui.OutputWriter.String()) + require.Zero(t, ec) + reset(ui) + + t.Cleanup(func() { os.Remove(path.Join(dir, "myTest.hcl")) }) + content, err = os.ReadFile("myTest.hcl") + require.NoError(t, err) + require.Equal(t, defaultHclVarSpec, string(content)) + }) + t.Run("json", func(t *testing.T) { + ci.Parallel(t) + dir := dir + ui := cli.NewMockUi() + cmd := &VarInitCommand{Meta: Meta{Ui: ui}} + + // Fails on misuse + code := cmd.Run([]string{"some", "bad", "args"}) + require.Equal(t, 1, code) + require.Contains(t, ui.ErrorWriter.String(), "This command takes no arguments or one") + require.Empty(t, ui.OutputWriter.String()) + reset(ui) + + // Works if the file doesn't exist + code = cmd.Run([]string{"-out", "json"}) + require.Contains(t, ui.ErrorWriter.String(), "REMINDER: While keys") + require.Contains(t, ui.OutputWriter.String(), "Example variable specification written to spec.nsv.json\n") + require.Zero(t, code) + reset(ui) + + t.Cleanup(func() { os.Remove(path.Join(dir, "spec.nsv.json")) }) + content, err := os.ReadFile(DefaultJsonVarInitName) + require.NoError(t, err) + require.Equal(t, defaultJsonVarSpec, string(content)) + + // Fails if the file exists + code = cmd.Run([]string{"-out", "json"}) + require.Contains(t, ui.ErrorWriter.String(), "exists") + require.Empty(t, ui.OutputWriter.String()) + require.Equal(t, 1, code) + reset(ui) + + // Works if file is passed + code = cmd.Run([]string{"-out", "json", "myTest.json"}) + require.Contains(t, ui.ErrorWriter.String(), "REMINDER: While keys") + require.Contains(t, ui.OutputWriter.String(), "Example variable specification written to myTest.json\n") + require.Zero(t, code) + reset(ui) + + t.Cleanup(func() { os.Remove(path.Join(dir, "myTest.json")) }) + content, err = os.ReadFile("myTest.json") + require.NoError(t, err) + require.Equal(t, defaultJsonVarSpec, string(content)) + }) +} + +func reset(ui *cli.MockUi) { + ui.OutputWriter.Reset() + ui.ErrorWriter.Reset() +} diff --git a/command/var_list.go b/command/var_list.go index 6b174f0a0..fc45c7091 100644 --- a/command/var_list.go +++ b/command/var_list.go @@ -4,7 +4,6 @@ import ( "fmt" "sort" "strings" - "time" "github.com/hashicorp/nomad/api" "github.com/posener/complete" @@ -27,8 +26,8 @@ Usage: nomad var list [options] List is used to list available variables. Supplying an optional prefix, filters the list to variables having a path starting with the prefix. - If ACLs are enabled, this command will return only variables stored at - namespaced paths where the token has the ` + "`read`" + ` capability. + If ACLs are enabled, this command will only return variables stored in + namespaces where the token has the 'variables:list' capability. General Options: @@ -55,7 +54,7 @@ List Options: -q Output matching variable paths with no additional information. - This option overrides the ` + "`-t`" + ` option. + This option overrides the '-t' option. ` return strings.TrimSpace(helpText) } @@ -222,7 +221,7 @@ func formatVarStubs(vars []*api.VariableMetadata) string { rows[i+1] = fmt.Sprintf("%s|%s|%s", sv.Namespace, sv.Path, - time.Unix(0, sv.ModifyTime), + formatUnixNanoTime(sv.ModifyTime), ) } return formatList(rows) diff --git a/command/var_list_test.go b/command/var_list_test.go index 024ebeeb9..ceee8d3b2 100644 --- a/command/var_list_test.go +++ b/command/var_list_test.go @@ -5,7 +5,6 @@ import ( "fmt" "strings" "testing" - "time" "github.com/hashicorp/nomad/api" "github.com/hashicorp/nomad/ci" @@ -19,7 +18,7 @@ func TestVarListCommand_Implements(t *testing.T) { } // TestVarListCommand_Offline contains all of the tests that do not require a -// testagent to complete +// testServer to complete func TestVarListCommand_Offline(t *testing.T) { ci.Parallel(t) ui := cli.NewMockUi() @@ -95,8 +94,8 @@ func TestVarListCommand_Offline(t *testing.T) { } } -// TestVarListCommand_Online contains all of the tests that use a testagent. -// They reuse the same testagent so that they can run in parallel and minimize +// TestVarListCommand_Online contains all of the tests that use a testServer. +// They reuse the same testServer so that they can run in parallel and minimize // test startup time costs. func TestVarListCommand_Online(t *testing.T) { ci.Parallel(t) @@ -161,7 +160,7 @@ func TestVarListCommand_Online(t *testing.T) { "Namespace|Path|Last Updated", fmt.Sprintf( "default|a/b/c/d|%s", - time.Unix(0, variables.HavingPrefix("a/b/c/d")[0].ModifyTime), + formatUnixNanoTime(variables.HavingPrefix("a/b/c/d")[0].ModifyTime), ), }, ), diff --git a/command/var_purge.go b/command/var_purge.go new file mode 100644 index 000000000..f29f701f9 --- /dev/null +++ b/command/var_purge.go @@ -0,0 +1,121 @@ +package command + +import ( + "errors" + "fmt" + "strconv" + "strings" + + "github.com/mitchellh/cli" + "github.com/posener/complete" +) + +type VarPurgeCommand struct { + Meta +} + +func (c *VarPurgeCommand) Help() string { + helpText := ` +Usage: nomad var purge [options] + + Purge is used to permanently delete an existing variable. + + If ACLs are enabled, this command requires a token with the 'variables:destroy' + capability for the target variable's namespace. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault) + ` + +Purge Options: + + -check-index + If set, the variable is only acted upon if the server side version's modify + index matches the provided value. +` + + return strings.TrimSpace(helpText) +} + +func (c *VarPurgeCommand) AutocompleteFlags() complete.Flags { + return c.Meta.AutocompleteFlags(FlagSetClient) +} + +func (c *VarPurgeCommand) AutocompleteArgs() complete.Predictor { + return VariablePathPredictor(c.Meta.Client) +} + +func (c *VarPurgeCommand) Synopsis() string { + return "Purge a variable" +} + +func (c *VarPurgeCommand) Name() string { return "var purge" } + +func (c *VarPurgeCommand) Run(args []string) int { + var checkIndexStr string + + flags := c.Meta.FlagSet(c.Name(), FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + flags.StringVar(&checkIndexStr, "check-index", "", "") + + if err := flags.Parse(args); err != nil { + return 1 + } + + // Check that we got one argument + args = flags.Args() + if l := len(args); l != 1 { + c.Ui.Error("This command takes one argument: ") + c.Ui.Error(commandErrorText(c)) + return 1 + } + + // Parse the check-index + checkIndex, enforce, err := parseCheckIndex(checkIndexStr) + if err != nil { + switch { + case errors.Is(err, strconv.ErrRange): + c.Ui.Error(fmt.Sprintf("Invalid -check-index value %q: out of range for uint64", checkIndexStr)) + case errors.Is(err, strconv.ErrSyntax): + c.Ui.Error(fmt.Sprintf("Invalid -check-index value %q: not parsable as uint64", checkIndexStr)) + default: + c.Ui.Error(fmt.Sprintf("Error parsing -check-index value %q: %v", checkIndexStr, err)) + } + return 1 + } + + if c.Meta.namespace == "*" { + c.Ui.Error(errWildcardNamespaceNotAllowed) + return 1 + } + + path := args[0] + + // Get the HTTP client + client, err := c.Meta.Client() + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + if enforce { + _, err = client.Variables().CheckedDelete(path, checkIndex, nil) + } else { + _, err = client.Variables().Delete(path, nil) + } + + if err != nil { + if handled := handleCASError(err, c); handled { + return 1 + } + c.Ui.Error(fmt.Sprintf("Error purging variable: %s", err)) + return 1 + } + + c.Ui.Output(fmt.Sprintf("Successfully purged variable %q!", path)) + return 0 +} + +func (c *VarPurgeCommand) GetConcurrentUI() cli.ConcurrentUi { + return cli.ConcurrentUi{Ui: c.Ui} +} diff --git a/command/var_purge_test.go b/command/var_purge_test.go new file mode 100644 index 000000000..e0bea3ae8 --- /dev/null +++ b/command/var_purge_test.go @@ -0,0 +1,133 @@ +package command + +import ( + "fmt" + "strings" + "testing" + + "github.com/hashicorp/nomad/ci" + "github.com/mitchellh/cli" + "github.com/posener/complete" + "github.com/stretchr/testify/require" +) + +func TestVarPurgeCommand_Implements(t *testing.T) { + ci.Parallel(t) + var _ cli.Command = &VarPurgeCommand{} +} + +func TestVarPurgeCommand_Fails(t *testing.T) { + ci.Parallel(t) + t.Run("bad_args", func(t *testing.T) { + ci.Parallel(t) + ui := cli.NewMockUi() + cmd := &VarPurgeCommand{Meta: Meta{Ui: ui}} + code := cmd.Run([]string{"some", "bad", "args"}) + out := ui.ErrorWriter.String() + require.Equal(t, 1, code, "expected exit code 1, got: %d") + require.Contains(t, out, commandErrorText(cmd), "expected help output, got: %s", out) + }) + t.Run("bad_address", func(t *testing.T) { + ci.Parallel(t) + ui := cli.NewMockUi() + cmd := &VarPurgeCommand{Meta: Meta{Ui: ui}} + code := cmd.Run([]string{"-address=nope", "foo"}) + out := ui.ErrorWriter.String() + require.Equal(t, 1, code, "expected exit code 1, got: %d") + require.Contains(t, ui.ErrorWriter.String(), "purging variable", "connection error, got: %s", out) + require.Zero(t, ui.OutputWriter.String()) + }) + t.Run("bad_check_index/syntax", func(t *testing.T) { + ci.Parallel(t) + ui := cli.NewMockUi() + cmd := &VarPurgeCommand{Meta: Meta{Ui: ui}} + code := cmd.Run([]string{`-check-index=a`, "foo"}) + out := strings.TrimSpace(ui.ErrorWriter.String()) + require.Equal(t, 1, code, "expected exit code 1, got: %d", code) + require.Equal(t, `Invalid -check-index value "a": not parsable as uint64`, out) + require.Zero(t, ui.OutputWriter.String()) + }) + t.Run("bad_check_index/range", func(t *testing.T) { + ci.Parallel(t) + ui := cli.NewMockUi() + cmd := &VarPurgeCommand{Meta: Meta{Ui: ui}} + code := cmd.Run([]string{`-check-index=18446744073709551616`, "foo"}) + out := strings.TrimSpace(ui.ErrorWriter.String()) + require.Equal(t, 1, code, "expected exit code 1, got: %d", code) + require.Equal(t, `Invalid -check-index value "18446744073709551616": out of range for uint64`, out) + require.Zero(t, ui.OutputWriter.String()) + }) +} + +func TestVarPurgeCommand_Online(t *testing.T) { + ci.Parallel(t) + + // Create a server + srv, client, url := testServer(t, true, nil) + t.Cleanup(func() { + srv.Shutdown() + }) + + t.Run("unchecked", func(t *testing.T) { + ui := cli.NewMockUi() + cmd := &VarPurgeCommand{Meta: Meta{Ui: ui}} + + // Create a var to delete + sv := testVariable() + _, _, err := client.Variables().Create(sv, nil) + require.NoError(t, err) + t.Cleanup(func() { _, _ = client.Variables().Delete(sv.Path, nil) }) + + // Delete the variable + code := cmd.Run([]string{"-address=" + url, sv.Path}) + require.Equal(t, 0, code, "expected exit 0, got: %d; %v", code, ui.ErrorWriter.String()) + + vars, _, err := client.Variables().List(nil) + require.NoError(t, err) + require.Len(t, vars, 0) + }) + + t.Run("unchecked", func(t *testing.T) { + ci.Parallel(t) + ui := cli.NewMockUi() + cmd := &VarPurgeCommand{Meta: Meta{Ui: ui}} + + // Create a var to delete + sv := testVariable() + sv, _, err := client.Variables().Create(sv, nil) + require.NoError(t, err) + + // Delete a variable + code := cmd.Run([]string{"-address=" + url, "-check-index=1", sv.Path}) + stderr := ui.ErrorWriter.String() + require.Equal(t, 1, code, "expected exit 1, got: %d; %v", code, stderr) + require.Contains(t, stderr, "\nCheck-and-Set conflict\n\n Your provided check-index (1)") + + code = cmd.Run([]string{"-address=" + url, fmt.Sprintf("-check-index=%v", sv.ModifyIndex), sv.Path}) + require.Equal(t, 0, code, "expected exit 0, got: %d; %v", code, ui.ErrorWriter.String()) + + vars, _, err := client.Variables().List(nil) + require.NoError(t, err) + require.Len(t, vars, 0) + }) + + t.Run("autocompleteArgs", func(t *testing.T) { + ci.Parallel(t) + ui := cli.NewMockUi() + cmd := &VarPurgeCommand{Meta: Meta{Ui: ui, flagAddress: url}} + + // Create a var + sv := testVariable() + sv.Path = "autocomplete/test" + _, _, err := client.Variables().Create(sv, nil) + require.NoError(t, err) + t.Cleanup(func() { client.Variables().Delete(sv.Path, nil) }) + + args := complete.Args{Last: "aut"} + predictor := cmd.AutocompleteArgs() + + res := predictor.Predict(args) + require.Equal(t, 1, len(res)) + require.Equal(t, sv.Path, res[0]) + }) +} diff --git a/command/var_put.go b/command/var_put.go new file mode 100644 index 000000000..cadc1176f --- /dev/null +++ b/command/var_put.go @@ -0,0 +1,523 @@ +package command + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/hashicorp/hcl" + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/helper" + "github.com/mitchellh/cli" + "github.com/mitchellh/mapstructure" + "github.com/posener/complete" +) + +type VarPutCommand struct { + Meta + + contents []byte + inFmt string + outFmt string + tmpl string + testStdin io.Reader // for tests + verbose func(string) +} + +func (c *VarPutCommand) Help() string { + helpText := ` +Usage: +nomad var put [options] [=]... +nomad var put [options] [] [=]... + + The 'var put' command is used to create or update an existing variable. + Variable metadata and items can be supplied using a variable specification, + by using command arguments, or by a combination of the two techniques. + + An entire variable specification can be provided to the command via standard + input (stdin) by setting the first argument to "-" or from a file by using an + @-prefixed path to a variable specification file. When providing variable + data via stdin, you must provide the "-in" flag with the format of the + specification, either "hcl" or "json" + + Items to be stored in the variable can be supplied using the specification, + as a series of key-value pairs, or both. The value for a key-value pair can + be a string, an @-prefixed file reference, or a '-' to get the value from + stdin. Item values provided from file references or stdin are consumed as-is + with no additional processing and do not require the input format to be + specified. + + Values supplied as command line arguments supersede values provided in the + any variable specification piped into the command or loaded from file. + + If ACLs are enabled, this command requires the 'variables:write' capability + for the destination namespace. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault) + ` + +Apply Options: + + -check-index + If set, the variable is only acted upon if the server-side version's index + matches the provided value. When a variable specification contains + a modify index, that modify index is used as the check-index for the + check-and-set operation and can be overridden using this flag. + + -force + Perform this operation regardless of the state or index of the variable + on the server-side. + + -in (hcl | json) + Parser to use for data supplied via standard input or when the variable + specification's type can not be known using the file extension. Defaults + to "json". + + -out (go-template | hcl | json | none | table) + Format to render created or updated variable. Defaults to "none" when + stdout is a terminal and "json" when the output is redirected. + + -template + Template to render output with. Required when format is "go-template", + invalid for other formats. + + -verbose + Provides additional information via standard error to preserve standard + output (stdout) for redirected output. + +` + return strings.TrimSpace(helpText) +} + +func (c *VarPutCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{ + "-in": complete.PredictSet("hcl", "json"), + "-out": complete.PredictSet("none", "hcl", "json", "go-template", "table"), + }, + ) +} + +func (c *VarPutCommand) AutocompleteArgs() complete.Predictor { + return VariablePathPredictor(c.Meta.Client) +} + +func (c *VarPutCommand) Synopsis() string { + return "Create or update a variable" +} + +func (c *VarPutCommand) Name() string { return "var put" } + +func (c *VarPutCommand) Run(args []string) int { + var force, enforce, doVerbose bool + var path, checkIndexStr string + var checkIndex uint64 + var err error + + flags := c.Meta.FlagSet(c.Name(), FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + + flags.BoolVar(&force, "force", false, "") + flags.BoolVar(&doVerbose, "verbose", false, "") + flags.StringVar(&checkIndexStr, "check-index", "", "") + flags.StringVar(&c.inFmt, "in", "json", "") + flags.StringVar(&c.tmpl, "template", "", "") + + if fileInfo, _ := os.Stdout.Stat(); (fileInfo.Mode() & os.ModeCharDevice) != 0 { + flags.StringVar(&c.outFmt, "out", "none", "") + } else { + flags.StringVar(&c.outFmt, "out", "json", "") + } + + if err := flags.Parse(args); err != nil { + c.Ui.Error(commandErrorText(c)) + return 1 + } + + args = flags.Args() + + // Manage verbose output + verbose := func(_ string) {} //no-op + if doVerbose { + verbose = func(msg string) { + c.Ui.Warn(msg) + } + } + c.verbose = verbose + + // Parse the check-index + checkIndex, enforce, err = parseCheckIndex(checkIndexStr) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error parsing check-index value %q: %v", checkIndexStr, err)) + return 1 + } + + if c.Meta.namespace == "*" { + c.Ui.Error(errWildcardNamespaceNotAllowed) + return 1 + } + + // Pull our fake stdin if needed + stdin := (io.Reader)(os.Stdin) + if c.testStdin != nil { + stdin = c.testStdin + } + + switch { + case len(args) < 1: + c.Ui.Error(fmt.Sprintf("Not enough arguments (expected >1, got %d)", len(args))) + c.Ui.Error(commandErrorText(c)) + return 1 + case len(args) == 1 && !isArgStdinRef(args[0]) && !isArgFileRef(args[0]): + c.Ui.Error("Must supply data") + c.Ui.Error(commandErrorText(c)) + return 1 + } + + if err = c.validateInputFlag(); err != nil { + c.Ui.Error(err.Error()) + c.Ui.Error(commandErrorText(c)) + return 1 + } + + if err := c.validateOutputFlag(); err != nil { + c.Ui.Error(err.Error()) + c.Ui.Error(commandErrorText(c)) + return 1 + } + + arg := args[0] + switch { + // Handle first argument: can be -, @file, «var path» + case isArgStdinRef(arg): + + // read the specification into memory from stdin + stat, _ := os.Stdin.Stat() + if (stat.Mode() & os.ModeCharDevice) == 0 { + c.contents, err = io.ReadAll(os.Stdin) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error reading from stdin: %s", err)) + return 1 + } + } + verbose(fmt.Sprintf("Reading whole %s variable specification from stdin", strings.ToUpper(c.inFmt))) + + case isArgFileRef(arg): + // ArgFileRefs start with "@" so we need to peel that off + // detect format based on file extension + specPath := arg[1:] + switch filepath.Ext(specPath) { + case ".json": + c.inFmt = "json" + case ".hcl": + c.inFmt = "hcl" + default: + c.Ui.Error(fmt.Sprintf("Unable to determine format of %s; Use the -in flag to specify it.", specPath)) + return 1 + } + + verbose(fmt.Sprintf("Reading whole %s variable specification from %q", strings.ToUpper(c.inFmt), specPath)) + c.contents, err = os.ReadFile(specPath) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error reading %q: %s", specPath, err)) + return 1 + } + default: + path = sanitizePath(arg) + verbose(fmt.Sprintf("Writing to path %q", path)) + } + + args = args[1:] + switch { + // Handle second argument: can be -, @file, or kv + case len(args) == 0: + // no-op + case isArgStdinRef(args[0]): + verbose(fmt.Sprintf("Creating variable %q using specification from stdin", path)) + stat, _ := os.Stdin.Stat() + if (stat.Mode() & os.ModeCharDevice) == 0 { + c.contents, err = io.ReadAll(os.Stdin) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error reading from stdin: %s", err)) + return 1 + } + } + args = args[1:] + + case isArgFileRef(args[0]): + arg := args[0] + verbose(fmt.Sprintf("Creating variable %q from specification file %q", path, arg)) + fPath := arg[1:] + c.contents, err = os.ReadFile(fPath) + if err != nil { + c.Ui.Error(fmt.Sprintf("error reading %q: %s", fPath, err)) + return 1 + } + args = args[1:] + default: + // no-op - should be KV arg + } + + sv, err := c.makeVariable(path) + if err != nil { + c.Ui.Error(fmt.Sprintf("Failed to parse variable data: %s", err)) + return 1 + } + + if len(args) > 0 { + data, err := parseArgsData(stdin, args) + if err != nil { + c.Ui.Error(fmt.Sprintf("Failed to parse K=V data: %s", err)) + return 1 + } + + for k, v := range data { + vs := v.(string) + if vs == "" { + if _, ok := sv.Items[k]; ok { + verbose(fmt.Sprintf("Removed item %q", k)) + delete(sv.Items, k) + } else { + verbose(fmt.Sprintf("Item %q does not exist, continuing...", k)) + } + continue + } + sv.Items[k] = vs + } + } + // Get the HTTP client + client, err := c.Meta.Client() + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + if enforce { + sv.ModifyIndex = checkIndex + } + + if force { + sv, _, err = client.Variables().Update(sv, nil) + } else { + sv, _, err = client.Variables().CheckedUpdate(sv, nil) + } + if err != nil { + if handled := handleCASError(err, c); handled { + return 1 + } + c.Ui.Error(fmt.Sprintf("Error creating variable: %s", err)) + return 1 + } + + verbose(fmt.Sprintf("Created variable %q with modify index %v", sv.Path, sv.ModifyIndex)) + + var out string + switch c.outFmt { + case "json": + out = sv.AsPrettyJSON() + case "hcl": + out = renderAsHCL(sv) + case "go-template": + if out, err = renderWithGoTemplate(sv, c.tmpl); err != nil { + c.Ui.Error(err.Error()) + return 1 + } + case "table": + // the renderSVAsUiTable func writes directly to the ui and doesn't error. + renderSVAsUiTable(sv, c) + return 0 + default: + return 0 + } + c.Ui.Output(out) + return 0 +} + +// makeVariable creates a variable based on whether or not there is data in +// content and the format is set. +func (c *VarPutCommand) makeVariable(path string) (*api.Variable, error) { + var err error + out := new(api.Variable) + if len(c.contents) == 0 { + out.Path = path + out.Namespace = c.Meta.namespace + out.Items = make(map[string]string) + return out, nil + } + switch c.inFmt { + case "json": + err = json.Unmarshal(c.contents, out) + if err != nil { + return nil, fmt.Errorf("error unmarshaling json: %w", err) + } + case "hcl": + out, err = parseVariableSpec(c.contents, c.verbose) + if err != nil { + return nil, fmt.Errorf("error parsing hcl: %w", err) + } + case "": + return nil, errors.New("format flag required") + default: + return nil, fmt.Errorf("unknown format flag value") + } + + // Handle cases where values are provided by CLI flags that modify the + // the created variable. Typical of a "copy" operation, it is a convenience + // to reset the Create and Modify metadata to zero. + var resetIndex bool + + // Step on the namespace in the object if one is provided by flag + if c.Meta.namespace != "" && c.Meta.namespace != out.Namespace { + out.Namespace = c.Meta.namespace + resetIndex = true + } + + // Step on the path in the object if one is provided by argument. + if path != "" && path != out.Path { + out.Path = path + resetIndex = true + } + + if resetIndex { + out.CreateIndex = 0 + out.CreateTime = 0 + out.ModifyIndex = 0 + out.ModifyTime = 0 + } + return out, nil +} + +// parseVariableSpec is used to parse the variable specification +// from HCL +func parseVariableSpec(input []byte, verbose func(string)) (*api.Variable, error) { + root, err := hcl.ParseBytes(input) + if err != nil { + return nil, err + } + + // Top-level item should be a list + list, ok := root.Node.(*ast.ObjectList) + if !ok { + return nil, fmt.Errorf("error parsing: root should be an object") + } + + var out api.Variable + if err := parseVariableSpecImpl(&out, list); err != nil { + return nil, err + } + return &out, nil +} + +// parseVariableSpecImpl parses the variable taking as input the AST tree +func parseVariableSpecImpl(result *api.Variable, list *ast.ObjectList) error { + // Decode the full thing into a map[string]interface for ease + var m map[string]interface{} + if err := hcl.DecodeObject(&m, list); err != nil { + return err + } + + // Check for invalid keys + valid := []string{ + "namespace", + "path", + "create_index", + "modify_index", + "create_time", + "modify_time", + "items", + } + if err := helper.CheckHCLKeys(list, valid); err != nil { + return err + } + + for _, index := range []string{"create_index", "modify_index"} { + if value, ok := m[index]; ok { + vInt, ok := value.(int) + if !ok { + return fmt.Errorf("%s must be integer; got (%T) %[2]v", index, value) + } + idx := uint64(vInt) + n := strings.ReplaceAll(strings.Title(strings.ReplaceAll(index, "_", " ")), " ", "") + m[n] = idx + delete(m, index) + } + } + + for _, index := range []string{"create_time", "modify_time"} { + if value, ok := m[index]; ok { + vInt, ok := value.(int) + if !ok { + return fmt.Errorf("%s must be a int64; got a (%T) %[2]v", index, value) + } + n := strings.ReplaceAll(strings.Title(strings.ReplaceAll(index, "_", " ")), " ", "") + m[n] = vInt + delete(m, index) + } + } + + // Decode the rest + if err := mapstructure.WeakDecode(m, result); err != nil { + return err + } + + return nil +} + +func isArgFileRef(a string) bool { + return strings.HasPrefix(a, "@") && !strings.HasPrefix(a, "\\@") +} + +func isArgStdinRef(a string) bool { + return a == "-" +} + +// sanitizePath removes any leading or trailing things from a "path". +func sanitizePath(s string) string { + return strings.Trim(strings.TrimSpace(s), "/") +} + +// parseArgsData parses the given args in the format key=value into a map of +// the provided arguments. The given reader can also supply key=value pairs. +func parseArgsData(stdin io.Reader, args []string) (map[string]interface{}, error) { + builder := &KVBuilder{Stdin: stdin} + if err := builder.Add(args...); err != nil { + return nil, err + } + return builder.Map(), nil +} + +func (c *VarPutCommand) GetConcurrentUI() cli.ConcurrentUi { + return cli.ConcurrentUi{Ui: c.Ui} +} + +func (c *VarPutCommand) validateInputFlag() error { + switch c.inFmt { + case "hcl", "json": + return nil + default: + return errors.New(errInvalidInFormat) + } +} + +func (c *VarPutCommand) validateOutputFlag() error { + if c.outFmt != "go-template" && c.tmpl != "" { + return errors.New(errUnexpectedTemplate) + } + switch c.outFmt { + case "none", "json", "hcl", "table": + return nil + case "go-template": + if c.tmpl == "" { + return errors.New(errMissingTemplate) + } + return nil + default: + return errors.New(errInvalidOutFormat) + } +} diff --git a/command/var_put_test.go b/command/var_put_test.go new file mode 100644 index 000000000..2e64cfa8f --- /dev/null +++ b/command/var_put_test.go @@ -0,0 +1,123 @@ +package command + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/ci" + "github.com/mitchellh/cli" + "github.com/posener/complete" + "github.com/stretchr/testify/require" +) + +func TestVarPutCommand_Implements(t *testing.T) { + ci.Parallel(t) + var _ cli.Command = &VarPutCommand{} +} +func TestVarPutCommand_Fails(t *testing.T) { + ci.Parallel(t) + t.Run("bad_args", func(t *testing.T) { + ci.Parallel(t) + ui := cli.NewMockUi() + cmd := &VarPutCommand{Meta: Meta{Ui: ui}} + code := cmd.Run([]string{"-bad-flag"}) + out := ui.ErrorWriter.String() + require.Equal(t, 1, code, "expected exit code 1, got: %d") + require.Contains(t, out, commandErrorText(cmd), "expected help output, got: %s", out) + }) + t.Run("bad_address", func(t *testing.T) { + ci.Parallel(t) + ui := cli.NewMockUi() + cmd := &VarPutCommand{Meta: Meta{Ui: ui}} + code := cmd.Run([]string{"-address=nope", "foo", "-"}) + out := ui.ErrorWriter.String() + require.Equal(t, 1, code, "expected exit code 1, got: %d") + require.Contains(t, out, "Error creating variable", "expected error creating variable, got: %s", out) + }) + t.Run("missing_template", func(t *testing.T) { + ci.Parallel(t) + ui := cli.NewMockUi() + cmd := &VarPutCommand{Meta: Meta{Ui: ui}} + code := cmd.Run([]string{`-out=go-template`, "foo", "-"}) + out := strings.TrimSpace(ui.ErrorWriter.String()) + require.Equal(t, 1, code, "expected exit code 1, got: %d", code) + require.Equal(t, errMissingTemplate+"\n"+commandErrorText(cmd), out) + }) + t.Run("unexpected_template", func(t *testing.T) { + ci.Parallel(t) + ui := cli.NewMockUi() + cmd := &VarPutCommand{Meta: Meta{Ui: ui}} + code := cmd.Run([]string{`-out=json`, `-template="bad"`, "foo", "-"}) + out := strings.TrimSpace(ui.ErrorWriter.String()) + require.Equal(t, 1, code, "expected exit code 1, got: %d", code) + require.Equal(t, errUnexpectedTemplate+"\n"+commandErrorText(cmd), out) + }) + t.Run("bad_in", func(t *testing.T) { + ci.Parallel(t) + ui := cli.NewMockUi() + cmd := &VarPutCommand{Meta: Meta{Ui: ui}} + code := cmd.Run([]string{`-in=bad`, "foo", "-"}) + out := strings.TrimSpace(ui.ErrorWriter.String()) + require.Equal(t, 1, code, "expected exit code 1, got: %d", code) + require.Equal(t, errInvalidInFormat+"\n"+commandErrorText(cmd), out) + }) + t.Run("wildcard_namespace", func(t *testing.T) { + ci.Parallel(t) + ui := cli.NewMockUi() + cmd := &VarPutCommand{Meta: Meta{Ui: ui}} + code := cmd.Run([]string{`-namespace=*`, "foo", "-"}) + out := strings.TrimSpace(ui.ErrorWriter.String()) + require.Equal(t, 1, code, "expected exit code 1, got: %d", code) + require.Equal(t, errWildcardNamespaceNotAllowed, out) + }) +} + +func TestVarPutCommand_GoodJson(t *testing.T) { + ci.Parallel(t) + + // Create a server + srv, client, url := testServer(t, true, nil) + defer srv.Shutdown() + + ui := cli.NewMockUi() + cmd := &VarPutCommand{Meta: Meta{Ui: ui}} + + // Get the variable + code := cmd.Run([]string{"-address=" + url, "-out=json", "test/var", "k1=v1", "k2=v2"}) + require.Equal(t, 0, code, "expected exit 0, got: %d; %v", code, ui.ErrorWriter.String()) + + t.Cleanup(func() { + _, _ = client.Variables().Delete("test/var", nil) + }) + + var outVar api.Variable + b := ui.OutputWriter.Bytes() + err := json.Unmarshal(b, &outVar) + require.NoError(t, err, "error unmarshaling json: %v\nb: %s", err, b) + require.Equal(t, "default", outVar.Namespace) + require.Equal(t, "test/var", outVar.Path) + require.Equal(t, api.VariableItems{"k1": "v1", "k2": "v2"}, outVar.Items) +} + +func TestVarPutCommand_AutocompleteArgs(t *testing.T) { + ci.Parallel(t) + _, client, url, shutdownFn := testAPIClient(t) + defer shutdownFn() + + ui := cli.NewMockUi() + cmd := &VarPutCommand{Meta: Meta{Ui: ui, flagAddress: url}} + + // Create a var + sv := testVariable() + _, _, err := client.Variables().Create(sv, nil) + require.NoError(t, err) + + args := complete.Args{Last: "t"} + predictor := cmd.AutocompleteArgs() + + res := predictor.Predict(args) + require.Equal(t, 1, len(res)) + require.Equal(t, sv.Path, res[0]) +} diff --git a/command/var_test.go b/command/var_test.go new file mode 100644 index 000000000..92c21f172 --- /dev/null +++ b/command/var_test.go @@ -0,0 +1,26 @@ +package command + +import ( + "testing" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/nomad/command/agent" +) + +// testVariable returns a test variable spec +func testVariable() *api.Variable { + return &api.Variable{ + Namespace: "default", + Path: "test/var", + Items: map[string]string{ + "keyA": "valueA", + "keyB": "valueB", + }, + } +} + +func testAPIClient(t *testing.T) (srv *agent.TestAgent, client *api.Client, url string, shutdownFn func() error) { + srv, client, url = testServer(t, true, nil) + shutdownFn = srv.Shutdown + return +} diff --git a/nomad/variables_endpoint.go b/nomad/variables_endpoint.go index 3217d30b8..b68e82d81 100644 --- a/nomad/variables_endpoint.go +++ b/nomad/variables_endpoint.go @@ -237,7 +237,7 @@ func (sv *Variables) Read(args *structs.VariablesReadRequest, reply *structs.Var reply.Data = &ov reply.Index = out.ModifyIndex } else { - sv.srv.replySetIndex(state.TableVariables, &reply.QueryMeta) + sv.srv.setReplyQueryMeta(s, state.TableVariables, &reply.QueryMeta) } return nil }} @@ -266,9 +266,6 @@ func (sv *Variables) List( if err != nil { return err } - if err != nil { - return err - } // Set up and return the blocking query. return sv.srv.blockingRPC(&blockingOptions{