From b52f42db9a7bab1cf03cec60e4422f566c2d3efd Mon Sep 17 00:00:00 2001 From: Florian Apolloner Date: Thu, 4 Nov 2021 16:33:09 +0100 Subject: [PATCH] Added a `-hcl2-strict` flag to allow for lenient hcl variable parsing. (#11284) Co-authored-by: James Rasell --- .changelog/11284.txt | 3 ++ command/helpers.go | 6 +-- command/helpers_test.go | 53 ++++++++++++++++++- command/job_plan.go | 11 +++- command/job_run.go | 11 +++- command/job_validate.go | 17 ++++-- website/content/docs/commands/job/plan.mdx | 8 +++ website/content/docs/commands/job/run.mdx | 14 +++-- .../content/docs/commands/job/validate.mdx | 12 +++++ 9 files changed, 117 insertions(+), 18 deletions(-) create mode 100644 .changelog/11284.txt diff --git a/.changelog/11284.txt b/.changelog/11284.txt new file mode 100644 index 000000000..d767686e5 --- /dev/null +++ b/.changelog/11284.txt @@ -0,0 +1,3 @@ +```release-note:improvement +cli: added `hcl2-strict` flag to control HCL2 parsing errors where variable passed without root +``` diff --git a/command/helpers.go b/command/helpers.go index ee0de86a8..863dc823e 100644 --- a/command/helpers.go +++ b/command/helpers.go @@ -388,10 +388,10 @@ type JobGetter struct { // ApiJob returns the Job struct from jobfile. func (j *JobGetter) ApiJob(jpath string) (*api.Job, error) { - return j.ApiJobWithArgs(jpath, nil, nil) + return j.ApiJobWithArgs(jpath, nil, nil, true) } -func (j *JobGetter) ApiJobWithArgs(jpath string, vars []string, varfiles []string) (*api.Job, error) { +func (j *JobGetter) ApiJobWithArgs(jpath string, vars []string, varfiles []string, strict bool) (*api.Job, error) { var jobfile io.Reader pathName := filepath.Base(jpath) switch jpath { @@ -459,7 +459,7 @@ func (j *JobGetter) ApiJobWithArgs(jpath string, vars []string, varfiles []strin AllowFS: true, VarFiles: varfiles, Envs: os.Environ(), - Strict: true, + Strict: strict, }) if err != nil { diff --git a/command/helpers_test.go b/command/helpers_test.go index c5e54e853..275a5d249 100644 --- a/command/helpers_test.go +++ b/command/helpers_test.go @@ -368,7 +368,58 @@ job "example" { _, err = vf.WriteString(fileVars + "\n") require.NoError(t, err) - j, err := (&JobGetter{}).ApiJobWithArgs(hclf.Name(), cliArgs, []string{vf.Name()}) + j, err := (&JobGetter{}).ApiJobWithArgs(hclf.Name(), cliArgs, []string{vf.Name()}, true) + require.NoError(t, err) + + require.NotNil(t, j) + require.Equal(t, expected, j.Datacenters) +} + +func TestJobGetter_HCL2_Variables_StrictFalse(t *testing.T) { + t.Parallel() + + hcl := ` +variables { + var1 = "default-val" + var2 = "default-val" + var3 = "default-val" + var4 = "default-val" +} + +job "example" { + datacenters = ["${var.var1}", "${var.var2}", "${var.var3}", "${var.var4}"] +} +` + + os.Setenv("NOMAD_VAR_var4", "from-envvar") + defer os.Unsetenv("NOMAD_VAR_var4") + + // Both the CLI and var file contain variables that are not used with the + // template and therefore would error, if hcl2-strict was true. + cliArgs := []string{`var2=from-cli`,`unsedVar1=from-cli`} + fileVars := ` +var3 = "from-varfile" +unsedVar2 = "from-varfile" +` + expected := []string{"default-val", "from-cli", "from-varfile", "from-envvar"} + + hclf, err := ioutil.TempFile("", "hcl") + require.NoError(t, err) + defer os.Remove(hclf.Name()) + defer hclf.Close() + + _, err = hclf.WriteString(hcl) + require.NoError(t, err) + + vf, err := ioutil.TempFile("", "var.hcl") + require.NoError(t, err) + defer os.Remove(vf.Name()) + defer vf.Close() + + _, err = vf.WriteString(fileVars + "\n") + require.NoError(t, err) + + j, err := (&JobGetter{}).ApiJobWithArgs(hclf.Name(), cliArgs, []string{vf.Name()}, false) require.NoError(t, err) require.NotNil(t, j) diff --git a/command/job_plan.go b/command/job_plan.go index bd43b0eab..9ab331de9 100644 --- a/command/job_plan.go +++ b/command/job_plan.go @@ -79,6 +79,11 @@ Plan Options: -hcl1 Parses the job file as HCLv1. + -hcl2-strict + Whether an error should be produced from the HCL2 parser where a variable + has been supplied which is not defined within the root variables. Defaults + to true. + -policy-override Sets the flag to force override any soft mandatory Sentinel policies. @@ -105,6 +110,7 @@ func (c *JobPlanCommand) AutocompleteFlags() complete.Flags { "-policy-override": complete.PredictNothing, "-verbose": complete.PredictNothing, "-hcl1": complete.PredictNothing, + "-hcl2-strict": complete.PredictNothing, "-var": complete.PredictAnything, "-var-file": complete.PredictFiles("*.var"), }) @@ -116,7 +122,7 @@ func (c *JobPlanCommand) AutocompleteArgs() complete.Predictor { func (c *JobPlanCommand) Name() string { return "job plan" } func (c *JobPlanCommand) Run(args []string) int { - var diff, policyOverride, verbose bool + var diff, policyOverride, verbose, hcl2Strict bool var varArgs, varFiles flaghelper.StringFlag flagSet := c.Meta.FlagSet(c.Name(), FlagSetClient) @@ -125,6 +131,7 @@ func (c *JobPlanCommand) Run(args []string) int { flagSet.BoolVar(&policyOverride, "policy-override", false, "") flagSet.BoolVar(&verbose, "verbose", false, "") flagSet.BoolVar(&c.JobGetter.hcl1, "hcl1", false, "") + flagSet.BoolVar(&hcl2Strict, "hcl2-strict", true, "") flagSet.Var(&varArgs, "var", "") flagSet.Var(&varFiles, "var-file", "") @@ -142,7 +149,7 @@ func (c *JobPlanCommand) Run(args []string) int { path := args[0] // Get Job struct from Jobfile - job, err := c.JobGetter.ApiJobWithArgs(args[0], varArgs, varFiles) + job, err := c.JobGetter.ApiJobWithArgs(args[0], varArgs, varFiles, hcl2Strict) if err != nil { c.Ui.Error(fmt.Sprintf("Error getting job struct: %s", err)) return 255 diff --git a/command/job_run.go b/command/job_run.go index 77bd4b2b3..a2ef4cb40 100644 --- a/command/job_run.go +++ b/command/job_run.go @@ -89,6 +89,11 @@ Run Options: -hcl1 Parses the job file as HCLv1. + -hcl2-strict + Whether an error should be produced from the HCL2 parser where a variable + has been supplied which is not defined within the root variables. Defaults + to true. + -output Output the JSON that would be submitted to the HTTP API without submitting the job. @@ -144,6 +149,7 @@ func (c *JobRunCommand) AutocompleteFlags() complete.Flags { "-policy-override": complete.PredictNothing, "-preserve-counts": complete.PredictNothing, "-hcl1": complete.PredictNothing, + "-hcl2-strict": complete.PredictNothing, "-var": complete.PredictAnything, "-var-file": complete.PredictFiles("*.var"), }) @@ -156,7 +162,7 @@ func (c *JobRunCommand) AutocompleteArgs() complete.Predictor { func (c *JobRunCommand) Name() string { return "job run" } func (c *JobRunCommand) Run(args []string) int { - var detach, verbose, output, override, preserveCounts bool + var detach, verbose, output, override, preserveCounts, hcl2Strict bool var checkIndexStr, consulToken, consulNamespace, vaultToken, vaultNamespace string var varArgs, varFiles flaghelper.StringFlag @@ -168,6 +174,7 @@ func (c *JobRunCommand) Run(args []string) int { flagSet.BoolVar(&override, "policy-override", false, "") flagSet.BoolVar(&preserveCounts, "preserve-counts", false, "") flagSet.BoolVar(&c.JobGetter.hcl1, "hcl1", false, "") + flagSet.BoolVar(&hcl2Strict, "hcl2-strict", true, "") flagSet.StringVar(&checkIndexStr, "check-index", "", "") flagSet.StringVar(&consulToken, "consul-token", "", "") flagSet.StringVar(&consulNamespace, "consul-namespace", "", "") @@ -195,7 +202,7 @@ func (c *JobRunCommand) Run(args []string) int { } // Get Job struct from Jobfile - job, err := c.JobGetter.ApiJobWithArgs(args[0], varArgs, varFiles) + job, err := c.JobGetter.ApiJobWithArgs(args[0], varArgs, varFiles, hcl2Strict) if err != nil { c.Ui.Error(fmt.Sprintf("Error getting job struct: %s", err)) return 1 diff --git a/command/job_validate.go b/command/job_validate.go index bae381357..bea9d119e 100644 --- a/command/job_validate.go +++ b/command/job_validate.go @@ -37,12 +37,16 @@ Validate Options: -hcl1 Parses the job file as HCLv1. + -hcl2-strict + Whether an error should be produced from the HCL2 parser where a variable + has been supplied which is not defined within the root variables. Defaults + to true. + -var 'key=value' Variable for template, can be used multiple times. -var-file=path Path to HCL2 file containing user variables. - ` return strings.TrimSpace(helpText) } @@ -53,9 +57,10 @@ func (c *JobValidateCommand) Synopsis() string { func (c *JobValidateCommand) AutocompleteFlags() complete.Flags { return complete.Flags{ - "-hcl1": complete.PredictNothing, - "-var": complete.PredictAnything, - "-var-file": complete.PredictFiles("*.var"), + "-hcl1": complete.PredictNothing, + "-hcl2-strict": complete.PredictNothing, + "-var": complete.PredictAnything, + "-var-file": complete.PredictFiles("*.var"), } } @@ -67,10 +72,12 @@ func (c *JobValidateCommand) Name() string { return "job validate" } func (c *JobValidateCommand) Run(args []string) int { var varArgs, varFiles flaghelper.StringFlag + var hcl2Strict bool flagSet := c.Meta.FlagSet(c.Name(), FlagSetNone) flagSet.Usage = func() { c.Ui.Output(c.Help()) } flagSet.BoolVar(&c.JobGetter.hcl1, "hcl1", false, "") + flagSet.BoolVar(&hcl2Strict, "hcl2-strict", true, "") flagSet.Var(&varArgs, "var", "") flagSet.Var(&varFiles, "var-file", "") @@ -87,7 +94,7 @@ func (c *JobValidateCommand) Run(args []string) int { } // Get Job struct from Jobfile - job, err := c.JobGetter.ApiJobWithArgs(args[0], varArgs, varFiles) + job, err := c.JobGetter.ApiJobWithArgs(args[0], varArgs, varFiles, hcl2Strict) if err != nil { c.Ui.Error(fmt.Sprintf("Error getting job struct: %s", err)) return 1 diff --git a/website/content/docs/commands/job/plan.mdx b/website/content/docs/commands/job/plan.mdx index 263f361a6..fd09bc589 100644 --- a/website/content/docs/commands/job/plan.mdx +++ b/website/content/docs/commands/job/plan.mdx @@ -65,6 +65,14 @@ capability for the job's namespace. - `-hcl1`: If set, HCL1 parser is used for parsing the job spec. +- `-hcl2-strict`: Whether an error should be produced from the HCL2 parser where + a variable has been supplied which is not defined within the root variables. + Defaults to true. + +- `-var=`: Variable for template, can be used multiple times. + +- `-var-file=`: Path to HCL2 file containing user variables. + - `-verbose`: Increase diff verbosity. ## Examples diff --git a/website/content/docs/commands/job/run.mdx b/website/content/docs/commands/job/run.mdx index f6cffefb9..1f032a843 100644 --- a/website/content/docs/commands/job/run.mdx +++ b/website/content/docs/commands/job/run.mdx @@ -27,8 +27,8 @@ downloaded and read from URL specified. Nomad downloads the job file using By default, on successful job submission the run command will enter an interactive monitor and display log information detailing the scheduling -decisions, placement information, and [deployment status] for the provided job -if applicable ([`batch`] and [`system`] jobs don't create deployments). The monitor will +decisions, placement information, and [deployment status] for the provided job +if applicable ([`batch`] and [`system`] jobs don't create deployments). The monitor will exit after scheduling and deployment have finished or failed. On successful job submission and scheduling, exit code 0 will be returned. If @@ -72,6 +72,10 @@ that volume. - `-hcl1`: If set, HCL1 parser is used for parsing the job spec. +- `-hcl2-strict`: Whether an error should be produced from the HCL2 parser where + a variable has been supplied which is not defined within the root variables. + Defaults to true. + - `-output`: Output the JSON that would be submitted to the HTTP API without submitting the job. @@ -133,7 +137,7 @@ $ nomad job run job1.nomad Deployed Task Group Desired Placed Healthy Unhealthy Progress Deadline - cache 2 2 1 0 2021-06-09T15:32:58-07:00 + cache 2 2 1 0 2021-06-09T15:32:58-07:00 web 1 1 1 0 2021-06-09T15:32:58-07:00 ``` @@ -154,7 +158,7 @@ $ nomad job run -check-index 6 example.nomad ==> 2021-06-09T16:57:30-07:00: Evaluation "5ef16dff" finished with status "complete" ==> 2021-06-09T16:57:30-07:00: Monitoring deployment "62eb607c" ✓ Deployment "62eb607c" successful - + 2021-06-09T16:57:30-07:00 ID = 62eb607c Job ID = example @@ -218,7 +222,7 @@ $ nomad job run example.nomad ``` [`go-getter`]: https://github.com/hashicorp/go-getter -[deployment status]: /docs/commands/deployment#status +[deployment status]: /docs/commands/deployment#status [`batch`]: /docs/schedulers#batch [`system`]: /docs/schedulers#system [`job plan` command]: /docs/commands/job/plan diff --git a/website/content/docs/commands/job/validate.mdx b/website/content/docs/commands/job/validate.mdx index 9b3d17268..ed10d2322 100644 --- a/website/content/docs/commands/job/validate.mdx +++ b/website/content/docs/commands/job/validate.mdx @@ -32,6 +32,18 @@ of 1 indicates an error. When ACLs are enabled, this command requires a token with the `read-job` capability for the job's namespace. +## Validate Options + +- `-hcl1`: If set, HCL1 parser is used for parsing the job spec. + +- `-hcl2-strict`: Whether an error should be produced from the HCL2 parser where +a variable has been supplied which is not defined within the root variables. +Defaults to true. + +- `-var=`: Variable for template, can be used multiple times. + +- `-var-file=`: Path to HCL2 file containing user variables. + ## Examples Validate a job with invalid syntax: