diff --git a/api/tasks.go b/api/tasks.go index aac90ee40..bedbaa149 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -2,6 +2,9 @@ package api import ( "fmt" + "path" + + "path/filepath" "strings" "time" @@ -323,12 +326,30 @@ func (t *Task) Canonicalize(tg *TaskGroup, job *Job) { type TaskArtifact struct { GetterSource *string `mapstructure:"source"` GetterOptions map[string]string `mapstructure:"options"` + GetterMode *string `mapstructure:"mode"` RelativeDest *string `mapstructure:"destination"` } func (a *TaskArtifact) Canonicalize() { + if a.GetterMode == nil { + a.GetterMode = helper.StringToPtr("any") + } + if a.GetterSource == nil { + // Shouldn't be possible, but we don't want to panic + a.GetterSource = helper.StringToPtr("") + } if a.RelativeDest == nil { - a.RelativeDest = helper.StringToPtr("local/") + switch *a.GetterMode { + case "file": + // File mode should default to local/filename + dest := *a.GetterSource + dest = path.Base(dest) + dest = filepath.Join("local", dest) + a.RelativeDest = &dest + default: + // Default to a directory + a.RelativeDest = helper.StringToPtr("local/") + } } } diff --git a/api/tasks_test.go b/api/tasks_test.go index 7756bfe52..334a1cdf3 100644 --- a/api/tasks_test.go +++ b/api/tasks_test.go @@ -219,3 +219,17 @@ func TestTask_Constrain(t *testing.T) { t.Fatalf("expect: %#v, got: %#v", expect, task.Constraints) } } + +func TestTask_Artifact(t *testing.T) { + a := TaskArtifact{ + GetterSource: helper.StringToPtr("http://localhost/foo.txt"), + GetterMode: helper.StringToPtr("file"), + } + a.Canonicalize() + if *a.GetterMode != "file" { + t.Errorf("expected file but found %q", *a.GetterMode) + } + if *a.RelativeDest != "local/foo.txt" { + t.Errorf("expected local/foo.txt but found %q", *a.RelativeDest) + } +} diff --git a/client/getter/getter.go b/client/getter/getter.go index c55dafd27..eeb9bb62c 100644 --- a/client/getter/getter.go +++ b/client/getter/getter.go @@ -33,7 +33,7 @@ type EnvReplacer interface { } // getClient returns a client that is suitable for Nomad downloading artifacts. -func getClient(src, dst string) *gg.Client { +func getClient(src string, mode gg.ClientMode, dst string) *gg.Client { lock.Lock() defer lock.Unlock() @@ -50,7 +50,7 @@ func getClient(src, dst string) *gg.Client { return &gg.Client{ Src: src, Dst: dst, - Mode: gg.ClientModeAny, + Mode: mode, Getters: getters, } } @@ -97,7 +97,17 @@ func GetArtifact(taskEnv EnvReplacer, artifact *structs.TaskArtifact, taskDir st // Download the artifact dest := filepath.Join(taskDir, artifact.RelativeDest) - if err := getClient(url, dest).Get(); err != nil { + + // Convert from string getter mode to go-getter const + mode := gg.ClientModeAny + switch artifact.GetterMode { + case structs.GetterModeFile: + mode = gg.ClientModeFile + case structs.GetterModeDir: + mode = gg.ClientModeDir + } + + if err := getClient(url, mode, dest).Get(); err != nil { return newGetError(url, err, true) } diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 7c975d617..b5bed9142 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -654,6 +654,7 @@ func ApiTaskToStructsTask(apiTask *api.Task, structsTask *structs.Task) { structsTask.Artifacts[k] = &structs.TaskArtifact{ GetterSource: *ta.GetterSource, GetterOptions: ta.GetterOptions, + GetterMode: *ta.GetterMode, RelativeDest: *ta.RelativeDest, } } diff --git a/jobspec/parse.go b/jobspec/parse.go index 02f1dcbf1..34b6f32f4 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -788,6 +788,7 @@ func parseArtifacts(result *[]*api.TaskArtifact, list *ast.ObjectList) error { valid := []string{ "source", "options", + "mode", "destination", } if err := checkHCLKeys(o.Val, valid); err != nil { diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 9fc57ba92..012108be5 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -166,6 +166,7 @@ func TestParse(t *testing.T) { GetterOptions: map[string]string{ "checksum": "md5:ff1cc0d3432dad54d607c1505fb7245c", }, + GetterMode: helper.StringToPtr("file"), }, }, Vault: &api.Vault{ diff --git a/jobspec/test-fixtures/basic.hcl b/jobspec/test-fixtures/basic.hcl index f7a38ee84..cfcfc5a1d 100644 --- a/jobspec/test-fixtures/basic.hcl +++ b/jobspec/test-fixtures/basic.hcl @@ -140,6 +140,7 @@ job "binstore-storagelocker" { artifact { source = "http://bar.com/artifact" destination = "test/foo/" + mode = "file" options { checksum = "md5:ff1cc0d3432dad54d607c1505fb7245c" diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index 4ba261de3..7e90dd9b0 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -2469,6 +2469,7 @@ func TestTaskDiff(t *testing.T) { GetterOptions: map[string]string{ "bar": "baz", }, + GetterMode: "dir", RelativeDest: "bar", }, }, @@ -2487,6 +2488,7 @@ func TestTaskDiff(t *testing.T) { GetterOptions: map[string]string{ "bam": "baz", }, + GetterMode: "file", RelativeDest: "bam", }, }, @@ -2498,6 +2500,12 @@ func TestTaskDiff(t *testing.T) { Type: DiffTypeAdded, Name: "Artifact", Fields: []*FieldDiff{ + { + Type: DiffTypeAdded, + Name: "GetterMode", + Old: "", + New: "file", + }, { Type: DiffTypeAdded, Name: "GetterOptions[bam]", @@ -2522,6 +2530,12 @@ func TestTaskDiff(t *testing.T) { Type: DiffTypeDeleted, Name: "Artifact", Fields: []*FieldDiff{ + { + Type: DiffTypeDeleted, + Name: "GetterMode", + Old: "dir", + New: "", + }, { Type: DiffTypeDeleted, Name: "GetterOptions[bar]", diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index daa461fb4..13a2c5015 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -78,6 +78,10 @@ const ( ProtocolVersion = "protocol" APIMajorVersion = "api.major" APIMinorVersion = "api.minor" + + GetterModeAny = "any" + GetterModeFile = "file" + GetterModeDir = "dir" ) // RPCInfo is used to describe common information about query @@ -3405,6 +3409,10 @@ type TaskArtifact struct { // go-getter. GetterOptions map[string]string + // GetterMode is the go-getter.ClientMode for fetching resources. + // Defaults to "any" but can be set to "file" or "dir". + GetterMode string + // RelativeDest is the download destination given relative to the task's // directory. RelativeDest string @@ -3453,6 +3461,17 @@ func (ta *TaskArtifact) Validate() error { mErr.Errors = append(mErr.Errors, fmt.Errorf("source must be specified")) } + switch ta.GetterMode { + case "": + // Default to any + ta.GetterMode = GetterModeAny + case GetterModeAny, GetterModeFile, GetterModeDir: + // Ok + default: + mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid artifact mode %q; must be one of: %s, %s, %s", + ta.GetterMode, GetterModeAny, GetterModeFile, GetterModeDir)) + } + escaped, err := PathEscapesAllocDir("task", ta.RelativeDest) if err != nil { mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid destination path: %v", err)) diff --git a/vendor/github.com/hashicorp/go-getter/appveyor.yml b/vendor/github.com/hashicorp/go-getter/appveyor.yml index 159dad4dc..ec48d45ec 100644 --- a/vendor/github.com/hashicorp/go-getter/appveyor.yml +++ b/vendor/github.com/hashicorp/go-getter/appveyor.yml @@ -1,5 +1,5 @@ version: "build-{branch}-{build}" -image: Visual Studio 2015 +image: Visual Studio 2017 clone_folder: c:\gopath\github.com\hashicorp\go-getter environment: GOPATH: c:\gopath