Update go-getter and add support for git and hg

Fixes https://github.com/hashicorp/nomad/issues/2042

client/getter/getter.go

@@ -18,7 +18,7 @@ var (
 	lock    sync.Mutex
 
 	// supported is the set of download schemes supported by Nomad
-	supported = []string{"http", "https", "s3"}
+	supported = []string{"http", "https", "s3", "hg", "git"}
 )
 
 // getClient returns a client that is suitable for Nomad downloading artifacts.

client/task_runner.go

@@ -793,6 +793,7 @@ func (r *TaskRunner) prestart(resultCh chan bool) {
 			for _, artifact := range r.task.Artifacts {
 				if err := getter.GetArtifact(r.getTaskEnv(), artifact, r.taskDir.Dir); err != nil {
 					wrapped := fmt.Errorf("failed to download artifact %q: %v", artifact.GetterSource, err)
+					r.logger.Printf("[DEBUG] client: %v", wrapped)
 					r.setState(structs.TaskStatePending,
 						structs.NewTaskEvent(structs.TaskArtifactDownloadFailed).SetDownloadError(wrapped))
 					r.restartTracker.SetStartError(structs.NewRecoverableError(wrapped, true))

vendor/github.com/bgentry/go-netrc/LICENSE

@@ -0,0 +1,20 @@
+Original version Copyright © 2010 Fazlul Shahriar <fshahriar@gmail.com>. Newer
+portions Copyright © 2014 Blake Gentry <blakesgentry@gmail.com>.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/bgentry/go-netrc/netrc/netrc.go

@@ -0,0 +1,510 @@
+package netrc
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+	"sync"
+	"unicode"
+	"unicode/utf8"
+)
+
+type tkType int
+
+const (
+	tkMachine tkType = iota
+	tkDefault
+	tkLogin
+	tkPassword
+	tkAccount
+	tkMacdef
+	tkComment
+	tkWhitespace
+)
+
+var keywords = map[string]tkType{
+	"machine":  tkMachine,
+	"default":  tkDefault,
+	"login":    tkLogin,
+	"password": tkPassword,
+	"account":  tkAccount,
+	"macdef":   tkMacdef,
+	"#":        tkComment,
+}
+
+type Netrc struct {
+	tokens     []*token
+	machines   []*Machine
+	macros     Macros
+	updateLock sync.Mutex
+}
+
+// FindMachine returns the Machine in n named by name. If a machine named by
+// name exists, it is returned. If no Machine with name name is found and there
+// is a ``default'' machine, the ``default'' machine is returned. Otherwise, nil
+// is returned.
+func (n *Netrc) FindMachine(name string) (m *Machine) {
+	// TODO(bgentry): not safe for concurrency
+	var def *Machine
+	for _, m = range n.machines {
+		if m.Name == name {
+			return m
+		}
+		if m.IsDefault() {
+			def = m
+		}
+	}
+	if def == nil {
+		return nil
+	}
+	return def
+}
+
+// MarshalText implements the encoding.TextMarshaler interface to encode a
+// Netrc into text format.
+func (n *Netrc) MarshalText() (text []byte, err error) {
+	// TODO(bgentry): not safe for concurrency
+	for i := range n.tokens {
+		switch n.tokens[i].kind {
+		case tkComment, tkDefault, tkWhitespace: // always append these types
+			text = append(text, n.tokens[i].rawkind...)
+		default:
+			if n.tokens[i].value != "" { // skip empty-value tokens
+				text = append(text, n.tokens[i].rawkind...)
+			}
+		}
+		if n.tokens[i].kind == tkMacdef {
+			text = append(text, ' ')
+			text = append(text, n.tokens[i].macroName...)
+		}
+		text = append(text, n.tokens[i].rawvalue...)
+	}
+	return
+}
+
+func (n *Netrc) NewMachine(name, login, password, account string) *Machine {
+	n.updateLock.Lock()
+	defer n.updateLock.Unlock()
+
+	prefix := "\n"
+	if len(n.tokens) == 0 {
+		prefix = ""
+	}
+	m := &Machine{
+		Name:     name,
+		Login:    login,
+		Password: password,
+		Account:  account,
+
+		nametoken: &token{
+			kind:     tkMachine,
+			rawkind:  []byte(prefix + "machine"),
+			value:    name,
+			rawvalue: []byte(" " + name),
+		},
+		logintoken: &token{
+			kind:     tkLogin,
+			rawkind:  []byte("\n\tlogin"),
+			value:    login,
+			rawvalue: []byte(" " + login),
+		},
+		passtoken: &token{
+			kind:     tkPassword,
+			rawkind:  []byte("\n\tpassword"),
+			value:    password,
+			rawvalue: []byte(" " + password),
+		},
+		accounttoken: &token{
+			kind:     tkAccount,
+			rawkind:  []byte("\n\taccount"),
+			value:    account,
+			rawvalue: []byte(" " + account),
+		},
+	}
+	n.insertMachineTokensBeforeDefault(m)
+	for i := range n.machines {
+		if n.machines[i].IsDefault() {
+			n.machines = append(append(n.machines[:i], m), n.machines[i:]...)
+			return m
+		}
+	}
+	n.machines = append(n.machines, m)
+	return m
+}
+
+func (n *Netrc) insertMachineTokensBeforeDefault(m *Machine) {
+	newtokens := []*token{m.nametoken}
+	if m.logintoken.value != "" {
+		newtokens = append(newtokens, m.logintoken)
+	}
+	if m.passtoken.value != "" {
+		newtokens = append(newtokens, m.passtoken)
+	}
+	if m.accounttoken.value != "" {
+		newtokens = append(newtokens, m.accounttoken)
+	}
+	for i := range n.tokens {
+		if n.tokens[i].kind == tkDefault {
+			// found the default, now insert tokens before it
+			n.tokens = append(n.tokens[:i], append(newtokens, n.tokens[i:]...)...)
+			return
+		}
+	}
+	// didn't find a default, just add the newtokens to the end
+	n.tokens = append(n.tokens, newtokens...)
+	return
+}
+
+func (n *Netrc) RemoveMachine(name string) {
+	n.updateLock.Lock()
+	defer n.updateLock.Unlock()
+
+	for i := range n.machines {
+		if n.machines[i] != nil && n.machines[i].Name == name {
+			m := n.machines[i]
+			for _, t := range []*token{
+				m.nametoken, m.logintoken, m.passtoken, m.accounttoken,
+			} {
+				n.removeToken(t)
+			}
+			n.machines = append(n.machines[:i], n.machines[i+1:]...)
+			return
+		}
+	}
+}
+
+func (n *Netrc) removeToken(t *token) {
+	if t != nil {
+		for i := range n.tokens {
+			if n.tokens[i] == t {
+				n.tokens = append(n.tokens[:i], n.tokens[i+1:]...)
+				return
+			}
+		}
+	}
+}
+
+// Machine contains information about a remote machine.
+type Machine struct {
+	Name     string
+	Login    string
+	Password string
+	Account  string
+
+	nametoken    *token
+	logintoken   *token
+	passtoken    *token
+	accounttoken *token
+}
+
+// IsDefault returns true if the machine is a "default" token, denoted by an
+// empty name.
+func (m *Machine) IsDefault() bool {
+	return m.Name == ""
+}
+
+// UpdatePassword sets the password for the Machine m.
+func (m *Machine) UpdatePassword(newpass string) {
+	m.Password = newpass
+	updateTokenValue(m.passtoken, newpass)
+}
+
+// UpdateLogin sets the login for the Machine m.
+func (m *Machine) UpdateLogin(newlogin string) {
+	m.Login = newlogin
+	updateTokenValue(m.logintoken, newlogin)
+}
+
+// UpdateAccount sets the login for the Machine m.
+func (m *Machine) UpdateAccount(newaccount string) {
+	m.Account = newaccount
+	updateTokenValue(m.accounttoken, newaccount)
+}
+
+func updateTokenValue(t *token, value string) {
+	oldvalue := t.value
+	t.value = value
+	newraw := make([]byte, len(t.rawvalue))
+	copy(newraw, t.rawvalue)
+	t.rawvalue = append(
+		bytes.TrimSuffix(newraw, []byte(oldvalue)),
+		[]byte(value)...,
+	)
+}
+
+// Macros contains all the macro definitions in a netrc file.
+type Macros map[string]string
+
+type token struct {
+	kind      tkType
+	macroName string
+	value     string
+	rawkind   []byte
+	rawvalue  []byte
+}
+
+// Error represents a netrc file parse error.
+type Error struct {
+	LineNum int    // Line number
+	Msg     string // Error message
+}
+
+// Error returns a string representation of error e.
+func (e *Error) Error() string {
+	return fmt.Sprintf("line %d: %s", e.LineNum, e.Msg)
+}
+
+func (e *Error) BadDefaultOrder() bool {
+	return e.Msg == errBadDefaultOrder
+}
+
+const errBadDefaultOrder = "default token must appear after all machine tokens"
+
+// scanLinesKeepPrefix is a split function for a Scanner that returns each line
+// of text. The returned token may include newlines if they are before the
+// first non-space character. The returned line may be empty. The end-of-line
+// marker is one optional carriage return followed by one mandatory newline. In
+// regular expression notation, it is `\r?\n`. The last non-empty line of
+// input will be returned even if it has no newline.
+func scanLinesKeepPrefix(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	if atEOF && len(data) == 0 {
+		return 0, nil, nil
+	}
+	// Skip leading spaces.
+	start := 0
+	for width := 0; start < len(data); start += width {
+		var r rune
+		r, width = utf8.DecodeRune(data[start:])
+		if !unicode.IsSpace(r) {
+			break
+		}
+	}
+	if i := bytes.IndexByte(data[start:], '\n'); i >= 0 {
+		// We have a full newline-terminated line.
+		return start + i, data[0 : start+i], nil
+	}
+	// If we're at EOF, we have a final, non-terminated line. Return it.
+	if atEOF {
+		return len(data), data, nil
+	}
+	// Request more data.
+	return 0, nil, nil
+}
+
+// scanWordsKeepPrefix is a split function for a Scanner that returns each
+// space-separated word of text, with prefixing spaces included. It will never
+// return an empty string. The definition of space is set by unicode.IsSpace.
+//
+// Adapted from bufio.ScanWords().
+func scanTokensKeepPrefix(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	// Skip leading spaces.
+	start := 0
+	for width := 0; start < len(data); start += width {
+		var r rune
+		r, width = utf8.DecodeRune(data[start:])
+		if !unicode.IsSpace(r) {
+			break
+		}
+	}
+	if atEOF && len(data) == 0 || start == len(data) {
+		return len(data), data, nil
+	}
+	if len(data) > start && data[start] == '#' {
+		return scanLinesKeepPrefix(data, atEOF)
+	}
+	// Scan until space, marking end of word.
+	for width, i := 0, start; i < len(data); i += width {
+		var r rune
+		r, width = utf8.DecodeRune(data[i:])
+		if unicode.IsSpace(r) {
+			return i, data[:i], nil
+		}
+	}
+	// If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
+	if atEOF && len(data) > start {
+		return len(data), data, nil
+	}
+	// Request more data.
+	return 0, nil, nil
+}
+
+func newToken(rawb []byte) (*token, error) {
+	_, tkind, err := bufio.ScanWords(rawb, true)
+	if err != nil {
+		return nil, err
+	}
+	var ok bool
+	t := token{rawkind: rawb}
+	t.kind, ok = keywords[string(tkind)]
+	if !ok {
+		trimmed := strings.TrimSpace(string(tkind))
+		if trimmed == "" {
+			t.kind = tkWhitespace // whitespace-only, should happen only at EOF
+			return &t, nil
+		}
+		if strings.HasPrefix(trimmed, "#") {
+			t.kind = tkComment // this is a comment
+			return &t, nil
+		}
+		return &t, fmt.Errorf("keyword expected; got " + string(tkind))
+	}
+	return &t, nil
+}
+
+func scanValue(scanner *bufio.Scanner, pos int) ([]byte, string, int, error) {
+	if scanner.Scan() {
+		raw := scanner.Bytes()
+		pos += bytes.Count(raw, []byte{'\n'})
+		return raw, strings.TrimSpace(string(raw)), pos, nil
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, "", pos, &Error{pos, err.Error()}
+	}
+	return nil, "", pos, nil
+}
+
+func parse(r io.Reader, pos int) (*Netrc, error) {
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	nrc := Netrc{machines: make([]*Machine, 0, 20), macros: make(Macros, 10)}
+
+	defaultSeen := false
+	var currentMacro *token
+	var m *Machine
+	var t *token
+	scanner := bufio.NewScanner(bytes.NewReader(b))
+	scanner.Split(scanTokensKeepPrefix)
+
+	for scanner.Scan() {
+		rawb := scanner.Bytes()
+		if len(rawb) == 0 {
+			break
+		}
+		pos += bytes.Count(rawb, []byte{'\n'})
+		t, err = newToken(rawb)
+		if err != nil {
+			if currentMacro == nil {
+				return nil, &Error{pos, err.Error()}
+			}
+			currentMacro.rawvalue = append(currentMacro.rawvalue, rawb...)
+			continue
+		}
+
+		if currentMacro != nil && bytes.Contains(rawb, []byte{'\n', '\n'}) {
+			// if macro rawvalue + rawb would contain \n\n, then macro def is over
+			currentMacro.value = strings.TrimLeft(string(currentMacro.rawvalue), "\r\n")
+			nrc.macros[currentMacro.macroName] = currentMacro.value
+			currentMacro = nil
+		}
+
+		switch t.kind {
+		case tkMacdef:
+			if _, t.macroName, pos, err = scanValue(scanner, pos); err != nil {
+				return nil, &Error{pos, err.Error()}
+			}
+			currentMacro = t
+		case tkDefault:
+			if defaultSeen {
+				return nil, &Error{pos, "multiple default token"}
+			}
+			if m != nil {
+				nrc.machines, m = append(nrc.machines, m), nil
+			}
+			m = new(Machine)
+			m.Name = ""
+			defaultSeen = true
+		case tkMachine:
+			if defaultSeen {
+				return nil, &Error{pos, errBadDefaultOrder}
+			}
+			if m != nil {
+				nrc.machines, m = append(nrc.machines, m), nil
+			}
+			m = new(Machine)
+			if t.rawvalue, m.Name, pos, err = scanValue(scanner, pos); err != nil {
+				return nil, &Error{pos, err.Error()}
+			}
+			t.value = m.Name
+			m.nametoken = t
+		case tkLogin:
+			if m == nil || m.Login != "" {
+				return nil, &Error{pos, "unexpected token login "}
+			}
+			if t.rawvalue, m.Login, pos, err = scanValue(scanner, pos); err != nil {
+				return nil, &Error{pos, err.Error()}
+			}
+			t.value = m.Login
+			m.logintoken = t
+		case tkPassword:
+			if m == nil || m.Password != "" {
+				return nil, &Error{pos, "unexpected token password"}
+			}
+			if t.rawvalue, m.Password, pos, err = scanValue(scanner, pos); err != nil {
+				return nil, &Error{pos, err.Error()}
+			}
+			t.value = m.Password
+			m.passtoken = t
+		case tkAccount:
+			if m == nil || m.Account != "" {
+				return nil, &Error{pos, "unexpected token account"}
+			}
+			if t.rawvalue, m.Account, pos, err = scanValue(scanner, pos); err != nil {
+				return nil, &Error{pos, err.Error()}
+			}
+			t.value = m.Account
+			m.accounttoken = t
+		}
+
+		nrc.tokens = append(nrc.tokens, t)
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	if m != nil {
+		nrc.machines, m = append(nrc.machines, m), nil
+	}
+	return &nrc, nil
+}
+
+// ParseFile opens the file at filename and then passes its io.Reader to
+// Parse().
+func ParseFile(filename string) (*Netrc, error) {
+	fd, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer fd.Close()
+	return Parse(fd)
+}
+
+// Parse parses from the the Reader r as a netrc file and returns the set of
+// machine information and macros defined in it. The ``default'' machine,
+// which is intended to be used when no machine name matches, is identified
+// by an empty machine name. There can be only one ``default'' machine.
+//
+// If there is a parsing error, an Error is returned.
+func Parse(r io.Reader) (*Netrc, error) {
+	return parse(r, 1)
+}
+
+// FindMachine parses the netrc file identified by filename and returns the
+// Machine named by name. If a problem occurs parsing the file at filename, an
+// error is returned. If a machine named by name exists, it is returned. If no
+// Machine with name name is found and there is a ``default'' machine, the
+// ``default'' machine is returned. Otherwise, nil is returned.
+func FindMachine(filename, name string) (m *Machine, err error) {
+	n, err := ParseFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	return n.FindMachine(name), nil
+}

vendor/github.com/hashicorp/go-getter/.travis.yml

@@ -1,10 +0,0 @@
-sudo: false
-
-language: go
-
-go:
-  - 1.5
-
-branches:
-  only:
-    - master

vendor/github.com/hashicorp/go-getter/README.md

@@ -210,6 +210,12 @@ None
     a commit SHA, a branch name, etc. If it is a named ref such as a branch
     name, go-getter will update it to the latest on each get.
 
+  * `sshkey` - An SSH private key to use during clones. The provided key must
+    be a base64-encoded string. For example, to generate a suitable `sshkey`
+    from a private key file on disk, you would run `base64 -w0 <file>`.
+
+    **Note**: Git 2.3+ is required to use this feature.
+
 ### Mercurial (`hg`)
 
   * `rev` - The Mercurial revision to checkout.
@@ -227,3 +233,21 @@ the query parameters are present, these take priority.
   * `aws_access_key_id` - AWS access key.
   * `aws_access_key_secret` - AWS access key secret.
   * `aws_access_token` - AWS access token if this is being used.
+
+#### Using IAM Instance Profiles with S3
+
+If you use go-getter and want to use an EC2 IAM Instance Profile to avoid
+using credentials, then just omit these and the profile, if available will
+be used automatically.
+
+#### S3 Bucket Examples
+
+S3 has several addressing schemes used to reference your bucket. These are
+listed here: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro
+
+Some examples for these addressing schemes:
+- s3::https://s3.amazonaws.com/bucket/foo
+- s3::https://s3-eu-west-1.amazonaws.com/bucket/foo
+- bucket.s3.amazonaws.com/foo
+- bucket.s3-eu-west-1.amazonaws.com/foo/bar
+

vendor/github.com/hashicorp/go-getter/client.go

@@ -153,7 +153,7 @@ func (c *Client) Get() error {
 		// We don't appear to... but is it part of the filename?
 		matchingLen := 0
 		for k, _ := range decompressors {
-			if strings.HasSuffix(u.Path, k) && len(k) > matchingLen {
+			if strings.HasSuffix(u.Path, "."+k) && len(k) > matchingLen {
 				archiveV = k
 				matchingLen = len(k)
 			}
@@ -222,13 +222,18 @@ func (c *Client) Get() error {
 		checksumValue = b
 	}
 
-	// For now, any means file. In the future, we'll ask the getter
-	// what it thinks it is.
 	if mode == ClientModeAny {
-		mode = ClientModeFile
+		// Ask the getter which client mode to use
+		mode, err = g.ClientMode(u)
+		if err != nil {
+			return err
+		}
 
-		// Destination is the base name of the URL path
-		dst = filepath.Join(dst, filepath.Base(u.Path))
+		// Destination is the base name of the URL path in "any" mode when
+		// a file source is detected.
+		if mode == ClientModeFile {
+			dst = filepath.Join(dst, filepath.Base(u.Path))
+		}
 	}
 
 	// If we're not downloading a directory, then just download the file

vendor/github.com/hashicorp/go-getter/detect_file.go

@@ -36,6 +36,13 @@ func (d *FileDetector) Detect(src, pwd string) (string, bool, error) {
 				if err != nil {
 					return "", true, err
 				}
+
+				// The symlink itself might be a relative path, so we have to
+				// resolve this to have a correctly rooted URL.
+				pwd, err = filepath.Abs(pwd)
+				if err != nil {
+					return "", true, err
+				}
 			}
 		}
 

vendor/github.com/hashicorp/go-getter/get.go

@@ -35,6 +35,10 @@ type Getter interface {
 	// reference a single file. If possible, the Getter should check if
 	// the remote end contains the same file and no-op this operation.
 	GetFile(string, *url.URL) error
+
+	// ClientMode returns the mode based on the given URL. This is used to
+	// allow clients to let the getters decide which mode to use.
+	ClientMode(*url.URL) (ClientMode, error)
 }
 
 // Getters is the mapping of scheme to the Getter implementation that will
@@ -46,7 +50,7 @@ var Getters map[string]Getter
 var forcedRegexp = regexp.MustCompile(`^([A-Za-z0-9]+)::(.+)$`)
 
 func init() {
-	httpGetter := new(HttpGetter)
+	httpGetter := &HttpGetter{Netrc: true}
 
 	Getters = map[string]Getter{
 		"file":  new(FileGetter),

vendor/github.com/hashicorp/go-getter/get_file.go

@@ -1,8 +1,32 @@
 package getter
 
+import (
+	"net/url"
+	"os"
+)
+
 // FileGetter is a Getter implementation that will download a module from
 // a file scheme.
 type FileGetter struct {
 	// Copy, if set to true, will copy data instead of using a symlink
 	Copy bool
 }
+
+func (g *FileGetter) ClientMode(u *url.URL) (ClientMode, error) {
+	path := u.Path
+	if u.RawPath != "" {
+		path = u.RawPath
+	}
+
+	fi, err := os.Stat(path)
+	if err != nil {
+		return 0, err
+	}
+
+	// Check if the source is a directory.
+	if fi.IsDir() {
+		return ClientModeDir, nil
+	}
+
+	return ClientModeFile, nil
+}

vendor/github.com/hashicorp/go-getter/get_file_unix.go

@@ -11,8 +11,13 @@ import (
 )
 
 func (g *FileGetter) Get(dst string, u *url.URL) error {
+	path := u.Path
+	if u.RawPath != "" {
+		path = u.RawPath
+	}
+
 	// The source path must exist and be a directory to be usable.
-	if fi, err := os.Stat(u.Path); err != nil {
+	if fi, err := os.Stat(path); err != nil {
 		return fmt.Errorf("source path error: %s", err)
 	} else if !fi.IsDir() {
 		return fmt.Errorf("source path must be a directory")
@@ -41,12 +46,17 @@ func (g *FileGetter) Get(dst string, u *url.URL) error {
 		return err
 	}
 
-	return os.Symlink(u.Path, dst)
+	return os.Symlink(path, dst)
 }
 
 func (g *FileGetter) GetFile(dst string, u *url.URL) error {
-	// The source path must exist and be a directory to be usable.
-	if fi, err := os.Stat(u.Path); err != nil {
+	path := u.Path
+	if u.RawPath != "" {
+		path = u.RawPath
+	}
+
+	// The source path must exist and be a file to be usable.
+	if fi, err := os.Stat(path); err != nil {
 		return fmt.Errorf("source path error: %s", err)
 	} else if fi.IsDir() {
 		return fmt.Errorf("source path must be a file")
@@ -72,11 +82,11 @@ func (g *FileGetter) GetFile(dst string, u *url.URL) error {
 
 	// If we're not copying, just symlink and we're done
 	if !g.Copy {
-		return os.Symlink(u.Path, dst)
+		return os.Symlink(path, dst)
 	}
 
 	// Copy
-	srcF, err := os.Open(u.Path)
+	srcF, err := os.Open(path)
 	if err != nil {
 		return err
 	}

vendor/github.com/hashicorp/go-getter/get_file_windows.go

@@ -13,8 +13,13 @@ import (
 )
 
 func (g *FileGetter) Get(dst string, u *url.URL) error {
+	path := u.Path
+	if u.RawPath != "" {
+		path = u.RawPath
+	}
+
 	// The source path must exist and be a directory to be usable.
-	if fi, err := os.Stat(u.Path); err != nil {
+	if fi, err := os.Stat(path); err != nil {
 		return fmt.Errorf("source path error: %s", err)
 	} else if !fi.IsDir() {
 		return fmt.Errorf("source path must be a directory")
@@ -43,7 +48,7 @@ func (g *FileGetter) Get(dst string, u *url.URL) error {
 		return err
 	}
 
-	sourcePath := toBackslash(u.Path)
+	sourcePath := toBackslash(path)
 
 	// Use mklink to create a junction point
 	output, err := exec.Command("cmd", "/c", "mklink", "/J", dst, sourcePath).CombinedOutput()
@@ -55,8 +60,13 @@ func (g *FileGetter) Get(dst string, u *url.URL) error {
 }
 
 func (g *FileGetter) GetFile(dst string, u *url.URL) error {
+	path := u.Path
+	if u.RawPath != "" {
+		path = u.RawPath
+	}
+
 	// The source path must exist and be a directory to be usable.
-	if fi, err := os.Stat(u.Path); err != nil {
+	if fi, err := os.Stat(path); err != nil {
 		return fmt.Errorf("source path error: %s", err)
 	} else if fi.IsDir() {
 		return fmt.Errorf("source path must be a file")
@@ -82,11 +92,11 @@ func (g *FileGetter) GetFile(dst string, u *url.URL) error {
 
 	// If we're not copying, just symlink and we're done
 	if !g.Copy {
-		return os.Symlink(u.Path, dst)
+		return os.Symlink(path, dst)
 	}
 
 	// Copy
-	srcF, err := os.Open(u.Path)
+	srcF, err := os.Open(path)
 	if err != nil {
 		return err
 	}

vendor/github.com/hashicorp/go-getter/get_git.go

@@ -1,58 +1,105 @@
 package getter
 
 import (
+	"encoding/base64"
 	"fmt"
 	"io/ioutil"
 	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 
 	urlhelper "github.com/hashicorp/go-getter/helper/url"
+	"github.com/hashicorp/go-version"
 )
 
 // GitGetter is a Getter implementation that will download a module from
 // a git repository.
 type GitGetter struct{}
 
+func (g *GitGetter) ClientMode(_ *url.URL) (ClientMode, error) {
+	return ClientModeDir, nil
+}
+
 func (g *GitGetter) Get(dst string, u *url.URL) error {
 	if _, err := exec.LookPath("git"); err != nil {
 		return fmt.Errorf("git must be available and on the PATH")
 	}
 
 	// Extract some query parameters we use
-	var ref string
+	var ref, sshKey string
 	q := u.Query()
 	if len(q) > 0 {
 		ref = q.Get("ref")
 		q.Del("ref")
 
+		sshKey = q.Get("sshkey")
+		q.Del("sshkey")
+
 		// Copy the URL
 		var newU url.URL = *u
 		u = &newU
 		u.RawQuery = q.Encode()
 	}
 
-	// First: clone or update the repository
+	var sshKeyFile string
+	if sshKey != "" {
+		// Check that the git version is sufficiently new.
+		if err := checkGitVersion("2.3"); err != nil {
+			return fmt.Errorf("Error using ssh key: %v", err)
+		}
+
+		// We have an SSH key - decode it.
+		raw, err := base64.StdEncoding.DecodeString(sshKey)
+		if err != nil {
+			return err
+		}
+
+		// Create a temp file for the key and ensure it is removed.
+		fh, err := ioutil.TempFile("", "go-getter")
+		if err != nil {
+			return err
+		}
+		sshKeyFile = fh.Name()
+		defer os.Remove(sshKeyFile)
+
+		// Set the permissions prior to writing the key material.
+		if err := os.Chmod(sshKeyFile, 0600); err != nil {
+			return err
+		}
+
+		// Write the raw key into the temp file.
+		_, err = fh.Write(raw)
+		fh.Close()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Clone or update the repository
 	_, err := os.Stat(dst)
 	if err != nil && !os.IsNotExist(err) {
 		return err
 	}
 	if err == nil {
-		err = g.update(dst, ref)
+		err = g.update(dst, sshKeyFile, ref)
 	} else {
-		err = g.clone(dst, u)
+		err = g.clone(dst, sshKeyFile, u)
 	}
 	if err != nil {
 		return err
 	}
 
 	// Next: check out the proper tag/branch if it is specified, and checkout
-	if ref == "" {
-		return nil
+	if ref != "" {
+		if err := g.checkout(dst, ref); err != nil {
+			return err
+		}
 	}
 
-	return g.checkout(dst, ref)
+	// Lastly, download any/all submodules.
+	return g.fetchSubmodules(dst, sshKeyFile)
 }
 
 // GetFile for Git doesn't support updating at this time. It will download
@@ -92,16 +139,18 @@ func (g *GitGetter) checkout(dst string, ref string) error {
 	return getRunCommand(cmd)
 }
 
-func (g *GitGetter) clone(dst string, u *url.URL) error {
+func (g *GitGetter) clone(dst, sshKeyFile string, u *url.URL) error {
 	cmd := exec.Command("git", "clone", u.String(), dst)
+	setupGitEnv(cmd, sshKeyFile)
 	return getRunCommand(cmd)
 }
 
-func (g *GitGetter) update(dst string, ref string) error {
+func (g *GitGetter) update(dst, sshKeyFile, ref string) error {
 	// Determine if we're a branch. If we're NOT a branch, then we just
 	// switch to master prior to checking out
 	cmd := exec.Command("git", "show-ref", "-q", "--verify", "refs/heads/"+ref)
 	cmd.Dir = dst
+
 	if getRunCommand(cmd) != nil {
 		// Not a branch, switch to master. This will also catch non-existent
 		// branches, in which case we want to switch to master and then
@@ -116,5 +165,61 @@ func (g *GitGetter) update(dst string, ref string) error {
 
 	cmd = exec.Command("git", "pull", "--ff-only")
 	cmd.Dir = dst
+	setupGitEnv(cmd, sshKeyFile)
 	return getRunCommand(cmd)
 }
+
+// fetchSubmodules downloads any configured submodules recursively.
+func (g *GitGetter) fetchSubmodules(dst, sshKeyFile string) error {
+	cmd := exec.Command("git", "submodule", "update", "--init", "--recursive")
+	cmd.Dir = dst
+	setupGitEnv(cmd, sshKeyFile)
+	return getRunCommand(cmd)
+}
+
+// setupGitEnv sets up the environment for the given command. This is used to
+// pass configuration data to git and ssh and enables advanced cloning methods.
+func setupGitEnv(cmd *exec.Cmd, sshKeyFile string) {
+	var sshOpts []string
+
+	if sshKeyFile != "" {
+		// We have an SSH key temp file configured, tell ssh about this.
+		sshOpts = append(sshOpts, "-i", sshKeyFile)
+	}
+
+	cmd.Env = append(os.Environ(),
+		// Set the ssh command to use for clones.
+		"GIT_SSH_COMMAND=ssh "+strings.Join(sshOpts, " "),
+	)
+}
+
+// checkGitVersion is used to check the version of git installed on the system
+// against a known minimum version. Returns an error if the installed version
+// is older than the given minimum.
+func checkGitVersion(min string) error {
+	want, err := version.NewVersion(min)
+	if err != nil {
+		return err
+	}
+
+	out, err := exec.Command("git", "version").Output()
+	if err != nil {
+		return err
+	}
+
+	fields := strings.Fields(string(out))
+	if len(fields) != 3 {
+		return fmt.Errorf("Unexpected 'git version' output: %q", string(out))
+	}
+
+	have, err := version.NewVersion(fields[2])
+	if err != nil {
+		return err
+	}
+
+	if have.LessThan(want) {
+		return fmt.Errorf("Required git version = %s, have %s", want, have)
+	}
+
+	return nil
+}

vendor/github.com/hashicorp/go-getter/get_hg.go

@@ -16,6 +16,10 @@ import (
 // a Mercurial repository.
 type HgGetter struct{}
 
+func (g *HgGetter) ClientMode(_ *url.URL) (ClientMode, error) {
+	return ClientModeDir, nil
+}
+
 func (g *HgGetter) Get(dst string, u *url.URL) error {
 	if _, err := exec.LookPath("hg"); err != nil {
 		return fmt.Errorf("hg must be available and on the PATH")

vendor/github.com/hashicorp/go-getter/get_http.go

@@ -32,13 +32,31 @@ import (
 // The source URL, whether from the header or meta tag, must be a fully
 // formed URL. The shorthand syntax of "github.com/foo/bar" or relative
 // paths are not allowed.
-type HttpGetter struct{}
+type HttpGetter struct {
+	// Netrc, if true, will lookup and use auth information found
+	// in the user's netrc file if available.
+	Netrc bool
+}
+
+func (g *HttpGetter) ClientMode(u *url.URL) (ClientMode, error) {
+	if strings.HasSuffix(u.Path, "/") {
+		return ClientModeDir, nil
+	}
+	return ClientModeFile, nil
+}
 
 func (g *HttpGetter) Get(dst string, u *url.URL) error {
 	// Copy the URL so we can modify it
 	var newU url.URL = *u
 	u = &newU
 
+	if g.Netrc {
+		// Add auth from netrc if we can
+		if err := addAuthFromNetrc(u); err != nil {
+			return err
+		}
+	}
+
 	// Add terraform-get to the parameter.
 	q := u.Query()
 	q.Add("terraform-get", "1")

vendor/github.com/hashicorp/go-getter/get_mock.go

@@ -43,3 +43,10 @@ func (g *MockGetter) GetFile(dst string, u *url.URL) error {
 	}
 	return g.GetFileErr
 }
+
+func (g *MockGetter) ClientMode(u *url.URL) (ClientMode, error) {
+	if l := len(u.Path); l > 0 && u.Path[l-1:] == "/" {
+		return ClientModeDir, nil
+	}
+	return ClientModeFile, nil
+}

vendor/github.com/hashicorp/go-getter/get_s3.go

@@ -20,6 +20,45 @@ import (
 // a S3 bucket.
 type S3Getter struct{}
 
+func (g *S3Getter) ClientMode(u *url.URL) (ClientMode, error) {
+	// Parse URL
+	region, bucket, path, _, creds, err := g.parseUrl(u)
+	if err != nil {
+		return 0, err
+	}
+
+	// Create client config
+	config := g.getAWSConfig(region, creds)
+	sess := session.New(config)
+	client := s3.New(sess)
+
+	// List the object(s) at the given prefix
+	req := &s3.ListObjectsInput{
+		Bucket: aws.String(bucket),
+		Prefix: aws.String(path),
+	}
+	resp, err := client.ListObjects(req)
+	if err != nil {
+		return 0, err
+	}
+
+	for _, o := range resp.Contents {
+		// Use file mode on exact match.
+		if *o.Key == path {
+			return ClientModeFile, nil
+		}
+
+		// Use dir mode if child keys are found.
+		if strings.HasPrefix(*o.Key, path+"/") {
+			return ClientModeDir, nil
+		}
+	}
+
+	// There was no match, so just return file mode. The download is going
+	// to fail but we will let S3 return the proper error later.
+	return ClientModeFile, nil
+}
+
 func (g *S3Getter) Get(dst string, u *url.URL) error {
 	// Parse URL
 	region, bucket, path, _, creds, err := g.parseUrl(u)

vendor/github.com/hashicorp/go-getter/netrc.go

@@ -0,0 +1,67 @@
+package getter
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"runtime"
+
+	"github.com/bgentry/go-netrc/netrc"
+	"github.com/mitchellh/go-homedir"
+)
+
+// addAuthFromNetrc adds auth information to the URL from the user's
+// netrc file if it can be found. This will only add the auth info
+// if the URL doesn't already have auth info specified and the
+// the username is blank.
+func addAuthFromNetrc(u *url.URL) error {
+	// If the URL already has auth information, do nothing
+	if u.User != nil && u.User.Username() != "" {
+		return nil
+	}
+
+	// Get the netrc file path
+	path := os.Getenv("NETRC")
+	if path == "" {
+		filename := ".netrc"
+		if runtime.GOOS == "windows" {
+			filename = "_netrc"
+		}
+
+		var err error
+		path, err = homedir.Expand("~/" + filename)
+		if err != nil {
+			return err
+		}
+	}
+
+	// If the file is not a file, then do nothing
+	if fi, err := os.Stat(path); err != nil {
+		// File doesn't exist, do nothing
+		if os.IsNotExist(err) {
+			return nil
+		}
+
+		// Some other error!
+		return err
+	} else if fi.IsDir() {
+		// File is directory, ignore
+		return nil
+	}
+
+	// Load up the netrc file
+	net, err := netrc.ParseFile(path)
+	if err != nil {
+		return fmt.Errorf("Error parsing netrc file at %q: %s", path, err)
+	}
+
+	machine := net.FindMachine(u.Host)
+	if machine == nil {
+		// Machine not found, no problem
+		return nil
+	}
+
+	// Set the user info
+	u.User = url.UserPassword(machine.Login, machine.Password)
+	return nil
+}

vendor/vendor.json

@@ -200,6 +200,12 @@
 			"path": "github.com/aws/aws-sdk-go/service/s3",
 			"revision": "80dd4951fdb3f711d31843b8d87871130ef2df67"
 		},
+		{
+			"checksumSHA1": "nqw2Qn5xUklssHTubS5HDvEL9L4=",
+			"path": "github.com/bgentry/go-netrc/netrc",
+			"revision": "9fd32a8b3d3d3f9d43c341bfe098430e07609480",
+			"revisionTime": "2014-04-22T17:41:19Z"
+		},
 		{
 			"path": "github.com/bgentry/speakeasy",
 			"revision": "36e9cfdd690967f4f690c6edcc9ffacd006014a0"
@@ -354,14 +360,14 @@
 		{
 			"checksumSHA1": "iP5slJJPRZUm0rfdII8OiATAACA=",
 			"path": "github.com/docker/docker/pkg/idtools",
-			"revision": "52debcd58ac91bf68503ce60561536911b74ff05",
-			"revisionTime": "2016-05-20T15:17:10Z"
+			"revision": "02caa73df411debed164f520a6a1304778f8b88c",
+			"revisionTime": "2016-05-28T10:48:36Z"
 		},
 		{
 			"checksumSHA1": "iP5slJJPRZUm0rfdII8OiATAACA=",
 			"path": "github.com/docker/docker/pkg/idtools",
-			"revision": "02caa73df411debed164f520a6a1304778f8b88c",
-			"revisionTime": "2016-05-28T10:48:36Z"
+			"revision": "52debcd58ac91bf68503ce60561536911b74ff05",
+			"revisionTime": "2016-05-20T15:17:10Z"
 		},
 		{
 			"checksumSHA1": "tdhmIGUaoOMEDymMC23qTS7bt0g=",
@@ -396,18 +402,12 @@
 		{
 			"checksumSHA1": "rArZ5mYIe9I1L5PRQOJu8BwafFw=",
 			"path": "github.com/docker/docker/pkg/pools",
-			"revision": "da39e9a4f920a15683dd0f23923c302d4db6eed5",
-			"revisionTime": "2016-05-28T08:11:04Z"
+			"revision": "52debcd58ac91bf68503ce60561536911b74ff05",
+			"revisionTime": "2016-05-20T15:17:10Z"
 		},
 		{
 			"checksumSHA1": "rArZ5mYIe9I1L5PRQOJu8BwafFw=",
 			"path": "github.com/docker/docker/pkg/pools",
-			"revision": "52debcd58ac91bf68503ce60561536911b74ff05",
-			"revisionTime": "2016-05-20T15:17:10Z"
-		},
-		{
-			"checksumSHA1": "txf3EORYff4hO6PEvwBm2lyh1MU=",
-			"path": "github.com/docker/docker/pkg/promise",
 			"revision": "da39e9a4f920a15683dd0f23923c302d4db6eed5",
 			"revisionTime": "2016-05-28T08:11:04Z"
 		},
@@ -417,6 +417,12 @@
 			"revision": "52debcd58ac91bf68503ce60561536911b74ff05",
 			"revisionTime": "2016-05-20T15:17:10Z"
 		},
+		{
+			"checksumSHA1": "txf3EORYff4hO6PEvwBm2lyh1MU=",
+			"path": "github.com/docker/docker/pkg/promise",
+			"revision": "da39e9a4f920a15683dd0f23923c302d4db6eed5",
+			"revisionTime": "2016-05-28T08:11:04Z"
+		},
 		{
 			"checksumSHA1": "lThih54jzz9A4zHKEFb9SIV3Ed0=",
 			"path": "github.com/docker/docker/pkg/random",
@@ -678,8 +684,10 @@
 			"revisionTime": "2017-02-11T00:33:01Z"
 		},
 		{
+			"checksumSHA1": "nsL2kI426RMuq1jw15e7igFqdIY=",
 			"path": "github.com/hashicorp/go-getter",
-			"revision": "3142ddc1d627a166970ddd301bc09cb510c74edc"
+			"revision": "c3d66e76678dce180a7b452653472f949aedfbcd",
+			"revisionTime": "2017-02-07T21:55:32Z"
 		},
 		{
 			"path": "github.com/hashicorp/go-getter/helper/url",

website/source/docs/job-specification/artifact.html.md

@@ -40,9 +40,9 @@ job "docs" {
 }
 ```
 
-Nomad supports downloading `http`, `https`, and `S3` artifacts. If these
-artifacts are archived (`zip`, `tgz`, `bz2`), they are automatically unarchived
-before the starting the task.
+Nomad supports downloading `http`, `https`, `git`, `hg` and `S3` artifacts. If
+these artifacts are archived (`zip`, `tgz`, `bz2`), they are automatically
+unarchived before the starting the task.
 
 ## `artifact` Parameters