diff --git a/CHANGELOG.md b/CHANGELOG.md index 92015c50a..8e50b11ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,14 @@ IMPROVEMENTS: * core: Allow count zero task groups to enable blue/green deploys [GH-931] + * cli: `alloc-status` display is less verbose by default [GH-946] * cli: `server-members` displays the current leader in each region [GH-935] + * cli: New `inspect` command to display a submitted job's specification + [GH-952] + * cli: `node-status` display is less verbose by default and shows a node's + total resources [GH-946] + * client: `artifact` block now supports downloading paths relative to the + task's directory [GH-944] ## 0.3.1 diff --git a/Vagrantfile b/Vagrantfile index 58b00db46..275748026 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -57,6 +57,9 @@ sudo apt-get install -y docker-engine # Restart docker to make sure we get the latest version of the daemon if there is an upgrade sudo service docker restart +# Install rkt +bash /opt/gopath/src/github.com/hashicorp/nomad/scripts/install_rkt.sh + # Make sure we can actually use docker as the vagrant user sudo usermod -aG docker vagrant diff --git a/api/api.go b/api/api.go index 32d9b87b0..f21455bba 100644 --- a/api/api.go +++ b/api/api.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "net/http" "net/url" "os" @@ -34,6 +35,9 @@ type QueryOptions struct { // If set, used as prefix for resource list searches Prefix string + + // If set, pretty print the response json. + Pretty bool } // WriteOptions are used to parameterize a write @@ -156,6 +160,9 @@ func (r *request) setQueryOptions(q *QueryOptions) { if q.Prefix != "" { r.params.Set("prefix", q.Prefix) } + if q.Pretty { + r.params.Set("pretty", "true") + } } // durToMsec converts a duration to a millisecond specified string @@ -265,6 +272,29 @@ func (c *Client) query(endpoint string, out interface{}, q *QueryOptions) (*Quer return qm, nil } +// rawQuery is used to do a GET request against an endpoint and return the raw +// string result. +func (c *Client) rawQuery(endpoint string, q *QueryOptions) (string, *QueryMeta, error) { + r := c.newRequest("GET", endpoint) + r.setQueryOptions(q) + rtt, resp, err := requireOK(c.doRequest(r)) + if err != nil { + return "", nil, err + } + defer resp.Body.Close() + + qm := &QueryMeta{} + parseQueryMeta(resp, qm) + qm.RequestTime = rtt + + raw, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", nil, err + } + + return string(raw), qm, nil +} + // write is used to do a PUT request against an endpoint // and serialize/deserialized using the standard Nomad conventions. func (c *Client) write(endpoint string, in, out interface{}, q *WriteOptions) (*WriteMeta, error) { diff --git a/api/jobs.go b/api/jobs.go index a00c3a169..1af973dd9 100644 --- a/api/jobs.go +++ b/api/jobs.go @@ -63,6 +63,20 @@ func (j *Jobs) Info(jobID string, q *QueryOptions) (*Job, *QueryMeta, error) { return &resp, qm, nil } +// RawJob is used to retrieve information about a particular +// job given its unique ID and return the raw json. +func (j *Jobs) RawJob(jobID string, q *QueryOptions) (string, *QueryMeta, error) { + if q == nil { + q = &QueryOptions{} + } + q.Pretty = true + raw, qm, err := j.client.rawQuery("/v1/job/"+jobID, q) + if err != nil { + return "", nil, err + } + return raw, qm, nil +} + // Allocations is used to return the allocs for a given job ID. func (j *Jobs) Allocations(jobID string, q *QueryOptions) ([]*AllocationListStub, *QueryMeta, error) { var resp []*AllocationListStub diff --git a/api/jobs_test.go b/api/jobs_test.go index cb998ac89..d696f2c6e 100644 --- a/api/jobs_test.go +++ b/api/jobs_test.go @@ -83,6 +83,39 @@ func TestJobs_Info(t *testing.T) { } } +func TestJobs_RawJob(t *testing.T) { + c, s := makeClient(t, nil, nil) + defer s.Stop() + jobs := c.Jobs() + + // Trying to retrieve a job by ID before it exists + // returns an error + _, _, err := jobs.RawJob("job1", nil) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("expected not found error, got: %#v", err) + } + + // Register the job + job := testJob() + _, wm, err := jobs.Register(job, nil) + if err != nil { + t.Fatalf("err: %s", err) + } + assertWriteMeta(t, wm) + + // Query the job again and ensure it exists + result, qm, err := jobs.RawJob("job1", nil) + if err != nil { + t.Fatalf("err: %s", err) + } + assertQueryMeta(t, qm) + + // Check that the result is what we expect + if result == "" || !strings.Contains(result, job.ID) { + t.Fatalf("expect: %#v, got: %#v", job, result) + } +} + func TestJobs_PrefixList(t *testing.T) { c, s := makeClient(t, nil, nil) defer s.Stop() diff --git a/api/tasks.go b/api/tasks.go index d09339f34..84765c71a 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -99,6 +99,7 @@ type Task struct { type TaskArtifact struct { GetterSource string GetterOptions map[string]string + RelativeDest string } // NewTask creates and initializes a new Task. diff --git a/client/driver/exec.go b/client/driver/exec.go index f097d0079..672f0e96c 100644 --- a/client/driver/exec.go +++ b/client/driver/exec.go @@ -113,7 +113,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, }, executorCtx) if err != nil { pluginClient.Kill() - return nil, fmt.Errorf("error starting process via the plugin: %v", err) + return nil, err } d.logger.Printf("[DEBUG] driver.exec: started process via plugin with pid: %v", ps.Pid) diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go index ceba6b0ea..904bd3448 100644 --- a/client/driver/executor/executor.go +++ b/client/driver/executor/executor.go @@ -7,6 +7,7 @@ import ( "net" "os" "os/exec" + "path/filepath" "runtime" "strings" "sync" @@ -159,20 +160,36 @@ func (e *UniversalExecutor) LaunchCmd(command *ExecCommand, ctx *ExecutorContext e.cmd.Stdout = e.lro e.cmd.Stderr = e.lre - // setting the env, path and args for the command e.ctx.TaskEnv.Build() - e.cmd.Env = ctx.TaskEnv.EnvList() - e.cmd.Path = ctx.TaskEnv.ReplaceEnv(command.Cmd) - e.cmd.Args = append([]string{e.cmd.Path}, ctx.TaskEnv.ParseAndReplace(command.Args)...) - // Ensure that the binary being started is executable. - if err := e.makeExecutable(e.cmd.Path); err != nil { + // Look up the binary path and make it executable + absPath, err := e.lookupBin(ctx.TaskEnv.ReplaceEnv(command.Cmd)) + if err != nil { return nil, err } - // starting the process + if err := e.makeExecutable(absPath); err != nil { + return nil, err + } + + // Determine the path to run as it may have to be relative to the chroot. + path := absPath + if e.command.FSIsolation { + rel, err := filepath.Rel(e.taskDir, absPath) + if err != nil { + return nil, err + } + path = rel + } + + // Set the commands arguments + e.cmd.Path = path + e.cmd.Args = append([]string{path}, ctx.TaskEnv.ParseAndReplace(command.Args)...) + e.cmd.Env = ctx.TaskEnv.EnvList() + + // Start the process if err := e.cmd.Start(); err != nil { - return nil, fmt.Errorf("error starting command: %v", err) + return nil, err } go e.wait() ic := &cstructs.IsolationConfig{Cgroup: e.groups} @@ -328,8 +345,36 @@ func (e *UniversalExecutor) configureTaskDir() error { return nil } -// makeExecutablePosix makes the given file executable for root,group,others. -func (e *UniversalExecutor) makeExecutablePosix(binPath string) error { +// lookupBin looks for path to the binary to run by looking for the binary in +// the following locations, in-order: task/local/, task/, based on host $PATH. +// The return path is absolute. +func (e *UniversalExecutor) lookupBin(bin string) (string, error) { + // Check in the local directory + local := filepath.Join(e.taskDir, allocdir.TaskLocal, bin) + if _, err := os.Stat(local); err == nil { + return local, nil + } + + // Check at the root of the task's directory + root := filepath.Join(e.taskDir, bin) + if _, err := os.Stat(root); err == nil { + return root, nil + } + + // Check the $PATH + if host, err := exec.LookPath(bin); err == nil { + return host, nil + } + + return "", fmt.Errorf("binary %q could not be found", bin) +} + +// makeExecutable makes the given file executable for root,group,others. +func (e *UniversalExecutor) makeExecutable(binPath string) error { + if runtime.GOOS == "windows" { + return nil + } + fi, err := os.Stat(binPath) if err != nil { if os.IsNotExist(err) { diff --git a/client/driver/executor/executor_basic.go b/client/driver/executor/executor_basic.go index 00480e9dc..7d59ee366 100644 --- a/client/driver/executor/executor_basic.go +++ b/client/driver/executor/executor_basic.go @@ -2,25 +2,7 @@ package executor -import ( - "path/filepath" - "runtime" - - cgroupConfig "github.com/opencontainers/runc/libcontainer/configs" -) - -func (e *UniversalExecutor) makeExecutable(binPath string) error { - if runtime.GOOS == "windows" { - return nil - } - - path := binPath - if !filepath.IsAbs(binPath) { - // The path must be relative the allocations directory. - path = filepath.Join(e.taskDir, binPath) - } - return e.makeExecutablePosix(path) -} +import cgroupConfig "github.com/opencontainers/runc/libcontainer/configs" func (e *UniversalExecutor) configureChroot() error { return nil diff --git a/client/driver/executor/executor_linux.go b/client/driver/executor/executor_linux.go index d6c4cf38b..48ec5f729 100644 --- a/client/driver/executor/executor_linux.go +++ b/client/driver/executor/executor_linux.go @@ -36,18 +36,6 @@ var ( } ) -func (e *UniversalExecutor) makeExecutable(binPath string) error { - path := binPath - if e.command.FSIsolation { - // The path must be relative the chroot - path = filepath.Join(e.taskDir, binPath) - } else if !filepath.IsAbs(binPath) { - // The path must be relative the allocations directory. - path = filepath.Join(e.taskDir, binPath) - } - return e.makeExecutablePosix(path) -} - // configureIsolation configures chroot and creates cgroups func (e *UniversalExecutor) configureIsolation() error { if e.command.FSIsolation { diff --git a/client/driver/executor/executor_test.go b/client/driver/executor/executor_test.go index 0a43e3494..f60f97807 100644 --- a/client/driver/executor/executor_test.go +++ b/client/driver/executor/executor_test.go @@ -11,11 +11,11 @@ import ( "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/driver/env" + cstructs "github.com/hashicorp/nomad/client/driver/structs" "github.com/hashicorp/nomad/client/testutil" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" tu "github.com/hashicorp/nomad/testutil" - cstructs "github.com/hashicorp/nomad/client/driver/structs" ) var ( @@ -232,3 +232,38 @@ func TestExecutor_Start_Kill(t *testing.T) { t.Fatalf("Command output incorrectly: want %v; got %v", expected, act) } } + +func TestExecutor_MakeExecutable(t *testing.T) { + // Create a temp file + f, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal(err) + } + defer f.Close() + defer os.Remove(f.Name()) + + // Set its permissions to be non-executable + f.Chmod(os.FileMode(0610)) + + // Make a fake exececutor + ctx := testExecutorContext(t) + defer ctx.AllocDir.Destroy() + executor := NewExecutor(log.New(os.Stdout, "", log.LstdFlags)) + + err = executor.(*UniversalExecutor).makeExecutable(f.Name()) + if err != nil { + t.Fatalf("makeExecutable() failed: %v", err) + } + + // Check the permissions + stat, err := f.Stat() + if err != nil { + t.Fatalf("Stat() failed: %v", err) + } + + act := stat.Mode().Perm() + exp := os.FileMode(0755) + if act != exp { + t.Fatalf("expected permissions %v; got %v", err) + } +} diff --git a/client/driver/java.go b/client/driver/java.go index 19b66ff84..d4863df57 100644 --- a/client/driver/java.go +++ b/client/driver/java.go @@ -173,7 +173,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, }, executorCtx) if err != nil { pluginClient.Kill() - return nil, fmt.Errorf("error starting process via the plugin: %v", err) + return nil, err } d.logger.Printf("[DEBUG] driver.java: started process with pid: %v", ps.Pid) diff --git a/client/driver/qemu.go b/client/driver/qemu.go index 47d0ed5c5..60c3354d3 100644 --- a/client/driver/qemu.go +++ b/client/driver/qemu.go @@ -198,7 +198,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, ps, err := exec.LaunchCmd(&executor.ExecCommand{Cmd: args[0], Args: args[1:]}, executorCtx) if err != nil { pluginClient.Kill() - return nil, fmt.Errorf("error starting process via the plugin: %v", err) + return nil, err } d.logger.Printf("[INFO] Started new QemuVM: %s", vmID) diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go index 564d8061a..2dc0aceeb 100644 --- a/client/driver/raw_exec.go +++ b/client/driver/raw_exec.go @@ -103,7 +103,7 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl ps, err := exec.LaunchCmd(&executor.ExecCommand{Cmd: command, Args: driverConfig.Args}, executorCtx) if err != nil { pluginClient.Kill() - return nil, fmt.Errorf("error starting process via the plugin: %v", err) + return nil, err } d.logger.Printf("[DEBUG] driver.raw_exec: started process with pid: %v", ps.Pid) diff --git a/client/driver/rkt.go b/client/driver/rkt.go index 1c73400ac..76a1fa64e 100644 --- a/client/driver/rkt.go +++ b/client/driver/rkt.go @@ -246,7 +246,7 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e ps, err := execIntf.LaunchCmd(&executor.ExecCommand{Cmd: absPath, Args: cmdArgs}, executorCtx) if err != nil { pluginClient.Kill() - return nil, fmt.Errorf("error starting process via the plugin: %v", err) + return nil, err } d.logger.Printf("[DEBUG] driver.rkt: started ACI %q with: %v", img, cmdArgs) diff --git a/client/getter/getter.go b/client/getter/getter.go index 92d295f47..22a3f369e 100644 --- a/client/getter/getter.go +++ b/client/getter/getter.go @@ -4,6 +4,7 @@ import ( "fmt" "log" "net/url" + "path/filepath" "sync" gg "github.com/hashicorp/go-getter" @@ -59,16 +60,17 @@ func getGetterUrl(artifact *structs.TaskArtifact) (string, error) { return u.String(), nil } -// GetArtifact downloads an artifact into the specified destination directory. -func GetArtifact(artifact *structs.TaskArtifact, destDir string, logger *log.Logger) error { +// GetArtifact downloads an artifact into the specified task directory. +func GetArtifact(artifact *structs.TaskArtifact, taskDir string, logger *log.Logger) error { url, err := getGetterUrl(artifact) if err != nil { return err } // Download the artifact - if err := getClient(url, destDir).Get(); err != nil { - return err + dest := filepath.Join(taskDir, artifact.RelativeDest) + if err := getClient(url, dest).Get(); err != nil { + return fmt.Errorf("GET error: %v", err) } return nil diff --git a/client/getter/getter_test.go b/client/getter/getter_test.go index 7282000df..208f9efe5 100644 --- a/client/getter/getter_test.go +++ b/client/getter/getter_test.go @@ -21,11 +21,11 @@ func TestGetArtifact_FileAndChecksum(t *testing.T) { defer ts.Close() // Create a temp directory to download into - destDir, err := ioutil.TempDir("", "nomad-test") + taskDir, err := ioutil.TempDir("", "nomad-test") if err != nil { t.Fatalf("failed to make temp directory: %v", err) } - defer os.RemoveAll(destDir) + defer os.RemoveAll(taskDir) // Create the artifact file := "test.sh" @@ -38,13 +38,48 @@ func TestGetArtifact_FileAndChecksum(t *testing.T) { // Download the artifact logger := log.New(os.Stderr, "", log.LstdFlags) - if err := GetArtifact(artifact, destDir, logger); err != nil { + if err := GetArtifact(artifact, taskDir, logger); err != nil { t.Fatalf("GetArtifact failed: %v", err) } // Verify artifact exists - if _, err := os.Stat(filepath.Join(destDir, file)); err != nil { - t.Fatalf("source path error: %s", err) + if _, err := os.Stat(filepath.Join(taskDir, file)); err != nil { + t.Fatalf("file not found: %s", err) + } +} + +func TestGetArtifact_File_RelativeDest(t *testing.T) { + // Create the test server hosting the file to download + ts := httptest.NewServer(http.FileServer(http.Dir(filepath.Dir("./test-fixtures/")))) + defer ts.Close() + + // Create a temp directory to download into + taskDir, err := ioutil.TempDir("", "nomad-test") + if err != nil { + t.Fatalf("failed to make temp directory: %v", err) + } + defer os.RemoveAll(taskDir) + + // Create the artifact + file := "test.sh" + relative := "foo/" + artifact := &structs.TaskArtifact{ + GetterSource: fmt.Sprintf("%s/%s", ts.URL, file), + GetterOptions: map[string]string{ + "checksum": "md5:bce963762aa2dbfed13caf492a45fb72", + }, + RelativeDest: relative, + } + + // Download the artifact + logger := log.New(os.Stderr, "", log.LstdFlags) + if err := GetArtifact(artifact, taskDir, logger); err != nil { + t.Fatalf("GetArtifact failed: %v", err) + } + + // Verify artifact was downloaded to the correct path + if _, err := os.Stat(filepath.Join(taskDir, relative, file)); err != nil { + t.Fatalf("file not found: %s", err) } } @@ -54,11 +89,11 @@ func TestGetArtifact_InvalidChecksum(t *testing.T) { defer ts.Close() // Create a temp directory to download into - destDir, err := ioutil.TempDir("", "nomad-test") + taskDir, err := ioutil.TempDir("", "nomad-test") if err != nil { t.Fatalf("failed to make temp directory: %v", err) } - defer os.RemoveAll(destDir) + defer os.RemoveAll(taskDir) // Create the artifact with an incorrect checksum file := "test.sh" @@ -71,7 +106,7 @@ func TestGetArtifact_InvalidChecksum(t *testing.T) { // Download the artifact and expect an error logger := log.New(os.Stderr, "", log.LstdFlags) - if err := GetArtifact(artifact, destDir, logger); err == nil { + if err := GetArtifact(artifact, taskDir, logger); err == nil { t.Fatalf("GetArtifact should have failed") } } @@ -116,17 +151,17 @@ func TestGetArtifact_Archive(t *testing.T) { // Create a temp directory to download into and create some of the same // files that exist in the artifact to ensure they are overriden - destDir, err := ioutil.TempDir("", "nomad-test") + taskDir, err := ioutil.TempDir("", "nomad-test") if err != nil { t.Fatalf("failed to make temp directory: %v", err) } - defer os.RemoveAll(destDir) + defer os.RemoveAll(taskDir) create := map[string]string{ "exist/my.config": "to be replaced", "untouched": "existing top-level", } - createContents(destDir, create, t) + createContents(taskDir, create, t) file := "archive.tar.gz" artifact := &structs.TaskArtifact{ @@ -137,7 +172,7 @@ func TestGetArtifact_Archive(t *testing.T) { } logger := log.New(os.Stderr, "", log.LstdFlags) - if err := GetArtifact(artifact, destDir, logger); err != nil { + if err := GetArtifact(artifact, taskDir, logger); err != nil { t.Fatalf("GetArtifact failed: %v", err) } @@ -148,5 +183,5 @@ func TestGetArtifact_Archive(t *testing.T) { "new/my.config": "hello world\n", "test.sh": "sleep 1\n", } - checkContents(destDir, expected, t) + checkContents(taskDir, expected, t) } diff --git a/client/task_runner.go b/client/task_runner.go index 32571acb7..afe481ad6 100644 --- a/client/task_runner.go +++ b/client/task_runner.go @@ -240,7 +240,18 @@ func (r *TaskRunner) run() { return } - for _, artifact := range r.task.Artifacts { + for i, artifact := range r.task.Artifacts { + // Verify the artifact doesn't escape the task directory. + if err := artifact.Validate(); err != nil { + // If this error occurs there is potentially a server bug or + // mallicious, server spoofing. + r.setState(structs.TaskStateDead, + structs.NewTaskEvent(structs.TaskArtifactDownloadFailed).SetDownloadError(err)) + r.logger.Printf("[ERR] client: allocation %q, task %v, artifact %v (%v) fails validation", + r.alloc.ID, r.task.Name, artifact, i) + return + } + if err := getter.GetArtifact(artifact, taskDir, r.logger); err != nil { r.setState(structs.TaskStateDead, structs.NewTaskEvent(structs.TaskArtifactDownloadFailed).SetDownloadError(err)) diff --git a/command/agent/agent.go b/command/agent/agent.go index fea10ddf2..d8e555159 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -127,9 +127,29 @@ func (a *Agent) serverConfig() (*nomad.Config, error) { } if addr := a.config.Addresses.RPC; addr != "" { conf.RPCAddr.IP = net.ParseIP(addr) + } else if device := a.config.Interfaces.RPC; device != "" { + ip, err := ipOfDevice(device) + if err != nil { + return nil, err + } + conf.RPCAddr.IP = ip } if addr := a.config.Addresses.Serf; addr != "" { conf.SerfConfig.MemberlistConfig.BindAddr = addr + } else if device := a.config.Interfaces.Serf; device != "" { + ip, err := ipOfDevice(device) + if err != nil { + return nil, err + } + conf.SerfConfig.MemberlistConfig.BindAddr = ip.String() + } + + if device := a.config.Interfaces.HTTP; device != "" && a.config.Addresses.HTTP == "" { + ip, err := ipOfDevice(device) + if err != nil { + return nil, err + } + a.config.Addresses.HTTP = ip.String() } // Set up the ports @@ -209,12 +229,21 @@ func (a *Agent) clientConfig() (*clientconfig.Config, error) { conf.Node.Name = a.config.NodeName conf.Node.Meta = a.config.Client.Meta conf.Node.NodeClass = a.config.Client.NodeClass + + // Setting the proper HTTP Addr httpAddr := fmt.Sprintf("%s:%d", a.config.BindAddr, a.config.Ports.HTTP) - if a.config.Addresses.HTTP != "" && a.config.AdvertiseAddrs.HTTP == "" { + if a.config.Addresses.HTTP != "" && a.config.AdvertiseAddrs.HTTP == "" && a.config.Interfaces.HTTP == "" { httpAddr = fmt.Sprintf("%s:%d", a.config.Addresses.HTTP, a.config.Ports.HTTP) if _, err := net.ResolveTCPAddr("tcp", httpAddr); err != nil { return nil, fmt.Errorf("error resolving http addr: %v:", err) } + } else if a.config.Interfaces.HTTP != "" && a.config.AdvertiseAddrs.HTTP == "" { + ip, err := ipOfDevice(a.config.Interfaces.HTTP) + if err != nil { + return nil, fmt.Errorf("error finding ip address from interface %q: %v", a.config.Interfaces.HTTP, err) + } + a.config.Addresses.HTTP = ip.String() + httpAddr = fmt.Sprintf("%s:%d", ip.String(), a.config.Ports.HTTP) } else if a.config.AdvertiseAddrs.HTTP != "" { addr, err := net.ResolveTCPAddr("tcp", a.config.AdvertiseAddrs.HTTP) if err != nil { @@ -223,7 +252,6 @@ func (a *Agent) clientConfig() (*clientconfig.Config, error) { httpAddr = fmt.Sprintf("%s:%d", addr.IP.String(), addr.Port) } conf.Node.HTTPAddr = httpAddr - conf.Version = a.config.Version // Reserve resources on the node. r := conf.Node.Reserved @@ -237,6 +265,8 @@ func (a *Agent) clientConfig() (*clientconfig.Config, error) { r.IOPS = a.config.Client.Reserved.IOPS conf.GloballyReservedPorts = a.config.Client.Reserved.ParsedReservedPorts + conf.Version = a.config.Version + return conf, nil } diff --git a/command/agent/config.go b/command/agent/config.go index 6137ea4f0..e6e173db1 100644 --- a/command/agent/config.go +++ b/command/agent/config.go @@ -46,6 +46,10 @@ type Config struct { // Addresses is used to override the network addresses we bind to. Addresses *Addresses `mapstructure:"addresses"` + // Interfaces is used to override the network addresses we bind to by + // providing device names + Interfaces *Interfaces `mapstructure:"interfaces"` + // AdvertiseAddrs is used to control the addresses we advertise. AdvertiseAddrs *AdvertiseAddrs `mapstructure:"advertise"` @@ -255,6 +259,14 @@ type Addresses struct { Serf string `mapstructure:"serf"` } +// Interfaces provides an alternative to the Addresses configuration. We pick an +// ip configured on the devide specified and use that to bind. +type Interfaces struct { + HTTP string `mapstructure:"http"` + RPC string `mapstructure:"rpc"` + Serf string `mapstructure:"serf"` +} + // AdvertiseAddrs is used to control the addresses we advertise out for // different network services. Not all network services support an // advertise address. All are optional and default to BindAddr. @@ -366,6 +378,7 @@ func DefaultConfig() *Config { Serf: 4648, }, Addresses: &Addresses{}, + Interfaces: &Interfaces{}, AdvertiseAddrs: &AdvertiseAddrs{}, Atlas: &AtlasConfig{}, Client: &ClientConfig{ @@ -496,6 +509,14 @@ func (c *Config) Merge(b *Config) *Config { result.Addresses = result.Addresses.Merge(b.Addresses) } + // Apply the interfaces config + if result.Interfaces == nil && b.Interfaces != nil { + interfaces := *b.Interfaces + result.Interfaces = &interfaces + } else if b.Interfaces != nil { + result.Interfaces = result.Interfaces.Merge(b.Interfaces) + } + // Apply the advertise addrs config if result.AdvertiseAddrs == nil && b.AdvertiseAddrs != nil { advertise := *b.AdvertiseAddrs @@ -675,6 +696,22 @@ func (a *Addresses) Merge(b *Addresses) *Addresses { return &result } +// Merge is used to merge two interfaces configs together. +func (i *Interfaces) Merge(b *Interfaces) *Interfaces { + result := *i + + if b.HTTP != "" { + result.HTTP = b.HTTP + } + if b.RPC != "" { + result.RPC = b.RPC + } + if b.Serf != "" { + result.Serf = b.Serf + } + return &result +} + // Merge merges two advertise addrs configs together. func (a *AdvertiseAddrs) Merge(b *AdvertiseAddrs) *AdvertiseAddrs { result := *a diff --git a/command/agent/config_parse.go b/command/agent/config_parse.go index ac77252f6..e9d237678 100644 --- a/command/agent/config_parse.go +++ b/command/agent/config_parse.go @@ -78,6 +78,7 @@ func parseConfig(result *Config, list *ast.ObjectList) error { "enable_debug", "ports", "addresses", + "interfaces", "advertise", "client", "server", @@ -102,6 +103,7 @@ func parseConfig(result *Config, list *ast.ObjectList) error { } delete(m, "ports") delete(m, "addresses") + delete(m, "interfaces") delete(m, "advertise") delete(m, "client") delete(m, "server") @@ -128,6 +130,13 @@ func parseConfig(result *Config, list *ast.ObjectList) error { } } + // Parse interfaces + if o := list.Filter("interfaces"); len(o.Items) > 0 { + if err := parseInterfaces(&result.Interfaces, o); err != nil { + return multierror.Prefix(err, "interfaces ->") + } + } + // Parse advertise if o := list.Filter("advertise"); len(o.Items) > 0 { if err := parseAdvertise(&result.AdvertiseAddrs, o); err != nil { @@ -244,6 +253,38 @@ func parseAddresses(result **Addresses, list *ast.ObjectList) error { return nil } +func parseInterfaces(result **Interfaces, list *ast.ObjectList) error { + list = list.Elem() + if len(list.Items) > 1 { + return fmt.Errorf("only one 'interfaces' block allowed") + } + + // Get our interfaces object + listVal := list.Items[0].Val + + // Check for the invalid keys + valid := []string{ + "http", + "rpc", + "serf", + } + if err := checkHCLKeys(listVal, valid); err != nil { + return err + } + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, listVal); err != nil { + return err + } + + var interfaces Interfaces + if err := mapstructure.WeakDecode(m, &interfaces); err != nil { + return err + } + *result = &interfaces + return nil +} + func parseAdvertise(result **AdvertiseAddrs, list *ast.ObjectList) error { list = list.Elem() if len(list.Items) > 1 { diff --git a/command/agent/util.go b/command/agent/util.go index ceb333ed9..2fa4993ae 100644 --- a/command/agent/util.go +++ b/command/agent/util.go @@ -1,7 +1,9 @@ package agent import ( + "fmt" "math/rand" + "net" "time" ) @@ -9,3 +11,43 @@ import ( func randomStagger(intv time.Duration) time.Duration { return time.Duration(uint64(rand.Int63()) % uint64(intv)) } + +// IpOfDevice returns a routable ip addr of a device +func ipOfDevice(name string) (net.IP, error) { + intf, err := net.InterfaceByName(name) + if err != nil { + return nil, err + } + addrs, err := intf.Addrs() + if err != nil { + return nil, err + } + if len(addrs) == 0 { + return nil, fmt.Errorf("no ips were detected on the interface: %v", name) + } + + // Iterating through the IPs configured for that device and returning the + // the first ipv4 address configured. If no ipv4 addresses are configured, + // we return the first ipv6 addr if any ipv6 addr is configured. + var ipv6Addrs []net.IP + for _, addr := range addrs { + var ip net.IP + switch v := (addr).(type) { + case *net.IPNet: + ip = v.IP + if ip.To4() != nil { + return ip, nil + } + if ip.To16() != nil { + ipv6Addrs = append(ipv6Addrs, ip) + continue + } + case *net.IPAddr: + continue + } + } + if len(ipv6Addrs) > 0 { + return ipv6Addrs[0], nil + } + return nil, fmt.Errorf("no ips were detected on the interface: %v", name) +} diff --git a/command/alloc_status.go b/command/alloc_status.go index 5e6807e49..c41eb3c05 100644 --- a/command/alloc_status.go +++ b/command/alloc_status.go @@ -126,14 +126,22 @@ func (c *AllocStatusCommand) Run(args []string) int { fmt.Sprintf("Node ID|%s", limit(alloc.NodeID, length)), fmt.Sprintf("Job ID|%s", alloc.JobID), fmt.Sprintf("Client Status|%s", alloc.ClientStatus), - fmt.Sprintf("Evaluated Nodes|%d", alloc.Metrics.NodesEvaluated), - fmt.Sprintf("Filtered Nodes|%d", alloc.Metrics.NodesFiltered), - fmt.Sprintf("Exhausted Nodes|%d", alloc.Metrics.NodesExhausted), - fmt.Sprintf("Allocation Time|%s", alloc.Metrics.AllocationTime), - fmt.Sprintf("Failures|%d", alloc.Metrics.CoalescedFailures), + } + + if verbose { + basic = append(basic, + fmt.Sprintf("Evaluated Nodes|%d", alloc.Metrics.NodesEvaluated), + fmt.Sprintf("Filtered Nodes|%d", alloc.Metrics.NodesFiltered), + fmt.Sprintf("Exhausted Nodes|%d", alloc.Metrics.NodesExhausted), + fmt.Sprintf("Allocation Time|%s", alloc.Metrics.AllocationTime), + fmt.Sprintf("Failures|%d", alloc.Metrics.CoalescedFailures)) } c.Ui.Output(formatKV(basic)) + if !short { + c.taskResources(alloc) + } + // Print the state of each task. if short { c.shortTaskStatus(alloc) @@ -142,12 +150,9 @@ func (c *AllocStatusCommand) Run(args []string) int { } // Format the detailed status - c.Ui.Output("\n==> Status") - dumpAllocStatus(c.Ui, alloc, length) - - if !short { - c.Ui.Output("\n==> Task Resources") - c.taskResources(alloc) + if verbose || alloc.DesiredStatus == "failed" { + c.Ui.Output("\n==> Status") + dumpAllocStatus(c.Ui, alloc, length) } return 0 @@ -283,6 +288,11 @@ func (c *AllocStatusCommand) allocResources(alloc *api.Allocation) { // taskResources prints out the tasks current resource usage func (c *AllocStatusCommand) taskResources(alloc *api.Allocation) { + if len(alloc.TaskResources) == 0 { + return + } + + c.Ui.Output("\n==> Task Resources") firstLine := true for task, resource := range alloc.TaskResources { header := fmt.Sprintf("\nTask: %q", task) diff --git a/command/inspect.go b/command/inspect.go new file mode 100644 index 000000000..a38430164 --- /dev/null +++ b/command/inspect.go @@ -0,0 +1,86 @@ +package command + +import ( + "fmt" + "strings" +) + +type InspectCommand struct { + Meta +} + +func (c *InspectCommand) Help() string { + helpText := ` +Usage: nomad inspect [options] + + Inspect is used to see the specification of a submitted job. + +General Options: + + ` + generalOptionsUsage() + + return strings.TrimSpace(helpText) +} + +func (c *InspectCommand) Synopsis() string { + return "Inspect a submitted job" +} + +func (c *InspectCommand) Run(args []string) int { + flags := c.Meta.FlagSet("inspect", FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + + if err := flags.Parse(args); err != nil { + return 1 + } + + // Check that we got exactly one job + args = flags.Args() + if len(args) != 1 { + c.Ui.Error(c.Help()) + return 1 + } + jobID := args[0] + + // Get the HTTP client + client, err := c.Meta.Client() + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + // Check if the job exists + jobs, _, err := client.Jobs().PrefixList(jobID) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error inspecting job: %s", err)) + return 1 + } + if len(jobs) == 0 { + c.Ui.Error(fmt.Sprintf("No job(s) with prefix or id %q found", jobID)) + return 1 + } + if len(jobs) > 1 { + out := make([]string, len(jobs)+1) + out[0] = "ID|Type|Priority|Status" + for i, job := range jobs { + out[i+1] = fmt.Sprintf("%s|%s|%d|%s", + job.ID, + job.Type, + job.Priority, + job.Status) + } + c.Ui.Output(fmt.Sprintf("Prefix matched multiple jobs\n\n%s", formatList(out))) + return 0 + } + + // Prefix lookup matched a single job + job, _, err := client.Jobs().RawJob(jobs[0].ID, nil) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error inspecting job: %s", err)) + return 1 + } + + // Print the contents of the job + c.Ui.Output(job) + return 0 +} diff --git a/command/inspect_test.go b/command/inspect_test.go new file mode 100644 index 000000000..ba7539abe --- /dev/null +++ b/command/inspect_test.go @@ -0,0 +1,46 @@ +package command + +import ( + "strings" + "testing" + + "github.com/mitchellh/cli" +) + +func TestInspectCommand_Implements(t *testing.T) { + var _ cli.Command = &InspectCommand{} +} + +func TestInspectCommand_Fails(t *testing.T) { + srv, _, url := testServer(t, nil) + defer srv.Stop() + + ui := new(cli.MockUi) + cmd := &InspectCommand{Meta: Meta{Ui: ui}} + + // Fails on misuse + if code := cmd.Run([]string{"some", "bad", "args"}); code != 1 { + t.Fatalf("expected exit code 1, got: %d", code) + } + if out := ui.ErrorWriter.String(); !strings.Contains(out, cmd.Help()) { + t.Fatalf("expected help output, got: %s", out) + } + ui.ErrorWriter.Reset() + + // Fails on non-existent job ID + if code := cmd.Run([]string{"-address=" + url, "nope"}); code != 1 { + t.Fatalf("expect exit 1, got: %d", code) + } + if out := ui.ErrorWriter.String(); !strings.Contains(out, "No job(s) with prefix or id") { + t.Fatalf("expect not found error, got: %s", out) + } + ui.ErrorWriter.Reset() + + // Fails on connection failure + if code := cmd.Run([]string{"-address=nope", "nope"}); code != 1 { + t.Fatalf("expected exit code 1, got: %d", code) + } + if out := ui.ErrorWriter.String(); !strings.Contains(out, "Error inspecting job") { + t.Fatalf("expected failed query error, got: %s", out) + } +} diff --git a/command/node_status.go b/command/node_status.go index efbbab312..949b7bb38 100644 --- a/command/node_status.go +++ b/command/node_status.go @@ -2,9 +2,10 @@ package command import ( "fmt" - "github.com/hashicorp/nomad/api" "sort" "strings" + + "github.com/hashicorp/nomad/api" ) type NodeStatusCommand struct { @@ -179,20 +180,6 @@ func (c *NodeStatusCommand) Run(args []string) int { return 1 } - m := node.Attributes - keys := make([]string, len(m)) - for k := range m { - keys = append(keys, k) - } - sort.Strings(keys) - - var attributes []string - for _, k := range keys { - if k != "" { - attributes = append(attributes, fmt.Sprintf("%s:%s", k, m[k])) - } - } - // Format the output basic := []string{ fmt.Sprintf("ID|%s", limit(node.ID, length)), @@ -201,19 +188,10 @@ func (c *NodeStatusCommand) Run(args []string) int { fmt.Sprintf("DC|%s", node.Datacenter), fmt.Sprintf("Drain|%v", node.Drain), fmt.Sprintf("Status|%s", node.Status), - fmt.Sprintf("Attributes|%s", strings.Join(attributes, ", ")), } - - // Dump the output c.Ui.Output(formatKV(basic)) + if !short { - allocs, err := getAllocs(client, node, length) - if err != nil { - c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err)) - return 1 - } - c.Ui.Output("\n==> Allocations") - c.Ui.Output(formatList(allocs)) resources, err := getResources(client, node) if err != nil { c.Ui.Error(fmt.Sprintf("Error querying node resources: %s", err)) @@ -221,7 +199,37 @@ func (c *NodeStatusCommand) Run(args []string) int { } c.Ui.Output("\n==> Resource Utilization") c.Ui.Output(formatList(resources)) + + allocs, err := getAllocs(client, node, length) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error querying node allocations: %s", err)) + return 1 + } + + if len(allocs) > 1 { + c.Ui.Output("\n==> Allocations") + c.Ui.Output(formatList(allocs)) + } } + + if verbose { + // Print the attributes + keys := make([]string, len(node.Attributes)) + for k := range node.Attributes { + keys = append(keys, k) + } + sort.Strings(keys) + + var attributes []string + for _, k := range keys { + if k != "" { + attributes = append(attributes, fmt.Sprintf("%s|%s", k, node.Attributes[k])) + } + } + c.Ui.Output("\n==> Attributes") + c.Ui.Output(formatKV(attributes)) + } + return 0 } @@ -260,9 +268,19 @@ func getAllocs(client *api.Client, node *api.Node, length int) ([]string, error) return allocs, err } +// getResources returns the resource usage of the node. func getResources(client *api.Client, node *api.Node) ([]string, error) { var resources []string var cpu, mem, disk, iops int + var totalCpu, totalMem, totalDisk, totalIops int + + // Compute the total + r := node.Resources + res := node.Reserved + totalCpu = r.CPU - res.CPU + totalMem = r.MemoryMB - res.MemoryMB + totalDisk = r.DiskMB - res.DiskMB + totalIops = r.IOPS - res.IOPS // Get list of running allocations on the node runningAllocs, err := getRunningAllocs(client, node.ID) @@ -277,11 +295,15 @@ func getResources(client *api.Client, node *api.Node) ([]string, error) { resources = make([]string, 2) resources[0] = "CPU|Memory MB|Disk MB|IOPS" - resources[1] = fmt.Sprintf("%v|%v|%v|%v", + resources[1] = fmt.Sprintf("%v/%v|%v/%v|%v/%v|%v/%v", cpu, + totalCpu, mem, + totalMem, disk, - iops) + totalDisk, + iops, + totalIops) return resources, err } diff --git a/command/node_status_test.go b/command/node_status_test.go index 14324bca4..e4924d9c8 100644 --- a/command/node_status_test.go +++ b/command/node_status_test.go @@ -58,9 +58,6 @@ func TestNodeStatusCommand_Run(t *testing.T) { if !strings.Contains(out, "mynode") { t.Fatalf("expect to find mynode, got: %s", out) } - if !strings.Contains(out, "Allocations") { - t.Fatalf("expected allocations, got: %s", out) - } ui.OutputWriter.Reset() // Query single node in short view @@ -83,9 +80,6 @@ func TestNodeStatusCommand_Run(t *testing.T) { if !strings.Contains(out, "mynode") { t.Fatalf("expect to find mynode, got: %s", out) } - if !strings.Contains(out, "Allocations") { - t.Fatalf("expected allocations, got: %s", out) - } if strings.Contains(out, nodeID) { t.Fatalf("expected truncated node id, got: %s", out) } diff --git a/command/stop.go b/command/stop.go index dfea7ffbe..b970ca847 100644 --- a/command/stop.go +++ b/command/stop.go @@ -75,36 +75,33 @@ func (c *StopCommand) Run(args []string) int { } // Check if the job exists - job, _, err := client.Jobs().Info(jobID, nil) + jobs, _, err := client.Jobs().PrefixList(jobID) if err != nil { - jobs, _, err := client.Jobs().PrefixList(jobID) - if err != nil { - c.Ui.Error(fmt.Sprintf("Error deregistering job: %s", err)) - return 1 - } - if len(jobs) == 0 { - c.Ui.Error(fmt.Sprintf("No job(s) with prefix or id %q found", jobID)) - return 1 - } - if len(jobs) > 1 { - out := make([]string, len(jobs)+1) - out[0] = "ID|Type|Priority|Status" - for i, job := range jobs { - out[i+1] = fmt.Sprintf("%s|%s|%d|%s", - job.ID, - job.Type, - job.Priority, - job.Status) - } - c.Ui.Output(fmt.Sprintf("Prefix matched multiple jobs\n\n%s", formatList(out))) - return 0 - } - // Prefix lookup matched a single job - job, _, err = client.Jobs().Info(jobs[0].ID, nil) - if err != nil { - c.Ui.Error(fmt.Sprintf("Error deregistering job: %s", err)) - return 1 + c.Ui.Error(fmt.Sprintf("Error deregistering job: %s", err)) + return 1 + } + if len(jobs) == 0 { + c.Ui.Error(fmt.Sprintf("No job(s) with prefix or id %q found", jobID)) + return 1 + } + if len(jobs) > 1 { + out := make([]string, len(jobs)+1) + out[0] = "ID|Type|Priority|Status" + for i, job := range jobs { + out[i+1] = fmt.Sprintf("%s|%s|%d|%s", + job.ID, + job.Type, + job.Priority, + job.Status) } + c.Ui.Output(fmt.Sprintf("Prefix matched multiple jobs\n\n%s", formatList(out))) + return 0 + } + // Prefix lookup matched a single job + job, _, err := client.Jobs().Info(jobs[0].ID, nil) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error deregistering job: %s", err)) + return 1 } // Invoke the stop diff --git a/commands.go b/commands.go index c77c8482a..7a6709d12 100644 --- a/commands.go +++ b/commands.go @@ -87,7 +87,11 @@ func Commands(metaPtr *command.Meta) map[string]cli.CommandFactory { Meta: meta, }, nil }, - + "inspect": func() (cli.Command, error) { + return &command.InspectCommand{ + Meta: meta, + }, nil + }, "node-drain": func() (cli.Command, error) { return &command.NodeDrainCommand{ Meta: meta, diff --git a/jobspec/parse.go b/jobspec/parse.go index 853640f75..1baf29cf6 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -617,6 +617,7 @@ func parseArtifacts(result *[]*structs.TaskArtifact, list *ast.ObjectList) error valid := []string{ "source", "options", + "destination", } if err := checkHCLKeys(o.Val, valid); err != nil { return err @@ -629,6 +630,11 @@ func parseArtifacts(result *[]*structs.TaskArtifact, list *ast.ObjectList) error delete(m, "options") + // Default to downloading to the local directory. + if _, ok := m["destination"]; !ok { + m["destination"] = "local/" + } + var ta structs.TaskArtifact if err := mapstructure.WeakDecode(m, &ta); err != nil { return err diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 28cc6afe8..4616db771 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -131,12 +131,14 @@ func TestParse(t *testing.T) { Artifacts: []*structs.TaskArtifact{ { GetterSource: "http://foo.com/artifact", + RelativeDest: "local/", GetterOptions: map[string]string{ "checksum": "md5:b8a4f3f72ecab0510a6a31e997461c5f", }, }, { GetterSource: "http://bar.com/artifact", + RelativeDest: "local/", GetterOptions: map[string]string{ "checksum": "md5:ff1cc0d3432dad54d607c1505fb7245c", }, @@ -320,6 +322,58 @@ func TestParse(t *testing.T) { nil, true, }, + + { + "artifacts.hcl", + &structs.Job{ + ID: "binstore-storagelocker", + Name: "binstore-storagelocker", + Type: "service", + Priority: 50, + Region: "global", + + TaskGroups: []*structs.TaskGroup{ + &structs.TaskGroup{ + Name: "binsl", + Count: 1, + Tasks: []*structs.Task{ + &structs.Task{ + Name: "binstore", + Driver: "docker", + Resources: &structs.Resources{ + CPU: 100, + MemoryMB: 10, + DiskMB: 300, + IOPS: 0, + }, + LogConfig: &structs.LogConfig{ + MaxFiles: 10, + MaxFileSizeMB: 10, + }, + Artifacts: []*structs.TaskArtifact{ + { + GetterSource: "http://foo.com/bar", + GetterOptions: map[string]string{}, + RelativeDest: "", + }, + { + GetterSource: "http://foo.com/baz", + GetterOptions: map[string]string{}, + RelativeDest: "local/", + }, + { + GetterSource: "http://foo.com/bam", + GetterOptions: map[string]string{}, + RelativeDest: "var/foo", + }, + }, + }, + }, + }, + }, + }, + false, + }, } for _, tc := range cases { diff --git a/jobspec/test-fixtures/artifacts.hcl b/jobspec/test-fixtures/artifacts.hcl new file mode 100644 index 000000000..361e8dc61 --- /dev/null +++ b/jobspec/test-fixtures/artifacts.hcl @@ -0,0 +1,21 @@ +job "binstore-storagelocker" { + group "binsl" { + task "binstore" { + driver = "docker" + + artifact { + source = "http://foo.com/bar" + destination = "" + } + + artifact { + source = "http://foo.com/baz" + } + artifact { + source = "http://foo.com/bam" + destination = "var/foo" + } + resources {} + } + } +} diff --git a/main.go b/main.go index 11d237709..073e46d52 100644 --- a/main.go +++ b/main.go @@ -34,6 +34,7 @@ func RunCustom(args []string, commands map[string]cli.CommandFactory) int { switch k { case "executor": case "syslog": + case "fs ls", "fs cat", "fs stat": default: commandsInclude = append(commandsInclude, k) } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 21153f9f6..a3fad3ee5 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "net/url" + "path/filepath" "reflect" "regexp" "strconv" @@ -1913,6 +1914,10 @@ type TaskArtifact struct { // GetterOptions are options to use when downloading the artifact using // go-getter. GetterOptions map[string]string `mapstructure:"options"` + + // RelativeDest is the download destination given relative to the task's + // directory. + RelativeDest string `mapstructure:"destination"` } func (ta *TaskArtifact) Copy() *TaskArtifact { @@ -1925,16 +1930,36 @@ func (ta *TaskArtifact) Copy() *TaskArtifact { return nta } +func (ta *TaskArtifact) GoString() string { + return fmt.Sprintf("%+v", ta) +} + func (ta *TaskArtifact) Validate() error { // Verify the source var mErr multierror.Error if ta.GetterSource == "" { mErr.Errors = append(mErr.Errors, fmt.Errorf("source must be specified")) + } else { + _, err := url.Parse(ta.GetterSource) + if err != nil { + mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid source URL %q: %v", ta.GetterSource, err)) + } } - _, err := url.Parse(ta.GetterSource) + // Verify the destination doesn't escape the tasks directory + alloc := "/foo/bar/" + abs, err := filepath.Abs(filepath.Join(alloc, ta.RelativeDest)) if err != nil { - mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid source URL %q: %v", ta.GetterSource, err)) + mErr.Errors = append(mErr.Errors, err) + return mErr.ErrorOrNil() + } + rel, err := filepath.Rel(alloc, abs) + if err != nil { + mErr.Errors = append(mErr.Errors, err) + return mErr.ErrorOrNil() + } + if strings.HasPrefix(rel, "..") { + mErr.Errors = append(mErr.Errors, fmt.Errorf("destination escapes task's directory")) } // Verify the checksum diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index 1b2ce9517..04f96635d 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -777,6 +777,28 @@ func TestTaskArtifact_Validate_Source(t *testing.T) { } } +func TestTaskArtifact_Validate_Dest(t *testing.T) { + valid := &TaskArtifact{GetterSource: "google.com"} + if err := valid.Validate(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + valid.RelativeDest = "local/" + if err := valid.Validate(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + valid.RelativeDest = "local/.." + if err := valid.Validate(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + valid.RelativeDest = "local/../.." + if err := valid.Validate(); err == nil { + t.Fatalf("expected error: %v", err) + } +} + func TestTaskArtifact_Validate_Checksum(t *testing.T) { cases := []struct { Input *TaskArtifact diff --git a/website/source/docs/agent/config.html.md b/website/source/docs/agent/config.html.md index cdd50c09b..de337423d 100644 --- a/website/source/docs/agent/config.html.md +++ b/website/source/docs/agent/config.html.md @@ -144,6 +144,22 @@ nodes, unless otherwise specified: server nodes from the same datacenter if possible. Used only on server nodes. +* `interfaces`: Provides an alternative to the + `addresses` configuration. Operators can provide network device names to which + Nomad binds individual network services. Nomad looks for the first IPv4 + address configured for the device and uses it, and if no IPv4 address is + present then it looks for an IPv6 address. The value is a map of device names of + network interfaces and supports the following keys: +
+ * `http`: The device name the HTTP server is bound to. Applies to both clients and servers. + * `rpc`: The device name to bind the internal RPC interfaces to. Should be exposed + only to other cluster members if possible. Used only on server nodes, but + must be accessible from all agents. + * `serf`: The device name used to bind the gossip layer to. Both a TCP and UDP + listener will be exposed on this address. Should be restricted to only + server nodes from the same datacenter if possible. Used only on server + nodes. + * `advertise`: Controls the advertise address for individual network services. This can be used to advertise a different address to the peers of a server node to support more complex network configurations such as NAT. This diff --git a/website/source/docs/jobspec/index.html.md b/website/source/docs/jobspec/index.html.md index 0a68cb5f0..7d277bbc2 100644 --- a/website/source/docs/jobspec/index.html.md +++ b/website/source/docs/jobspec/index.html.md @@ -430,6 +430,10 @@ The `artifact` object maps supports the following keys: * `source` - The path to the artifact to download. +* `destination` - An optional path to download the artifact into relative to the + root of the task's directory. If the `destination` key is omitted, it will + default to `local/`. + * `options` - The `options` block allows setting parameters for `go-getter`. An example is given below: diff --git a/website/source/docs/jobspec/json.html.md b/website/source/docs/jobspec/json.html.md index d8ea02245..c62a8d1ec 100644 --- a/website/source/docs/jobspec/json.html.md +++ b/website/source/docs/jobspec/json.html.md @@ -414,13 +414,16 @@ is started. The `Artifact` object maps supports the following keys: -* `Source` - The path to the artifact to download. +* `GetterSource` - The path to the artifact to download. -* `Options` - The `options` block allows setting parameters for `go-getter`. An - example is given below: +* `RelativeDest` - The destination to download the artifact relative the task's + directory. + +* `GetterOptions` - A `map[string]string` block of options for `go-getter`. An + example is given below: ``` -"Options": { +"GetterOptions": { "checksum": "md5:c4aa853ad2215426eb7d70a21922e794", "aws_access_key_id": "",