From 078a025725fcd94716f1c5f224d246a5f1bab3db Mon Sep 17 00:00:00 2001 From: Carlos Diaz-Padron Date: Mon, 28 Sep 2015 16:54:32 -0700 Subject: [PATCH 1/6] Use environment to connect to Docker by default Uses the environment definition for docker by default. Docker will default to the unix/tcp socket if the environment is not set. --- client/driver/docker.go | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 6995d2b25..bb0795ca9 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -39,9 +39,10 @@ func NewDockerDriver(ctx *DriverContext) Driver { func (d *DockerDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { // Initialize docker API client - dockerEndpoint := d.config.ReadDefault("docker.endpoint", "unix:///var/run/docker.sock") - client, err := docker.NewClient(dockerEndpoint) + client, err := d.dockerClient() + if err != nil { + d.logger.Printf("[DEBUG] driver.docker: could not connect to docker daemon: %v", err) return false, nil } @@ -56,6 +57,7 @@ func (d *DockerDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool env, err := client.Version() if err != nil { + d.logger.Printf("[DEBUG] driver.docker: could not read version from daemon: %v", err) // Check the "no such file" error if the unix file is missing if strings.Contains(err.Error(), "no such file") { return false, nil @@ -195,10 +197,9 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle } // Initialize docker API client - dockerEndpoint := d.config.ReadDefault("docker.endpoint", "unix:///var/run/docker.sock") - client, err := docker.NewClient(dockerEndpoint) + client, err := d.dockerClient() if err != nil { - return nil, fmt.Errorf("Failed to connect to docker.endpoint (%s): %s", dockerEndpoint, err) + return nil, fmt.Errorf("Failed to connect to docker.endpoint (%s): %s", client.Endpoint(), err) } repo, tag := docker.ParseRepositoryTag(image) @@ -292,10 +293,9 @@ func (d *DockerDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, er d.logger.Printf("[INFO] driver.docker: re-attaching to docker process: %s", handleID) // Initialize docker API client - dockerEndpoint := d.config.ReadDefault("docker.endpoint", "unix:///var/run/docker.sock") - client, err := docker.NewClient(dockerEndpoint) + client, err := d.dockerClient() if err != nil { - return nil, fmt.Errorf("Failed to connect to docker.endpoint (%s): %s", dockerEndpoint, err) + return nil, fmt.Errorf("Failed to connect to docker.endpoint (%s): %s", client.Endpoint(), err) } // Look for a running container with this ID @@ -333,6 +333,18 @@ func (d *DockerDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, er return h, nil } +// dockerClient returns a configured *docker.Client from the ClientConfig +func (d *DockerDriver) dockerClient() (*docker.Client, error) { + dockerEndpoint := d.config.Read("docker.endpoint") + if dockerEndpoint == "" { + client, err := docker.NewClientFromEnv() + return client, err + } else { + client, err := docker.NewClient(dockerEndpoint) + return client, err + } +} + func (h *dockerHandle) ID() string { // Return a handle to the PID pid := dockerPID{ From 127dc127b2173726c74501a3361c4e8d6b9e34a6 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 6 Oct 2015 16:26:31 -0700 Subject: [PATCH 2/6] Use docker.NewClient; move dockerClient before it's used --- client/driver/docker.go | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 4e0e035ad..896529513 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -37,10 +37,17 @@ func NewDockerDriver(ctx *DriverContext) Driver { return &DockerDriver{*ctx} } +// dockerClient creates *docker.Client using ClientConfig so we can get the +// correct socket for the daemon +func (d *DockerDriver) dockerClient() (*docker.Client, error) { + dockerEndpoint := d.config.Read("docker.endpoint") + client, err := docker.NewClient(dockerEndpoint) + return client, err +} + func (d *DockerDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { // Initialize docker API client client, err := d.dockerClient() - if err != nil { d.logger.Printf("[DEBUG] driver.docker: could not connect to docker daemon: %v", err) return false, nil @@ -350,18 +357,6 @@ func (d *DockerDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, er return h, nil } -// dockerClient returns a configured *docker.Client from the ClientConfig -func (d *DockerDriver) dockerClient() (*docker.Client, error) { - dockerEndpoint := d.config.Read("docker.endpoint") - if dockerEndpoint == "" { - client, err := docker.NewClientFromEnv() - return client, err - } else { - client, err := docker.NewClient(dockerEndpoint) - return client, err - } -} - func (h *dockerHandle) ID() string { // Return a handle to the PID pid := dockerPID{ From f3b5d553ad17571bc881890b1d2157eae0268933 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 6 Oct 2015 17:53:05 -0700 Subject: [PATCH 3/6] Support boot2docker or VM for dev/test --- client/driver/docker.go | 44 ++++++++++++++++++++++++++++++++---- client/driver/docker_test.go | 26 +++++++++++++-------- 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 896529513..b7b4e3eb2 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -4,10 +4,12 @@ import ( "encoding/json" "fmt" "log" + "runtime" "strconv" "strings" docker "github.com/fsouza/go-dockerclient" + opts "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/opts" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/nomad/structs" @@ -33,16 +35,47 @@ type dockerHandle struct { doneCh chan struct{} } +// getDefaultDockerHost is copied from fsouza. If it were exported we woudn't +// need this here. +func getDefaultDockerHost() (string, error) { + var defaultHost string + if runtime.GOOS == "windows" { + // If we do not have a host, default to TCP socket on Windows + defaultHost = fmt.Sprintf("tcp://%s:%d", opts.DefaultHTTPHost, opts.DefaultHTTPPort) + } else { + // If we do not have a host, default to unix socket + defaultHost = fmt.Sprintf("unix://%s", opts.DefaultUnixSocket) + } + return opts.ValidateHost(defaultHost) +} + func NewDockerDriver(ctx *DriverContext) Driver { return &DockerDriver{*ctx} } -// dockerClient creates *docker.Client using ClientConfig so we can get the -// correct socket for the daemon +// dockerClient creates *docker.Client. In test / dev mode we can use ENV vars +// to connect to the docker daemon. In production mode we will read +// docker.endpoint from the config file. func (d *DockerDriver) dockerClient() (*docker.Client, error) { - dockerEndpoint := d.config.Read("docker.endpoint") - client, err := docker.NewClient(dockerEndpoint) - return client, err + // In dev mode, read DOCKER_* environment variables DOCKER_HOST, + // DOCKER_TLS_VERIFY, and DOCKER_CERT_PATH. This allows you to run tests and + // demo against boot2docker or a VM on OSX and Windows. This falls back on + // the default unix socket on linux if tests are run on linux. + // + // Also note that we need to turn on DevMode in the test configs. + if d.config.DevMode { + return docker.NewClientFromEnv() + } + + // In prod mode we'll read the docker.endpoint configuration and fall back + // on the host-specific default. We do not read from the environment. + defaultEndpoint, err := getDefaultDockerHost() + if err != nil { + return nil, fmt.Errorf("Unable to determine default docker endpoint: %s", err) + } + dockerEndpoint := d.config.ReadDefault("docker.endpoint", defaultEndpoint) + + return docker.NewClient(dockerEndpoint) } func (d *DockerDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { @@ -408,6 +441,7 @@ func (h *dockerHandle) Kill() error { err = h.client.RemoveImage(h.imageID) if err != nil { containers, err := h.client.ListContainers(docker.ListContainersOptions{ + // The image might be in use by a stopped container, so check everything All: true, Filters: map[string][]string{ "image": []string{h.imageID}, diff --git a/client/driver/docker_test.go b/client/driver/docker_test.go index 3bea8235a..2d436092d 100644 --- a/client/driver/docker_test.go +++ b/client/driver/docker_test.go @@ -9,6 +9,12 @@ import ( "github.com/hashicorp/nomad/nomad/structs" ) +func testDockerDriverContext(task string) *DriverContext { + cfg := testConfig() + cfg.DevMode = true + return NewDriverContext(task, cfg, cfg.Node, testLogger()) +} + // dockerLocated looks to see whether docker is available on this system before // we try to run tests. We'll keep it simple and just check for the CLI. func dockerLocated() bool { @@ -33,7 +39,7 @@ func TestDockerDriver_Handle(t *testing.T) { // The fingerprinter test should always pass, even if Docker is not installed. func TestDockerDriver_Fingerprint(t *testing.T) { - d := NewDockerDriver(testDriverContext("")) + d := NewDockerDriver(testDockerDriverContext("")) node := &structs.Node{ Attributes: make(map[string]string), } @@ -56,14 +62,14 @@ func TestDockerDriver_StartOpen_Wait(t *testing.T) { } task := &structs.Task{ - Name: "python-demo", + Name: "redis-demo", Config: map[string]string{ "image": "redis", }, Resources: basicResources, } - driverCtx := testDriverContext(task.Name) + driverCtx := testDockerDriverContext(task.Name) ctx := testDriverExecContext(task, driverCtx) defer ctx.AllocDir.Destroy() d := NewDockerDriver(driverCtx) @@ -93,7 +99,7 @@ func TestDockerDriver_Start_Wait(t *testing.T) { } task := &structs.Task{ - Name: "python-demo", + Name: "redis-demo", Config: map[string]string{ "image": "redis", "command": "redis-server -v", @@ -104,7 +110,7 @@ func TestDockerDriver_Start_Wait(t *testing.T) { }, } - driverCtx := testDriverContext(task.Name) + driverCtx := testDockerDriverContext(task.Name) ctx := testDriverExecContext(task, driverCtx) defer ctx.AllocDir.Destroy() d := NewDockerDriver(driverCtx) @@ -140,7 +146,7 @@ func TestDockerDriver_Start_Kill_Wait(t *testing.T) { } task := &structs.Task{ - Name: "python-demo", + Name: "redis-demo", Config: map[string]string{ "image": "redis", "command": "sleep 10", @@ -148,7 +154,7 @@ func TestDockerDriver_Start_Kill_Wait(t *testing.T) { Resources: basicResources, } - driverCtx := testDriverContext(task.Name) + driverCtx := testDockerDriverContext(task.Name) ctx := testDriverExecContext(task, driverCtx) defer ctx.AllocDir.Destroy() d := NewDockerDriver(driverCtx) @@ -219,7 +225,7 @@ func TestDocker_StartN(t *testing.T) { // Let's spin up a bunch of things var err error for idx, task := range taskList { - driverCtx := testDriverContext(task.Name) + driverCtx := testDockerDriverContext(task.Name) ctx := testDriverExecContext(task, driverCtx) defer ctx.AllocDir.Destroy() d := NewDockerDriver(driverCtx) @@ -265,7 +271,7 @@ func TestDocker_StartNVersions(t *testing.T) { // Let's spin up a bunch of things var err error for idx, task := range taskList { - driverCtx := testDriverContext(task.Name) + driverCtx := testDockerDriverContext(task.Name) ctx := testDriverExecContext(task, driverCtx) defer ctx.AllocDir.Destroy() d := NewDockerDriver(driverCtx) @@ -299,7 +305,7 @@ func TestDockerHostNet(t *testing.T) { CPU: 512, }, } - driverCtx := testDriverContext(task.Name) + driverCtx := testDockerDriverContext(task.Name) ctx := testDriverExecContext(task, driverCtx) defer ctx.AllocDir.Destroy() d := NewDockerDriver(driverCtx) From 8b0c18db5f3a9a2b7bdab68858afba790fe05729 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Tue, 6 Oct 2015 19:09:59 -0700 Subject: [PATCH 4/6] Remove panic -- client is nil when there is an error --- client/driver/docker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index ffbd05dcd..5c5a82e91 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -256,7 +256,7 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle // Initialize docker API client client, err := d.dockerClient() if err != nil { - return nil, fmt.Errorf("Failed to connect to docker.endpoint (%s): %s", client.Endpoint(), err) + return nil, fmt.Errorf("Failed to connect to docker daemon: %s", err) } repo, tag := docker.ParseRepositoryTag(image) @@ -352,7 +352,7 @@ func (d *DockerDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, er // Initialize docker API client client, err := d.dockerClient() if err != nil { - return nil, fmt.Errorf("Failed to connect to docker.endpoint (%s): %s", client.Endpoint(), err) + return nil, fmt.Errorf("Failed to connect to docker daemon: %s", err) } // Look for a running container with this ID From dc1d845e2c09ee559980e1e8010e888034a0181a Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Thu, 8 Oct 2015 00:08:54 -0700 Subject: [PATCH 5/6] Added documentation for DOCKER_HOST and docker.endpoint; also filled in docs for docker.cleanup --- website/source/docs/drivers/docker.html.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/website/source/docs/drivers/docker.html.md b/website/source/docs/drivers/docker.html.md index 78ca25e45..c6ea40a77 100644 --- a/website/source/docs/drivers/docker.html.md +++ b/website/source/docs/drivers/docker.html.md @@ -111,11 +111,21 @@ The `docker` driver has the following configuration options: * `docker.endpoint` - Defaults to `unix:///var/run/docker.sock`. You will need to customize this if you use a non-standard socket (http or another location). +* `docker.cleanup.container` Defaults to `true`. Changing this to `false` will + prevent Nomad from removing containers from stopped tasks. + +* `docker.cleanup.image` Defaults to `true`. Changing this to `false` will + prevent Nomad from removing images from stopped tasks. + +Note: When testing or using the `-dev` flag you can use `DOCKER_HOST`, +`DOCKER_TLS_VERIFY`, and `DOCKER_CERT_PATH` to customize Nomad's behavior. In +production Nomad will always read `docker.endpoint`. + ## Client Attributes The `docker` driver will set the following client attributes: -* `driver.Docker` - This will be set to "1", indicating the +* `driver.docker` - This will be set to "1", indicating the driver is available. ## Resource Isolation From 6e43a2ba3389aa4481a113f42861024625b7ca67 Mon Sep 17 00:00:00 2001 From: Chris Bednarski Date: Thu, 8 Oct 2015 12:35:19 -0700 Subject: [PATCH 6/6] Use DefaultDockerHost from fsouza upstream --- client/driver/docker.go | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 5c5a82e91..0fbf64714 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -4,12 +4,10 @@ import ( "encoding/json" "fmt" "log" - "runtime" "strconv" "strings" docker "github.com/fsouza/go-dockerclient" - opts "github.com/fsouza/go-dockerclient/external/github.com/docker/docker/opts" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/nomad/structs" @@ -35,20 +33,6 @@ type dockerHandle struct { doneCh chan struct{} } -// getDefaultDockerHost is copied from fsouza. If it were exported we woudn't -// need this here. -func getDefaultDockerHost() (string, error) { - var defaultHost string - if runtime.GOOS == "windows" { - // If we do not have a host, default to TCP socket on Windows - defaultHost = fmt.Sprintf("tcp://%s:%d", opts.DefaultHTTPHost, opts.DefaultHTTPPort) - } else { - // If we do not have a host, default to unix socket - defaultHost = fmt.Sprintf("unix://%s", opts.DefaultUnixSocket) - } - return opts.ValidateHost(defaultHost) -} - func NewDockerDriver(ctx *DriverContext) Driver { return &DockerDriver{*ctx} } @@ -69,7 +53,7 @@ func (d *DockerDriver) dockerClient() (*docker.Client, error) { // In prod mode we'll read the docker.endpoint configuration and fall back // on the host-specific default. We do not read from the environment. - defaultEndpoint, err := getDefaultDockerHost() + defaultEndpoint, err := docker.DefaultDockerHost() if err != nil { return nil, fmt.Errorf("Unable to determine default docker endpoint: %s", err) }