From 95c90d27d84ccd71708c5d22542077d0ee2e0974 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antonio=20Fern=C3=A1ndez?= Date: Fri, 13 Nov 2015 22:30:58 +0100 Subject: [PATCH 1/4] POC: allow to add labels for docker containers from job metadata --- client/driver/docker.go | 29 +++++++++++++++++++++++++++-- client/driver/docker_test.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 862587dd2..ac073cda2 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -14,6 +14,7 @@ import ( "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/args" + "github.com/hashicorp/nomad/client/driver/environment" "github.com/hashicorp/nomad/client/fingerprint" "github.com/hashicorp/nomad/nomad/structs" ) @@ -43,6 +44,10 @@ func NewDockerDriver(ctx *DriverContext) Driver { return &DockerDriver{DriverContext: *ctx} } +const ( + labelPrefix = environment.MetaPrefix + "LABEL_" +) + // dockerClient creates *docker.Client. In test / dev mode we can use ENV vars // to connect to the docker daemon. In production mode we will read // docker.endpoint from the config file. @@ -266,9 +271,12 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task) (do env.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)) env.SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)) + envVars, labels := splitEnvVarsLabels(env.List()) + config := &docker.Config{ - Env: env.List(), - Image: task.Config["image"], + Env: envVars, + Labels: labels, + Image: task.Config["image"], } rawArgs, hasArgs := task.Config["args"] @@ -557,3 +565,20 @@ func (h *dockerHandle) run() { } close(h.waitCh) } + +// splitEnvVarsLabels returns environment vars and labels to set on the container +func splitEnvVarsLabels(envVars []string) ([]string, map[string]string) { + var envs []string + labels := make(map[string]string) + + for _, value := range envVars { + if strings.HasPrefix(value, labelPrefix) { + parts := strings.SplitN(value, "=", 2) + labels[strings.TrimPrefix(parts[0], labelPrefix)] = parts[1] + } else { + envs = append(envs, value) + } + } + + return envs, labels +} diff --git a/client/driver/docker_test.go b/client/driver/docker_test.go index 872c2419b..246950333 100644 --- a/client/driver/docker_test.go +++ b/client/driver/docker_test.go @@ -432,3 +432,39 @@ func TestDockerHostNet(t *testing.T) { } defer handle.Kill() } + +func TestsSlitEnvVarsLabels(t *testing.T) { + + environment := []string{ + "NOMAD_IP=1.1.1.1", + "NOMAD_CPU_LIMIT=500", + "NOMAD_META_TEST1=true_one", + "NOMAD_META_LABEL_LAB1=one", + "NOMAD_META_TEST2=test_two", + "NOMAD_META_LABEL_LAB2=two", + "NOMAD_META_TEST3=true_three", + "NOMAD_META_LABEL_LAB3=three", + "NOMAD_MEMORY_LIMIT=1024", + } + + envVars, labels := splitEnvVarsLabels(environment) + if got, want := len(envVars), 6; got != want { + t.Errorf("Error on len. Got: %v. Expect %v items.", got, want) + } + if got, want := len(labels), 3; got != want { + t.Errorf("Error on len. Got: %v. Expect %v items.", got, want) + } + if got, want := envVars[3], environment[4]; got != want { + t.Errorf("Error. Got: '%v'. Expect '%v'.", got, want) + } + if got, want := envVars[4], environment[6]; got != want { + t.Errorf("Error. Got: '%v'. Expect '%v'.", got, want) + } + if got, want := labels["LAB2"], "two"; got != want { + t.Errorf("Error. Got: '%v'. Expect '%v'.", got, want) + } + if got, want := labels["LAB3"], "three"; got != want { + t.Errorf("Error. Got: '%v'. Expect '%v'.", got, want) + } + +} From f0da375e7c0b84bb51d44f4cab9b234b51f9623b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antonio=20Fern=C3=A1ndez?= Date: Tue, 17 Nov 2015 07:05:11 +0100 Subject: [PATCH 2/4] revert previous commit --- client/driver/docker.go | 29 ++--------------------------- client/driver/docker_test.go | 36 ------------------------------------ 2 files changed, 2 insertions(+), 63 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index ac073cda2..862587dd2 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -14,7 +14,6 @@ import ( "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/args" - "github.com/hashicorp/nomad/client/driver/environment" "github.com/hashicorp/nomad/client/fingerprint" "github.com/hashicorp/nomad/nomad/structs" ) @@ -44,10 +43,6 @@ func NewDockerDriver(ctx *DriverContext) Driver { return &DockerDriver{DriverContext: *ctx} } -const ( - labelPrefix = environment.MetaPrefix + "LABEL_" -) - // dockerClient creates *docker.Client. In test / dev mode we can use ENV vars // to connect to the docker daemon. In production mode we will read // docker.endpoint from the config file. @@ -271,12 +266,9 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task) (do env.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)) env.SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)) - envVars, labels := splitEnvVarsLabels(env.List()) - config := &docker.Config{ - Env: envVars, - Labels: labels, - Image: task.Config["image"], + Env: env.List(), + Image: task.Config["image"], } rawArgs, hasArgs := task.Config["args"] @@ -565,20 +557,3 @@ func (h *dockerHandle) run() { } close(h.waitCh) } - -// splitEnvVarsLabels returns environment vars and labels to set on the container -func splitEnvVarsLabels(envVars []string) ([]string, map[string]string) { - var envs []string - labels := make(map[string]string) - - for _, value := range envVars { - if strings.HasPrefix(value, labelPrefix) { - parts := strings.SplitN(value, "=", 2) - labels[strings.TrimPrefix(parts[0], labelPrefix)] = parts[1] - } else { - envs = append(envs, value) - } - } - - return envs, labels -} diff --git a/client/driver/docker_test.go b/client/driver/docker_test.go index 246950333..872c2419b 100644 --- a/client/driver/docker_test.go +++ b/client/driver/docker_test.go @@ -432,39 +432,3 @@ func TestDockerHostNet(t *testing.T) { } defer handle.Kill() } - -func TestsSlitEnvVarsLabels(t *testing.T) { - - environment := []string{ - "NOMAD_IP=1.1.1.1", - "NOMAD_CPU_LIMIT=500", - "NOMAD_META_TEST1=true_one", - "NOMAD_META_LABEL_LAB1=one", - "NOMAD_META_TEST2=test_two", - "NOMAD_META_LABEL_LAB2=two", - "NOMAD_META_TEST3=true_three", - "NOMAD_META_LABEL_LAB3=three", - "NOMAD_MEMORY_LIMIT=1024", - } - - envVars, labels := splitEnvVarsLabels(environment) - if got, want := len(envVars), 6; got != want { - t.Errorf("Error on len. Got: %v. Expect %v items.", got, want) - } - if got, want := len(labels), 3; got != want { - t.Errorf("Error on len. Got: %v. Expect %v items.", got, want) - } - if got, want := envVars[3], environment[4]; got != want { - t.Errorf("Error. Got: '%v'. Expect '%v'.", got, want) - } - if got, want := envVars[4], environment[6]; got != want { - t.Errorf("Error. Got: '%v'. Expect '%v'.", got, want) - } - if got, want := labels["LAB2"], "two"; got != want { - t.Errorf("Error. Got: '%v'. Expect '%v'.", got, want) - } - if got, want := labels["LAB3"], "three"; got != want { - t.Errorf("Error. Got: '%v'. Expect '%v'.", got, want) - } - -} From 1e1be3c08252aad4ea0939a0b1425efcd26314fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antonio=20Fern=C3=A1ndez?= Date: Tue, 17 Nov 2015 14:12:49 +0100 Subject: [PATCH 3/4] allow to set labels on docker containers --- client/driver/docker.go | 26 +++++++++++----- client/driver/docker_test.go | 57 ++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 8 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index f2d25f9ba..c862b58e3 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -35,14 +35,15 @@ type DockerAuthConfig struct { type DockerDriverConfig struct { DockerAuthConfig - ImageName string `mapstructure:"image"` // Container's Image Name - Command string `mapstructure:"command"` // The Command/Entrypoint to run when the container starts up - Args string `mapstructure:"args"` // The arguments to the Command/Entrypoint - NetworkMode string `mapstructure:"network_mode"` // The network mode of the container - host, net and none - PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and the ports exposed on the container - Privileged bool `mapstructure:"privileged"` // Flag to run the container in priviledged mode - DNS string `mapstructure:"dns_server"` // DNS Server for containers - SearchDomains string `mapstructure:"search_domains"` // DNS Search domains for containers + ImageName string `mapstructure:"image"` // Container's Image Name + Command string `mapstructure:"command"` // The Command/Entrypoint to run when the container starts up + Args string `mapstructure:"args"` // The arguments to the Command/Entrypoint + NetworkMode string `mapstructure:"network_mode"` // The network mode of the container - host, net and none + PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and the ports exposed on the container + Privileged bool `mapstructure:"privileged"` // Flag to run the container in priviledged mode + DNS string `mapstructure:"dns_server"` // DNS Server for containers + SearchDomains string `mapstructure:"search_domains"` // DNS Search domains for containers + Labels []map[string]string `mapstructure:"labels"` // Labels to set when the container starts up } func (c *DockerDriverConfig) Validate() error { @@ -53,6 +54,10 @@ func (c *DockerDriverConfig) Validate() error { if len(c.PortMap) > 1 { return fmt.Errorf("Only one port_map block is allowed in the docker driver config") } + + if len(c.Labels) > 1 { + return fmt.Errorf("Only one labels block is allowed in the docker driver config") + } return nil } @@ -321,6 +326,11 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri d.logger.Println("[DEBUG] driver.docker: ignoring args because command not specified") } + if len(driverConfig.Labels) == 1 { + config.Labels = driverConfig.Labels[0] + d.logger.Println("[DEBUG] driver.docker: applied labels on the container") + } + config.Env = env.List() return docker.CreateContainerOptions{ Config: config, diff --git a/client/driver/docker_test.go b/client/driver/docker_test.go index 87bb58e6f..05df5a647 100644 --- a/client/driver/docker_test.go +++ b/client/driver/docker_test.go @@ -1,10 +1,12 @@ package driver import ( + "encoding/json" "fmt" "io/ioutil" "path/filepath" "reflect" + "strings" "testing" "time" @@ -433,3 +435,58 @@ func TestDockerHostNet(t *testing.T) { } defer handle.Kill() } + +func TestDockerLabels(t *testing.T) { + if !dockerIsConnected(t) { + t.SkipNow() + } + + task := taskTemplate() + task.Config["labels"] = []map[string]string{ + map[string]string{ + "label1": "value1", + "label2": "value2", + }, + } + + driverCtx := testDockerDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + d := NewDockerDriver(driverCtx) + + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + client, err := docker.NewClientFromEnv() + if err != nil { + t.Fatalf("err: %v", err) + } + + // don't know if is queriable in a clean way + parts := strings.SplitN(handle.ID(), ":", 2) + var pid dockerPID + err = json.Unmarshal([]byte(parts[1]), &pid) + if err != nil { + t.Fatalf("err: %v", err) + } + + container, err := client.InspectContainer(pid.ContainerID) + if err != nil { + t.Fatalf("err: %v", err) + } + + if want, got := 2, len(container.Config.Labels); want != got { + t.Errorf("Wrong labels count for docker job. Expect: %d, got: %d", want, got) + } + + if want, got := "value1", container.Config.Labels["label1"]; want != got { + t.Errorf("Wrong label value docker job. Expect: %s, got: %s", want, got) + } + + defer handle.Kill() +} From e42f0831f8c90d1e22aa8e35ce1945fc2257a3d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antonio=20Fern=C3=A1ndez?= Date: Tue, 17 Nov 2015 14:14:35 +0100 Subject: [PATCH 4/4] updating website: docker driver configuration --- website/source/docs/drivers/docker.html.md | 23 ++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/website/source/docs/drivers/docker.html.md b/website/source/docs/drivers/docker.html.md index dfa4c85a5..5a097fe36 100644 --- a/website/source/docs/drivers/docker.html.md +++ b/website/source/docs/drivers/docker.html.md @@ -32,25 +32,28 @@ The `docker` driver supports the following configuration in the job specificatio network mode is not supported right now and is reported as an invalid option. -* `privileged` - (optional) Privileged mode gives the container full access to +* `privileged` - (Optional) Privileged mode gives the container full access to the host. Valid options are `"true"` and `"false"` (defaults to `"false"`). Tasks with `privileged` set can only run on Nomad Agents with `docker.privileged.enabled = "true"`. -* `dns-servers` - (optional) A comma separated list of DNS servers for the container +* `dns-servers` - (Optional) A comma separated list of DNS servers for the container to use (e.g. "8.8.8.8,8.8.4.4"). *Docker API v1.10 and above only* -* `search-domains` - (optional) A comma separated list of DNS search domains for the +* `search-domains` - (Optional) A comma separated list of DNS search domains for the container to use. - + +* `labels` - (Optional) A key/value map of labels to set to the containers on start. + + **Authentication** -Registry authentication can be set per task with the following authentication -parameters. These options can provide access to private repositories that +Registry authentication can be set per task with the following authentication +parameters. These options can provide access to private repositories that utilize the docker remote api (e.g. dockerhub, quay.io) - - `auth.username` - (optional) The account username - - `auth.password` - (optional) The account password - - `auth.email` - (optional) The account email - - `auth.server-address` - (optional) The server domain/ip without the protocol + - `auth.username` - (Optional) The account username + - `auth.password` - (Optional) The account password + - `auth.email` - (Optional) The account email + - `auth.server-address` - (Optional) The server domain/ip without the protocol ### Port Mapping