From f71db5e645b586567571399f1e5c3c36331c5819 Mon Sep 17 00:00:00 2001 From: Ivo Verberk Date: Sun, 24 Jan 2016 10:31:03 +0100 Subject: [PATCH 1/3] Pass a combination of ip and port to the task environment. The different labels can be on different IPs. --- client/driver/docker.go | 5 ++--- client/driver/docker_test.go | 8 +++---- client/driver/driver.go | 7 +----- client/driver/driver_test.go | 12 +++++----- client/driver/env/env.go | 36 ++++++++++++++++-------------- client/driver/env/env_test.go | 42 ++++++++++++++++++++++++++++++----- 6 files changed, 69 insertions(+), 41 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index be27837a5..9a77328f3 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -347,9 +347,8 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri // TODO refactor the implementation in GetTaskEnv to match // the 0.2 ports world view. Docker seems to be the only place where // this is actually needed, but this is kinda hacky. - if len(driverConfig.PortMap) > 0 { - d.taskEnv.SetPorts(network.MapLabelToValues(driverConfig.PortMap)) - } + d.taskEnv.SetPortMap(driverConfig.PortMap) + hostConfig.PortBindings = publishedPorts config.ExposedPorts = exposedPorts } diff --git a/client/driver/docker_test.go b/client/driver/docker_test.go index baa57762a..ae451f540 100644 --- a/client/driver/docker_test.go +++ b/client/driver/docker_test.go @@ -551,8 +551,8 @@ func TestDockerPortsNoMap(t *testing.T) { } expectedEnvironment := map[string]string{ - "NOMAD_PORT_main": fmt.Sprintf("%d", res), - "NOMAD_PORT_REDIS": fmt.Sprintf("%d", dyn), + "NOMAD_IP_main": fmt.Sprintf("127.0.0.1:%d", res), + "NOMAD_IP_REDIS": fmt.Sprintf("127.0.0.1:%d", dyn), } for key, val := range expectedEnvironment { @@ -606,8 +606,8 @@ func TestDockerPortsMapping(t *testing.T) { } expectedEnvironment := map[string]string{ - "NOMAD_PORT_main": "8080", - "NOMAD_PORT_REDIS": "6379", + "NOMAD_IP_main": "127.0.0.1:8080", + "NOMAD_IP_REDIS": "127.0.0.1:6379", } for key, val := range expectedEnvironment { diff --git a/client/driver/driver.go b/client/driver/driver.go index 89a48f569..5fd1238b3 100644 --- a/client/driver/driver.go +++ b/client/driver/driver.go @@ -148,12 +148,7 @@ func GetTaskEnv(alloc *allocdir.AllocDir, node *structs.Node, task *structs.Task if task.Resources != nil { env.SetMemLimit(task.Resources.MemoryMB) env.SetCpuLimit(task.Resources.CPU) - - if len(task.Resources.Networks) > 0 { - network := task.Resources.Networks[0] - env.SetTaskIp(network.IP) - env.SetPorts(network.MapLabelToValues(nil)) - } + env.SetNetworks(task.Resources.Networks) } return env.Build(), nil diff --git a/client/driver/driver_test.go b/client/driver/driver_test.go index 02b74f835..19109def7 100644 --- a/client/driver/driver_test.go +++ b/client/driver/driver_test.go @@ -114,12 +114,12 @@ func TestDriver_GetTaskEnv(t *testing.T) { "NOMAD_CPU_LIMIT": "1000", "NOMAD_MEMORY_LIMIT": "500", "NOMAD_IP": "1.2.3.4", - "NOMAD_PORT_one": "80", - "NOMAD_PORT_two": "443", - "NOMAD_PORT_three": "8080", - "NOMAD_PORT_four": "12345", - "NOMAD_PORT_admin": "8081", - "NOMAD_PORT_web": "8086", + "NOMAD_IP_one": "1.2.3.4:80", + "NOMAD_IP_two": "1.2.3.4:443", + "NOMAD_IP_three": "1.2.3.4:8080", + "NOMAD_IP_four": "1.2.3.4:12345", + "NOMAD_IP_admin": "1.2.3.4:8081", + "NOMAD_IP_web": "1.2.3.4:8086", "NOMAD_META_CHOCOLATE": "cake", "NOMAD_META_STRAWBERRY": "icecream", "HELLO": "world", diff --git a/client/driver/env/env.go b/client/driver/env/env.go index 5ed202355..736f4c38a 100644 --- a/client/driver/env/env.go +++ b/client/driver/env/env.go @@ -30,8 +30,8 @@ const ( // Prefix for passing both dynamic and static port allocations to // tasks. - // E.g. $NOMAD_PORT_1 or $NOMAD_PORT_http - PortPrefix = "NOMAD_PORT_" + // E.g. $NOMAD_IP_1=127.0.0.1:1 or $NOMAD_IP_http=127.0.0.1:80 + IPPortPrefix = "NOMAD_IP_" // Prefix for passing task meta data. MetaPrefix = "NOMAD_META_" @@ -54,13 +54,13 @@ const ( type TaskEnvironment struct { env map[string]string meta map[string]string - ports map[string]int allocDir string taskDir string cpuLimit int memLimit int - ip string node *structs.Node + networks []*structs.NetworkResource + portMap map[string]int // taskEnv is the variables that will be set in the tasks environment taskEnv map[string]string @@ -103,8 +103,11 @@ func (t *TaskEnvironment) Build() *TaskEnvironment { } // Build the ports - for label, port := range t.ports { - t.taskEnv[fmt.Sprintf("%s%s", PortPrefix, label)] = strconv.Itoa(port) + for _, network := range t.networks { + for label, value := range network.MapLabelToValues(t.portMap) { + IPPort := fmt.Sprintf("%s:%d", network.IP, value) + t.taskEnv[fmt.Sprintf("%s%s", IPPortPrefix, label)] = IPPort + } } // Build the directories @@ -124,8 +127,8 @@ func (t *TaskEnvironment) Build() *TaskEnvironment { } // Build the IP - if t.ip != "" { - t.taskEnv[TaskIP] = t.ip + if len(t.networks) > 0 { + t.taskEnv[TaskIP] = t.networks[0].IP } // Build the node @@ -221,24 +224,23 @@ func (t *TaskEnvironment) ClearCpuLimit() *TaskEnvironment { return t } -func (t *TaskEnvironment) SetTaskIp(ip string) *TaskEnvironment { - t.ip = ip +func (t *TaskEnvironment) SetNetworks(networks []*structs.NetworkResource) *TaskEnvironment { + t.networks = networks return t } -func (t *TaskEnvironment) ClearTaskIp() *TaskEnvironment { - t.ip = "" +func (t *TaskEnvironment) clearNetworks() *TaskEnvironment { + t.networks = nil return t } -// Takes a map of port labels to their port value. -func (t *TaskEnvironment) SetPorts(ports map[string]int) *TaskEnvironment { - t.ports = ports +func (t *TaskEnvironment) SetPortMap(portMap map[string]int) *TaskEnvironment { + t.portMap = portMap return t } -func (t *TaskEnvironment) ClearPorts() *TaskEnvironment { - t.ports = nil +func (t *TaskEnvironment) clearPortMap() *TaskEnvironment { + t.portMap = nil return t } diff --git a/client/driver/env/env_test.go b/client/driver/env/env_test.go index 43a8eb65d..ed4a509e6 100644 --- a/client/driver/env/env_test.go +++ b/client/driver/env/env_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/hashicorp/nomad/nomad/mock" + "github.com/hashicorp/nomad/nomad/structs" ) const ( @@ -25,6 +26,20 @@ const ( envTwoVal = ":80" ) +var ( + // Networks that tests can rely on + networks = []*structs.NetworkResource{ + &structs.NetworkResource{ + IP: "127.0.0.1", + ReservedPorts: []structs.Port{{"http", 80}}, + DynamicPorts: []structs.Port{{"https", 8080}}, + }, + } + portMap = map[string]int{ + "https": 443, + } +) + func testTaskEnvironment() *TaskEnvironment { n := mock.Node() n.Attributes = map[string]string{ @@ -121,11 +136,17 @@ func TestEnvironment_ReplaceEnv_Mixed(t *testing.T) { func TestEnvironment_AsList(t *testing.T) { n := mock.Node() env := NewTaskEnvironment(n). - SetTaskIp("127.0.0.1").SetPorts(map[string]int{"http": 80}). + SetNetworks(networks). + SetPortMap(portMap). SetMeta(map[string]string{"foo": "baz"}).Build() act := env.EnvList() - exp := []string{"NOMAD_IP=127.0.0.1", "NOMAD_PORT_http=80", "NOMAD_META_FOO=baz"} + exp := []string{ + "NOMAD_IP=127.0.0.1", + "NOMAD_IP_http=127.0.0.1:80", + "NOMAD_IP_https=127.0.0.1:443", + "NOMAD_META_FOO=baz", + } sort.Strings(act) sort.Strings(exp) if !reflect.DeepEqual(act, exp) { @@ -136,11 +157,18 @@ func TestEnvironment_AsList(t *testing.T) { func TestEnvironment_ClearEnvvars(t *testing.T) { n := mock.Node() env := NewTaskEnvironment(n). - SetTaskIp("127.0.0.1"). + SetNetworks(networks). + SetPortMap(portMap). SetEnvvars(map[string]string{"foo": "baz", "bar": "bang"}).Build() act := env.EnvList() - exp := []string{"NOMAD_IP=127.0.0.1", "bar=bang", "foo=baz"} + exp := []string{ + "NOMAD_IP=127.0.0.1", + "NOMAD_IP_http=127.0.0.1:80", + "NOMAD_IP_https=127.0.0.1:443", + "bar=bang", + "foo=baz", + } sort.Strings(act) sort.Strings(exp) if !reflect.DeepEqual(act, exp) { @@ -151,7 +179,11 @@ func TestEnvironment_ClearEnvvars(t *testing.T) { env.ClearEnvvars().Build() act = env.EnvList() - exp = []string{"NOMAD_IP=127.0.0.1"} + exp = []string{ + "NOMAD_IP=127.0.0.1", + "NOMAD_IP_http=127.0.0.1:80", + "NOMAD_IP_https=127.0.0.1:443", + } sort.Strings(act) sort.Strings(exp) if !reflect.DeepEqual(act, exp) { From b9de8704eeb98e7a864206908e872e0dbc6ca7a9 Mon Sep 17 00:00:00 2001 From: Ivo Verberk Date: Mon, 25 Jan 2016 20:46:01 +0100 Subject: [PATCH 2/3] Rename IPPort env variable and add a mapped host port to the env. --- client/driver/docker.go | 6 ------ client/driver/docker_test.go | 9 +++++---- client/driver/driver_test.go | 13 ++++++------- client/driver/env/env.go | 17 ++++++++++------- client/driver/env/env_test.go | 18 +++++++++--------- 5 files changed, 30 insertions(+), 33 deletions(-) diff --git a/client/driver/docker.go b/client/driver/docker.go index 9a77328f3..631babdfd 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -341,12 +341,6 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri d.logger.Printf("[DEBUG] driver.docker: exposed port %s", containerPort) } - // This was set above in a call to GetTaskEnv but if we - // have mapped any ports we will need to override them. - // - // TODO refactor the implementation in GetTaskEnv to match - // the 0.2 ports world view. Docker seems to be the only place where - // this is actually needed, but this is kinda hacky. d.taskEnv.SetPortMap(driverConfig.PortMap) hostConfig.PortBindings = publishedPorts diff --git a/client/driver/docker_test.go b/client/driver/docker_test.go index ae451f540..0eb09ce2e 100644 --- a/client/driver/docker_test.go +++ b/client/driver/docker_test.go @@ -551,8 +551,8 @@ func TestDockerPortsNoMap(t *testing.T) { } expectedEnvironment := map[string]string{ - "NOMAD_IP_main": fmt.Sprintf("127.0.0.1:%d", res), - "NOMAD_IP_REDIS": fmt.Sprintf("127.0.0.1:%d", dyn), + "NOMAD_ADDR_main": fmt.Sprintf("127.0.0.1:%d", res), + "NOMAD_ADDR_REDIS": fmt.Sprintf("127.0.0.1:%d", dyn), } for key, val := range expectedEnvironment { @@ -606,8 +606,9 @@ func TestDockerPortsMapping(t *testing.T) { } expectedEnvironment := map[string]string{ - "NOMAD_IP_main": "127.0.0.1:8080", - "NOMAD_IP_REDIS": "127.0.0.1:6379", + "NOMAD_ADDR_main": "127.0.0.1:8080", + "NOMAD_ADDR_REDIS": "127.0.0.1:6379", + "NOMAD_HOST_PORT_main": "8080", } for key, val := range expectedEnvironment { diff --git a/client/driver/driver_test.go b/client/driver/driver_test.go index 19109def7..0ab20d813 100644 --- a/client/driver/driver_test.go +++ b/client/driver/driver_test.go @@ -113,13 +113,12 @@ func TestDriver_GetTaskEnv(t *testing.T) { exp := map[string]string{ "NOMAD_CPU_LIMIT": "1000", "NOMAD_MEMORY_LIMIT": "500", - "NOMAD_IP": "1.2.3.4", - "NOMAD_IP_one": "1.2.3.4:80", - "NOMAD_IP_two": "1.2.3.4:443", - "NOMAD_IP_three": "1.2.3.4:8080", - "NOMAD_IP_four": "1.2.3.4:12345", - "NOMAD_IP_admin": "1.2.3.4:8081", - "NOMAD_IP_web": "1.2.3.4:8086", + "NOMAD_ADDR_one": "1.2.3.4:80", + "NOMAD_ADDR_two": "1.2.3.4:443", + "NOMAD_ADDR_three": "1.2.3.4:8080", + "NOMAD_ADDR_four": "1.2.3.4:12345", + "NOMAD_ADDR_admin": "1.2.3.4:8081", + "NOMAD_ADDR_web": "1.2.3.4:8086", "NOMAD_META_CHOCOLATE": "cake", "NOMAD_META_STRAWBERRY": "icecream", "HELLO": "world", diff --git a/client/driver/env/env.go b/client/driver/env/env.go index 736f4c38a..c620599a6 100644 --- a/client/driver/env/env.go +++ b/client/driver/env/env.go @@ -31,7 +31,10 @@ const ( // Prefix for passing both dynamic and static port allocations to // tasks. // E.g. $NOMAD_IP_1=127.0.0.1:1 or $NOMAD_IP_http=127.0.0.1:80 - IPPortPrefix = "NOMAD_IP_" + AddrPrefix = "NOMAD_ADDR_" + + // Prefix for passing the host port when a portmap is specified. + HostPortPrefix = "NOMAD_HOST_PORT_" // Prefix for passing task meta data. MetaPrefix = "NOMAD_META_" @@ -106,7 +109,12 @@ func (t *TaskEnvironment) Build() *TaskEnvironment { for _, network := range t.networks { for label, value := range network.MapLabelToValues(t.portMap) { IPPort := fmt.Sprintf("%s:%d", network.IP, value) - t.taskEnv[fmt.Sprintf("%s%s", IPPortPrefix, label)] = IPPort + t.taskEnv[fmt.Sprintf("%s%s", AddrPrefix, label)] = IPPort + + // Pass an explicit port mapping to the environment + if port, ok := t.portMap[label]; ok { + t.taskEnv[fmt.Sprintf("%s%s", HostPortPrefix, label)] = strconv.Itoa(port) + } } } @@ -126,11 +134,6 @@ func (t *TaskEnvironment) Build() *TaskEnvironment { t.taskEnv[CpuLimit] = strconv.Itoa(t.cpuLimit) } - // Build the IP - if len(t.networks) > 0 { - t.taskEnv[TaskIP] = t.networks[0].IP - } - // Build the node if t.node != nil { // Set up the node values. diff --git a/client/driver/env/env_test.go b/client/driver/env/env_test.go index ed4a509e6..6cd476b10 100644 --- a/client/driver/env/env_test.go +++ b/client/driver/env/env_test.go @@ -142,9 +142,9 @@ func TestEnvironment_AsList(t *testing.T) { act := env.EnvList() exp := []string{ - "NOMAD_IP=127.0.0.1", - "NOMAD_IP_http=127.0.0.1:80", - "NOMAD_IP_https=127.0.0.1:443", + "NOMAD_ADDR_http=127.0.0.1:80", + "NOMAD_ADDR_https=127.0.0.1:443", + "NOMAD_HOST_PORT_https=443", "NOMAD_META_FOO=baz", } sort.Strings(act) @@ -163,9 +163,9 @@ func TestEnvironment_ClearEnvvars(t *testing.T) { act := env.EnvList() exp := []string{ - "NOMAD_IP=127.0.0.1", - "NOMAD_IP_http=127.0.0.1:80", - "NOMAD_IP_https=127.0.0.1:443", + "NOMAD_ADDR_http=127.0.0.1:80", + "NOMAD_ADDR_https=127.0.0.1:443", + "NOMAD_HOST_PORT_https=443", "bar=bang", "foo=baz", } @@ -180,9 +180,9 @@ func TestEnvironment_ClearEnvvars(t *testing.T) { act = env.EnvList() exp = []string{ - "NOMAD_IP=127.0.0.1", - "NOMAD_IP_http=127.0.0.1:80", - "NOMAD_IP_https=127.0.0.1:443", + "NOMAD_ADDR_http=127.0.0.1:80", + "NOMAD_ADDR_https=127.0.0.1:443", + "NOMAD_HOST_PORT_https=443", } sort.Strings(act) sort.Strings(exp) From c5ef55bf7c0023576e5062e42a8b4c80d5eb5e8d Mon Sep 17 00:00:00 2001 From: Ivo Verberk Date: Fri, 29 Jan 2016 17:48:24 +0100 Subject: [PATCH 3/3] Documentation update for the new port environment variables. --- website/source/docs/jobspec/networking.html.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/website/source/docs/jobspec/networking.html.md b/website/source/docs/jobspec/networking.html.md index 8062f6727..3c53d3636 100644 --- a/website/source/docs/jobspec/networking.html.md +++ b/website/source/docs/jobspec/networking.html.md @@ -89,10 +89,10 @@ port "http" {} ``` When the task is started, it is passed an environment variable named -`NOMAD_PORT_http` which indicates the port. +`NOMAD_ADDR_http` which indicates a combination of the interface IP and port. ``` -NOMAD_PORT_http=53423 ./start-command +NOMAD_ADDR_http=127.0.0.1:53423 ./start-command ``` ### Mapped Ports @@ -118,4 +118,8 @@ The above example is for the Docker driver. The service is listening on port `8080` inside the container. The driver will automatically map the dynamic port to this service. +When the task is started, it is passed an additional environment variable named +`NOMAD_HOST_PORT_http` which indicates the host port that the http service is +bound to. + Please refer to the [Docker](/docs/drivers/docker.html) and [QEMU](/docs/drivers/qemu.html) drivers for additional information.