diff --git a/CHANGELOG.md b/CHANGELOG.md index 91b792ed5..13a6934cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ __BACKWARDS INCOMPATIBILITIES:__ * csi: The `attachment_mode` and `access_mode` field are required for `volume` blocks in job specifications. Registering a volume requires at least one `capability` block with the `attachment_mode` and `access_mode` fields set. [[GH-10330](https://github.com/hashicorp/nomad/issues/10330)] * licensing: Enterprise licenses are no longer stored in raft or synced between servers. Loading the Enterprise license from disk or environment is required. The `nomad license put` command has been removed. [[GH-10458](https://github.com/hashicorp/nomad/issues/10458)] +SECURITY: + * drivers/docker+exec+java: Disable `CAP_NET_RAW` linux capability by default to prevent ARP spoofing. CVE-2021-32575 [GH-10568](https://github.com/hashicorp/nomad/issues/10568) + IMPROVEMENTS: * api: Added an API endpoint for fuzzy search queries [[GH-10184](https://github.com/hashicorp/nomad/pull/10184)] * api: Removed unimplemented `CSIVolumes.PluginList` API. [[GH-10158](https://github.com/hashicorp/nomad/issues/10158)] @@ -72,7 +75,7 @@ BUG FIXES: * server: Fixed a panic that may arise on submission of jobs containing invalid service checks [[GH-10154](https://github.com/hashicorp/nomad/issues/10154)] * ui: Fixed the rendering of interstitial components shown after processing a dynamic application sizing recommendation. [[GH-10094](https://github.com/hashicorp/nomad/pull/10094)] -## 1.0.5 (Unreleased) +## 1.0.6 (Unreleased) BUG FIXES: * core (Enterprise): Update licensing library to v0.0.11 to include race condition fix. [[GH-10253](https://github.com/hashicorp/nomad/issues/10253)] @@ -101,6 +104,11 @@ BUG FIXES: * server: Fixed a panic that may arise on submission of jobs containing invalid service checks [[GH-10154](https://github.com/hashicorp/nomad/issues/10154)] * ui: Fixed the rendering of interstitial components shown after processing a dynamic application sizing recommendation. [[GH-10094](https://github.com/hashicorp/nomad/pull/10094)] +## 1.0.5 (May 11, 2021) + +SECURITY: + * drivers/docker+exec+java: Disable `CAP_NET_RAW` linux capability by default to prevent ARP spoofing. CVE-2021-32575 [GH-10568](https://github.com/hashicorp/nomad/issues/10568) + ## 1.0.4 (February 24, 2021) FEATURES: @@ -277,6 +285,11 @@ BUG FIXES: * ui: Fixed a bug in the volume status page where read allocations and write allocations were not displayed. [[GH-9377](https://github.com/hashicorp/nomad/issues/9377)] * ui: Fixed a bug in the CSI volume and plugin status pages where plugins that don't require controllers were shown as unhealthy. [[GH-9416](https://github.com/hashicorp/nomad/issues/9416)] +## 0.12.12 (May 11, 2021) + +SECURITY: + * drivers/docker+exec+java: Disable `CAP_NET_RAW` linux capability by default to prevent ARP spoofing. CVE-2021-32575 [GH-10568](https://github.com/hashicorp/nomad/issues/10568) + ## 0.12.11 (March 18, 2021) BUG FIXES: diff --git a/drivers/docker/config.go b/drivers/docker/config.go index c92e42784..ec919360d 100644 --- a/drivers/docker/config.go +++ b/drivers/docker/config.go @@ -36,17 +36,41 @@ const ( // it is timed out. dockerTimeout = 5 * time.Minute - // dockerBasicCaps is comma-separated list of Linux capabilities that are - // allowed by docker by default, as documented in - // https://docs.docker.com/engine/reference/run/#block-io-bandwidth-blkio-constraint - dockerBasicCaps = "CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID," + - "SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE" - // dockerAuthHelperPrefix is the prefix to attach to the credential helper // and should be found in the $PATH. Example: ${prefix-}${helper-name} dockerAuthHelperPrefix = "docker-credential-" ) +// nomadDefaultCaps is the subset of dockerDefaultCaps that Nomad enables by +// default and is used to compute the set of capabilities to add/drop given +// docker driver configuration. +func nomadDefaultCaps() []string { + return []string{ + "AUDIT_WRITE", + "CHOWN", + "DAC_OVERRIDE", + "FOWNER", + "FSETID", + "KILL", + "MKNOD", + "NET_BIND_SERVICE", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_CHROOT", + } +} + +// dockerDefaultCaps is a list of Linux capabilities enabled by docker by default +// and is used to compute the set of capabilities to add/drop given docker driver +// configuration, as well as Nomad built-in limitations. +// +// https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities +func dockerDefaultCaps() []string { + return append(nomadDefaultCaps(), "NET_RAW") +} + func PluginLoader(opts map[string]string) (map[string]interface{}, error) { conf := map[string]interface{}{} if v, ok := opts["docker.endpoint"]; ok { @@ -263,7 +287,7 @@ var ( "allow_privileged": hclspec.NewAttr("allow_privileged", "bool", false), "allow_caps": hclspec.NewDefault( hclspec.NewAttr("allow_caps", "list(string)", false), - hclspec.NewLiteral(`["CHOWN","DAC_OVERRIDE","FSETID","FOWNER","MKNOD","NET_RAW","SETGID","SETUID","SETFCAP","SETPCAP","NET_BIND_SERVICE","SYS_CHROOT","KILL","AUDIT_WRITE"]`), + hclspec.NewLiteral(`["CHOWN","DAC_OVERRIDE","FSETID","FOWNER","MKNOD","SETGID","SETUID","SETFCAP","SETPCAP","NET_BIND_SERVICE","SYS_CHROOT","KILL","AUDIT_WRITE"]`), ), "nvidia_runtime": hclspec.NewDefault( hclspec.NewAttr("nvidia_runtime", "string", false), diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go index bf0f6d380..c8a2981ec 100644 --- a/drivers/docker/driver.go +++ b/drivers/docker/driver.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "runtime" + "sort" "strconv" "strings" "sync" @@ -23,7 +24,9 @@ import ( "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/drivers/docker/docklog" "github.com/hashicorp/nomad/drivers/shared/eventer" + "github.com/hashicorp/nomad/drivers/shared/executor" "github.com/hashicorp/nomad/drivers/shared/resolvconf" + "github.com/hashicorp/nomad/helper" nstructs "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" @@ -909,38 +912,12 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T } hostConfig.Privileged = driverConfig.Privileged - // set capabilities - hostCapsWhitelistConfig := d.config.AllowCaps - hostCapsWhitelist := make(map[string]struct{}) - for _, cap := range hostCapsWhitelistConfig { - cap = strings.ToLower(strings.TrimSpace(cap)) - hostCapsWhitelist[cap] = struct{}{} + // set add/drop capabilities + hostConfig.CapAdd, hostConfig.CapDrop, err = d.getCaps(driverConfig) + if err != nil { + return c, err } - if _, ok := hostCapsWhitelist["all"]; !ok { - effectiveCaps, err := tweakCapabilities( - strings.Split(dockerBasicCaps, ","), - driverConfig.CapAdd, - driverConfig.CapDrop, - ) - if err != nil { - return c, err - } - var missingCaps []string - for _, cap := range effectiveCaps { - cap = strings.ToLower(cap) - if _, ok := hostCapsWhitelist[cap]; !ok { - missingCaps = append(missingCaps, cap) - } - } - if len(missingCaps) > 0 { - return c, fmt.Errorf("Docker driver doesn't have the following caps allowlisted on this Nomad agent: %s", missingCaps) - } - } - - hostConfig.CapAdd = driverConfig.CapAdd - hostConfig.CapDrop = driverConfig.CapDrop - // set SHM size if driverConfig.ShmSize != 0 { hostConfig.ShmSize = driverConfig.ShmSize @@ -1207,6 +1184,119 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T }, nil } +// getCaps computes the capabilities to supply to the --add-cap and --drop-cap +// options to the docker driver, which override the default capabilities enabled +// by docker itself. +func (d *Driver) getCaps(taskConfig *TaskConfig) ([]string, []string, error) { + + // capabilities allowable by client docker plugin configuration + allowCaps := expandAllowCaps(d.config.AllowCaps) + + // capabilities the task docker config is asking for based on the default + // capabilities allowable by nomad + desiredCaps, err := tweakCapabilities(nomadDefaultCaps(), taskConfig.CapAdd, taskConfig.CapDrop) + if err != nil { + return nil, nil, err + } + + // capabilities the task is requesting that are NOT allowed by the docker plugin + if missing := missingCaps(allowCaps, desiredCaps); len(missing) > 0 { + return nil, nil, fmt.Errorf("Docker driver does not have the following caps allow-listed on this Nomad agent: %s", missing) + } + + // capabilities that should be dropped relative to the docker default capabilities + dropCaps := capDrops(taskConfig.CapDrop, allowCaps) + + return taskConfig.CapAdd, dropCaps, nil +} + +// capDrops will compute the total dropped capabilities set +// +// {task cap_drop} U ({docker defaults} \ {driver allow caps}) +func capDrops(dropCaps []string, allowCaps []string) []string { + dropSet := make(map[string]struct{}) + + for _, c := range normalizeCaps(dropCaps) { + dropSet[c] = struct{}{} + } + + // if dropCaps includes ALL, no need to iterate every capability + if _, exists := dropSet["ALL"]; exists { + return []string{"ALL"} + } + + dockerDefaults := helper.SliceStringToSet(normalizeCaps(dockerDefaultCaps())) + allowedCaps := helper.SliceStringToSet(normalizeCaps(allowCaps)) + + // find the docker default caps not in allowed caps + for dCap := range dockerDefaults { + if _, exists := allowedCaps[dCap]; !exists { + dropSet[dCap] = struct{}{} + } + } + + drops := make([]string, 0, len(dropSet)) + for c := range dropSet { + drops = append(drops, c) + } + sort.Strings(drops) + return drops +} + +// expandAllowCaps returns the normalized set of allowable capabilities set +// for the docker plugin configuration. +func expandAllowCaps(allowCaps []string) []string { + if len(allowCaps) == 0 { + return nil + } + + set := make(map[string]struct{}, len(allowCaps)) + + for _, rawCap := range allowCaps { + capability := strings.ToUpper(rawCap) + if capability == "ALL" { + for _, defCap := range normalizeCaps(executor.SupportedCaps(true)) { + set[defCap] = struct{}{} + } + } else { + set[capability] = struct{}{} + } + } + + result := make([]string, 0, len(set)) + for capability := range set { + result = append(result, capability) + } + sort.Strings(result) + return result +} + +// missingCaps returns the set of elements in desired that are not present in +// allowed. The elements in desired are first upper-cased before comparison. +// The elements in allowed are assumed to be upper-cased. +func missingCaps(allowed, desired []string) []string { + _, missing := helper.SliceStringIsSubset(allowed, normalizeCaps(desired)) + sort.Strings(missing) + return missing +} + +// normalizeCaps returns a copy of caps with duplicate elements removed and all +// elements upper-cased. +func normalizeCaps(caps []string) []string { + set := make(map[string]struct{}, len(caps)) + for _, c := range caps { + normal := strings.TrimPrefix(strings.ToUpper(c), "CAP_") + set[strings.ToUpper(normal)] = struct{}{} + } + + result := make([]string, 0, len(set)) + for c := range set { + result = append(result, c) + } + sort.Strings(result) + return result +} + func (d *Driver) toDockerMount(m *DockerMount, task *drivers.TaskConfig) (*docker.HostMount, error) { hm, err := m.toDockerHostMount() if err != nil { diff --git a/drivers/docker/driver_default.go b/drivers/docker/driver_default.go index 295706bd9..45086f7bf 100644 --- a/drivers/docker/driver_default.go +++ b/drivers/docker/driver_default.go @@ -27,5 +27,6 @@ func tweakCapabilities(basics, adds, drops []string) ([]string, error) { for i, cap := range effectiveCaps { effectiveCaps[i] = cap[len("CAP_"):] } + return effectiveCaps, nil } diff --git a/drivers/docker/driver_test.go b/drivers/docker/driver_test.go index abfef16be..67c22e88e 100644 --- a/drivers/docker/driver_test.go +++ b/drivers/docker/driver_test.go @@ -19,6 +19,7 @@ import ( hclog "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/client/testutil" + "github.com/hashicorp/nomad/drivers/shared/executor" "github.com/hashicorp/nomad/helper/freeport" "github.com/hashicorp/nomad/helper/pluginutils/hclspecutils" "github.com/hashicorp/nomad/helper/pluginutils/hclutils" @@ -1386,44 +1387,44 @@ func TestDockerDriver_Capabilities(t *testing.T) { { Name: "default-allowlist-add-allowed", CapAdd: []string{"fowner", "mknod"}, - CapDrop: []string{"all"}, + CapDrop: []string{"ALL"}, }, { Name: "default-allowlist-add-forbidden", CapAdd: []string{"net_admin"}, - StartError: "net_admin", + StartError: "NET_ADMIN", }, { Name: "default-allowlist-drop-existing", - CapDrop: []string{"fowner", "mknod"}, + CapDrop: []string{"FOWNER", "MKNOD", "NET_RAW"}, }, { Name: "restrictive-allowlist-drop-all", - CapDrop: []string{"all"}, - Allowlist: "fowner,mknod", + CapDrop: []string{"ALL"}, + Allowlist: "FOWNER,MKNOD", }, { Name: "restrictive-allowlist-add-allowed", CapAdd: []string{"fowner", "mknod"}, - CapDrop: []string{"all"}, + CapDrop: []string{"ALL"}, Allowlist: "fowner,mknod", }, { Name: "restrictive-allowlist-add-forbidden", CapAdd: []string{"net_admin", "mknod"}, - CapDrop: []string{"all"}, + CapDrop: []string{"ALL"}, Allowlist: "fowner,mknod", - StartError: "net_admin", + StartError: "NET_ADMIN", }, { Name: "permissive-allowlist", CapAdd: []string{"net_admin", "mknod"}, - Allowlist: "all", + Allowlist: "ALL", }, { Name: "permissive-allowlist-add-all", CapAdd: []string{"all"}, - Allowlist: "all", + Allowlist: "ALL", }, } @@ -3063,3 +3064,169 @@ func TestDockerDriver_StopSignal(t *testing.T) { }) } } + +func TestDockerCaps_normalizeCaps(t *testing.T) { + t.Run("empty", func(t *testing.T) { + result := normalizeCaps(nil) + require.Len(t, result, 0) + }) + + t.Run("mixed", func(t *testing.T) { + result := normalizeCaps([]string{ + "DAC_OVERRIDE", "sys_chroot", "kill", "KILL", + }) + require.Equal(t, []string{ + "DAC_OVERRIDE", "KILL", "SYS_CHROOT", + }, result) + }) +} + +func TestDockerCaps_missingCaps(t *testing.T) { + allowed := []string{ + "DAC_OVERRIDE", "SYS_CHROOT", "KILL", "CHOWN", + } + + t.Run("none missing", func(t *testing.T) { + result := missingCaps(allowed, []string{ + "SYS_CHROOT", "chown", "KILL", + }) + require.Equal(t, []string(nil), result) + }) + + t.Run("some missing", func(t *testing.T) { + result := missingCaps(allowed, []string{ + "chown", "audit_write", "SETPCAP", "dac_override", + }) + require.Equal(t, []string{"AUDIT_WRITE", "SETPCAP"}, result) + }) +} + +func TestDockerCaps_expandAllowCaps(t *testing.T) { + t.Run("empty", func(t *testing.T) { + result := expandAllowCaps(nil) + require.Empty(t, result) + }) + + t.Run("manual", func(t *testing.T) { + result := expandAllowCaps([]string{ + "DAC_OVERRIDE", "SYS_CHROOT", "KILL", "CHOWN", + }) + require.Equal(t, []string{ + "CHOWN", "DAC_OVERRIDE", "KILL", "SYS_CHROOT", + }, result) + }) + + t.Run("all", func(t *testing.T) { + result := expandAllowCaps([]string{"all"}) + exp := normalizeCaps(executor.SupportedCaps(true)) + sort.Strings(exp) + require.Equal(t, exp, result) + }) +} + +func TestDockerCaps_capDrops(t *testing.T) { + // docker default caps is always the same, task configured drop_caps and + // plugin config allow_caps may be altered + + // This is the 90% use case, where NET_RAW is dropped, as Nomad's default + // capability allow-list is a subset of the docker default cap list. + t.Run("defaults", func(t *testing.T) { + result := capDrops(nil, nomadDefaultCaps()) + require.Equal(t, []string{"NET_RAW"}, result) + }) + + // Users want to use ICMP (ping). + t.Run("enable net_raw", func(t *testing.T) { + result := capDrops(nil, append(nomadDefaultCaps(), "net_raw")) + require.Empty(t, result) + }) + + // The plugin is reduced in ability. + t.Run("enable minimal", func(t *testing.T) { + allow := []string{"setgid", "setuid", "chown", "kill"} + exp := []string{"AUDIT_WRITE", "DAC_OVERRIDE", "FOWNER", "FSETID", "MKNOD", "NET_BIND_SERVICE", "NET_RAW", "SETFCAP", "SETPCAP", "SYS_CHROOT"} + result := capDrops(nil, allow) + require.Equal(t, exp, result) + }) + + // The task drops abilities. + t.Run("task drops", func(t *testing.T) { + drops := []string{"audit_write", "fowner", "kill", "chown"} + exp := []string{"AUDIT_WRITE", "CHOWN", "FOWNER", "KILL", "NET_RAW"} + result := capDrops(drops, nomadDefaultCaps()) + require.Equal(t, exp, result) + }) + + // Drop all mixed with others. + t.Run("task drops mix", func(t *testing.T) { + drops := []string{"audit_write", "all", "chown"} + exp := []string{"ALL"} // minimized + result := capDrops(drops, nomadDefaultCaps()) + require.Equal(t, exp, result) + }) +} + +func TestDockerCaps_getCaps(t *testing.T) { + testutil.ExecCompatible(t) // tests require linux + + t.Run("defaults", func(t *testing.T) { + d := Driver{config: &DriverConfig{ + AllowCaps: nomadDefaultCaps(), + }} + add, drop, err := d.getCaps(&TaskConfig{ + CapAdd: nil, CapDrop: nil, + }) + require.NoError(t, err) + require.Empty(t, add) + require.Equal(t, []string{"NET_RAW"}, drop) + }) + + t.Run("enable net_raw", func(t *testing.T) { + d := Driver{config: &DriverConfig{ + AllowCaps: append(nomadDefaultCaps(), "net_raw"), + }} + add, drop, err := d.getCaps(&TaskConfig{ + CapAdd: nil, CapDrop: nil, + }) + require.NoError(t, err) + require.Empty(t, add) + require.Empty(t, drop) + }) + + t.Run("block sys_time", func(t *testing.T) { + d := Driver{config: &DriverConfig{ + AllowCaps: nomadDefaultCaps(), + }} + _, _, err := d.getCaps(&TaskConfig{ + CapAdd: []string{"SYS_TIME"}, + CapDrop: nil, + }) + require.EqualError(t, err, `Docker driver does not have the following caps allow-listed on this Nomad agent: [SYS_TIME]`) + }) + + t.Run("enable sys_time", func(t *testing.T) { + d := Driver{config: &DriverConfig{ + AllowCaps: append(nomadDefaultCaps(), "sys_time"), + }} + add, drop, err := d.getCaps(&TaskConfig{ + CapAdd: []string{"SYS_TIME"}, + CapDrop: nil, + }) + require.NoError(t, err) + require.Equal(t, []string{"SYS_TIME"}, add) + require.Equal(t, []string{"NET_RAW"}, drop) + }) + + t.Run("task drops chown", func(t *testing.T) { + d := Driver{config: &DriverConfig{ + AllowCaps: nomadDefaultCaps(), + }} + add, drop, err := d.getCaps(&TaskConfig{ + CapAdd: nil, + CapDrop: []string{"chown"}, + }) + require.NoError(t, err) + require.Empty(t, add) + require.Equal(t, []string{"CHOWN", "NET_RAW"}, drop) + }) +} diff --git a/drivers/shared/executor/executor.go b/drivers/shared/executor/executor.go index bdee704a9..a87674d2a 100644 --- a/drivers/shared/executor/executor.go +++ b/drivers/shared/executor/executor.go @@ -23,6 +23,7 @@ import ( cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/plugins/drivers" "github.com/kr/pty" + "github.com/syndtr/gocapability/capability" shelpers "github.com/hashicorp/nomad/helper/stats" ) @@ -684,3 +685,23 @@ func makeExecutable(binPath string) error { } return nil } + +// SupportedCaps returns a list of all supported capabilities in kernel. +func SupportedCaps(allowNetRaw bool) []string { + var allCaps []string + last := capability.CAP_LAST_CAP + // workaround for RHEL6 which has no /proc/sys/kernel/cap_last_cap + if last == capability.Cap(63) { + last = capability.CAP_BLOCK_SUSPEND + } + for _, cap := range capability.List() { + if cap > last { + continue + } + if !allowNetRaw && cap == capability.CAP_NET_RAW { + continue + } + allCaps = append(allCaps, fmt.Sprintf("CAP_%s", strings.ToUpper(cap.String()))) + } + return allCaps +} diff --git a/drivers/shared/executor/executor_linux.go b/drivers/shared/executor/executor_linux.go index 7c358e379..16eb27b4c 100644 --- a/drivers/shared/executor/executor_linux.go +++ b/drivers/shared/executor/executor_linux.go @@ -32,7 +32,6 @@ import ( ldevices "github.com/opencontainers/runc/libcontainer/devices" "github.com/opencontainers/runc/libcontainer/specconv" lutils "github.com/opencontainers/runc/libcontainer/utils" - "github.com/syndtr/gocapability/capability" "golang.org/x/sys/unix" ) @@ -534,12 +533,12 @@ func (l *LibcontainerExecutor) handleExecWait(ch chan *waitResult, process *libc } func configureCapabilities(cfg *lconfigs.Config, command *ExecCommand) error { - // TODO: allow better control of these + // TODO(shoenig): allow better control of these // use capabilities list as prior to adopting libcontainer in 0.9 - allCaps := supportedCaps() // match capabilities used in Nomad 0.8 if command.User == "root" { + allCaps := SupportedCaps(true) cfg.Capabilities = &lconfigs.Capabilities{ Bounding: allCaps, Permitted: allCaps, @@ -548,6 +547,7 @@ func configureCapabilities(cfg *lconfigs.Config, command *ExecCommand) error { Inheritable: nil, } } else { + allCaps := SupportedCaps(false) cfg.Capabilities = &lconfigs.Capabilities{ Bounding: allCaps, } @@ -556,23 +556,6 @@ func configureCapabilities(cfg *lconfigs.Config, command *ExecCommand) error { return nil } -// supportedCaps returns a list of all supported capabilities in kernel -func supportedCaps() []string { - allCaps := []string{} - last := capability.CAP_LAST_CAP - // workaround for RHEL6 which has no /proc/sys/kernel/cap_last_cap - if last == capability.Cap(63) { - last = capability.CAP_BLOCK_SUSPEND - } - for _, cap := range capability.List() { - if cap > last { - continue - } - allCaps = append(allCaps, fmt.Sprintf("CAP_%s", strings.ToUpper(cap.String()))) - } - return allCaps -} - func configureNamespaces(pidMode, ipcMode string) lconfigs.Namespaces { namespaces := lconfigs.Namespaces{{Type: lconfigs.NEWNS}} if pidMode == IsolationModePrivate { diff --git a/drivers/shared/executor/executor_linux_test.go b/drivers/shared/executor/executor_linux_test.go index 244862a04..a8bfba0a4 100644 --- a/drivers/shared/executor/executor_linux_test.go +++ b/drivers/shared/executor/executor_linux_test.go @@ -478,7 +478,7 @@ func TestExecutor_Capabilities(t *testing.T) { CapInh: 0000000000000000 CapPrm: 0000000000000000 CapEff: 0000000000000000 -CapBnd: 0000003fffffffff +CapBnd: 0000003fffffdfff CapAmb: 0000000000000000`, }, { diff --git a/website/content/docs/upgrade/upgrade-specific.mdx b/website/content/docs/upgrade/upgrade-specific.mdx index da376c91e..2ff1ade19 100644 --- a/website/content/docs/upgrade/upgrade-specific.mdx +++ b/website/content/docs/upgrade/upgrade-specific.mdx @@ -63,6 +63,51 @@ inserting them as the first rule. This allows better control for user-defined iptables rules but users who append rules currently should verify that their rules are being appended in the correct order. +## Nomad 1.1.0, 1.0.5, 0.12.12 + +Nomad versions 1.1.0, 1.0.5 and 0.12.12 change the behavior of the `docker`, `exec`, +and `java` task drivers so that the [`CAP_NET_RAW`] linux capability is disabled +by default. This is one of the [`linux capabilities`] that Docker itself enables +by default, as this capability enables the generation of ICMP packets - used by +the common `ping` utility for performing network diagnostics. When used by groups in +`bridge` networking mode, the `CAP_NET_RAW` capability also exposes tasks to ARP spoofing, +enabling DoS and MITM attacks against other tasks running in `bridge` networking +on the same host. Operators should weigh potential impact of an upgrade on their +applications against the security consequences inherit with `CAP_NET_RAW`. Typical +applications using `tcp` or `udp` based networking should not be affected. + +This is the sole change for Nomad 1.0.5 and 0.12.12, intended to provide better +task network isolation by default. + +Users of the `docker` driver can restore the previous behavior by configuring the +[`allow_caps`] driver configuration option to explicitly enable the `CAP_NET_RAW` +capability. + +```hcl +plugin "docker" { + config { + allow_caps = [ + "CHOWN", "DAC_OVERRIDE", "FSETID", "FOWNER", "MKNOD", + "SETGID", "SETUID", "SETFCAP", "SETPCAP", "NET_BIND_SERVICE", + "SYS_CHROOT", "KILL", "AUDIT_WRITE", "NET_RAW", + ] + } +} +``` + +An upcoming version of Nomad will include similar configuration options for the +`exec` and `java` task drivers. + +This change is limited to `docker`, `exec`, and `java` driver plugins. It does +not affect the Nomad server. This only affects Nomad clients running Linux, with +tasks using `bridge` networking and one of these task drivers, or third-party +plugins which relied on the shared Nomad executor library. + +Upgrading a Nomad client to 1.0.5 or 0.12.12 will not restart existing tasks. As +such, processes from existing `docker`, `exec`, or `java` tasks will need to be +manually restarted (using `alloc stop` or another mechanism) in order to be +fully isolated. + ## Nomad 1.0.3, 0.12.10 Nomad versions 1.0.3 and 0.12.10 change the behavior of the `exec` and `java` drivers so that @@ -1063,3 +1108,6 @@ deleted and then Nomad 0.3.0 can be launched. [`volume register`]: /docs/commands/volume/register [`volume`]: /docs/job-specification/volume [Enterprise licensing]: /docs/enterprise/license +[`CAP_NET_RAW`]: https://security.stackexchange.com/a/128988 +[`linux capabilities`]: https://docs.docker.com/engine/reference/run/#runtime-privilege-and-linux-capabilities +[`allow_caps`]: /docs/drivers/docker#allow_caps