diff --git a/api/tasks.go b/api/tasks.go index 9c017887a..578188cc2 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -370,12 +370,28 @@ type VolumeRequest struct { ReadOnly bool `mapstructure:"read_only"` } +const ( + VolumeMountPropagationPrivate = "private" + VolumeMountPropagationHostToTask = "host-to-task" + VolumeMountPropagationBidirectional = "bidirectional" +) + // VolumeMount represents the relationship between a destination path in a task // and the task group volume that should be mounted there. type VolumeMount struct { - Volume string - Destination string - ReadOnly bool `mapstructure:"read_only"` + Volume *string + Destination *string + ReadOnly *bool `mapstructure:"read_only"` + PropagationMode *string `mapstructure:"propagation_mode"` +} + +func (vm *VolumeMount) Canonicalize() { + if vm.PropagationMode == nil { + vm.PropagationMode = stringToPtr(VolumeMountPropagationPrivate) + } + if vm.ReadOnly == nil { + vm.ReadOnly = boolToPtr(false) + } } // TaskGroup is the unit of scheduling. @@ -632,6 +648,9 @@ func (t *Task) Canonicalize(tg *TaskGroup, job *Job) { for _, a := range t.Affinities { a.Canonicalize() } + for _, vm := range t.VolumeMounts { + vm.Canonicalize() + } } // TaskArtifact is used to download artifacts before running a task. diff --git a/api/tasks_test.go b/api/tasks_test.go index 29d9cf691..f83a91e24 100644 --- a/api/tasks_test.go +++ b/api/tasks_test.go @@ -368,6 +368,14 @@ func TestTask_Artifact(t *testing.T) { } } +func TestTask_VolumeMount(t *testing.T) { + t.Parallel() + vm := &VolumeMount{} + vm.Canonicalize() + require.NotNil(t, vm.PropagationMode) + require.Equal(t, *vm.PropagationMode, "private") +} + // Ensures no regression on https://github.com/hashicorp/nomad/issues/3132 func TestTaskGroup_Canonicalize_Update(t *testing.T) { // Job with an Empty() Update diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 7e7f206b9..54cac029f 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -812,9 +812,10 @@ func ApiTaskToStructsTask(apiTask *api.Task, structsTask *structs.Task) { structsTask.VolumeMounts = make([]*structs.VolumeMount, l) for i, mount := range apiTask.VolumeMounts { structsTask.VolumeMounts[i] = &structs.VolumeMount{ - Volume: mount.Volume, - Destination: mount.Destination, - ReadOnly: mount.ReadOnly, + Volume: *mount.Volume, + Destination: *mount.Destination, + ReadOnly: *mount.ReadOnly, + PropagationMode: *mount.PropagationMode, } } } diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go index 5f5784a7e..67b578bc3 100644 --- a/drivers/docker/driver.go +++ b/drivers/docker/driver.go @@ -642,6 +642,15 @@ func (d *Driver) containerBinds(task *drivers.TaskConfig, driverConfig *TaskConf return binds, nil } +var userMountToUnixMount = map[string]string{ + // Empty string maps to `rprivate` for backwards compatibility in restored + // older tasks, where mount propagation will not be present. + "": "rprivate", + nstructs.VolumeMountPropagationPrivate: "rprivate", + nstructs.VolumeMountPropagationHostToTask: "rslave", + nstructs.VolumeMountPropagationBidirectional: "rshared", +} + func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *TaskConfig, imageID string) (docker.CreateContainerOptions, error) { @@ -833,13 +842,24 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T hostConfig.Mounts = append(hostConfig.Mounts, hm) } + for _, m := range task.Mounts { - hostConfig.Mounts = append(hostConfig.Mounts, docker.HostMount{ + hm := docker.HostMount{ Type: "bind", Target: m.TaskPath, Source: m.HostPath, ReadOnly: m.Readonly, - }) + } + + // MountPropagation is only supported by Docker on Linux: + // https://docs.docker.com/storage/bind-mounts/#configure-bind-propagation + if runtime.GOOS == "linux" { + hm.BindOptions = &docker.BindOptions{ + Propagation: userMountToUnixMount[m.PropagationMode], + } + } + + hostConfig.Mounts = append(hostConfig.Mounts, hm) } // set DNS search domains and extra hosts diff --git a/drivers/docker/driver_unix_test.go b/drivers/docker/driver_unix_test.go index 9faa95326..6f312a1ec 100644 --- a/drivers/docker/driver_unix_test.go +++ b/drivers/docker/driver_unix_test.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "runtime" "sort" "strconv" "strings" @@ -598,22 +599,32 @@ func TestDockerDriver_CreateContainerConfig_MountsCombined(t *testing.T) { c, err := driver.createContainerConfig(task, cfg, "org/repo:0.1") require.NoError(t, err) - expectedMounts := []docker.HostMount{ { - Type: "bind", - Source: "/tmp/cfg-mount", - Target: "/container/tmp/cfg-mount", - ReadOnly: false, - BindOptions: &docker.BindOptions{}, + Type: "bind", + Source: "/tmp/cfg-mount", + Target: "/container/tmp/cfg-mount", + ReadOnly: false, + BindOptions: &docker.BindOptions{ + Propagation: "", + }, }, { Type: "bind", Source: "/tmp/task-mount", Target: "/container/tmp/task-mount", ReadOnly: true, + BindOptions: &docker.BindOptions{ + Propagation: "rprivate", + }, }, } + + if runtime.GOOS != "linux" { + expectedMounts[0].BindOptions = &docker.BindOptions{} + expectedMounts[1].BindOptions = &docker.BindOptions{} + } + foundMounts := c.HostConfig.Mounts sort.Slice(foundMounts, func(i, j int) bool { return foundMounts[i].Target < foundMounts[j].Target diff --git a/drivers/shared/executor/executor_linux.go b/drivers/shared/executor/executor_linux.go index df5c5a6b3..f81090a56 100644 --- a/drivers/shared/executor/executor_linux.go +++ b/drivers/shared/executor/executor_linux.go @@ -813,6 +813,15 @@ func cmdDevices(devices []*drivers.DeviceConfig) ([]*lconfigs.Device, error) { return r, nil } +var userMountToUnixMount = map[string]int{ + // Empty string maps to `rprivate` for backwards compatibility in restored + // older tasks, where mount propagation will not be present. + "": unix.MS_PRIVATE | unix.MS_REC, // rprivate + structs.VolumeMountPropagationPrivate: unix.MS_PRIVATE | unix.MS_REC, // rprivate + structs.VolumeMountPropagationHostToTask: unix.MS_SLAVE | unix.MS_REC, // rslave + structs.VolumeMountPropagationBidirectional: unix.MS_SHARED | unix.MS_REC, // rshared +} + // cmdMounts converts a list of driver.MountConfigs into excutor.Mounts. func cmdMounts(mounts []*drivers.MountConfig) []*lconfigs.Mount { if len(mounts) == 0 { @@ -826,11 +835,13 @@ func cmdMounts(mounts []*drivers.MountConfig) []*lconfigs.Mount { if m.Readonly { flags |= unix.MS_RDONLY } + r[i] = &lconfigs.Mount{ - Source: m.HostPath, - Destination: m.TaskPath, - Device: "bind", - Flags: flags, + Source: m.HostPath, + Destination: m.TaskPath, + Device: "bind", + Flags: flags, + PropagationFlags: []int{userMountToUnixMount[m.PropagationMode]}, } } diff --git a/drivers/shared/executor/executor_linux_test.go b/drivers/shared/executor/executor_linux_test.go index c8e07370b..1861ea5ae 100644 --- a/drivers/shared/executor/executor_linux_test.go +++ b/drivers/shared/executor/executor_linux_test.go @@ -467,16 +467,18 @@ func TestExecutor_cmdMounts(t *testing.T) { expected := []*lconfigs.Mount{ { - Source: "/host/path-ro", - Destination: "/task/path-ro", - Flags: unix.MS_BIND | unix.MS_RDONLY, - Device: "bind", + Source: "/host/path-ro", + Destination: "/task/path-ro", + Flags: unix.MS_BIND | unix.MS_RDONLY, + Device: "bind", + PropagationFlags: []int{unix.MS_PRIVATE | unix.MS_REC}, }, { - Source: "/host/path-rw", - Destination: "/task/path-rw", - Flags: unix.MS_BIND, - Device: "bind", + Source: "/host/path-rw", + Destination: "/task/path-rw", + Flags: unix.MS_BIND, + Device: "bind", + PropagationFlags: []int{unix.MS_PRIVATE | unix.MS_REC}, }, } diff --git a/jobspec/parse_task.go b/jobspec/parse_task.go index 7edccb430..a045ce06c 100644 --- a/jobspec/parse_task.go +++ b/jobspec/parse_task.go @@ -522,6 +522,7 @@ func parseVolumeMounts(out *[]*api.VolumeMount, list *ast.ObjectList) error { "volume", "read_only", "destination", + "propagation_mode", } if err := helper.CheckHCLKeys(item.Val, valid); err != nil { return err diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index cd3857388..ef2236f9a 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -197,8 +197,8 @@ func TestParse(t *testing.T) { }, VolumeMounts: []*api.VolumeMount{ { - Volume: "foo", - Destination: "/mnt/foo", + Volume: helper.StringToPtr("foo"), + Destination: helper.StringToPtr("/mnt/foo"), }, }, Affinities: []*api.Affinity{ diff --git a/nomad/job_endpoint.go b/nomad/job_endpoint.go index 6c7c3ad28..a89d94e34 100644 --- a/nomad/job_endpoint.go +++ b/nomad/job_endpoint.go @@ -110,6 +110,7 @@ func (j *Job) Register(args *structs.JobRegisterRequest, reply *structs.JobRegis if !aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilitySubmitJob) { return structs.ErrPermissionDenied } + // Validate Volume Permsissions for _, tg := range args.Job.TaskGroups { for _, vol := range tg.Volumes { @@ -131,6 +132,16 @@ func (j *Job) Register(args *structs.JobRegisterRequest, reply *structs.JobRegis } } } + + for _, t := range tg.Tasks { + for _, vm := range t.VolumeMounts { + vol := tg.Volumes[vm.Volume] + if vm.PropagationMode == structs.VolumeMountPropagationBidirectional && + !aclObj.AllowHostVolumeOperation(vol.Source, acl.HostVolumeCapabilityMountReadWrite) { + return structs.ErrPermissionDenied + } + } + } } // Check if override is set and we do not have permissions diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 1f3104b7d..78250d999 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -5525,6 +5525,14 @@ func (t *Task) Validate(ephemeralDisk *EphemeralDisk, jobType string, tgServices mErr.Errors = append(mErr.Errors, serviceErr) } } + + // Validation for volumes + for idx, vm := range t.VolumeMounts { + if !MountPropagationModeIsValid(vm.PropagationMode) { + mErr.Errors = append(mErr.Errors, fmt.Errorf("Volume Mount (%d) has an invalid propagation mode: \"%s\"", idx, vm.PropagationMode)) + } + } + return mErr.ErrorOrNil() } diff --git a/nomad/structs/volumes.go b/nomad/structs/volumes.go index 8d750fdc3..fe44e4830 100644 --- a/nomad/structs/volumes.go +++ b/nomad/structs/volumes.go @@ -4,6 +4,21 @@ const ( VolumeTypeHost = "host" ) +const ( + VolumeMountPropagationPrivate = "private" + VolumeMountPropagationHostToTask = "host-to-task" + VolumeMountPropagationBidirectional = "bidirectional" +) + +func MountPropagationModeIsValid(propagationMode string) bool { + switch propagationMode { + case "", VolumeMountPropagationPrivate, VolumeMountPropagationHostToTask, VolumeMountPropagationBidirectional: + return true + default: + return false + } +} + // ClientHostVolumeConfig is used to configure access to host paths on a Nomad Client type ClientHostVolumeConfig struct { Name string `hcl:",key"` @@ -103,9 +118,10 @@ func CopyMapVolumeRequest(s map[string]*VolumeRequest) map[string]*VolumeRequest // VolumeMount represents the relationship between a destination path in a task // and the task group volume that should be mounted there. type VolumeMount struct { - Volume string - Destination string - ReadOnly bool + Volume string + Destination string + ReadOnly bool + PropagationMode string } func (v *VolumeMount) Copy() *VolumeMount { diff --git a/plugins/drivers/driver.go b/plugins/drivers/driver.go index 4c127393d..a4259a6f8 100644 --- a/plugins/drivers/driver.go +++ b/plugins/drivers/driver.go @@ -357,15 +357,17 @@ func (d *DeviceConfig) Copy() *DeviceConfig { } type MountConfig struct { - TaskPath string - HostPath string - Readonly bool + TaskPath string + HostPath string + Readonly bool + PropagationMode string } func (m *MountConfig) IsEqual(o *MountConfig) bool { return m.TaskPath == o.TaskPath && m.HostPath == o.HostPath && - m.Readonly == o.Readonly + m.Readonly == o.Readonly && + m.PropagationMode == o.PropagationMode } func (m *MountConfig) Copy() *MountConfig {