Merge pull request #3687 from mikemccracken/add-lxc-bindmounts

Add volumes config to LXC driver

client/driver/lxc.go

@@ -31,6 +31,11 @@ const (
 	// Config.Options map.
 	lxcConfigOption = "driver.lxc.enable"
 
+	// lxcVolumesConfigOption is the key for enabling the use of
+	// custom bind volumes to arbitrary host paths
+	lxcVolumesConfigOption  = "lxc.volumes.enabled"
+	lxcVolumesConfigDefault = true
+
 	// containerMonitorIntv is the interval at which the driver checks if the
 	// container is still alive
 	containerMonitorIntv = 2 * time.Second
@@ -69,6 +74,7 @@ type LxcDriverConfig struct {
 	TemplateArgs         []string `mapstructure:"template_args"`
 	LogLevel             string   `mapstructure:"log_level"`
 	Verbosity            string
+	Volumes              []string `mapstructure:"volumes"`
 }
 
 // NewLxcDriver returns a new instance of the LXC driver
@@ -137,6 +143,10 @@ func (d *LxcDriver) Validate(config map[string]interface{}) error {
 				Type:     fields.TypeString,
 				Required: false,
 			},
+			"volumes": {
+				Type:     fields.TypeArray,
+				Required: false,
+			},
 		},
 	}
 
@@ -144,6 +154,21 @@ func (d *LxcDriver) Validate(config map[string]interface{}) error {
 		return err
 	}
 
+	volumes, _ := fd.GetOk("volumes")
+	for _, volDesc := range volumes.([]interface{}) {
+		volStr := volDesc.(string)
+		paths := strings.Split(volStr, ":")
+		if len(paths) != 2 {
+			return fmt.Errorf("invalid volume bind mount entry: '%s'", volStr)
+		}
+		if len(paths[0]) == 0 || len(paths[1]) == 0 {
+			return fmt.Errorf("invalid volume bind mount entry: '%s'", volStr)
+		}
+		if paths[1][0] == '/' {
+			return fmt.Errorf("unsupported absolute container mount point: '%s'", paths[1])
+		}
+	}
+
 	return nil
 }
 
@@ -170,6 +195,12 @@ func (d *LxcDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, e
 	}
 	node.Attributes["driver.lxc.version"] = version
 	node.Attributes["driver.lxc"] = "1"
+
+	// Advertise if this node supports lxc volumes
+	if d.config.ReadBoolDefault(lxcVolumesConfigOption, lxcVolumesConfigDefault) {
+		node.Attributes["driver."+lxcVolumesConfigOption] = "1"
+	}
+
 	return true, nil
 }
 
@@ -250,6 +281,25 @@ func (d *LxcDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse,
 		fmt.Sprintf("%s alloc none rw,bind,create=dir", ctx.TaskDir.SharedAllocDir),
 		fmt.Sprintf("%s secrets none rw,bind,create=dir", ctx.TaskDir.SecretsDir),
 	}
+
+	volumesEnabled := d.config.ReadBoolDefault(lxcVolumesConfigOption, lxcVolumesConfigDefault)
+
+	for _, volDesc := range driverConfig.Volumes {
+		// the format was checked in Validate()
+		paths := strings.Split(volDesc, ":")
+
+		if filepath.IsAbs(paths[0]) {
+			if !volumesEnabled {
+				return nil, fmt.Errorf("absolute bind-mount volume in config but '%v' is false", lxcVolumesConfigOption)
+			}
+		} else {
+			// Relative source paths are treated as relative to alloc dir
+			paths[0] = filepath.Join(ctx.TaskDir.Dir, paths[0])
+		}
+
+		mounts = append(mounts, fmt.Sprintf("%s %s none rw,bind,create=dir", paths[0], paths[1]))
+	}
+
 	for _, mnt := range mounts {
 		if err := c.SetConfigItem("lxc.mount.entry", mnt); err != nil {
 			return nil, fmt.Errorf("error setting bind mount %q error: %v", mnt, err)

client/driver/lxc_test.go

@@ -3,8 +3,11 @@
 package driver
 
 import (
+	"bytes"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"testing"
 	"time"
@@ -69,11 +72,25 @@ func TestLxcDriver_Start_Wait(t *testing.T) {
 		Driver: "lxc",
 		Config: map[string]interface{}{
 			"template": "/usr/share/lxc/templates/lxc-busybox",
+			"volumes":  []string{"/tmp/:mnt/tmp"},
 		},
 		KillTimeout: 10 * time.Second,
 		Resources:   structs.DefaultResources(),
 	}
 
+	testFileContents := []byte("this should be visible under /mnt/tmp")
+	tmpFile, err := ioutil.TempFile("/tmp", "testlxcdriver_start_wait")
+	if err != nil {
+		t.Fatalf("error writing temp file: %v", err)
+	}
+	defer os.Remove(tmpFile.Name())
+	if _, err := tmpFile.Write(testFileContents); err != nil {
+		t.Fatalf("error writing temp file: %v", err)
+	}
+	if err := tmpFile.Close(); err != nil {
+		t.Fatalf("error closing temp file: %v", err)
+	}
+
 	ctx := testDriverContexts(t, task)
 	defer ctx.AllocDir.Destroy()
 	d := NewLxcDriver(ctx.DriverCtx)
@@ -106,7 +123,7 @@ func TestLxcDriver_Start_Wait(t *testing.T) {
 
 	// Look for mounted directories in their proper location
 	containerName := fmt.Sprintf("%s-%s", task.Name, ctx.DriverCtx.allocID)
-	for _, mnt := range []string{"alloc", "local", "secrets"} {
+	for _, mnt := range []string{"alloc", "local", "secrets", "mnt/tmp"} {
 		fullpath := filepath.Join(lxcHandle.lxcPath, containerName, "rootfs", mnt)
 		stat, err := os.Stat(fullpath)
 		if err != nil {
@@ -117,6 +134,16 @@ func TestLxcDriver_Start_Wait(t *testing.T) {
 		}
 	}
 
+	// Test that /mnt/tmp/$tempFile exists in the container:
+	mountedContents, err := exec.Command("lxc-attach", "-n", containerName, "--", "cat", filepath.Join("/mnt/", tmpFile.Name())).Output()
+	if err != nil {
+		t.Fatalf("err reading temp file in bind mount: %v", err)
+	}
+
+	if !bytes.Equal(mountedContents, testFileContents) {
+		t.Fatalf("contents of temp bind mounted file did not match, was '%s'", mountedContents)
+	}
+
 	// Desroy the container
 	if err := sresp.Handle.Kill(); err != nil {
 		t.Fatalf("err: %v", err)
@@ -200,3 +227,98 @@ func TestLxcDriver_Open_Wait(t *testing.T) {
 func lxcPresent(t *testing.T) bool {
 	return lxc.Version() != ""
 }
+
+func TestLxcDriver_Volumes_ConfigValidation(t *testing.T) {
+	if !testutil.IsTravis() {
+		t.Parallel()
+	}
+	if !lxcPresent(t) {
+		t.Skip("lxc not present")
+	}
+	ctestutil.RequireRoot(t)
+
+	brokenVolumeConfigs := [][]string{
+		{
+			"foo:/var",
+		},
+		{
+			":",
+		},
+		{
+			"abc:",
+		},
+		{
+			":def",
+		},
+		{
+			"abc:def:ghi",
+		},
+	}
+
+	for _, bc := range brokenVolumeConfigs {
+		if err := testVolumeConfig(t, bc); err == nil {
+			t.Fatalf("error expected in validate for config %+v", bc)
+		}
+	}
+	if err := testVolumeConfig(t, []string{"abc:def"}); err != nil {
+		t.Fatalf("error in validate for syntactically valid config abc:def")
+	}
+}
+
+func testVolumeConfig(t *testing.T, volConfig []string) error {
+	task := &structs.Task{
+		Name:        "voltest",
+		Driver:      "lxc",
+		KillTimeout: 10 * time.Second,
+		Resources:   structs.DefaultResources(),
+		Config: map[string]interface{}{
+			"template": "busybox",
+		},
+	}
+	task.Config["volumes"] = volConfig
+
+	ctx := testDriverContexts(t, task)
+	defer ctx.AllocDir.Destroy()
+
+	driver := NewLxcDriver(ctx.DriverCtx)
+
+	err := driver.Validate(task.Config)
+	return err
+
+}
+
+func TestLxcDriver_Start_NoVolumes(t *testing.T) {
+	if !testutil.IsTravis() {
+		t.Parallel()
+	}
+	if !lxcPresent(t) {
+		t.Skip("lxc not present")
+	}
+	ctestutil.RequireRoot(t)
+
+	task := &structs.Task{
+		Name:   "foo",
+		Driver: "lxc",
+		Config: map[string]interface{}{
+			"template": "/usr/share/lxc/templates/lxc-busybox",
+			"volumes":  []string{"/tmp/:mnt/tmp"},
+		},
+		KillTimeout: 10 * time.Second,
+		Resources:   structs.DefaultResources(),
+	}
+
+	ctx := testDriverContexts(t, task)
+	defer ctx.AllocDir.Destroy()
+
+	ctx.DriverCtx.config.Options = map[string]string{lxcVolumesConfigOption: "false"}
+
+	d := NewLxcDriver(ctx.DriverCtx)
+
+	if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
+		t.Fatalf("prestart err: %v", err)
+	}
+	_, err := d.Start(ctx.ExecCtx, task)
+	if err == nil {
+		t.Fatalf("expected error in start, got nil.")
+	}
+}

website/source/docs/drivers/lxc.html.md

@@ -64,6 +64,31 @@ The `lxc` driver supports the following configuration in the job spec:
     }
     ```
 
+* `volumes` - (Optional) A list of `host_path:container_path` strings to bind-mount
+  host paths to container paths. Mounting host paths outside of the allocation
+  directory can be disabled on clients by setting the `lxc.volumes.enabled`
+  option set to false. This will limit volumes to directories that exist inside
+  the allocation directory.
+
+  Note that unlike the similar option for the docker driver, this
+  option must not have an absolute path as the `container_path`
+  component. This will cause an error when submitting a job.
+
+  Setting this does not affect the standard bind-mounts of `alloc`,
+  `local`, and `secrets`, which are always created.
+
+    ```hcl
+    config {
+      volumes = [
+        # Use absolute paths to mount arbitrary paths on the host
+        "/path/on/host:path/in/container",
+
+        # Use relative paths to rebind paths already in the allocation dir
+        "relative/to/task:also/in/container"
+      ]
+    }
+    ```
+
 ## Networking
 
 Currently the `lxc` driver only supports host networking. See the `none`