diff --git a/client/allocrunner/taskrunner/plugin_supervisor_hook.go b/client/allocrunner/taskrunner/plugin_supervisor_hook.go index 122386fd7..415b6e20b 100644 --- a/client/allocrunner/taskrunner/plugin_supervisor_hook.go +++ b/client/allocrunner/taskrunner/plugin_supervisor_hook.go @@ -170,19 +170,21 @@ func (h *csiPluginSupervisorHook) Prestart(ctx context.Context, h.setSocketHook() - switch h.caps.FSIsolation { - case drivers.FSIsolationNone: - // Plugin tasks with no filesystem isolation won't have the - // plugin dir bind-mounted to their alloc dir, but we can - // provide them the path to the socket. These Nomad-only - // plugins will need to be aware of the csi directory layout - // in the client data dir - resp.Env = map[string]string{ - "CSI_ENDPOINT": h.socketPath} - default: - resp.Env = map[string]string{ - "CSI_ENDPOINT": filepath.Join( - h.task.CSIPluginConfig.MountDir, structs.CSISocketName)} + if _, ok := h.task.Env["CSI_ENDPOINT"]; !ok { + switch h.caps.FSIsolation { + case drivers.FSIsolationNone: + // Plugin tasks with no filesystem isolation won't have the + // plugin dir bind-mounted to their alloc dir, but we can + // provide them the path to the socket. These Nomad-only + // plugins will need to be aware of the csi directory layout + // in the client data dir + resp.Env = map[string]string{ + "CSI_ENDPOINT": "unix://" + h.socketPath} + default: + resp.Env = map[string]string{ + "CSI_ENDPOINT": "unix://" + filepath.Join( + h.task.CSIPluginConfig.MountDir, structs.CSISocketName)} + } } mounts := ensureMountpointInserted(h.runner.hookResources.getMounts(), configMount) diff --git a/demo/csi/ceph-csi-plugin/README.md b/demo/csi/ceph-csi-plugin/README.md index 0eaa1aa63..f00b30ed6 100644 --- a/demo/csi/ceph-csi-plugin/README.md +++ b/demo/csi/ceph-csi-plugin/README.md @@ -19,7 +19,8 @@ Refer to the official plugin * `--type=rbd`: driver type `rbd` (or alternately `cephfs`) -* `--endpoint=unix:///csi/csi.sock`: this option must match the `mount_dir` +* `--endpoint=${CSI_ENDPOINT}`: if you don't use the `CSI_ENDPOINT` + environment variable, this option must match the `mount_dir` specified in the `csi_plugin` stanza for the task. * `--nodeid=${node.unique.id}`: a unique ID for the node the task is running diff --git a/demo/csi/cinder-csi-plugin/README.md b/demo/csi/cinder-csi-plugin/README.md index a6c133441..43b1652d0 100644 --- a/demo/csi/cinder-csi-plugin/README.md +++ b/demo/csi/cinder-csi-plugin/README.md @@ -3,7 +3,7 @@ ## Requirements The containers that run the Node/Controller applications require a cloud-config file be mounted in the containers and the path specified in the containers `args`. - + The example plugin job creates a file at `local/cloud.conf` using a [`template`](https://www.nomadproject.io/docs/job-specification/template) stanza which pulls the necessary credentials from a [Vault kv-v2](https://www.vaultproject.io/docs/secrets/kv/kv-v2) secrets store. However, other methods, such as using the [`artifact`](https://www.nomadproject.io/docs/job-specification/artifact) stanza, will work as well for delivering the `cloud.conf` file to the CSI drivers. ### Example cloud.conf @@ -24,21 +24,18 @@ The Cinder CSI Node task requires that [`privileged = true`](https://www.nomadpr ## Container Arguments -- `--endpoint=unix:///csi/csi.sock` +* `--endpoint=${CSI_ENDPOINT}`: If you don't use the `CSI_ENDPOINT` + environment variable, this option must match the `mount_dir` + specified in the `csi_plugin` stanza for the task. - - This option must match the `mount_dir` specified in the `csi_plugin` stanza for the task. +* `--cloud-config=/etc/config/cloud.conf`: The location that the + cloud.conf file was mounted inside the container -- `--cloud-config=/etc/config/cloud.conf` - - - The location that the cloud.conf file was mounted inside the container - -- `--nodeid=${node.unique.name}` +* `--nodeid=${node.unique.name}`: A unique ID for the node the task is + running on. Recommend using `${node.unique.name}` - - A unique ID for the node the task is running on. Recommend using `${node.unique.name}` - -- `--cluster=${NOMAD_DC}` - - - The cluster the Controller/Node is a part of. Recommend using `${NOMAD_DC}` +* `--cluster=${NOMAD_DC}`: The cluster the Controller/Node is a part + of. Recommend using `${NOMAD_DC}` ## Deployment diff --git a/demo/csi/cinder-csi-plugin/cinder-csi-plugin.hcl b/demo/csi/cinder-csi-plugin/cinder-csi-plugin.hcl index 92235bf4b..e35b95334 100644 --- a/demo/csi/cinder-csi-plugin/cinder-csi-plugin.hcl +++ b/demo/csi/cinder-csi-plugin/cinder-csi-plugin.hcl @@ -34,7 +34,7 @@ EOF args = [ "/bin/cinder-csi-plugin", "-v=4", - "--endpoint=unix:///csi/csi.sock", + "--endpoint=${CSI_ENDPOINT}", "--cloud-config=/etc/config/cloud.conf", "--nodeid=${node.unique.name}", ] @@ -73,7 +73,7 @@ EOF args = [ "/bin/cinder-csi-plugin", "-v=4", - "--endpoint=unix:///csi/csi.sock", + "--endpoint=${CSI_ENDPOINT}", "--cloud-config=/etc/config/cloud.conf", "--nodeid=${node.unique.name}", "--cluster=${NOMAD_DC}" @@ -88,4 +88,3 @@ EOF } } } - diff --git a/demo/csi/digitalocean/plugin.nomad b/demo/csi/digitalocean/plugin.nomad index 998142948..321234e16 100644 --- a/demo/csi/digitalocean/plugin.nomad +++ b/demo/csi/digitalocean/plugin.nomad @@ -10,7 +10,7 @@ job "digitalocean" { config { image = "digitalocean/do-csi-plugin:v2.1.1" args = [ - "--endpoint=unix://csi/csi.sock", + "--endpoint=${CSI_ENDPOINT}", "--token=${token}", "--url=https://api.digitalocean.com/", ] diff --git a/demo/csi/hostpath/plugin.nomad b/demo/csi/hostpath/plugin.nomad index a1b43f326..60637d10a 100644 --- a/demo/csi/hostpath/plugin.nomad +++ b/demo/csi/hostpath/plugin.nomad @@ -13,7 +13,7 @@ job "csi-plugin" { args = [ "--drivername=csi-hostpath", "--v=5", - "--endpoint=unix://csi/csi.sock", + "--endpoint=${CSI_ENDPOINT}", "--nodeid=node-${NOMAD_ALLOC_INDEX}", ] diff --git a/e2e/csi/input/plugin-aws-ebs-controller.nomad b/e2e/csi/input/plugin-aws-ebs-controller.nomad index dd0b675c7..2f022c6e0 100644 --- a/e2e/csi/input/plugin-aws-ebs-controller.nomad +++ b/e2e/csi/input/plugin-aws-ebs-controller.nomad @@ -26,7 +26,7 @@ job "plugin-aws-ebs-controller" { args = [ "controller", - "--endpoint=unix://csi/csi.sock", + "--endpoint=${CSI_ENDPOINT}", "--logtostderr", "--v=5", ] diff --git a/e2e/csi/input/plugin-aws-ebs-nodes.nomad b/e2e/csi/input/plugin-aws-ebs-nodes.nomad index 206b1df81..8bb9691f6 100644 --- a/e2e/csi/input/plugin-aws-ebs-nodes.nomad +++ b/e2e/csi/input/plugin-aws-ebs-nodes.nomad @@ -23,7 +23,7 @@ job "plugin-aws-ebs-nodes" { args = [ "node", - "--endpoint=unix://csi/csi.sock", + "--endpoint=${CSI_ENDPOINT}", "--logtostderr", "--v=5", ] diff --git a/e2e/csi/input/plugin-aws-efs-nodes.nomad b/e2e/csi/input/plugin-aws-efs-nodes.nomad index 3cf5358e9..67601f78a 100644 --- a/e2e/csi/input/plugin-aws-efs-nodes.nomad +++ b/e2e/csi/input/plugin-aws-efs-nodes.nomad @@ -19,13 +19,10 @@ job "plugin-aws-efs-nodes" { driver = "docker" config { - image = "amazon/aws-efs-csi-driver:v1.2.0" - - # note: the EFS driver doesn't seem to respect the --endpoint - # flag and always sets up the listener at '/tmp/csi.sock' + image = "amazon/aws-efs-csi-driver:v1.3.6" args = [ "node", - "--endpoint=unix://tmp/csi.sock", + "--endpoint=${CSI_ENDPOINT}", "--logtostderr", "--v=5", ] @@ -33,6 +30,9 @@ job "plugin-aws-efs-nodes" { privileged = true } + # note: the EFS driver doesn't seem to respect the --endpoint + # flag or CSI_ENDPOINT env var and always sets up the listener + # at '/tmp/csi.sock' csi_plugin { id = "aws-efs0" type = "node"