CSI: set plugin CSI_ENDPOINT env var only if unset by user (#12257)

* Use unix:// prefix for CSI_ENDPOINT variable by default
* Some plugins have strict validation over the format of the
  `CSI_ENDPOINT` variable, and unfortunately not all plugins
  agree. Allow the user to override the `CSI_ENDPOINT` to workaround
  those cases.
* Update all demos and tests with CSI_ENDPOINT

client/allocrunner/taskrunner/plugin_supervisor_hook.go

@@ -170,19 +170,21 @@ func (h *csiPluginSupervisorHook) Prestart(ctx context.Context,
 
 	h.setSocketHook()
 
-	switch h.caps.FSIsolation {
-	case drivers.FSIsolationNone:
-		// Plugin tasks with no filesystem isolation won't have the
-		// plugin dir bind-mounted to their alloc dir, but we can
-		// provide them the path to the socket. These Nomad-only
-		// plugins will need to be aware of the csi directory layout
-		// in the client data dir
-		resp.Env = map[string]string{
-			"CSI_ENDPOINT": h.socketPath}
-	default:
-		resp.Env = map[string]string{
-			"CSI_ENDPOINT": filepath.Join(
-				h.task.CSIPluginConfig.MountDir, structs.CSISocketName)}
+	if _, ok := h.task.Env["CSI_ENDPOINT"]; !ok {
+		switch h.caps.FSIsolation {
+		case drivers.FSIsolationNone:
+			// Plugin tasks with no filesystem isolation won't have the
+			// plugin dir bind-mounted to their alloc dir, but we can
+			// provide them the path to the socket. These Nomad-only
+			// plugins will need to be aware of the csi directory layout
+			// in the client data dir
+			resp.Env = map[string]string{
+				"CSI_ENDPOINT": "unix://" + h.socketPath}
+		default:
+			resp.Env = map[string]string{
+				"CSI_ENDPOINT": "unix://" + filepath.Join(
+					h.task.CSIPluginConfig.MountDir, structs.CSISocketName)}
+		}
 	}
 
 	mounts := ensureMountpointInserted(h.runner.hookResources.getMounts(), configMount)

demo/csi/ceph-csi-plugin/README.md

@@ -19,7 +19,8 @@ Refer to the official plugin
 
 * `--type=rbd`: driver type `rbd` (or alternately `cephfs`)
 
-* `--endpoint=unix:///csi/csi.sock`: this option must match the `mount_dir`
+* `--endpoint=${CSI_ENDPOINT}`: if you don't use the `CSI_ENDPOINT`
+    environment variable, this option must match the `mount_dir`
     specified in the `csi_plugin` stanza for the task.
 
 * `--nodeid=${node.unique.id}`: a unique ID for the node the task is running

demo/csi/cinder-csi-plugin/README.md

@@ -3,7 +3,7 @@
 ## Requirements
 
 The containers that run the Node/Controller applications require a cloud-config file be mounted in the containers and the path specified in the containers `args`.
-  
+
 The example plugin job creates a file at `local/cloud.conf` using a [`template`](https://www.nomadproject.io/docs/job-specification/template) stanza which pulls the necessary credentials from a [Vault kv-v2](https://www.vaultproject.io/docs/secrets/kv/kv-v2) secrets store. However, other methods, such as using the [`artifact`](https://www.nomadproject.io/docs/job-specification/artifact) stanza, will work as well for delivering the `cloud.conf` file to the CSI drivers.
 
 ### Example cloud.conf
@@ -24,21 +24,18 @@ The Cinder CSI Node task requires that [`privileged = true`](https://www.nomadpr
 
 ## Container Arguments
 
-- `--endpoint=unix:///csi/csi.sock`
+* `--endpoint=${CSI_ENDPOINT}`: If you don't use the `CSI_ENDPOINT`
+    environment variable, this option must match the `mount_dir`
+    specified in the `csi_plugin` stanza for the task.
 
-  - This option must match the `mount_dir` specified in the `csi_plugin` stanza for the task.
+* `--cloud-config=/etc/config/cloud.conf`: The location that the
+  cloud.conf file was mounted inside the container
 
-- `--cloud-config=/etc/config/cloud.conf`
-  
-  - The location that the cloud.conf file was mounted inside the container
-  
-- `--nodeid=${node.unique.name}`
+* `--nodeid=${node.unique.name}`: A unique ID for the node the task is
+  running on. Recommend using `${node.unique.name}`
 
-  - A unique ID for the node the task is running on. Recommend using `${node.unique.name}`
-
-- `--cluster=${NOMAD_DC}`
-
-  - The cluster the Controller/Node is a part of. Recommend using `${NOMAD_DC}`
+* `--cluster=${NOMAD_DC}`: The cluster the Controller/Node is a part
+  of. Recommend using `${NOMAD_DC}`
 
 ## Deployment
 

demo/csi/cinder-csi-plugin/cinder-csi-plugin.hcl

@@ -34,7 +34,7 @@ EOF
         args = [
           "/bin/cinder-csi-plugin",
           "-v=4",
-          "--endpoint=unix:///csi/csi.sock",
+          "--endpoint=${CSI_ENDPOINT}",
           "--cloud-config=/etc/config/cloud.conf",
           "--nodeid=${node.unique.name}",
         ]
@@ -73,7 +73,7 @@ EOF
         args = [
           "/bin/cinder-csi-plugin",
           "-v=4",
-          "--endpoint=unix:///csi/csi.sock",
+          "--endpoint=${CSI_ENDPOINT}",
           "--cloud-config=/etc/config/cloud.conf",
           "--nodeid=${node.unique.name}",
           "--cluster=${NOMAD_DC}"
@@ -88,4 +88,3 @@ EOF
     }
   }
 }
-

demo/csi/digitalocean/plugin.nomad

@@ -10,7 +10,7 @@ job "digitalocean" {
       config {
         image = "digitalocean/do-csi-plugin:v2.1.1"
         args = [
-          "--endpoint=unix://csi/csi.sock",
+          "--endpoint=${CSI_ENDPOINT}",
           "--token=${token}",
           "--url=https://api.digitalocean.com/",
         ]

demo/csi/hostpath/plugin.nomad

@@ -13,7 +13,7 @@ job "csi-plugin" {
         args = [
           "--drivername=csi-hostpath",
           "--v=5",
-          "--endpoint=unix://csi/csi.sock",
+          "--endpoint=${CSI_ENDPOINT}",
           "--nodeid=node-${NOMAD_ALLOC_INDEX}",
         ]
 

e2e/csi/input/plugin-aws-ebs-controller.nomad

@@ -26,7 +26,7 @@ job "plugin-aws-ebs-controller" {
 
         args = [
           "controller",
-          "--endpoint=unix://csi/csi.sock",
+          "--endpoint=${CSI_ENDPOINT}",
           "--logtostderr",
           "--v=5",
         ]

e2e/csi/input/plugin-aws-ebs-nodes.nomad

@@ -23,7 +23,7 @@ job "plugin-aws-ebs-nodes" {
 
         args = [
           "node",
-          "--endpoint=unix://csi/csi.sock",
+          "--endpoint=${CSI_ENDPOINT}",
           "--logtostderr",
           "--v=5",
         ]

e2e/csi/input/plugin-aws-efs-nodes.nomad

@@ -19,13 +19,10 @@ job "plugin-aws-efs-nodes" {
       driver = "docker"
 
       config {
-        image = "amazon/aws-efs-csi-driver:v1.2.0"
-
-        # note: the EFS driver doesn't seem to respect the --endpoint
-        # flag and always sets up the listener at '/tmp/csi.sock'
+        image = "amazon/aws-efs-csi-driver:v1.3.6"
         args = [
           "node",
-          "--endpoint=unix://tmp/csi.sock",
+          "--endpoint=${CSI_ENDPOINT}",
           "--logtostderr",
           "--v=5",
         ]
@@ -33,6 +30,9 @@ job "plugin-aws-efs-nodes" {
         privileged = true
       }
 
+      # note: the EFS driver doesn't seem to respect the --endpoint
+      # flag or CSI_ENDPOINT env var and always sets up the listener
+      # at '/tmp/csi.sock'
       csi_plugin {
         id        = "aws-efs0"
         type      = "node"