Add stage_publish_base_dir field to csi_plugin stanza of a job (#13919)

* Allow specification of CSI staging and publishing directory path
* Add website documentation for stage_publish_dir
* Replace erroneous reference to csi_plugin.mount_config with csi_plugin.mount_dir
* Avoid requiring CSI plugins to be redeployed after introducing StagePublishDir

.changelog/13919.txt

+```release-note:improvement
+csi: Add `stage_publish_base_dir` field to `csi_plugin` block to support plugins that require a specific staging/publishing directory for mounts
+```

api/tasks.go

@@ -1032,14 +1032,17 @@ type TaskCSIPluginConfig struct {
 	// CSIPluginType instructs Nomad on how to handle processing a plugin
 	Type CSIPluginType `mapstructure:"type" hcl:"type,optional"`
 
-	// MountDir is the destination that nomad should mount in its CSI
-	// directory for the plugin. It will then expect a file called CSISocketName
-	// to be created by the plugin, and will provide references into
-	// "MountDir/CSIIntermediaryDirname/VolumeName/AllocID for mounts.
-	//
-	// Default is /csi.
+	// MountDir is the directory (within its container) in which the plugin creates a
+	// socket (called CSISocketName) for communication with Nomad. Default is /csi.
 	MountDir string `mapstructure:"mount_dir" hcl:"mount_dir,optional"`
 
+	// StagePublishBaseDir is the base directory (within its container) in which the plugin
+	// mounts volumes being staged and bind mounts volumes being published.
+	// e.g. staging_target_path = {StagePublishBaseDir}/staging/{volume-id}/{usage-mode}
+	// e.g. target_path = {StagePublishBaseDir}/per-alloc/{alloc-id}/{volume-id}/{usage-mode}
+	// Default is /local/csi.
+	StagePublishBaseDir string `mapstructure:"stage_publish_base_dir" hcl:"stage_publish_base_dir,optional"`
+
 	// HealthTimeout is the time after which the CSI plugin tasks will be killed
 	// if the CSI Plugin is not healthy.
 	HealthTimeout time.Duration `mapstructure:"health_timeout" hcl:"health_timeout,optional"`
@@ -1050,6 +1053,10 @@ func (t *TaskCSIPluginConfig) Canonicalize() {
 		t.MountDir = "/csi"
 	}
 
+	if t.StagePublishBaseDir == "" {
+		t.StagePublishBaseDir = filepath.Join("/local", "csi")
+	}
+
 	if t.HealthTimeout == 0 {
 		t.HealthTimeout = 30 * time.Second
 	}

client/allocrunner/taskrunner/plugin_supervisor_hook.go

@@ -81,7 +81,7 @@ var _ interfaces.TaskStopHook = &csiPluginSupervisorHook{}
 //       Per-allocation directories of unix domain sockets used to communicate
 //       with the CSI plugin. Nomad creates the directory and the plugin creates
 //       the socket file. This directory is bind-mounted to the
-//       csi_plugin.mount_config dir in the plugin task.
+//       csi_plugin.mount_dir in the plugin task.
 //
 // {plugin-type}/{plugin-id}/
 //    staging/
@@ -103,6 +103,16 @@ func newCSIPluginSupervisorHook(config *csiPluginSupervisorHookConfig) *csiPlugi
 	socketMountPoint := filepath.Join(config.clientStateDirPath, "csi",
 		"plugins", config.runner.Alloc().ID)
 
+	// In v1.3.0, Nomad started instructing CSI plugins to stage and publish
+	// within /local/csi. Plugins deployed after the introduction of
+	// StagePublishBaseDir default to StagePublishBaseDir = /local/csi. However,
+	// plugins deployed between v1.3.0 and the introduction of
+	// StagePublishBaseDir have StagePublishBaseDir = "". Default to /local/csi here
+	// to avoid breaking plugins that aren't redeployed.
+	if task.CSIPluginConfig.StagePublishBaseDir == "" {
+		task.CSIPluginConfig.StagePublishBaseDir = filepath.Join("/local", "csi")
+	}
+
 	if task.CSIPluginConfig.HealthTimeout == 0 {
 		task.CSIPluginConfig.HealthTimeout = 30 * time.Second
 	}
@@ -157,8 +167,7 @@ func (h *csiPluginSupervisorHook) Prestart(ctx context.Context,
 	}
 	// where the staging and per-alloc directories will be mounted
 	volumeStagingMounts := &drivers.MountConfig{
-		// TODO(tgross): add this TaskPath to the CSIPluginConfig as well
-		TaskPath:        "/local/csi",
+		TaskPath:        h.task.CSIPluginConfig.StagePublishBaseDir,
 		HostPath:        h.mountPoint,
 		Readonly:        false,
 		PropagationMode: "bidirectional",
@@ -360,7 +369,7 @@ func (h *csiPluginSupervisorHook) registerPlugin(client csi.CSIPlugin, socketPat
 			Options: map[string]string{
 				"Provider":            info.Name, // vendor name
 				"MountPoint":          h.mountPoint,
-				"ContainerMountPoint": "/local/csi",
+				"ContainerMountPoint": h.task.CSIPluginConfig.StagePublishBaseDir,
 			},
 		}
 	}

command/agent/job_endpoint.go

@@ -1263,6 +1263,7 @@ func ApiCSIPluginConfigToStructsCSIPluginConfig(apiConfig *api.TaskCSIPluginConf
 	sc.ID = apiConfig.ID
 	sc.Type = structs.CSIPluginType(apiConfig.Type)
 	sc.MountDir = apiConfig.MountDir
+	sc.StagePublishBaseDir = apiConfig.StagePublishBaseDir
 	sc.HealthTimeout = apiConfig.HealthTimeout
 	return sc
 }

nomad/structs/csi.go

@@ -62,12 +62,17 @@ type TaskCSIPluginConfig struct {
 	// Type instructs Nomad on how to handle processing a plugin
 	Type CSIPluginType
 
-	// MountDir is the destination that nomad should mount in its CSI
-	// directory for the plugin. It will then expect a file called CSISocketName
-	// to be created by the plugin, and will provide references into
-	// "MountDir/CSIIntermediaryDirname/{VolumeName}/{AllocID} for mounts.
+	// MountDir is the directory (within its container) in which the plugin creates a
+	// socket (called CSISocketName) for communication with Nomad. Default is /csi.
 	MountDir string
 
+	// StagePublishBaseDir is the base directory (within its container) in which the plugin
+	// mounts volumes being staged and bind mount volumes being published.
+	// e.g. staging_target_path = {StagePublishBaseDir}/staging/{volume-id}/{usage-mode}
+	// e.g. target_path = {StagePublishBaseDir}/per-alloc/{alloc-id}/{volume-id}/{usage-mode}
+	// Default is /local/csi.
+	StagePublishBaseDir string
+
 	// HealthTimeout is the time after which the CSI plugin tasks will be killed
 	// if the CSI Plugin is not healthy.
 	HealthTimeout time.Duration `mapstructure:"health_timeout" hcl:"health_timeout,optional"`

website/content/docs/concepts/plugins/csi.mdx

@@ -38,9 +38,10 @@ A CSI plugin task requires the [`csi_plugin`][csi_plugin] block:
 
 ```hcl
 csi_plugin {
-  id        = "csi-hostpath"
-  type      = "monolith"
-  mount_dir = "/csi"
+  id                     = "csi-hostpath"
+  type                   = "monolith"
+  mount_dir              = "/csi"
+  stage_publish_base_dir = "/local/csi"
 }
 ```
 
@@ -73,7 +74,11 @@ Nomad exposes a Unix domain socket named `csi.sock` inside each CSI
 plugin task, and communicates over the gRPC protocol expected by the
 CSI specification. The `mount_dir` field tells Nomad where the plugin
 expects to find the socket file. The path to this socket is exposed in
-the container as the `CSI_ENDPOINT` environment variable.
+the container as the `CSI_ENDPOINT` environment variable. 
+
+Some plugins also require the `stage_publish_base_dir` field, which
+tells Nomad where to instruct the plugin to mount volumes for staging 
+and/or publishing.
 
 ### Plugin Lifecycle and State
 

website/content/docs/job-specification/csi_plugin.mdx

@@ -17,10 +17,11 @@ to claim [volumes][csi_volumes].
 
 ```hcl
 csi_plugin {
-  id             = "csi-hostpath"
-  type           = "monolith"
-  mount_dir      = "/csi"
-  health_timeout = "30s"
+  id                      = "csi-hostpath"
+  type                    = "monolith"
+  mount_dir               = "/csi"
+  stage_publish__base_dir = "/local/csi"
+  health_timeout          = "30s"
 }
 ```
 
@@ -40,9 +41,15 @@ csi_plugin {
   `node` at the same time, and these are called `monolith`
   plugins. Refer to your CSI plugin's documentation.
 
-- `mount_dir` `(string: <required>)` - The directory path inside the
+- `mount_dir` `(string: <optional>)` - The directory path inside the
   container where the plugin will expect a Unix domain socket for
-  bidirectional communication with Nomad.
+  bidirectional communication with Nomad. This field is typically not
+  required. Refer to your CSI plugin's documentation for details.
+
+- `stage_publish_base_dir` `(string: <optional>)` - The base directory 
+  path inside the container where the plugin will be instructed to
+  stage and publish volumes. This field is typically not required. 
+  Refer to your CSI plugin's documentation for details.
 
 - `health_timeout` `(duration: <optional>)` - The duration that 
   the plugin supervisor will wait before restarting an unhealthy