Merge pull request #4962 from hashicorp/f-exec-device-mounts

drivers/exec: Support devices mounts

drivers/exec/driver.go

@@ -308,6 +308,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstru
 		TaskDir:        cfg.TaskDir().Dir,
 		StdoutPath:     cfg.StdoutPath,
 		StderrPath:     cfg.StderrPath,
+		Mounts:         cfg.Mounts,
+		Devices:        cfg.Devices,
 	}
 
 	ps, err := exec.Launch(execCmd)

drivers/exec/driver_test.go

@@ -490,6 +490,97 @@ func TestExecDriver_HandlerExec(t *testing.T) {
 	require.NoError(harness.DestroyTask(task.ID, true))
 }
 
+func TestExecDriver_DevicesAndMounts(t *testing.T) {
+	t.Parallel()
+	require := require.New(t)
+	ctestutils.ExecCompatible(t)
+
+	tmpDir, err := ioutil.TempDir("", "exec_binds_mounts")
+	require.NoError(err)
+	defer os.RemoveAll(tmpDir)
+
+	err = ioutil.WriteFile(filepath.Join(tmpDir, "testfile"), []byte("from-host"), 600)
+	require.NoError(err)
+
+	d := NewExecDriver(testlog.HCLogger(t))
+	harness := dtestutil.NewDriverHarness(t, d)
+	task := &drivers.TaskConfig{
+		ID:         uuid.Generate(),
+		Name:       "test",
+		StdoutPath: filepath.Join(tmpDir, "task-stdout"),
+		StderrPath: filepath.Join(tmpDir, "task-stderr"),
+		Devices: []*drivers.DeviceConfig{
+			{
+				TaskPath:    "/dev/inserted-random",
+				HostPath:    "/dev/random",
+				Permissions: "rw",
+			},
+		},
+		Mounts: []*drivers.MountConfig{
+			{
+				TaskPath: "/tmp/task-path-rw",
+				HostPath: tmpDir,
+				Readonly: false,
+			},
+			{
+				TaskPath: "/tmp/task-path-ro",
+				HostPath: tmpDir,
+				Readonly: true,
+			},
+		},
+	}
+
+	require.NoError(ioutil.WriteFile(task.StdoutPath, []byte{}, 660))
+	require.NoError(ioutil.WriteFile(task.StderrPath, []byte{}, 660))
+
+	taskConfig := map[string]interface{}{
+		"command": "/bin/bash",
+		"args": []string{"-c", `
+export LANG=en.UTF-8
+echo "mounted device /inserted-random: $(stat -c '%t:%T' /dev/inserted-random)"
+echo "reading from ro path: $(cat /tmp/task-path-ro/testfile)"
+echo "reading from rw path: $(cat /tmp/task-path-rw/testfile)"
+touch /tmp/task-path-rw/testfile && echo 'overwriting file in rw succeeded'
+touch /tmp/task-path-rw/testfile-from-rw && echo from-exec >  /tmp/task-path-rw/testfile-from-rw && echo 'writing new file in rw succeeded'
+touch /tmp/task-path-ro/testfile && echo 'overwriting file in ro succeeded'
+touch /tmp/task-path-ro/testfile-from-ro && echo from-exec >  /tmp/task-path-ro/testfile-from-ro && echo 'writing new file in ro succeeded'
+exit 0
+`},
+	}
+	encodeDriverHelper(require, task, taskConfig)
+
+	cleanup := harness.MkAllocDir(task, false)
+	defer cleanup()
+
+	handle, _, err := harness.StartTask(task)
+	require.NoError(err)
+
+	ch, err := harness.WaitTask(context.Background(), handle.Config.ID)
+	require.NoError(err)
+	result := <-ch
+	require.NoError(harness.DestroyTask(task.ID, true))
+
+	stdout, err := ioutil.ReadFile(task.StdoutPath)
+	require.NoError(err)
+	require.Equal(`mounted device /inserted-random: 1:8
+reading from ro path: from-host
+reading from rw path: from-host
+overwriting file in rw succeeded
+writing new file in rw succeeded`, strings.TrimSpace(string(stdout)))
+
+	stderr, err := ioutil.ReadFile(task.StderrPath)
+	require.NoError(err)
+	require.Equal(`touch: cannot touch '/tmp/task-path-ro/testfile': Read-only file system
+touch: cannot touch '/tmp/task-path-ro/testfile-from-ro': Read-only file system`, strings.TrimSpace(string(stderr)))
+
+	// testing exit code last so we can inspect output first
+	require.Zero(result.ExitCode)
+
+	fromRWContent, err := ioutil.ReadFile(filepath.Join(tmpDir, "testfile-from-rw"))
+	require.NoError(err)
+	require.Equal("from-exec", strings.TrimSpace(string(fromRWContent)))
+}
+
 func encodeDriverHelper(require *require.Assertions, task *drivers.TaskConfig, taskConfig map[string]interface{}) {
 	evalCtx := &hcl.EvalContext{
 		Functions: shared.GetStdlibFuncs(),

drivers/java/driver.go

@@ -344,6 +344,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstru
 		TaskDir:    cfg.TaskDir().Dir,
 		StdoutPath: cfg.StdoutPath,
 		StderrPath: cfg.StderrPath,
+		Mounts:     cfg.Mounts,
+		Devices:    cfg.Devices,
 	}
 
 	ps, err := exec.Launch(execCmd)

drivers/shared/executor/executor.go

@@ -22,6 +22,7 @@ import (
 	"github.com/hashicorp/nomad/client/stats"
 	cstructs "github.com/hashicorp/nomad/client/structs"
 	shelpers "github.com/hashicorp/nomad/helper/stats"
+	"github.com/hashicorp/nomad/plugins/drivers"
 )
 
 const (
@@ -120,6 +121,12 @@ type ExecCommand struct {
 	// doesn't enforce resource limits. To enforce limits, set ResourceLimits.
 	// Using the cgroup does allow more precise cleanup of processes.
 	BasicProcessCgroup bool
+
+	// Mounts are the host paths to be be made available inside rootfs
+	Mounts []*drivers.MountConfig
+
+	// Devices are the the device nodes to be created in isolation environment
+	Devices []*drivers.DeviceConfig
 }
 
 type nopCloser struct {

drivers/shared/executor/executor_linux.go

@@ -22,11 +22,14 @@ import (
 	"github.com/hashicorp/nomad/helper/discover"
 	shelpers "github.com/hashicorp/nomad/helper/stats"
 	"github.com/hashicorp/nomad/helper/uuid"
+	"github.com/hashicorp/nomad/plugins/drivers"
 	"github.com/opencontainers/runc/libcontainer"
 	"github.com/opencontainers/runc/libcontainer/cgroups"
 	cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
 	lconfigs "github.com/opencontainers/runc/libcontainer/configs"
+	ldevices "github.com/opencontainers/runc/libcontainer/devices"
 	"github.com/syndtr/gocapability/capability"
+	"golang.org/x/sys/unix"
 )
 
 const (
@@ -125,7 +128,11 @@ func (l *LibcontainerExecutor) Launch(command *ExecCommand) (*ProcessState, erro
 	}
 
 	// A container groups processes under the same isolation enforcement
-	container, err := factory.Create(l.id, newLibcontainerConfig(command))
+	containerCfg, err := newLibcontainerConfig(command)
+	if err != nil {
+		return nil, fmt.Errorf("failed to configure container(%s): %v", l.id, err)
+	}
+	container, err := factory.Create(l.id, containerCfg)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create container(%s): %v", l.id, err)
 	}
@@ -468,7 +475,7 @@ func configureCapabilities(cfg *lconfigs.Config, command *ExecCommand) {
 
 }
 
-func configureIsolation(cfg *lconfigs.Config, command *ExecCommand) {
+func configureIsolation(cfg *lconfigs.Config, command *ExecCommand) error {
 	defaultMountFlags := syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV
 
 	// set the new root directory for the container
@@ -491,6 +498,14 @@ func configureIsolation(cfg *lconfigs.Config, command *ExecCommand) {
 	}
 
 	cfg.Devices = lconfigs.DefaultAutoCreatedDevices
+	if len(command.Devices) > 0 {
+		devs, err := cmdDevices(command.Devices)
+		if err != nil {
+			return err
+		}
+		cfg.Devices = append(cfg.Devices, devs...)
+	}
+
 	cfg.Mounts = []*lconfigs.Mount{
 		{
 			Source:      "tmpfs",
@@ -532,6 +547,12 @@ func configureIsolation(cfg *lconfigs.Config, command *ExecCommand) {
 			Flags:       defaultMountFlags | syscall.MS_RDONLY,
 		},
 	}
+
+	if len(command.Mounts) > 0 {
+		cfg.Mounts = append(cfg.Mounts, cmdMounts(command.Mounts)...)
+	}
+
+	return nil
 }
 
 func configureCgroups(cfg *lconfigs.Config, command *ExecCommand) error {
@@ -592,7 +613,7 @@ func configureBasicCgroups(cfg *lconfigs.Config) error {
 	return nil
 }
 
-func newLibcontainerConfig(command *ExecCommand) *lconfigs.Config {
+func newLibcontainerConfig(command *ExecCommand) (*lconfigs.Config, error) {
 	cfg := &lconfigs.Config{
 		Cgroups: &lconfigs.Cgroup{
 			Resources: &lconfigs.Resources{
@@ -605,9 +626,13 @@ func newLibcontainerConfig(command *ExecCommand) *lconfigs.Config {
 	}
 
 	configureCapabilities(cfg, command)
-	configureIsolation(cfg, command)
-	configureCgroups(cfg, command)
-	return cfg
+	if err := configureIsolation(cfg, command); err != nil {
+		return nil, err
+	}
+	if err := configureCgroups(cfg, command); err != nil {
+		return nil, err
+	}
+	return cfg, nil
 }
 
 // JoinRootCgroup moves the current process to the cgroups of the init process
@@ -631,3 +656,47 @@ func JoinRootCgroup(subsystems []string) error {
 
 	return mErrs.ErrorOrNil()
 }
+
+// cmdDevices converts a list of driver.DeviceConfigs into excutor.Devices.
+func cmdDevices(devices []*drivers.DeviceConfig) ([]*lconfigs.Device, error) {
+	if len(devices) == 0 {
+		return nil, nil
+	}
+
+	r := make([]*lconfigs.Device, len(devices))
+
+	for i, d := range devices {
+		ed, err := ldevices.DeviceFromPath(d.HostPath, d.Permissions)
+		if err != nil {
+			return nil, fmt.Errorf("failed to make device out for %s: %v", d.HostPath, err)
+		}
+		ed.Path = d.TaskPath
+		r[i] = ed
+	}
+
+	return r, nil
+}
+
+// cmdMounts converts a list of driver.MountConfigs into excutor.Mounts.
+func cmdMounts(mounts []*drivers.MountConfig) []*lconfigs.Mount {
+	if len(mounts) == 0 {
+		return nil
+	}
+
+	r := make([]*lconfigs.Mount, len(mounts))
+
+	for i, m := range mounts {
+		flags := unix.MS_BIND
+		if m.Readonly {
+			flags |= unix.MS_RDONLY
+		}
+		r[i] = &lconfigs.Mount{
+			Source:      m.HostPath,
+			Destination: m.TaskPath,
+			Device:      "bind",
+			Flags:       flags,
+		}
+	}
+
+	return r
+}

drivers/shared/executor/executor_linux_test.go

@@ -17,8 +17,11 @@ import (
 	"github.com/hashicorp/nomad/client/testutil"
 	"github.com/hashicorp/nomad/helper/testlog"
 	"github.com/hashicorp/nomad/nomad/mock"
+	"github.com/hashicorp/nomad/plugins/drivers"
 	tu "github.com/hashicorp/nomad/testutil"
+	lconfigs "github.com/opencontainers/runc/libcontainer/configs"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
 )
 
 func init() {
@@ -194,3 +197,66 @@ func TestExecutor_ClientCleanup(t *testing.T) {
 	output1 := execCmd.stdout.(*bufferCloser).String()
 	require.Equal(len(output), len(output1))
 }
+
+func TestExecutor_cmdDevices(t *testing.T) {
+	input := []*drivers.DeviceConfig{
+		{
+			HostPath:    "/dev/null",
+			TaskPath:    "/task/dev/null",
+			Permissions: "rwm",
+		},
+	}
+
+	expected := &lconfigs.Device{
+		Path:        "/task/dev/null",
+		Type:        99,
+		Major:       1,
+		Minor:       3,
+		Permissions: "rwm",
+	}
+
+	found, err := cmdDevices(input)
+	require.NoError(t, err)
+	require.Len(t, found, 1)
+
+	// ignore file permission and ownership
+	// as they are host specific potentially
+	d := found[0]
+	d.FileMode = 0
+	d.Uid = 0
+	d.Gid = 0
+
+	require.EqualValues(t, expected, d)
+}
+
+func TestExecutor_cmdMounts(t *testing.T) {
+	input := []*drivers.MountConfig{
+		{
+			HostPath: "/host/path-ro",
+			TaskPath: "/task/path-ro",
+			Readonly: true,
+		},
+		{
+			HostPath: "/host/path-rw",
+			TaskPath: "/task/path-rw",
+			Readonly: false,
+		},
+	}
+
+	expected := []*lconfigs.Mount{
+		{
+			Source:      "/host/path-ro",
+			Destination: "/task/path-ro",
+			Flags:       unix.MS_BIND | unix.MS_RDONLY,
+			Device:      "bind",
+		},
+		{
+			Source:      "/host/path-rw",
+			Destination: "/task/path-rw",
+			Flags:       unix.MS_BIND,
+			Device:      "bind",
+		},
+	}
+
+	require.EqualValues(t, expected, cmdMounts(input))
+}