From 4bae54741e71ac281ea591731b384ea3719f9f2b Mon Sep 17 00:00:00 2001
From: Alex Dadgar <alex.dadgar@gmail.com>
Date: Fri, 2 Sep 2016 12:44:05 -0700
Subject: [PATCH] Secret dir materialized in alloc/task directory
---
client/allocdir/alloc_dir.go | 22 +++++++++++
client/allocdir/alloc_dir_darwin.go | 11 ++++++
client/allocdir/alloc_dir_freebsd.go | 11 ++++++
client/allocdir/alloc_dir_linux.go | 37 +++++++++++++++++++
client/allocdir/alloc_dir_test.go | 4 ++
client/allocdir/alloc_dir_unix.go | 10 ++++-
client/allocdir/alloc_dir_windows.go | 20 +++++++++-
client/client_test.go | 18 +++++++++
client/driver/docker.go | 1 +
client/driver/docker_test.go | 3 +-
client/driver/driver.go | 1 +
client/driver/env/env.go | 18 +++++++++
client/driver/exec_test.go | 3 +-
client/driver/executor/executor_linux_test.go | 18 ++++++++-
client/driver/java_test.go | 3 +-
client/driver/qemu_test.go | 3 +-
client/driver/raw_exec_test.go | 3 +-
client/driver/rkt_test.go | 1 +
client/task_runner_test.go | 2 +
19 files changed, 179 insertions(+), 10 deletions(-)
diff --git a/client/allocdir/alloc_dir.go b/client/allocdir/alloc_dir.go
index 6d0a41ee07..312bb5d797 100644
--- a/client/allocdir/alloc_dir.go
+++ b/client/allocdir/alloc_dir.go
@@ -46,6 +46,10 @@ var (
// regardless of driver.
TaskLocal = "local"
+ // TaskSecrets is the the name of the secret directory inside each task
+ // directory
+ TaskSecrets = "secrets"
+
// TaskDirs is the set of directories created in each tasks directory.
TaskDirs = []string{"tmp"}
)
@@ -154,6 +158,14 @@ func (d *AllocDir) UnmountAll() error {
}
}
+ taskSecret := filepath.Join(dir, TaskSecrets)
+ if d.pathExists(taskSecret) {
+ if err := d.removeSecretDir(taskSecret); err != nil {
+ mErr.Errors = append(mErr.Errors,
+ fmt.Errorf("failed to remove the secret dir %q: %v", taskSecret, err))
+ }
+ }
+
// Unmount dev/ and proc/ have been mounted.
d.unmountSpecialDirs(dir)
}
@@ -223,6 +235,16 @@ func (d *AllocDir) Build(tasks []*structs.Task) error {
return err
}
}
+
+ // Create the secret directory
+ secret := filepath.Join(taskDir, TaskSecrets)
+ if err := d.createSecretDir(secret); err != nil {
+ return err
+ }
+
+ if err := d.dropDirPermissions(secret); err != nil {
+ return err
+ }
}
return nil
diff --git a/client/allocdir/alloc_dir_darwin.go b/client/allocdir/alloc_dir_darwin.go
index 2cfdd38c31..9b94ec6e52 100644
--- a/client/allocdir/alloc_dir_darwin.go
+++ b/client/allocdir/alloc_dir_darwin.go
@@ -1,6 +1,7 @@
package allocdir
import (
+ "os"
"syscall"
)
@@ -14,6 +15,16 @@ func (d *AllocDir) unmountSharedDir(dir string) error {
return syscall.Unlink(dir)
}
+// createSecretDir creates the secrets dir folder at the given path
+func (d *AllocDir) createSecretDir(dir string) error {
+ return os.MkdirAll(dir, 0777)
+}
+
+// removeSecretDir removes the secrets dir folder
+func (d *AllocDir) removeSecretDir(dir string) error {
+ return os.RemoveAll(dir)
+}
+
// MountSpecialDirs mounts the dev and proc file system on the chroot of the
// task. It's a no-op on darwin.
func (d *AllocDir) MountSpecialDirs(taskDir string) error {
diff --git a/client/allocdir/alloc_dir_freebsd.go b/client/allocdir/alloc_dir_freebsd.go
index a4d3801db7..0f95e056be 100644
--- a/client/allocdir/alloc_dir_freebsd.go
+++ b/client/allocdir/alloc_dir_freebsd.go
@@ -1,6 +1,7 @@
package allocdir
import (
+ "os"
"syscall"
)
@@ -14,6 +15,16 @@ func (d *AllocDir) unmountSharedDir(dir string) error {
return syscall.Unlink(dir)
}
+// createSecretDir creates the secrets dir folder at the given path
+func (d *AllocDir) createSecretDir(dir string) error {
+ return os.MkdirAll(dir, 0777)
+}
+
+// removeSecretDir removes the secrets dir folder
+func (d *AllocDir) removeSecretDir(dir string) error {
+ return os.RemoveAll(dir)
+}
+
// MountSpecialDirs mounts the dev and proc file system on the chroot of the
// task. It's a no-op on FreeBSD right now.
func (d *AllocDir) MountSpecialDirs(taskDir string) error {
diff --git a/client/allocdir/alloc_dir_linux.go b/client/allocdir/alloc_dir_linux.go
index 9b2c670352..48ab1c3ad7 100644
--- a/client/allocdir/alloc_dir_linux.go
+++ b/client/allocdir/alloc_dir_linux.go
@@ -6,9 +6,16 @@ import (
"path/filepath"
"syscall"
+ "golang.org/x/sys/unix"
+
"github.com/hashicorp/go-multierror"
)
+const (
+ // secretDirTmpfsSize is the size of the tmpfs per task in MBs
+ secretDirTmpfsSize = 1
+)
+
// Bind mounts the shared directory into the task directory. Must be root to
// run.
func (d *AllocDir) mountSharedDir(taskDir string) error {
@@ -23,6 +30,36 @@ func (d *AllocDir) unmountSharedDir(dir string) error {
return syscall.Unmount(dir, 0)
}
+// createSecretDir creates the secrets dir folder at the given path using a
+// tmpfs
+func (d *AllocDir) createSecretDir(dir string) error {
+ // Only mount the tmpfs if we are root
+ if unix.Geteuid() == 0 {
+ if err := os.MkdirAll(dir, 0777); err != nil {
+ return err
+ }
+
+ var flags uintptr
+ flags = syscall.MS_NOEXEC
+ options := fmt.Sprintf("size=%dm", secretDirTmpfsSize)
+ err := syscall.Mount("tmpfs", dir, "tmpfs", flags, options)
+ return os.NewSyscallError("mount", err)
+ }
+
+ return os.MkdirAll(dir, 0777)
+}
+
+// createSecretDir removes the secrets dir folder
+func (d *AllocDir) removeSecretDir(dir string) error {
+ if unix.Geteuid() == 0 {
+ if err := syscall.Unmount(dir, 0); err != nil {
+ return err
+ }
+ }
+
+ return os.RemoveAll(dir)
+}
+
// MountSpecialDirs mounts the dev and proc file system from the host to the
// chroot
func (d *AllocDir) MountSpecialDirs(taskDir string) error {
diff --git a/client/allocdir/alloc_dir_test.go b/client/allocdir/alloc_dir_test.go
index aa0f5e49b5..9c2d42c6e4 100644
--- a/client/allocdir/alloc_dir_test.go
+++ b/client/allocdir/alloc_dir_test.go
@@ -73,6 +73,10 @@ func TestAllocDir_BuildAlloc(t *testing.T) {
if _, err := os.Stat(tDir); os.IsNotExist(err) {
t.Fatalf("Build(%v) didn't create TaskDir %v", tasks, tDir)
}
+
+ if _, err := os.Stat(filepath.Join(tDir, TaskSecrets)); os.IsNotExist(err) {
+ t.Fatalf("Build(%v) didn't create secret dir %v", tasks)
+ }
}
}
diff --git a/client/allocdir/alloc_dir_unix.go b/client/allocdir/alloc_dir_unix.go
index 339e59d5d1..0807505c4b 100644
--- a/client/allocdir/alloc_dir_unix.go
+++ b/client/allocdir/alloc_dir_unix.go
@@ -14,11 +14,17 @@ import (
)
var (
- //Path inside container for mounted directory shared across tasks in a task group.
+ // SharedAllocContainerPath is the path inside container for mounted
+ // directory shared across tasks in a task group.
SharedAllocContainerPath = filepath.Join("/", SharedAllocName)
- //Path inside container for mounted directory for local storage.
+ // TaskLocalContainer is the path inside a container for mounted directory
+ // for local storage.
TaskLocalContainerPath = filepath.Join("/", TaskLocal)
+
+ // TaskSecretsContainerPath is the path inside a container for mounted
+ // secrets directory
+ TaskSecretsContainerPath = filepath.Join("/", TaskSecrets)
)
func (d *AllocDir) linkOrCopy(src, dst string, perm os.FileMode) error {
diff --git a/client/allocdir/alloc_dir_windows.go b/client/allocdir/alloc_dir_windows.go
index 112fe9b63b..cacc9da829 100644
--- a/client/allocdir/alloc_dir_windows.go
+++ b/client/allocdir/alloc_dir_windows.go
@@ -7,11 +7,17 @@ import (
)
var (
- //Path inside container for mounted directory that is shared across tasks in a task group.
+ // SharedAllocContainerPath is the path inside container for mounted
+ // directory shared across tasks in a task group.
SharedAllocContainerPath = filepath.Join("c:\\", SharedAllocName)
- //Path inside container for mounted directory for local storage.
+ // TaskLocalContainer is the path inside a container for mounted directory
+ // for local storage.
TaskLocalContainerPath = filepath.Join("c:\\", TaskLocal)
+
+ // TaskSecretsContainerPath is the path inside a container for mounted
+ // secrets directory
+ TaskSecretsContainerPath = filepath.Join("c:\\", TaskSecrets)
)
func (d *AllocDir) linkOrCopy(src, dst string, perm os.FileMode) error {
@@ -23,6 +29,16 @@ func (d *AllocDir) mountSharedDir(dir string) error {
return errors.New("Mount on Windows not supported.")
}
+// createSecretDir creates the secrets dir folder at the given path
+func (d *AllocDir) createSecretDir(dir string) error {
+ return os.MkdirAll(dir, 0777)
+}
+
+// removeSecretDir removes the secrets dir folder
+func (d *AllocDir) removeSecretDir(dir string) error {
+ return os.RemoveAll(dir)
+}
+
// The windows version does nothing currently.
func (d *AllocDir) dropDirPermissions(path string) error {
return nil
diff --git a/client/client_test.go b/client/client_test.go
index ba98dc054d..cab6e7fa13 100644
--- a/client/client_test.go
+++ b/client/client_test.go
@@ -575,6 +575,14 @@ func TestClient_SaveRestoreState(t *testing.T) {
}, func(err error) {
t.Fatalf("err: %v", err)
})
+
+ // Destroy all the allocations
+ c2.allocLock.Lock()
+ for _, ar := range c2.allocs {
+ ar.Destroy()
+ <-ar.WaitCh()
+ }
+ c2.allocLock.Unlock()
}
func TestClient_Init(t *testing.T) {
@@ -608,6 +616,7 @@ func TestClient_BlockedAllocations(t *testing.T) {
c1 := testClient(t, func(c *config.Config) {
c.RPCHandler = s1
})
+ defer c1.Shutdown()
// Wait for the node to be ready
state := s1.State()
@@ -691,4 +700,13 @@ func TestClient_BlockedAllocations(t *testing.T) {
}, func(err error) {
t.Fatalf("err: %v", err)
})
+
+ // Destroy all the allocations
+ c1.allocLock.Lock()
+ for _, ar := range c1.allocs {
+ ar.Destroy()
+ <-ar.WaitCh()
+ }
+ c1.allocLock.Unlock()
+
}
diff --git a/client/driver/docker.go b/client/driver/docker.go
index 6c1390aa12..c2fdc92c2a 100644
--- a/client/driver/docker.go
+++ b/client/driver/docker.go
@@ -381,6 +381,7 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task,
// Set environment variables.
d.taskEnv.SetAllocDir(allocdir.SharedAllocContainerPath)
d.taskEnv.SetTaskLocalDir(allocdir.TaskLocalContainerPath)
+ d.taskEnv.SetTaskLocalDir(allocdir.TaskSecretsContainerPath)
config := &docker.Config{
Image: driverConfig.ImageName,
diff --git a/client/driver/docker_test.go b/client/driver/docker_test.go
index 97ed914eb8..5029d9e787 100644
--- a/client/driver/docker_test.go
+++ b/client/driver/docker_test.go
@@ -117,7 +117,8 @@ func dockerSetup(t *testing.T, task *structs.Task) (*docker.Client, DriverHandle
// This test should always pass, even if docker daemon is not available
func TestDockerDriver_Fingerprint(t *testing.T) {
- driverCtx, _ := testDriverContexts(&structs.Task{Name: "foo", Resources: basicResources})
+ driverCtx, execCtx := testDriverContexts(&structs.Task{Name: "foo", Resources: basicResources})
+ defer execCtx.AllocDir.Destroy()
d := NewDockerDriver(driverCtx)
node := &structs.Node{
Attributes: make(map[string]string),
diff --git a/client/driver/driver.go b/client/driver/driver.go
index 2ad38835ab..60adf9c7d7 100644
--- a/client/driver/driver.go
+++ b/client/driver/driver.go
@@ -153,6 +153,7 @@ func GetTaskEnv(allocDir *allocdir.AllocDir, node *structs.Node,
}
env.SetTaskLocalDir(filepath.Join(taskdir, allocdir.TaskLocal))
+ env.SetSecretDir(filepath.Join(taskdir, allocdir.TaskSecrets))
}
if task.Resources != nil {
diff --git a/client/driver/env/env.go b/client/driver/env/env.go
index 1ac9b7510b..e2ff660b89 100644
--- a/client/driver/env/env.go
+++ b/client/driver/env/env.go
@@ -21,6 +21,10 @@ const (
// removed.
TaskLocalDir = "NOMAD_TASK_DIR"
+ // SecretDir is the environment variable with the path to the tasks secret
+ // directory where it can store sensitive data.
+ SecretDir = "NOMAD_SECRET_DIR"
+
// MemLimit is the environment variable with the tasks memory limit in MBs.
MemLimit = "NOMAD_MEMORY_LIMIT"
@@ -79,6 +83,7 @@ type TaskEnvironment struct {
JobMeta map[string]string
AllocDir string
TaskDir string
+ SecretDir string
CpuLimit int
MemLimit int
TaskName string
@@ -153,6 +158,9 @@ func (t *TaskEnvironment) Build() *TaskEnvironment {
if t.TaskDir != "" {
t.TaskEnv[TaskLocalDir] = t.TaskDir
}
+ if t.SecretDir != "" {
+ t.TaskEnv[SecretDir] = t.SecretDir
+ }
// Build the resource limits
if t.MemLimit != 0 {
@@ -249,6 +257,16 @@ func (t *TaskEnvironment) ClearTaskLocalDir() *TaskEnvironment {
return t
}
+func (t *TaskEnvironment) SetSecretDir(dir string) *TaskEnvironment {
+ t.SecretDir = dir
+ return t
+}
+
+func (t *TaskEnvironment) ClearSecretDir() *TaskEnvironment {
+ t.SecretDir = ""
+ return t
+}
+
func (t *TaskEnvironment) SetMemLimit(limit int) *TaskEnvironment {
t.MemLimit = limit
return t
diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go
index 55e4f7b9dd..f2e72637fc 100644
--- a/client/driver/exec_test.go
+++ b/client/driver/exec_test.go
@@ -26,7 +26,8 @@ func TestExecDriver_Fingerprint(t *testing.T) {
Name: "foo",
Resources: structs.DefaultResources(),
}
- driverCtx, _ := testDriverContexts(task)
+ driverCtx, execCtx := testDriverContexts(task)
+ defer execCtx.AllocDir.Destroy()
d := NewExecDriver(driverCtx)
node := &structs.Node{
Attributes: map[string]string{
diff --git a/client/driver/executor/executor_linux_test.go b/client/driver/executor/executor_linux_test.go
index c3006071a4..321e0e0125 100644
--- a/client/driver/executor/executor_linux_test.go
+++ b/client/driver/executor/executor_linux_test.go
@@ -81,7 +81,23 @@ func TestExecutor_IsolationAndConstraints(t *testing.T) {
t.Fatalf("file %v hasn't been removed", memLimits)
}
- expected := "/:\nalloc/\nbin/\ndev/\netc/\nlib/\nlib64/\nlocal/\nproc/\ntmp/\nusr/\n\n/etc/:\nld.so.cache\nld.so.conf\nld.so.conf.d/"
+ expected := `/:
+alloc/
+bin/
+dev/
+etc/
+lib/
+lib64/
+local/
+proc/
+secrets/
+tmp/
+usr/
+
+/etc/:
+ld.so.cache
+ld.so.conf
+ld.so.conf.d/`
file := filepath.Join(ctx.AllocDir.LogDir(), "web.stdout.0")
output, err := ioutil.ReadFile(file)
if err != nil {
diff --git a/client/driver/java_test.go b/client/driver/java_test.go
index 553bb727f3..40e7b46c62 100644
--- a/client/driver/java_test.go
+++ b/client/driver/java_test.go
@@ -35,7 +35,8 @@ func TestJavaDriver_Fingerprint(t *testing.T) {
Name: "foo",
Resources: structs.DefaultResources(),
}
- driverCtx, _ := testDriverContexts(task)
+ driverCtx, execCtx := testDriverContexts(task)
+ defer execCtx.AllocDir.Destroy()
d := NewJavaDriver(driverCtx)
node := &structs.Node{
Attributes: map[string]string{
diff --git a/client/driver/qemu_test.go b/client/driver/qemu_test.go
index cd139c72dc..4f31220805 100644
--- a/client/driver/qemu_test.go
+++ b/client/driver/qemu_test.go
@@ -19,7 +19,8 @@ func TestQemuDriver_Fingerprint(t *testing.T) {
Name: "foo",
Resources: structs.DefaultResources(),
}
- driverCtx, _ := testDriverContexts(task)
+ driverCtx, execCtx := testDriverContexts(task)
+ defer execCtx.AllocDir.Destroy()
d := NewQemuDriver(driverCtx)
node := &structs.Node{
Attributes: make(map[string]string),
diff --git a/client/driver/raw_exec_test.go b/client/driver/raw_exec_test.go
index 1a5666f794..17023393fd 100644
--- a/client/driver/raw_exec_test.go
+++ b/client/driver/raw_exec_test.go
@@ -21,7 +21,8 @@ func TestRawExecDriver_Fingerprint(t *testing.T) {
Name: "foo",
Resources: structs.DefaultResources(),
}
- driverCtx, _ := testDriverContexts(task)
+ driverCtx, execCtx := testDriverContexts(task)
+ defer execCtx.AllocDir.Destroy()
d := NewRawExecDriver(driverCtx)
node := &structs.Node{
Attributes: make(map[string]string),
diff --git a/client/driver/rkt_test.go b/client/driver/rkt_test.go
index f8ed842432..fc68e47807 100644
--- a/client/driver/rkt_test.go
+++ b/client/driver/rkt_test.go
@@ -376,6 +376,7 @@ func TestRktTaskValidate(t *testing.T) {
"dns_servers": []string{"8.8.8.8", "8.8.4.4"},
"dns_search_domains": []string{"example.com", "example.org", "example.net"},
},
+ Resources: basicResources,
}
driverCtx, execCtx := testDriverContexts(task)
defer execCtx.AllocDir.Destroy()
diff --git a/client/task_runner_test.go b/client/task_runner_test.go
index 0fc7316c23..88c2b6be41 100644
--- a/client/task_runner_test.go
+++ b/client/task_runner_test.go
@@ -385,6 +385,8 @@ func TestTaskRunner_Download_Retries(t *testing.T) {
func TestTaskRunner_Validate_UserEnforcement(t *testing.T) {
_, tr := testTaskRunner(false)
+ defer tr.Destroy(structs.NewTaskEvent(structs.TaskKilled))
+ defer tr.ctx.AllocDir.Destroy()
// Try to run as root with exec.
tr.task.Driver = "exec"
--
GitLab