Commit 73579790 authored by Alex Dadgar's avatar Alex Dadgar
Browse files

Revert "Revert "Make drivers take arguments as a list and not as a string""

parent 0003310a
Showing with 84 additions and 149 deletions
+84 -149
......@@ -65,7 +65,7 @@ func TestAllocRunner_Destroy(t *testing.T) {
// Ensure task takes some time
task := ar.alloc.Job.TaskGroups[0].Tasks[0]
task.Config["command"] = "/bin/sleep"
task.Config["args"] = "10"
task.Config["args"] = []string{"10"}
go ar.Run()
start := time.Now()
......@@ -97,7 +97,7 @@ func TestAllocRunner_Update(t *testing.T) {
// Ensure task takes some time
task := ar.alloc.Job.TaskGroups[0].Tasks[0]
task.Config["command"] = "/bin/sleep"
task.Config["args"] = "10"
task.Config["args"] = []string{"10"}
go ar.Run()
defer ar.Destroy()
start := time.Now()
......@@ -130,7 +130,7 @@ func TestAllocRunner_SaveRestoreState(t *testing.T) {
// Ensure task takes some time
task := ar.alloc.Job.TaskGroups[0].Tasks[0]
task.Config["command"] = "/bin/sleep"
task.Config["args"] = "10"
task.Config["args"] = []string{"10"}
go ar.Run()
defer ar.Destroy()
......
......@@ -336,7 +336,7 @@ func TestClient_SaveRestoreState(t *testing.T) {
alloc1.NodeID = c1.Node().ID
task := alloc1.Job.TaskGroups[0].Tasks[0]
task.Config["command"] = "/bin/sleep"
task.Config["args"] = "10"
task.Config["args"] = []string{"10"}
state := s1.State()
err := state.UpsertAllocs(100,
......
package args
import (
"fmt"
"regexp"
"github.com/mattn/go-shellwords"
)
import "regexp"
var (
envRe = regexp.MustCompile(`\$({[a-zA-Z0-9_]+}|[a-zA-Z0-9_]+)`)
......@@ -13,27 +8,17 @@ var (
// ParseAndReplace takes the user supplied args and a map of environment
// variables. It replaces any instance of an environment variable in the args
// with the actual value and does correct splitting of the arg list.
func ParseAndReplace(args string, env map[string]string) ([]string, error) {
// Set up parser.
p := shellwords.NewParser()
p.ParseEnv = false
p.ParseBacktick = false
parsed, err := p.Parse(args)
if err != nil {
return nil, fmt.Errorf("Couldn't parse args %v: %v", args, err)
}
replaced := make([]string, len(parsed))
for i, arg := range parsed {
// with the actual value.
func ParseAndReplace(args []string, env map[string]string) []string {
replaced := make([]string, len(args))
for i, arg := range args {
replaced[i] = ReplaceEnv(arg, env)
}
return replaced, nil
return replaced
}
// replaceEnv takes an arg and replaces all occurences of environment variables.
// ReplaceEnv takes an arg and replaces all occurences of environment variables.
// If the variable is found in the passed map it is replaced, otherwise the
// original string is returned.
func ReplaceEnv(arg string, env map[string]string) string {
......
......@@ -21,12 +21,9 @@ var (
)
func TestDriverArgs_ParseAndReplaceInvalidEnv(t *testing.T) {
input := "invalid $FOO"
input := []string{"invalid", "$FOO"}
exp := []string{"invalid", "$FOO"}
act, err := ParseAndReplace(input, envVars)
if err != nil {
t.Fatalf("Failed to parse valid input args %v: %v", input, err)
}
act := ParseAndReplace(input, envVars)
if !reflect.DeepEqual(act, exp) {
t.Fatalf("ParseAndReplace(%v, %v) returned %#v; want %#v", input, envVars, act, exp)
......@@ -34,12 +31,9 @@ func TestDriverArgs_ParseAndReplaceInvalidEnv(t *testing.T) {
}
func TestDriverArgs_ParseAndReplaceValidEnv(t *testing.T) {
input := fmt.Sprintf("nomad_ip \\\"$%v\\\"!", ipKey)
input := []string{"nomad_ip", fmt.Sprintf(`"$%v"!`, ipKey)}
exp := []string{"nomad_ip", fmt.Sprintf("\"%s\"!", ipVal)}
act, err := ParseAndReplace(input, envVars)
if err != nil {
t.Fatalf("Failed to parse valid input args %v: %v", input, err)
}
act := ParseAndReplace(input, envVars)
if !reflect.DeepEqual(act, exp) {
t.Fatalf("ParseAndReplace(%v, %v) returned %#v; want %#v", input, envVars, act, exp)
......@@ -47,32 +41,9 @@ func TestDriverArgs_ParseAndReplaceValidEnv(t *testing.T) {
}
func TestDriverArgs_ParseAndReplaceChainedEnv(t *testing.T) {
input := fmt.Sprintf("-foo $%s$%s", ipKey, portKey)
input := []string{"-foo", fmt.Sprintf("$%s$%s", ipKey, portKey)}
exp := []string{"-foo", fmt.Sprintf("%s%s", ipVal, portVal)}
act, err := ParseAndReplace(input, envVars)
if err != nil {
t.Fatalf("Failed to parse valid input args %v: %v", input, err)
}
if !reflect.DeepEqual(act, exp) {
t.Fatalf("ParseAndReplace(%v, %v) returned %#v; want %#v", input, envVars, act, exp)
}
}
func TestDriverArgs_ParseAndReplaceInvalidArgEscape(t *testing.T) {
input := "-c \"echo \"foo\\\" > bar.txt\""
if _, err := ParseAndReplace(input, envVars); err == nil {
t.Fatalf("ParseAndReplace(%v, %v) should have failed", input, envVars)
}
}
func TestDriverArgs_ParseAndReplaceValidArgEscape(t *testing.T) {
input := "-c \"echo \\\"foo\\\" > bar.txt\""
exp := []string{"-c", "echo \"foo\" > bar.txt"}
act, err := ParseAndReplace(input, envVars)
if err != nil {
t.Fatalf("Failed to parse valid input args %v: %v", input, err)
}
act := ParseAndReplace(input, envVars)
if !reflect.DeepEqual(act, exp) {
t.Fatalf("ParseAndReplace(%v, %v) returned %#v; want %#v", input, envVars, act, exp)
......
......@@ -35,7 +35,7 @@ type DockerDriverAuth struct {
type DockerDriverConfig struct {
ImageName string `mapstructure:"image"` // Container's Image Name
Command string `mapstructure:"command"` // The Command/Entrypoint to run when the container starts up
Args string `mapstructure:"args"` // The arguments to the Command/Entrypoint
Args []string `mapstructure:"args"` // The arguments to the Command/Entrypoint
NetworkMode string `mapstructure:"network_mode"` // The network mode of the container - host, net and none
PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and the ports exposed on the container
Privileged bool `mapstructure:"privileged"` // Flag to run the container in priviledged mode
......@@ -293,21 +293,18 @@ func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, dri
config.ExposedPorts = exposedPorts
}
parsedArgs, err := args.ParseAndReplace(driverConfig.Args, env.Map())
if err != nil {
return c, err
}
parsedArgs := args.ParseAndReplace(driverConfig.Args, env.Map())
// If the user specified a custom command to run as their entrypoint, we'll
// inject it here.
if driverConfig.Command != "" {
cmd := []string{driverConfig.Command}
if driverConfig.Args != "" {
if len(driverConfig.Args) != 0 {
cmd = append(cmd, parsedArgs...)
}
d.logger.Printf("[DEBUG] driver.docker: setting container startup command to: %s", strings.Join(cmd, " "))
config.Cmd = cmd
} else if driverConfig.Args != "" {
} else if len(driverConfig.Args) != 0 {
d.logger.Println("[DEBUG] driver.docker: ignoring command arguments because command is not specified")
}
......@@ -431,7 +428,7 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle
if len(containers) != 1 {
log.Printf("[ERR] driver.docker: failed to get id for container %s", config.Name)
return nil, fmt.Errorf("Failed to get id for container %s: %s", config.Name, err)
return nil, fmt.Errorf("Failed to get id for container %s", config.Name)
}
log.Printf("[INFO] driver.docker: a container with the name %s already exists; will attempt to purge and re-create", config.Name)
......
......@@ -137,7 +137,7 @@ func TestDockerDriver_Start_Wait(t *testing.T) {
Config: map[string]interface{}{
"image": "redis",
"command": "redis-server",
"args": "-v",
"args": []string{"-v"},
},
Resources: &structs.Resources{
MemoryMB: 256,
......@@ -190,7 +190,11 @@ func TestDockerDriver_Start_Wait_AllocDir(t *testing.T) {
Config: map[string]interface{}{
"image": "redis",
"command": "/bin/bash",
"args": fmt.Sprintf(`-c "sleep 1; echo -n %s > $%s/%s"`, string(exp), environment.AllocDir, file),
"args": []string{
"-c",
fmt.Sprintf(`sleep 1; echo -n %s > $%s/%s`,
string(exp), environment.AllocDir, file),
},
},
Resources: &structs.Resources{
MemoryMB: 256,
......@@ -243,7 +247,7 @@ func TestDockerDriver_Start_Kill_Wait(t *testing.T) {
Config: map[string]interface{}{
"image": "redis",
"command": "/bin/sleep",
"args": "10",
"args": []string{"10"},
},
Resources: basicResources,
}
......
......@@ -24,10 +24,10 @@ type ExecDriver struct {
fingerprint.StaticFingerprinter
}
type ExecDriverConfig struct {
ArtifactSource string `mapstructure:"artifact_source"`
Checksum string `mapstructure:"checksum"`
Command string `mapstructure:"command"`
Args string `mapstructure:"args"`
ArtifactSource string `mapstructure:"artifact_source"`
Checksum string `mapstructure:"checksum"`
Command string `mapstructure:"command"`
Args []string `mapstructure:"args"`
}
// execHandle is returned from Start/Open as a handle to the PID
......@@ -91,14 +91,8 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
// Get the environment variables.
envVars := TaskEnvironmentVariables(ctx, task)
// Look for arguments
var args []string
if driverConfig.Args != "" {
args = append(args, driverConfig.Args)
}
// Setup the command
cmd := executor.Command(command, args...)
cmd := executor.Command(command, driverConfig.Args...)
if err := cmd.Limit(task.Resources); err != nil {
return nil, fmt.Errorf("failed to constrain resources: %s", err)
}
......
......@@ -39,7 +39,7 @@ func TestExecDriver_StartOpen_Wait(t *testing.T) {
Name: "sleep",
Config: map[string]interface{}{
"command": "/bin/sleep",
"args": "5",
"args": []string{"5"},
},
Resources: basicResources,
}
......@@ -73,7 +73,7 @@ func TestExecDriver_Start_Wait(t *testing.T) {
Name: "sleep",
Config: map[string]interface{}{
"command": "/bin/sleep",
"args": "2",
"args": []string{"2"},
},
Resources: basicResources,
}
......@@ -161,7 +161,10 @@ func TestExecDriver_Start_Artifact_expanded(t *testing.T) {
Config: map[string]interface{}{
"artifact_source": fmt.Sprintf("https://dl.dropboxusercontent.com/u/47675/jar_thing/%s", file),
"command": "/bin/bash",
"args": fmt.Sprintf("-c '/bin/sleep 1 && %s'", filepath.Join("$NOMAD_TASK_DIR", file)),
"args": []string{
"-c",
fmt.Sprintf(`/bin/sleep 1 && %s`, filepath.Join("$NOMAD_TASK_DIR", file)),
},
},
Resources: basicResources,
}
......@@ -204,7 +207,10 @@ func TestExecDriver_Start_Wait_AllocDir(t *testing.T) {
Name: "sleep",
Config: map[string]interface{}{
"command": "/bin/bash",
"args": fmt.Sprintf("-c \"sleep 1; echo -n %s > $%s/%s\"", string(exp), environment.AllocDir, file),
"args": []string{
"-c",
fmt.Sprintf(`sleep 1; echo -n %s > $%s/%s`, string(exp), environment.AllocDir, file),
},
},
Resources: basicResources,
}
......@@ -250,7 +256,7 @@ func TestExecDriver_Start_Kill_Wait(t *testing.T) {
Name: "sleep",
Config: map[string]interface{}{
"command": "/bin/sleep",
"args": "1",
"args": []string{"1"},
},
Resources: basicResources,
}
......
......@@ -29,7 +29,6 @@ type BasicExecutor struct {
allocDir string
}
// TODO: Have raw_exec use this as well.
func NewBasicExecutor() Executor {
return &BasicExecutor{}
}
......@@ -63,12 +62,7 @@ func (e *BasicExecutor) Start() error {
}
e.cmd.Path = args.ReplaceEnv(e.cmd.Path, envVars.Map())
combined := strings.Join(e.cmd.Args, " ")
parsed, err := args.ParseAndReplace(combined, envVars.Map())
if err != nil {
return err
}
e.cmd.Args = parsed
e.cmd.Args = args.ParseAndReplace(e.cmd.Args, envVars.Map())
spawnState := filepath.Join(e.allocDir, fmt.Sprintf("%s_%s", e.taskName, "exit_status"))
e.spawn = spawn.NewSpawner(spawnState)
......
......@@ -167,12 +167,7 @@ func (e *LinuxExecutor) Start() error {
}
e.cmd.Path = args.ReplaceEnv(e.cmd.Path, envVars.Map())
combined := strings.Join(e.cmd.Args, " ")
parsed, err := args.ParseAndReplace(combined, envVars.Map())
if err != nil {
return err
}
e.cmd.Args = parsed
e.cmd.Args = args.ParseAndReplace(e.cmd.Args, envVars.Map())
spawnState := filepath.Join(e.allocDir, fmt.Sprintf("%s_%s", e.taskName, "exit_status"))
e.spawn = spawn.NewSpawner(spawnState)
......
......@@ -112,7 +112,7 @@ func Executor_Start_Wait(t *testing.T, command buildExecCommand) {
expected := "hello world"
file := filepath.Join(allocdir.TaskLocal, "output.txt")
absFilePath := filepath.Join(taskDir, file)
cmd := fmt.Sprintf(`"%v \"%v\" > %v"`, "/bin/sleep 1 ; echo -n", expected, file)
cmd := fmt.Sprintf(`/bin/sleep 1 ; echo -n %v > %v`, expected, file)
e := command("/bin/bash", "-c", cmd)
if err := e.Limit(constraint); err != nil {
......@@ -190,7 +190,7 @@ func Executor_Open(t *testing.T, command buildExecCommand, newExecutor func() Ex
expected := "hello world"
file := filepath.Join(allocdir.TaskLocal, "output.txt")
absFilePath := filepath.Join(taskDir, file)
cmd := fmt.Sprintf(`"%v \"%v\" > %v"`, "/bin/sleep 1 ; echo -n", expected, file)
cmd := fmt.Sprintf(`/bin/sleep 1 ; echo -n %v > %v`, expected, file)
e := command("/bin/bash", "-c", cmd)
if err := e.Limit(constraint); err != nil {
......
......@@ -28,10 +28,10 @@ type JavaDriver struct {
}
type JavaDriverConfig struct {
JvmOpts string `mapstructure:"jvm_options"`
ArtifactSource string `mapstructure:"artifact_source"`
Checksum string `mapstructure:"checksum"`
Args string `mapstructure:"args"`
JvmOpts []string `mapstructure:"jvm_options"`
ArtifactSource string `mapstructure:"artifact_source"`
Checksum string `mapstructure:"checksum"`
Args []string `mapstructure:"args"`
}
// javaHandle is returned from Start/Open as a handle to the PID
......@@ -126,15 +126,15 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle,
args := []string{}
// Look for jvm options
if driverConfig.JvmOpts != "" {
if len(driverConfig.JvmOpts) != 0 {
d.logger.Printf("[DEBUG] driver.java: found JVM options: %s", driverConfig.JvmOpts)
args = append(args, driverConfig.JvmOpts)
args = append(args, driverConfig.JvmOpts...)
}
// Build the argument list.
args = append(args, "-jar", filepath.Join(allocdir.TaskLocal, jarName))
if driverConfig.Args != "" {
args = append(args, driverConfig.Args)
if len(driverConfig.Args) != 0 {
args = append(args, driverConfig.Args...)
}
// Setup the command
......
......@@ -51,7 +51,7 @@ func TestJavaDriver_StartOpen_Wait(t *testing.T) {
Name: "demo-app",
Config: map[string]interface{}{
"artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar",
"jvm_options": "-Xmx2048m -Xms256m",
"jvm_options": []string{"-Xmx64m", "-Xms32m"},
"checksum": "sha256:58d6e8130308d32e197c5108edd4f56ddf1417408f743097c2e662df0f0b17c8",
},
Resources: basicResources,
......@@ -97,7 +97,6 @@ func TestJavaDriver_Start_Wait(t *testing.T) {
Name: "demo-app",
Config: map[string]interface{}{
"artifact_source": "https://dl.dropboxusercontent.com/u/47675/jar_thing/demoapp.jar",
"jvm_options": "-Xmx2048m -Xms256m",
"checksum": "sha256:58d6e8130308d32e197c5108edd4f56ddf1417408f743097c2e662df0f0b17c8",
},
Resources: basicResources,
......
......@@ -93,15 +93,9 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl
// Get the environment variables.
envVars := TaskEnvironmentVariables(ctx, task)
// Look for arguments
var args []string
if driverConfig.Args != "" {
args = append(args, driverConfig.Args)
}
// Setup the command
cmd := executor.NewBasicExecutor()
executor.SetCommand(cmd, command, args)
executor.SetCommand(cmd, command, driverConfig.Args)
if err := cmd.Limit(task.Resources); err != nil {
return nil, fmt.Errorf("failed to constrain resources: %s", err)
}
......
......@@ -53,7 +53,7 @@ func TestRawExecDriver_StartOpen_Wait(t *testing.T) {
Name: "sleep",
Config: map[string]interface{}{
"command": "/bin/sleep",
"args": "1",
"args": []string{"1"},
},
Resources: basicResources,
}
......@@ -151,7 +151,10 @@ func TestRawExecDriver_Start_Artifact_expanded(t *testing.T) {
Config: map[string]interface{}{
"artifact_source": fmt.Sprintf("https://dl.dropboxusercontent.com/u/47675/jar_thing/%s", file),
"command": "/bin/bash",
"args": fmt.Sprintf("-c '/bin/sleep 1 && %s'", filepath.Join("$NOMAD_TASK_DIR", file)),
"args": []string{
"-c",
fmt.Sprintf(`'/bin/sleep 1 && %s'`, filepath.Join("$NOMAD_TASK_DIR", file)),
},
},
Resources: basicResources,
}
......@@ -190,7 +193,7 @@ func TestRawExecDriver_Start_Wait(t *testing.T) {
Name: "sleep",
Config: map[string]interface{}{
"command": "/bin/sleep",
"args": "1",
"args": []string{"1"},
},
Resources: basicResources,
}
......@@ -232,7 +235,10 @@ func TestRawExecDriver_Start_Wait_AllocDir(t *testing.T) {
Name: "sleep",
Config: map[string]interface{}{
"command": "/bin/bash",
"args": fmt.Sprintf(`-c "sleep 1; echo -n %s > $%s/%s"`, string(exp), environment.AllocDir, file),
"args": []string{
"-c",
fmt.Sprintf(`sleep 1; echo -n %s > $%s/%s`, string(exp), environment.AllocDir, file),
},
},
Resources: basicResources,
}
......@@ -277,7 +283,7 @@ func TestRawExecDriver_Start_Kill_Wait(t *testing.T) {
Name: "sleep",
Config: map[string]interface{}{
"command": "/bin/sleep",
"args": "1",
"args": []string{"1"},
},
Resources: basicResources,
}
......
......@@ -37,8 +37,8 @@ type RktDriver struct {
}
type RktDriverConfig struct {
ImageName string `mapstructure:"image"`
Args string `mapstructure:"args"`
ImageName string `mapstructure:"image"`
Args []string `mapstructure:"args"`
}
// rktHandle is returned from Start/Open as a handle to the PID
......@@ -150,11 +150,8 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e
}
// Add user passed arguments.
if driverConfig.Args != "" {
parsed, err := args.ParseAndReplace(driverConfig.Args, envVars.Map())
if err != nil {
return nil, err
}
if len(driverConfig.Args) != 0 {
parsed := args.ParseAndReplace(driverConfig.Args, envVars.Map())
// Need to start arguments with "--"
if len(parsed) > 0 {
......
......@@ -119,7 +119,7 @@ func TestRktDriver_Start_Wait(t *testing.T) {
"trust_prefix": "coreos.com/etcd",
"image": "coreos.com/etcd:v2.0.4",
"command": "/etcd",
"args": "--version",
"args": []string{"--version"},
},
}
......@@ -160,7 +160,7 @@ func TestRktDriver_Start_Wait_Skip_Trust(t *testing.T) {
Config: map[string]interface{}{
"image": "coreos.com/etcd:v2.0.4",
"command": "/etcd",
"args": "--version",
"args": []string{"--version"},
},
}
......@@ -202,7 +202,7 @@ func TestRktDriver_Start_Wait_Logs(t *testing.T) {
"trust_prefix": "coreos.com/etcd",
"image": "coreos.com/etcd:v2.0.4",
"command": "/etcd",
"args": "--version",
"args": []string{"--version"},
},
}
......
......@@ -89,7 +89,7 @@ func TestTaskRunner_Destroy(t *testing.T) {
// Change command to ensure we run for a bit
tr.task.Config["command"] = "/bin/sleep"
tr.task.Config["args"] = "10"
tr.task.Config["args"] = []string{"10"}
go tr.Run()
// Begin the tear down
......@@ -128,7 +128,7 @@ func TestTaskRunner_Update(t *testing.T) {
// Change command to ensure we run for a bit
tr.task.Config["command"] = "/bin/sleep"
tr.task.Config["args"] = "10"
tr.task.Config["args"] = []string{"10"}
go tr.Run()
defer tr.Destroy()
defer tr.ctx.AllocDir.Destroy()
......@@ -153,7 +153,7 @@ func TestTaskRunner_SaveRestoreState(t *testing.T) {
// Change command to ensure we run for a bit
tr.task.Config["command"] = "/bin/sleep"
tr.task.Config["args"] = "10"
tr.task.Config["args"] = []string{"10"}
go tr.Run()
defer tr.Destroy()
......
......@@ -13,13 +13,6 @@ import (
"github.com/hashicorp/raft"
)
var (
msgpackHandle = &codec.MsgpackHandle{
RawToString: true,
WriteExt: true,
}
)
const (
// timeTableGranularity is the granularity of index to time tracking
timeTableGranularity = 5 * time.Minute
......@@ -328,7 +321,7 @@ func (n *nomadFSM) Restore(old io.ReadCloser) error {
defer restore.Abort()
// Create a decoder
dec := codec.NewDecoder(old, msgpackHandle)
dec := codec.NewDecoder(old, structs.MsgpackHandle)
// Read in the header
var header snapshotHeader
......@@ -412,7 +405,7 @@ func (n *nomadFSM) Restore(old io.ReadCloser) error {
func (s *nomadSnapshot) Persist(sink raft.SnapshotSink) error {
defer metrics.MeasureSince([]string{"nomad", "fsm", "persist"}, time.Now())
// Register the nodes
encoder := codec.NewEncoder(sink, msgpackHandle)
encoder := codec.NewEncoder(sink, structs.MsgpackHandle)
// Write the header
header := snapshotHeader{}
......
......@@ -86,7 +86,7 @@ func Job() *structs.Job {
Driver: "exec",
Config: map[string]interface{}{
"command": "/bin/date",
"args": "+%s",
"args": []string{"+%s"},
},
Env: map[string]string{
"FOO": "bar",
......@@ -151,7 +151,7 @@ func SystemJob() *structs.Job {
Driver: "exec",
Config: map[string]interface{}{
"command": "/bin/date",
"args": "+%s",
"args": []string{"+%s"},
},
Resources: &structs.Resources{
CPU: 500,
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment