Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
小 白蛋
Nomad
Commits
af5d42c0
Unverified
Commit
af5d42c0
authored
5 years ago
by
Danielle Lancashire
Browse files
Options
Download
Email Patches
Plain Diff
structs: Unify Volume and VolumeRequest
parent
0f5cf5fa
Branches unavailable
v1.4.3
v1.4.2
v1.4.1
v1.4.0
v1.4.0-rc.1
v1.4.0-beta.1
v1.3.8
v1.3.7
v1.3.6
v1.3.5
v1.3.4
v1.3.3
v1.3.2
v1.3.1
v1.3.0
v1.3.0-rc.1
v1.3.0-beta.1
v1.2.15
v1.2.14
v1.2.13
v1.2.12
v1.2.11
v1.2.10
v1.2.9
v1.2.8
v1.2.7
v1.2.6
v1.2.5
v1.2.4
v1.2.3
v1.2.2
v1.2.1
v1.2.0
v1.2.0-rc1
v1.2.0-beta1
v1.1.18
v1.1.17
v1.1.16
v1.1.15
v1.1.14
v1.1.13
v1.1.12
v1.1.11
v1.1.10
v1.1.9
v1.1.8
v1.1.7
v1.1.6
v1.1.5
v1.1.4
v1.1.3
v1.1.2
v1.1.1
v1.1.0
v1.1.0-rc1
v1.1.0-beta1
v1.0.18
v1.0.17
v1.0.16
v1.0.15
v1.0.14
v1.0.13
v1.0.12
v1.0.11
v1.0.10
v1.0.9
v1.0.8
v1.0.7
v1.0.6
v1.0.5
v1.0.4
v1.0.3
v1.0.2
v1.0.1
v1.0.0
v1.0.0-rc1
v1.0.0-beta3
v1.0.0-beta2
v0.12.12
v0.12.11
v0.12.10
v0.12.9
v0.12.8
v0.12.7
v0.12.6
v0.12.5
v0.12.4
v0.12.4-rc1
v0.12.3
v0.12.2
v0.12.1
v0.12.0
v0.12.0-rc1
v0.12.0-beta2
v0.12.0-beta1
v0.11.8
v0.11.7
v0.11.6
v0.11.5
v0.11.4
v0.11.3
v0.11.2
v0.11.1
v0.11.0
v0.11.0-rc1
v0.11.0-beta2
v0.11.0-beta1
v0.10.9
v0.10.8
v0.10.7
v0.10.6
v0.10.5
v0.10.4
v0.10.4-rc1
v0.10.3
v0.10.2
v0.10.2-rc1
v0.10.1
v0.10.0
v0.10.0-rc1
v0.10.0-beta1
nightly
No related merge requests found
Changes
8
Hide whitespace changes
Inline
Side-by-side
Showing
8 changed files
client/allocrunner/taskrunner/volume_hook.go
+13
-9
client/allocrunner/taskrunner/volume_hook.go
command/agent/config.go
+1
-5
command/agent/config.go
command/agent/job_endpoint.go
+2
-5
command/agent/job_endpoint.go
nomad/structs/structs.go
+2
-2
nomad/structs/structs.go
nomad/structs/structs_test.go
+3
-11
nomad/structs/structs_test.go
nomad/structs/volumes.go
+30
-50
nomad/structs/volumes.go
scheduler/feasible.go
+13
-5
scheduler/feasible.go
scheduler/feasible_test.go
+3
-3
scheduler/feasible_test.go
with
67 additions
and
90 deletions
+67
-90
client/allocrunner/taskrunner/volume_hook.go
+
13
-
9
View file @
af5d42c0
...
...
@@ -30,20 +30,21 @@ func (*volumeHook) Name() string {
return
"volumes"
}
func
validateHostVolumes
(
requested
map
[
string
]
*
structs
.
VolumeRequest
,
client
map
[
string
]
*
structs
.
ClientHostVolumeConfig
)
error
{
func
validateHostVolumes
(
requested
ByAlias
map
[
string
]
*
structs
.
VolumeRequest
,
client
VolumesByName
map
[
string
]
*
structs
.
ClientHostVolumeConfig
)
error
{
var
result
error
for
n
,
req
:=
range
requested
{
if
req
.
Volume
.
Type
!=
"h
ost
"
{
for
n
,
req
:=
range
requested
ByAlias
{
if
req
.
Type
!=
structs
.
VolumeType
H
ost
{
continue
}
cfg
,
err
:=
structs
.
ParseHostVolumeConfig
(
req
.
Config
)
if
err
!=
nil
{
result
=
multierror
.
Append
(
result
,
fmt
.
Errorf
(
"failed to parse config for %s: %v"
,
n
,
err
))
continue
}
_
,
ok
:=
client
[
cfg
.
Source
]
_
,
ok
:=
client
VolumesByName
[
cfg
.
Source
]
if
!
ok
{
result
=
multierror
.
Append
(
result
,
fmt
.
Errorf
(
"missing %s"
,
cfg
.
Source
))
}
...
...
@@ -52,10 +53,13 @@ func validateHostVolumes(requested map[string]*structs.VolumeRequest, client map
return
result
}
func
(
h
*
volumeHook
)
hostVolumeMountConfigurations
(
vmounts
[]
*
structs
.
VolumeMount
,
volumes
map
[
string
]
*
structs
.
VolumeRequest
,
client
map
[
string
]
*
structs
.
ClientHostVolumeConfig
)
([]
*
drivers
.
MountConfig
,
error
)
{
// hostVolumeMountConfigurations takes the users requested volume mounts,
// volumes, and the client host volume configuration and converts them into a
// format that can be used by drivers.
func
(
h
*
volumeHook
)
hostVolumeMountConfigurations
(
taskMounts
[]
*
structs
.
VolumeMount
,
taskVolumesByAlias
map
[
string
]
*
structs
.
VolumeRequest
,
clientVolumesByName
map
[
string
]
*
structs
.
ClientHostVolumeConfig
)
([]
*
drivers
.
MountConfig
,
error
)
{
var
mounts
[]
*
drivers
.
MountConfig
for
_
,
m
:=
range
vm
ounts
{
req
,
ok
:=
volume
s
[
m
.
Volume
]
for
_
,
m
:=
range
taskM
ounts
{
req
,
ok
:=
taskVolumesByAlia
s
[
m
.
Volume
]
if
!
ok
{
// Should never happen unless we misvalidated on job submission
return
nil
,
fmt
.
Errorf
(
"No group volume declaration found named: %s"
,
m
.
Volume
)
...
...
@@ -66,7 +70,7 @@ func (h *volumeHook) hostVolumeMountConfigurations(vmounts []*structs.VolumeMoun
return
nil
,
fmt
.
Errorf
(
"failed to parse config for %s: %v"
,
m
.
Volume
,
err
)
}
hostVolume
,
ok
:=
client
[
cfg
.
Source
]
hostVolume
,
ok
:=
client
VolumesByName
[
cfg
.
Source
]
if
!
ok
{
// Should never happen, but unless the client volumes were mutated during
// the execution of this hook.
...
...
@@ -76,7 +80,7 @@ func (h *volumeHook) hostVolumeMountConfigurations(vmounts []*structs.VolumeMoun
mcfg
:=
&
drivers
.
MountConfig
{
HostPath
:
hostVolume
.
Source
,
TaskPath
:
m
.
Destination
,
Readonly
:
hostVolume
.
ReadOnly
||
req
.
Volume
.
ReadOnly
||
m
.
ReadOnly
,
Readonly
:
hostVolume
.
ReadOnly
||
req
.
ReadOnly
||
m
.
ReadOnly
,
}
mounts
=
append
(
mounts
,
mcfg
)
}
...
...
This diff is collapsed.
Click to expand it.
command/agent/config.go
+
1
-
5
View file @
af5d42c0
...
...
@@ -1327,11 +1327,7 @@ func (a *ClientConfig) Merge(b *ClientConfig) *ClientConfig {
}
if
len
(
a
.
HostVolumes
)
==
0
&&
len
(
b
.
HostVolumes
)
!=
0
{
cc
:=
make
([]
*
structs
.
ClientHostVolumeConfig
,
len
(
b
.
HostVolumes
))
for
k
,
v
:=
range
b
.
HostVolumes
{
cc
[
k
]
=
v
.
Copy
()
}
result
.
HostVolumes
=
cc
result
.
HostVolumes
=
structs
.
CopySliceClientHostVolumeConfig
(
b
.
HostVolumes
)
}
else
if
len
(
b
.
HostVolumes
)
!=
0
{
result
.
HostVolumes
=
structs
.
HostVolumeSliceMerge
(
a
.
HostVolumes
,
b
.
HostVolumes
)
}
...
...
This diff is collapsed.
Click to expand it.
command/agent/job_endpoint.go
+
2
-
5
View file @
af5d42c0
...
...
@@ -749,7 +749,7 @@ func ApiTgToStructsTG(taskGroup *api.TaskGroup, tg *structs.TaskGroup) {
continue
}
vol
:=
&
structs
.
Volume
{
vol
:=
&
structs
.
Volume
Request
{
Name
:
v
.
Name
,
Type
:
v
.
Type
,
ReadOnly
:
v
.
ReadOnly
,
...
...
@@ -757,10 +757,7 @@ func ApiTgToStructsTG(taskGroup *api.TaskGroup, tg *structs.TaskGroup) {
Config
:
v
.
Config
,
}
tg
.
Volumes
[
k
]
=
&
structs
.
VolumeRequest
{
Volume
:
vol
,
Config
:
v
.
Config
,
}
tg
.
Volumes
[
k
]
=
vol
}
}
...
...
This diff is collapsed.
Click to expand it.
nomad/structs/structs.go
+
2
-
2
View file @
af5d42c0
...
...
@@ -4885,9 +4885,9 @@ func (tg *TaskGroup) Validate(j *Job) error {
// Validate the Host Volumes
for
name
,
decl
:=
range
tg
.
Volumes
{
if
decl
.
Volume
.
Type
!=
VolumeTypeHost
{
if
decl
.
Type
!=
VolumeTypeHost
{
// TODO: Remove this error when adding new volume types
mErr
.
Errors
=
append
(
mErr
.
Errors
,
fmt
.
Errorf
(
"Volume %s has unrecognised type %s"
,
name
,
decl
.
Volume
.
Type
))
mErr
.
Errors
=
append
(
mErr
.
Errors
,
fmt
.
Errorf
(
"Volume %s has unrecognised type %s"
,
name
,
decl
.
Type
))
continue
}
...
...
This diff is collapsed.
Click to expand it.
nomad/structs/structs_test.go
+
3
-
11
View file @
af5d42c0
...
...
@@ -872,9 +872,7 @@ func TestTaskGroup_Validate(t *testing.T) {
tg
=
&
TaskGroup
{
Volumes
:
map
[
string
]
*
VolumeRequest
{
"foo"
:
{
Volume
:
&
Volume
{
Type
:
"nothost"
,
},
Type
:
"nothost"
,
Config
:
map
[
string
]
interface
{}{
"sOuRcE"
:
"foo"
,
},
...
...
@@ -893,10 +891,7 @@ func TestTaskGroup_Validate(t *testing.T) {
tg
=
&
TaskGroup
{
Volumes
:
map
[
string
]
*
VolumeRequest
{
"foo"
:
{
Volume
:
&
Volume
{
Type
:
"host"
,
},
Config
:
nil
,
Type
:
"host"
,
},
},
Tasks
:
[]
*
Task
{
...
...
@@ -912,10 +907,7 @@ func TestTaskGroup_Validate(t *testing.T) {
tg
=
&
TaskGroup
{
Volumes
:
map
[
string
]
*
VolumeRequest
{
"foo"
:
{
Volume
:
&
Volume
{
Type
:
"host"
,
},
Config
:
nil
,
Type
:
"host"
,
},
},
Tasks
:
[]
*
Task
{
...
...
This diff is collapsed.
Click to expand it.
nomad/structs/volumes.go
+
30
-
50
View file @
af5d42c0
...
...
@@ -40,23 +40,36 @@ func CopyMapStringClientHostVolumeConfig(m map[string]*ClientHostVolumeConfig) m
return
nm
}
func
CopySliceClientHostVolumeConfig
(
s
[]
*
ClientHostVolumeConfig
)
[]
*
ClientHostVolumeConfig
{
l
:=
len
(
s
)
if
l
==
0
{
return
nil
}
ns
:=
make
([]
*
ClientHostVolumeConfig
,
l
)
for
idx
,
cfg
:=
range
s
{
ns
[
idx
]
=
cfg
.
Copy
()
}
return
ns
}
func
HostVolumeSliceMerge
(
a
,
b
[]
*
ClientHostVolumeConfig
)
[]
*
ClientHostVolumeConfig
{
n
:=
make
([]
*
ClientHostVolumeConfig
,
len
(
a
))
seenKeys
:=
make
(
map
[
string
]
struct
{}
,
len
(
a
))
seenKeys
:=
make
(
map
[
string
]
int
,
len
(
a
))
for
k
,
v
:=
range
a
{
if
_
,
ok
:=
seenKeys
[
v
.
Name
];
ok
{
continue
}
n
[
k
]
=
v
.
Copy
()
seenKeys
[
v
.
Name
]
=
struct
{}{}
for
i
,
config
:=
range
a
{
n
[
i
]
=
config
.
Copy
()
seenKeys
[
config
.
Name
]
=
i
}
for
k
,
v
:=
range
b
{
if
_
,
ok
:=
seenKeys
[
v
.
Name
];
ok
{
for
_
,
config
:=
range
b
{
if
fIndex
,
ok
:=
seenKeys
[
config
.
Name
];
ok
{
n
[
fIndex
]
=
config
.
Copy
()
continue
}
n
[
k
]
=
v
.
Copy
()
seenKeys
[
v
.
Name
]
=
struct
{}{}
n
=
append
(
n
,
config
.
Copy
())
}
return
n
...
...
@@ -78,8 +91,8 @@ func (h *HostVolumeConfig) Copy() *HostVolumeConfig {
return
nh
}
// Volume is a representation of a storage volume that a TaskGroup wishes to use.
type
Volume
struct
{
// Volume
Request
is a representation of a storage volume that a TaskGroup wishes to use.
type
Volume
Request
struct
{
Name
string
Type
string
ReadOnly
bool
...
...
@@ -88,11 +101,11 @@ type Volume struct {
Config
map
[
string
]
interface
{}
}
func
(
v
*
Volume
)
Copy
()
*
Volume
{
func
(
v
*
Volume
Request
)
Copy
()
*
Volume
Request
{
if
v
==
nil
{
return
nil
}
nv
:=
new
(
Volume
)
nv
:=
new
(
Volume
Request
)
*
nv
=
*
v
if
i
,
err
:=
copystructure
.
Copy
(
nv
.
Config
);
err
!=
nil
{
...
...
@@ -104,13 +117,13 @@ func (v *Volume) Copy() *Volume {
return
nv
}
func
CopyMapVolume
s
(
s
map
[
string
]
*
Volume
)
map
[
string
]
*
Volume
{
func
CopyMapVolume
Request
(
s
map
[
string
]
*
Volume
Request
)
map
[
string
]
*
Volume
Request
{
if
s
==
nil
{
return
nil
}
l
:=
len
(
s
)
c
:=
make
(
map
[
string
]
*
Volume
,
l
)
c
:=
make
(
map
[
string
]
*
Volume
Request
,
l
)
for
k
,
v
:=
range
s
{
c
[
k
]
=
v
.
Copy
()
}
...
...
@@ -147,39 +160,6 @@ func CopySliceVolumeMount(s []*VolumeMount) []*VolumeMount {
return
c
}
type
VolumeRequest
struct
{
Volume
*
Volume
Config
map
[
string
]
interface
{}
}
func
(
h
*
VolumeRequest
)
Copy
()
*
VolumeRequest
{
if
h
==
nil
{
return
nil
}
c
:=
new
(
VolumeRequest
)
c
.
Volume
=
h
.
Volume
.
Copy
()
if
i
,
err
:=
copystructure
.
Copy
(
h
.
Config
);
err
!=
nil
{
panic
(
err
.
Error
())
}
else
{
c
.
Config
=
i
.
(
map
[
string
]
interface
{})
}
return
c
}
func
CopyMapVolumeRequest
(
m
map
[
string
]
*
VolumeRequest
)
map
[
string
]
*
VolumeRequest
{
if
m
==
nil
{
return
nil
}
l
:=
len
(
m
)
c
:=
make
(
map
[
string
]
*
VolumeRequest
,
l
)
for
k
,
v
:=
range
m
{
c
[
k
]
=
v
.
Copy
()
}
return
c
}
func
ParseHostVolumeConfig
(
m
map
[
string
]
interface
{})
(
*
HostVolumeConfig
,
error
)
{
var
c
HostVolumeConfig
err
:=
mapstructure
.
Decode
(
m
,
&
c
)
...
...
This diff is collapsed.
Click to expand it.
scheduler/feasible.go
+
13
-
5
View file @
af5d42c0
...
...
@@ -99,7 +99,10 @@ func NewRandomIterator(ctx Context, nodes []*structs.Node) *StaticIterator {
// HostVolumeChecker is a FeasibilityChecker which returns whether a node has
// the host volumes necessary to schedule a task group.
type
HostVolumeChecker
struct
{
ctx
Context
ctx
Context
// volumes is a map[HostVolumeName][]RequestedVolume. The requested volumes are
// a slice because a single task group may request the same volume multiple times.
volumes
map
[
string
][]
*
structs
.
VolumeRequest
}
...
...
@@ -117,7 +120,7 @@ func (h *HostVolumeChecker) SetVolumes(volumes map[string]*structs.VolumeRequest
// Convert the map from map[DesiredName]Request to map[Source][]Request to improve
// lookup performance. Also filter non-host volumes.
for
_
,
req
:=
range
volumes
{
if
req
.
Volume
.
Type
!=
structs
.
VolumeTypeHost
{
if
req
.
Type
!=
structs
.
VolumeTypeHost
{
continue
}
...
...
@@ -142,11 +145,16 @@ func (h *HostVolumeChecker) Feasible(candidate *structs.Node) bool {
}
func
(
h
*
HostVolumeChecker
)
hasVolumes
(
n
*
structs
.
Node
)
bool
{
hLen
:=
len
(
h
.
volumes
)
nLen
:=
len
(
n
.
HostVolumes
)
rLen
:=
len
(
h
.
volumes
)
hLen
:=
len
(
n
.
HostVolumes
)
// Fast path: Requested no volumes. No need to check further.
if
rLen
==
0
{
return
true
}
// Fast path: Requesting more volumes than the node has, can't meet the criteria.
if
h
Len
>
n
Len
{
if
r
Len
>
h
Len
{
return
false
}
...
...
This diff is collapsed.
Click to expand it.
scheduler/feasible_test.go
+
3
-
3
View file @
af5d42c0
...
...
@@ -109,15 +109,15 @@ func TestHostVolumeChecker(t *testing.T) {
volumes
:=
map
[
string
]
*
structs
.
VolumeRequest
{
"foo"
:
{
Volume
:
&
structs
.
Volume
{
Type
:
"host"
}
,
Type
:
"host"
,
Config
:
map
[
string
]
interface
{}{
"source"
:
"foo"
},
},
"bar"
:
{
Volume
:
&
structs
.
Volume
{
Type
:
"host"
}
,
Type
:
"host"
,
Config
:
map
[
string
]
interface
{}{
"source"
:
"bar"
},
},
"baz"
:
{
Volume
:
&
structs
.
Volume
{
Type
:
"nothost"
}
,
Type
:
"nothost"
,
Config
:
map
[
string
]
interface
{}{
"source"
:
"baz"
},
},
}
...
...
This diff is collapsed.
Click to expand it.
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment
Menu
Projects
Groups
Snippets
Help