Allow template to set Vault grace

This PR allows a template to specify the Vault grace duration.

Fixes https://github.com/hashicorp/nomad/issues/2922

api/jobs_test.go

@@ -275,6 +275,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 										EmbeddedTmpl: helper.StringToPtr("FOO=bar\n"),
 										DestPath:     helper.StringToPtr("local/file.env"),
 										Envvars:      helper.BoolToPtr(true),
+										VaultGrace:   helper.TimeToPtr(3 * time.Second),
 									},
 								},
 							},
@@ -389,6 +390,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 										LeftDelim:    helper.StringToPtr("{{"),
 										RightDelim:   helper.StringToPtr("}}"),
 										Envvars:      helper.BoolToPtr(false),
+										VaultGrace:   helper.TimeToPtr(5 * time.Minute),
 									},
 									{
 										SourcePath:   helper.StringToPtr(""),
@@ -401,6 +403,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 										LeftDelim:    helper.StringToPtr("{{"),
 										RightDelim:   helper.StringToPtr("}}"),
 										Envvars:      helper.BoolToPtr(true),
+										VaultGrace:   helper.TimeToPtr(3 * time.Second),
 									},
 								},
 							},

api/tasks.go

@@ -364,6 +364,7 @@ type Template struct {
 	LeftDelim    *string        `mapstructure:"left_delimiter"`
 	RightDelim   *string        `mapstructure:"right_delimiter"`
 	Envvars      *bool          `mapstructure:"env"`
+	VaultGrace   *time.Duration `mapstructure:"vault_grace"`
 }
 
 func (tmpl *Template) Canonicalize() {
@@ -404,6 +405,9 @@ func (tmpl *Template) Canonicalize() {
 	if tmpl.Envvars == nil {
 		tmpl.Envvars = helper.BoolToPtr(false)
 	}
+	if tmpl.VaultGrace == nil {
+		tmpl.VaultGrace = helper.TimeToPtr(5 * time.Minute)
+	}
 }
 
 type Vault struct {

client/consul_template.go

@@ -17,6 +17,7 @@ import (
 	multierror "github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/driver/env"
+	"github.com/hashicorp/nomad/helper"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
@@ -341,11 +342,6 @@ func templateRunner(tmpls []*structs.Template, config *config.Config,
 		return nil, nil, nil
 	}
 
-	runnerConfig, err := runnerConfig(config, vaultToken)
-	if err != nil {
-		return nil, nil, err
-	}
-
 	// Parse the templates
 	allowAbs := config.ReadBoolDefault(hostSrcOption, true)
 	ctmplMapping, err := parseTemplateConfigs(tmpls, taskDir, taskEnv, allowAbs)
@@ -353,13 +349,11 @@ func templateRunner(tmpls []*structs.Template, config *config.Config,
 		return nil, nil, err
 	}
 
-	// Set the config
-	flat := ctconf.TemplateConfigs(make([]*ctconf.TemplateConfig, 0, len(ctmplMapping)))
-	for ctmpl := range ctmplMapping {
-		local := ctmpl
-		flat = append(flat, &local)
+	// Create the runner configuration.
+	runnerConfig, err := runnerConfig(config, vaultToken, ctmplMapping)
+	if err != nil {
+		return nil, nil, err
 	}
-	runnerConfig.Templates = &flat
 
 	runner, err := manager.NewRunner(runnerConfig, false, false)
 	if err != nil {
@@ -430,11 +424,32 @@ func parseTemplateConfigs(tmpls []*structs.Template, taskDir string,
 }
 
 // runnerConfig returns a consul-template runner configuration, setting the
-// Vault and Consul configurations based on the clients configs.
-func runnerConfig(config *config.Config, vaultToken string) (*ctconf.Config, error) {
+// Vault and Consul configurations based on the clients configs. The parameters
+// are the client config, Vault token if set and the mapping of consul-templates
+// to Nomad templates.
+func runnerConfig(config *config.Config, vaultToken string,
+	templateMapping map[ctconf.TemplateConfig]*structs.Template) (*ctconf.Config, error) {
+
 	conf := ctconf.DefaultConfig()
 
-	t, f := true, false
+	// Gather the consul-template tempates
+	flat := ctconf.TemplateConfigs(make([]*ctconf.TemplateConfig, 0, len(templateMapping)))
+	for ctmpl := range templateMapping {
+		local := ctmpl
+		flat = append(flat, &local)
+	}
+	conf.Templates = &flat
+
+	// Go through the templates and determine the minimum Vault grace
+	vaultGrace := time.Duration(-1)
+	for _, tmpl := range templateMapping {
+		// Initial condition
+		if vaultGrace < 0 {
+			vaultGrace = tmpl.VaultGrace
+		} else if tmpl.VaultGrace < vaultGrace {
+			vaultGrace = tmpl.VaultGrace
+		}
+	}
 
 	// Force faster retries
 	if testRetryRate != 0 {
@@ -450,7 +465,7 @@ func runnerConfig(config *config.Config, vaultToken string) (*ctconf.Config, err
 		if config.ConsulConfig.EnableSSL != nil && *config.ConsulConfig.EnableSSL {
 			verify := config.ConsulConfig.VerifySSL != nil && *config.ConsulConfig.VerifySSL
 			conf.Consul.SSL = &ctconf.SSLConfig{
-				Enabled: &t,
+				Enabled: helper.BoolToPtr(true),
 				Verify:  &verify,
 				Cert:    &config.ConsulConfig.CertFile,
 				Key:     &config.ConsulConfig.KeyFile,
@@ -465,7 +480,7 @@ func runnerConfig(config *config.Config, vaultToken string) (*ctconf.Config, err
 			}
 
 			conf.Consul.Auth = &ctconf.AuthConfig{
-				Enabled:  &t,
+				Enabled:  helper.BoolToPtr(true),
 				Username: &parts[0],
 				Password: &parts[1],
 			}
@@ -475,18 +490,18 @@ func runnerConfig(config *config.Config, vaultToken string) (*ctconf.Config, err
 	// Setup the Vault config
 	// Always set these to ensure nothing is picked up from the environment
 	emptyStr := ""
-	conf.Vault.RenewToken = &f
+	conf.Vault.RenewToken = helper.BoolToPtr(false)
 	conf.Vault.Token = &emptyStr
 	if config.VaultConfig != nil && config.VaultConfig.IsEnabled() {
 		conf.Vault.Address = &config.VaultConfig.Addr
 		conf.Vault.Token = &vaultToken
-		// XXX
+		conf.Vault.Grace = helper.TimeToPtr(vaultGrace)
 
 		if strings.HasPrefix(config.VaultConfig.Addr, "https") || config.VaultConfig.TLSCertFile != "" {
 			skipVerify := config.VaultConfig.TLSSkipVerify != nil && *config.VaultConfig.TLSSkipVerify
 			verify := !skipVerify
 			conf.Vault.SSL = &ctconf.SSLConfig{
-				Enabled:    &t,
+				Enabled:    helper.BoolToPtr(true),
 				Verify:     &verify,
 				Cert:       &config.VaultConfig.TLSCertFile,
 				Key:        &config.VaultConfig.TLSKeyFile,
@@ -496,8 +511,8 @@ func runnerConfig(config *config.Config, vaultToken string) (*ctconf.Config, err
 			}
 		} else {
 			conf.Vault.SSL = &ctconf.SSLConfig{
-				Enabled:    &f,
-				Verify:     &f,
+				Enabled:    helper.BoolToPtr(false),
+				Verify:     helper.BoolToPtr(false),
 				Cert:       &emptyStr,
 				Key:        &emptyStr,
 				CaCert:     &emptyStr,

client/consul_template_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/hashicorp/nomad/nomad/structs"
 	sconfig "github.com/hashicorp/nomad/nomad/structs/config"
 	"github.com/hashicorp/nomad/testutil"
+	"github.com/stretchr/testify/assert"
 )
 
 const (
@@ -1062,7 +1063,7 @@ func TestTaskTemplateManager_Config_ServerName(t *testing.T) {
 		Addr:          "https://localhost/",
 		TLSServerName: "notlocalhost",
 	}
-	ctconf, err := runnerConfig(c, "token")
+	ctconf, err := runnerConfig(c, "token", nil)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -1071,3 +1072,41 @@ func TestTaskTemplateManager_Config_ServerName(t *testing.T) {
 		t.Fatalf("expected %q but found %q", c.VaultConfig.TLSServerName, *ctconf.Vault.SSL.ServerName)
 	}
 }
+
+// TestTaskTemplateManager_Config_VaultGrace asserts the vault_grace setting is
+// propogated to consul-template's configuration.
+func TestTaskTemplateManager_Config_VaultGrace(t *testing.T) {
+	t.Parallel()
+	assert := assert.New(t)
+	c := config.DefaultConfig()
+	c.VaultConfig = &sconfig.VaultConfig{
+		Enabled:       helper.BoolToPtr(true),
+		Addr:          "https://localhost/",
+		TLSServerName: "notlocalhost",
+	}
+
+	// Make a template that will render immediately
+	templates := []*structs.Template{
+		{
+			EmbeddedTmpl: "bar",
+			DestPath:     "foo",
+			ChangeMode:   structs.TemplateChangeModeNoop,
+			VaultGrace:   10 * time.Second,
+		},
+		{
+			EmbeddedTmpl: "baz",
+			DestPath:     "bam",
+			ChangeMode:   structs.TemplateChangeModeNoop,
+			VaultGrace:   100 * time.Second,
+		},
+	}
+
+	taskEnv := env.NewTaskEnv(nil, nil)
+	ctmplMapping, err := parseTemplateConfigs(templates, "/fake/dir", taskEnv, false)
+	assert.Nil(err, "Parsing Templates")
+
+	ctconf, err := runnerConfig(c, "token", ctmplMapping)
+	assert.Nil(err, "Building Runner Config")
+	assert.NotNil(ctconf.Vault.Grace, "Vault Grace Pointer")
+	assert.Equal(10*time.Second, *ctconf.Vault.Grace, "Vault Grace Value")
+}

command/agent/job_endpoint.go

@@ -781,6 +781,7 @@ func ApiTaskToStructsTask(apiTask *api.Task, structsTask *structs.Task) {
 				LeftDelim:    *template.LeftDelim,
 				RightDelim:   *template.RightDelim,
 				Envvars:      *template.Envvars,
+				VaultGrace:   *template.VaultGrace,
 			}
 		}
 	}

command/agent/job_endpoint_test.go

@@ -1170,6 +1170,8 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 								Perms:        helper.StringToPtr("666"),
 								LeftDelim:    helper.StringToPtr("abc"),
 								RightDelim:   helper.StringToPtr("def"),
+								Envvars:      helper.BoolToPtr(true),
+								VaultGrace:   helper.TimeToPtr(3 * time.Second),
 							},
 						},
 						DispatchPayload: &api.DispatchPayloadConfig{
@@ -1357,6 +1359,8 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) {
 								Perms:        "666",
 								LeftDelim:    "abc",
 								RightDelim:   "def",
+								Envvars:      true,
+								VaultGrace:   3 * time.Second,
 							},
 						},
 						DispatchPayload: &structs.DispatchPayloadConfig{

jobspec/parse.go

@@ -863,6 +863,7 @@ func parseTemplates(result *[]*api.Template, list *ast.ObjectList) error {
 			"source",
 			"splay",
 			"env",
+			"vault_grace",
 		}
 		if err := checkHCLKeys(o.Val, valid); err != nil {
 			return err

jobspec/parse_test.go

@@ -183,6 +183,7 @@ func TestParse(t *testing.T) {
 										Splay:        helper.TimeToPtr(10 * time.Second),
 										Perms:        helper.StringToPtr("0644"),
 										Envvars:      helper.BoolToPtr(true),
+										VaultGrace:   helper.TimeToPtr(33 * time.Second),
 									},
 									{
 										SourcePath: helper.StringToPtr("bar"),

jobspec/test-fixtures/basic.hcl

@@ -158,6 +158,7 @@ job "binstore-storagelocker" {
         change_signal = "foo"
         splay = "10s"
         env = true
+        vault_grace = "33s"
       }
 
       template {

nomad/structs/structs.go

@@ -3287,6 +3287,11 @@ type Template struct {
 	// Empty lines and lines starting with # will be ignored, but to avoid
 	// escaping issues #s within lines will not be treated as comments.
 	Envvars bool
+
+	// VaultGrace is the grace duration between lease renewal and reacquiring a
+	// secret. If the lease of a secret is less than the grace, a new secret is
+	// acquired.
+	VaultGrace time.Duration
 }
 
 // DefaultTemplate returns a default template.
@@ -3360,6 +3365,10 @@ func (t *Template) Validate() error {
 		}
 	}
 
+	if t.VaultGrace.Nanoseconds() < 0 {
+		multierror.Append(&mErr, fmt.Errorf("Vault grace must be greater than zero: %v < 0", t.VaultGrace))
+	}
+
 	return mErr.ErrorOrNil()
 }
 

website/source/api/json-jobs.html.md

@@ -734,6 +734,14 @@ README][ct].
   splay value before invoking the change mode. Should be specified in
   nanoseconds.
 
+- `VaultGrace` - Specifies the grace period between lease renewal and secret
+  re-acquisition. When renewing a secret, if the remaining lease is less than or
+  equal to the configured grace, the template will request a new credential.
+  This prevents Vault from revoking the secret at its expiration and the task
+  having a stale secret. If the grace is set to a value that is higher than your
+  default TTL or max TTL, the template will always read a new secret. If the
+  task defines several templates, the `vault_grace` will be set to the lowest
+  value across all the templates.
 
 ```json
 {

website/source/docs/job-specification/template.html.md

@@ -94,6 +94,15 @@ README][ct]. Since Nomad v0.6.0, templates can be read as environment variables.
   prevent a thundering herd problem where all task instances restart at the same
   time.
 
+- `vault_grace` `(string: "5m")` - Specifies the grace period between lease
+  renewal and secret re-acquisition. When renewing a secret, if the remaining
+  lease is less than or equal to the configured grace, the template will request
+  a new credential. This prevents Vault from revoking the secret at its
+  expiration and the task having a stale secret. If the grace is set to a value
+  that is higher than your default TTL or max TTL, the template will always read
+  a new secret. If the task defines several templates, the `vault_grace` will be
+  set to the lowest value across all the templates.
+
 ## `template` Examples
 
 The following examples only show the `template` stanzas. Remember that the