Merge branch 'master' into external-group-creation

CHANGELOG.md

@@ -32,6 +32,7 @@ BUG FIXES:
 * secrets/gcp: Ensure that the IAM policy version is appropriately set after a roleset's bindings have changed. [[GH-9603](https://github.com/hashicorp/vault/pull/9603)]
 * replication (enterprise): Fix status API output incorrectly stating replication is in `idle` state.
 * core: Fix panic when printing over-long info fields at startup [[GH-9681](https://github.com/hashicorp/vault/pull/9681)]
+* core: Seal migration using the new minimal-downtime strategy didn't work properly with performance standbys. [[GH-9690](https://github.com/hashicorp/vault/pull/9690)]
 
 ## 1.5.0
 ### July 21st, 2020
@@ -72,6 +73,7 @@ IMPROVEMENTS:
 * audit: Replication status requests are no longer audited. [[GH-8877](https://github.com/hashicorp/vault/pull/8877)]
 * audit: Added mount_type field to requests and responses. [[GH-9167](https://github.com/hashicorp/vault/pull/9167)]
 * auth/aws: Add support for Web Identity credentials [[GH-7738](https://github.com/hashicorp/vault/pull/7738)]
+* auth/aws: Retry on transient failures during AWS IAM auth login attempts [[GH-8727](https://github.com/hashicorp/vault/pull/8727)]
 * auth/jwt: Support users that are members of more than 200 groups on Azure [[GH-120](https://github.com/hashicorp/vault-plugin-auth-jwt/pull/120)]
 * auth/kerberos: Support identities without userPrincipalName [[GH-44](https://github.com/hashicorp/vault-plugin-auth-kerberos/issues/44)]
 * auth/kubernetes: Allow disabling `iss` validation [[GH-91](https://github.com/hashicorp/vault-plugin-auth-kubernetes/pull/91)]
@@ -124,6 +126,7 @@ BUG FIXES:
 BUG FIXES:
 
 * seal/awskms: fix AWS KMS auto-unseal when AWS_ROLE_SESSION_NAME not set [[GH-9416](https://github.com/hashicorp/vault/pull/9416)]
+* auth/okta: fix bug introduced in 1.4.0 that broke handling of external groups with > 200 members [[GH-9580](https://github.com/hashicorp/vault/pull/9580)]
 
 IMPROVEMENTS:
 * ui: Add transit key algorithms aes128-gcm96, ecdsa-p384, ecdsa-p521 to the UI. [[GH-9070](https://github.com/hashicorp/vault/pull/9070)] & [[GH-9520](https://github.com/hashicorp/vault/pull/9520)]

command/agent.go

@@ -349,6 +349,7 @@ func (c *AgentCommand) Run(args []string) int {
 					Client:  client,
 					WrapTTL: sc.WrapTTL,
 					DHType:  sc.DHType,
+					DeriveKey: sc.DeriveKey,
 					DHPath:  sc.DHPath,
 					AAD:     sc.AAD,
 				}

command/agent/cert_with_name_end_to_end_test.go

@@ -11,6 +11,9 @@ import (
 	hclog "github.com/hashicorp/go-hclog"
 
 	"github.com/hashicorp/vault/api"
+	"github.com/hashicorp/vault/sdk/helper/jsonutil"
+	"github.com/hashicorp/vault/sdk/helper/logging"
+	"github.com/hashicorp/vault/sdk/logical"
 	vaultcert "github.com/hashicorp/vault/builtin/credential/cert"
 	"github.com/hashicorp/vault/command/agent/auth"
 	agentcert "github.com/hashicorp/vault/command/agent/auth/cert"
@@ -18,9 +21,6 @@ import (
 	"github.com/hashicorp/vault/command/agent/sink/file"
 	"github.com/hashicorp/vault/helper/dhutil"
 	vaulthttp "github.com/hashicorp/vault/http"
-	"github.com/hashicorp/vault/sdk/helper/jsonutil"
-	"github.com/hashicorp/vault/sdk/helper/logging"
-	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/hashicorp/vault/vault"
 )
 
@@ -137,6 +137,7 @@ func testCertWithNameEndToEnd(t *testing.T, ahWrapping bool) {
 		AAD:    "foobar",
 		DHType: "curve25519",
 		DHPath: dhpath,
+		DeriveKey: true,
 		Config: map[string]interface{}{
 			"path": out,
 		},
@@ -186,14 +187,17 @@ func testCertWithNameEndToEnd(t *testing.T, ahWrapping bool) {
 					continue
 				}
 
-				aesKey, err := dhutil.GenerateSharedKey(pri, resp.Curve25519PublicKey)
+				shared, err := dhutil.GenerateSharedSecret(pri, resp.Curve25519PublicKey)
+				if err != nil {
+					t.Fatal(err)
+				}
+				aesKey, err := dhutil.DeriveSharedKey(shared, pub, resp.Curve25519PublicKey)
 				if err != nil {
 					t.Fatal(err)
 				}
 				if len(aesKey) == 0 {
 					t.Fatal("got empty aes key")
 				}
-
 				val, err = dhutil.DecryptAES(aesKey, resp.EncryptedPayload, resp.Nonce, []byte("foobar"))
 				if err != nil {
 					t.Fatalf("error: %v\nresp: %v", err, string(val))

command/agent/cert_with_no_name_end_to_end_test.go

@@ -183,7 +183,7 @@ func testCertWithNoNAmeEndToEnd(t *testing.T, ahWrapping bool) {
 					continue
 				}
 
-				aesKey, err := dhutil.GenerateSharedKey(pri, resp.Curve25519PublicKey)
+				aesKey, err := dhutil.GenerateSharedSecret(pri, resp.Curve25519PublicKey)
 				if err != nil {
 					t.Fatal(err)
 				}

command/agent/config/config.go

@@ -75,6 +75,7 @@ type Sink struct {
 	WrapTTLRaw interface{}   `hcl:"wrap_ttl"`
 	WrapTTL    time.Duration `hcl:"-"`
 	DHType     string        `hcl:"dh_type"`
+	DeriveKey  bool          `hcl:"derive_key"`
 	DHPath     string        `hcl:"dh_path"`
 	AAD        string        `hcl:"aad"`
 	AADEnvVar  string        `hcl:"aad_env_var"`
@@ -395,6 +396,9 @@ func parseSinks(result *Config, list *ast.ObjectList) error {
 			if s.AAD != "" {
 				return multierror.Prefix(errors.New("specifying AAD data without 'dh_type' does not make sense"), fmt.Sprintf("sink.%s", s.Type))
 			}
+			if s.DeriveKey {
+				return multierror.Prefix(errors.New("specifying 'derive_key' data without 'dh_type' does not make sense"), fmt.Sprintf("sink.%s", s.Type))
+			}
 		case s.DHPath != "" && s.DHType != "":
 		default:
 			return multierror.Prefix(errors.New("'dh_type' and 'dh_path' must be specified together"), fmt.Sprintf("sink.%s", s.Type))

command/agent/config/config_test.go

@@ -143,6 +143,7 @@ func TestLoadConfigFile(t *testing.T) {
 					DHType:  "curve25519",
 					DHPath:  "/tmp/file-foo-dhpath2",
 					AAD:     "aad",
+					DeriveKey: true,
 					Config: map[string]interface{}{
 						"path": "/tmp/file-bar",
 					},

command/agent/config/test-fixtures/config-embedded-type.hcl

@@ -23,6 +23,7 @@ auto_auth {
 		aad_env_var = "TEST_AAD_ENV"
 		dh_type = "curve25519"
 		dh_path = "/tmp/file-foo-dhpath2"
+		derive_key = true
 		config = {
 			path = "/tmp/file-bar"
 		}

command/agent/config/test-fixtures/config.hcl

@@ -25,6 +25,7 @@ auto_auth {
 		aad_env_var = "TEST_AAD_ENV"
 		dh_type = "curve25519"
 		dh_path = "/tmp/file-foo-dhpath2"
+		derive_key = true
 		config = {
 			path = "/tmp/file-bar"
 		}

command/agent/jwt_end_to_end_test.go

@@ -158,6 +158,7 @@ func testJWTEndToEnd(t *testing.T, ahWrapping bool) {
 		AAD:    "foobar",
 		DHType: "curve25519",
 		DHPath: dhpath,
+		DeriveKey: true,
 		Config: map[string]interface{}{
 			"path": out,
 		},
@@ -231,7 +232,11 @@ func testJWTEndToEnd(t *testing.T, ahWrapping bool) {
 					continue
 				}
 
-				aesKey, err := dhutil.GenerateSharedKey(pri, resp.Curve25519PublicKey)
+				shared, err := dhutil.GenerateSharedSecret(pri, resp.Curve25519PublicKey)
+				if err != nil {
+					t.Fatal(err)
+				}
+				aesKey, err := dhutil.DeriveSharedKey(shared, pub, resp.Curve25519PublicKey)
 				if err != nil {
 					t.Fatal(err)
 				}

command/agent/sink/sink.go

@@ -32,6 +32,7 @@ type SinkConfig struct {
 	WrapTTL            time.Duration
 	DHType             string
 	DHPath             string
+	DeriveKey          bool
 	AAD                string
 	cachedRemotePubKey []byte
 	cachedPubKey       []byte
@@ -205,7 +206,16 @@ func (s *SinkConfig) encryptToken(token string) (string, error) {
 		resp.Curve25519PublicKey = s.cachedPubKey
 	}
 
-	aesKey, err = dhutil.GenerateSharedKey(s.cachedPriKey, s.cachedRemotePubKey)
+	secret, err := dhutil.GenerateSharedSecret(s.cachedPriKey, s.cachedRemotePubKey)
+	if err != nil {
+		return "", errwrap.Wrapf("error calculating shared key: {{err}}", err)
+	}
+	if s.DeriveKey {
+		aesKey, err = dhutil.DeriveSharedKey(secret, s.cachedPubKey, s.cachedRemotePubKey)
+	} else {
+		aesKey = secret
+	}
+
 	if err != nil {
 		return "", errwrap.Wrapf("error deriving shared key: {{err}}", err)
 	}

helper/dhutil/dhutil.go

 package dhutil
 
 import (
+	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"fmt"
+	"golang.org/x/crypto/hkdf"
 	"io"
 
 	"golang.org/x/crypto/curve25519"
@@ -34,9 +38,9 @@ func GeneratePublicPrivateKey() ([]byte, []byte, error) {
 	return public[:], scalar[:], nil
 }
 
-// generateSharedKey uses the private key and the other party's public key to
+// GenerateSharedSecret uses the private key and the other party's public key to
 // generate the shared secret.
-func GenerateSharedKey(ourPrivate, theirPublic []byte) ([]byte, error) {
+func GenerateSharedSecret(ourPrivate, theirPublic []byte) ([]byte, error) {
 	if len(ourPrivate) != 32 {
 		return nil, fmt.Errorf("invalid private key length: %d", len(ourPrivate))
 	}
@@ -44,13 +48,50 @@ func GenerateSharedKey(ourPrivate, theirPublic []byte) ([]byte, error) {
 		return nil, fmt.Errorf("invalid public key length: %d", len(theirPublic))
 	}
 
-	var scalar, pub, secret [32]byte
-	copy(scalar[:], ourPrivate)
-	copy(pub[:], theirPublic)
+	return curve25519.X25519(ourPrivate, theirPublic)
+}
+
+// DeriveSharedKey uses HKDF to derive a key from a shared secret and public keys
+func DeriveSharedKey(secret, ourPublic, theirPublic []byte) ([]byte, error) {
+	// Derive the final key from the HKDF of the secret and public keys.
+
+/*
+	Internally, HKDF hashes the secret and two public keys. If Alice and Bob are doing DH key exchange, Alice calculates:
+
+	HKDF(secret, A, B) since ourPublic is A.
+
+		Bob calculates HKDF(secret, B, A), since Bob's ours is B. That produces a different value. Now we only care
+	    that both public keys participate in the derivation, so simply sorting them so they are in a consistent
+	    numerical order (either one would do) arrives at an agreed value.
+*/
+
+	var pub1 []byte
+	var pub2 []byte
+	switch bytes.Compare(ourPublic, theirPublic) {
+	case 0:
+		return nil, errors.New("same public key supplied for both participants")
+	case -1:
+		pub1 = ourPublic
+		pub2 = theirPublic
+	case 1:
+		pub1 = theirPublic
+		pub2 = ourPublic
+	}
+
+	kio := hkdf.New(sha256.New, secret, pub1, pub2)
 
-	curve25519.ScalarMult(&secret, &scalar, &pub)
+	var key [32]byte
+	n, err := io.ReadFull(kio, key[:])
+	if err != nil {
+		// Don't return the key along with the error to prevent misuse
+		return nil, err
+	}
+	if n != 32 {
+		return nil, errors.New("short read from hkdf")
+	}
+	fmt.Printf("Key: %s\n", hex.EncodeToString(key[:]))
 
-	return secret[:], nil
+	return key[:], nil
 }
 
 // Use AES256-GCM to encrypt some plaintext with a provided key. The returned values are

helper/dhutil/dhutil_test.go

+package dhutil

vault/logical_system_quotas.go

@@ -73,6 +73,11 @@ The 'rate' must be positive.`,
 					Type:        framework.TypeDurationSecond,
 					Description: "The duration to enforce rate limiting for (default '1s').",
 				},
+				"block_interval": {
+					Type: framework.TypeDurationSecond,
+					Description: `If set, when a client reaches a rate limit threshold, the client will be prohibited
+from any further requests until after the 'block_interval' has elapsed.`,
+				},
 			},
 			Operations: map[logical.Operation]framework.OperationHandler{
 				logical.UpdateOperation: &framework.PathOperation{
@@ -154,6 +159,11 @@ func (b *SystemBackend) handleRateLimitQuotasUpdate() framework.OperationFunc {
 			interval = time.Second
 		}
 
+		blockInterval := time.Second * time.Duration(d.Get("block_interval").(int))
+		if blockInterval < 0 {
+			return logical.ErrorResponse("'block' is invalid"), nil
+		}
+
 		mountPath := sanitizePath(d.Get("path").(string))
 		ns := b.Core.namespaceByPath(mountPath)
 		if ns.ID != namespace.RootNamespaceID {
@@ -185,13 +195,14 @@ func (b *SystemBackend) handleRateLimitQuotasUpdate() framework.OperationFunc {
 				return logical.ErrorResponse("quota rule with similar properties exists under the name %q", quotaByFactors.QuotaName()), nil
 			}
 
-			quota = quotas.NewRateLimitQuota(name, ns.Path, mountPath, rate, interval)
+			quota = quotas.NewRateLimitQuota(name, ns.Path, mountPath, rate, interval, blockInterval)
 		default:
 			rlq := quota.(*quotas.RateLimitQuota)
 			rlq.NamespacePath = ns.Path
 			rlq.MountPath = mountPath
 			rlq.Rate = rate
 			rlq.Interval = interval
+			rlq.BlockInterval = blockInterval
 		}
 
 		entry, err := logical.StorageEntryJSON(quotas.QuotaStoragePath(qType, name), quota)
@@ -232,11 +243,12 @@ func (b *SystemBackend) handleRateLimitQuotasRead() framework.OperationFunc {
 		}
 
 		data := map[string]interface{}{
-			"type":     qType,
-			"name":     rlq.Name,
-			"path":     nsPath + rlq.MountPath,
-			"rate":     rlq.Rate,
-			"interval": int(rlq.Interval.Seconds()),
+			"type":           qType,
+			"name":           rlq.Name,
+			"path":           nsPath + rlq.MountPath,
+			"rate":           rlq.Rate,
+			"interval":       int(rlq.Interval.Seconds()),
+			"block_interval": int(rlq.BlockInterval.Seconds()),
 		}
 
 		return &logical.Response{

vault/quotas/quotas_rate_limit.go

@@ -73,18 +73,28 @@ type RateLimitQuota struct {
 	// Interval defines the duration to which rate limiting is applied.
 	Interval time.Duration `json:"interval"`
 
-	lock          *sync.RWMutex
-	store         limiter.Store
-	logger        log.Logger
-	metricSink    *metricsutil.ClusterMetricSink
-	purgeInterval time.Duration
-	staleAge      time.Duration
+	// BlockInterval defines the duration during which all requests are blocked for
+	// a given client. This interval is enforced only if non-zero and a client
+	// reaches the rate limit.
+	BlockInterval time.Duration `json:"block_interval"`
+
+	lock                *sync.RWMutex
+	store               limiter.Store
+	logger              log.Logger
+	metricSink          *metricsutil.ClusterMetricSink
+	purgeInterval       time.Duration
+	staleAge            time.Duration
+	blockedClients      sync.Map
+	purgeBlocked        bool
+	closePurgeBlockedCh chan struct{}
 }
 
 // NewRateLimitQuota creates a quota checker for imposing limits on the number
 // of requests in a given interval. An interval time duration of zero may be
-// provided, which will default to 1s when initialized.
-func NewRateLimitQuota(name, nsPath, mountPath string, rate float64, interval time.Duration) *RateLimitQuota {
+// provided, which will default to 1s when initialized. An optional block
+// duration may be provided, where if set, when a client reaches the rate limit,
+// subsequent requests will fail until the block duration has passed.
+func NewRateLimitQuota(name, nsPath, mountPath string, rate float64, interval, block time.Duration) *RateLimitQuota {
 	return &RateLimitQuota{
 		Name:          name,
 		Type:          TypeRateLimit,
@@ -92,6 +102,7 @@ func NewRateLimitQuota(name, nsPath, mountPath string, rate float64, interval ti
 		MountPath:     mountPath,
 		Rate:          rate,
 		Interval:      interval,
+		BlockInterval: block,
 		purgeInterval: DefaultRateLimitPurgeInterval,
 		staleAge:      DefaultRateLimitStaleAge,
 	}
@@ -119,7 +130,11 @@ func (rlq *RateLimitQuota) initialize(logger log.Logger, ms *metricsutil.Cluster
 	}
 
 	if rlq.Rate <= 0 {
-		return fmt.Errorf("invalid avg rps: %v", rlq.Rate)
+		return fmt.Errorf("invalid rate: %v", rlq.Rate)
+	}
+
+	if rlq.BlockInterval < 0 {
+		return fmt.Errorf("invalid block interval: %v", rlq.BlockInterval)
 	}
 
 	if logger != nil {
@@ -150,10 +165,73 @@ func (rlq *RateLimitQuota) initialize(logger log.Logger, ms *metricsutil.Cluster
 	}
 
 	rlq.store = rlStore
+	rlq.blockedClients = sync.Map{}
+
+	if rlq.BlockInterval > 0 && !rlq.purgeBlocked {
+		rlq.purgeBlocked = true
+		rlq.closePurgeBlockedCh = make(chan struct{})
+		go rlq.purgeBlockedClients()
+	}
 
 	return nil
 }
 
+// purgeBlockedClients performs a blocking process where every purgeInterval
+// duration, we look at all blocked clients to potentially remove from the blocked
+// clients map.
+//
+// A blocked client will only be removed if the current time minus the time the
+// client was blocked at is greater than or equal to the block duration. The loop
+// will continue to run indefinitely until a value is	sent on the closePurgeBlockedCh
+// in which we stop the ticker and return.
+func (rlq *RateLimitQuota) purgeBlockedClients() {
+	rlq.lock.RLock()
+	ticker := time.NewTicker(rlq.purgeInterval)
+	rlq.lock.RUnlock()
+
+	for {
+		select {
+		case t := <-ticker.C:
+			rlq.blockedClients.Range(func(key, value interface{}) bool {
+				blockedAt := value.(time.Time)
+				if t.Sub(blockedAt) >= rlq.BlockInterval {
+					rlq.blockedClients.Delete(key)
+				}
+
+				return true
+			})
+
+		case <-rlq.closePurgeBlockedCh:
+			ticker.Stop()
+
+			rlq.lock.Lock()
+			rlq.purgeBlocked = false
+			rlq.lock.Unlock()
+
+			return
+		}
+	}
+}
+
+func (rlq *RateLimitQuota) getPurgeBlocked() bool {
+	rlq.lock.RLock()
+	defer rlq.lock.RUnlock()
+	return rlq.purgeBlocked
+}
+
+func (rlq *RateLimitQuota) numBlockedClients() int {
+	rlq.lock.RLock()
+	defer rlq.lock.RUnlock()
+
+	size := 0
+	rlq.blockedClients.Range(func(_, _ interface{}) bool {
+		size++
+		return true
+	})
+
+	return size
+}
+
 // quotaID returns the identifier of the quota rule
 func (rlq *RateLimitQuota) quotaID() string {
 	return rlq.ID
@@ -169,7 +247,9 @@ func (rlq *RateLimitQuota) QuotaName() string {
 // quota will not be evaluated. Otherwise, the client rate limiter is retrieved
 // by address and the rate limit quota is checked against that limiter.
 func (rlq *RateLimitQuota) allow(req *Request) (Response, error) {
-	var resp Response
+	resp := Response{
+		Headers: make(map[string]string),
+	}
 
 	// Skip rate limit checks for paths that are exempt from rate limiting.
 	if rateLimitExemptPaths.HasPath(req.Path) {
@@ -181,17 +261,46 @@ func (rlq *RateLimitQuota) allow(req *Request) (Response, error) {
 		return resp, fmt.Errorf("missing request client address in quota request")
 	}
 
-	limit, remaining, reset, allow := rlq.store.Take(req.ClientAddress)
-	resp.Allowed = allow
-	resp.Headers = map[string]string{
-		httplimit.HeaderRateLimitLimit:     strconv.FormatUint(limit, 10),
-		httplimit.HeaderRateLimitRemaining: strconv.FormatUint(remaining, 10),
-		httplimit.HeaderRateLimitReset:     time.Unix(0, int64(reset)).UTC().Format(time.RFC1123),
+	var retryAfter string
+
+	defer func() {
+		if !resp.Allowed {
+			resp.Headers[httplimit.HeaderRetryAfter] = retryAfter
+			rlq.metricSink.IncrCounterWithLabels([]string{"quota", "rate_limit", "violation"}, 1, []metrics.Label{{"name", rlq.Name}})
+		}
+	}()
+
+	// Check if the client is currently blocked and if so, deny the request. Note,
+	// we cannot simply rely on the presence of the client in the map as the timing
+	// of purging blocked clients may not yield a false negative. In other words,
+	// a client may no longer be considered blocked whereas the purging interval
+	// has yet to run.
+	if v, ok := rlq.blockedClients.Load(req.ClientAddress); ok {
+		blockedAt := v.(time.Time)
+		if time.Since(blockedAt) >= rlq.BlockInterval {
+			// allow the request and remove the blocked client
+			rlq.blockedClients.Delete(req.ClientAddress)
+		} else {
+			// deny the request and return early
+			resp.Allowed = false
+			retryAfter = strconv.Itoa(int(time.Until(blockedAt.Add(rlq.BlockInterval)).Seconds()))
+			return resp, nil
+		}
 	}
 
-	if !resp.Allowed {
-		resp.Headers[httplimit.HeaderRetryAfter] = resp.Headers[httplimit.HeaderRateLimitReset]
-		rlq.metricSink.IncrCounterWithLabels([]string{"quota", "rate_limit", "violation"}, 1, []metrics.Label{{"name", rlq.Name}})
+	limit, remaining, reset, allow := rlq.store.Take(req.ClientAddress)
+	resp.Allowed = allow
+	resp.Headers[httplimit.HeaderRateLimitLimit] = strconv.FormatUint(limit, 10)
+	resp.Headers[httplimit.HeaderRateLimitRemaining] = strconv.FormatUint(remaining, 10)
+	resp.Headers[httplimit.HeaderRateLimitReset] = strconv.Itoa(int(time.Until(time.Unix(0, int64(reset))).Seconds()))
+	retryAfter = resp.Headers[httplimit.HeaderRateLimitReset]
+
+	// If the request is not allowed (i.e. rate limit threshold reached) and blocking
+	// is enabled, we add the client to the set of blocked clients.
+	if !resp.Allowed && rlq.purgeBlocked {
+		blockedAt := time.Now()
+		retryAfter = strconv.Itoa(int(time.Until(blockedAt.Add(rlq.BlockInterval)).Seconds()))
+		rlq.blockedClients.Store(req.ClientAddress, blockedAt)
 	}
 
 	return resp, nil
@@ -199,6 +308,10 @@ func (rlq *RateLimitQuota) allow(req *Request) (Response, error) {
 
 // close stops the current running client purge loop.
 func (rlq *RateLimitQuota) close() error {
+	if rlq.purgeBlocked {
+		close(rlq.closePurgeBlockedCh)
+	}
+
 	if rlq.store != nil {
 		return rlq.store.Close()
 	}

vault/quotas/quotas_rate_limit_test.go

@@ -14,13 +14,18 @@ import (
 	"go.uber.org/atomic"
 )
 
+type clientResult struct {
+	atomicNumAllow *atomic.Int32
+	atomicNumFail  *atomic.Int32
+}
+
 func TestNewRateLimitQuota(t *testing.T) {
 	testCases := []struct {
 		name      string
 		rlq       *RateLimitQuota
 		expectErr bool
 	}{
-		{"valid rate", NewRateLimitQuota("test-rate-limiter", "qa", "/foo/bar", 16.7, time.Second), false},
+		{"valid rate", NewRateLimitQuota("test-rate-limiter", "qa", "/foo/bar", 16.7, time.Second, 0), false},
 	}
 
 	for _, tc := range testCases {
@@ -34,9 +39,12 @@ func TestNewRateLimitQuota(t *testing.T) {
 }
 
 func TestRateLimitQuota_Close(t *testing.T) {
-	rlq := NewRateLimitQuota("test-rate-limiter", "qa", "/foo/bar", 16.7, time.Second)
+	rlq := NewRateLimitQuota("test-rate-limiter", "qa", "/foo/bar", 16.7, time.Second, time.Minute)
 	require.NoError(t, rlq.initialize(logging.NewVaultLogger(log.Trace), metricsutil.BlackholeSink()))
 	require.NoError(t, rlq.close())
+
+	time.Sleep(time.Second) // allow enough time for purgeClientsLoop to receive on closeCh
+	require.False(t, rlq.getPurgeBlocked(), "expected blocked client purging to be disabled after explicit close")
 }
 
 func TestRateLimitQuota_Allow(t *testing.T) {
@@ -56,11 +64,6 @@ func TestRateLimitQuota_Allow(t *testing.T) {
 
 	var wg sync.WaitGroup
 
-	type clientResult struct {
-		atomicNumAllow *atomic.Int32
-		atomicNumFail  *atomic.Int32
-	}
-
 	reqFunc := func(addr string, atomicNumAllow, atomicNumFail *atomic.Int32) {
 		defer wg.Done()
 
@@ -80,8 +83,8 @@ func TestRateLimitQuota_Allow(t *testing.T) {
 
 	start := time.Now()
 	end := start.Add(5 * time.Second)
-	for time.Now().Before(end) {
 
+	for time.Now().Before(end) {
 		for i := 0; i < 5; i++ {
 			wg.Add(1)
 
@@ -116,3 +119,88 @@ func TestRateLimitQuota_Allow(t *testing.T) {
 		require.Falsef(t, numAllow > want, "too many successful requests; addr: %s, want: %d, numSuccess: %d, numFail: %d, elapsed: %d", addr, want, numAllow, numFail, elapsed)
 	}
 }
+
+func TestRateLimitQuota_Allow_WithBlock(t *testing.T) {
+	rlq := &RateLimitQuota{
+		Name:          "test-rate-limiter",
+		Type:          TypeRateLimit,
+		NamespacePath: "qa",
+		MountPath:     "/foo/bar",
+		Rate:          16.7,
+		BlockInterval: 10 * time.Second,
+
+		// override values to lower durations for testing purposes
+		purgeInterval: 10 * time.Second,
+		staleAge:      10 * time.Second,
+	}
+
+	require.NoError(t, rlq.initialize(logging.NewVaultLogger(log.Trace), metricsutil.BlackholeSink()))
+	require.True(t, rlq.getPurgeBlocked())
+
+	var wg sync.WaitGroup
+
+	reqFunc := func(addr string, atomicNumAllow, atomicNumFail *atomic.Int32) {
+		defer wg.Done()
+
+		resp, err := rlq.allow(&Request{ClientAddress: addr})
+		if err != nil {
+			return
+		}
+
+		if resp.Allowed {
+			atomicNumAllow.Add(1)
+		} else {
+			atomicNumFail.Add(1)
+		}
+	}
+
+	results := make(map[string]*clientResult)
+
+	start := time.Now()
+	end := start.Add(5 * time.Second)
+
+	for time.Now().Before(end) {
+		for i := 0; i < 5; i++ {
+			wg.Add(1)
+
+			addr := fmt.Sprintf("127.0.0.%d", i)
+			cr, ok := results[addr]
+			if !ok {
+				results[addr] = &clientResult{atomicNumAllow: atomic.NewInt32(0), atomicNumFail: atomic.NewInt32(0)}
+				cr = results[addr]
+			}
+
+			go reqFunc(addr, cr.atomicNumAllow, cr.atomicNumFail)
+
+			time.Sleep(2 * time.Millisecond)
+		}
+	}
+
+	wg.Wait()
+
+	for _, cr := range results {
+		numAllow := cr.atomicNumAllow.Load()
+		numFail := cr.atomicNumFail.Load()
+
+		// Since blocking is enabled, each client should only have 'rate' successful
+		// requests, whereas all subsequent requests fail.
+		require.Equal(t, int32(17), numAllow)
+		require.NotZero(t, numFail)
+	}
+
+	func() {
+		timeout := time.After(rlq.purgeInterval * 2)
+		ticker := time.Tick(time.Second)
+		for {
+			select {
+			case <-timeout:
+				require.Failf(t, "timeout exceeded waiting for blocked clients to be purged", "num blocked: %d", rlq.numBlockedClients())
+
+			case <-ticker:
+				if rlq.numBlockedClients() == 0 {
+					return
+				}
+			}
+		}
+	}()
+}

vault/quotas/quotas_test.go

@@ -18,7 +18,7 @@ func TestQuotas_Precedence(t *testing.T) {
 
 	setQuotaFunc := func(t *testing.T, name, nsPath, mountPath string) Quota {
 		t.Helper()
-		quota := NewRateLimitQuota(name, nsPath, mountPath, 10, time.Second)
+		quota := NewRateLimitQuota(name, nsPath, mountPath, 10, time.Second, 0)
 		require.NoError(t, qm.SetQuota(context.Background(), TypeRateLimit.String(), quota, true))
 		return quota
 	}

website/pages/api-docs/secret/ssh/index.mdx

@@ -211,6 +211,11 @@ This endpoint creates or updates a named role.
 - `allowed_user_key_lengths` `(map<string|int>: "")` – Specifies a map of ssh key types
   and their expected sizes which are allowed to be signed by the CA type.
 
+- `algorithm_signer` `(string: "ssh-rsa")` - Algorithm to sign keys with.  Valid
+  values are `ssh-rsa`, `rsa-sha2-256`, and `rsa-sha2-512`.  Note that `ssh-rsa`
+  is now considered insecure and is not supported by current OpenSSH versions.
+  Defaults to `ssh-rsa` for backwards compatibility.
+
 ### Sample Payload
 
 ```json

website/pages/api-docs/secret/transit/index.mdx

@@ -891,9 +891,9 @@ supports signing.
   only for legacy applications. Signing using SHA-1 can be blocked by operators by
   assigning the following policy corresponding to a named key:
 
-  ```
-  {
-  ]
+  ```hcl
+  path "/transit/sign/:name/sha1" {
+    capabilities = ["deny"]
   }
   ```
 
@@ -1046,9 +1046,9 @@ data.
   be blocked by operators by assigning the following policy corresponding to a
   named key:
 
-  ```
-  {
-  ]
+  ```hcl
+  path "/transit/verify/:name/sha1" {
+    capabilities = ["deny"]
   }
   ```
 

website/pages/api-docs/system/rate-limit-quotas.mdx

@@ -32,6 +32,9 @@ either be a namespace or mount.
 - `rate` `(float: 0.0)` - The maximum number of requests in a given interval to
   be allowed by the quota rule. The `rate` must be positive.
 - `interval` `(string: "")` - The duration to enforce rate limiting for (default `"1s"`).
+- `block_interval` `(string: "")` - If set, when a client reaches a rate limit
+threshold, the client will be prohibited from any further requests until after
+the 'block_interval' has elapsed.
 
 ### Sample Payload
 
@@ -39,7 +42,8 @@ either be a namespace or mount.
 {
   "path": "",
   "rate": 897.3,
-  "interval": "2m"
+  "interval": "2m",
+  "block_interval": "5m"
 }
 ```
 
@@ -96,7 +100,8 @@ $ curl \
   "lease_duration": 0,
   "renewable": false,
   "data": {
-    "interval": "2m0s",
+    "block_interval": 300,
+    "interval": 2,
     "name": "global-rate-limiter",
     "path": "",
     "rate": 897.3,

website/pages/docs/agent/autoauth/index.mdx

@@ -155,6 +155,11 @@ These configuration values are common to all Sinks:
   agent should read the client's initial parameters (e.g. curve25519 public
   key).
 
+- `derive_key` `(bool: false)` - If specified, the final encryption key is
+  calculated by using HKDF-SHA256 to derive a key from the calculated shared
+  secret and the two public keys for enhanced security.  This is recommended
+  if backward compatibility isn't a concern.
+
 - `aad` `(string: optional)` - If specified, additional authenticated data to
   use with the AES-GCM encryption of the token. Can be any string, including
   serialized data.