Skip to content
GitLab
Unverified Commit b7fea76f authored by Tim Gross's avatar Tim Gross Committed by GitHub
Browse files

keyring: wrap root key in key encryption key (#14388) 

Update the on-disk format for the root key so that it's wrapped with a unique
per-key/per-server key encryption key. This is a bit of security theatre for the
current implementation, but it uses `go-kms-wrapping` as the interface for
wrapping the key. This provides a shim for future support of external KMS such
as cloud provider APIs or Vault transit encryption.

* Removes the JSON serialization extension we had on the `RootKey` struct; this
  struct is now only used for key replication and not for disk serialization, so
  we don't need this helper.

* Creates a helper for generating cryptographically random slices of bytes that
  properly accounts for short reads from the source.

* No observable functional changes outside of the on-disk format, so there are
  no test updates.
parent 8dfa6b06
Hide whitespace changes
Inline Side-by-side
Showing
with 126 additions and 93 deletions
+126 -93
helper/crypto/crypto.go 0 → 100644
+ 24
- 0
View file @ b7fea76f
package crypto
import (
"errors"
"fmt"
// note: this is aliased so that it's more noticeable if someone
// accidentally swaps it out for math/rand via running goimports
cryptorand "crypto/rand"
)
// Bytes gets a slice of cryptographically random bytes of the given length and
// enforces that we check for short reads to avoid entropy exhaustion.
func Bytes(length int) ([]byte, error) {
key := make([]byte, length)
n, err := cryptorand.Read(key)
if err != nil {
return nil, fmt.Errorf("could not read from random source: %v", err)
}
if n < length {
return nil, errors.New("entropy exhausted")
}
return key, nil
}
helper/uuid/uuid.go
+ 4
- 3
View file @ b7fea76f
package uuid
import (
"crypto/rand"
"fmt"
"github.com/hashicorp/nomad/helper/crypto"
)
// Generate is used to generate a random UUID.
func Generate() string {
buf := make([]byte, 16)
if _, err := rand.Read(buf); err != nil {
buf, err := crypto.Bytes(16)
if err != nil {
panic(fmt.Errorf("failed to read random bytes: %v", err))
}
......
nomad/encrypter.go
+ 73
- 53
View file @ b7fea76f
package nomad
import (
"bytes"
"context"
"crypto/aes"
"crypto/cipher"
"crypto/ed25519"
"encoding/base64"
"encoding/json"
"fmt"
"io/fs"
......@@ -16,16 +14,14 @@ import (
"sync"
"time"
// note: this is aliased so that it's more noticeable if someone
// accidentally swaps it out for math/rand via running goimports
cryptorand "crypto/rand"
jwt "github.com/golang-jwt/jwt/v4"
log "github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-msgpack/codec"
kms "github.com/hashicorp/go-kms-wrapping/v2"
"github.com/hashicorp/go-kms-wrapping/v2/aead"
"golang.org/x/time/rate"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/helper/crypto"
"github.com/hashicorp/nomad/nomad/structs"
)
......@@ -50,26 +46,27 @@ type keyset struct {
// NewEncrypter loads or creates a new local keystore and returns an
// encryption keyring with the keys it finds.
func NewEncrypter(srv *Server, keystorePath string) (*Encrypter, error) {
err := os.MkdirAll(keystorePath, 0700)
if err != nil {
return nil, err
encrypter := &Encrypter{
srv: srv,
keystorePath: keystorePath,
keyring: make(map[string]*keyset),
}
encrypter, err := encrypterFromKeystore(keystorePath)
err := encrypter.loadKeystore()
if err != nil {
return nil, err
}
encrypter.srv = srv
return encrypter, nil
}
func encrypterFromKeystore(keystoreDirectory string) (*Encrypter, error) {
func (e *Encrypter) loadKeystore() error {
encrypter := &Encrypter{
keyring: make(map[string]*keyset),
keystorePath: keystoreDirectory,
if err := os.MkdirAll(e.keystorePath, 0o700); err != nil {
return err
}
err := filepath.Walk(keystoreDirectory, func(path string, info fs.FileInfo, err error) error {
return filepath.Walk(e.keystorePath, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return fmt.Errorf("could not read path %s from keystore: %v", path, err)
}
......@@ -77,7 +74,7 @@ func encrypterFromKeystore(keystoreDirectory string) (*Encrypter, error) {
// skip over subdirectories and non-key files; they shouldn't
// be here but there's no reason to fail startup for it if the
// administrator has left something there
if path != keystoreDirectory && info.IsDir() {
if path != e.keystorePath && info.IsDir() {
return filepath.SkipDir
}
if !strings.HasSuffix(path, nomadKeystoreExtension) {
......@@ -88,7 +85,7 @@ func encrypterFromKeystore(keystoreDirectory string) (*Encrypter, error) {
return nil
}
key, err := encrypter.loadKeyFromStore(path)
key, err := e.loadKeyFromStore(path)
if err != nil {
return fmt.Errorf("could not load key file %s from keystore: %v", path, err)
}
......@@ -96,17 +93,12 @@ func encrypterFromKeystore(keystoreDirectory string) (*Encrypter, error) {
return fmt.Errorf("root key ID %s must match key file %s", key.Meta.KeyID, path)
}
err = encrypter.AddKey(key)
err = e.AddKey(key)
if err != nil {
return fmt.Errorf("could not add key file %s to keystore: %v", path, err)
}
return nil
})
if err != nil {
return nil, err
}
return encrypter, nil
}
// Encrypt encrypts the clear data with the cipher for the current
......@@ -121,14 +113,9 @@ func (e *Encrypter) Encrypt(cleartext []byte) ([]byte, string, error) {
return nil, "", err
}
nonceSize := keyset.cipher.NonceSize()
nonce := make([]byte, nonceSize)
n, err := cryptorand.Read(nonce)
nonce, err := crypto.Bytes(keyset.cipher.NonceSize())
if err != nil {
return nil, "", err
}
if n < nonceSize {
return nil, "", fmt.Errorf("failed to encrypt: entropy exhausted")
return nil, "", fmt.Errorf("failed to generate key wrapper nonce: %v", err)
}
keyID := keyset.rootKey.Meta.KeyID
......@@ -306,9 +293,6 @@ func (e *Encrypter) keysetByIDLocked(keyID string) (*keyset, error) {
// RemoveKey removes a key by ID from the keyring
func (e *Encrypter) RemoveKey(keyID string) error {
// TODO: should the server remove the serialized file here?
// TODO: given that it's irreversible, should the server *ever*
// remove the serialized file?
e.lock.Lock()
defer e.lock.Unlock()
delete(e.keyring, keyID)
......@@ -317,14 +301,33 @@ func (e *Encrypter) RemoveKey(keyID string) error {
// saveKeyToStore serializes a root key to the on-disk keystore.
func (e *Encrypter) saveKeyToStore(rootKey *structs.RootKey) error {
var buf bytes.Buffer
enc := codec.NewEncoder(&buf, structs.JsonHandleWithExtensions)
err := enc.Encode(rootKey)
kek, err := crypto.Bytes(32)
if err != nil {
return fmt.Errorf("failed to generate key wrapper key: %v", err)
}
wrapper, err := e.newKMSWrapper(rootKey.Meta.KeyID, kek)
if err != nil {
return fmt.Errorf("failed to create encryption wrapper: %v", err)
}
blob, err := wrapper.Encrypt(e.srv.shutdownCtx, rootKey.Key)
if err != nil {
return fmt.Errorf("failed to encrypt root key: %v", err)
}
kekWrapper := &structs.KeyEncryptionKeyWrapper{
Meta: rootKey.Meta,
EncryptedDataEncryptionKey: blob.Ciphertext,
KeyEncryptionKey: kek,
}
buf, err := json.Marshal(kekWrapper)
if err != nil {
return err
}
path := filepath.Join(e.keystorePath, rootKey.Meta.KeyID+nomadKeystoreExtension)
err = os.WriteFile(path, buf.Bytes(), 0600)
err = os.WriteFile(path, buf, 0o600)
if err != nil {
return err
}
......@@ -339,28 +342,27 @@ func (e *Encrypter) loadKeyFromStore(path string) (*structs.RootKey, error) {
return nil, err
}
storedKey := &struct {
Meta *structs.RootKeyMetaStub
Key string
}{}
if err := json.Unmarshal(raw, storedKey); err != nil {
kekWrapper := &structs.KeyEncryptionKeyWrapper{}
if err := json.Unmarshal(raw, kekWrapper); err != nil {
return nil, err
}
meta := &structs.RootKeyMeta{
State: storedKey.Meta.State,
KeyID: storedKey.Meta.KeyID,
Algorithm: storedKey.Meta.Algorithm,
CreateTime: storedKey.Meta.CreateTime,
}
meta := kekWrapper.Meta
if err = meta.Validate(); err != nil {
return nil, err
}
key, err := base64.StdEncoding.DecodeString(storedKey.Key)
// the errors that bubble up from this library can be a bit opaque, so make
// sure we wrap them with as much context as possible
wrapper, err := e.newKMSWrapper(meta.KeyID, kekWrapper.KeyEncryptionKey)
if err != nil {
return nil, fmt.Errorf("could not decode key: %v", err)
return nil, fmt.Errorf("unable to create key wrapper cipher: %v", err)
}
key, err := wrapper.Decrypt(e.srv.shutdownCtx, &kms.BlobInfo{
Ciphertext: kekWrapper.EncryptedDataEncryptionKey,
})
if err != nil {
return nil, fmt.Errorf("unable to decrypt wrapped root key: %v", err)
}
return &structs.RootKey{
......@@ -369,6 +371,24 @@ func (e *Encrypter) loadKeyFromStore(path string) (*structs.RootKey, error) {
}, nil
}
// newKMSWrapper returns a go-kms-wrapping interface the caller can use to
// encrypt the RootKey with a key encryption key (KEK). This is a bit of
// security theatre for local on-disk key material, but gives us a shim for
// external KMS providers in the future.
func (e *Encrypter) newKMSWrapper(keyID string, kek []byte) (kms.Wrapper, error) {
wrapper := aead.NewWrapper()
wrapper.SetConfig(context.Background(),
aead.WithAeadType(kms.AeadTypeAesGcm),
aead.WithHashType(kms.HashTypeSha256),
kms.WithKeyId(keyID),
)
err := wrapper.SetAesGcmKeyBytes(kek)
if err != nil {
return nil, err
}
return wrapper, nil
}
type KeyringReplicator struct {
srv *Server
encrypter *Encrypter
......
nomad/encrypter_test.go
+ 1
- 1
View file @ b7fea76f
......@@ -21,7 +21,7 @@ func TestEncrypter_LoadSave(t *testing.T) {
ci.Parallel(t)
tmpDir := t.TempDir()
encrypter, err := NewEncrypter(nil, tmpDir)
encrypter, err := NewEncrypter(&Server{shutdownCtx: context.Background()}, tmpDir)
require.NoError(t, err)
algos := []structs.EncryptionAlgorithm{
......
nomad/structs/extensions.go
+ 0
- 21
View file @ b7fea76f
package structs
import (
"encoding/base64"
"reflect"
)
......@@ -13,8 +12,6 @@ var (
reflect.TypeOf(&Node{}): nodeExt,
reflect.TypeOf(CSIVolume{}): csiVolumeExt,
reflect.TypeOf(&CSIVolume{}): csiVolumeExt,
reflect.TypeOf(&RootKey{}): rootKeyExt,
reflect.TypeOf(RootKey{}): rootKeyExt,
}
)
......@@ -79,21 +76,3 @@ func csiVolumeExt(v interface{}) interface{} {
return apiVol
}
// rootKeyExt safely serializes a RootKey by base64 encoding the key
// material and extracting the metadata stub. We only store the root
// key in the keystore and never in raft or return it via the API, so
// by having this extension as the default we make it slightly harder
// to misuse.
func rootKeyExt(v interface{}) interface{} {
key := v.(*RootKey)
encodedKey := base64.StdEncoding.EncodeToString(key.Key)
return &struct {
Meta *RootKeyMetaStub
Key string
}{
Meta: key.Meta.Stub(),
Key: encodedKey,
}
}
nomad/structs/keyring.go
+ 15
- 12
View file @ b7fea76f
......@@ -4,11 +4,8 @@ import (
"fmt"
"time"
// note: this is aliased so that it's more noticeable if someone
// accidentally swaps it out for math/rand via running goimports
cryptorand "crypto/rand"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/helper/crypto"
"github.com/hashicorp/nomad/helper/uuid"
)
......@@ -29,14 +26,9 @@ func NewRootKey(algorithm EncryptionAlgorithm) (*RootKey, error) {
switch algorithm {
case EncryptionAlgorithmAES256GCM:
const keyBytes = 32
key := make([]byte, keyBytes)
n, err := cryptorand.Read(key)
key, err := crypto.Bytes(32)
if err != nil {
return nil, err
}
if n < keyBytes {
return nil, fmt.Errorf("failed to generate key: entropy exhausted")
return nil, fmt.Errorf("failed to generate key: %v", err)
}
rootKey.Key = key
}
......@@ -160,6 +152,15 @@ func (rkm *RootKeyMeta) Validate() error {
return nil
}
// KeyEncryptionKeyWrapper is the struct that gets serialized for the on-disk
// KMS wrapper. This struct includes the server-specific key-wrapping key and
// should never be sent over RPC.
type KeyEncryptionKeyWrapper struct {
Meta *RootKeyMeta
EncryptedDataEncryptionKey []byte `json:"DEK"`
KeyEncryptionKey []byte `json:"KEK"`
}
// EncryptionAlgorithm chooses which algorithm is used for
// encrypting / decrypting entries with this key
type EncryptionAlgorithm string
......@@ -168,6 +169,7 @@ const (
EncryptionAlgorithmAES256GCM EncryptionAlgorithm = "aes256-gcm"
)
// KeyringRotateRootKeyRequest is the argument to the Keyring.Rotate RPC
type KeyringRotateRootKeyRequest struct {
Algorithm EncryptionAlgorithm
Full bool
......@@ -180,11 +182,12 @@ type KeyringRotateRootKeyResponse struct {
WriteMeta
}
// KeyringListRootKeyMetaRequest is the argument to the Keyring.List RPC
type KeyringListRootKeyMetaRequest struct {
// TODO: do we need any fields here?
QueryOptions
}
// KeyringListRootKeyMetaRequest is the response value of the List RPC
type KeyringListRootKeyMetaResponse struct {
Keys []*RootKeyMeta
QueryMeta
......
website/content/docs/operations/key-management.mdx
+ 6
- 3
View file @ b7fea76f
......@@ -10,7 +10,8 @@ Nomad servers maintain an encryption keyring used to encrypt [Variables][] and
sign task [workload identities][]. The servers store key metadata in raft, but
the encryption key material is stored in a separate file in the `keystore`
subdirectory of the Nomad [data directory][]. These files have the extension
`.nks.json`.
`.nks.json`. The key material in each file is wrapped in a unique key encryption
key (KEK) that is not shared between servers.
Under normal operations the keyring is entirely managed by Nomad, but this
section provides administrators additional context around key replication and
......@@ -49,8 +50,10 @@ snapshots you might provide without exposing any of your keys or variables.
However, this means that to restore a cluster from snapshot you need to also
provide the keystore directory with the `.nks.json` key files on at least one
server. Operators should include these files as part of your organization's
backup and recovery strategy for the cluster.
server. The `.nks.json` key files are unique per server, but only one server's
key files are needed to recover the cluster. Operators should include these
files as part of your organization's backup and recovery strategy for the
cluster.
[Variables]: /docs/concepts/variables
[workload identities]: /docs/concepts/workload-identity
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

免费DevSecOps平台，让您的项目体验完整的DevSecOps流程，让项目更安全