Fix for deleting unused etcd and kubelet certs

pki/services.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"reflect"
 	"sort"
+	"strings"
 
 	"github.com/rancher/rke/hosts"
 	"github.com/rancher/rke/log"
@@ -370,7 +371,6 @@ func GenerateEtcdCertificates(ctx context.Context, certs map[string]CertificateP
 		ips = append(ips, ip.String())
 	}
 	sort.Strings(ips)
-
 	for _, host := range etcdHosts {
 		etcdName := GetCrtNameForHost(host, EtcdCertName)
 		if _, ok := certs[etcdName]; ok && certs[etcdName].CertificatePEM != "" && !rotate {
@@ -403,6 +403,8 @@ func GenerateEtcdCertificates(ctx context.Context, certs map[string]CertificateP
 		}
 		certs[etcdName] = ToCertObject(etcdName, "", "", etcdCrt, etcdKey, nil)
 	}
+	log.Debugf(ctx, "Checking and deleting unused etcd certificates, current etcd nodes are: %v", etcdHosts)
+	deleteUnusedCerts(ctx, certs, EtcdCertName, etcdHosts)
 	return nil
 }
 
@@ -516,6 +518,8 @@ func GenerateKubeletCertificate(ctx context.Context, certs map[string]Certificat
 		}
 		certs[kubeletName] = ToCertObject(kubeletName, "", "", kubeletCrt, kubeletKey, nil)
 	}
+	log.Debugf(ctx, "Checking and deleting unused kubelet certificates, current nodes are : %v", allHosts)
+	deleteUnusedCerts(ctx, certs, KubeletCertName, allHosts)
 	return nil
 }
 
@@ -555,6 +559,15 @@ func GenerateRKEServicesCerts(ctx context.Context, certs map[string]CertificateP
 	}
 	if IsKubeletGenerateServingCertificateEnabledinConfig(&rkeConfig) {
 		RKECerts = append(RKECerts, GenerateKubeletCertificate)
+	} else {
+		//Clean up kubelet certs when GenerateServingCertificate is disabled
+		log.Infof(ctx, "[certificates] GenerateServingCertificate is disabled, checking if there are unused kubelet certificates")
+		for k := range certs {
+			if strings.HasPrefix(k, KubeletCertName) {
+				log.Infof(ctx, "[certificates] Deleting unused kubelet certificate: %s", k)
+				delete(certs, k)
+			}
+		}
 	}
 	for _, gen := range RKECerts {
 		if err := gen(ctx, certs, rkeConfig, configPath, configDir, rotate); err != nil {
@@ -588,3 +601,21 @@ func GenerateRKEServicesCSRs(ctx context.Context, certs map[string]CertificatePK
 	}
 	return nil
 }
+
+func deleteUnusedCerts(ctx context.Context, certs map[string]CertificatePKI, certName string, hosts []*hosts.Host) {
+	log.Infof(ctx, "[certificates] Checking and deleting unused %s certificates", certName)
+	unusedCerts := make(map[string]bool)
+	for k := range certs {
+		if strings.HasPrefix(k, certName) {
+			unusedCerts[k] = true
+		}
+	}
+	for _, host := range hosts {
+		Name := GetCrtNameForHost(host, certName)
+		delete(unusedCerts, Name)
+	}
+	for k := range unusedCerts {
+		log.Infof(ctx, "[certificates] Deleting unused certificate: %s", k)
+		delete(certs, k)
+	}
+}

pki/services_test.go

+package pki
+
+import (
+	"context"
+	"github.com/rancher/rke/hosts"
+	v3 "github.com/rancher/types/apis/management.cattle.io/v3"
+	"github.com/stretchr/testify/assert"
+	"reflect"
+	"testing"
+)
+
+func TestDeleteUnusedCerts(t *testing.T) {
+	tests := []struct {
+		ctx             context.Context
+		name            string
+		certs           map[string]CertificatePKI
+		certName        string
+		hosts           []*hosts.Host
+		expectLeftCerts map[string]CertificatePKI
+	}{
+		{
+			ctx:  context.Background(),
+			name: "Keep valid etcd certs",
+			certs: map[string]CertificatePKI{
+				"kube-etcd-172-17-0-3":    CertificatePKI{},
+				"kube-etcd-172-17-0-4":    CertificatePKI{},
+				"kube-node":               CertificatePKI{},
+				"kube-kubelet-172-17-0-4": CertificatePKI{},
+				"kube-apiserver":          CertificatePKI{},
+				"kube-proxy":              CertificatePKI{},
+			},
+			certName: EtcdCertName,
+			hosts: []*hosts.Host{
+				{RKEConfigNode: v3.RKEConfigNode{
+					Address: "172.17.0.3",
+				}},
+				{RKEConfigNode: v3.RKEConfigNode{
+					Address: "172.17.0.4",
+				}},
+			},
+			expectLeftCerts: map[string]CertificatePKI{
+				"kube-etcd-172-17-0-3":    CertificatePKI{},
+				"kube-etcd-172-17-0-4":    CertificatePKI{},
+				"kube-node":               CertificatePKI{},
+				"kube-kubelet-172-17-0-4": CertificatePKI{},
+				"kube-apiserver":          CertificatePKI{},
+				"kube-proxy":              CertificatePKI{},
+			},
+		},
+		{
+			ctx:  context.Background(),
+			name: "Keep valid kubelet certs",
+			certs: map[string]CertificatePKI{
+				"kube-kubelet-172-17-0-5": CertificatePKI{},
+				"kube-kubelet-172-17-0-6": CertificatePKI{},
+				"kube-node":               CertificatePKI{},
+				"kube-apiserver":          CertificatePKI{},
+				"kube-proxy":              CertificatePKI{},
+				"kube-etcd-172-17-0-6":    CertificatePKI{},
+			},
+			certName: KubeletCertName,
+			hosts: []*hosts.Host{
+				{RKEConfigNode: v3.RKEConfigNode{
+					Address: "172.17.0.5",
+				}},
+				{RKEConfigNode: v3.RKEConfigNode{
+					Address: "172.17.0.6",
+				}},
+			},
+			expectLeftCerts: map[string]CertificatePKI{
+				"kube-kubelet-172-17-0-5": CertificatePKI{},
+				"kube-kubelet-172-17-0-6": CertificatePKI{},
+				"kube-node":               CertificatePKI{},
+				"kube-apiserver":          CertificatePKI{},
+				"kube-proxy":              CertificatePKI{},
+				"kube-etcd-172-17-0-6":    CertificatePKI{},
+			},
+		},
+		{
+			ctx:  context.Background(),
+			name: "Remove unused etcd certs",
+			certs: map[string]CertificatePKI{
+				"kube-etcd-172-17-0-11":    CertificatePKI{},
+				"kube-etcd-172-17-0-10":    CertificatePKI{},
+				"kube-kubelet-172-17-0-11": CertificatePKI{},
+				"kube-node":                CertificatePKI{},
+				"kube-apiserver":           CertificatePKI{},
+				"kube-proxy":               CertificatePKI{},
+			},
+			certName: EtcdCertName,
+			hosts: []*hosts.Host{
+				{RKEConfigNode: v3.RKEConfigNode{
+					Address: "172.17.0.11",
+				}},
+				{RKEConfigNode: v3.RKEConfigNode{
+					Address: "172.17.0.12",
+				}},
+			},
+			expectLeftCerts: map[string]CertificatePKI{
+				"kube-etcd-172-17-0-11":    CertificatePKI{},
+				"kube-kubelet-172-17-0-11": CertificatePKI{},
+				"kube-node":                CertificatePKI{},
+				"kube-apiserver":           CertificatePKI{},
+				"kube-proxy":               CertificatePKI{},
+			},
+		},
+		{
+			ctx:  context.Background(),
+			name: "Remove unused kubelet certs",
+			certs: map[string]CertificatePKI{
+				"kube-kubelet-172-17-0-11": CertificatePKI{},
+				"kube-kubelet-172-17-0-10": CertificatePKI{},
+				"kube-etcd-172-17-0-10":    CertificatePKI{},
+				"kube-node":                CertificatePKI{},
+				"kube-apiserver":           CertificatePKI{},
+				"kube-proxy":               CertificatePKI{},
+			},
+			certName: KubeletCertName,
+			hosts: []*hosts.Host{
+				{RKEConfigNode: v3.RKEConfigNode{
+					Address: "172.17.0.11",
+				}},
+				{RKEConfigNode: v3.RKEConfigNode{
+					Address: "172.17.0.12",
+				}},
+			},
+			expectLeftCerts: map[string]CertificatePKI{
+				"kube-kubelet-172-17-0-11": CertificatePKI{},
+				"kube-etcd-172-17-0-10":    CertificatePKI{},
+				"kube-node":                CertificatePKI{},
+				"kube-apiserver":           CertificatePKI{},
+				"kube-proxy":               CertificatePKI{},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			deleteUnusedCerts(tt.ctx, tt.certs, tt.certName, tt.hosts)
+			assert.Equal(t, true, reflect.DeepEqual(tt.certs, tt.expectLeftCerts))
+		})
+	}
+}