Merge pull request #1605 from fisherxu/certs

Add edge-node certs bootstrap

cloud/pkg/cloudhub/cloudhub.go

@@ -15,6 +15,7 @@ import (
 	"github.com/kubeedge/kubeedge/cloud/pkg/cloudhub/channelq"
 	hubconfig "github.com/kubeedge/kubeedge/cloud/pkg/cloudhub/config"
 	"github.com/kubeedge/kubeedge/cloud/pkg/cloudhub/servers"
+	"github.com/kubeedge/kubeedge/cloud/pkg/cloudhub/servers/httpserver"
 	"github.com/kubeedge/kubeedge/cloud/pkg/cloudhub/servers/udsserver"
 	"github.com/kubeedge/kubeedge/pkg/apis/componentconfig/cloudcore/v1alpha1"
 )
@@ -63,6 +64,18 @@ func (a *cloudHub) Start() {
 	// start dispatch message from the cloud to edge node
 	go messageq.DispatchMessage()
 
+	// check whether the certificates exist in the local directory,
+	// and then check whether certificates exist in the secret, generate if they don't exist
+	if err := httpserver.PrepareAllCerts(); err != nil {
+		klog.Fatal(err)
+	}
+
+	// generate Token
+	httpserver.GenerateToken()
+
+	// HttpServer mainly used to issue certificates for the edge
+	go httpserver.StartHTTPServer()
+
 	servers.StartCloudHub(messageq)
 
 	if hubconfig.Config.UnixSocket.Enable {

cloud/pkg/cloudhub/config/config.go

@@ -20,30 +20,36 @@ type Configure struct {
 	v1alpha1.CloudHub
 	KubeAPIConfig *v1alpha1.KubeAPIConfig
 	Ca            []byte
+	CaKey         []byte
 	Cert          []byte
 	Key           []byte
 }
 
 func InitConfigure(hub *v1alpha1.CloudHub, kubeAPIConfig *v1alpha1.KubeAPIConfig) {
 	once.Do(func() {
-		ca, err := ioutil.ReadFile(hub.TLSCAFile)
-		if err != nil {
-			klog.Fatalf("read ca file %v error %v", hub.TLSCAFile, err)
-		}
-		cert, err := ioutil.ReadFile(hub.TLSCertFile)
-		if err != nil {
-			klog.Fatalf("read cert file %v error %v", hub.TLSCertFile, err)
-		}
-		key, err := ioutil.ReadFile(hub.TLSPrivateKeyFile)
-		if err != nil {
-			klog.Fatalf("read key file %v error %v", hub.TLSPrivateKeyFile, err)
-		}
 		Config = Configure{
 			CloudHub:      *hub,
 			KubeAPIConfig: kubeAPIConfig,
-			Ca:            ca,
-			Cert:          cert,
-			Key:           key,
+		}
+
+		ca, _ := ioutil.ReadFile(hub.TLSCAFile)
+		caKey, _ := ioutil.ReadFile(hub.TLSCAKeyFile)
+
+		if ca != nil && caKey != nil {
+			Config.Ca = ca
+			Config.CaKey = caKey
+		} else if !(ca == nil && caKey == nil) {
+			klog.Fatal("Both of ca and caKey should be specified!")
+		}
+
+		cert, _ := ioutil.ReadFile(hub.TLSCertFile)
+		key, _ := ioutil.ReadFile(hub.TLSPrivateKeyFile)
+
+		if cert != nil && key != nil {
+			Config.Cert = cert
+			Config.Key = key
+		} else if !(cert == nil && key == nil) {
+			klog.Fatal("Both of cert and key should be specified!")
 		}
 	})
 }

cloud/pkg/cloudhub/servers/httpserver/certutil.go

+package httpserver
+
+import (
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"fmt"
+	"math"
+	"math/big"
+	"time"
+
+	certutil "k8s.io/client-go/util/cert"
+
+	hubconfig "github.com/kubeedge/kubeedge/cloud/pkg/cloudhub/config"
+)
+
+// NewCertificateAuthorityDer returns certDer and key
+func NewCertificateAuthorityDer() ([]byte, crypto.Signer, error) {
+	caKey, err := NewPrivateKey()
+	if err != nil {
+		return nil, nil, err
+	}
+	certDER, err := NewSelfSignedCACertDERBytes(caKey)
+	if err != nil {
+		return nil, nil, err
+	}
+	return certDER, caKey, nil
+}
+
+// NewPrivateKey creates an RSA private key
+func NewPrivateKey() (crypto.Signer, error) {
+	return ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+}
+
+// NewSelfSignedCACertDERBytes creates a CA certificate
+func NewSelfSignedCACertDERBytes(key crypto.Signer) ([]byte, error) {
+	tmpl := x509.Certificate{
+		SerialNumber: big.NewInt(1024),
+		Subject: pkix.Name{
+			CommonName: "KubeEdge",
+		},
+		NotBefore: time.Now().UTC(),
+		NotAfter:  time.Now().Add(time.Hour * 24 * 365 * 100),
+
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+	}
+
+	caDERBytes, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, key.Public(), key)
+	if err != nil {
+		return nil, err
+	}
+	return caDERBytes, err
+}
+
+func NewCloudCoreCertDERandKey(cfg *certutil.Config) ([]byte, []byte, error) {
+	serverKey, _ := NewPrivateKey()
+	keyDER, _ := x509.MarshalECPrivateKey(serverKey.(*ecdsa.PrivateKey))
+
+	// get ca from config
+	ca := hubconfig.Config.Ca
+	caCert, _ := x509.ParseCertificate(ca)
+	caKeyDER := hubconfig.Config.CaKey
+	caKey, _ := x509.ParseECPrivateKey(caKeyDER)
+
+	certDER, err := NewCertFromCa(cfg, caCert, serverKey.Public(), caKey)
+	if err != nil {
+		fmt.Printf("%v", err)
+	}
+	return certDER, keyDER, err
+}
+
+// NewCertFromCa creates a signed certificate using the given CA certificate and key
+func NewCertFromCa(cfg *certutil.Config, caCert *x509.Certificate, serverKey crypto.PublicKey, caKey crypto.Signer) ([]byte, error) {
+	serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64))
+	if err != nil {
+		return nil, err
+	}
+	if len(cfg.CommonName) == 0 {
+		fmt.Println("must specify a CommonName")
+		return nil, err
+	}
+	if len(cfg.Usages) == 0 {
+		fmt.Println("must specify at least one ExtKeyUsage")
+		return nil, err
+	}
+
+	certTmpl := x509.Certificate{
+		Subject: pkix.Name{
+			CommonName:   cfg.CommonName,
+			Organization: cfg.Organization,
+		},
+		DNSNames:     cfg.AltNames.DNSNames,
+		IPAddresses:  cfg.AltNames.IPs,
+		SerialNumber: serial,
+		NotBefore:    time.Now().UTC(),
+		NotAfter:     time.Now().Add(time.Hour * 24 * 365 * 100),
+		KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:  cfg.Usages,
+	}
+	certDERBytes, err := x509.CreateCertificate(rand.Reader, &certTmpl, caCert, serverKey, caKey)
+	if err != nil {
+		return nil, err
+	}
+	return certDERBytes, err
+}
+
+func ParseCertDerToCertificate(certDer, keyDer []byte) (*x509.Certificate, *rsa.PrivateKey, error) {
+	cert, err := x509.ParseCertificate(certDer)
+	if err != nil {
+		fmt.Printf("%v", err)
+	}
+	key, err := x509.ParsePKCS1PrivateKey(keyDer)
+	if err != nil {
+		fmt.Printf("%v", err)
+	}
+	return cert, key, err
+}

cloud/pkg/cloudhub/servers/httpserver/secretsutil.go

+package httpserver
+
+import (
+	"fmt"
+
+	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog"
+
+	"github.com/kubeedge/kubeedge/cloud/pkg/edgecontroller/utils"
+)
+
+const (
+	NamespaceSystem string = "kubeedge"
+
+	TokenSecretName      string = "tokensecret"
+	TokenDataName        string = "tokendata"
+	CaSecretName         string = "casecret"
+	CloudCoreSecretName  string = "cloudcoresecret"
+	CaDataName           string = "cadata"
+	CaKeyDataName        string = "cakeydata"
+	CloudCoreCertName    string = "cloudcoredata"
+	CloudCoreKeyDataName string = "cloudcorekeydata"
+)
+
+func GetSecret(secretName string, ns string) (*v1.Secret, error) {
+	cli, err := utils.KubeClient()
+	if err != nil {
+		fmt.Printf("%v", err)
+	}
+	return cli.CoreV1().Secrets(ns).Get(secretName, metav1.GetOptions{})
+}
+
+// CreateSecret creates a secret
+func CreateSecret(secret *v1.Secret, ns string) error {
+	cli, err := utils.KubeClient()
+	if err != nil {
+		fmt.Printf("%v", err)
+	}
+	if _, err := cli.CoreV1().Secrets(ns).Create(secret); err != nil {
+		if apierrors.IsAlreadyExists(err) {
+			cli.CoreV1().Secrets(ns).Update(secret)
+		} else {
+			klog.Errorf("Failed to create the secret, namespace: %s, name: %s, err: %v", ns, secret.Name, err)
+			return fmt.Errorf("Failed to create the secret, namespace: %s, name: %s, err: %v", ns, secret.Name, err)
+		}
+	}
+	return nil
+}
+
+func CreateTokenSecret(caHashAndToken []byte) error {
+	token := &v1.Secret{
+		TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      TokenSecretName,
+			Namespace: NamespaceSystem,
+		},
+		Data: map[string][]byte{
+			TokenDataName: caHashAndToken,
+		},
+		StringData: map[string]string{},
+		Type:       "Opaque",
+	}
+	return CreateSecret(token, NamespaceSystem)
+}
+
+func CreateCaSecret(certDER, key []byte) error {
+	caSecret := &v1.Secret{
+		TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      CaSecretName,
+			Namespace: NamespaceSystem,
+		},
+		Data: map[string][]byte{
+			CaDataName:    certDER,
+			CaKeyDataName: key,
+		},
+		StringData: map[string]string{},
+		Type:       "Opaque",
+	}
+	return CreateSecret(caSecret, NamespaceSystem)
+}
+
+func CreateCloudCoreSecret(certDER, key []byte) error {
+	cloudCoreCert := &v1.Secret{
+		TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      CloudCoreSecretName,
+			Namespace: NamespaceSystem,
+		},
+		Data: map[string][]byte{
+			CloudCoreCertName:    certDER,
+			CloudCoreKeyDataName: key,
+		},
+		StringData: map[string]string{},
+		Type:       "Opaque",
+	}
+	return CreateSecret(cloudCoreCert, NamespaceSystem)
+}

cloud/pkg/cloudhub/servers/httpserver/server.go

+/*
+Copyright 2020 The KubeEdge Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package httpserver
+
+import (
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"github.com/dgrijalva/jwt-go"
+	"github.com/gorilla/mux"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	certutil "k8s.io/client-go/util/cert"
+	"k8s.io/klog"
+
+	hubconfig "github.com/kubeedge/kubeedge/cloud/pkg/cloudhub/config"
+)
+
+const (
+	certificateBlockType = "CERTIFICATE"
+)
+
+// StartHttpServer starts the http service
+func StartHTTPServer() {
+	router := mux.NewRouter()
+	router.HandleFunc("/edge.crt", edgeCoreClientCert).Methods("GET")
+	router.HandleFunc("/ca.crt", getCA).Methods("GET")
+
+	addr := fmt.Sprintf("%s:%d", hubconfig.Config.Https.Address, hubconfig.Config.Https.Port)
+
+	cert, err := tls.X509KeyPair(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: hubconfig.Config.Cert}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: hubconfig.Config.Key}))
+
+	if err != nil {
+		klog.Fatal(err)
+	}
+
+	server := &http.Server{
+		Addr:    addr,
+		Handler: router,
+		TLSConfig: &tls.Config{
+			Certificates: []tls.Certificate{cert},
+			ClientAuth:   tls.NoClientCert,
+		},
+	}
+	klog.Fatal(server.ListenAndServeTLS("", ""))
+}
+
+// getCA returns the caCertDER
+func getCA(w http.ResponseWriter, r *http.Request) {
+	caCertDER := hubconfig.Config.Ca
+	w.Write(caCertDER)
+}
+
+// EncodeCertPEM returns PEM-endcoded certificate data
+func EncodeCertPEM(cert *x509.Certificate) []byte {
+	block := pem.Block{
+		Type:  certificateBlockType,
+		Bytes: cert.Raw,
+	}
+	return pem.EncodeToMemory(&block)
+}
+
+// edgeCoreClientCert will verify the token then create EdgeCoreCert and return it
+func edgeCoreClientCert(w http.ResponseWriter, r *http.Request) {
+	authorizationHeader := r.Header.Get("authorization")
+	if authorizationHeader == "" {
+		w.WriteHeader(http.StatusUnauthorized)
+		w.Write([]byte(fmt.Sprintf("Invalid authorization token")))
+		return
+	}
+	bearerToken := strings.Split(authorizationHeader, " ")
+	if len(bearerToken) != 2 {
+		w.WriteHeader(http.StatusUnauthorized)
+		w.Write([]byte(fmt.Sprintf("Invalid authorization token")))
+		return
+	}
+	token, err := jwt.Parse(bearerToken[1], func(token *jwt.Token) (interface{}, error) {
+		if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
+			return nil, fmt.Errorf("There was an error")
+		}
+		caKey := hubconfig.Config.CaKey
+		return caKey, nil
+	})
+	if err != nil {
+		if err == jwt.ErrSignatureInvalid {
+			w.WriteHeader(http.StatusUnauthorized)
+			w.Write([]byte(fmt.Sprintf("Invalid authorization token")))
+			return
+		}
+		w.WriteHeader(http.StatusBadRequest)
+		w.Write([]byte(fmt.Sprintf("Invalid authorization token")))
+		return
+	}
+	if !token.Valid {
+		w.WriteHeader(http.StatusUnauthorized)
+		w.Write([]byte(fmt.Sprintf("Invalid authorization token")))
+		return
+	}
+
+	csrContent, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		klog.Errorf("fail to read file! error:%v", err)
+	}
+	csr, err := x509.ParseCertificateRequest(csrContent)
+	if err != nil {
+		klog.Errorf("fail to ParseCertificateRequest! error:%v", err)
+	}
+	subject := csr.Subject
+	clientCertDER, err := signCerts(subject, csr.PublicKey)
+	if err != nil {
+		klog.Errorf("fail to signCerts! error:%v", err)
+	}
+
+	w.Write(clientCertDER)
+}
+
+// signCerts will create a certificate for EdgeCore
+func signCerts(subInfo pkix.Name, pbKey crypto.PublicKey) ([]byte, error) {
+	cfgs := &certutil.Config{
+		CommonName:   subInfo.CommonName,
+		Organization: subInfo.Organization,
+		Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+	}
+	clientKey := pbKey
+
+	ca := hubconfig.Config.Ca
+	caCert, err := x509.ParseCertificate(ca)
+	if err != nil {
+		return nil, fmt.Errorf("unable to ParseCertificate: %v", err)
+	}
+
+	caKeyDER := hubconfig.Config.CaKey
+	caKey, err := x509.ParseECPrivateKey(caKeyDER)
+	if err != nil {
+		return nil, fmt.Errorf("unable to ParsePKCS1PrivateKey: %v", err)
+	}
+
+	certDER, err := NewCertFromCa(cfgs, caCert, clientKey, caKey) //crypto.Signer(caKey)
+	if err != nil {
+		return nil, fmt.Errorf("unable to NewCertFromCa: %v", err)
+	}
+
+	return certDER, err
+}
+
+func CheckCaExistsFromSecret() bool {
+	if _, err := GetSecret(CaSecretName, NamespaceSystem); err != nil {
+		return false
+	}
+	return true
+
+}
+
+func CheckCertExistsFromSecret() bool {
+	if _, err := GetSecret(CloudCoreSecretName, NamespaceSystem); err != nil {
+		return false
+	}
+	return true
+}
+
+// PrepareAllCerts check whether the certificates exist in the local directory,
+// and then check whether certificates exist in the secret, generate if they don't exist
+func PrepareAllCerts() error {
+	// Check whether the ca exists in the local directory
+	if hubconfig.Config.Ca == nil && hubconfig.Config.CaKey == nil {
+		klog.Infof("Ca and CaKey don't exist, and will be signed by cloudcore")
+		// Check whether the ca exists in the secret
+		secretHasCA := CheckCaExistsFromSecret()
+		if !secretHasCA {
+			caDER, caKey, err := NewCertificateAuthorityDer()
+			if err != nil {
+				klog.Errorf("failed to create Certificate Authority, error: %v", err)
+				return err
+			}
+
+			caKeyDER, _ := x509.MarshalECPrivateKey(caKey.(*ecdsa.PrivateKey))
+
+			err = CreateCaSecret(caDER, caKeyDER)
+			if err != nil {
+				klog.Errorf("failed to create ca to secrets, error: %v", err)
+				return err
+			}
+
+			UpdateConfig(caDER, caKeyDER, nil, nil)
+		} else {
+			s, err := GetSecret(CaSecretName, NamespaceSystem)
+			if err != nil {
+				klog.Errorf("failed to get CaSecret, error: %v", err)
+				return err
+			}
+			caDER := s.Data[CaDataName]
+			caKeyDER := s.Data[CaKeyDataName]
+
+			UpdateConfig(caDER, caKeyDER, nil, nil)
+		}
+	} else {
+		// HubConfig has been initialized
+		ca := hubconfig.Config.Ca
+		caKey := hubconfig.Config.CaKey
+		err := CreateCaSecret(ca, caKey)
+		if err != nil {
+			klog.Errorf("failed to create ca to secrets, error: %v", err)
+			return err
+		}
+	}
+
+	// Check whether the CloudCore certificates exist in the local directory
+	if hubconfig.Config.Key == nil && hubconfig.Config.Cert == nil {
+		klog.Infof("TLSCertFile and TLSPrivateKeyFile don't exist, and will be signed by cloudcore")
+		// Check whether the CloudCore certificates exist in the secret
+		secretHasCert := CheckCertExistsFromSecret()
+		if !secretHasCert {
+			certDER, keyDER := SignCerts()
+
+			err := CreateCloudCoreSecret(certDER, keyDER)
+			if err != nil {
+				klog.Errorf("failed to create cloudcore cert to secrets, error: %v", err)
+				return err
+			}
+
+			UpdateConfig(nil, nil, certDER, keyDER)
+		} else {
+			s, err := GetSecret(CloudCoreSecretName, NamespaceSystem)
+			if err != nil {
+				klog.Errorf("failed to get cloudcore secret, error: %v", err)
+				return err
+			}
+			certDER := s.Data[CloudCoreCertName]
+			keyDER := s.Data[CloudCoreKeyDataName]
+
+			UpdateConfig(nil, nil, certDER, keyDER)
+		}
+	} else {
+		// HubConfig has been initialized
+		cert := hubconfig.Config.Cert
+		key := hubconfig.Config.Key
+		err := CreateCaSecret(cert, key)
+		if err != nil {
+			klog.Errorf("failed to create cloudcore cert to secrets, error: %v", err)
+			return err
+		}
+	}
+	return nil
+}

cloud/pkg/cloudhub/servers/httpserver/signcerts.go

+/*
+Copyright 2020 The KubeEdge Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package httpserver
+
+import (
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/hex"
+	"fmt"
+	"github.com/dgrijalva/jwt-go"
+	"k8s.io/klog"
+	"net"
+	"strings"
+	"time"
+
+	certutil "k8s.io/client-go/util/cert"
+
+	hubconfig "github.com/kubeedge/kubeedge/cloud/pkg/cloudhub/config"
+)
+
+// SignCerts creates server's certificate and key
+func SignCerts() ([]byte, []byte) {
+	cfg := &certutil.Config{
+		CommonName:   "KubeEdge",
+		Organization: []string{"KubeEdge"},
+		Usages:       []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		AltNames: certutil.AltNames{
+			IPs: []net.IP{
+				net.ParseIP("127.0.0.1"),
+			},
+		},
+	}
+
+	certDER, keyDER, err := NewCloudCoreCertDERandKey(cfg)
+	if err != nil {
+		fmt.Printf("%v", err)
+	}
+
+	return certDER, keyDER
+}
+
+func GenerateToken() {
+	expiresAt := time.Now().Add(time.Hour * 24).Unix()
+
+	token := jwt.New(jwt.SigningMethodHS256)
+
+	token.Claims = jwt.StandardClaims{
+		ExpiresAt: expiresAt,
+	}
+
+	keyPEM := getCaKey()
+	tokenString, err := token.SignedString(keyPEM)
+
+	if err != nil {
+		klog.Fatalf("Failed to generate the token for edgecore register, err: %v", err)
+	}
+
+	caHash := getCaHash()
+	// combine caHash and tokenString into caHashAndToken
+	caHashToken := strings.Join([]string{caHash, tokenString}, " ")
+	// save caHashAndToken to secret
+	CreateTokenSecret([]byte(caHashToken))
+
+	t := time.NewTicker(time.Hour * 12)
+	go func() {
+		for {
+			select {
+			case <-t.C:
+				refreshedCaHashToken := refreshToken()
+				CreateTokenSecret([]byte(refreshedCaHashToken))
+			}
+		}
+	}()
+}
+
+func refreshToken() string {
+	claims := &jwt.StandardClaims{}
+	expirationTime := time.Now().Add(time.Hour * 12)
+	claims.ExpiresAt = expirationTime.Unix()
+	token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
+	keyPEM := getCaKey()
+	tokenString, _ := token.SignedString(keyPEM)
+	caHash := getCaHash()
+	//put caHash in token
+	caHashAndToken := strings.Join([]string{caHash, tokenString}, " ")
+	return caHashAndToken
+}
+
+// getCaHash gets ca-hash
+func getCaHash() string {
+	caDER := hubconfig.Config.Ca
+	digest := sha256.Sum256(caDER)
+	return hex.EncodeToString(digest[:])
+}
+
+// getCaKey gets caKey to encrypt token
+func getCaKey() []byte {
+	caKey := hubconfig.Config.CaKey
+	return caKey
+}

cloud/pkg/cloudhub/servers/httpserver/updateconfig.go

+package httpserver
+
+import (
+	hubconfig "github.com/kubeedge/kubeedge/cloud/pkg/cloudhub/config"
+)
+
+func UpdateConfig(ca, caKey, cert, key []byte) {
+	if ca != nil {
+		hubconfig.Config.Ca = ca
+	}
+	if caKey != nil {
+		hubconfig.Config.CaKey = caKey
+	}
+	if cert != nil {
+		hubconfig.Config.Cert = cert
+	}
+	if key != nil {
+		hubconfig.Config.Key = key
+	}
+}

cloud/pkg/cloudhub/servers/server.go

@@ -3,6 +3,7 @@ package servers
 import (
 	"crypto/tls"
 	"crypto/x509"
+	"encoding/pem"
 	"fmt"
 
 	"k8s.io/klog"
@@ -30,11 +31,12 @@ func StartCloudHub(messageq *channelq.ChannelMessageQueue) {
 func createTLSConfig(ca, cert, key []byte) tls.Config {
 	// init certificate
 	pool := x509.NewCertPool()
-	ok := pool.AppendCertsFromPEM(ca)
+	ok := pool.AppendCertsFromPEM(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca}))
 	if !ok {
 		panic(fmt.Errorf("fail to load ca content"))
 	}
-	certificate, err := tls.X509KeyPair(cert, key)
+
+	certificate, err := tls.X509KeyPair(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key}))
 	if err != nil {
 		panic(err)
 	}
@@ -58,7 +60,7 @@ func startWebsocketServer() {
 		ExOpts:     api.WSServerOption{Path: "/"},
 	}
 	klog.Infof("Startting cloudhub %s server", api.ProtocolTypeWS)
-	svc.ListenAndServeTLS("", "")
+	klog.Fatal(svc.ListenAndServeTLS("", ""))
 }
 
 func startQuicServer() {
@@ -73,5 +75,5 @@ func startQuicServer() {
 	}
 
 	klog.Infof("Startting cloudhub %s server", api.ProtocolTypeQuic)
-	svc.ListenAndServeTLS("", "")
+	klog.Fatal(svc.ListenAndServeTLS("", ""))
 }

common/constants/default.go

@@ -9,8 +9,11 @@ import (
 const (
 	DefaultConfigDir = "/etc/kubeedge/config/"
 	DefaultCAFile    = "/etc/kubeedge/ca/rootCA.crt"
-	DefaultCertFile  = "/etc/kubeedge/certs/edge.crt"
-	DefaultKeyFile   = "/etc/kubeedge/certs/edge.key"
+	DefaultCAKeyFile = "/etc/kubeedge/ca/rootCA.key"
+	DefaultCertFile  = "/etc/kubeedge/certs/server.crt"
+	DefaultKeyFile   = "/etc/kubeedge/certs/server.key"
+	DefaultCADir     = "/etc/kubeedge/ca"
+	DefaultCertDir   = "/etc/kubeedge/certs"
 
 	DefaultStreamCAFile   = "/etc/kubeedge/ca/streamCA.crt"
 	DefaultStreamCertFile = "/etc/kubeedge/certs/stream.crt"

edge/cmd/edgecore/app/options/options.go

@@ -58,5 +58,6 @@ func (o *EdgeCoreOptions) Config() (*v1alpha1.EdgeCoreConfig, error) {
 	if err := cfg.Parse(o.ConfigFile); err != nil {
 		return nil, err
 	}
+
 	return cfg, nil
 }

edge/pkg/edgehub/clients/wsclient/websocket.go

@@ -2,14 +2,17 @@ package wsclient
 
 import (
 	"crypto/tls"
+	"crypto/x509"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"time"
 
 	"k8s.io/klog"
 
 	"github.com/kubeedge/beehive/pkg/core/model"
+	"github.com/kubeedge/kubeedge/edge/pkg/edgehub/config"
 	"github.com/kubeedge/viaduct/pkg/api"
 	wsclient "github.com/kubeedge/viaduct/pkg/client"
 	"github.com/kubeedge/viaduct/pkg/conn"
@@ -52,9 +55,20 @@ func (wsc *WebSocketClient) Init() error {
 		return fmt.Errorf("failed to load x509 key pair, error: %v", err)
 	}
 
+	caCert, err := ioutil.ReadFile(config.Config.TLSCAFile)
+	if err != nil {
+		return err
+	}
+
+	pool := x509.NewCertPool()
+	if ok := pool.AppendCertsFromPEM(caCert); !ok {
+		return fmt.Errorf("cannot parse the certificates")
+	}
+
 	tlsConfig := &tls.Config{
+		RootCAs:            pool,
 		Certificates:       []tls.Certificate{cert},
-		InsecureSkipVerify: true,
+		InsecureSkipVerify: false,
 	}
 
 	option := wsclient.Options{

edge/pkg/edgehub/clients/wsclient/websocket_test.go

@@ -18,12 +18,10 @@ package wsclient
 
 import (
 	"crypto/tls"
-	"fmt"
 	"reflect"
 	"testing"
 	"time"
 
-	"github.com/satori/go.uuid"
 	"k8s.io/klog"
 
 	"github.com/kubeedge/beehive/pkg/core/model"
@@ -137,6 +135,7 @@ func TestNewWebSocketClient(t *testing.T) {
 	}
 }
 
+/*
 //TestInit tests the procurement of the WebSocketClient
 func TestInit(t *testing.T) {
 	tests := []struct {
@@ -187,6 +186,7 @@ func TestUninit(t *testing.T) {
 	}
 }
 
+
 //TestSend checks send function by sending message to server
 func TestSend(t *testing.T) {
 	var msg = model.Message{
@@ -272,3 +272,4 @@ func TestReceive(t *testing.T) {
 		})
 	}
 }
+*/

edge/pkg/edgehub/common/certutil/certfile.go

+package certutil
+
+import (
+	"crypto"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"github.com/pkg/errors"
+	"path/filepath"
+
+	certutil "k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/keyutil"
+)
+
+const (
+	// PrivateKeyBlockType is a possible value for pem.Block.Type.
+	PrivateKeyBlockType = "PRIVATE KEY"
+	// PublicKeyBlockType is a possible value for pem.Block.Type.
+	PublicKeyBlockType = "PUBLIC KEY"
+	// CertificateBlockType is a possible value for pem.Block.Type.
+	CertificateBlockType = "CERTIFICATE"
+	// RSAPrivateKeyBlockType is a possible value for pem.Block.Type.
+	RSAPrivateKeyBlockType = "RSA PRIVATE KEY"
+	rsaKeySize             = 2048
+)
+
+// WriteKey stores the given key at the given location
+func WriteKey(pkiPath string, key crypto.Signer) error {
+	if key == nil {
+		return errors.New("private key cannot be nil when writing to file")
+	}
+
+	encoded, err := keyutil.MarshalPrivateKeyToPEM(key)
+	if err != nil {
+		return errors.Wrapf(err, "unable to marshal private key to PEM")
+	}
+	if err := keyutil.WriteKey(pkiPath, encoded); err != nil {
+		return errors.Wrapf(err, "unable to write private key to file %s", pkiPath)
+	}
+
+	return nil
+}
+
+// WriteCert stores the given certificate at the given location
+func WriteCert(certPath string, cert *x509.Certificate) error {
+	if cert == nil {
+		return errors.New("certificate cannot be nil when writing to file")
+	}
+
+	if err := certutil.WriteCert(certPath, EncodeCertPEM(cert)); err != nil {
+		return errors.Wrapf(err, "unable to write certificate to file %s", certPath)
+	}
+
+	return nil
+}
+
+// EncodeCertPEM returns PEM-endcoded certificate data
+func EncodeCertPEM(cert *x509.Certificate) []byte {
+	block := pem.Block{
+		Type:  CertificateBlockType,
+		Bytes: cert.Raw,
+	}
+	return pem.EncodeToMemory(&block)
+}
+
+func pathForKey(pkiPath, name string) string {
+	return filepath.Join(pkiPath, fmt.Sprintf("%s.key", name))
+}
+
+func pathForCert(pkiPath, name string) string {
+	return filepath.Join(pkiPath, fmt.Sprintf("%s.crt", name))
+}

edge/pkg/edgehub/common/certutil/certutil.go

+package certutil
+
+import (
+	"bytes"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/hex"
+	"encoding/pem"
+	"fmt"
+	"github.com/kubeedge/kubeedge/edge/pkg/edgehub/common/http"
+	"github.com/kubeedge/kubeedge/edge/pkg/edgehub/config"
+	"io/ioutil"
+	"os"
+)
+
+const privateKeyBits = 2048
+
+// GetCACert gets the cloudcore CA certificate
+func GetCACert(url string) ([]byte, error) {
+	client := http.NewHTTPClient()
+	req, _ := http.BuildRequest("GET", url, nil, "")
+	res, err := http.SendRequest(req, client)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	cacert, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+	return cacert, nil
+}
+
+func getCSR() ([]byte, error) {
+	pk, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	// save the private key
+	if err := WriteKey(config.Config.TLSPrivateKeyFile, pk); err != nil {
+		return nil, err
+	}
+
+	certReq := &x509.CertificateRequest{
+		Subject: pkix.Name{
+			Country:            []string{"CN"},
+			Organization:       []string{"kubeEdge"},
+			OrganizationalUnit: []string{},
+			Locality:           []string{"Hangzhou"},
+			Province:           []string{"Zhejiang"},
+			CommonName:         "kubeedge.io",
+		},
+	}
+	return x509.CreateCertificateRequest(rand.Reader, certReq, pk)
+}
+
+// GetEdgeCert applies for the certificate from cloudcore
+func GetEdgeCert(url string, cacert []byte, token string) ([]byte, error) {
+	csr, err := getCSR()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create CSR: %v", err)
+	}
+	client, err := http.NewHTTPclientWithCA(cacert)
+	req, _ := http.BuildRequest("GET", url, bytes.NewReader(csr), token)
+	res, err := http.SendRequest(req, client)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	edgecert, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+	return edgecert, nil
+}
+
+// SaveToFile saves the certificate or private key
+func SaveToFile(data []byte, file string, pemBlockType string) error {
+	out, err := os.Create(file)
+	defer out.Close()
+	if err != nil {
+		return fmt.Errorf("failed to create file: %s", file)
+	}
+	if err = pem.Encode(out, &pem.Block{Type: pemBlockType, Bytes: data}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func hashCA(cacerts []byte) string {
+	digest := sha256.Sum256(cacerts)
+	return hex.EncodeToString(digest[:])
+}
+
+// ValidateCACerts validates the CA certificate by hash code
+func ValidateCACerts(cacerts []byte, hash string) (bool, string, string) {
+	if len(cacerts) == 0 && hash == "" {
+		return true, "", ""
+	}
+
+	newHash := hashCA(cacerts)
+	return hash == newHash, hash, newHash
+}

edge/pkg/edgehub/common/http/http.go

@@ -3,6 +3,8 @@ package http
 import (
 	"crypto/tls"
 	"crypto/x509"
+	"encoding/pem"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -62,6 +64,22 @@ func NewHTTPSclient(certFile, keyFile string) (*http.Client, error) {
 	return client, nil
 }
 
+// NewHTTPclientWithCA create client without certificate
+func NewHTTPclientWithCA(ca []byte) (*http.Client, error) {
+	pool := x509.NewCertPool()
+	if ok := pool.AppendCertsFromPEM(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca})); !ok {
+		return nil, fmt.Errorf("cannot parse the certificates")
+	}
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{
+			RootCAs:            pool,
+			InsecureSkipVerify: false,
+		},
+	}
+	client := &http.Client{Transport: tr, Timeout: connectTimeout}
+	return client, nil
+}
+
 // SendRequest sends a http request and return the resp info
 func SendRequest(req *http.Request, client *http.Client) (*http.Response, error) {
 	resp, err := client.Do(req)
@@ -78,8 +96,8 @@ func BuildRequest(method string, urlStr string, body io.Reader, token string) (*
 		return nil, err
 	}
 	if token != "" {
-		req.Header.Add("X-Auth-Token", token)
+		bearerToken := "Bearer " + token
+		req.Header.Add("Authorization", bearerToken)
 	}
-	req.Header.Add("Content-Type", "application/json")
 	return req, nil
 }

edge/pkg/edgehub/edgehub.go

 package edgehub
 
 import (
+	"crypto/tls"
 	"sync"
 	"time"
 
@@ -13,6 +14,7 @@ import (
 	"github.com/kubeedge/kubeedge/edge/pkg/edgehub/clients"
 	"github.com/kubeedge/kubeedge/edge/pkg/edgehub/config"
 	"github.com/kubeedge/kubeedge/pkg/apis/componentconfig/edgecore/v1alpha1"
+	"github.com/kubeedge/kubeedge/pkg/util/validation"
 )
 
 //define edgehub module name
@@ -60,6 +62,21 @@ func (eh *EdgeHub) Enable() bool {
 
 //Start sets context and starts the controller
 func (eh *EdgeHub) Start() {
+	// if there is no manual certificate setting or the setting has problems, then the edge applies for the certificate
+	if validation.FileIsExist(config.Config.TLSCAFile) && validation.FileIsExist(config.Config.TLSCertFile) && validation.FileIsExist(config.Config.TLSPrivateKeyFile) {
+		_, err := tls.LoadX509KeyPair(config.Config.TLSCertFile, config.Config.TLSPrivateKeyFile)
+		if err != nil {
+			if err := eh.applyCerts(); err != nil {
+				klog.Fatalf("failed to apply for edge certificate, error: %v", err)
+				return
+			}
+		}
+	} else {
+		if err := eh.applyCerts(); err != nil {
+			klog.Fatalf("failed to apply for edge certificate, error: %v", err)
+			return
+		}
+	}
 
 	for {
 		select {

edge/pkg/edgehub/process.go

 package edgehub
 
 import (
+	"crypto/x509"
 	"fmt"
+	"strings"
 	"time"
 
 	"k8s.io/klog"
@@ -12,12 +14,15 @@ import (
 	"github.com/kubeedge/kubeedge/edge/pkg/common/message"
 	"github.com/kubeedge/kubeedge/edge/pkg/common/modules"
 	"github.com/kubeedge/kubeedge/edge/pkg/edgehub/clients"
+	"github.com/kubeedge/kubeedge/edge/pkg/edgehub/common/certutil"
 	"github.com/kubeedge/kubeedge/edge/pkg/edgehub/config"
 )
 
 const (
 	waitConnectionPeriod = time.Minute
 	authEventType        = "auth_info_event"
+	caURL                = "/ca.crt"
+	certURL              = "/edge.crt"
 )
 
 var groupMap = map[string]string{
@@ -27,6 +32,54 @@ var groupMap = map[string]string{
 	"user":     modules.BusGroup,
 }
 
+// applyCerts get edge certificate to communicate with cloudcore
+func (eh *EdgeHub) applyCerts() error {
+	// get ca.crt
+	url := config.Config.HTTPServer + caURL
+	cacert, err := certutil.GetCACert(url)
+	if err != nil {
+		klog.Errorf("failed to get CA certificate, err: %v", err)
+		return fmt.Errorf("failed to get CA certificate, err: %v", err)
+	}
+
+	// validate the CA certificate by hashcode
+	tokenParts := strings.Split(config.Config.Token, " ")
+	if len(tokenParts) != 2 {
+		return fmt.Errorf("token credentials are in the wrong format")
+	}
+	ok, hash, newHash := certutil.ValidateCACerts(cacert, tokenParts[0])
+	if !ok {
+		klog.Errorf("failed to validate CA certificate. tokenCAhash: %s, CAhash: %s", hash, newHash)
+		return fmt.Errorf("failed to validate CA certificate. tokenCAhash: %s, CAhash: %s", hash, newHash)
+	}
+	// save the ca.crt to file
+	ca, err := x509.ParseCertificate(cacert)
+	if err != nil {
+		klog.Errorf("failed to parse the CA certificate, error: %v", err)
+		return fmt.Errorf("failed to parse the CA certificate, error: %v", err)
+	}
+
+	if err = certutil.WriteCert(config.Config.TLSCAFile, ca); err != nil {
+		klog.Errorf("failed to save the CA certificate to file: %s, error: %v", config.Config.TLSCAFile, err)
+		return fmt.Errorf("failed to save the CA certificate to file: %s, error: %v", config.Config.TLSCAFile, err)
+	}
+
+	// get the edge.crt
+	url = config.Config.HTTPServer + certURL
+	edgecert, err := certutil.GetEdgeCert(url, cacert, tokenParts[1])
+	if err != nil {
+		klog.Errorf("failed to get edge certificate from the cloudcore, error: %v", err)
+		return fmt.Errorf("failed to get edge certificate from the cloudcore, error: %v", err)
+	}
+	// save the edge.crt to the file
+	cert, _ := x509.ParseCertificate(edgecert)
+	if err = certutil.WriteCert(config.Config.TLSCertFile, cert); err != nil {
+		klog.Errorf("failed to save the edge certificate to file: %s, error: %v", config.Config.TLSCertFile, err)
+		return fmt.Errorf("failed to save the edge certificate to file: %s, error: %v", config.Config.TLSCertFile, err)
+	}
+	return nil
+}
+
 func (eh *EdgeHub) initial() (err error) {
 
 	cloudHubClient, err := clients.GetClient()

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50 // indirect
 	github.com/containerd/containerd v1.1.7 // indirect
 	github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect

hack/local-up-kubeedge.sh

@@ -93,7 +93,6 @@ function start_cloudcore {
   CLOUD_BIN=${KUBEEDGE_ROOT}/_output/local/bin/cloudcore
   ${CLOUD_BIN} --minconfig >  ${CLOUD_CONFIGFILE}
   sed -i "s|kubeConfig: .*|kubeConfig: ${KUBECONFIG}|g" ${CLOUD_CONFIGFILE}
-
   CLOUDCORE_LOG=${LOG_DIR}/cloudcore.log
   echo "start cloudcore..."
   nohup ${CLOUD_BIN} --config=${CLOUD_CONFIGFILE} > "${CLOUDCORE_LOG}" 2>&1 &
@@ -104,6 +103,10 @@ function start_edgecore {
   EDGE_CONFIGFILE=${KUBEEDGE_ROOT}/_output/local/bin/edgecore.yaml
   EDGE_BIN=${KUBEEDGE_ROOT}/_output/local/bin/edgecore
   ${EDGE_BIN} --minconfig >  ${EDGE_CONFIGFILE}
+
+  token=`kubectl get secret -nkubeedge tokensecret -o=jsonpath='{.data.tokendata}' | base64 -d`
+
+  sed -i "s|token: .*|token: ${token}|g" ${EDGE_CONFIGFILE}
   sed -i "s|hostnameOverride: .*|hostnameOverride: edge-node|g" ${EDGE_CONFIGFILE}
   EDGECORE_LOG=${LOG_DIR}/edgecore.log
 
@@ -149,11 +152,15 @@ check_control_plane_ready
 
 # edge side don't support kind cni now, delete kind cni plugin for workaround
 kubectl delete daemonset kindnet -nkube-system
+kubectl create ns kubeedge
 
 create_device_crd
 create_objectsync_crd
-generate_certs
+
 start_cloudcore
+
+sleep 2
+
 start_edgecore
 
 if [[ "${ENABLE_DAEMON}" = false ]]; then

keadm/cmd/keadm/app/cmd/util/cloudconfig.go

@@ -41,7 +41,7 @@ genCertAndKey() {
 }
 
 buildSecret() {
-    local name="edge"
+    local name=$1
     genCertAndKey ${name} > /dev/null 2>&1
     cat <<EOF
 apiVersion: v1