Merge pull request #578 from oldthreefeng/fix_join_node

 join node use config file

.github/workflows/go.yml

@@ -25,11 +25,11 @@ jobs:
         version: latest
         args: build --snapshot --rm-dist  --timeout=1h
 
-    - name: Build binareies to oss
-      env:
-        BUCKETNAME: ${{ secrets.BUCKETNAME }}
-      run: |
-        echo ${{ secrets.OSS_CONFIG }} |base64 --decode >> .ossutilconfig
-        wget http://gosspublic.alicdn.com/ossutil/1.6.19/ossutil64  && chmod 755 ossutil64
-        ./ossutil64 -c .ossutilconfig cp -f dist/sealos_linux_amd64/sealos oss://${BUCKETNAME:-sealyun}/latest/sealos
-        ./ossutil64 -c .ossutilconfig cp -f dist/sealos_linux_arm64/sealos oss://${BUCKETNAME:-sealyun}/latest/sealos-arm64
+    # - name: Build binareies to oss
+    #   env:
+    #     BUCKETNAME: ${{ secrets.BUCKETNAME }}
+    #   run: |
+    #     echo ${{ secrets.OSS_CONFIG }} |base64 --decode >> .ossutilconfig
+    #     wget http://gosspublic.alicdn.com/ossutil/1.6.19/ossutil64  && chmod 755 ossutil64
+    #     ./ossutil64 -c .ossutilconfig cp -f dist/sealos_linux_amd64/sealos oss://${BUCKETNAME:-sealyun}/latest/sealos
+    #     ./ossutil64 -c .ossutilconfig cp -f dist/sealos_linux_arm64/sealos oss://${BUCKETNAME:-sealyun}/latest/sealos-arm64

install/constants.go

@@ -23,6 +23,10 @@ const (
 	// kube file
 	KUBECONTROLLERCONFIGFILE = "/etc/kubernetes/controller-manager.conf"
 	KUBESCHEDULERCONFIGFILE  = "/etc/kubernetes/scheduler.conf"
+
+	// CriSocket
+	DefaultDockerCRISocket     = "/var/run/dockershim.sock"
+	DefaultContainerdCRISocket = "/run/containerd/containerd.sock"
 )
 
 const InitTemplateTextV1beta1 = string(`apiVersion: kubeadm.k8s.io/v1beta1
@@ -86,38 +90,28 @@ ipvs:
   excludeCIDRs:
   - "{{.VIP}}/32"`)
 
-const JoinCPTemplateTextV1beate2Container = string(`apiVersion: kubeadm.k8s.io/v1beta2	
+const JoinCPTemplateTextV1beta2 = string(`apiVersion: kubeadm.k8s.io/v1beta2
 caCertPath: /etc/kubernetes/pki/ca.crt
 discovery:
   bootstrapToken:
+    {{- if .Master}}
     apiServerEndpoint: {{.Master0}}:6443
+    {{else}}
+    apiServerEndpoint: {{.VIP}}:6443
+    {{end -}}
     token: {{.TokenDiscovery}}
     caCertHashes:
     - {{.TokenDiscoveryCAHash}}
   timeout: 5m0s
 kind: JoinConfiguration
+{{- if .Master }}
 controlPlane:
   localAPIEndpoint:
     advertiseAddress: {{.Master}}
     bindPort: 6443
+{{- end}}
 nodeRegistration:
-  criSocket: /run/containerd/containerd.sock`)
-
-const JoinCPTemplateTextV1beta2 = string(`apiVersion: kubeadm.k8s.io/v1beta2
-caCertPath: /etc/kubernetes/pki/ca.crt
-discovery:
-  bootstrapToken:
-    apiServerEndpoint: {{.Master0}}:6443
-    token: {{.TokenDiscovery}}
-    caCertHashes:
-    - {{.TokenDiscoveryCAHash}}
-  timeout: 5m0s
-kind: JoinConfiguration
-controlPlane:
-  localAPIEndpoint:
-    advertiseAddress: {{.Master}}
-    bindPort: 6443`)
-
+  criSocket: {{.CriSocket}}`)
 
 const InitTemplateTextV1bate2 = string(`apiVersion: kubeadm.k8s.io/v1beta2
 kind: InitConfiguration
@@ -180,4 +174,4 @@ kind: KubeProxyConfiguration
 mode: "ipvs"
 ipvs:
   excludeCIDRs:
-  - "{{.VIP}}/32"`)
\ No newline at end of file
+  - "{{.VIP}}/32"`)

install/generator.go

@@ -24,11 +24,7 @@ func Config() {
 
 func joinKubeadmConfig() string {
 	var sb strings.Builder
-	if For120(Version) {
-		sb.Write([]byte(JoinCPTemplateTextV1beate2Container))
-	} else {
-		sb.Write([]byte(JoinCPTemplateTextV1beta2))
-	}
+	sb.Write([]byte(JoinCPTemplateTextV1beta2))
 	return sb.String()
 }
 
@@ -74,9 +70,16 @@ func JoinTemplateFromTemplateContent(templateContent, ip string) []byte {
 	}
 	var envMap = make(map[string]interface{})
 	envMap["Master0"] = IpFormat(MasterIPs[0])
-	envMap["Master"] = IpFormat(ip)
+	envMap["Master"] = ip
 	envMap["TokenDiscovery"] = JoinToken
 	envMap["TokenDiscoveryCAHash"] = TokenCaCertHash
+	envMap["VIP"] = VIP
+	if For120(Version) {
+		CriSocket = DefaultContainerdCRISocket
+	} else {
+		CriSocket = DefaultDockerCRISocket
+	}
+	envMap["CriSocket"] = CriSocket
 	var buffer bytes.Buffer
 	_ = tmpl.Execute(&buffer, envMap)
 	return buffer.Bytes()

install/generator_test.go

@@ -19,6 +19,7 @@ func TestTemplate(t *testing.T) {
 	}
 	MasterIPs = masters
 	VIP = vip
+	ApiServer = "apiserver.cluster.local"
 	config.Cmd("127.0.0.1", "echo \""+string(Template())+"\" > ~/aa")
 	t.Log(string(Template()))
 }
@@ -114,15 +115,21 @@ func TestJoinTemplate(t *testing.T) {
 	config := sshutil.SSH{
 		User:     "louis",
 		Password: "210010",
+		PkFile: "/home/louis/.ssh/id_rsa",
 	}
+	Version = "v1.20.0"
 	MasterIPs = masters
 	JoinToken = "1y6yyl.ramfafiy99vz3tbw"
 	TokenCaCertHash = "sha256:a68c79c87368ff794ae50c5fd6a8ce13fdb2778764f1080614ddfeaa0e2b9d14"
 
 
 	VIP = vip
-	config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate(masters[0]))+"\" > ~/aa")
-	t.Log(string(JoinTemplate(masters[0])))
+	config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate(IpFormat(masters[0])))+"\" > ~/aa")
+	t.Log(string(JoinTemplate(IpFormat(masters[0]))))
+	
+	Version = "v1.19.0"
+	config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate(""))+"\" > ~/aa")
+	t.Log(string(JoinTemplate("")))
 }
 
 var tepJoin = `apiVersion: kubeadm.k8s.io/v1beta2

install/join.go

@@ -90,7 +90,7 @@ func sendJoinCPConfig(joinMaster []string) {
 		wg.Add(1)
 		go func(master string) {
 			defer wg.Done()
-			templateData := string(JoinTemplate(master))
+			templateData := string(JoinTemplate(IpFormat(master)))
 			cmd := fmt.Sprintf(`echo "%s" > /root/kubeadm-join-config.yaml`, templateData)
 			_ = SSHConfig.CmdAsync(master, cmd)
 		}(master)
@@ -140,11 +140,15 @@ func (s *SealosInstaller) JoinNodes() {
 		masters += fmt.Sprintf(" --rs %s:6443", IpFormat(master))
 	}
 	ipvsCmd := fmt.Sprintf("sealos ipvs --vs %s:6443 %s --health-path /healthz --health-schem https --run-once", VIP, masters)
-
+	templateData := string(JoinTemplate(""))
 	for _, node := range s.Nodes {
 		wg.Add(1)
 		go func(node string) {
 			defer wg.Done()
+			// send join node config
+			cmdJoinConfig := fmt.Sprintf(`echo "%s" > /root/kubeadm-join-config.yaml`, templateData)
+			_ = SSHConfig.CmdAsync(node, cmdJoinConfig)
+
 			cmdHosts := fmt.Sprintf("echo %s %s >> /etc/hosts", VIP, ApiServer)
 			_ = SSHConfig.CmdAsync(node, cmdHosts)
 

install/sealos.go

@@ -54,16 +54,19 @@ const JoinNode CommandType = "joinNode"
 
 func (s *SealosInstaller) Command(version string, name CommandType) (cmd string) {
 	cmds := make(map[CommandType]string)
+	// Please convert your v1beta1 configuration files to v1beta2 using the 
+	// "kubeadm config migrate" command of kubeadm v1.15.x, 因此1.14 版本不支持双网卡. 
 	cmds = map[CommandType]string{
 		InitMaster: `kubeadm init --config=/root/kubeadm-config.yaml --experimental-upload-certs` + vlogToStr(),
-		JoinMaster: "kubeadm join %s:6443 --config=/root/kubeadm-join-config.yaml "+vlogToStr(),
+		JoinMaster: fmt.Sprintf("kubeadm join %s:6443 --token %s --discovery-token-ca-cert-hash %s --experimental-control-plane --certificate-key %s"+vlogToStr(), IpFormat(s.Masters[0]), JoinToken, TokenCaCertHash, CertificateKey),
 		JoinNode:   fmt.Sprintf("kubeadm join %s:6443 --token %s --discovery-token-ca-cert-hash %s"+vlogToStr(), VIP, JoinToken, TokenCaCertHash),
 	}
-	//other version
+	//other version >= 1.15.x
 	//todo
 	if VersionToInt(version) >= 115 {
 		cmds[InitMaster] = `kubeadm init --config=/root/kubeadm-config.yaml --upload-certs` + vlogToStr()
 		cmds[JoinMaster] = "kubeadm join --config=/root/kubeadm-join-config.yaml "+vlogToStr()
+		cmds[JoinNode] =   "kubeadm join --config=/root/kubeadm-join-config.yaml "+vlogToStr()
 	}
 
 	v, ok := cmds[name]

install/utils_test.go

 package install
 
 import (
+	"fmt"
 	"reflect"
 	"strings"
 	"testing"
@@ -23,6 +24,8 @@ func TestPrint(t *testing.T) {
 
 func TestVersionToInt(t *testing.T) {
 	t.Log(ParseIPs([]string{"172.26.13.133-172.26.13.136:2222"}))
+	fmt.Println(VersionToInt("v1.15.6"))
+	fmt.Println(VersionToInt("v1.18.6-rc.0"))
 }
 
 func TestSliceRemoveStr(t *testing.T) {

install/vars.go

@@ -28,6 +28,9 @@ var (
 	EtcdCert     = cert.SealosConfigDir + "/pki/etcd/healthcheck-client.crt"
 	EtcdKey      = cert.SealosConfigDir + "/pki/etcd/healthcheck-client.key"
 
+	//criSocket
+	CriSocket string
+
 	VIP     string
 	PkgUrl  string
 	Version string