etcd init

Makefile

@@ -15,10 +15,10 @@ cicd: build ## 运行CI/CD测试
 ssh: ## CentOS使用root登录
 	ssh -q root@10.24.0.10 -i .vagrant/machines/master0/virtualbox/private_key
 
-certs: build
+certs: build ## 生成docker证书
 	./bin/vik8s -f ./bin docker --tls.enable --hosts "tcp://{IP}:2375"
 
-testcerts:
+testcerts: ## 测试证书
 	DOCKER_TLS_VERIFY="0" \
    	DOCKER_HOST="tcp://10.24.0.10:2375" \
 	DOCKER_CERT_PATH=~/workbench/self/go/vik8s/bin/default/ \

Vagrantfile

@@ -24,6 +24,12 @@ Vagrant.configure("2") do |config|
   config.vbguest.iso_path = guest_iso_path
   config.vbguest.auto_update = false
 
+  if box != "ubuntu"
+    config.vm.provision "ansible" do |ansible|
+      ansible.playbook = "playbook.yml"
+    end
+  end
+
   config.vm.define "master0" do |node|
     node.vm.hostname = "master0.vik8s.com"
     node.vm.network "private_network", ip: "10.24.0.10"
@@ -31,11 +37,6 @@ Vagrant.configure("2") do |config|
       v.cpus = "2"
       v.memory = "2048"
     end
-    if box != "ubuntu"
-      node.vm.provision "ansible" do |ansible|
-        ansible.playbook = "playbook.yml"
-      end
-    end
   end
 
   # config.vm.define "slave20" do |node|

certs/docker/generate.go

@@ -41,7 +41,7 @@ func GenerateServerCertificates(node *ssh.Node, options *config.DockerCertsConfi
 		{
 			config.Organization = []string{"system:server"}
 			config.Usages = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}
-			config.AltNames = *certs.GetAltNames([]string{node.Host, "127.0.0.1", "localhost"}, "server")
+			config.AltNames = *certs.GetAltNames([]string{node.Hostname, node.Host, "127.0.0.1", "localhost"}, "server")
 		}
 		serverCert, serverKey := certs.NewCertAndKey(caCert, caKey, config)
 		certs.WriteCertAndKey(dir, "server-"+node.Host, serverCert, serverKey)

cmd/etcd.go

@@ -16,18 +16,14 @@ var etcdCmd = &cobra.Command{
 This program uses etcdadm for installation, for details https://github.com/kubernetes-sigs/etcdadm`,
 }
 
-func init() {
-	etcdCmd.AddCommand(etcdInitCmd, etcdJoinCmd, etcdResetCmd)
-}
-
 var etcdConfig = new(config.ETCD)
 var etcdInitCmd = &cobra.Command{
 	Use: "init", Short: "Initialize a new etcd cluster", Args: cobra.MinimumNArgs(1),
 	PreRunE: configLoad(hostsLoad(none)), PostRunE: configDown(none),
-	Example: `
-	vik8s etcd init 172.16.100.11-172.16.100.13
-	vik8s etcd init 172.16.100.11 172.16.100.12 172.16.100.13`,
+	Example: `  vik8s etcd init 172.16.100.11-172.16.100.13
+  vik8s etcd init 172.16.100.11 172.16.100.12 172.16.100.13`,
 	RunE: func(cmd *cobra.Command, args []string) (err error) {
+		config.Config.ETCD = etcdConfig
 		ips := hosts.Add(args...)
 		hosts.MustGatheringFacts(ips...)
 		etcd.InitCluster(ips[0])
@@ -51,7 +47,6 @@ var etcdJoinCmd = &cobra.Command{
 	Args:    cobra.MinimumNArgs(1),
 	PreRunE: configLoad(hostsLoad(none)), PostRunE: configDown(none),
 	Run: func(cmd *cobra.Command, args []string) {
-		etcd.Config.MustRead()
 		ips := hosts.Add(args...)
 		for _, ip := range ips {
 			etcd.JoinCluster(ip)
@@ -66,10 +61,9 @@ var etcdResetCmd = &cobra.Command{
 reset one node: vik8s etcd reset 172.16.100.10`,
 	PreRunE: configLoad(hostsLoad(none)), PostRunE: configDown(none),
 	Run: func(cmd *cobra.Command, args []string) {
-		etcd.Config.MustRead()
 		ips := utils.ParseIPS(args)
 		if len(ips) == 0 {
-			ips = etcd.Config.Nodes
+			ips = config.Config.ETCD.Nodes
 		}
 		nodes := hosts.Add(ips...)
 		for _, node := range nodes {
@@ -78,3 +72,7 @@ reset one node: vik8s etcd reset 172.16.100.10`,
 		fmt.Println("-=-=-=- SUCCESS -=-=-=-")
 	},
 }
+
+func init() {
+	etcdCmd.AddCommand(etcdInitCmd, etcdJoinCmd, etcdResetCmd)
+}

config/etcd.go

@@ -6,14 +6,13 @@ import (
 
 type ETCD struct {
 	Nodes               []string `flag:"-"`
-	Version             string   `def:"3.4.13" help:"etcd version"`
+	Version             string   `def:"v3.4.13" help:"etcd version"`
 	ServerCertExtraSans []string `help:"optional extra Subject Alternative Names for the etcd server signing cert, can be multiple comma separated DNS names or IPs"`
 
 	CertsValidity time.Duration `def:"4y" help:"Certificate validity time"`
 	CertsDir      string        `def:"/etc/etcd/pki" help:"certificates directory"`
+	Data          string        `def:"/var/lib/etcd" help:"etcd data dir"`
 
 	Snapshot       string `help:"Etcd v3 snapshot (local disk) file used to initialize member"`
 	RemoteSnapshot string `help:"Etcd v3 snapshot (remote disk at first node) file used to initialize member"`
-
-	Source string `help:"the etcdadm source. if chain https://gitee.com/ihaiker/etcdadm else https://github.com/kubernetes-sigs/etcdadm"`
 }

install/etcd/config.go

-package etcd
-
-import (
-	"encoding/json"
-	"fmt"
-	"github.com/ihaiker/vik8s/install/hosts"
-	"github.com/ihaiker/vik8s/install/paths"
-	"github.com/ihaiker/vik8s/libs/utils"
-	"io/ioutil"
-	"os"
-	"time"
-)
-
-type ETCD struct {
-	SSH hosts.Option `json:"ssh"`
-
-	Nodes               []string `json:"nodes,omitempty"`
-	Version             string   `json:"version,omitempty"`
-	ServerCertExtraSans []string `json:"server-cert-extra-sans,omitempty"`
-
-	CertsValidity time.Duration `json:"certsValidity"`
-	CertsDir      string        `json:"certDir,omitempty"`
-
-	Snapshot       string `json:"snapshot,omitempty"`
-	RemoteSnapshot string `json:"remoteSnapshot,omitempty"`
-
-	Source string `json:"source,omitempty"`
-}
-
-func (etcd *ETCD) Write() {
-	etcdConfig := paths.Join("etcd.json")
-
-	if len(Config.Nodes) == 0 {
-		_ = os.Remove(etcdConfig)
-		return
-	}
-
-	bs, _ := json.MarshalIndent(etcd, "", "    ")
-	defer utils.Catch(func(err error) {
-		fmt.Println("write config error ", err)
-		fmt.Println("Be sure to save the following content to " + etcdConfig + ", very important! very important! very important! 重要！重要！重要！")
-		fmt.Println(string(bs))
-	})
-	utils.Panic(os.MkdirAll(paths.Join(), os.ModePerm), "mkdir config file dir")
-	utils.Panic(ioutil.WriteFile(etcdConfig, bs, 0666), "write config file")
-}
-
-func (etcd *ETCD) MustRead() {
-	etcdConfigLoc := paths.Join("etcd.json")
-
-	etcdConfigBytes, err := ioutil.ReadFile(etcdConfigLoc)
-	utils.Panic(err, "read etcd config file %s", etcdConfigLoc)
-
-	err = json.Unmarshal(etcdConfigBytes, Config)
-	utils.Panic(err, "read etcd config file %s", etcdConfigLoc)
-}
-func (etcd *ETCD) Read() error {
-	etcdConfigLoc := paths.Join("etcd.json")
-
-	etcdConfigBytes, err := ioutil.ReadFile(etcdConfigLoc)
-	if err != nil {
-		return utils.Wrap(err, "read etcd config file %s", etcdConfigLoc)
-	}
-	err = json.Unmarshal(etcdConfigBytes, Config)
-	if err != nil {
-		return utils.Wrap(err, "read etcd config file %s", etcdConfigLoc)
-	}
-	return nil
-}
-
-func (etcd *ETCD) Exists(ip string) bool {
-	for _, node := range etcd.Nodes {
-		if node == ip {
-			return true
-		}
-	}
-	return false
-}
-
-func (etcd *ETCD) Join(ip string) {
-	etcd.Nodes = append(etcd.Nodes, ip)
-	etcd.Write()
-}
-
-func (etcd *ETCD) Remove(ip string) {
-	for i, node := range etcd.Nodes {
-		if node == ip {
-			etcd.Nodes = append(etcd.Nodes[0:i], etcd.Nodes[i+1:]...)
-			break
-		}
-	}
-	etcd.Write()
-}
-
-var Config = new(ETCD)

install/etcd/init.go

@@ -3,6 +3,7 @@ package etcd
 import (
 	"fmt"
 	etcdcerts "github.com/ihaiker/vik8s/certs/etcd"
+	"github.com/ihaiker/vik8s/config"
 	"github.com/ihaiker/vik8s/install/bases"
 	"github.com/ihaiker/vik8s/install/cri"
 	"github.com/ihaiker/vik8s/install/paths"
@@ -17,24 +18,30 @@ func InitCluster(node *ssh.Node) {
 	node.Logger("install etcd server")
 	bases.Check(node)
 	cri.Install(node)
-	if true {
-		return
-	}
-
-	checkEtcdadm(node)
+	pullContainerImage(node)
 	makeAndPushCerts(node)
-	etcdadmInit(node)
-	Config.Join(node.Host)
+}
+
+func pullContainerImage(node *ssh.Node) {
+	var err error
+	if config.Config.Docker != nil {
+		dockerUrl := fmt.Sprintf("docker pull %s/%s:%s", repo.QuayIO(""), "coreos/etcd", config.Config.ETCD.Version)
+		err = node.SudoCmdOutput(dockerUrl, os.Stdout)
+	} else {
+		err = node.SudoCmdOutput("ctr pull ", os.Stdout)
+	}
+	utils.Panic(err, "pull image")
 }
 
 func makeAndPushCerts(node *ssh.Node) {
 	node.Logger("make certs files")
+
 	name := node.Hostname
 	dir := paths.Join("etcd", "pki")
 	sans := []string{"127.0.0.1", "localhost", node.Hostname, node.Host}
-	sans = append(sans, utils.ParseIPS(Config.Nodes)...)
-	sans = append(sans, Config.ServerCertExtraSans...)
-	vt := Config.CertsValidity
+	sans = append(sans, utils.ParseIPS(config.Config.ETCD.Nodes)...)
+	sans = append(sans, config.Config.ETCD.ServerCertExtraSans...)
+	vt := config.Config.ETCD.CertsValidity
 	etcdcerts.CreatePKIAssets(name, dir, sans, vt)
 
 	certsFiles := map[string]string{
@@ -46,97 +53,11 @@ func makeAndPushCerts(node *ssh.Node) {
 		"healthcheck-client":    "healthcheck-client",
 	}
 
-	for lf, rf := range certsFiles {
+	for localFile, remoteFile := range certsFiles {
 		for _, exp := range []string{".key", ".crt"} {
-			local := filepath.Join(dir, lf+exp)
-			remote := filepath.Join(Config.CertsDir, rf+exp)
-			utils.Panic(node.Scp(local, remote), "scp %s %s", local, remote)
-		}
-	}
-}
-
-func checkEtcdadm(node *ssh.Node) {
-	utils.Line("check and install etcdadm")
-
-	etcdadm, err := node.Cmd2String("command -v etcdadm")
-	if err != nil {
-		etcdadm = installEtcdadm(node)
-	}
-	//etcdadm
-	{
-		local := paths.Join("etcd", "etcdadm")
-		if utils.NotExists(local) {
-			err = node.Pull(etcdadm, local)
-			utils.Panic(err, "pull etcdadm")
-		}
-	}
-	//etcd.tar.gz
-	{
-		tar := fmt.Sprintf("etcd-v%s-linux-amd64.tar.gz", Config.Version)
-		local := paths.Join("etcd", tar)
-		if utils.Exists(local) {
-			remote := fmt.Sprintf("/var/cache/etcdadm/etcd/v%s/%s", Config.Version, tar)
-			err := node.Scp(local, remote)
-			utils.Panic(err, "scp %s %s", local, remote)
-		}
-	}
-}
-
-func installEtcdadm(node *ssh.Node) string {
-	remoteBin := "/usr/local/bin/etcdadm"
-	localBin := paths.Join("etcd", "etcdadm")
-
-	if utils.NotExists(localBin) {
-		node.Logger("build etcdadm")
-		bases.Install("git", "", node)
-		bases.Install("golang", "", node)
-		source := Config.Source
-		if source == "" {
-			source = repo.Etcdadm()
-		}
-		goProxy := ""
-		if paths.China {
-			goProxy = `export GOPROXY="https://goproxy.io"`
+			local := filepath.Join(dir, localFile+exp)
+			remote := filepath.Join(config.Config.ETCD.CertsDir, remoteFile+exp)
+			utils.Panic(node.SudoScp(local, remote), "scp %s %s", local, remote)
 		}
-		shell := fmt.Sprintf(`
-cd /tmp
-git clone %s --local etcdadm
-cd etcdadm
-%s
-go build
-mv -f etcdadm %s`, source, goProxy, remoteBin)
-		err := node.ShellChannel(shell, utils.Stdout(node.Hostname))
-		utils.Panic(err, "make etcdadm")
-	} else {
-		err := node.Scp(localBin, remoteBin)
-		utils.Panic(err, "scp etcdadm")
-		node.MustCmd(fmt.Sprintf("chmod +x %s", remoteBin))
-	}
-	return remoteBin
-}
-
-func etcdadmInit(master *ssh.Node) {
-	utils.Line("etcdadm init")
-	cmd := "etcdadm init --name " + master.Hostname +
-		" --install-dir /usr/local/bin " +
-		" --certs-dir " + Config.CertsDir +
-		" --version " + Config.Version
-	if Config.Snapshot != "" {
-		cmd += " --snapshot " + Config.Snapshot
-	}
-	/* use certs make
-	for _, san := range Config.ServerCertExtraSans {
-		cmd += " --server-cert-extra-sans " + san
-	}
-	*/
-	err := master.CmdStd(cmd, os.Stdout)
-	utils.Panic(err, "etcdadm init")
-
-	tar := fmt.Sprintf("etcd-v%s-linux-amd64.tar.gz", Config.Version)
-	local := paths.Join("etcd", tar)
-	if utils.NotExists(local) {
-		remote := fmt.Sprintf("/var/cache/etcdadm/etcd/v%s/%s", Config.Version, tar)
-		err = master.Pull(remote, local)
-		utils.Panic(err, "pull %s %s", remote, local)
 	}
 }

install/etcd/join.go

 package etcd
 
 import (
-	"fmt"
+	"github.com/ihaiker/vik8s/config"
 	"github.com/ihaiker/vik8s/install/bases"
+	"github.com/ihaiker/vik8s/install/cri"
 	"github.com/ihaiker/vik8s/install/hosts"
 	"github.com/ihaiker/vik8s/libs/ssh"
 	"github.com/ihaiker/vik8s/libs/utils"
-	"gopkg.in/fatih/color.v1"
-	"os"
 )
 
 func JoinCluster(node *ssh.Node) {
-
-	if Config.Exists(node.Host) {
-		fmt.Printf("%s already in the cluster\n", color.RedString(node.Host))
-		return
-	}
-	master := hosts.Get(Config.Nodes[0])
+	master := hosts.Get(config.Config.ETCD.Nodes[0])
 
 	bases.Check(node)
-	checkEtcdadm(node)
+	cri.Install(node)
+	pullContainerImage(node)
 	makeAndPushCerts(node)
-	etcdadmJoin(master, node)
 
-	Config.Join(node.Host)
+	etcdadmJoin(master, node)
 }
 
 func etcdadmJoin(master *ssh.Node, node *ssh.Node) {
-	utils.Line("etcdadm join")
-	cmd := "etcdadm join --name " + node.Hostname +
-		" --install-dir /usr/local/bin " +
-		" --certs-dir " + Config.CertsDir +
-		" --version " + Config.Version
-	/*
-		for _, san := range Config.ServerCertExtraSans {
-			cmd += " --server-cert-extra-sans " + san
-		}
-	*/
-	cmd += " https://" + master.Host + ":2379"
-	err := node.CmdStd(cmd, os.Stdout)
-	utils.Panic(err, "etcdadm join")
+	utils.Line("etcd join")
 }

install/etcd/reset.go

@@ -2,15 +2,15 @@ package etcd
 
 import (
 	"fmt"
+	"github.com/ihaiker/vik8s/config"
 	"github.com/ihaiker/vik8s/libs/ssh"
+	"github.com/ihaiker/vik8s/libs/utils"
 	"gopkg.in/fatih/color.v1"
 )
 
 func ResetCluster(node *ssh.Node) {
-	if !Config.Exists(node.Host) {
+	if utils.Search(config.Config.ETCD.Nodes, node.Host) == -1 {
 		fmt.Printf("%s not in the cluster\n", color.New(color.FgRed).Sprint(node.Host))
 		return
 	}
-	_ = node.MustCmd2String("etcdadm reset")
-	Config.Remove(node.Host)
 }

install/k8s/config.go

@@ -3,7 +3,6 @@ package k8s
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/ihaiker/vik8s/install/etcd"
 	"github.com/ihaiker/vik8s/install/hosts"
 	"github.com/ihaiker/vik8s/install/paths"
 	"github.com/ihaiker/vik8s/install/repo"
@@ -108,8 +107,9 @@ func (cfg *config) Write() {
 
 func (cfg *config) readInstallETCDCluster() {
 	//用户自己架设的外部etcd集群
-	if len(cfg.ETCD.Nodes) == 0 {
-		if err := etcd.Config.Read(); err == nil {
+	//TODO .......-----==-=-=-=-
+	/*if len(cfg.ETCD.Nodes) == 0 {
+		if err := config.Config.Read(); err == nil {
 			if len(etcd.Config.Nodes) > 0 {
 				cfg.ETCD.External = true
 				cfg.ETCD.Nodes = utils.Append(utils.ParseIPS(etcd.Config.Nodes), ":2379")
@@ -118,7 +118,7 @@ func (cfg *config) readInstallETCDCluster() {
 				cfg.ETCD.ApiServerCertFile = paths.Join("etcd/pki/apiserver-etcd-client.crt")
 			}
 		}
-	}
+	}*/
 }
 
 func (cfg *config) Parse() {

install/k8s/kubeadm_init.go

@@ -3,7 +3,6 @@ package k8s
 import (
 	"fmt"
 	"github.com/ihaiker/vik8s/install/bases"
-	"github.com/ihaiker/vik8s/install/etcd"
 	"github.com/ihaiker/vik8s/install/hosts"
 	"github.com/ihaiker/vik8s/install/paths"
 	"github.com/ihaiker/vik8s/install/tools"
@@ -40,10 +39,11 @@ func ResetNode(node *ssh.Node) {
 	Config.RemoveNode(node.Host)
 	if len(Config.Masters) == 0 && len(Config.Nodes) == 0 {
 		_ = os.RemoveAll(paths.Join("kube"))
-		if err := etcd.Config.Read(); err == nil {
+		//TODO 这里设置
+		/*if err := etcd.Config.Read(); err == nil {
 			_, _ = hosts.Get(etcd.Config.Nodes[0]).Cmd("etcdctl.sh del /registry --prefix")
 			_, _ = hosts.Get(etcd.Config.Nodes[0]).Cmd("etcdctl.sh del /calico --prefix")
-		}
+		}*/
 	}
 }
 

install/repo/main.go

@@ -86,14 +86,6 @@ func KubeletImage() string {
 	}
 }
 
-func Ceph() string {
-	if paths.China {
-		return "http://mirrors.aliyun.com/ceph"
-	} else {
-		return "https://download.ceph.com"
-	}
-}
-
 func QuayIO(repo string) string {
 	if repo != "" {
 		return repo