--story=878643981 cluster-manager支持IPv6 (merge request !454)

Squash merge branch 'clustermanager-master.2' into 'master'
feature: bcs-cluster-manager support ipv6

bcs-services/bcs-cluster-manager/api/clustermanager/Makefile

@@ -28,7 +28,7 @@ $(SWAGGEROBJ):$(PROTO)
 		--proto_path=/proto \
 		--go_out=plugins=grpc:/proto \
 		--validate_out=lang=go:/proto $<
-	sed -i 's/json:"-"/json:"-" bson:"-"/g' clustermanager.pb.go
+	sed -i '' 's/json:"-"/json:"-" bson:"-"/g' clustermanager.pb.go
 
 %.pb.gw.go: %.proto
 	docker run --rm \

bcs-services/bcs-cluster-manager/api/clustermanager/clustermanager.pb.validate.go

@@ -574,6 +574,8 @@ func (m *Node) validate(all bool) error {
 	// no validation rules for NodeName
+	// no validation rules for InnerIPv6
+
 	if len(errors) > 0 {
 		return NodeMultiError(errors)
 	}
@@ -16442,6 +16444,8 @@ func (m *ClusterNode) validate(all bool) error {
 	// no validation rules for DeviceClass
+	// no validation rules for InnerIPv6
+
 	if len(errors) > 0 {
 		return ClusterNodeMultiError(errors)
 	}

bcs-services/bcs-cluster-manager/api/clustermanager/clustermanager.proto

@@ -1277,6 +1277,10 @@ message Node {
     title: "nodeName",
     description: "节点名称, 存储各个云节点名称"
   }];
+  string innerIPv6 = 19 [(grpc.gateway.protoc_gen_swagger.options.openapiv2_field) = {
+    title: "innerIPv6",
+    description: "节点ipv6地址"
+  }];
 }
 
 message NetworkSetting {
@@ -5027,6 +5031,10 @@ message ClusterNode {
     title: "deviceClass",
     description: "机型"
   }];
+  string innerIPv6 = 24 [(grpc.gateway.protoc_gen_swagger.options.openapiv2_field) = {
+    title: "innerIPv6",
+    description: "节点ipv6地址"
+  }];
 }
 
 message ListMastersInClusterRequest {

bcs-services/bcs-cluster-manager/api/clustermanager/clustermanager.swagger.json

@@ -5969,6 +5969,11 @@
           "type": "string",
           "description": "机型",
           "title": "deviceClass"
+        },
+        "innerIPv6": {
+          "type": "string",
+          "description": "节点ipv6地址",
+          "title": "innerIPv6"
         }
       },
       "description": "记录节点信息，用于节点列表展示",
@@ -10243,6 +10248,11 @@
           "type": "string",
           "description": "节点名称, 存储各个云节点名称",
           "title": "nodeName"
+        },
+        "innerIPv6": {
+          "type": "string",
+          "description": "节点ipv6地址",
+          "title": "innerIPv6"
         }
       },
       "description": "记录节点信息，用于维护节点状态，被cluster与nodegroup使用",

bcs-services/bcs-cluster-manager/conf/bcs-cluster-manager.json.template

@@ -128,6 +128,7 @@
     "resourceSchemaPath": "${bcsResourceSchemaPath}",
     "debug": ${bcsClusterManagerDebug},
     "address": "${localIp}",
+    "ipv6Address": "${localIpv6}",
     "port": ${bcsClusterManagerPort},
     "httpport": ${bcsClusterManagerHTTPPort},
     "metricport": ${bcsClusterManagerMetricPort},

bcs-services/bcs-cluster-manager/internal/actions/cluster/list.go

@@ -536,12 +536,12 @@ func (la *ListMastersInClusterAction) validate() error {
 }
 
 func (la *ListMastersInClusterAction) listNodes() error {
-	cluster, err := la.model.GetCluster(la.ctx, la.req.ClusterID)
+	cls, err := la.model.GetCluster(la.ctx, la.req.ClusterID)
 	if err != nil {
 		blog.Errorf("get cluster %s failed, %s", la.req.ClusterID, err.Error())
 		return err
 	}
-	for _, v := range cluster.Master {
+	for _, v := range cls.Master {
 		la.nodes = append(la.nodes, transNodeToClusterNode(v))
 	}
 

bcs-services/bcs-cluster-manager/internal/actions/cluster/utils.go

@@ -21,19 +21,18 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/cloudprovider"
-	provider "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/cloudprovider/common"
-
-	"github.com/Tencent/bk-bcs/bcs-common/common/blog"
-	"github.com/Tencent/bk-bcs/bcs-common/pkg/auth/iam"
-	"github.com/Tencent/bk-bcs/bcs-common/pkg/odm/operator"
-	"github.com/Tencent/bk-bcs/bcs-services/pkg/bcs-auth/cluster"
 	spb "google.golang.org/protobuf/types/known/structpb"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 
+	"github.com/Tencent/bk-bcs/bcs-common/common/blog"
+	"github.com/Tencent/bk-bcs/bcs-common/common/util"
+	"github.com/Tencent/bk-bcs/bcs-common/pkg/auth/iam"
+	"github.com/Tencent/bk-bcs/bcs-common/pkg/odm/operator"
 	proto "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/api/clustermanager"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/actions"
+	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/cloudprovider"
+	provider "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/cloudprovider/common"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/common"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/remote/auth"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/remote/cmdb"
@@ -41,6 +40,7 @@ import (
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/store"
 	storeopt "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/store/options"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/utils"
+	"github.com/Tencent/bk-bcs/bcs-services/pkg/bcs-auth/cluster"
 )
 
 type clusterInfo struct {
@@ -455,31 +455,27 @@ func filterNodesRole(k8sNodes []*corev1.Node, master bool) []*corev1.Node {
 // mergeClusterNodes merge k8s nodes and db nodes
 // 1. 集群中不存在的节点，并且在cluster manager中状态处于初始化中、初始化失败、移除中、移除失败状态时，需要展示cluster manager中数据
 // 2. 集群中存在的节点，则以集群中为准，注意状态的转换
+// 3. 适配双栈, 通过nodeName 作为唯一值, 当前数据库nodeName 可能为空, 因此需要适配转换
 func mergeClusterNodes(cmNodes []*proto.ClusterNode, k8sNodes []*corev1.Node) []*proto.ClusterNode {
 	clusterID := ""
+	// cnNodes exist in k8s cluster and get nodeName
+	GetCmNodeNames(cmNodes, k8sNodes)
+
 	cmNodesMap := make(map[string]*proto.ClusterNode, 0)
 	k8sNodesMap := make(map[string]*corev1.Node, 0)
+
 	for i := range cmNodes {
 		clusterID = cmNodes[i].ClusterID
-		cmNodesMap[cmNodes[i].InnerIP] = cmNodes[i]
+		cmNodesMap[cmNodes[i].NodeName] = cmNodes[i]
 	}
 	for i := range k8sNodes {
-		innerIP := ""
-		for _, v := range k8sNodes[i].Status.Addresses {
-			if v.Type == corev1.NodeInternalIP {
-				innerIP = v.Address
-			}
-		}
-		if len(innerIP) == 0 {
-			continue
-		}
-		k8sNodesMap[innerIP] = k8sNodes[i]
+		k8sNodesMap[k8sNodes[i].Name] = k8sNodes[i]
 	}
 
 	// 处理在 cluster manager 中的节点，但是状态为非正常状态数据，非正常数据放在列表前面
 	nodes := make([]*proto.ClusterNode, 0)
 	for _, v := range cmNodes {
-		if _, ok := k8sNodesMap[v.InnerIP]; ok {
+		if _, ok := k8sNodesMap[v.NodeName]; ok {
 			continue
 		}
 		if v.Status == common.StatusRunning || v.Status == common.StatusNodeRemovable {
@@ -490,16 +486,18 @@ func mergeClusterNodes(cmNodes []*proto.ClusterNode, k8sNodes []*corev1.Node) []
 
 	nodes2 := make([]*proto.ClusterNode, 0)
 	// 集群中存在的节点，则以集群中为准
-	for k, v := range k8sNodesMap {
-		if n, ok := cmNodesMap[k]; ok {
+	for name, node := range k8sNodesMap {
+		ipv4, ipv6 := getNodeDualAddress(node)
+		// 集群中存在节点且存在cm数据库, 提取其他信息
+		if n, ok := cmNodesMap[name]; ok {
 			nodes2 = append(nodes2, &proto.ClusterNode{
 				NodeID:       n.NodeID,
-				InnerIP:      n.InnerIP,
+				InnerIP:      ipv4,
 				InstanceType: n.InstanceType,
 				CPU:          n.CPU,
 				Mem:          n.Mem,
 				GPU:          n.GPU,
-				Status:       transNodeStatus(n.Status, v),
+				Status:       transNodeStatus(n.Status, node),
 				ZoneID:       n.ZoneID,
 				NodeGroupID:  n.NodeGroupID,
 				ClusterID:    n.ClusterID,
@@ -508,30 +506,32 @@ func mergeClusterNodes(cmNodes []*proto.ClusterNode, k8sNodes []*corev1.Node) []
 				Passwd:       n.Passwd,
 				Zone:         n.Zone,
 				DeviceID:     n.DeviceID,
-				NodeName:     v.Name,
-				Labels:       v.Labels,
-				Taints:       actions.K8sTaintToTaint(v.Spec.Taints),
+				NodeName:     node.Name,
+				Labels:       node.Labels,
+				Taints:       actions.K8sTaintToTaint(node.Spec.Taints),
 				UnSchedulable: func(u bool) uint32 {
 					if u {
 						return 1
 					}
 					return 0
-				}(v.Spec.Unschedulable),
+				}(node.Spec.Unschedulable),
+				InnerIPv6: ipv6,
 			})
 		} else {
 			nodes2 = append(nodes2, &proto.ClusterNode{
-				InnerIP:   k,
-				Status:    transNodeStatus("", v),
+				InnerIP:   ipv4,
+				Status:    transNodeStatus("", node),
 				ClusterID: clusterID,
-				NodeName:  v.Name,
-				Labels:    v.Labels,
-				Taints:    actions.K8sTaintToTaint(v.Spec.Taints),
+				NodeName:  node.Name,
+				Labels:    node.Labels,
+				Taints:    actions.K8sTaintToTaint(node.Spec.Taints),
 				UnSchedulable: func(u bool) uint32 {
 					if u {
 						return 1
 					}
 					return 0
-				}(v.Spec.Unschedulable),
+				}(node.Spec.Unschedulable),
+				InnerIPv6: ipv6,
 			})
 		}
 	}
@@ -554,3 +554,46 @@ func (n NodeSlice) Less(i, j int) bool {
 func (n NodeSlice) Swap(i, j int) {
 	n[i], n[j] = n[j], n[i]
 }
+
+func GetCmNodeNames(cmNodes []*proto.ClusterNode, k8sNodes []*corev1.Node) {
+	for i := range cmNodes {
+		ipv4 := cmNodes[i].InnerIP
+		ipv6 := cmNodes[i].InnerIPv6
+
+		if ipv4 == "" && ipv6 == "" {
+			continue
+		}
+
+		for _, node := range k8sNodes {
+			ipv4s, ipv6s := getNodeIPAddress(node)
+			if utils.StringInSlice(ipv4, ipv4s) || utils.StringInSlice(ipv6, ipv6s) {
+				cmNodes[i].NodeName = node.Name
+			}
+		}
+	}
+}
+
+func getNodeIPAddress(node *corev1.Node) ([]string, []string) {
+	ipv4Address := make([]string, 0)
+	ipv6Address := make([]string, 0)
+
+	for _, address := range node.Status.Addresses {
+		if address.Type == corev1.NodeInternalIP {
+			switch {
+			case util.IsIPv6(address.Address):
+				ipv6Address = append(ipv6Address, address.Address)
+			case util.IsIPv4(address.Address):
+				ipv4Address = append(ipv4Address, address.Address)
+			default:
+				blog.Errorf("unsupported ip type")
+			}
+		}
+	}
+
+	return ipv4Address, ipv6Address
+}
+
+func getNodeDualAddress(node *corev1.Node) (string, string) {
+	ipv4s, ipv6s := getNodeIPAddress(node)
+	return utils.SliceToString(ipv4s), utils.SliceToString(ipv6s)
+}

bcs-services/bcs-cluster-manager/internal/app/app.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"net/http/pprof"
 	"os"
@@ -38,18 +39,10 @@ import (
 	"github.com/Tencent/bk-bcs/bcs-common/pkg/auth/iam"
 	"github.com/Tencent/bk-bcs/bcs-common/pkg/odm/drivers"
 	"github.com/Tencent/bk-bcs/bcs-common/pkg/odm/drivers/mongo"
-	"github.com/Tencent/bk-bcs/bcs-services/pkg/bcs-auth/middleware"
-	restful "github.com/emicklei/go-restful"
-	"github.com/gorilla/mux"
-	"github.com/grpc-ecosystem/grpc-gateway/runtime"
-	"github.com/micro/go-micro/v2/registry"
-	"github.com/micro/go-micro/v2/registry/etcd"
-	microsvc "github.com/micro/go-micro/v2/service"
-	microgrpcsvc "github.com/micro/go-micro/v2/service/grpc"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"google.golang.org/grpc"
-	grpccred "google.golang.org/grpc/credentials"
 
+	"github.com/Tencent/bk-bcs/bcs-common/common/http/ipv6server"
+	"github.com/Tencent/bk-bcs/bcs-common/common/tcp/listener"
+	"github.com/Tencent/bk-bcs/bcs-common/common/types"
 	cmproto "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/api/clustermanager"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/auth"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/cloudprovider"
@@ -75,6 +68,19 @@ import (
 	mesostunnel "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/tunnelhandler/mesos"
 	mesoswebconsole "github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/tunnelhandler/mesoswebconsole"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-cluster-manager/internal/utils"
+	"github.com/Tencent/bk-bcs/bcs-services/pkg/bcs-auth/middleware"
+
+	restful "github.com/emicklei/go-restful"
+	"github.com/gorilla/mux"
+	"github.com/grpc-ecosystem/grpc-gateway/runtime"
+	"github.com/micro/go-micro/v2/registry"
+	"github.com/micro/go-micro/v2/registry/etcd"
+	microgrpcserver "github.com/micro/go-micro/v2/server/grpc"
+	microsvc "github.com/micro/go-micro/v2/service"
+	microgrpcsvc "github.com/micro/go-micro/v2/service/grpc"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"google.golang.org/grpc"
+	grpccred "google.golang.org/grpc/credentials"
 )
 
 // ClusterManager cluster manager
@@ -98,10 +104,10 @@ type ClusterManager struct {
 	mux *http.ServeMux
 
 	// http server
-	httpServer *http.Server
+	httpServer *ipv6server.IPv6Server
 
 	// extra module server, [pprof, metrics, swagger]
-	extraServer *http.Server
+	extraServer *ipv6server.IPv6Server
 
 	// discovery
 	disc *discovery.ModuleDiscovery
@@ -675,18 +681,20 @@ func (cm *ClusterManager) initHTTPService() error {
 		return err
 	}
 
-	mux := http.NewServeMux()
-	mux.Handle("/", router)
-	cm.initSwagger(mux)
+	muxServe := http.NewServeMux()
+	muxServe.Handle("/", router)
+	cm.initSwagger(muxServe)
 
-	httpAddr := cm.opt.Address + ":" + strconv.Itoa(int(cm.opt.HTTPPort))
-	cm.httpServer = &http.Server{
-		Addr:    httpAddr,
-		Handler: mux,
+	// server address
+	addresses := []string{cm.opt.Address}
+	if len(cm.opt.Ipv6Address) > 0 {
+		addresses = append(addresses, cm.opt.Ipv6Address)
 	}
+	cm.httpServer = ipv6server.NewIPv6Server(addresses, strconv.Itoa(int(cm.opt.HTTPPort)), "", muxServe)
+
 	go func() {
 		var err error
-		blog.Infof("start http gateway server on address %s", httpAddr)
+		blog.Infof("start http gateway server on address %+v", addresses)
 		if cm.tlsConfig != nil {
 			cm.httpServer.TLSConfig = cm.tlsConfig
 			err = cm.httpServer.ListenAndServeTLS("", "")
@@ -733,15 +741,16 @@ func (cm *ClusterManager) initExtraModules() {
 	extraMux := http.NewServeMux()
 	cm.initPProf(extraMux)
 	cm.initMetric(extraMux)
-	extraServerEndpoint := cm.opt.Address + ":" + strconv.Itoa(int(cm.opt.MetricPort))
-	cm.extraServer = &http.Server{
-		Addr:    extraServerEndpoint,
-		Handler: extraMux,
+
+	ips := []string{cm.opt.Address}
+	if len(cm.opt.Ipv6Address) > 0 {
+		ips = append(ips, cm.opt.Ipv6Address)
 	}
+	cm.extraServer = ipv6server.NewIPv6Server(ips, strconv.Itoa(int(cm.opt.MetricPort)), "", extraMux)
 
 	go func() {
 		var err error
-		blog.Infof("start extra modules [pprof, metric] server %s", extraServerEndpoint)
+		blog.Infof("start extra modules [pprof, metric] server %+v", ips)
 		err = cm.extraServer.ListenAndServe()
 		if err != nil {
 			blog.Errorf("extra modules server listen failed, err %s", err.Error())
@@ -751,6 +760,20 @@ func (cm *ClusterManager) initExtraModules() {
 }
 
 func (cm *ClusterManager) initMicro() error {
+	// server listen ip
+	ipv4 := cm.opt.Address
+	ipv6 := cm.opt.Ipv6Address
+	port := strconv.Itoa(int(cm.opt.Port))
+
+	// service inject metadata to discovery center
+	metadata := make(map[string]string)
+	metadata[cmcommon.MicroMetaKeyHTTPPort] = strconv.Itoa(int(cm.opt.HTTPPort))
+
+	// 适配单栈环境（ipv6注册地址不能是本地回环地址）
+	if v := net.ParseIP(ipv6); v != nil && !v.IsLoopback() {
+		metadata[types.IPV6] = net.JoinHostPort(ipv6, port)
+	}
+
 	authWrapper := middleware.NewGoMicroAuth(auth.GetJWTClient()).
 		EnableSkipHandler(auth.SkipHandler).
 		EnableSkipClient(auth.SkipClient).
@@ -758,11 +781,9 @@ func (cm *ClusterManager) initMicro() error {
 	// New Service
 	microService := microgrpcsvc.NewService(
 		microsvc.Name(cmcommon.ClusterManagerServiceDomain),
-		microsvc.Metadata(map[string]string{
-			cmcommon.MicroMetaKeyHTTPPort: strconv.Itoa(int(cm.opt.HTTPPort)),
-		}),
+		microsvc.Metadata(metadata),
 		microgrpcsvc.WithTLS(cm.tlsConfig),
-		microsvc.Address(cm.opt.Address+":"+strconv.Itoa(int(cm.opt.Port))),
+		microsvc.Address(net.JoinHostPort(ipv4, port)),
 		microsvc.Registry(cm.microRegistry),
 		microsvc.Version(version.BcsVersion),
 		microsvc.RegisterTTL(30*time.Second),
@@ -794,8 +815,22 @@ func (cm *ClusterManager) initMicro() error {
 		IAMClient:  cm.iamClient,
 		CmOptions:  cm.opt,
 	})
+	// 创建双栈监听
+	dualStackListener := listener.NewDualStackListener()
+	if err := dualStackListener.AddListener(ipv4, port); err != nil { // 添加IPv4地址监听
+		return err
+	}
+	if err := dualStackListener.AddListener(ipv6, port); err != nil { // 添加IPv6地址监听
+		return err
+	}
+
+	// grpc server
+	grpcServer := microService.Server()
+	if err := grpcServer.Init(microgrpcserver.Listener(dualStackListener)); err != nil {
+		return err
+	}
 	// Register handler
-	cmproto.RegisterClusterManagerHandler(microService.Server(), cm.serverHandler)
+	cmproto.RegisterClusterManagerHandler(grpcServer, cm.serverHandler)
 	cm.microService = microService
 	return nil
 }

bcs-services/bcs-cluster-manager/internal/cloudprovider/blueking/tasks/importCluster.go

@@ -18,6 +18,7 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/Tencent/bk-bcs/bcs-common/common/blog"
@@ -153,9 +154,8 @@ func importClusterInstances(data *cloudprovider.CloudDependBasicInfo) error {
 	return nil
 }
 
-func importClusterNodesToCM(ctx context.Context, ipList []string, opt *cloudprovider.ListNodesOption) error {
-	nodeMgr := api.NodeManager{}
-	nodes, err := nodeMgr.ListNodesByIP(ipList, &cloudprovider.ListNodesOption{
+func importClusterNodesToCM(ctx context.Context, ipList []types.NodeAddress, opt *cloudprovider.ListNodesOption) error {
+	nodes, err := transInstanceIPToNodes(ipList, &cloudprovider.ListNodesOption{
 		Common: opt.Common,
 	})
 	if err != nil {
@@ -188,19 +188,26 @@ func importClusterNodesToCM(ctx context.Context, ipList []string, opt *cloudprov
 	return nil
 }
 
-func getNodeIP(node v1.Node) string {
-	nodeIP := ""
+func getNodeIP(node v1.Node) types.NodeAddress {
+	var nodeAddress types.NodeAddress
+	nodeAddress.NodeName = node.Name
 
 	for _, address := range node.Status.Addresses {
 		if address.Type == v1.NodeInternalIP {
-			nodeIP = address.Address
+			switch {
+			case strings.Contains(address.Address, ":"):
+				nodeAddress.IPv6Address = address.Address
+			default:
+				nodeAddress.IPv4Address = address.Address
+			}
 		}
 	}
 
-	return nodeIP
+	return nodeAddress
 }
 
-func getClusterInstancesByKubeConfig(data *cloudprovider.CloudDependBasicInfo) ([]string, []string, error) {
+func getClusterInstancesByKubeConfig(data *cloudprovider.CloudDependBasicInfo) ([]types.NodeAddress,
+	[]types.NodeAddress, error) {
 	kubeCli, err := clusterops.NewKubeClient(data.Cluster.KubeConfig)
 	if err != nil {
 		return nil, nil, err
@@ -211,7 +218,7 @@ func getClusterInstancesByKubeConfig(data *cloudprovider.CloudDependBasicInfo) (
 		return nil, nil, err
 	}
 
-	masterIPs, nodeIPs := make([]string, 0), make([]string, 0)
+	masterIPs, nodeIPs := make([]types.NodeAddress, 0), make([]types.NodeAddress, 0)
 	for i := range nodeList.Items {
 		ip := getNodeIP(nodeList.Items[i])
 		_, ok := nodeList.Items[i].Labels[icommon.MasterRole]
@@ -226,14 +233,29 @@ func getClusterInstancesByKubeConfig(data *cloudprovider.CloudDependBasicInfo) (
 	return masterIPs, nodeIPs, nil
 }
 
-func transInstanceIPToNodes(ipList []string, opt *cloudprovider.ListNodesOption) ([]*proto.Node, error) {
+func transInstanceIPToNodes(ipList []types.NodeAddress, opt *cloudprovider.ListNodesOption) ([]*proto.Node, error) {
+	var (
+		ipAddressList = make([]string, 0)
+		ipAddressMap  = make(map[string]types.NodeAddress, 0)
+	)
+	for _, ip := range ipList {
+		ipAddressList = append(ipAddressList, ip.IPv4Address)
+		ipAddressMap[ip.IPv4Address] = ip
+	}
+
 	nodeMgr := api.NodeManager{}
-	nodes, err := nodeMgr.ListNodesByIP(ipList, &cloudprovider.ListNodesOption{
+	nodes, err := nodeMgr.ListNodesByIP(ipAddressList, &cloudprovider.ListNodesOption{
 		Common: opt.Common,
 	})
 	if err != nil {
 		return nil, err
 	}
+	for i := range nodes {
+		if address, ok := ipAddressMap[nodes[i].InnerIP]; ok {
+			nodes[i].NodeName = address.NodeName
+			nodes[i].InnerIPv6 = address.IPv6Address
+		}
+	}
 
 	return nodes, nil
 }

bcs-services/bcs-cluster-manager/internal/cloudprovider/qcloud/api/node.go

@@ -486,6 +486,7 @@ func InstanceToNode(inst *cvm.Instance, zoneInfo map[string]uint32) *proto.Node
 		VPC:          *inst.VirtualPrivateCloud.VpcId,
 		ZoneID:       *inst.Placement.Zone,
 		Zone:         zoneID,
+		InnerIPv6:    utils.SlicePtrToString(inst.IPv6Addresses),
 	}
 	return node
 }

bcs-services/bcs-cluster-manager/internal/clusterops/clusterclient.go

@@ -114,7 +114,7 @@ func (ko *K8SOperator) GetClusterClient(clusterID string) (k8scorecliset.Interfa
 		rand.Seed(time.Now().Unix())
 		cfg.Host = addressList[rand.Intn(len(addressList))]
 		cfg.TLSClientConfig = rest.TLSClientConfig{
-			Insecure: true,
+			Insecure: false,
 			CAData:   []byte(cred.CaCertData),
 			CertData: []byte(cred.ClientCert),
 			KeyData:  []byte(cred.ClientKey),

bcs-services/bcs-cluster-manager/internal/options/options.go

@@ -46,6 +46,7 @@ type SwaggerConfig struct {
 // ServerConfig option for server
 type ServerConfig struct {
 	Address         string `json:"address"`
+	Ipv6Address     string `json:"ipv6Address"`
 	InsecureAddress string `json:"insecureaddress"`
 	Port            uint   `json:"port"`
 	HTTPPort        uint   `json:"httpport"`

bcs-services/bcs-cluster-manager/internal/types/types.go

@@ -115,3 +115,10 @@ type TkeCidrCount struct {
 	IPNumber uint64 `bson:"ipNumber"`
 	Status   string `bson:"status"`
 }
+
+// NodeAddress node address
+type NodeAddress struct {
+	NodeName    string
+	IPv4Address string
+	IPv6Address string
+}

bcs-services/bcs-cluster-manager/internal/utils/utils.go

@@ -10,18 +10,30 @@
  * limitations under the License.
  */
 
-// Package utils xxx
 package utils
 
 import (
 	"encoding/json"
+	"errors"
+	"fmt"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"runtime/debug"
 	"strings"
 
-	"github.com/Tencent/bk-bcs/bcs-common/common/blog"
 	"github.com/kirito41dd/xslice"
+	"github.com/micro/go-micro/v2/registry"
+
+	"github.com/Tencent/bk-bcs/bcs-common/common/blog"
+	"github.com/Tencent/bk-bcs/bcs-common/common/types"
+)
+
+const (
+	// IPV4 ipv4 flag
+	IPV4 = "ipv4"
+	// IPV6 ipv6 flag
+	IPV6 = "ipv6"
 )
 
 // SplitAddrString split address string
@@ -31,6 +43,37 @@ func SplitAddrString(addrs string) []string {
 	return addrArray
 }
 
+// SlicePtrToString to string by ","
+func SlicePtrToString(ips []*string) string {
+	if len(ips) == 0 {
+		return ""
+	}
+
+	ipList := make([]string, 0)
+	for _, ip := range ips {
+		ipList = append(ipList, *ip)
+	}
+
+	return strings.Join(ipList, ",")
+}
+
+// SliceToString to string by ","
+func SliceToString(slice []string) string {
+	if len(slice) == 0 {
+		return ""
+	}
+	if len(slice) == 1 {
+		return slice[0]
+	}
+
+	sList := make([]string, 0)
+	for _, s := range slice {
+		sList = append(sList, s)
+	}
+
+	return strings.Join(sList, ",")
+}
+
 // GetXRequestIDFromHTTPRequest get X-Request-Id from http request
 func GetXRequestIDFromHTTPRequest(req *http.Request) string {
 	if req == nil {
@@ -108,3 +151,36 @@ func GetFileContent(file string) (string, error) {
 
 	return string(body), nil
 }
+
+// GetServerEndpointsFromRegistryNode get dual address
+func GetServerEndpointsFromRegistryNode(nodeServer *registry.Node) []string {
+	// ipv4 server address
+	endpoints := []string{nodeServer.Address}
+	// ipv6 server address
+	if ipv6Address := nodeServer.Metadata[types.IPV6]; ipv6Address != "" {
+		endpoints = append(endpoints, ipv6Address)
+	}
+
+	return endpoints
+}
+
+// CheckIPAddressType check ip address type
+func CheckIPAddressType(ip string) (string, error) {
+	if net.ParseIP(ip) == nil {
+		errMsg := fmt.Sprintf("Invalid IP Address: %s", ip)
+		blog.Errorf(errMsg)
+		return "", errors.New(errMsg)
+	}
+
+	for i := 0; i < len(ip); i++ {
+		switch ip[i] {
+		case '.':
+			return IPV4, nil
+		case ':':
+			fmt.Printf("Given IP Address %s is IPV6 type\n", ip)
+			return IPV6, nil
+		}
+	}
+
+	return "", fmt.Errorf("not supported ip type")
+}
\ No newline at end of file

bcs-services/bcs-cluster-manager/main.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/Tencent/bk-bcs/bcs-common/common/blog"
 	bcsconf "github.com/Tencent/bk-bcs/bcs-common/common/conf"
+	"github.com/Tencent/bk-bcs/bcs-common/common/util"
 	mconfig "github.com/micro/go-micro/v2/config"
 	mfile "github.com/micro/go-micro/v2/config/source/file"
 	mflag "github.com/micro/go-micro/v2/config/source/flag"
@@ -35,6 +36,7 @@ func main() {
 	flag.String("etcd_ca", "", "ca file for etcd")
 	// server config
 	flag.String("address", "127.0.0.1", "grpc server address")
+	flag.String("ipv6Address", "", "grpc server ipv6 address")
 	flag.String("insecureaddress", "127.0.0.1", "insecure server address")
 	flag.Uint("port", 8081, "grpc server port")
 	flag.Uint("httpport", 8080, "http server port")
@@ -102,6 +104,9 @@ func main() {
 	if err != nil {
 		blog.Fatalf("scan config failed, err %s", err.Error())
 	}
+	// init serverConfig Ipv6Address
+	opt.ServerConfig.Ipv6Address = util.InitIPv6Address(opt.ServerConfig.Ipv6Address)
+	blog.Infof("service ipv6 server address: %s", opt.ServerConfig.Ipv6Address)
 
 	blog.InitLogs(bcsconf.LogConfig{
 		LogDir:          opt.BcsLog.LogDir,

install/conf/bcs-services/bcs-cluster-manager/bcs-cluster-manager.json.template

@@ -128,6 +128,7 @@
     "resourceSchemaPath": "${bcsResourceSchemaPath}",
     "debug": ${bcsClusterManagerDebug},
     "address": "${localIp}",
+    "ipv6Address": "${localIpv6}",
     "port": ${bcsClusterManagerPort},
     "httpport": ${bcsClusterManagerHTTPPort},
     "metricport": ${bcsClusterManagerMetricPort},