Merge pull request #776 from DeveloperJim/master

自动化测试问题修复

bcs-common/pkg/bcsapi/clustermanager.go

@@ -17,6 +17,7 @@ import (
 	"fmt"
 	"math/rand"
 	"time"
+	"context"
 
 	"github.com/Tencent/bk-bcs/bcs-common/common/blog"
 	cm "github.com/Tencent/bk-bcs/bcs-common/pkg/bcsapi/clustermanager"
@@ -28,14 +29,12 @@ import (
 )
 
 // XRequestID insert X-Request-ID
-func XRequestID() *grpc.HeaderCallOption {
+func XRequestID() context.Context {
 	requestID := map[string]string{
 		"X-Request-Id": uuid.New().String(),
 	}
 	md := metadata.New(requestID)
-	return &grpc.HeaderCallOption{
-		HeaderAddr: &md,
-	}
+	return metadata.NewOutgoingContext(context.Background(), md)
 }
 
 // NewClusterManager create ClusterManager SDK implementation
@@ -61,11 +60,12 @@ func NewClusterManager(config *Config) cm.ClusterManagerClient {
 		opts = append(opts, grpc.WithInsecure())
 	}
 	var conn *grpc.ClientConn
+	var err error
 	maxTries := 3
 	for i := 0; i < maxTries; i++ {
 		selected := rand.Intn(1024) % len(config.Hosts)
 		addr := config.Hosts[selected]
-		conn, err := grpc.Dial(addr, opts...)
+		conn, err = grpc.Dial(addr, opts...)
 		if err != nil {
 			blog.Errorf("Create clsuter manager grpc client with %s error: %s", addr, err.Error())
 			continue

bcs-k8s/bcs-kube-agent/app/report.go

@@ -103,11 +103,13 @@ func reportToBke(kubeClient *kubernetes.Clientset, cfg *rest.Config) {
 			request = gorequest.New().TLSClientConfig(tlsConfig)
 		}
 		userToken := os.Getenv("USER_TOKEN")
-		if len(userToken) != 0 {
-			request.AppendHeader("Authorization", "Bearer "+userToken)
+		if len(userToken) == 0 {
+			blog.Errorf("lost USER_TOKEN env parameter")
+			panic("lost USER_TOKEN env parameter")
 		}
-
-		resp, respBody, errs := request.Put(bkeURL).Send(clusterInfoParams).End()
+		resp, respBody, errs := request.Put(bkeURL).
+			Set("Authorization", "Bearer "+userToken).
+			Send(clusterInfoParams).End()
 		if len(errs) > 0 {
 			blog.Errorf("unable to connect to the bke server: %s", errs[0].Error())
 			// sleep a while to try again, avoid trying in loop
@@ -115,9 +117,13 @@ func reportToBke(kubeClient *kubernetes.Clientset, cfg *rest.Config) {
 			continue
 		}
 		if resp.StatusCode >= 400 {
-			codeName := json.Get([]byte(respBody), "code_name").ToString()
+			codeName := json.Get([]byte(respBody), "code").ToInt()
 			message := json.Get([]byte(respBody), "message").ToString()
-			blog.Errorf("Error updating cluster credential to bke, response code: %s, response message: %s", codeName, message)
+			blog.Errorf(
+				"Error updating cluster credential to bke, response code: %s, response message: %s",
+				codeName,
+				message,
+			)
 		}
 
 		select {

bcs-k8s/bcs-kube-agent/app/websocket.go

@@ -43,7 +43,7 @@ const (
 	Cluster = "BCS-API-Tunnel-ClusterId"
 
 	// ModuleName module name for kube agent
-	ModuleName = "kube-agent"
+	ModuleName = "kubeagent"
 )
 
 func getenv(env string) (string, error) {

bcs-mesos/bcs-mesos-driver/mesosdriver/websocket.go

@@ -39,7 +39,7 @@ const (
 	// Cluster ID for ws tunnel
 	Cluster = "BCS-API-Tunnel-ClusterId"
 	// ModuleName definition
-	ModuleName = "mesos-driver"
+	ModuleName = "mesosdriver"
 )
 
 func (m *MesosDriver) buildWebsocketToAPI() error {
@@ -97,7 +97,7 @@ func (m *MesosDriver) buildWebsocketToAPI() error {
 
 	go func() {
 		for {
-			wsURL := fmt.Sprintf("wss://%s/bcsapi/v4/usermanager/v1/websocket/connect", bcsAPIURL.Host)
+			wsURL := fmt.Sprintf("wss://%s/bcsapi/v4/clustermanager/v1/websocket/connect", bcsAPIURL.Host)
 			blog.Infof("Connecting to %s with token %s", wsURL, m.config.RegisterToken)
 
 			websocketDialer.ClientConnect(context.Background(), wsURL, headers, tlsConfig, nil, func(proto, address string) bool {

bcs-services/bcs-cluster-manager/internal/actions/clustercredential/list.go

@@ -55,7 +55,7 @@ func (la *ListAction) listClusterCredential() error {
 		condM["clusterID"] = la.req.ClusterID
 	}
 	if len(la.req.ClientMode) != 0 {
-		condM["clientMode"] = la.req.ClientMode
+		condM["clientModule"] = la.req.ClientMode
 	}
 	if len(la.req.ConnectMode) != 0 {
 		condM["connectMode"] = la.req.ConnectMode

bcs-services/bcs-cluster-manager/internal/actions/clustercredential/update.go

@@ -52,7 +52,7 @@ func (ua *UpdateAction) updateCredential() error {
 		ServerAddress: ua.req.ServerAddress,
 		CaCertData:    ua.req.CaCertData,
 		UserToken:     ua.req.UserToken,
-		ConnectMode:   modules.BCSConnectModeTunnel,
+		ConnectMode:   modules.BCSConnectModeDirect,
 	}
 	if err := ua.model.PutClusterCredential(ua.ctx, newCredential); err != nil {
 		return err

bcs-services/bcs-cluster-manager/internal/tunnel/tunnelserver.go

@@ -37,11 +37,9 @@ const (
 	// Cluster http tunnel header
 	Cluster = "BCS-API-Tunnel-ClusterId"
 	// KubeAgentModule http tunnel header
-	KubeAgentModule = "kube-agent"
-	// K8sDriverModule http tunnel header
-	K8sDriverModule = "k8s-driver"
+	KubeAgentModule = "kubeagent"
 	// MesosDriverModule http tunnel header
-	MesosDriverModule = "mesos-driver"
+	MesosDriverModule = "mesosdriver"
 )
 
 // RegisterCluster definition of tunnel cluster info
@@ -138,7 +136,7 @@ func (wts *WsTunnelServerCallback) authorizeTunnel(req *http.Request) (string, b
 			return "", false, err
 		}
 		return clusterID, true, nil
-	} else if moduleName == MesosDriverModule || moduleName == K8sDriverModule {
+	} else if moduleName == MesosDriverModule {
 		// for mesos, the registerCluster.Address is mesos-driver url.
 		// one mesos cluster may have 3 or more mesos-driver,
 		// so we should distinguish them, so use {clusterID}-{ip:port} as serverKey

bcs-services/bcs-gateway-discovery/app/zookeeper.go

@@ -14,7 +14,6 @@
 package app
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"net/url"
@@ -172,7 +171,12 @@ func (s *DiscoveryServer) formatKubeAPIServerInfo(module string) ([]*register.Se
 		ClientMode:  modules.BCSModuleKubeagent,
 		ConnectMode: modules.BCSConnectModeDirect,
 	}
-	clusterResp, err := clusterCli.ListClusterCredential(context.Background(), req, bcsapi.XRequestID())
+	if clusterCli == nil {
+		blog.Errorf("create cluster manager cli from config: %+v failed, please check discovery", config)
+		return nil, fmt.Errorf("no available clustermanager client")
+	}
+
+	clusterResp, err := clusterCli.ListClusterCredential(bcsapi.XRequestID(), req)
 	if err != nil {
 		blog.Errorf("request all kube-apiserver cluster info from bcs-cluster-manager %s failed, %s", node.Address, err.Error())
 		return nil, err

bcs-services/bcs-gateway-discovery/go.mod

@@ -3,12 +3,12 @@ module github.com/Tencent/bk-bcs/bcs-services/bcs-gateway-discovery
 go 1.14
 
 replace (
-	github.com/Tencent/bk-bcs/bcs-common => github.com/Tencent/bk-bcs/bcs-common v0.0.0-20210204134536-62cbdd86fc35
+	github.com/Tencent/bk-bcs/bcs-common => ../../bcs-common
 	github.com/coreos/bbolt v1.3.4 => go.etcd.io/bbolt v1.3.4
+	github.com/coreos/etcd => github.com/coreos/etcd v3.3.18+incompatible
 	github.com/kevholditch/gokong => github.com/DeveloperJim/gokong v1.9.2
 	github.com/micro/go-micro/v2 => github.com/OvertimeDog/go-micro/v2 v2.9.3
 	go.etcd.io/bbolt v1.3.4 => github.com/coreos/bbolt v1.3.4
-	github.com/coreos/etcd => github.com/coreos/etcd v3.3.18+incompatible
 	google.golang.org/grpc => google.golang.org/grpc v1.26.0
 )
 

bcs-services/bcs-gateway-discovery/plugins/apisix/bkbcs-auth/bkbcs.lua

@@ -162,12 +162,7 @@ function BKUserCli:construct_identity(conf, request)
   else
     auth.resource_type = conf.module
   end
-  -- kubeagent & networkdetection has no ClusterId
-  local headers = request.get_headers()
-  if not headers[CLUSTER_HEADER] and is_cluster_resource(conf.module) then
-    core.log.error(" user_cli get no BCS-ClusterID from request ", ngx.var.uri)
-    return nil, "lost BCS-ClusterID in header"
-  end
+  -- kubeagent & mesosdriver construct ClusterId
   if conf.module == KUBEAGENT then
     -- retrieve bcs-cluster-id from url as resource
     local id_iterator, id_err = ngx.re.gmatch(ngx.var.uri, "BCS-K8S-([0-9]+[^/])")
@@ -178,10 +173,15 @@ function BKUserCli:construct_identity(conf, request)
     local id, err = id_iterator()
     if not id or #id < 1 then
       core.log.error(" user_cli parse kubernetes BCS-ClusterID in request ", ngx.var.uri, " failed, ", err)
-      return nil, "kuberentes BCS-ClusterID "
+      return nil, "kuberentes BCS-ClusterID parse failed"
     end
     auth.resource = id[0]
-  else
+  else if conf.module == MESOSDRIVER then
+    local headers = request.get_headers()
+    if not headers[CLUSTER_HEADER] then
+      core.log.error(" user_cli get no BCS-ClusterID from request ", ngx.var.uri)
+      return nil, "lost BCS-ClusterID in header"
+    end
     -- retrieve bcs-cluster-id from header
     auth.resource = headers[CLUSTER_HEADER]
   end
@@ -189,12 +189,12 @@ function BKUserCli:construct_identity(conf, request)
   -- get token for http header
   local auth_header = request.get_headers()
   if not auth_header["Authorization"] then
-    core.log.error(" user_cli get no Authorization from http header, request path:", ngx.var.uri)
+    core.log.error(" user_cli get no Authorization from http header, request path:", ngx.var.uri, " header details: ", core.json.encode(auth_header))
     return nil, "lost Authorization"
   end
   local iterator, iter_err = ngx.re.gmatch(auth_header["Authorization"], "\\s*[Bb]earer\\s+(.+)")
   if not iterator then
-    core.log.error(" user_cli search token for request ", ngx.var.uri, " failed, ", iter_err)
+    core.log.error(" user_cli search token for request ", ngx.var.uri, " failed, ", iter_err, " Authorization: ", auth_header["Authorization"])
     return nil, "Authorization format error"
   end
   local m, err = iterator()