Merge branch 'master' of github.com:tkestack/tke into listai-bug-fix

.github/workflows/e2e.yml

@@ -19,6 +19,7 @@ env:
   CREATE_INSTANCES_PARAM: ${{ secrets.CREATE_INSTANCES_PARAM }}
   PASSWORD: ${{ secrets.PASSWORD }}
   NEED_DELETE: ${{ secrets.NEED_DELETE }}
+  DOCKER_HUB_ACTION_AUTH: ${{ secrets.DOCKER_HUB_ACTION_AUTH }}
 jobs:
   image:
     name: build tke image

docs/devel/running-locally.md

-# How to run TKEStack locally
+# How to run TKEStack locally 
+
+## This document is out of date, maybe doesn't work as expected
+
 This guide will walk you through deploying the full TKEStack on you local machine and allow you to play with the core components. It is highly recommended if you want to develop TKEStack and contribute regularly.
 
 ## Table of Contents
@@ -940,4 +943,4 @@ the `tke-auth` component:
 
 **> Question:** How do I find the config options of TKEStack services?
 
-**Answer:** Instead of using `-C` to pass the configuration file to run TKEStack services, you can simply use `-h` to get a full list of options.
\ No newline at end of file
+**Answer:** Instead of using `-C` to pass the configuration file to run TKEStack services, you can simply use `-h` to get a full list of options.

pkg/monitor/controller/prometheus/yamls.go

@@ -25,7 +25,7 @@ import (
 func scrapeConfigForPrometheus() string {
 	cfgStr := `
-    # Use kubelet_running_pod_count to get kube node labels
+    # Use kubelet_running_pod_count or kubelet_running_pods to get kube node labels
     - job_name: 'kubernetes-nodes'
       scrape_timeout: 60s
       scheme: https
@@ -42,7 +42,7 @@ func scrapeConfigForPrometheus() string {
         target_label: device_type
       metric_relabel_configs:
       - source_labels: [ __name__ ]
-        regex: 'kubelet_running_pod_count|kubelet_volume_stats_used_bytes|kubelet_volume_stats_available_bytes|kubelet_volume_stats_capacity_bytes|kubelet_pleg_relist_duration_seconds_sum|kubelet_pleg_relist_duration_seconds_count|kubelet_docker_operations_errors|kubelet_docker_operations_errors_total|kubelet_running_container_count|volume_manager_total_volumes|kubelet_node_config_error|kubelet_runtime_operations_total|kubelet_runtime_operations_errors_total|kubelet_pod_start_duration_seconds_(.*)|kubelet_pod_worker_duration_seconds_(.*)|storage_operation_duration_seconds_(.*)|storage_operation_errors_total|kubelet_runtime_operations_duration_seconds_(.*)|kubelet_cgroup_manager_duration_seconds_(.*)|go_goroutines|process_resident_memory_bytes|process_cpu_seconds_total|rest_client_requests_total|rest_client_request_latency_seconds_(.*)'
+        regex: 'kubelet_running_pods|kubelet_running_pod_count|kubelet_volume_stats_used_bytes|kubelet_volume_stats_available_bytes|kubelet_volume_stats_capacity_bytes|kubelet_pleg_relist_duration_seconds_sum|kubelet_pleg_relist_duration_seconds_count|kubelet_docker_operations_errors|kubelet_docker_operations_errors_total|kubelet_running_containers|kubelet_running_container_count|volume_manager_total_volumes|kubelet_node_config_error|kubelet_runtime_operations_total|kubelet_runtime_operations_errors_total|kubelet_pod_start_duration_seconds_(.*)|kubelet_pod_worker_duration_seconds_(.*)|storage_operation_duration_seconds_(.*)|storage_operation_errors_total|kubelet_runtime_operations_duration_seconds_(.*)|kubelet_cgroup_manager_duration_seconds_(.*)|go_goroutines|process_resident_memory_bytes|process_cpu_seconds_total|rest_client_requests_total|rest_client_request_latency_seconds_(.*)'
         action: keep
       - regex: (__name__|instance|node_role_kubernetes_io_master|device_type|state|namespace|persistentvolumeclaim|operation_type|container_state|plugin_name|operation_name|volume_plugin)
         action: labelkeep
@@ -453,7 +453,7 @@ groups:
 - name: k8s-ag-data
   rules:
   - record: kube_node_labels
-    expr: kubelet_running_pod_count*0 + 1
+    expr: '{__name__=~"kubelet_running_pod_count|kubelet_running_pods"}*0 + 1'
   - record: kube_node_status_capacity_gpu
     expr: sum by(node) (kube_node_status_capacity{resource="tencent_com_vcuda_core"})

test/e2e/constants.go

@@ -29,4 +29,7 @@ const (
 	EtcdYamlFile                  = ManifestDir + "etcd/*.yaml"
 	TKEPlatformAPIYAMLFile        = ManifestDir + "tke-platform-api/*.yaml"
 	TKEPlatformControllerYAMLFile = ManifestDir + "tke-platform-controller/*.yaml"
+	DstDockerConfPath             = "/root/.docker/"
+	DockerConfName                = "config.json"
+	DockerConfigFile              = ManifestDir + "docker/config.json"
 )

test/e2e/manifests/docker/config.json

+{
+    "auths": {
+        "https://index.docker.io/v1/": {
+            "auth": "{{ .Auth }}"
+        }
+    },
+    "HttpHeaders": {
+        "User-Agent": "Docker-Client/19.03.14 (linux)"
+    }
+}
\ No newline at end of file

test/tke/tke.go

 package tke
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
+	"time"
+
 	"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
 	corev1 "k8s.io/api/core/v1"
 	k8serror "k8s.io/apimachinery/pkg/api/errors"
@@ -12,10 +16,12 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/klog"
-	"time"
 	tkeclientset "tkestack.io/tke/api/client/clientset/versioned"
 	platformv1 "tkestack.io/tke/api/platform/v1"
 	typesv1 "tkestack.io/tke/pkg/platform/types/v1"
+	"tkestack.io/tke/pkg/util/ssh"
+	"tkestack.io/tke/pkg/util/template"
+	"tkestack.io/tke/test/e2e"
 	"tkestack.io/tke/test/util"
 	"tkestack.io/tke/test/util/cloudprovider"
 	"tkestack.io/tke/test/util/env"
@@ -72,7 +78,41 @@ func (testTke *TestTKE) ClusterTemplate(nodes ...cloudprovider.Instance) *platfo
 			panic(fmt.Errorf("CreateInstance failed. %v", err))
 		}
 	}
+	if len(os.Getenv(env.DOCKERHUBACTIONAUTH)) == 0 {
+		panic(fmt.Errorf("can't get %v from env", env.DOCKERHUBACTIONAUTH))
+	}
+	dockerconf, err := template.ParseFile(e2e.DockerConfigFile,
+		map[string]interface{}{
+			"Auth": os.Getenv(env.DOCKERHUBACTIONAUTH),
+		})
+	if err != nil {
+		panic(fmt.Errorf("create docker config failed. %v", err))
+	}
 	for _, one := range nodes {
+		sshConfig := &ssh.Config{
+			User:        one.Username,
+			Host:        one.InternalIP,
+			Port:        int(one.Port),
+			Password:    one.Password,
+			DialTimeOut: 30 * time.Second,
+			Retry:       5,
+		}
+		s, err := ssh.New(sshConfig)
+		if err != nil {
+			panic(fmt.Errorf("create ssh failed: %v", err))
+		}
+		out, err := s.CombinedOutput("mkdir " + e2e.DstDockerConfPath)
+		if err != nil {
+			panic(fmt.Errorf("mkdir %v failed: %v, out: %v", e2e.DstDockerConfPath, err, string(out)))
+		}
+
+		err = s.WriteFile(bytes.NewReader(dockerconf), e2e.DstDockerConfPath+e2e.DockerConfName)
+		if err != nil {
+			panic(fmt.Errorf("copy docker config to %v:%v failed: %v",
+				one.InternalIP,
+				e2e.DstDockerConfPath+e2e.DockerConfName,
+				err))
+		}
 		cluster.Spec.Machines = append(cluster.Spec.Machines, platformv1.ClusterMachine{
 			IP:       one.InternalIP,
 			Port:     one.Port,
@@ -85,7 +125,16 @@ func (testTke *TestTKE) ClusterTemplate(nodes ...cloudprovider.Instance) *platfo
 
 func (testTke *TestTKE) CreateClusterInternal(cls *platformv1.Cluster) (cluster *platformv1.Cluster, err error) {
 	klog.Info("Create cluster: ", cls.String())
-	cluster, err = testTke.TkeClient.PlatformV1().Clusters().Create(context.Background(), cls, metav1.CreateOptions{})
+
+	err = wait.PollImmediate(30*time.Second, 5*time.Minute, func() (bool, error) {
+		cluster, err = testTke.TkeClient.PlatformV1().Clusters().Create(context.Background(), cls, metav1.CreateOptions{})
+		if err != nil {
+			klog.Warningf("Create cluster failed: %v", err)
+			return false, err
+		}
+		return true, nil
+	})
+
 	if err != nil {
 		return
 	}

test/util/env/env.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path"
 	"strings"
+
 	"tkestack.io/tke/cmd/tke-installer/app/installer/images"
 	"tkestack.io/tke/pkg/spec"
 
@@ -52,6 +53,7 @@ const (
 	PASSWORD             = "PASSWORD"
 	NEEDDELETE           = "NEED_DELETE"
 	INSTANCETYPES        = "INSTANCE_TYPES"
+	DOCKERHUBACTIONAUTH  = "DOCKER_HUB_ACTION_AUTH"
 )
 
 func ImageVersion() string {

web/console/helpers/downloadCrt.ts

@@ -74,7 +74,7 @@ export function getKubectlConfig({ caCert, token, host, clusterId, clientKey, cl
 apiVersion: v1
 clusters:
 - cluster:
-    certificate-authority-data: ${caCert}
+    ${caCert ? `certificate-authority-data: ${caCert}` : `insecure-skip-tls-verify: true`}
     server: ${host}
   name: ${clusterId}
 contexts:

web/console/helpers/version.ts

+import * as compareVersions from 'compare-versions';
+
+export const compareVersion = (firstVersion: string, secondVersion: string) => {
+  // 判断是否包含“-”， 因为项目这边：1.20.4-tke.1 版本号是大于1.20.4的
+  const [firstVersionPart1, firstVersionPart2] = firstVersion.split('-');
+  const [secondVersionPart1, secondVersionPart2] = secondVersion.split('-');
+
+  const compareResult = compareVersions(firstVersionPart1, secondVersionPart1);
+
+  // 第一部分可以判断出大小
+  if (compareResult !== 0) return compareResult;
+
+  // 第二部分都不存在
+  if (firstVersionPart2 === undefined && secondVersionPart2 === undefined) return 0;
+
+  // 第二部分也相同
+  if (firstVersionPart2 === secondVersionPart2) return 0;
+
+  // firstVersionPart2不存在
+  if (firstVersionPart2 === undefined) return -1;
+
+  // secondVersionPart2不存在
+  if (secondVersionPart2 === undefined) return 1;
+
+  // 第二部分都存在
+  return compareVersions(firstVersion, secondVersion);
+};

web/console/src/modules/cluster/components/clusterManage/ClusterUpdate.tsx

@@ -2,7 +2,7 @@ import React, { useEffect, useState } from 'react';
 import { Space, Button, Form, Select, Checkbox, InputNumber, Typography } from 'antd';
 import { AntdLayout } from '@src/modules/common/layouts';
 import { getK8sValidVersions } from '@src/webApi/cluster';
-import { compare } from 'compare-versions';
+import { compareVersion } from '@helper/version';
 import { RootProps } from '../ClusterApp';
 import { updateCluster } from '@src/webApi/cluster';
 
@@ -89,7 +89,7 @@ export function ClusterUpdate({ route, actions }: RootProps) {
         >
           <Select style={ItemStyle()}>
             {k8sValidVersions.map(v => (
-              <Select.Option key={v} disabled={compare(clusterVersion, v, '>=')} value={v}>
+              <Select.Option key={v} disabled={compareVersion(clusterVersion, v) >= 0} value={v}>
                 {v}
               </Select.Option>
             ))}

web/console/src/modules/cluster/components/resource/resourceDetail/ResourcePodPanel.tsx

@@ -229,7 +229,7 @@ export class ResourcePodPanel extends React.Component<RootProps, ResourcePodPane
         // render: x => this._getDays(x.status.startTime)
         render: x => (
           <Bubble content={`创建时间：${moment(x.metadata.creationTimestamp).format('YYYY-MM-DD HH:mm:ss')}`}>
-            <Text>{this.runningTime(x.status.startTime)}</Text>
+            <Text>{this._getDays(x.status.startTime)}</Text>
           </Bubble>
         )
       },
@@ -522,7 +522,7 @@ export class ResourcePodPanel extends React.Component<RootProps, ResourcePodPane
     return <Text>{`${first} ${second}`}</Text>;
   }
 
-  private runningTime = (startTime: string) => moment.duration(moment().diff(moment(startTime))).humanize(true);
+  private runningTime = (startTime: string) => moment.duration(moment().diff(moment(startTime))).humanize();
 
   /** 运行时间 */
   private _getDays(startTime: string) {

web/console/src/webApi/cluster.ts

 import Request from './request';
-import { compare } from 'compare-versions';
+import { compareVersion } from '@helper/version';
 
 function transObjToelector(obj: { [key: string]: string | number }) {
   return Object.entries(obj)
@@ -90,7 +90,7 @@ export const checkClusterIsNeedUpdate = async ({
 
   // checkversion
   const canUpdateVersions = await getK8sValidVersions();
-  if (canUpdateVersions.findIndex(v => compare(v, clusterVersion, '>')) < 0) {
+  if (canUpdateVersions.findIndex(v => compareVersion(v, clusterVersion) === 1) < 0) {
     return {
       master: {
         isNeed: false,