Merge remote-tracking branch 'woa-bk-bcs/master'

* woa-bk-bcs/master:
  [feat] bcs-monitor request_id (merge request !297)

bcs-services/bcs-monitor/go.mod

@@ -209,7 +209,7 @@ require (
 replace (
 	github.com/coreos/bbolt v1.3.4 => go.etcd.io/bbolt v1.3.4
 	github.com/prometheus/prometheus => github.com/prometheus/prometheus v1.8.2-0.20220308163432-03831554a519
-	github.com/thanos-io/thanos => github.com/ifooth/thanos v0.26.1-0.20220817131253-e89847de197e
+	github.com/thanos-io/thanos => github.com/ifooth/thanos v0.26.1-0.20220907075819-8a2eac0dd9d5
 	github.com/vimeo/galaxycache => github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e
 	go.etcd.io/bbolt v1.3.4 => github.com/coreos/bbolt v1.3.4
 )

bcs-services/bcs-monitor/go.sum

@@ -1225,8 +1225,8 @@ github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
-github.com/ifooth/thanos v0.26.1-0.20220817131253-e89847de197e h1:YdOvBW3WcSSMyXxQwvzpvNqEeLL7gYaU9aoQWjJ+X+Q=
-github.com/ifooth/thanos v0.26.1-0.20220817131253-e89847de197e/go.mod h1:yE4GgQ4dRxDScBTSQGwSFJ/yzPR1q2Szu9iGCLmcFXA=
+github.com/ifooth/thanos v0.26.1-0.20220907075819-8a2eac0dd9d5 h1:tBcuy7vKjp1Ksh5naXlBthNFjWKoEORcLFvD081YBCY=
+github.com/ifooth/thanos v0.26.1-0.20220907075819-8a2eac0dd9d5/go.mod h1:yE4GgQ4dRxDScBTSQGwSFJ/yzPR1q2Szu9iGCLmcFXA=
 github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df/go.mod h1:QMZY7/J/KSQEhKWFeDesPjMj+wCHReeknARU3wqlyN4=
 github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=

bcs-services/bcs-monitor/pkg/api/metrics/cluster.go

@@ -52,7 +52,7 @@ func handleClusterMetric(c *rest.Context, promql string) (interface{}, error) {
 
 	end := time.Now()
 	start := end.Add(-time.Hour)
-	result, err := bcsmonitor.QueryRange(c.Context, c.ProjectCode, promql, params, start, end, time.Minute)
+	result, err := bcsmonitor.QueryRange(c.Request.Context(), c.ProjectCode, promql, params, start, end, time.Minute)
 	if err != nil {
 		return nil, err
 	}
@@ -79,7 +79,7 @@ func GetClusterOverview(c *rest.Context) (interface{}, error) {
 		"disk_total":   `bcs:cluster:disk:total{cluster_id="%<clusterId>s", %<provider>s}`,
 	}
 
-	result, err := bcsmonitor.QueryMultiValues(c.Context, c.ProjectId, promqlMap, params, time.Now())
+	result, err := bcsmonitor.QueryMultiValues(c.Request.Context(), c.ProjectId, promqlMap, params, time.Now())
 	if err != nil {
 		return nil, err
 	}

bcs-services/bcs-monitor/pkg/api/metrics/container.go

@@ -45,7 +45,7 @@ func handleContainerMetric(c *rest.Context, promql string) (interface{}, error)
 		"provider":      PROVIDER,
 	}
 
-	result, err := bcsmonitor.QueryRange(c.Context, c.ProjectCode, promql, params, queryTime.Start, queryTime.End,
+	result, err := bcsmonitor.QueryRange(c.Request.Context(), c.ProjectCode, promql, params, queryTime.Start, queryTime.End,
 		queryTime.Step)
 	if err != nil {
 		return nil, err

bcs-services/bcs-monitor/pkg/api/metrics/node.go

@@ -103,7 +103,7 @@ func handleNodeMetric(c *rest.Context, promql string) (interface{}, error) {
 		"provider":  PROVIDER,
 	}
 
-	result, err := bcsmonitor.QueryRange(c.Context, c.ProjectCode, promql, params, queryTime.Start, queryTime.End,
+	result, err := bcsmonitor.QueryRange(c.Request.Context(), c.ProjectCode, promql, params, queryTime.Start, queryTime.End,
 		queryTime.Step)
 	if err != nil {
 		return nil, err
@@ -124,7 +124,7 @@ func GetNodeInfo(c *rest.Context) (interface{}, error) {
 	}
 
 	promql := `bcs:node:info{cluster_id="%<clusterId>s", ip="%<ip>s", %<provider>s}`
-	labelSet, err := bcsmonitor.QueryLabelSet(c.Context, c.ProjectId, promql, params, time.Now())
+	labelSet, err := bcsmonitor.QueryLabelSet(c.Request.Context(), c.ProjectId, promql, params, time.Now())
 	if err != nil {
 		return nil, err
 	}
@@ -152,7 +152,7 @@ func GetNodeOverview(c *rest.Context) (interface{}, error) {
 		"pod_count":       `bcs:node:pod_count{cluster_id="%<clusterId>s", ip="%<ip>s", %<provider>s}`,
 	}
 
-	result, err := bcsmonitor.QueryMultiValues(c.Context, c.ProjectId, promqlMap, params, time.Now())
+	result, err := bcsmonitor.QueryMultiValues(c.Request.Context(), c.ProjectId, promqlMap, params, time.Now())
 	if err != nil {
 		return nil, err
 	}

bcs-services/bcs-monitor/pkg/api/metrics/pod.go

@@ -44,7 +44,7 @@ func handlePodMetric(c *rest.Context, promql string) (interface{}, error) {
 		"provider":    PROVIDER,
 	}
 
-	result, err := bcsmonitor.QueryRange(c.Context, c.ProjectCode, promql, params, queryTime.Start, queryTime.End,
+	result, err := bcsmonitor.QueryRange(c.Request.Context(), c.ProjectCode, promql, params, queryTime.Start, queryTime.End,
 		queryTime.Step)
 
 	if err != nil {

bcs-services/bcs-monitor/pkg/api/routes.go

@@ -33,6 +33,7 @@ import (
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-monitor/pkg/config"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-monitor/pkg/rest"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-monitor/pkg/rest/middleware"
+	"github.com/Tencent/bk-bcs/bcs-services/bcs-monitor/pkg/rest/tracing"
 	"github.com/Tencent/bk-bcs/bcs-services/bcs-monitor/pkg/storegw"
 )
 
@@ -77,7 +78,7 @@ func (a *APIServer) newRoutes(engine *gin.Engine) {
 	// 添加 X-Request-Id 头部
 	requestIdMiddleware := requestid.New(
 		requestid.WithGenerator(func() string {
-			return rest.RequestIdGenerator()
+			return tracing.RequestIdGenerator()
 		}),
 	)
 

bcs-services/bcs-monitor/pkg/component/client.go

@@ -21,13 +21,15 @@ import (
 	"net/http"
 	"net/url"
 	"strconv"
-	"sync"
 	"time"
 
 	"github.com/dustin/go-humanize"
 	resty "github.com/go-resty/resty/v2"
 	"github.com/pkg/errors"
+	"github.com/thanos-io/thanos/pkg/store"
 	"k8s.io/klog/v2"
+
+	"github.com/Tencent/bk-bcs/bcs-services/bcs-monitor/pkg/rest/tracing"
 )
 
 const (
@@ -35,9 +37,7 @@ const (
 )
 
 var (
-	clientOnce   sync.Once
-	globalClient *resty.Client
-	maskKeys     = map[string]struct{}{
+	maskKeys = map[string]struct{}{
 		"bk_app_secret": {},
 	}
 )
@@ -61,7 +61,7 @@ func restyReqToCurl(r *resty.Request) string {
 	}
 	rawURL.RawQuery = queryValue.Encode()
 
-	reqMsg := fmt.Sprintf("curl -X %s %s%s", r.Method, rawURL.String(), headers)
+	reqMsg := fmt.Sprintf("curl -X %s '%s'%s", r.Method, rawURL.String(), headers)
 	if r.Body != nil {
 		switch body := r.Body.(type) {
 		case []byte:
@@ -102,29 +102,31 @@ func restyResponseToCurl(resp *resty.Response) string {
 }
 
 func restyErrHook(r *resty.Request, err error) {
-	klog.Infof("REQ: %s", restyReqToCurl(r))
-	klog.Infof("RESP: [err] %s", err)
+	klog.Infof("[%s] REQ: %s", store.RequestIDValue(r.RawRequest.Context()), restyReqToCurl(r))
+	klog.Infof("[%s] RESP: [err] %s", store.RequestIDValue(r.RawRequest.Context()), err)
 }
 
 func restyAfterResponseHook(c *resty.Client, r *resty.Response) error {
-	klog.Infof("REQ: %s", restyReqToCurl(r.Request))
-	klog.Infof("RESP: %s", restyResponseToCurl(r))
+	klog.Infof("[%s] REQ: %s", store.RequestIDValue(r.Request.Context()), restyReqToCurl(r.Request))
+	klog.Infof("[%s] RESP: %s", store.RequestIDValue(r.Request.Context()), restyResponseToCurl(r))
 	return nil
 }
 
-// GetClient xxx
+func restyBeforeRequestHook(c *resty.Client, r *http.Request) error {
+	tracing.SetRequestIDValue(r, store.RequestIDValue(r.Context()))
+	return nil
+}
+
+// GetClient : 新建Client, 设置公共参数，每次新建，cookies不复用
 func GetClient() *resty.Client {
-	if globalClient == nil {
-		clientOnce.Do(func() {
-			globalClient = resty.New().SetTimeout(timeout)
-			globalClient = globalClient.SetDebug(false) // 更多详情, 可以开启为 true
-			globalClient.SetDebugBodyLimit(1024)
-			globalClient.OnAfterResponse(restyAfterResponseHook)
-			globalClient.OnError(restyErrHook)
-			globalClient.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
-		})
-	}
-	return globalClient
+	client := resty.New().SetTimeout(timeout)
+	client = client.SetDebug(false) // 更多详情, 可以开启为 true
+	client.SetDebugBodyLimit(1024)
+	client.OnAfterResponse(restyAfterResponseHook)
+	client.SetPreRequestHook(restyBeforeRequestHook)
+	client.OnError(restyErrHook)
+	client.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
+	return client
 }
 
 // BKResult 蓝鲸返回规范的结构体

bcs-services/bcs-monitor/pkg/query/middleware.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/TencentBlueKing/bkmonitor-kits/logger"
 	"github.com/pkg/errors"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/promql/parser"
@@ -25,7 +26,7 @@ import (
 	extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http"
 	"github.com/thanos-io/thanos/pkg/store"
 
-	"github.com/TencentBlueKing/bkmonitor-kits/logger"
+	"github.com/Tencent/bk-bcs/bcs-services/bcs-monitor/pkg/rest/tracing"
 )
 
 const (
@@ -72,10 +73,19 @@ func (t *tenantAuthMiddleware) NewHandler(handlerName string, handler http.Handl
 			return
 		}
 
-		logger.Infow("handle request", "handler_name", handlerName, "label_matchers", fmt.Sprintf("%s", labelMatchers), "url",
-			r.URL)
+		requestID := tracing.RequestIDValue(r, true)
+		logger.Infow("handle request",
+			"request_id", requestID,
+			"handler_name", handlerName,
+			"label_matchers", fmt.Sprintf("%s", labelMatchers),
+			"req", fmt.Sprintf("%s %s", r.Method, r.URL),
+		)
+
+		// 返回的 header 写入 request_id
+		w.Header().Set(store.RequestIdHeaderKey(), requestID)
 
 		ctx := store.WithLabelMatchValue(r.Context(), labelMatchers)
+		ctx = store.WithRequestIDValue(ctx, requestID)
 		r = r.WithContext(ctx)
 		handleFunc(w, r)
 	}

bcs-services/bcs-monitor/pkg/rest/rest.go

@@ -8,7 +8,6 @@
  * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
  * either express or implied. See the License for the specific language governing permissions and
  * limitations under the License.
- *
  */
 
 // Package rest xxx
@@ -16,12 +15,13 @@ package rest
 
 import (
 	"net/http"
-	"strings"
 
 	"github.com/gin-contrib/requestid"
 	"github.com/gin-gonic/gin"
-	"github.com/google/uuid"
 	"github.com/pkg/errors"
+	"github.com/thanos-io/thanos/pkg/store"
+
+	"github.com/Tencent/bk-bcs/bcs-services/bcs-monitor/pkg/rest/tracing"
 )
 
 var (
@@ -73,22 +73,21 @@ func APIResponse(c *Context, data interface{}) {
 	c.JSON(http.StatusOK, result)
 }
 
-// RequestIdGenerator :
-func RequestIdGenerator() string {
-	uid := uuid.New().String()
-	requestId := strings.Replace(uid, "-", "", -1)
-	return requestId
-}
-
 // InitRestContext :
 func InitRestContext(c *gin.Context) *Context {
+	requestId := requestid.Get(c)
+
 	restContext := &Context{
 		Context:   c,
-		RequestId: requestid.Get(c),
+		RequestId: requestId,
 		ClusterId: c.Param("clusterId"),
 		ProjectId: c.Param("projectId"),
 	}
 	c.Set("rest_context", restContext)
+
+	tracing.SetRequestIDValue(c.Request, requestId)
+	ctx := store.WithRequestIDValue(c.Request.Context(), requestId)
+	restContext.Request = restContext.Request.WithContext(ctx)
 	return restContext
 }
 

bcs-services/bcs-monitor/pkg/rest/tracing/request_id.go

+/*
+ * Tencent is pleased to support the open source community by making Blueking Container Service available.
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ * http://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package tracing
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/thanos-io/thanos/pkg/store"
+)
+
+// RequestIdGenerator :
+func RequestIdGenerator() string {
+	uid := uuid.New().String()
+	requestId := strings.Replace(uid, "-", "", -1)
+	return requestId
+}
+
+// SetRequestIDValue :
+func SetRequestIDValue(req *http.Request, id string) {
+	req.Header.Set(store.RequestIdHeaderKey(), id)
+}
+
+// RequestIDValue :
+func RequestIDValue(req *http.Request, autoGen bool) string {
+	id := req.Header.Get(store.RequestIdHeaderKey())
+	if id == "" && autoGen {
+		id = RequestIdGenerator()
+	}
+	return id
+}

bcs-services/bcs-monitor/pkg/storegw/bcs_system/bcs_system.go

@@ -22,6 +22,7 @@ import (
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/thanos-io/thanos/pkg/component"
+	"github.com/thanos-io/thanos/pkg/store"
 	"github.com/thanos-io/thanos/pkg/store/labelpb"
 	"github.com/thanos-io/thanos/pkg/store/storepb"
 	"k8s.io/klog/v2"
@@ -89,7 +90,9 @@ func (s *BCSSystemStore) LabelValues(ctx context.Context, r *storepb.LabelValues
 
 // Series 返回时序数据
 func (s *BCSSystemStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesServer) error {
-	klog.InfoS(clientutil.DumpPromQL(r), "minTime=", r.MinTime, "maxTime", r.MaxTime, "step", r.QueryHints.StepMillis)
+	ctx := srv.Context()
+
+	klog.InfoS(clientutil.DumpPromQL(r), "request_id", store.RequestIDValue(ctx), "minTime=", r.MinTime, "maxTime", r.MaxTime, "step", r.QueryHints.StepMillis)
 
 	// 最小1分钟维度
 	step := r.QueryHints.StepMillis
@@ -147,15 +150,13 @@ func (s *BCSSystemStore) Series(r *storepb.SeriesRequest, srv storepb.Store_Seri
 		return err
 	}
 
-	ctx := srv.Context()
-
 	bcsConf := k8sclient.GetBCSConfByClusterId(clusterId)
 	cluster, err := bcs.GetCluster(ctx, bcsConf, clusterId)
 	if err != nil {
 		return err
 	}
 
-	client, err := source.ClientFactory(cluster.ClusterId)
+	client, err := source.ClientFactory(ctx, cluster.ClusterId)
 	if err != nil {
 		return err
 	}

bcs-services/bcs-monitor/pkg/storegw/bcs_system/source/client.go

@@ -51,8 +51,8 @@ func IsBKMonitorEnabled(ctx context.Context, clusterId string) (bool, error) {
 }
 
 // ClientFactory 自动切换Prometheus/蓝鲸监控
-func ClientFactory(clusterId string) (base.MetricHandler, error) {
-	ok, err := IsBKMonitorEnabled(context.Background(), clusterId)
+func ClientFactory(ctx context.Context, clusterId string) (base.MetricHandler, error) {
+	ok, err := IsBKMonitorEnabled(ctx, clusterId)
 	if err != nil {
 		return nil, err
 	}

bcs-services/bcs-monitor/pkg/storegw/bk_monitor/bk_monitor.go

@@ -24,6 +24,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/thanos-io/thanos/pkg/component"
+	"github.com/thanos-io/thanos/pkg/store"
 	"github.com/thanos-io/thanos/pkg/store/labelpb"
 	"github.com/thanos-io/thanos/pkg/store/storepb"
 	"gopkg.in/yaml.v2"
@@ -139,7 +140,8 @@ func (s *BKMonitorStore) LabelValues(ctx context.Context, r *storepb.LabelValues
 
 // Series 返回时序数据
 func (s *BKMonitorStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesServer) error {
-	klog.InfoS(clientutil.DumpPromQL(r), "minTime", r.MinTime, "maxTime", r.MaxTime, "step", r.QueryHints.StepMillis)
+	ctx := srv.Context()
+	klog.InfoS(clientutil.DumpPromQL(r), "request_id", store.RequestIDValue(ctx), "minTime", r.MinTime, "maxTime", r.MaxTime, "step", r.QueryHints.StepMillis)
 
 	if r.Step < 60 {
 		r.Step = 60

bcs-services/bcs-monitor/run-test.sh

@@ -23,6 +23,7 @@ function api() {
 function query {
     ./bin/bcs-monitor query \
     --http-address 0.0.0.0:10902 \
+    --store 11.187.114.115:30911 \
     --advertise-address clb:11902 \
     --config ./etc/config_dev.yaml \
     --credential-config ./etc/credentials_dev.yaml \