Commit 5d336b83 authored by ifooth's avatar ifooth Committed by schneesu
Browse files

--story=879120071 bcs-cluster-resources 支持 ipv6 双栈启动 (merge request !501)

parent 843b31e4
No related merge requests found
Showing with 155 additions and 175 deletions
+155 -175
......@@ -21,6 +21,8 @@ LDFLAGS="-X github.com/Tencent/bk-bcs/bcs-services/cluster-resources/pkg/version
LGFLAGS4TEST="-X github.com/Tencent/bk-bcs/bcs-services/cluster-resources/pkg/common/runtime.RunMode=UnitTest"
LGFLAGS4DEV="-X github.com/Tencent/bk-bcs/bcs-services/cluster-resources/pkg/common/runtime.RunMode=Dev"
proto:
# --micro_out => pb.micro.go,--go_out => pb.go,--grpc-gateway_out => pb.gw.go --validate_out => pb.validate.go,--swagger_out => swagger.json
protoc -I ./third_party/ --proto_path=. --micro_out=. --go_out=plugins=grpc:. \
......@@ -40,6 +42,10 @@ build: tidy
swagger: proto
go-bindata --nocompress -pkg swagger -o swagger/datafile.go third_party/swagger-ui/...
# build dev
dev: tidy
go build -ldflags ${LGFLAGS4DEV} -o bcs-cluster-resources *.go
# run unittest in ./pkg recursively
test: tidy
go test -ldflags ${LGFLAGS4TEST} -v ./pkg/... -cover -count=1
......
# BcsServices / ClusterResources
# BcsServices / ClusterResources
> 蓝鲸容器服务(Bcs)集群资源层,用于屏蔽底层集群类型,提供统一的 Restful 接口以供 SaaS / OpenAPI 使用
......@@ -9,11 +9,11 @@
```text
Go 1.17.5
etcd 3.5.0
protoc 3.12.3
protoc 3.20.3
micro v4
go-micro v4
protoc-gen-go latest
protoc-gen-micro v4
go-micro v1.1.4
protoc-gen-go 1.5.2
protoc-gen-micro v1.0.0
protoc-grpc-gateway v1.16.0
protoc-gen-swagger v1.16.0
grpc v1.42.0
......@@ -21,11 +21,20 @@ grpc v1.42.0
### 环境准备
protoc
```bash
# 解压到 $PATH 任意目录
wget https://github.com/protocolbuffers/protobuf/releases/download/v3.20.3/protoc-3.20.3-linux-x86_64.zip
```
```shell script
# 默认安装在 $GOPATH/bin 下
export GO111MODULE=on
go install go-micro.dev/v4/cmd/protoc-gen-micro@v4
go install github.com/golang/protobuf/protoc-gen-go@latest
# go-micro new service 等依赖
go install github.com/go-micro/cli/cmd/go-micro@v1.1.4
# proto 依赖
go install github.com/go-micro/generator/cmd/protoc-gen-micro@v1.0.0
go install github.com/golang/protobuf/protoc-gen-go@1.5.2
go install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger@v1.16.0
go install github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway@v1.16.0
......@@ -119,7 +128,7 @@ $ curl http://127.0.0.1:9091/clusterresources/v1/ping
│ │ │ └── ...
│ │ ├── errcode // 错误码常量
│ │ │ └── ...
│ │ ├── runmode // 运行模式常量
│ │ ├── runmode // 运行模式常量
│ │ │ └── ...
│ │ ├── runtime // 运行时配置
│ │ │ └── ...
......@@ -197,7 +206,7 @@ $ curl http://127.0.0.1:9091/clusterresources/v1/ping
│ │ │ └── ...
│ │ ├── config.go // BCS Cluster Config
│ │ ├── constants.go // 集群资源等常量
│ │ └── discovery.go // Redis Discover 实现
│ │ └── discovery.go // Redis Discover 实现
│ ├── service // 服务层逻辑
│ │ ├── util
│ │ │ └── ...
......
......@@ -22,7 +22,7 @@ import (
"time"
"github.com/Tencent/bk-bcs/bcs-common/common/ssl"
microEtcd "github.com/asim/go-micro/plugins/registry/etcd/v4"
microEtcd "github.com/go-micro/plugins/v4/registry/etcd"
"go-micro.dev/v4/registry"
"github.com/Tencent/bk-bcs/bcs-services/cluster-resources/pkg/cache/redis"
......
......@@ -17,6 +17,7 @@ package cmd
import (
"context"
"crypto/tls"
"net"
"net/http"
"path"
"strconv"
......@@ -24,9 +25,11 @@ import (
"time"
"github.com/Tencent/bk-bcs/bcs-common/common/ssl"
microEtcd "github.com/asim/go-micro/plugins/registry/etcd/v4"
microGrpc "github.com/asim/go-micro/plugins/server/grpc/v4"
"github.com/Tencent/bk-bcs/bcs-common/common/tcp/listener"
"github.com/Tencent/bk-bcs/bcs-common/common/types"
goBindataAssetfs "github.com/elazarl/go-bindata-assetfs"
microEtcd "github.com/go-micro/plugins/v4/registry/etcd"
microGrpc "github.com/go-micro/plugins/v4/server/grpc"
"github.com/gorilla/mux"
"github.com/grpc-ecosystem/grpc-gateway/runtime"
"github.com/prometheus/client_golang/prometheus/promhttp"
......@@ -114,11 +117,32 @@ func (crSvc *clusterResourcesService) Run() error {
// initMicro 初始化 MicroService
func (crSvc *clusterResourcesService) initMicro() error {
metadata := map[string]string{}
dualStackListener := listener.NewDualStackListener()
grpcPort := strconv.Itoa(crSvc.conf.Server.Port)
grpcAddr := net.JoinHostPort(crSvc.conf.Server.Address, grpcPort)
if err := dualStackListener.AddListenerWithAddr(grpcAddr); err != nil {
return err
}
if crSvc.conf.Server.AddressIPv6 != "" {
ipv6Addr := net.JoinHostPort(crSvc.conf.Server.AddressIPv6, grpcPort)
metadata[types.IPV6] = ipv6Addr
if err := dualStackListener.AddListenerWithAddr(ipv6Addr); err != nil {
return err
}
log.Info(crSvc.ctx, "grpc serve dualStackListener with ipv6: %s", ipv6Addr)
}
grpcServer := microGrpc.NewServer(
server.Name(conf.ServiceDomain),
microGrpc.AuthTLS(crSvc.tlsConfig),
microGrpc.MaxMsgSize(conf.MaxGrpcMsgSize),
server.Address(crSvc.conf.Server.Address+":"+strconv.Itoa(crSvc.conf.Server.Port)),
microGrpc.Listener(dualStackListener),
server.Address(grpcAddr),
server.Registry(crSvc.microRtr),
server.RegisterTTL(time.Duration(crSvc.conf.Server.RegisterTTL)*time.Second),
server.RegisterInterval(time.Duration(crSvc.conf.Server.RegisterInterval)*time.Second),
......@@ -144,7 +168,7 @@ func (crSvc *clusterResourcesService) initMicro() error {
return err
}
crSvc.microSvc = micro.NewService(micro.Server(grpcServer))
crSvc.microSvc = micro.NewService(micro.Server(grpcServer), micro.Metadata(metadata))
log.Info(crSvc.ctx, "register cluster resources handler to micro successfully.")
return nil
}
......@@ -309,7 +333,9 @@ func (crSvc *clusterResourcesService) initHTTPService() error {
originMux.Handle("/swagger-ui/", http.StripPrefix("/swagger-ui/", fileServer))
}
httpAddr := crSvc.conf.Server.Address + ":" + strconv.Itoa(crSvc.conf.Server.HTTPPort)
httpPort := strconv.Itoa(crSvc.conf.Server.HTTPPort)
httpAddr := net.JoinHostPort(crSvc.conf.Server.Address, httpPort)
crSvc.httpServer = &http.Server{
Addr: httpAddr,
Handler: wsproxy.WebsocketProxy(
......@@ -317,14 +343,27 @@ func (crSvc *clusterResourcesService) initHTTPService() error {
wsproxy.WithForwardedHeaders(httpUtil.WSHeaderForwarder),
),
}
dualStackListener := listener.NewDualStackListener()
if err := dualStackListener.AddListenerWithAddr(httpAddr); err != nil {
return err
}
if crSvc.conf.Server.AddressIPv6 != "" {
ipv6Addr := net.JoinHostPort(crSvc.conf.Server.AddressIPv6, httpPort)
if err := dualStackListener.AddListenerWithAddr(ipv6Addr); err != nil {
return err
}
log.Info(crSvc.ctx, "http serve dualStackListener with ipv6: %s", ipv6Addr)
}
go func() {
var err error
log.Info(crSvc.ctx, "start http gateway server on address %s", httpAddr)
if crSvc.tlsConfig != nil {
crSvc.httpServer.TLSConfig = crSvc.tlsConfig
err = crSvc.httpServer.ListenAndServeTLS("", "")
err = crSvc.httpServer.ServeTLS(dualStackListener, "", "")
} else {
err = crSvc.httpServer.ListenAndServe()
err = crSvc.httpServer.Serve(dualStackListener)
}
if err != nil {
log.Error(crSvc.ctx, "start http gateway server failed: %v", err)
......@@ -341,16 +380,29 @@ func (crSvc *clusterResourcesService) initMetricService() error {
metricMux := http.NewServeMux()
metricMux.Handle("/metrics", promhttp.Handler())
metricAddr := crSvc.conf.Server.Address + ":" + strconv.Itoa(crSvc.conf.Server.MetricPort)
metricPort := strconv.Itoa(crSvc.conf.Server.MetricPort)
metricAddr := net.JoinHostPort(crSvc.conf.Server.Address, metricPort)
crSvc.metricServer = &http.Server{
Addr: metricAddr,
Handler: metricMux,
}
dualStackListener := listener.NewDualStackListener()
if err := dualStackListener.AddListenerWithAddr(metricAddr); err != nil {
return err
}
if crSvc.conf.Server.AddressIPv6 != "" {
ipv6Addr := net.JoinHostPort(crSvc.conf.Server.AddressIPv6, metricPort)
if err := dualStackListener.AddListenerWithAddr(ipv6Addr); err != nil {
return err
}
log.Info(crSvc.ctx, "metric serve dualStackListener with ipv6: %s", ipv6Addr)
}
go func() {
var err error
log.Info(crSvc.ctx, "start metric server on address %s", metricAddr)
if err = crSvc.metricServer.ListenAndServe(); err != nil {
if err = crSvc.metricServer.Serve(dualStackListener); err != nil {
log.Error(crSvc.ctx, "start metric server failed: %v", err)
crSvc.stopCh <- struct{}{}
}
......
......@@ -2,29 +2,24 @@ module github.com/Tencent/bk-bcs/bcs-services/cluster-resources
go 1.17
replace (
github.com/coreos/bbolt v1.3.4 => go.etcd.io/bbolt v1.3.4
go.etcd.io/bbolt v1.3.4 => github.com/coreos/bbolt v1.3.4
)
require (
github.com/Masterminds/sprig/v3 v3.2.2
github.com/Tencent/bk-bcs/bcs-common v0.0.0-20220608030042-3606c580a7d1
github.com/Tencent/bk-bcs/bcs-common v0.0.0-20220915083455-f4c72dc09513
github.com/TencentBlueKing/gopkg v1.0.8
github.com/TencentBlueKing/iam-go-sdk v0.0.8
github.com/alicebob/miniredis v2.5.0+incompatible
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
github.com/asim/go-micro/plugins/registry/etcd/v4 v4.0.0-20220325022819-73eda3346d18
github.com/asim/go-micro/plugins/server/grpc/v4 v4.0.0-20220325022819-73eda3346d18
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/elazarl/go-bindata-assetfs v1.0.1
github.com/envoyproxy/protoc-gen-validate v0.1.0
github.com/fatih/structs v1.1.0
github.com/go-micro/plugins/v4/registry/etcd v1.1.0
github.com/go-micro/plugins/v4/server/grpc v1.1.1
github.com/go-redis/cache/v8 v8.4.3
github.com/go-redis/redis/v8 v8.11.4
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
github.com/golang/protobuf v1.5.2
github.com/google/uuid v1.2.0
github.com/google/uuid v1.3.0
github.com/googleapis/gnostic v0.5.5
github.com/gorilla/mux v1.8.0
github.com/grpc-ecosystem/grpc-gateway v1.16.0
......@@ -37,13 +32,13 @@ require (
github.com/ssrathi/go-attr v1.3.0
github.com/stretchr/testify v1.7.0
github.com/tmc/grpc-websocket-proxy v0.0.0-20200122045848-3419fae592fc
go-micro.dev/v4 v4.6.0
go-micro.dev/v4 v4.8.1
go.uber.org/zap v1.19.1
google.golang.org/genproto v0.0.0-20211020151524-b7c3a969101a
google.golang.org/grpc v1.42.0
google.golang.org/protobuf v1.27.1
google.golang.org/protobuf v1.28.1
gopkg.in/natefinch/lumberjack.v2 v2.0.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
gopkg.in/yaml.v3 v3.0.1
k8s.io/api v0.23.1
k8s.io/apimachinery v0.23.1
k8s.io/client-go v0.23.1
......@@ -52,21 +47,25 @@ require (
require (
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.1.1 // indirect
github.com/Microsoft/go-winio v0.5.0 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20220824120805-4b6e5c587895 // indirect
github.com/acomagu/bufpipe v1.0.3 // indirect
github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bitly/go-simplejson v0.5.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cloudflare/circl v1.2.0 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/emirpasic/gods v1.12.0 // indirect
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/evanphx/json-patch/v5 v5.5.0 // indirect
github.com/felixge/httpsnoop v1.0.2 // indirect
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-acme/lego/v4 v4.4.0 // indirect
github.com/go-git/gcfg v1.5.0 // indirect
github.com/go-git/go-billy/v5 v5.3.1 // indirect
github.com/go-git/go-git/v5 v5.4.2 // indirect
......@@ -74,20 +73,24 @@ require (
github.com/go-playground/locales v0.13.0 // indirect
github.com/go-playground/universal-translator v0.17.0 // indirect
github.com/go-playground/validator/v10 v10.4.1 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.0.4 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/gomodule/redigo v1.8.8 // indirect
github.com/google/go-cmp v0.5.6 // indirect
github.com/google/gofuzz v1.1.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/gorilla/handlers v1.5.1 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/huandu/xstrings v1.3.1 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/leodido/go-urn v1.2.0 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/miekg/dns v1.1.43 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/miekg/dns v1.1.50 // indirect
github.com/mitchellh/copystructure v1.0.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/hashstructure v1.1.0 // indirect
......@@ -103,33 +106,37 @@ require (
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.26.0 // indirect
github.com/prometheus/procfs v0.6.0 // indirect
github.com/russross/blackfriday/v2 v2.0.1 // indirect
github.com/sergi/go-diff v1.1.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sergi/go-diff v1.2.0 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
github.com/smartystreets/assertions v1.0.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/ugorji/go/codec v1.2.3 // indirect
github.com/urfave/cli/v2 v2.3.0 // indirect
github.com/vmihailenco/go-tinylfu v0.2.2 // indirect
github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/xanzy/ssh-agent v0.3.0 // indirect
github.com/xanzy/ssh-agent v0.3.2 // indirect
github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9 // indirect
go.etcd.io/etcd/api/v3 v3.5.0 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
go.etcd.io/etcd/client/v3 v3.5.0 // indirect
go.etcd.io/etcd/api/v3 v3.5.2 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.2 // indirect
go.etcd.io/etcd/client/v3 v3.5.2 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
golang.org/x/crypto v0.0.0-20210920023735-84f357641f63 // indirect
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
golang.org/x/exp v0.0.0-20210916165020-5cb4fee858ee // indirect
golang.org/x/net v0.0.0-20220111093109-d55c255bac03 // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/net v0.0.0-20220909164309-bea034e7d591 // indirect
golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e // indirect
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
golang.org/x/sync v0.0.0-20220907140024-f12130a52804 // indirect
golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2 // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
golang.org/x/tools v0.1.12 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/grpc/examples v0.0.0-20221025175633-3fd80b0c5298 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
......
......@@ -41,6 +41,10 @@ var (
BCSApiGWAuthToken = envx.Get("BCS_API_GW_AUTH_TOKEN", "")
// LocalIP 服务运行 Pod IP,容器化部署时候指定
LocalIP = envx.Get("LOCAL_IP", "")
// ipv6本地网关地址
IPv6Interface = envx.Get("IPV6_INTERFACE", "")
// 双栈集群, 多个ip地址
PodIPs = envx.Get("POD_IPs", "")
)
// 以下变量值可通过环境变量指定(仅用于单元测试)
......
......@@ -18,8 +18,10 @@ package config
import (
"crypto/rsa"
"io/ioutil"
"net"
"os"
"github.com/Tencent/bk-bcs/bcs-common/common/util"
bkiam "github.com/TencentBlueKing/iam-go-sdk"
"github.com/TencentBlueKing/iam-go-sdk/logger"
"github.com/TencentBlueKing/iam-go-sdk/metric"
......@@ -64,6 +66,26 @@ func LoadConf(filePath string) (*ClusterResourcesConf, error) {
return conf, nil
}
// getIPv6AddrFromEnv 解析ipv6
func getIPv6AddrFromEnv() string {
ipv6 := util.GetIPv6Address(envs.PodIPs)
if ipv6 == "" {
return ""
}
// 在实际中,ipv6不能是回环地址
if v := net.ParseIP(ipv6); v == nil || v.IsLoopback() || v.IsUnspecified() {
return ""
}
// local link ipv6 需要带上 interface, 格式如::%eth0
if envs.IPv6Interface != "" {
ipv6 = ipv6 + "%" + envs.IPv6Interface
}
return ipv6
}
// ClusterResourcesConf ClusterResources 服务启动配置
type ClusterResourcesConf struct {
Debug bool `yaml:"debug"`
......@@ -82,6 +104,7 @@ func (c *ClusterResourcesConf) initServerAddress() error {
c.Server.Address = envs.LocalIP
c.Server.InsecureAddress = envs.LocalIP
}
c.Server.AddressIPv6 = getIPv6AddrFromEnv()
return nil
}
......@@ -160,6 +183,7 @@ type EtcdConf struct {
type ServerConf struct {
UseLocalIP bool `yaml:"useLocalIP" usage:"是否使用 Local IP"`
Address string `yaml:"address" usage:"服务启动地址"`
AddressIPv6 string `yaml:"addressIPv6" usage:"服务启动ipv6地址"`
InsecureAddress string `yaml:"insecureAddress" usage:"服务启动地址(非安全)"`
Port int `yaml:"port" usage:"GRPC 服务端口"`
HTTPPort int `yaml:"httpPort" usage:"HTTP 服务端口"`
......
......@@ -13,7 +13,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.26.0
// protoc v3.12.3
// protoc v3.20.3
// source: proto/cluster-resources/cluster-resources.proto
 
package cluster_resources
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment