backend/cos: Add TencentCloud backend cos with lock (#22540)

* add TencentCloud COS backend for remote state

* add vendor of dependence

* fixed error not handle and remove default value for prefix argument

* get appid from TF_COS_APPID environment variables

vendor/github.com/tencentyun/cos-go-sdk-v5/auth.go

+package cos
+
+import (
+	"crypto/hmac"
+	"crypto/sha1"
+	"fmt"
+	"hash"
+	"net/http"
+	"net/url"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+const sha1SignAlgorithm = "sha1"
+const privateHeaderPrefix = "x-cos-"
+const defaultAuthExpire = time.Hour
+
+// 需要校验的 Headers 列表
+var needSignHeaders = map[string]bool{
+	"host":                           true,
+	"range":                          true,
+	"x-cos-acl":                      true,
+	"x-cos-grant-read":               true,
+	"x-cos-grant-write":              true,
+	"x-cos-grant-full-control":       true,
+	"response-content-type":          true,
+	"response-content-language":      true,
+	"response-expires":               true,
+	"response-cache-control":         true,
+	"response-content-disposition":   true,
+	"response-content-encoding":      true,
+	"cache-control":                  true,
+	"content-disposition":            true,
+	"content-encoding":               true,
+	"content-type":                   true,
+	"content-length":                 true,
+	"content-md5":                    true,
+	"expect":                         true,
+	"expires":                        true,
+	"x-cos-content-sha1":             true,
+	"x-cos-storage-class":            true,
+	"if-modified-since":              true,
+	"origin":                         true,
+	"access-control-request-method":  true,
+	"access-control-request-headers": true,
+	"x-cos-object-type":              true,
+}
+
+func safeURLEncode(s string) string {
+	s = encodeURIComponent(s)
+	s = strings.Replace(s, "!", "%21", -1)
+	s = strings.Replace(s, "'", "%27", -1)
+	s = strings.Replace(s, "(", "%28", -1)
+	s = strings.Replace(s, ")", "%29", -1)
+	s = strings.Replace(s, "*", "%2A", -1)
+	return s
+}
+
+type valuesSignMap map[string][]string
+
+func (vs valuesSignMap) Add(key, value string) {
+	key = strings.ToLower(key)
+	vs[key] = append(vs[key], value)
+}
+
+func (vs valuesSignMap) Encode() string {
+	var keys []string
+	for k := range vs {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	var pairs []string
+	for _, k := range keys {
+		items := vs[k]
+		sort.Strings(items)
+		for _, val := range items {
+			pairs = append(
+				pairs,
+				fmt.Sprintf("%s=%s", safeURLEncode(k), safeURLEncode(val)))
+		}
+	}
+	return strings.Join(pairs, "&")
+}
+
+// AuthTime 用于生成签名所需的 q-sign-time 和 q-key-time 相关参数
+type AuthTime struct {
+	SignStartTime time.Time
+	SignEndTime   time.Time
+	KeyStartTime  time.Time
+	KeyEndTime    time.Time
+}
+
+// NewAuthTime 生成 AuthTime 的便捷函数
+//
+//   expire: 从现在开始多久过期.
+func NewAuthTime(expire time.Duration) *AuthTime {
+	signStartTime := time.Now()
+	keyStartTime := signStartTime
+	signEndTime := signStartTime.Add(expire)
+	keyEndTime := signEndTime
+	return &AuthTime{
+		SignStartTime: signStartTime,
+		SignEndTime:   signEndTime,
+		KeyStartTime:  keyStartTime,
+		KeyEndTime:    keyEndTime,
+	}
+}
+
+// signString return q-sign-time string
+func (a *AuthTime) signString() string {
+	return fmt.Sprintf("%d;%d", a.SignStartTime.Unix(), a.SignEndTime.Unix())
+}
+
+// keyString return q-key-time string
+func (a *AuthTime) keyString() string {
+	return fmt.Sprintf("%d;%d", a.KeyStartTime.Unix(), a.KeyEndTime.Unix())
+}
+
+// newAuthorization 通过一系列步骤生成最终需要的 Authorization 字符串
+func newAuthorization(secretID, secretKey string, req *http.Request, authTime *AuthTime) string {
+	signTime := authTime.signString()
+	keyTime := authTime.keyString()
+	signKey := calSignKey(secretKey, keyTime)
+
+	formatHeaders := *new(string)
+	signedHeaderList := *new([]string)
+	formatHeaders, signedHeaderList = genFormatHeaders(req.Header)
+	formatParameters, signedParameterList := genFormatParameters(req.URL.Query())
+	formatString := genFormatString(req.Method, *req.URL, formatParameters, formatHeaders)
+
+	stringToSign := calStringToSign(sha1SignAlgorithm, keyTime, formatString)
+	signature := calSignature(signKey, stringToSign)
+
+	return genAuthorization(
+		secretID, signTime, keyTime, signature, signedHeaderList,
+		signedParameterList,
+	)
+}
+
+// AddAuthorizationHeader 给 req 增加签名信息
+func AddAuthorizationHeader(secretID, secretKey string, sessionToken string, req *http.Request, authTime *AuthTime) {
+	if secretID == "" {
+		return
+	}
+
+	auth := newAuthorization(secretID, secretKey, req,
+		authTime,
+	)
+	if len(sessionToken) > 0 {
+		req.Header.Set("x-cos-security-token", sessionToken)
+	}
+	req.Header.Set("Authorization", auth)
+}
+
+// calSignKey 计算 SignKey
+func calSignKey(secretKey, keyTime string) string {
+	digest := calHMACDigest(secretKey, keyTime, sha1SignAlgorithm)
+	return fmt.Sprintf("%x", digest)
+}
+
+// calStringToSign 计算 StringToSign
+func calStringToSign(signAlgorithm, signTime, formatString string) string {
+	h := sha1.New()
+	h.Write([]byte(formatString))
+	return fmt.Sprintf("%s\n%s\n%x\n", signAlgorithm, signTime, h.Sum(nil))
+}
+
+// calSignature 计算 Signature
+func calSignature(signKey, stringToSign string) string {
+	digest := calHMACDigest(signKey, stringToSign, sha1SignAlgorithm)
+	return fmt.Sprintf("%x", digest)
+}
+
+// genAuthorization 生成 Authorization
+func genAuthorization(secretID, signTime, keyTime, signature string, signedHeaderList, signedParameterList []string) string {
+	return strings.Join([]string{
+		"q-sign-algorithm=" + sha1SignAlgorithm,
+		"q-ak=" + secretID,
+		"q-sign-time=" + signTime,
+		"q-key-time=" + keyTime,
+		"q-header-list=" + strings.Join(signedHeaderList, ";"),
+		"q-url-param-list=" + strings.Join(signedParameterList, ";"),
+		"q-signature=" + signature,
+	}, "&")
+}
+
+// genFormatString 生成 FormatString
+func genFormatString(method string, uri url.URL, formatParameters, formatHeaders string) string {
+	formatMethod := strings.ToLower(method)
+	formatURI := uri.Path
+
+	return fmt.Sprintf("%s\n%s\n%s\n%s\n", formatMethod, formatURI,
+		formatParameters, formatHeaders,
+	)
+}
+
+// genFormatParameters 生成 FormatParameters 和 SignedParameterList
+// instead of the url.Values{}
+func genFormatParameters(parameters url.Values) (formatParameters string, signedParameterList []string) {
+	ps := valuesSignMap{}
+	for key, values := range parameters {
+		key = strings.ToLower(key)
+		for _, value := range values {
+			ps.Add(key, value)
+			signedParameterList = append(signedParameterList, key)
+		}
+	}
+	//formatParameters = strings.ToLower(ps.Encode())
+	formatParameters = ps.Encode()
+	sort.Strings(signedParameterList)
+	return
+}
+
+// genFormatHeaders 生成 FormatHeaders 和 SignedHeaderList
+func genFormatHeaders(headers http.Header) (formatHeaders string, signedHeaderList []string) {
+	hs := valuesSignMap{}
+	for key, values := range headers {
+		key = strings.ToLower(key)
+		for _, value := range values {
+			if isSignHeader(key) {
+				hs.Add(key, value)
+				signedHeaderList = append(signedHeaderList, key)
+			}
+		}
+	}
+	formatHeaders = hs.Encode()
+	sort.Strings(signedHeaderList)
+	return
+}
+
+// HMAC 签名
+func calHMACDigest(key, msg, signMethod string) []byte {
+	var hashFunc func() hash.Hash
+	switch signMethod {
+	case "sha1":
+		hashFunc = sha1.New
+	default:
+		hashFunc = sha1.New
+	}
+	h := hmac.New(hashFunc, []byte(key))
+	h.Write([]byte(msg))
+	return h.Sum(nil)
+}
+
+func isSignHeader(key string) bool {
+	for k, v := range needSignHeaders {
+		if key == k && v {
+			return true
+		}
+	}
+	return strings.HasPrefix(key, privateHeaderPrefix)
+}
+
+// AuthorizationTransport 给请求增加 Authorization header
+type AuthorizationTransport struct {
+	SecretID     string
+	SecretKey    string
+	SessionToken string
+	rwLocker     sync.RWMutex
+	// 签名多久过期
+	Expire    time.Duration
+	Transport http.RoundTripper
+}
+
+// SetCredential update the SecretID(ak), SercretKey(sk), sessiontoken
+func (t *AuthorizationTransport) SetCredential(ak, sk, token string) {
+	t.rwLocker.Lock()
+	defer t.rwLocker.Unlock()
+	t.SecretID = ak
+	t.SecretKey = sk
+	t.SessionToken = token
+}
+
+// GetCredential get the ak, sk, token
+func (t *AuthorizationTransport) GetCredential() (string, string, string) {
+	t.rwLocker.RLock()
+	defer t.rwLocker.RUnlock()
+	return t.SecretID, t.SecretKey, t.SessionToken
+}
+
+// RoundTrip implements the RoundTripper interface.
+func (t *AuthorizationTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	req = cloneRequest(req) // per RoundTrip contract
+	if t.Expire == time.Duration(0) {
+		t.Expire = defaultAuthExpire
+	}
+
+	ak, sk, token := t.GetCredential()
+	// 增加 Authorization header
+	authTime := NewAuthTime(t.Expire)
+	AddAuthorizationHeader(ak, sk, token, req, authTime)
+
+	resp, err := t.transport().RoundTrip(req)
+	return resp, err
+}
+
+func (t *AuthorizationTransport) transport() http.RoundTripper {
+	if t.Transport != nil {
+		return t.Transport
+	}
+	return http.DefaultTransport
+}

vendor/github.com/tencentyun/cos-go-sdk-v5/bucket.go

+package cos
+
+import (
+	"context"
+	"encoding/xml"
+	"net/http"
+)
+
+// BucketService 相关 API
+type BucketService service
+
+// BucketGetResult is the result of GetBucket
+type BucketGetResult struct {
+	XMLName        xml.Name `xml:"ListBucketResult"`
+	Name           string
+	Prefix         string `xml:"Prefix,omitempty"`
+	Marker         string `xml:"Marker,omitempty"`
+	NextMarker     string `xml:"NextMarker,omitempty"`
+	Delimiter      string `xml:"Delimiter,omitempty"`
+	MaxKeys        int
+	IsTruncated    bool
+	Contents       []Object `xml:"Contents,omitempty"`
+	CommonPrefixes []string `xml:"CommonPrefixes>Prefix,omitempty"`
+	EncodingType   string   `xml:"Encoding-Type,omitempty"`
+}
+
+// BucketGetOptions is the option of GetBucket
+type BucketGetOptions struct {
+	Prefix       string `url:"prefix,omitempty"`
+	Delimiter    string `url:"delimiter,omitempty"`
+	EncodingType string `url:"encoding-type,omitempty"`
+	Marker       string `url:"marker,omitempty"`
+	MaxKeys      int    `url:"max-keys,omitempty"`
+}
+
+// Get Bucket请求等同于 List Object请求，可以列出该Bucket下部分或者所有Object，发起该请求需要拥有Read权限。
+//
+// https://www.qcloud.com/document/product/436/7734
+func (s *BucketService) Get(ctx context.Context, opt *BucketGetOptions) (*BucketGetResult, *Response, error) {
+	var res BucketGetResult
+	sendOpt := sendOptions{
+		baseURL:  s.client.BaseURL.BucketURL,
+		uri:      "/",
+		method:   http.MethodGet,
+		optQuery: opt,
+		result:   &res,
+	}
+	resp, err := s.client.send(ctx, &sendOpt)
+	return &res, resp, err
+}
+
+// BucketPutOptions is same to the ACLHeaderOptions
+type BucketPutOptions ACLHeaderOptions
+
+// Put Bucket请求可以在指定账号下创建一个Bucket。
+//
+// https://www.qcloud.com/document/product/436/7738
+func (s *BucketService) Put(ctx context.Context, opt *BucketPutOptions) (*Response, error) {
+	sendOpt := sendOptions{
+		baseURL:   s.client.BaseURL.BucketURL,
+		uri:       "/",
+		method:    http.MethodPut,
+		optHeader: opt,
+	}
+	resp, err := s.client.send(ctx, &sendOpt)
+	return resp, err
+}
+
+// Delete Bucket请求可以在指定账号下删除Bucket，删除之前要求Bucket为空。
+//
+// https://www.qcloud.com/document/product/436/7732
+func (s *BucketService) Delete(ctx context.Context) (*Response, error) {
+	sendOpt := sendOptions{
+		baseURL: s.client.BaseURL.BucketURL,
+		uri:     "/",
+		method:  http.MethodDelete,
+	}
+	resp, err := s.client.send(ctx, &sendOpt)
+	return resp, err
+}
+
+// Head Bucket请求可以确认是否存在该Bucket，是否有权限访问，Head的权限与Read一致。
+//
+//   当其存在时，返回 HTTP 状态码200；
+//   当无权限时，返回 HTTP 状态码403；
+//   当不存在时，返回 HTTP 状态码404。
+//
+// https://www.qcloud.com/document/product/436/7735
+func (s *BucketService) Head(ctx context.Context) (*Response, error) {
+	sendOpt := sendOptions{
+		baseURL: s.client.BaseURL.BucketURL,
+		uri:     "/",
+		method:  http.MethodHead,
+	}
+	resp, err := s.client.send(ctx, &sendOpt)
+	return resp, err
+}
+
+// Bucket is the meta info of Bucket
+type Bucket struct {
+	Name         string
+	Region       string `xml:"Location,omitempty"`
+	CreationDate string `xml:",omitempty"`
+}

vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_acl.go

+package cos
+
+import (
+	"context"
+	"net/http"
+)
+
+// BucketGetACLResult is same to the ACLXml
+type BucketGetACLResult ACLXml
+
+// GetACL 使用API读取Bucket的ACL表，只有所有者有权操作。
+//
+// https://www.qcloud.com/document/product/436/7733
+func (s *BucketService) GetACL(ctx context.Context) (*BucketGetACLResult, *Response, error) {
+	var res BucketGetACLResult
+	sendOpt := sendOptions{
+		baseURL: s.client.BaseURL.BucketURL,
+		uri:     "/?acl",
+		method:  http.MethodGet,
+		result:  &res,
+	}
+	resp, err := s.client.send(ctx, &sendOpt)
+	return &res, resp, err
+}
+
+// BucketPutACLOptions is the option of PutBucketACL
+type BucketPutACLOptions struct {
+	Header *ACLHeaderOptions `url:"-" xml:"-"`
+	Body   *ACLXml           `url:"-" header:"-"`
+}
+
+// PutACL 使用API写入Bucket的ACL表，您可以通过Header："x-cos-acl","x-cos-grant-read",
+// "x-cos-grant-write","x-cos-grant-full-control"传入ACL信息，也可以通过body以XML格式传入ACL信息，
+//
+// 但是只能选择Header和Body其中一种，否则返回冲突。
+//
+// Put Bucket ACL是一个覆盖操作，传入新的ACL将覆盖原有ACL。只有所有者有权操作。
+//
+//   "x-cos-acl"：枚举值为public-read，private；public-read意味这个Bucket有公有读私有写的权限，
+//   private意味这个Bucket有私有读写的权限。
+//
+//   "x-cos-grant-read"：意味被赋予权限的用户拥有该Bucket的读权限
+//   "x-cos-grant-write"：意味被赋予权限的用户拥有该Bucket的写权限
+//   "x-cos-grant-full-control"：意味被赋予权限的用户拥有该Bucket的读写权限
+//
+// https://www.qcloud.com/document/product/436/7737
+func (s *BucketService) PutACL(ctx context.Context, opt *BucketPutACLOptions) (*Response, error) {
+	header := opt.Header
+	body := opt.Body
+	if body != nil {
+		header = nil
+	}
+	sendOpt := sendOptions{
+		baseURL:   s.client.BaseURL.BucketURL,
+		uri:       "/?acl",
+		method:    http.MethodPut,
+		body:      body,
+		optHeader: header,
+	}
+	resp, err := s.client.send(ctx, &sendOpt)
+	return resp, err
+}

vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_cors.go

+package cos
+
+import (
+	"context"
+	"encoding/xml"
+	"net/http"
+)
+
+// BucketCORSRule is the rule of BucketCORS
+type BucketCORSRule struct {
+	ID             string   `xml:"ID,omitempty"`
+	AllowedMethods []string `xml:"AllowedMethod"`
+	AllowedOrigins []string `xml:"AllowedOrigin"`
+	AllowedHeaders []string `xml:"AllowedHeader,omitempty"`
+	MaxAgeSeconds  int      `xml:"MaxAgeSeconds,omitempty"`
+	ExposeHeaders  []string `xml:"ExposeHeader,omitempty"`
+}
+
+// BucketGetCORSResult is the result of GetBucketCORS
+type BucketGetCORSResult struct {
+	XMLName xml.Name         `xml:"CORSConfiguration"`
+	Rules   []BucketCORSRule `xml:"CORSRule,omitempty"`
+}
+
+// GetCORS 实现 Bucket 跨域访问配置读取。
+//
+// https://www.qcloud.com/document/product/436/8274
+func (s *BucketService) GetCORS(ctx context.Context) (*BucketGetCORSResult, *Response, error) {
+	var res BucketGetCORSResult
+	sendOpt := sendOptions{
+		baseURL: s.client.BaseURL.BucketURL,
+		uri:     "/?cors",
+		method:  http.MethodGet,
+		result:  &res,
+	}
+	resp, err := s.client.send(ctx, &sendOpt)
+	return &res, resp, err
+}
+
+// BucketPutCORSOptions is the option of PutBucketCORS 
+type BucketPutCORSOptions struct {
+	XMLName xml.Name         `xml:"CORSConfiguration"`
+	Rules   []BucketCORSRule `xml:"CORSRule,omitempty"`
+}
+
+// PutCORS 实现 Bucket 跨域访问设置，您可以通过传入XML格式的配置文件实现配置，文件大小限制为64 KB。
+//
+// https://www.qcloud.com/document/product/436/8279
+func (s *BucketService) PutCORS(ctx context.Context, opt *BucketPutCORSOptions) (*Response, error) {
+	sendOpt := sendOptions{
+		baseURL: s.client.BaseURL.BucketURL,
+		uri:     "/?cors",
+		method:  http.MethodPut,
+		body:    opt,
+	}
+	resp, err := s.client.send(ctx, &sendOpt)
+	return resp, err
+}
+
+// DeleteCORS 实现 Bucket 跨域访问配置删除。
+//
+// https://www.qcloud.com/document/product/436/8283
+func (s *BucketService) DeleteCORS(ctx context.Context) (*Response, error) {
+	sendOpt := sendOptions{
+		baseURL: s.client.BaseURL.BucketURL,
+		uri:     "/?cors",
+		method:  http.MethodDelete,
+	}
+	resp, err := s.client.send(ctx, &sendOpt)
+	return resp, err
+}

vendor/github.com/tencentyun/cos-go-sdk-v5/doc.go

+// Package cos is COS(Cloud Object Storage) Go SDK. The V5 version(XML API).
+// There are examples of using each API in the project's 'example' directory.
+package cos

vendor/github.com/tencentyun/cos-go-sdk-v5/go.mod

+module github.com/tencentyun/cos-go-sdk-v5
+
+go 1.12
+
+require (
+	github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409
+	github.com/google/go-querystring v1.0.0
+	github.com/mozillazg/go-httpheader v0.2.1
+	github.com/stretchr/testify v1.3.0
+)