From 29bb8a283fed6400a3facf97920864e45ed58494 Mon Sep 17 00:00:00 2001 From: lentitude2tk Date: Wed, 31 Jan 2024 10:30:11 +0800 Subject: [PATCH] support tencent storage Signed-off-by: lentitude2tk --- go.mod | 1 + go.sum | 2 + storage/client.go | 11 ++++-- storage/tencent.go | 94 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 storage/tencent.go diff --git a/go.mod b/go.mod index 26b00f8..2b094bd 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.2 + github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.853 github.com/tidwall/gjson v1.17.0 github.com/uber/jaeger-client-go v2.30.0+incompatible go.uber.org/atomic v1.11.0 diff --git a/go.sum b/go.sum index 93dd009..22f8afe 100644 --- a/go.sum +++ b/go.sum @@ -266,6 +266,8 @@ github.com/swaggo/gin-swagger v1.6.0 h1:y8sxvQ3E20/RCyrXeFfg60r6H0Z+SwpTjMYsMm+z github.com/swaggo/gin-swagger v1.6.0/go.mod h1:BG00cCEy294xtVpyIAHG6+e2Qzj/xKlRdOqDkvq0uzo= github.com/swaggo/swag v1.16.2 h1:28Pp+8DkQoV+HLzLx8RGJZXNGKbFqnuvSbAAtoxiY04= github.com/swaggo/swag v1.16.2/go.mod h1:6YzXnDcpr0767iOejs318CwYkCQqyGer6BizOg03f+E= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.853 h1:TNYjF1jDLLNTirAkq7zRT9iF9xC2ZjgwpXsVSEBQvgQ= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.853/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0= github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= diff --git a/storage/client.go b/storage/client.go index 1a7e408..a332b48 100644 --- a/storage/client.go +++ b/storage/client.go @@ -11,14 +11,15 @@ type Provider string const ( AWS Provider = "aws" GCP Provider = "gcp" - ALI Provider = "ali" AZURE Provider = "azure" + ALI Provider = "ali" + TC Provider = "tc" unknown Provider = "unknown" ) const _defaultPageSize = 1000 -var _providerMap = map[string]Provider{"aws": AWS, "gcp": GCP, "ali": ALI, "azure": AZURE, "az": AZURE} +var _providerMap = map[string]Provider{"aws": AWS, "gcp": GCP, "ali": ALI, "azure": AZURE, "az": AZURE, "tc": TC} func ParseProvider(s string) Provider { if p, ok := _providerMap[s]; ok { @@ -53,10 +54,12 @@ func NewClient(cfg Cfg) (Client, error) { return NewAWSClient(cfg) case GCP: return NewGCPClient(cfg) - case ALI: - return NewAliyunClient(cfg) case AZURE: return NewAzureClient(cfg) + case ALI: + return NewAliyunClient(cfg) + case TC: + return NewTencentClient(cfg) default: return nil, fmt.Errorf("storage: unknown provide %s", cfg.Provider) } diff --git a/storage/tencent.go b/storage/tencent.go new file mode 100644 index 0000000..30d21a0 --- /dev/null +++ b/storage/tencent.go @@ -0,0 +1,94 @@ +package storage + +import ( + "fmt" + + "github.com/cockroachdb/errors" + "github.com/minio/minio-go/v7" + minioCred "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common" +) + +// NewTencentClient returns a minio.Client which is compatible for tencent COS +func NewTencentClient(cfg Cfg) (*MinioClient, error) { + opts := minio.Options{Secure: cfg.UseSSL, Region: cfg.Region, BucketLookup: minio.BucketLookupDNS} + if cfg.UseIAM { + provider, err := NewCredentialProvider() + if err != nil { + return nil, fmt.Errorf("storage: new tencent credential provider %w", err) + } + opts.Creds = minioCred.New(provider) + } else { + opts.Creds = minioCred.NewStaticV4(cfg.AK, cfg.SK, "") + } + + var addr string + if len(cfg.Endpoint) <= 0 { + addr = fmt.Sprintf("cos.%s.myqcloud.com", opts.Region) + opts.Secure = true + } else { + addr = cfg.Endpoint + } + cli, err := minio.New(addr, &opts) + if err != nil { + return nil, fmt.Errorf("storage: new tencent client %w", err) + } + + return &MinioClient{cli: cli, provider: TC}, nil +} + +// Credential is defined to mock tencent credential.Credentials +// +//go:generate mockery --name=Credential --with-expecter +type Credential interface { + common.CredentialIface +} + +// CredentialProvider implements "github.com/minio/minio-go/v7/pkg/credentials".Provider +// also implements transport +type CredentialProvider struct { + // tencentCreds doesn't provide a way to get the expired time, so we use the cache to check if it's expired + // when tencentCreds.GetSecretId is different from the cache, we know it's expired + akCache string + tencentCreds Credential +} + +func NewCredentialProvider() (minioCred.Provider, error) { + provider, err := common.DefaultTkeOIDCRoleArnProvider() + if err != nil { + return nil, errors.Wrap(err, "failed to create tencent credential provider") + } + + cred, err := provider.GetCredential() + if err != nil { + return nil, errors.Wrap(err, "failed to get tencent credential") + } + return &CredentialProvider{tencentCreds: cred}, nil +} + +// Retrieve returns nil if it successfully retrieved the value. +// Error is returned if the value were not obtainable, or empty. +// according to the caller minioCred.Credentials.Get(), +// it already has a lock, so we don't need to worry about concurrency +func (c *CredentialProvider) Retrieve() (minioCred.Value, error) { + ret := minioCred.Value{} + ak := c.tencentCreds.GetSecretId() + ret.AccessKeyID = ak + c.akCache = ak + + sk := c.tencentCreds.GetSecretKey() + ret.SecretAccessKey = sk + + securityToken := c.tencentCreds.GetToken() + ret.SessionToken = securityToken + return ret, nil +} + +// IsExpired returns if the credentials are no longer valid, and need +// to be retrieved. +// according to the caller minioCred.Credentials.IsExpired(), +// it already has a lock, so we don't need to worry about concurrency +func (c CredentialProvider) IsExpired() bool { + ak := c.tencentCreds.GetSecretId() + return ak != c.akCache +}