Skip to content

Commit 21ec012

Browse files
YangSen-qnYangSen-qn
authored
Get region support specified UC hosts and Client & List bucket support output file parts (#125)
1 parent 1d3fbec commit 21ec012

10 files changed

+157
-25
lines changed

‎CHANGELOG.md

+5
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# Changelog
2+
## 7.20.1
3+
* 新增
4+
* 获取区域 API 支持单独配置 UC 域名
5+
* BucketManager List Bucket 接口支持返回文件的 parts
6+
27
## 7.20.0
38
* 新增
49
* 新版存储客户端库 storagev2 包，包含

‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ github.com/qiniu/go-sdk
1717
在您的项目中的 `go.mod` 文件内添加这行代码
1818

1919
```
20-
require github.com/qiniu/go-sdk/v7 v7.20.0
20+
require github.com/qiniu/go-sdk/v7 v7.20.1
2121
```
2222

2323
并且在项目中使用 `"github.com/qiniu/go-sdk/v7"` 引用 Qiniu Go SDK。

‎conf/conf.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import (
55
"strings"
66
)
77

8-
const Version = "7.20.0"
8+
const Version = "7.20.1"
99

1010
const (
1111
CONTENT_TYPE_JSON = "application/json"

‎storage/bucket.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1023,7 +1023,7 @@ func (m *BucketManager) Zone(bucket string) (z *Zone, err error) {
10231023
}
10241024

10251025
func (m *BucketManager) makeRequestOptions() *apis.Options {
1026-
return &apis.Options{OverwrittenBucketHosts: getUcEndpoint(m.Cfg.UseHTTPS)}
1026+
return &apis.Options{OverwrittenBucketHosts: getUcEndpoint(m.Cfg.UseHTTPS, nil)}
10271027
}
10281028

10291029
// 构建op的方法，导出的方法支持在Batch操作中使用

‎storage/bucket_list.go

+15-2
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,11 @@ type ListItem struct {
5454
* 文件的 md5 值
5555
*/
5656
Md5 string `json:"md5"`
57+
58+
/**
59+
* 文件的分片信息
60+
*/
61+
Parts []uint `json:"parts"`
5762
}
5863

5964
// 接口可能返回空的记录
@@ -103,6 +108,7 @@ type listInputOptions struct {
103108
delimiter string
104109
marker string
105110
limit int
111+
needParts bool
106112
}
107113

108114
type ListInputOption func(options *listInputOptions)
@@ -131,6 +137,12 @@ func ListInputOptionsLimit(limit int) ListInputOption {
131137
}
132138
}
133139

140+
func ListInputOptionsNeedParts(needParts bool) ListInputOption {
141+
return func(input *listInputOptions) {
142+
input.needParts = needParts
143+
}
144+
}
145+
134146
// ListFilesWithContext
135147
//
136148
// @Description: 用来获取空间文件列表，可以根据需要指定文件的列举条件
@@ -166,7 +178,7 @@ func (m *BucketManager) ListFilesWithContext(ctx context.Context, bucket string,
166178
}
167179

168180
ret = &ListFilesRet{}
169-
reqURL := fmt.Sprintf("%s%s", host, uriListFiles(bucket, inputOptions.prefix, inputOptions.delimiter, inputOptions.marker, inputOptions.limit))
181+
reqURL := fmt.Sprintf("%s%s", host, uriListFiles(bucket, inputOptions.prefix, inputOptions.delimiter, inputOptions.marker, inputOptions.limit, inputOptions.needParts))
170182
err = m.Client.CredentialedCall(ctx, m.Mac, auth.TokenQiniu, ret, "POST", reqURL, nil)
171183
if err != nil {
172184
return nil, false, err
@@ -228,7 +240,7 @@ func (m *BucketManager) ListBucketContext(ctx context.Context, bucket, prefix, d
228240
return retCh, err
229241
}
230242

231-
func uriListFiles(bucket, prefix, delimiter, marker string, limit int) string {
243+
func uriListFiles(bucket, prefix, delimiter, marker string, limit int, needParts bool) string {
232244
query := make(url.Values)
233245
query.Add("bucket", bucket)
234246
if prefix != "" {
@@ -243,5 +255,6 @@ func uriListFiles(bucket, prefix, delimiter, marker string, limit int) string {
243255
if limit > 0 {
244256
query.Add("limit", strconv.FormatInt(int64(limit), 10))
245257
}
258+
query.Add("needparts", strconv.FormatBool(needParts))
246259
return fmt.Sprintf("/list?%s", query.Encode())
247260
}

‎storage/bucket_list_test.go

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
//go:build integration
2+
// +build integration
3+
4+
package storage
5+
6+
import (
7+
"context"
8+
"testing"
9+
)
10+
11+
func TestList(t *testing.T) {
12+
ret, _, err := bucketManager.ListFilesWithContext(context.Background(), testBucket,
13+
ListInputOptionsLimit(1000),
14+
ListInputOptionsNeedParts(false),
15+
)
16+
if err != nil {
17+
t.Fatalf("List bucket files error: %v\n", err)
18+
}
19+
20+
hasParts := false
21+
for _, item := range ret.Items {
22+
if len(item.Parts) > 0 {
23+
hasParts = true
24+
}
25+
}
26+
if hasParts {
27+
t.Fatal("list files: should no parts")
28+
}
29+
30+
ret, _, err = bucketManager.ListFilesWithContext(context.Background(), testBucket,
31+
ListInputOptionsLimit(1000),
32+
ListInputOptionsNeedParts(true),
33+
)
34+
if err != nil {
35+
t.Fatalf("List bucket files error: %v\n", err)
36+
}
37+
38+
hasParts = false
39+
for _, item := range ret.Items {
40+
if len(item.Parts) > 0 {
41+
hasParts = true
42+
}
43+
}
44+
if !hasParts {
45+
t.Fatal("list files: should parts")
46+
}
47+
}

‎storage/region.go

+28-12
Original file line numberDiff line numberDiff line change
@@ -290,18 +290,21 @@ func getUcBackupHosts() []string {
290290
return hosts
291291
}
292292

293-
func getUcEndpoint(useHttps bool) region_v2.EndpointsProvider {
294-
ucHosts := make([]string, 0, 1+len(ucHosts))
295-
if len(UcHost) > 0 {
296-
ucHosts = append(ucHosts, endpoint(useHttps, UcHost))
297-
}
298-
for _, host := range ucHosts {
299-
if len(host) > 0 {
300-
ucHosts = append(ucHosts, endpoint(useHttps, host))
293+
func getUcEndpoint(useHttps bool, hosts []string) region_v2.EndpointsProvider {
294+
if len(hosts) == 0 {
295+
if len(UcHost) > 0 {
296+
hosts = append(hosts, endpoint(useHttps, UcHost))
297+
}
298+
299+
for _, host := range ucHosts {
300+
if len(host) > 0 {
301+
hosts = append(hosts, endpoint(useHttps, host))
302+
}
301303
}
302304
}
303-
if len(ucHosts) > 0 {
304-
return region_v2.Endpoints{Preferred: ucHosts}
305+
306+
if len(hosts) > 0 {
307+
return region_v2.Endpoints{Preferred: hosts}
305308
} else {
306309
return nil
307310
}
@@ -347,19 +350,25 @@ func GetRegionsInfo(mac *auth.Credentials) ([]RegionInfo, error) {
347350
}
348351

349352
func GetRegionsInfoWithOptions(mac *auth.Credentials, options UCApiOptions) ([]RegionInfo, error) {
353+
var httpClient clientv2.Client
354+
if options.Client != nil {
355+
httpClient = options.Client.Client
356+
}
350357
response, err := apis.NewStorage(&http_client.Options{
358+
BasicHTTPClient: httpClient,
351359
HostFreezeDuration: options.HostFreezeDuration,
352360
HostRetryConfig: &clientv2.RetryConfig{
353361
RetryMax: options.RetryMax,
354362
},
355363
}).GetRegions(
356364
context.Background(),
357365
&apis.GetRegionsRequest{Credentials: mac},
358-
&apis.Options{OverwrittenBucketHosts: getUcEndpoint(options.UseHttps)},
366+
&apis.Options{OverwrittenBucketHosts: getUcEndpoint(options.UseHttps, options.Hosts)},
359367
)
360368
if err != nil {
361369
return nil, err
362370
}
371+
363372
regions := make([]RegionInfo, 0, len(response.Regions))
364373
for _, region := range response.Regions {
365374
regions = append(regions, RegionInfo{
@@ -377,14 +386,21 @@ type ucClientConfig struct {
377386
// 单域名重试次数
378387
RetryMax int
379388

389+
// 请求的域名
390+
Hosts []string
391+
380392
// 主备域名冻结时间（默认：600s），当一个域名请求失败（单个域名会被重试 TryTimes 次），会被冻结一段时间，使用备用域名进行重试，在冻结时间内，域名不能被使用，当一个操作中所有域名竣备冻结操作不在进行重试，返回最后一次操作的错误。
381393
HostFreezeDuration time.Duration
382394

383395
Client *client.Client
384396
}
385397

386398
func getUCClient(config ucClientConfig, mac *auth.Credentials) clientv2.Client {
387-
allHosts := getUcBackupHosts()
399+
allHosts := config.Hosts
400+
if len(allHosts) == 0 {
401+
allHosts = getUcBackupHosts()
402+
}
403+
388404
var hosts []string = nil
389405
if !config.IsUcQueryApi {
390406
// 非 uc query api 去除 defaultApiHost

‎storage/region_test.go

+24-1
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,10 @@ package storage
55

66
import (
77
"encoding/json"
8-
"github.com/qiniu/go-sdk/v7/client"
98
"strings"
109
"testing"
10+
11+
"github.com/qiniu/go-sdk/v7/client"
1112
)
1213

1314
func TestRegion(t *testing.T) {
@@ -150,6 +151,17 @@ func TestRegionWithSetHost(t *testing.T) {
150151
if !strings.HasPrefix(region1.IovipHost, "iovip") || !strings.HasSuffix(region1.IovipHost, ".qbox.me") {
151152
t.Fatalf("region1.IovipHost is wrong: %v\v", region1.IovipHost)
152153
}
154+
155+
region1, err = GetRegionWithOptions(testAK, testBucket, UCApiOptions{
156+
UseHttps: true,
157+
RetryMax: 0,
158+
Hosts: []string{"mock.uc.com"},
159+
HostFreezeDuration: 0,
160+
Client: nil,
161+
})
162+
if err == nil {
163+
t.Fatalf("request should be wrong")
164+
}
153165
}
154166

155167
func TestRegionV4(t *testing.T) {
@@ -161,6 +173,17 @@ func TestRegionV4(t *testing.T) {
161173
if len(regionGroup.regions) == 0 {
162174
t.Fatalf("region1.IovipHost is wrong")
163175
}
176+
177+
_, err = getRegionGroupWithOptions(testAK, testBucket, UCApiOptions{
178+
UseHttps: true,
179+
RetryMax: 0,
180+
Hosts: []string{"mock.uc.com"},
181+
HostFreezeDuration: 0,
182+
Client: nil,
183+
})
184+
if err == nil {
185+
t.Fatalf("request should be wrong")
186+
}
164187
}
165188

166189
func TestRegionV4WithNoProtocol(t *testing.T) {

‎storage/region_uc_v2.go

+29-5
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212

1313
"golang.org/x/sync/singleflight"
1414

15+
"github.com/qiniu/go-sdk/v7/client"
1516
"github.com/qiniu/go-sdk/v7/internal/clientv2"
1617
)
1718

@@ -209,9 +210,28 @@ func storeRegionV2Cache() {
209210

210211
type UCApiOptions struct {
211212
UseHttps bool //
212-
RetryMax int // 单域名重试次数
213+
214+
RetryMax int // 单域名重试次数
215+
216+
Hosts []string // api 请求的域名
217+
213218
// 主备域名冻结时间（默认：600s），当一个域名请求失败（单个域名会被重试 TryTimes 次），会被冻结一段时间，使用备用域名进行重试，在冻结时间内，域名不能被使用，当一个操作中所有域名竣备冻结操作不在进行重试，返回最后一次操作的错误。
214219
HostFreezeDuration time.Duration
220+
221+
Client *client.Client // api 请求使用的 client
222+
}
223+
224+
func (o *UCApiOptions) init() {
225+
if len(o.Hosts) == 0 {
226+
o.Hosts = getUcBackupHosts()
227+
}
228+
}
229+
230+
func (o *UCApiOptions) firstHost() string {
231+
if len(o.Hosts) == 0 {
232+
return ""
233+
}
234+
return o.Hosts[0]
215235
}
216236

217237
func DefaultUCApiOptions() UCApiOptions {
@@ -223,6 +243,7 @@ func DefaultUCApiOptions() UCApiOptions {
223243
}
224244

225245
func getRegionByV2(ak, bucket string, options UCApiOptions) (*Region, error) {
246+
options.init()
226247

227248
regionV2CacheLock.RLock()
228249
if regionV2CacheLoaded {
@@ -240,20 +261,22 @@ func getRegionByV2(ak, bucket string, options UCApiOptions) (*Region, error) {
240261
}()
241262
}
242263

243-
regionCacheKey := makeRegionCacheKey(ak, bucket)
264+
regionCacheKey := makeRegionCacheKey(ak, bucket, options.Hosts)
244265
//check from cache
245266
if v, ok := regionV2Cache.Load(regionCacheKey); ok && time.Now().Before(v.(regionV2CacheValue).Deadline) {
246267
return v.(regionV2CacheValue).Region, nil
247268
}
248269

249270
newRegion, err, _ := ucQueryV2Group.Do(regionCacheKey, func() (interface{}, error) {
250-
reqURL := fmt.Sprintf("%s/v2/query?ak=%s&bucket=%s", getUcHost(options.UseHttps), ak, bucket)
271+
reqURL := fmt.Sprintf("%s/v2/query?ak=%s&bucket=%s", endpoint(options.UseHttps, options.firstHost()), ak, bucket)
251272

252273
var ret UcQueryRet
253274
c := getUCClient(ucClientConfig{
254275
IsUcQueryApi: true,
255276
RetryMax: options.RetryMax,
277+
Hosts: options.Hosts,
256278
HostFreezeDuration: options.HostFreezeDuration,
279+
Client: options.Client,
257280
}, nil)
258281
err := clientv2.DoAndDecodeJsonResponse(c, clientv2.RequestParams{
259282
Context: context.Background(),
@@ -294,6 +317,7 @@ func getRegionByV2(ak, bucket string, options UCApiOptions) (*Region, error) {
294317
return newRegion.(*Region), err
295318
}
296319

297-
func makeRegionCacheKey(ak, bucket string) string {
298-
return fmt.Sprintf("%s:%s:%x", ak, bucket, md5.Sum([]byte(getUcHost(false))))
320+
func makeRegionCacheKey(ak, bucket string, ucHosts []string) string {
321+
hostStrings := fmt.Sprintf("%v", ucHosts)
322+
return fmt.Sprintf("%s:%s:%x", ak, bucket, md5.Sum([]byte(hostStrings)))
299323
}

‎storage/region_uc_v4.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,8 @@ func storeRegionV4Cache() {
123123
}
124124

125125
func getRegionByV4(ak, bucket string, options UCApiOptions) (*RegionGroup, error) {
126+
options.init()
127+
126128
regionV4CacheLock.RLock()
127129
if regionV4CacheLoaded {
128130
regionV4CacheLock.RUnlock()
@@ -139,21 +141,23 @@ func getRegionByV4(ak, bucket string, options UCApiOptions) (*RegionGroup, error
139141
}()
140142
}
141143

142-
regionCacheKey := makeRegionCacheKey(ak, bucket)
144+
regionCacheKey := makeRegionCacheKey(ak, bucket, options.Hosts)
143145
//check from cache
144146
if v, ok := regionV4Cache.Load(regionCacheKey); ok && time.Now().Before(v.(regionV4CacheValue).Deadline) {
145147
cacheValue, _ := v.(regionV4CacheValue)
146148
return NewRegionGroup(cacheValue.getRegions()...), nil
147149
}
148150

149151
newRegion, err, _ := ucQueryV4Group.Do(regionCacheKey, func() (interface{}, error) {
150-
reqURL := fmt.Sprintf("%s/v4/query?ak=%s&bucket=%s", getUcHost(options.UseHttps), ak, bucket)
152+
reqURL := fmt.Sprintf("%s/v4/query?ak=%s&bucket=%s", endpoint(options.UseHttps, options.firstHost()), ak, bucket)
151153

152154
var ret ucQueryV4Ret
153155
c := getUCClient(ucClientConfig{
154156
IsUcQueryApi: true,
155157
RetryMax: options.RetryMax,
158+
Hosts: options.Hosts,
156159
HostFreezeDuration: options.HostFreezeDuration,
160+
Client: options.Client,
157161
}, nil)
158162
err := clientv2.DoAndDecodeJsonResponse(c, clientv2.RequestParams{
159163
Context: context.Background(),

0 commit comments

Comments
 (0)