Skip to content

Commit 08feb95

Browse files
bachuebachue
committed
support AccelerateUploading in github.com/qiniu/go-sdk/v7/storage 
1 parent fe785ee commit 08feb95

13 files changed

+194
-50
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
## 7.24.0
44
* 新增
55
* 支持工作流模版
6+
* 在 [storage](github.com/qiniu/go-sdk/v7/storage) 包中支持上传加速
67
* 优化
78
* 超时连接重试机制
89
* 域名解析请求机制

‎storage/config.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,10 @@ type Config struct {
1111

1212
// 如果设置的Host本身是以http://开头的，又设置了该字段为true，那么优先使用该字段，使用https协议
1313
// 同理如果该字段为false, 但是设置的host以https开头，那么使用http协议通信
14-
UseHTTPS bool //是否使用https域名
15-
UseCdnDomains bool //是否使用cdn加速域名
16-
CentralRsHost string //中心机房的RsHost，用于list bucket
14+
UseHTTPS bool //是否使用https域名
15+
UseCdnDomains bool //是否使用cdn加速域名
16+
AccelerateUploading bool //是否启用上传加速
17+
CentralRsHost string //中心机房的RsHost，用于list bucket
1718

1819
// 兼容保留
1920
RsHost string

‎storage/form_upload.go

+1
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ func NewFormUploaderEx(cfg *Config, clt *client.Client) *FormUploader {
8080
opts := http_client.Options{
8181
BasicHTTPClient: clt.Client,
8282
UseInsecureProtocol: !cfg.UseHTTPS,
83+
AccelerateUploading: cfg.AccelerateUploading,
8384
HostRetryConfig: &clientv2.RetryConfig{},
8485
}
8586
if region := cfg.GetRegion(); region != nil {

‎storage/region_uc_v2.go

+44-7
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,9 @@ type UCApiOptions struct {
172172
// 是否使用 HTTPS 协议
173173
UseHttps bool
174174

175+
// 是否加速上传
176+
AccelerateUploading bool
177+
175178
// 单域名重试次数
176179
RetryMax int
177180

@@ -231,6 +234,7 @@ func (options *UCApiOptions) getApiStorageClient() *apis.Storage {
231234
return apis.NewStorage(&http_client.Options{
232235
Interceptors: []clientv2.Interceptor{},
233236
UseInsecureProtocol: !options.UseHttps,
237+
AccelerateUploading: options.AccelerateUploading,
234238
Resolver: options.Resolver,
235239
Chooser: options.Chooser,
236240
HostRetryConfig: &clientv2.RetryConfig{RetryMax: options.RetryMax, Retrier: options.Retrier},
@@ -276,7 +280,7 @@ func getRegionByV2(ak, bucket string, options UCApiOptions) (*Region, error) {
276280
}()
277281
}
278282

279-
regionCacheKey := makeRegionCacheKey(ak, bucket, options.Hosts)
283+
regionCacheKey := makeRegionCacheKey(ak, bucket, options.Hosts, options.AccelerateUploading)
280284
// check from cache
281285
if v, ok := regionV2Cache.Load(regionCacheKey); ok && time.Now().Before(v.(regionV2CacheValue).Deadline) {
282286
return v.(regionV2CacheValue).Region, nil
@@ -306,9 +310,13 @@ func getRegionByV2(ak, bucket string, options UCApiOptions) (*Region, error) {
306310
return newRegion.(*Region), err
307311
}
308312

309-
func makeRegionCacheKey(ak, bucket string, ucHosts []string) string {
313+
func makeRegionCacheKey(ak, bucket string, ucHosts []string, accelerateUploading bool) string {
310314
hostStrings := fmt.Sprintf("%v", ucHosts)
311-
return fmt.Sprintf("%s:%s:%x", ak, bucket, md5.Sum([]byte(hostStrings)))
315+
s := fmt.Sprintf("%s:%s:%x", ak, bucket, md5.Sum([]byte(hostStrings)))
316+
if accelerateUploading {
317+
s += ":1"
318+
}
319+
return s
312320
}
313321

314322
func _getRegionByV2WithoutCache(ak, bucket string, options UCApiOptions) (*Region, int64, error) {
@@ -323,11 +331,40 @@ func _getRegionByV2WithoutCache(ak, bucket string, options UCApiOptions) (*Regio
323331
if err != nil {
324332
return nil, 0, err
325333
}
334+
var srcUpHosts, cdnUpHosts []string
326335
var rsHost, rsfHost, apiHost, ioVipHost, ioSrcHost string
327-
srcUpHosts := append(response.UpDomains.SourceUpDomains.MainSourceUpDomains, response.UpDomains.SourceUpDomains.BackupSourceUpDomains...)
328-
srcUpHosts = append(srcUpHosts, response.UpDomains.OldSourceDomains.OldMainSourceUpDomains...)
329-
cdnUpHosts := append(response.UpDomains.AcceleratedUpDomains.MainAcceleratedUpDomains, response.UpDomains.AcceleratedUpDomains.BackupAcceleratedUpDomains...)
330-
cdnUpHosts = append(cdnUpHosts, response.UpDomains.OldAcceleratedDomains.OldMainAcceleratedUpDomains...)
336+
if options.AccelerateUploading && len(response.UpDomains.AcceleratedUpDomains.MainAcceleratedUpDomains) > 0 {
337+
srcUpHosts = make([]string, 0,
338+
len(response.UpDomains.AcceleratedUpDomains.MainAcceleratedUpDomains)+
339+
len(response.UpDomains.AcceleratedUpDomains.BackupAcceleratedUpDomains)+
340+
len(response.UpDomains.OldAcceleratedDomains.OldMainAcceleratedUpDomains)+
341+
len(response.UpDomains.SourceUpDomains.MainSourceUpDomains)+
342+
len(response.UpDomains.SourceUpDomains.BackupSourceUpDomains)+
343+
len(response.UpDomains.OldSourceDomains.OldMainSourceUpDomains))
344+
srcUpHosts = append(srcUpHosts, response.UpDomains.AcceleratedUpDomains.MainAcceleratedUpDomains...)
345+
srcUpHosts = append(srcUpHosts, response.UpDomains.AcceleratedUpDomains.BackupAcceleratedUpDomains...)
346+
srcUpHosts = append(srcUpHosts, response.UpDomains.OldAcceleratedDomains.OldMainAcceleratedUpDomains...)
347+
srcUpHosts = append(srcUpHosts, response.UpDomains.SourceUpDomains.MainSourceUpDomains...)
348+
srcUpHosts = append(srcUpHosts, response.UpDomains.SourceUpDomains.BackupSourceUpDomains...)
349+
srcUpHosts = append(srcUpHosts, response.UpDomains.OldSourceDomains.OldMainSourceUpDomains...)
350+
cdnUpHosts = make([]string, 0, len(response.UpDomains.AcceleratedUpDomains.MainAcceleratedUpDomains))
351+
cdnUpHosts = append(cdnUpHosts, response.UpDomains.AcceleratedUpDomains.MainAcceleratedUpDomains...)
352+
} else {
353+
srcUpHosts = make([]string, 0,
354+
len(response.UpDomains.SourceUpDomains.MainSourceUpDomains)+
355+
len(response.UpDomains.SourceUpDomains.BackupSourceUpDomains)+
356+
len(response.UpDomains.OldSourceDomains.OldMainSourceUpDomains))
357+
srcUpHosts = append(srcUpHosts, response.UpDomains.SourceUpDomains.MainSourceUpDomains...)
358+
srcUpHosts = append(srcUpHosts, response.UpDomains.SourceUpDomains.BackupSourceUpDomains...)
359+
srcUpHosts = append(srcUpHosts, response.UpDomains.OldSourceDomains.OldMainSourceUpDomains...)
360+
cdnUpHosts = make([]string, 0,
361+
len(response.UpDomains.AcceleratedUpDomains.MainAcceleratedUpDomains)+
362+
len(response.UpDomains.AcceleratedUpDomains.BackupAcceleratedUpDomains)+
363+
len(response.UpDomains.OldAcceleratedDomains.OldMainAcceleratedUpDomains))
364+
cdnUpHosts = append(cdnUpHosts, response.UpDomains.AcceleratedUpDomains.MainAcceleratedUpDomains...)
365+
cdnUpHosts = append(cdnUpHosts, response.UpDomains.AcceleratedUpDomains.BackupAcceleratedUpDomains...)
366+
cdnUpHosts = append(cdnUpHosts, response.UpDomains.OldAcceleratedDomains.OldMainAcceleratedUpDomains...)
367+
}
331368
if len(response.RsDomains.AcceleratedRsDomains.MainAcceleratedRsDomains) > 0 {
332369
rsHost = response.RsDomains.AcceleratedRsDomains.MainAcceleratedRsDomains[0]
333370
}

‎storage/region_uc_v4.go

+7-2
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ func getRegionByV4(ak, bucket string, options UCApiOptions) (*RegionGroup, error
110110
}()
111111
}
112112

113-
regionCacheKey := makeRegionCacheKey(ak, bucket, options.Hosts)
113+
regionCacheKey := makeRegionCacheKey(ak, bucket, options.Hosts, options.AccelerateUploading)
114114
// check from cache
115115
if v, ok := regionV4Cache.Load(regionCacheKey); ok && time.Now().Before(v.(regionV4CacheValue).Deadline) {
116116
cacheValue, _ := v.(regionV4CacheValue)
@@ -145,7 +145,12 @@ func getRegionByV4(ak, bucket string, options UCApiOptions) (*RegionGroup, error
145145
func _getRegionByV4WithoutCache(ak, bucket string, options UCApiOptions) ([]*Region, int64, error) {
146146
toRegion := func(r *query_bucket_v4.BucketQueryHost) *Region {
147147
var rsHost, rsfHost, apiHost, ioVipHost, ioSrcHost string
148-
upDomains := append(r.UpDomains.PreferedUpDomains, r.UpDomains.AlternativeUpDomains...)
148+
upDomains := make([]string, 0, len(r.UpDomains.AcceleratedUpDomains)+len(r.UpDomains.PreferedUpDomains)+len(r.UpDomains.AlternativeUpDomains))
149+
if options.AccelerateUploading && len(r.UpDomains.AcceleratedUpDomains) > 0 {
150+
upDomains = append(upDomains, r.UpDomains.AcceleratedUpDomains...)
151+
}
152+
upDomains = append(upDomains, r.UpDomains.PreferedUpDomains...)
153+
upDomains = append(upDomains, r.UpDomains.AlternativeUpDomains...)
149154
if len(r.RsDomains.PreferedRsDomains) > 0 {
150155
rsHost = r.RsDomains.PreferedRsDomains[0]
151156
}

‎storage/resume_uploader.go

+2
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ func NewResumeUploaderEx(cfg *Config, clt *client.Client) *ResumeUploader {
4444
opts := http_client.Options{
4545
BasicHTTPClient: clt.Client,
4646
UseInsecureProtocol: !cfg.UseHTTPS,
47+
AccelerateUploading: cfg.AccelerateUploading,
4748
HostRetryConfig: &clientv2.RetryConfig{},
4849
}
4950
if region := cfg.GetRegion(); region != nil {
@@ -259,6 +260,7 @@ func newResumeUploaderImpl(resumeUploader *ResumeUploader, bucket, key string, h
259260
opts := http_client.Options{
260261
BasicHTTPClient: resumeUploader.Client.Client,
261262
UseInsecureProtocol: !resumeUploader.Cfg.UseHTTPS,
263+
AccelerateUploading: resumeUploader.Cfg.AccelerateUploading,
262264
HostRetryConfig: &clientv2.RetryConfig{},
263265
}
264266
if region := resumeUploader.Cfg.GetRegion(); region != nil {

‎storage/resume_uploader_v2.go

+2
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ func NewResumeUploaderV2Ex(cfg *Config, clt *client.Client) *ResumeUploaderV2 {
4545
opts := http_client.Options{
4646
BasicHTTPClient: clt.Client,
4747
UseInsecureProtocol: !cfg.UseHTTPS,
48+
AccelerateUploading: cfg.AccelerateUploading,
4849
HostRetryConfig: &clientv2.RetryConfig{},
4950
}
5051
if region := cfg.GetRegion(); region != nil {
@@ -245,6 +246,7 @@ func newResumeUploaderV2Impl(resumeUploader *ResumeUploaderV2, bucket, key strin
245246
opts := http_client.Options{
246247
BasicHTTPClient: resumeUploader.Client.Client,
247248
UseInsecureProtocol: !resumeUploader.Cfg.UseHTTPS,
249+
AccelerateUploading: resumeUploader.Cfg.AccelerateUploading,
248250
HostRetryConfig: &clientv2.RetryConfig{},
249251
}
250252
if region := resumeUploader.Cfg.GetRegion(); region != nil {

‎storage/upload_manager.go

+26-21
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,10 @@ const (
2727

2828
// UploadConfig 为 UploadManager 提供配置信息
2929
type UploadConfig struct {
30-
UseHTTPS bool
31-
UseCdnDomains bool
32-
Regions *RegionGroup
30+
UseHTTPS bool
31+
UseCdnDomains bool
32+
AccelerateUploading bool
33+
Regions *RegionGroup
3334
}
3435

3536
// NewUploadConfig 创建默认的 UploadConfig 对象
@@ -434,31 +435,34 @@ func (manager *UploadManager) getRecoverRegion(key *string, upToken string, resu
434435

435436
func (manager *UploadManager) getFormUploader(region *Region) *FormUploader {
436437
return NewFormUploaderEx(&Config{
437-
Zone: region,
438-
Region: region,
439-
UseHTTPS: manager.cfg.UseHTTPS,
440-
UseCdnDomains: manager.cfg.UseCdnDomains,
441-
CentralRsHost: "",
438+
Zone: region,
439+
Region: region,
440+
UseHTTPS: manager.cfg.UseHTTPS,
441+
UseCdnDomains: manager.cfg.UseCdnDomains,
442+
AccelerateUploading: manager.cfg.AccelerateUploading,
443+
CentralRsHost: "",
442444
}, manager.client)
443445
}
444446

445447
func (manager *UploadManager) getResumeV1Uploader(region *Region) *ResumeUploader {
446448
return NewResumeUploaderEx(&Config{
447-
Zone: region,
448-
Region: region,
449-
UseHTTPS: manager.cfg.UseHTTPS,
450-
UseCdnDomains: manager.cfg.UseCdnDomains,
451-
CentralRsHost: "",
449+
Zone: region,
450+
Region: region,
451+
UseHTTPS: manager.cfg.UseHTTPS,
452+
UseCdnDomains: manager.cfg.UseCdnDomains,
453+
AccelerateUploading: manager.cfg.AccelerateUploading,
454+
CentralRsHost: "",
452455
}, manager.client)
453456
}
454457

455458
func (manager *UploadManager) getResumeV2Uploader(region *Region) *ResumeUploaderV2 {
456459
return NewResumeUploaderV2Ex(&Config{
457-
Zone: region,
458-
Region: region,
459-
UseHTTPS: manager.cfg.UseHTTPS,
460-
UseCdnDomains: manager.cfg.UseCdnDomains,
461-
CentralRsHost: "",
460+
Zone: region,
461+
Region: region,
462+
UseHTTPS: manager.cfg.UseHTTPS,
463+
UseCdnDomains: manager.cfg.UseCdnDomains,
464+
AccelerateUploading: manager.cfg.AccelerateUploading,
465+
CentralRsHost: "",
462466
}, manager.client)
463467
}
464468

@@ -468,9 +472,10 @@ func (manager *UploadManager) getRegionGroupWithUploadToken(upToken string, extr
468472
return nil, err
469473
}
470474
return getRegionGroupWithOptions(ak, bucket, UCApiOptions{
471-
UseHttps: manager.cfg.UseHTTPS,
472-
RetryMax: extra.TryTimes,
473-
HostFreezeDuration: extra.HostFreezeDuration,
475+
UseHttps: manager.cfg.UseHTTPS,
476+
AccelerateUploading: manager.cfg.AccelerateUploading,
477+
RetryMax: extra.TryTimes,
478+
HostFreezeDuration: extra.HostFreezeDuration,
474479
})
475480
}
476481

‎storage/uploader_base.go

+7-5
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,9 @@ func getUpHost(config *Config, retryMax int, hostFreezeDuration time.Duration, a
1313
region := config.GetRegion()
1414
if region == nil {
1515
if region, err = GetRegionWithOptions(ak, bucket, UCApiOptions{
16-
RetryMax: retryMax,
17-
HostFreezeDuration: hostFreezeDuration,
16+
RetryMax: retryMax,
17+
HostFreezeDuration: hostFreezeDuration,
18+
AccelerateUploading: config.AccelerateUploading,
1819
}); err != nil {
1920
return "", err
2021
}
@@ -24,7 +25,7 @@ func getUpHost(config *Config, retryMax int, hostFreezeDuration time.Duration, a
2425
return "", errors.New("can't get region with bucket:" + bucket)
2526
}
2627

27-
if config.UseCdnDomains {
28+
if config.UseCdnDomains || config.AccelerateUploading {
2829
if len(region.CdnUpHosts) == 0 {
2930
return "", errors.New("can't get region cdn host with bucket:" + bucket)
3031
}
@@ -46,8 +47,9 @@ func getUpHostProvider(config *Config, retryMax int, hostFreezeDuration time.Dur
4647
var err error
4748
if region == nil {
4849
if region, err = GetRegionWithOptions(ak, bucket, UCApiOptions{
49-
RetryMax: retryMax,
50-
HostFreezeDuration: hostFreezeDuration,
50+
RetryMax: retryMax,
51+
HostFreezeDuration: hostFreezeDuration,
52+
AccelerateUploading: config.AccelerateUploading,
5153
}); err != nil {
5254
return nil, err
5355
}

‎storagev2/apis/query_bucket_v2/api.go

+56-9
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,47 @@ type Response struct {
2626
ApiDomains ApiDomains // API 域名
2727
}
2828

29+
// 空间级别的主加速上传域名列表
30+
type MainBucketAcceleratedUpDomains = []string
31+
32+
// 空间级别的备用加速上传域名列表
33+
type BackupBucketAcceleratedUpDomains = []string
34+
35+
// 空间级别的加速上传域名
36+
type BucketAcceleratedUpDomains struct {
37+
MainAcceleratedUpDomains MainBucketAcceleratedUpDomains // 空间级别的主加速上传域名列表
38+
BackupAcceleratedUpDomains BackupBucketAcceleratedUpDomains // 空间级别的备用加速上传域名列表
39+
}
40+
type jsonBucketAcceleratedUpDomains struct {
41+
MainAcceleratedUpDomains MainBucketAcceleratedUpDomains `json:"main"` // 空间级别的主加速上传域名列表
42+
BackupAcceleratedUpDomains BackupBucketAcceleratedUpDomains `json:"backup"` // 空间级别的备用加速上传域名列表
43+
}
44+
45+
func (j *BucketAcceleratedUpDomains) MarshalJSON() ([]byte, error) {
46+
if err := j.validate(); err != nil {
47+
return nil, err
48+
}
49+
return json.Marshal(&jsonBucketAcceleratedUpDomains{MainAcceleratedUpDomains: j.MainAcceleratedUpDomains, BackupAcceleratedUpDomains: j.BackupAcceleratedUpDomains})
50+
}
51+
func (j *BucketAcceleratedUpDomains) UnmarshalJSON(data []byte) error {
52+
var nj jsonBucketAcceleratedUpDomains
53+
if err := json.Unmarshal(data, &nj); err != nil {
54+
return err
55+
}
56+
j.MainAcceleratedUpDomains = nj.MainAcceleratedUpDomains
57+
j.BackupAcceleratedUpDomains = nj.BackupAcceleratedUpDomains
58+
return nil
59+
}
60+
func (j *BucketAcceleratedUpDomains) validate() error {
61+
if len(j.MainAcceleratedUpDomains) == 0 {
62+
return errors.MissingRequiredFieldError{Name: "MainAcceleratedUpDomains"}
63+
}
64+
if len(j.BackupAcceleratedUpDomains) == 0 {
65+
return errors.MissingRequiredFieldError{Name: "BackupAcceleratedUpDomains"}
66+
}
67+
return nil
68+
}
69+
2970
// 主加速上传域名列表
3071
type MainAcceleratedUpDomains = []string
3172

@@ -192,36 +233,42 @@ func (j *OldSourceDomains) validate() error {
192233

193234
// 上传域名
194235
type UpDomains struct {
195-
AcceleratedUpDomains AcceleratedUpDomains // 加速上传域名
196-
SourceUpDomains SourceUpDomains // 源站上传域名
197-
OldAcceleratedDomains OldAcceleratedUpDomains // 已经过时的加速上传域名
198-
OldSourceDomains OldSourceUpDomains // 已经过时的源站上传域名
236+
BucketAcceleratedUpDomains BucketAcceleratedUpDomains // 空间级别的加速上传域名
237+
AcceleratedUpDomains AcceleratedUpDomains // 加速上传域名
238+
SourceUpDomains SourceUpDomains // 源站上传域名
239+
OldAcceleratedDomains OldAcceleratedUpDomains // 已经过时的加速上传域名
240+
OldSourceDomains OldSourceUpDomains // 已经过时的源站上传域名
199241
}
200242
type jsonUpDomains struct {
201-
AcceleratedUpDomains AcceleratedUpDomains `json:"acc"` // 加速上传域名
202-
SourceUpDomains SourceUpDomains `json:"src"` // 源站上传域名
203-
OldAcceleratedDomains OldAcceleratedUpDomains `json:"old_acc"` // 已经过时的加速上传域名
204-
OldSourceDomains OldSourceUpDomains `json:"old_src"` // 已经过时的源站上传域名
243+
BucketAcceleratedUpDomains BucketAcceleratedUpDomains `json:"bucket_acc"` // 空间级别的加速上传域名
244+
AcceleratedUpDomains AcceleratedUpDomains `json:"acc"` // 加速上传域名
245+
SourceUpDomains SourceUpDomains `json:"src"` // 源站上传域名
246+
OldAcceleratedDomains OldAcceleratedUpDomains `json:"old_acc"` // 已经过时的加速上传域名
247+
OldSourceDomains OldSourceUpDomains `json:"old_src"` // 已经过时的源站上传域名
205248
}
206249

207250
func (j *UpDomains) MarshalJSON() ([]byte, error) {
208251
if err := j.validate(); err != nil {
209252
return nil, err
210253
}
211-
return json.Marshal(&jsonUpDomains{AcceleratedUpDomains: j.AcceleratedUpDomains, SourceUpDomains: j.SourceUpDomains, OldAcceleratedDomains: j.OldAcceleratedDomains, OldSourceDomains: j.OldSourceDomains})
254+
return json.Marshal(&jsonUpDomains{BucketAcceleratedUpDomains: j.BucketAcceleratedUpDomains, AcceleratedUpDomains: j.AcceleratedUpDomains, SourceUpDomains: j.SourceUpDomains, OldAcceleratedDomains: j.OldAcceleratedDomains, OldSourceDomains: j.OldSourceDomains})
212255
}
213256
func (j *UpDomains) UnmarshalJSON(data []byte) error {
214257
var nj jsonUpDomains
215258
if err := json.Unmarshal(data, &nj); err != nil {
216259
return err
217260
}
261+
j.BucketAcceleratedUpDomains = nj.BucketAcceleratedUpDomains
218262
j.AcceleratedUpDomains = nj.AcceleratedUpDomains
219263
j.SourceUpDomains = nj.SourceUpDomains
220264
j.OldAcceleratedDomains = nj.OldAcceleratedDomains
221265
j.OldSourceDomains = nj.OldSourceDomains
222266
return nil
223267
}
224268
func (j *UpDomains) validate() error {
269+
if err := j.BucketAcceleratedUpDomains.validate(); err != nil {
270+
return err
271+
}
225272
if err := j.AcceleratedUpDomains.validate(); err != nil {
226273
return err
227274
}

0 commit comments

Comments
 (0)