Skip to content

Commit 34483b6

Browse files
bachuebachue
committed
little adjust for storagev2 interfaces
1 parent 728e24d commit 34483b6

26 files changed

+262
-256
lines changed

‎.github/workflows/ci-test.yml

+1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ jobs:
2626
GOPATH=$GITHUB_WORKSPACE go get github.com/qiniu/dyn
2727
GOPATH=$GITHUB_WORKSPACE go get github.com/gofrs/flock
2828
GOPATH=$GITHUB_WORKSPACE go get github.com/alex-ant/gomath/rational
29+
GOPATH=$GITHUB_WORKSPACE go get github.com/matishsiao/goInfo
2930
3031
# FIXME special package
3132
# github.com/go-playground/validator/v10

‎Makefile

+3
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ test:
44
unittest:
55
go test -tags=unit -failfast -v -coverprofile=coverage.txt `go list ./... | egrep -v 'examples|sms'`
66

7+
integrationtest:
8+
go test -tags=integration -failfast -parallel 1 -v -coverprofile=coverage.txt `go list ./... | egrep -v 'examples|sms'`
9+
710
staticcheck:
811
staticcheck -go 1.10 `go list ./... | egrep -v 'examples|sms'`
912

‎internal/clientv2/interceptor_retry_hosts.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,9 @@ func (c *HostsRetryConfig) getRetryDecision(req *http.Request, resp *http.Respon
4444
return retrier.DontRetry
4545
}
4646
} else if c.Retrier != nil {
47-
return c.Retrier.Retry(req, resp, err, &retrier.RetrierOptions{Attempts: attempts})
47+
return c.Retrier.Retry(resp, err, &retrier.RetrierOptions{Attempts: attempts})
4848
} else {
49-
return errorRetrier.Retry(req, resp, err, &retrier.RetrierOptions{Attempts: attempts})
49+
return errorRetrier.Retry(resp, err, &retrier.RetrierOptions{Attempts: attempts})
5050
}
5151
}
5252

‎internal/clientv2/interceptor_retry_simple.go

+12-12
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,13 @@ type (
2727
Chooser chooser.Chooser // IP 选择器
2828
Retrier retrier.Retrier // 重试器
2929

30-
BeforeResolve func(*http.Request) // 域名解析前回调函数
31-
AfterResolve func(*http.Request, []net.IP) // 域名解析后回调函数
32-
ResolveError func(*http.Request, error) // 域名解析错误回调函数
33-
BeforeBackoff func(*http.Request, *retrier.RetrierOptions, time.Duration) // 退避前回调函数
34-
AfterBackoff func(*http.Request, *retrier.RetrierOptions, time.Duration) // 退避后回调函数
35-
BeforeRequest func(*http.Request, *retrier.RetrierOptions) // 请求前回调函数
36-
AfterResponse func(*http.Request, *http.Response, *retrier.RetrierOptions, error) // 请求后回调函数
30+
BeforeResolve func(*http.Request) // 域名解析前回调函数
31+
AfterResolve func(*http.Request, []net.IP) // 域名解析后回调函数
32+
ResolveError func(*http.Request, error) // 域名解析错误回调函数
33+
BeforeBackoff func(*http.Request, *retrier.RetrierOptions, time.Duration) // 退避前回调函数
34+
AfterBackoff func(*http.Request, *retrier.RetrierOptions, time.Duration) // 退避后回调函数
35+
BeforeRequest func(*http.Request, *retrier.RetrierOptions) // 请求前回调函数
36+
AfterResponse func(*http.Response, *retrier.RetrierOptions, error) // 请求后回调函数
3737
}
3838

3939
simpleRetryInterceptor struct {
@@ -75,7 +75,7 @@ func (c *SimpleRetryConfig) getRetryDecision(req *http.Request, resp *http.Respo
7575
if c.Retrier != nil {
7676
r = c.Retrier
7777
}
78-
return r.Retry(req, resp, err, &retrier.RetrierOptions{Attempts: attempts})
78+
return r.Retry(resp, err, &retrier.RetrierOptions{Attempts: attempts})
7979
}
8080
}
8181

@@ -135,7 +135,7 @@ func (interceptor *simpleRetryInterceptor) callHandler(req *http.Request, option
135135
}
136136
resp, err = handler(req)
137137
if interceptor.config.AfterResponse != nil {
138-
interceptor.config.AfterResponse(req, resp, options, err)
138+
interceptor.config.AfterResponse(resp, options, err)
139139
}
140140
return
141141
}
@@ -163,7 +163,7 @@ func (interceptor *simpleRetryInterceptor) resolve(req *http.Request, hostname s
163163
func (interceptor *simpleRetryInterceptor) choose(req *http.Request, ips []net.IP, hostname string) (*http.Request, []net.IP) {
164164
if len(ips) > 0 {
165165
if cs := interceptor.config.Chooser; cs != nil {
166-
ips = cs.Choose(req.Context(), &chooser.ChooseOptions{IPs: ips, Domain: hostname})
166+
ips = cs.Choose(req.Context(), ips, &chooser.ChooseOptions{Domain: hostname})
167167
}
168168
req = req.WithContext(clientv1.WithResolvedIPs(req.Context(), hostname, ips))
169169
}
@@ -173,15 +173,15 @@ func (interceptor *simpleRetryInterceptor) choose(req *http.Request, ips []net.I
173173
func (interceptor *simpleRetryInterceptor) feedbackGood(req *http.Request, hostname string, ips []net.IP) {
174174
if len(ips) > 0 {
175175
if cs := interceptor.config.Chooser; cs != nil {
176-
cs.FeedbackGood(req.Context(), &chooser.FeedbackOptions{IPs: ips, Domain: hostname})
176+
cs.FeedbackGood(req.Context(), ips, &chooser.FeedbackOptions{Domain: hostname})
177177
}
178178
}
179179
}
180180

181181
func (interceptor *simpleRetryInterceptor) feedbackBad(req *http.Request, hostname string, ips []net.IP) {
182182
if len(ips) > 0 {
183183
if cs := interceptor.config.Chooser; cs != nil {
184-
cs.FeedbackBad(req.Context(), &chooser.FeedbackOptions{IPs: ips, Domain: hostname})
184+
cs.FeedbackBad(req.Context(), ips, &chooser.FeedbackOptions{Domain: hostname})
185185
}
186186
}
187187
}

‎internal/clientv2/interceptor_retry_simple_test.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ func TestSimpleAlwaysRetryInterceptor(t *testing.T) {
6767
t.Fatal("unexpected attempts", options.Attempts)
6868
}
6969
},
70-
AfterResponse: func(req *http.Request, resp *http.Response, options *retrier.RetrierOptions, err error) {
70+
AfterResponse: func(resp *http.Response, options *retrier.RetrierOptions, err error) {
7171
callbackedCount += 1
7272
if options.Attempts != (doCount - 1) {
7373
t.Fatal("unexpected attempts", options.Attempts)
@@ -174,7 +174,7 @@ func TestSimpleNotRetryInterceptor(t *testing.T) {
174174
t.Fatal("unexpected attempts", options.Attempts)
175175
}
176176
},
177-
AfterResponse: func(req *http.Request, resp *http.Response, options *retrier.RetrierOptions, err error) {
177+
AfterResponse: func(resp *http.Response, options *retrier.RetrierOptions, err error) {
178178
callbackedCount += 1
179179
if options.Attempts != (doCount - 1) {
180180
t.Fatal("unexpected attempts", options.Attempts)
@@ -278,7 +278,7 @@ func TestRetryInterceptorWithBackoff(t *testing.T) {
278278
t.Fatal("unexpected attempts", options.Attempts)
279279
}
280280
},
281-
AfterResponse: func(req *http.Request, resp *http.Response, options *retrier.RetrierOptions, err error) {
281+
AfterResponse: func(resp *http.Response, options *retrier.RetrierOptions, err error) {
282282
callbackedCount += 1
283283
if options.Attempts != (doCount - 1) {
284284
t.Fatal("unexpected attempts", options.Attempts)

‎storage/bucket.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -338,7 +338,7 @@ type BucketManagerOptions struct {
338338
BeforeRequest func(*http.Request, *retrier.RetrierOptions)
339339

340340
// 请求后回调函数
341-
AfterResponse func(*http.Request, *http.Response, *retrier.RetrierOptions, error)
341+
AfterResponse func(*http.Response, *retrier.RetrierOptions, error)
342342
}
343343

344344
// BucketManager 提供了对资源进行管理的操作

‎storage/form_upload.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,9 @@ type PutRet struct {
5959

6060
// FormUploader 表示一个表单上传的对象
6161
type FormUploader struct {
62-
Client *client.Client
62+
// Deprecated
63+
Client *client.Client
64+
// Deprecated
6365
Cfg *Config
6466
storage *apis.Storage
6567
}
@@ -222,6 +224,7 @@ func (p *FormUploader) putSeekableData(ctx context.Context, ret interface{}, upT
222224
return err
223225
}
224226

227+
// Deprecated
225228
func (p *FormUploader) UpHost(ak, bucket string) (upHost string, err error) {
226229
return getUpHost(p.Cfg, 0, 0, ak, bucket)
227230
}

‎storage/region_uc_v2.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ type UCApiOptions struct {
224224
BeforeRequest func(*http.Request, *retrier.RetrierOptions)
225225

226226
// 请求后回调函数
227-
AfterResponse func(*http.Request, *http.Response, *retrier.RetrierOptions, error)
227+
AfterResponse func(*http.Response, *retrier.RetrierOptions, error)
228228
}
229229

230230
func (options *UCApiOptions) getApiStorageClient() *apis.Storage {

‎storage/upload_manager_test.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,11 @@ import (
1515
)
1616

1717
func getUploadManager() *UploadManager {
18-
region01 := &Region{
19-
SrcUpHosts: []string{"mock01.qiniu.com", "mock02.qiniu.com"},
20-
}
18+
return getUploadManagerV2([]string{"mock01.qiniu.com", "mock02.qiniu.com"})
19+
}
2120

21+
func getUploadManagerV2(srcUpHosts []string) *UploadManager {
22+
region01 := &Region{SrcUpHosts: srcUpHosts}
2223
region02, err := GetRegion(testAK, testBucket)
2324
if err != nil {
2425
return nil

‎storage/upload_manager_uplog_test.go

+14-15
Original file line numberDiff line numberDiff line change
@@ -10,20 +10,14 @@ import (
1010
"os"
1111
"path/filepath"
1212
"strings"
13-
"sync"
1413
"testing"
1514
"time"
1615

1716
clientV1 "github.com/qiniu/go-sdk/v7/client"
1817
"github.com/qiniu/go-sdk/v7/storagev2/uplog"
1918
)
2019

21-
var testLock sync.Mutex
22-
2320
func TestUploadManagerUplogForm(t *testing.T) {
24-
testLock.Lock()
25-
defer testLock.Unlock()
26-
2721
tmpDir, err := ioutil.TempDir("", "test-uplog-*")
2822
if err != nil {
2923
t.Fatal(err)
@@ -37,6 +31,10 @@ func TestUploadManagerUplogForm(t *testing.T) {
3731
t.Fatal(err)
3832
}
3933

34+
writeFileBufferInterval := uplog.GetWriteFileBufferInterval()
35+
uplog.SetFlushFileBufferInterval(1 * time.Hour)
36+
defer uplog.SetFlushFileBufferInterval(writeFileBufferInterval)
37+
4038
clientV1.DebugMode = true
4139
defer func() {
4240
clientV1.DebugMode = false
@@ -54,7 +52,7 @@ func TestUploadManagerUplogForm(t *testing.T) {
5452
}()
5553
tempFile.Write(data)
5654

57-
uploadManager := getUploadManager()
55+
uploadManager := getUploadManagerV2([]string{"mock03.qiniu.com", "mock04.qiniu.com"})
5856
var ret Ret
5957

6058
// 上传 file
@@ -104,7 +102,7 @@ func TestUploadManagerUplogForm(t *testing.T) {
104102
if uplogs[0]["error_type"] != "unknown_host" {
105103
t.Fatalf("unexpected uplog error_type:%v", uplogs[0]["error_type"])
106104
}
107-
if uplogs[0]["host"] != "mock01.qiniu.com" {
105+
if uplogs[0]["host"] != "mock03.qiniu.com" {
108106
t.Fatalf("unexpected uplog host:%v", uplogs[0]["host"])
109107
}
110108
if uplogs[0]["path"] != "/" {
@@ -128,7 +126,7 @@ func TestUploadManagerUplogForm(t *testing.T) {
128126
if uplogs[1]["error_type"] != "unknown_host" {
129127
t.Fatalf("unexpected uplog error_type:%v", uplogs[1]["error_type"])
130128
}
131-
if uplogs[1]["host"] != "mock02.qiniu.com" {
129+
if uplogs[1]["host"] != "mock04.qiniu.com" {
132130
t.Fatalf("unexpected uplog host:%v", uplogs[1]["host"])
133131
}
134132
if uplogs[1]["target_bucket"] != testBucket {
@@ -194,9 +192,6 @@ func TestUploadManagerUplogForm(t *testing.T) {
194192
}
195193

196194
func TestUploadManagerUplogResumableV1(t *testing.T) {
197-
testLock.Lock()
198-
defer testLock.Unlock()
199-
200195
tmpDir, err := ioutil.TempDir("", "test-uplog-*")
201196
if err != nil {
202197
t.Fatal(err)
@@ -210,6 +205,10 @@ func TestUploadManagerUplogResumableV1(t *testing.T) {
210205
t.Fatal(err)
211206
}
212207

208+
writeFileBufferInterval := uplog.GetWriteFileBufferInterval()
209+
uplog.SetFlushFileBufferInterval(1 * time.Hour)
210+
defer uplog.SetFlushFileBufferInterval(writeFileBufferInterval)
211+
213212
clientV1.DebugMode = true
214213
defer func() {
215214
clientV1.DebugMode = false
@@ -229,7 +228,7 @@ func TestUploadManagerUplogResumableV1(t *testing.T) {
229228
}()
230229
tempFile.Write(data)
231230

232-
uploadManager := getUploadManager()
231+
uploadManager := getUploadManagerV2([]string{"mock05.qiniu.com", "mock06.qiniu.com"})
233232
var ret Ret
234233

235234
// 上传 file
@@ -283,7 +282,7 @@ func TestUploadManagerUplogResumableV1(t *testing.T) {
283282
if uplogs[i]["error_type"] != "unknown_host" {
284283
t.Fatalf("unexpected uplog error_type:%v", uplogs[i]["error_type"])
285284
}
286-
if uplogs[i]["host"] != "mock01.qiniu.com" {
285+
if uplogs[i]["host"] != "mock05.qiniu.com" {
287286
t.Fatalf("unexpected uplog host:%v", uplogs[i]["host"])
288287
}
289288
if uplogs[i]["path"] != "/mkblk/4194304" {
@@ -309,7 +308,7 @@ func TestUploadManagerUplogResumableV1(t *testing.T) {
309308
if uplogs[i]["error_type"] != "unknown_host" {
310309
t.Fatalf("unexpected uplog error_type:%v", uplogs[i]["error_type"])
311310
}
312-
if uplogs[i]["host"] != "mock02.qiniu.com" {
311+
if uplogs[i]["host"] != "mock06.qiniu.com" {
313312
t.Fatalf("unexpected uplog host:%v", uplogs[i]["host"])
314313
}
315314
if uplogs[i]["path"] != "/mkblk/4194304" {

‎storagev2/chooser/chooser.go

+13-17
Original file line numberDiff line numberDiff line change
@@ -8,30 +8,26 @@ import (
88
type (
99
// ChooseOptions 选择器的选项
1010
ChooseOptions struct {
11-
// IPs 是待选择的 IP 地址列表
12-
IPs []net.IP
13-
// Domain 是待使用的域名
11+
// Domain 是待选择的域名
1412
Domain string
1513
}
1614

1715
// FeedbackOptions 反馈的选项
1816
FeedbackOptions struct {
19-
// IPs 是待反馈的 IP 地址列表
20-
IPs []net.IP
21-
// Domain 是待使用的域名
17+
// Domain 是待反馈的域名
2218
Domain string
2319
}
2420

2521
// Chooser 选择器接口
2622
Chooser interface {
2723
// Choose 从给定的 IP 地址列表中选择一批 IP 地址用于发送请求
28-
Choose(context.Context, *ChooseOptions) []net.IP
24+
Choose(context.Context, []net.IP, *ChooseOptions) []net.IP
2925

3026
// FeedbackGood 反馈一批 IP 地址请求成功
31-
FeedbackGood(context.Context, *FeedbackOptions)
27+
FeedbackGood(context.Context, []net.IP, *FeedbackOptions)
3228

3329
// FeedbackBad 反馈一批 IP 地址请求失败
34-
FeedbackBad(context.Context, *FeedbackOptions)
30+
FeedbackBad(context.Context, []net.IP, *FeedbackOptions)
3531
}
3632

3733
directChooser struct{}
@@ -42,22 +38,22 @@ func NewDirectChooser() Chooser {
4238
return &directChooser{}
4339
}
4440

45-
func (chooser *directChooser) Choose(_ context.Context, options *ChooseOptions) []net.IP {
46-
return options.IPs
41+
func (chooser *directChooser) Choose(_ context.Context, ips []net.IP, _ *ChooseOptions) []net.IP {
42+
return ips
4743
}
4844

49-
func (chooser *directChooser) FeedbackGood(_ context.Context, _ *FeedbackOptions) {
45+
func (chooser *directChooser) FeedbackGood(_ context.Context, _ []net.IP, _ *FeedbackOptions) {
5046
// do nothing
5147
}
5248

53-
func (chooser *directChooser) FeedbackBad(_ context.Context, _ *FeedbackOptions) {
49+
func (chooser *directChooser) FeedbackBad(_ context.Context, _ []net.IP, _ *FeedbackOptions) {
5450
// do nothing
5551
}
5652

57-
func (options *ChooseOptions) makeSet(makeKey func(string, net.IP) string) map[string]struct{} {
58-
m := make(map[string]struct{}, len(options.IPs))
59-
for _, ip := range options.IPs {
60-
m[makeKey(options.Domain, ip)] = struct{}{}
53+
func makeSet(ips []net.IP, domain string, makeKey func(string, net.IP) string) map[string]struct{} {
54+
m := make(map[string]struct{}, len(ips))
55+
for _, ip := range ips {
56+
m[makeKey(domain, ip)] = struct{}{}
6157
}
6258
return m
6359
}

‎storagev2/chooser/chooser_test.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,7 @@ import (
1313

1414
func TestDirectChooser(t *testing.T) {
1515
cs := chooser.NewDirectChooser()
16-
ips := cs.Choose(context.Background(), &chooser.ChooseOptions{
17-
IPs: []net.IP{net.IPv4(1, 2, 3, 4), net.IPv4(5, 6, 7, 8)},
16+
ips := cs.Choose(context.Background(), []net.IP{net.IPv4(1, 2, 3, 4), net.IPv4(5, 6, 7, 8)}, &chooser.ChooseOptions{
1817
Domain: "www.qiniu.com",
1918
})
2019
assertIPs(t, ips, []net.IP{net.IPv4(1, 2, 3, 4), net.IPv4(5, 6, 7, 8)})

0 commit comments

Comments
 (0)