Skip to content

Commit 80e9973

Browse files
bachuebachue
committed
add chooser
1 parent 343f0e8 commit 80e9973

28 files changed

+843
-62
lines changed

‎.github/workflows/ci-test.yml

+1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ jobs:
2525
GOPATH=$GITHUB_WORKSPACE go get golang.org/x/sync/singleflight
2626
GOPATH=$GITHUB_WORKSPACE go get github.com/qiniu/dyn
2727
GOPATH=$GITHUB_WORKSPACE go get github.com/gofrs/flock
28+
GOPATH=$GITHUB_WORKSPACE go get github.com/alex-ant/gomath/rational
2829
2930
# FIXME special package
3031
# github.com/go-playground/validator/v10

‎client/dialer.go

-5
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package client
22

33
import (
44
"context"
5-
"fmt"
65
"net"
76
"time"
87
)
@@ -38,14 +37,10 @@ func defaultDialFunc(ctx context.Context, network string, address string) (net.C
3837
if port != "" {
3938
newAddr = net.JoinHostPort(newAddr, port)
4039
}
41-
fmt.Printf("***** defaultDialFunc 1: network: %s, newAddr: %s\n", network, newAddr)
4240
if conn, err := dialer.DialContext(ctx, network, newAddr); err == nil {
4341
return conn, nil
4442
}
4543
}
46-
fmt.Printf("***** defaultDialFunc 2: network: %s, host: %s\n", network, host)
47-
} else {
48-
fmt.Printf("***** defaultDialFunc 3: network: %s, host: %s\n", network, host)
4944
}
5045
return (&net.Dialer{Timeout: dialTimeout, KeepAlive: keepAliveInterval}).DialContext(ctx, network, address)
5146
}

‎go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ module github.com/qiniu/go-sdk/v7
33
go 1.14
44

55
require (
6+
github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82
67
github.com/dave/jennifer v1.6.1
78
github.com/davecgh/go-spew v1.1.1 // indirect
89
github.com/go-playground/universal-translator v0.18.0 // indirect

‎go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 h1:7dONQ3WNZ1zy960TmkxJPuwoolZwL7xKtpcM04MBnt4=
2+
github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82/go.mod h1:nLnM0KdK1CmygvjpDUO6m1TjSsiQtL61juhNsvV/JVI=
13
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
24
github.com/dave/jennifer v1.6.1 h1:T4T/67t6RAA5AIV6+NP8Uk/BIsXgDoqEowgycdQQLuk=
35
github.com/dave/jennifer v1.6.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc=

‎internal/clientv2/interceptor_retry_hosts.go

-9
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,11 @@ import (
66
"strings"
77
"time"
88

9-
clientV1 "github.com/qiniu/go-sdk/v7/client"
109
"github.com/qiniu/go-sdk/v7/internal/hostprovider"
1110
internal_io "github.com/qiniu/go-sdk/v7/internal/io"
12-
"github.com/qiniu/go-sdk/v7/storagev2/resolver"
1311
)
1412

1513
type HostsRetryConfig struct {
16-
Resolver resolver.Resolver // 主备域名解析器
1714
RetryConfig RetryConfig // 主备域名重试参数
1815
HostFreezeDuration time.Duration // 主备域名冻结时间（默认：600s），当一个域名请求失败被冻结的时间，最小 time.Millisecond
1916
HostProvider hostprovider.HostProvider // 备用域名获取方法
@@ -73,12 +70,6 @@ func (interceptor *hostsRetryInterceptor) Intercept(req *http.Request, handler H
7370
// Clone 防止后面 Handler 处理对 req 有污染
7471
reqBefore := cloneReq(req)
7572

76-
if resolver := interceptor.options.Resolver; resolver != nil {
77-
if ips, err := resolver.Resolve(req.Context(), req.URL.Hostname()); err == nil && len(ips) > 0 {
78-
req = req.WithContext(clientV1.WithResolvedIPs(req.Context(), req.URL.Hostname(), ips))
79-
}
80-
}
81-
8273
resp, err = handler(req)
8374

8475
if !interceptor.options.RetryConfig.ShouldRetry(reqBefore, resp, err) {

‎internal/clientv2/interceptor_retry_hosts_test.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ func TestHostsAlwaysRetryInterceptor(t *testing.T) {
3737
})
3838

3939
retryMax := 1
40-
sRetryInterceptor := NewSimpleRetryInterceptor(RetryConfig{
40+
sRetryInterceptor := NewSimpleRetryInterceptor(SimpleRetryConfig{
4141
RetryMax: retryMax,
4242
RetryInterval: func() time.Duration {
4343
return time.Second
@@ -119,7 +119,7 @@ func TestHostsNotRetryInterceptor(t *testing.T) {
119119
})
120120

121121
retryMax := 1
122-
sRetryInterceptor := NewSimpleRetryInterceptor(RetryConfig{
122+
sRetryInterceptor := NewSimpleRetryInterceptor(SimpleRetryConfig{
123123
RetryMax: retryMax,
124124
RetryInterval: func() time.Duration {
125125
return time.Second
@@ -198,7 +198,7 @@ func TestHostsRetryInterceptorByRequest(t *testing.T) {
198198
})
199199

200200
retryMax := 1
201-
sRetryInterceptor := NewSimpleRetryInterceptor(RetryConfig{
201+
sRetryInterceptor := NewSimpleRetryInterceptor(SimpleRetryConfig{
202202
RetryMax: retryMax,
203203
RetryInterval: func() time.Duration {
204204
return time.Second

‎internal/clientv2/interceptor_retry_simple.go

+59-9
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ import (
1313

1414
clientv1 "github.com/qiniu/go-sdk/v7/client"
1515
internal_io "github.com/qiniu/go-sdk/v7/internal/io"
16+
"github.com/qiniu/go-sdk/v7/storagev2/chooser"
17+
"github.com/qiniu/go-sdk/v7/storagev2/resolver"
1618
)
1719

1820
type contextKeyBufferResponse struct{}
@@ -43,16 +45,44 @@ func (c *RetryConfig) init() {
4345
}
4446
}
4547

46-
type simpleRetryInterceptor struct {
47-
config RetryConfig
48-
}
48+
type (
49+
SimpleRetryConfig struct {
50+
RetryMax int // 最大重试次数
51+
RetryInterval func() time.Duration // 重试时间间隔
52+
ShouldRetry func(req *http.Request, resp *http.Response, err error) bool
53+
Resolver resolver.Resolver // 主备域名解析器
54+
Chooser chooser.Chooser // 主备域名选择器
55+
}
56+
57+
simpleRetryInterceptor struct {
58+
config SimpleRetryConfig
59+
}
60+
)
61+
62+
func (c *SimpleRetryConfig) init() {
63+
if c == nil {
64+
return
65+
}
66+
67+
if c.RetryMax < 0 {
68+
c.RetryMax = 0
69+
}
70+
71+
if c.RetryInterval == nil {
72+
c.RetryInterval = func() time.Duration {
73+
return time.Duration(50+rand.Int()%50) * time.Millisecond
74+
}
75+
}
4976

50-
func NewSimpleRetryInterceptor(config RetryConfig) Interceptor {
51-
return &simpleRetryInterceptor{
52-
config: config,
77+
if c.ShouldRetry == nil {
78+
c.ShouldRetry = isSimpleRetryable
5379
}
5480
}
5581

82+
func NewSimpleRetryInterceptor(config SimpleRetryConfig) Interceptor {
83+
return &simpleRetryInterceptor{config: config}
84+
}
85+
5686
func (interceptor *simpleRetryInterceptor) Priority() InterceptorPriority {
5787
return InterceptorPriorityRetrySimple
5888
}
@@ -65,9 +95,18 @@ func (interceptor *simpleRetryInterceptor) Intercept(req *http.Request, handler
6595

6696
interceptor.config.init()
6797

68-
// 不重试
69-
if interceptor.config.RetryMax <= 0 {
70-
return handler(req)
98+
var ips []net.IP
99+
hostname := req.URL.Hostname()
100+
101+
if resolver := interceptor.config.Resolver; resolver != nil {
102+
if ips, err = resolver.Resolve(req.Context(), hostname); err == nil && len(ips) > 0 {
103+
if cs := interceptor.config.Chooser; cs != nil {
104+
ips = cs.Choose(req.Context(), &chooser.ChooseOptions{IPs: ips, Domain: hostname})
105+
}
106+
if len(ips) > 0 {
107+
req = req.WithContext(clientv1.WithResolvedIPs(req.Context(), hostname, ips))
108+
}
109+
}
71110
}
72111

73112
// 可能会被重试多次
@@ -83,8 +122,19 @@ func (interceptor *simpleRetryInterceptor) Intercept(req *http.Request, handler
83122
}
84123

85124
if !interceptor.config.ShouldRetry(reqBefore, resp, err) {
125+
if len(ips) > 0 {
126+
if cs := interceptor.config.Chooser; cs != nil {
127+
cs.FeedbackGood(req.Context(), &chooser.FeedbackOptions{IPs: ips, Domain: hostname})
128+
}
129+
}
86130
return resp, err
87131
}
132+
if len(ips) > 0 {
133+
if cs := interceptor.config.Chooser; cs != nil {
134+
cs.FeedbackBad(req.Context(), &chooser.FeedbackOptions{IPs: ips, Domain: hostname})
135+
}
136+
}
137+
88138
req = reqBefore
89139

90140
if i >= interceptor.config.RetryMax {

‎internal/clientv2/interceptor_retry_simple_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import (
1212
func TestSimpleAlwaysRetryInterceptor(t *testing.T) {
1313

1414
retryMax := 1
15-
rInterceptor := NewSimpleRetryInterceptor(RetryConfig{
15+
rInterceptor := NewSimpleRetryInterceptor(SimpleRetryConfig{
1616
RetryMax: retryMax,
1717
RetryInterval: func() time.Duration {
1818
return time.Second
@@ -67,7 +67,7 @@ func TestSimpleAlwaysRetryInterceptor(t *testing.T) {
6767
func TestSimpleNotRetryInterceptor(t *testing.T) {
6868

6969
retryMax := 1
70-
rInterceptor := NewSimpleRetryInterceptor(RetryConfig{
70+
rInterceptor := NewSimpleRetryInterceptor(SimpleRetryConfig{
7171
RetryMax: retryMax,
7272
RetryInterval: func() time.Duration {
7373
return time.Second

‎storage/bucket.go

+15-2
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,11 @@ import (
1414
"strings"
1515
"time"
1616

17+
"github.com/alex-ant/gomath/rational"
1718
"github.com/qiniu/go-sdk/v7/internal/clientv2"
1819
"github.com/qiniu/go-sdk/v7/storagev2/apis"
1920
"github.com/qiniu/go-sdk/v7/storagev2/apis/batch_ops"
21+
"github.com/qiniu/go-sdk/v7/storagev2/chooser"
2022
"github.com/qiniu/go-sdk/v7/storagev2/http_client"
2123
"github.com/qiniu/go-sdk/v7/storagev2/resolver"
2224

@@ -285,10 +287,14 @@ type BatchOpRet struct {
285287
}
286288

287289
type BucketManagerOptions struct {
288-
RetryMax int // 单域名重试次数，当前只有 uc 相关的服务有多域名
290+
// 单域名重试次数，当前只有 uc 相关的服务有多域名
291+
RetryMax int
289292
// 主备域名冻结时间（默认：600s），当一个域名请求失败（单个域名会被重试 TryTimes 次），会被冻结一段时间，使用备用域名进行重试，在冻结时间内，域名不能被使用，当一个操作中所有域名竣备冻结操作不在进行重试，返回最后一次操作的错误。
290293
HostFreezeDuration time.Duration
291-
Resolver resolver.Resolver
294+
// 域名解析器
295+
Resolver resolver.Resolver
296+
// 域名选择器
297+
Chooser chooser.Chooser
292298
}
293299

294300
// BucketManager 提供了对资源进行管理的操作
@@ -1056,6 +1062,13 @@ func (m *BucketManager) resolver() (resolver.Resolver, error) {
10561062
}
10571063
}
10581064

1065+
func (m *BucketManager) chooser() chooser.Chooser {
1066+
if m.options.Chooser != nil {
1067+
return m.options.Chooser
1068+
}
1069+
return chooser.NewNeverEmptyHandedChooser(chooser.NewShuffleChooser(chooser.NewSmartIPChooser(nil)), rational.New(1, 2))
1070+
}
1071+
10591072
// 构建op的方法，导出的方法支持在Batch操作中使用
10601073

10611074
// URIStat 构建 stat 接口的请求命令

‎storage/bucket_get.go

+1
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ func (m *BucketManager) Get(bucket, key string, options *GetObjectInput) (*GetOb
7979
RetryMax: m.options.RetryMax,
8080
HostFreezeDuration: m.options.HostFreezeDuration,
8181
Resolver: resolver,
82+
Chooser: m.chooser(),
8283
}); e != nil {
8384
return nil, e
8485
} else if len(rg.regions) == 0 {

‎storage/region.go

+6-2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"github.com/qiniu/go-sdk/v7/internal/clientv2"
1212
"github.com/qiniu/go-sdk/v7/internal/hostprovider"
1313
"github.com/qiniu/go-sdk/v7/storagev2/apis"
14+
"github.com/qiniu/go-sdk/v7/storagev2/chooser"
1415
"github.com/qiniu/go-sdk/v7/storagev2/http_client"
1516
region_v2 "github.com/qiniu/go-sdk/v7/storagev2/region"
1617
"github.com/qiniu/go-sdk/v7/storagev2/resolver"
@@ -383,6 +384,8 @@ type ucClientConfig struct {
383384

384385
Resolver resolver.Resolver
385386

387+
Chooser chooser.Chooser
388+
386389
Client *client.Client
387390
}
388391

@@ -410,12 +413,13 @@ func getUCClient(config ucClientConfig, mac *auth.Credentials) clientv2.Client {
410413
ShouldFreezeHost: nil,
411414
HostFreezeDuration: config.HostFreezeDuration,
412415
HostProvider: hostprovider.NewWithHosts(hosts),
413-
Resolver: config.Resolver,
414416
}),
415-
clientv2.NewSimpleRetryInterceptor(clientv2.RetryConfig{
417+
clientv2.NewSimpleRetryInterceptor(clientv2.SimpleRetryConfig{
416418
RetryMax: config.RetryMax,
417419
RetryInterval: nil,
418420
ShouldRetry: nil,
421+
Resolver: config.Resolver,
422+
Chooser: config.Chooser,
419423
}),
420424
}
421425

‎storage/region_uc_v2.go

+7
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@ import (
1212

1313
"golang.org/x/sync/singleflight"
1414

15+
"github.com/alex-ant/gomath/rational"
1516
"github.com/qiniu/go-sdk/v7/internal/clientv2"
17+
"github.com/qiniu/go-sdk/v7/storagev2/chooser"
1618
"github.com/qiniu/go-sdk/v7/storagev2/resolver"
1719
)
1820

@@ -214,6 +216,7 @@ type UCApiOptions struct {
214216
// 主备域名冻结时间（默认：600s），当一个域名请求失败（单个域名会被重试 TryTimes 次），会被冻结一段时间，使用备用域名进行重试，在冻结时间内，域名不能被使用，当一个操作中所有域名竣备冻结操作不在进行重试，返回最后一次操作的错误。
215217
HostFreezeDuration time.Duration
216218
Resolver resolver.Resolver
219+
Chooser chooser.Chooser
217220
}
218221

219222
func DefaultUCApiOptions() UCApiOptions {
@@ -258,13 +261,17 @@ func getRegionByV2(ak, bucket string, options UCApiOptions) (*Region, error) {
258261
return nil, err
259262
}
260263
}
264+
if options.Chooser == nil {
265+
options.Chooser = chooser.NewNeverEmptyHandedChooser(chooser.NewShuffleChooser(chooser.NewSmartIPChooser(nil)), rational.New(1, 2))
266+
}
261267

262268
var ret UcQueryRet
263269
c := getUCClient(ucClientConfig{
264270
IsUcQueryApi: true,
265271
RetryMax: options.RetryMax,
266272
HostFreezeDuration: options.HostFreezeDuration,
267273
Resolver: options.Resolver,
274+
Chooser: options.Chooser,
268275
}, nil)
269276
err = clientv2.DoAndDecodeJsonResponse(c, clientv2.RequestParams{
270277
Context: context.Background(),

‎storage/region_uc_v4.go

+6
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@ import (
1212

1313
"golang.org/x/sync/singleflight"
1414

15+
"github.com/alex-ant/gomath/rational"
1516
"github.com/qiniu/go-sdk/v7/internal/clientv2"
17+
"github.com/qiniu/go-sdk/v7/storagev2/chooser"
1618
"github.com/qiniu/go-sdk/v7/storagev2/resolver"
1719
)
1820

@@ -157,13 +159,17 @@ func getRegionByV4(ak, bucket string, options UCApiOptions) (*RegionGroup, error
157159
return nil, err
158160
}
159161
}
162+
if options.Chooser == nil {
163+
options.Chooser = chooser.NewNeverEmptyHandedChooser(chooser.NewShuffleChooser(chooser.NewSmartIPChooser(nil)), rational.New(1, 2))
164+
}
160165

161166
var ret ucQueryV4Ret
162167
c := getUCClient(ucClientConfig{
163168
IsUcQueryApi: true,
164169
RetryMax: options.RetryMax,
165170
HostFreezeDuration: options.HostFreezeDuration,
166171
Resolver: options.Resolver,
172+
Chooser: options.Chooser,
167173
}, nil)
168174
err = clientv2.DoAndDecodeJsonResponse(c, clientv2.RequestParams{
169175
Context: context.Background(),

‎storage/uc.go

+1
Original file line numberDiff line numberDiff line change
@@ -800,5 +800,6 @@ func (m *BucketManager) getUCClient() (clientv2.Client, error) {
800800
HostFreezeDuration: m.options.HostFreezeDuration,
801801
Client: m.Client,
802802
Resolver: resolver,
803+
Chooser: m.chooser(),
803804
}, m.Mac), nil
804805
}

0 commit comments

Comments
 (0)