Skip to content

Commit 38ac6b5

Browse files
bachuebachue
committed
add backoff
1 parent 09b9b4e commit 38ac6b5

14 files changed

+329
-92
lines changed

‎internal/clientv2/interceptor_retry_hosts.go

+9-16
Original file line numberDiff line numberDiff line change
@@ -11,24 +11,23 @@ import (
1111
)
1212

1313
type HostsRetryConfig struct {
14-
RetryConfig RetryConfig // 主备域名重试参数
14+
RetryMax int // 最大重试次数
15+
ShouldRetry func(req *http.Request, resp *http.Response, err error) bool
1516
HostFreezeDuration time.Duration // 主备域名冻结时间（默认：600s），当一个域名请求失败被冻结的时间，最小 time.Millisecond
1617
HostProvider hostprovider.HostProvider // 备用域名获取方法
1718
ShouldFreezeHost func(req *http.Request, resp *http.Response, err error) bool
1819
}
1920

2021
func (c *HostsRetryConfig) init() {
21-
if c.RetryConfig.ShouldRetry == nil {
22-
c.RetryConfig.ShouldRetry = func(req *http.Request, resp *http.Response, err error) bool {
22+
if c.ShouldRetry == nil {
23+
c.ShouldRetry = func(req *http.Request, resp *http.Response, err error) bool {
2324
return isHostRetryable(req, resp, err)
2425
}
2526
}
26-
if c.RetryConfig.RetryMax < 0 {
27-
c.RetryConfig.RetryMax = 1
27+
if c.RetryMax < 0 {
28+
c.RetryMax = 1
2829
}
2930

30-
c.RetryConfig.init()
31-
3231
if c.HostFreezeDuration < time.Millisecond {
3332
c.HostFreezeDuration = 600 * time.Second
3433
}
@@ -62,7 +61,7 @@ func (interceptor *hostsRetryInterceptor) Intercept(req *http.Request, handler H
6261
interceptor.options.init()
6362

6463
// 不重试
65-
if interceptor.options.RetryConfig.RetryMax <= 0 {
64+
if interceptor.options.RetryMax <= 0 {
6665
return handler(req)
6766
}
6867

@@ -72,7 +71,7 @@ func (interceptor *hostsRetryInterceptor) Intercept(req *http.Request, handler H
7271

7372
resp, err = handler(req)
7473

75-
if !interceptor.options.RetryConfig.ShouldRetry(reqBefore, resp, err) {
74+
if !interceptor.options.ShouldRetry(reqBefore, resp, err) {
7675
return resp, err
7776
}
7877

@@ -84,7 +83,7 @@ func (interceptor *hostsRetryInterceptor) Intercept(req *http.Request, handler H
8483
}
8584
}
8685

87-
if i >= interceptor.options.RetryConfig.RetryMax {
86+
if i >= interceptor.options.RetryMax {
8887
break
8988
}
9089

@@ -121,12 +120,6 @@ func (interceptor *hostsRetryInterceptor) Intercept(req *http.Request, handler H
121120
internal_io.SinkAll(resp.Body)
122121
resp.Body.Close()
123122
}
124-
125-
retryInterval := interceptor.options.RetryConfig.RetryInterval(&RetryInfo{Retried: i})
126-
if retryInterval < time.Microsecond {
127-
continue
128-
}
129-
time.Sleep(retryInterval)
130123
}
131124
return resp, err
132125
}

‎internal/clientv2/interceptor_retry_hosts_test.go

+8-33
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,9 @@ func TestHostsAlwaysRetryInterceptor(t *testing.T) {
2222
hostB := "bbb.bb.com"
2323
hRetryMax := 2
2424
hRetryInterceptor := NewHostsRetryInterceptor(HostsRetryConfig{
25-
RetryConfig: RetryConfig{
26-
RetryMax: hRetryMax,
27-
RetryInterval: func(_ *RetryInfo) time.Duration {
28-
return time.Second
29-
},
30-
ShouldRetry: func(req *http.Request, resp *http.Response, err error) bool {
31-
return true
32-
},
25+
RetryMax: hRetryMax,
26+
ShouldRetry: func(req *http.Request, resp *http.Response, err error) bool {
27+
return true
3328
},
3429
ShouldFreezeHost: nil,
3530
HostFreezeDuration: 0,
@@ -39,7 +34,7 @@ func TestHostsAlwaysRetryInterceptor(t *testing.T) {
3934
retryMax := 1
4035
sRetryInterceptor := NewSimpleRetryInterceptor(SimpleRetryConfig{
4136
RetryMax: retryMax,
42-
RetryInterval: func(_ *RetryInfo) time.Duration {
37+
RetryInterval: func() time.Duration {
4338
return time.Second
4439
},
4540
ShouldRetry: func(req *http.Request, resp *http.Response, err error) bool {
@@ -65,25 +60,18 @@ func TestHostsAlwaysRetryInterceptor(t *testing.T) {
6560

6661
c := NewClient(&testClient{}, interceptor, hRetryInterceptor, sRetryInterceptor)
6762

68-
start := time.Now()
69-
7063
resp, _ := Do(c, RequestParams{
7164
Context: nil,
7265
Method: RequestMethodGet,
7366
Url: "https://" + hostA + "/path/123",
7467
Header: nil,
7568
GetBody: nil,
7669
})
77-
duration := float32(time.Now().UnixNano()-start.UnixNano()) / 1e9
7870

7971
if (retryMax+1)*2 != doCount {
8072
t.Fatalf("retry count is not error:%d", doCount)
8173
}
8274

83-
if duration > float32(doCount-1)+0.3 || duration < float32(doCount-1)-0.3 {
84-
t.Fatalf("retry interval may be error:%f", duration)
85-
}
86-
8775
value := resp.Header.Get(headerKey)
8876
if value != " -> request -> Do -> response" {
8977
t.Fatalf("retry flow error")
@@ -104,15 +92,7 @@ func TestHostsNotRetryInterceptor(t *testing.T) {
10492
hostB := "bbb.bb.com"
10593
hRetryMax := 2
10694
hRetryInterceptor := NewHostsRetryInterceptor(HostsRetryConfig{
107-
RetryConfig: RetryConfig{
108-
RetryMax: hRetryMax,
109-
RetryInterval: func(_ *RetryInfo) time.Duration {
110-
return time.Second
111-
},
112-
//ShouldRetry: func(req *http.Request, resp *http.Response, err error) bool {
113-
// return true
114-
//},
115-
},
95+
RetryMax: hRetryMax,
11696
ShouldFreezeHost: nil,
11797
HostFreezeDuration: 0,
11898
HostProvider: hostprovider.NewWithHosts([]string{hostA, hostB}),
@@ -121,7 +101,7 @@ func TestHostsNotRetryInterceptor(t *testing.T) {
121101
retryMax := 1
122102
sRetryInterceptor := NewSimpleRetryInterceptor(SimpleRetryConfig{
123103
RetryMax: retryMax,
124-
RetryInterval: func(_ *RetryInfo) time.Duration {
104+
RetryInterval: func() time.Duration {
125105
return time.Second
126106
},
127107
//ShouldRetry: func(req *http.Request, resp *http.Response, err error) bool {
@@ -186,12 +166,7 @@ func TestHostsRetryInterceptorByRequest(t *testing.T) {
186166
hostB := "www.qiniu.com"
187167
hRetryMax := 30
188168
hRetryInterceptor := NewHostsRetryInterceptor(HostsRetryConfig{
189-
RetryConfig: RetryConfig{
190-
RetryMax: hRetryMax,
191-
RetryInterval: func(_ *RetryInfo) time.Duration {
192-
return time.Second
193-
},
194-
},
169+
RetryMax: hRetryMax,
195170
ShouldFreezeHost: nil,
196171
HostFreezeDuration: 0,
197172
HostProvider: hostprovider.NewWithHosts([]string{hostA, hostB}),
@@ -200,7 +175,7 @@ func TestHostsRetryInterceptorByRequest(t *testing.T) {
200175
retryMax := 1
201176
sRetryInterceptor := NewSimpleRetryInterceptor(SimpleRetryConfig{
202177
RetryMax: retryMax,
203-
RetryInterval: func(_ *RetryInfo) time.Duration {
178+
RetryInterval: func() time.Duration {
204179
return time.Second
205180
},
206181
})

‎internal/clientv2/interceptor_retry_simple.go

+22-20
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package clientv2
22

33
import (
4+
"context"
45
"io"
56
"math/rand"
67
"net"
@@ -13,20 +14,18 @@ import (
1314

1415
clientv1 "github.com/qiniu/go-sdk/v7/client"
1516
internal_io "github.com/qiniu/go-sdk/v7/internal/io"
17+
"github.com/qiniu/go-sdk/v7/storagev2/backoff"
1618
"github.com/qiniu/go-sdk/v7/storagev2/chooser"
1719
"github.com/qiniu/go-sdk/v7/storagev2/resolver"
1820
)
1921

2022
type (
2123
contextKeyBufferResponse struct{}
2224

23-
RetryInfo struct {
24-
Retried int
25-
}
26-
2725
SimpleRetryConfig struct {
28-
RetryMax int // 最大重试次数
29-
RetryInterval func(*RetryInfo) time.Duration // 重试时间间隔
26+
RetryMax int // 最大重试次数
27+
RetryInterval func() time.Duration // 重试时间间隔 v1
28+
Backoff backoff.Backoff // 重试时间间隔 v2，优先级高于 RetryInterval
3029
ShouldRetry func(req *http.Request, resp *http.Response, err error) bool
3130
Resolver resolver.Resolver // 主备域名解析器
3231
Chooser chooser.Chooser // 主备域名选择器
@@ -37,8 +36,9 @@ type (
3736
}
3837

3938
RetryConfig struct {
40-
RetryMax int // 最大重试次数
41-
RetryInterval func(*RetryInfo) time.Duration // 重试时间间隔
39+
RetryMax int // 最大重试次数
40+
RetryInterval func() time.Duration // 重试时间间隔 v1
41+
Backoff backoff.Backoff // 重试时间间隔 v2，优先级高于 RetryInterval
4242
ShouldRetry func(req *http.Request, resp *http.Response, err error) bool
4343
}
4444
)
@@ -52,10 +52,6 @@ func (c *RetryConfig) init() {
5252
c.RetryMax = 0
5353
}
5454

55-
if c.RetryInterval == nil {
56-
c.RetryInterval = defaultRetryInterval
57-
}
58-
5955
if c.ShouldRetry == nil {
6056
c.ShouldRetry = isSimpleRetryable
6157
}
@@ -70,15 +66,21 @@ func (c *SimpleRetryConfig) init() {
7066
c.RetryMax = 0
7167
}
7268

73-
if c.RetryInterval == nil {
74-
c.RetryInterval = defaultRetryInterval
75-
}
76-
7769
if c.ShouldRetry == nil {
7870
c.ShouldRetry = isSimpleRetryable
7971
}
8072
}
8173

74+
func (c *SimpleRetryConfig) getRetryInterval(ctx context.Context, attempts int) time.Duration {
75+
if bf := c.Backoff; bf != nil {
76+
return bf.Time(ctx, &backoff.BackoffOptions{Attempts: attempts})
77+
}
78+
if ri := c.RetryInterval; ri != nil {
79+
return ri()
80+
}
81+
return defaultRetryInterval()
82+
}
83+
8284
func NewSimpleRetryInterceptor(config SimpleRetryConfig) Interceptor {
8385
return &simpleRetryInterceptor{config: config}
8486
}
@@ -146,11 +148,11 @@ func (interceptor *simpleRetryInterceptor) Intercept(req *http.Request, handler
146148
resp.Body.Close()
147149
}
148150

149-
retryInterval := interceptor.config.RetryInterval(&RetryInfo{Retried: i})
150-
if retryInterval < time.Microsecond {
151+
if retryInterval := interceptor.config.getRetryInterval(req.Context(), i); retryInterval < time.Microsecond {
151152
continue
153+
} else {
154+
time.Sleep(retryInterval)
152155
}
153-
time.Sleep(retryInterval)
154156
}
155157
return resp, err
156158
}
@@ -265,6 +267,6 @@ func isNetworkErrorWithOpError(err *net.OpError) bool {
265267
return false
266268
}
267269

268-
func defaultRetryInterval(_ *RetryInfo) time.Duration {
270+
func defaultRetryInterval() time.Duration {
269271
return time.Duration(50+rand.Int()%50) * time.Millisecond
270272
}

‎internal/clientv2/interceptor_retry_simple_test.go

+56-2
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,16 @@ import (
77
"net/http"
88
"testing"
99
"time"
10+
11+
"github.com/qiniu/go-sdk/v7/storagev2/backoff"
1012
)
1113

1214
func TestSimpleAlwaysRetryInterceptor(t *testing.T) {
1315

1416
retryMax := 1
1517
rInterceptor := NewSimpleRetryInterceptor(SimpleRetryConfig{
1618
RetryMax: retryMax,
17-
RetryInterval: func(_ *RetryInfo) time.Duration {
19+
RetryInterval: func() time.Duration {
1820
return time.Second
1921
},
2022
ShouldRetry: func(req *http.Request, resp *http.Response, err error) bool {
@@ -69,7 +71,7 @@ func TestSimpleNotRetryInterceptor(t *testing.T) {
6971
retryMax := 1
7072
rInterceptor := NewSimpleRetryInterceptor(SimpleRetryConfig{
7173
RetryMax: retryMax,
72-
RetryInterval: func(_ *RetryInfo) time.Duration {
74+
RetryInterval: func() time.Duration {
7375
return time.Second
7476
},
7577
// 默认状态码是 400，400 不重试
@@ -120,3 +122,55 @@ func TestSimpleNotRetryInterceptor(t *testing.T) {
120122
t.Fatalf("retry flow error")
121123
}
122124
}
125+
126+
func TestRetryInterceptorWithBackoff(t *testing.T) {
127+
retryMax := 5
128+
rInterceptor := NewSimpleRetryInterceptor(SimpleRetryConfig{
129+
RetryMax: retryMax,
130+
Backoff: backoff.NewExponentialBackoff(100*time.Millisecond, 2),
131+
ShouldRetry: func(req *http.Request, resp *http.Response, err error) bool {
132+
return true
133+
},
134+
})
135+
136+
doCount := 0
137+
interceptor := NewSimpleInterceptor(func(req *http.Request, handler Handler) (*http.Response, error) {
138+
doCount += 1
139+
140+
value := req.Header.Get(headerKey)
141+
value += " -> request"
142+
req.Header.Set(headerKey, value)
143+
144+
resp, err := handler(req)
145+
146+
value = resp.Header.Get(headerKey)
147+
value += " -> response"
148+
resp.Header.Set(headerKey, value)
149+
return resp, err
150+
})
151+
152+
c := NewClient(&testClient{}, rInterceptor, interceptor)
153+
154+
start := time.Now()
155+
resp, _ := Do(c, RequestParams{
156+
Context: nil,
157+
Method: "",
158+
Url: "https://aaa.com",
159+
Header: nil,
160+
GetBody: nil,
161+
})
162+
duration := float32(time.Now().UnixNano()-start.UnixNano()) / float32(time.Millisecond)
163+
164+
if duration > 3100+10 || duration < 3100-10 {
165+
t.Fatalf("retry interval may be error:%f", duration)
166+
}
167+
168+
if (retryMax + 1) != doCount {
169+
t.Fatalf("retry count is not 2")
170+
}
171+
172+
value := resp.Header.Get(headerKey)
173+
if value != " -> request -> Do -> response" {
174+
t.Fatalf("retry flow error")
175+
}
176+
}

0 commit comments

Comments
 (0)