-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathretry.go
151 lines (133 loc) · 4.57 KB
/
retry.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
// Package retry invokes a given function until it succeeds. It sleeps in
// between attempts based the DelayMethod. It is useful in situations that an
// action might succeed after a few attempt due to unavailable resources or
// waiting for a condition to happen.
//
// The default DelayMethod sleeps exactly the same amount of time between
// attempts. You can use the IncrementalDelay method to increment the delays
// between attempts. It gives a jitter to the delay to prevent Thundering herd
// problems. If the delay is 0 in either case, it does not sleep between tries.
// The IncrementalDelay has a maximum delay of 1 second, but if you need a more
// flexible delay, you can use the IncrementalDelayMax method and give it a max
// delay.
package retry
import (
"context"
"errors"
"fmt"
"math/rand"
"runtime/debug"
"time"
)
// DelayMethod determines how the delay behaves. The current attempt is passed
// on each iteration, with the delay value of the Retry object.
type DelayMethod func(attempt int, delay time.Duration) time.Duration
// StopError causes the Do method stop trying and will return the Err. This
// error then is returned by the Do method.
type StopError struct {
Err error
}
func (s StopError) Error() string { return s.Err.Error() }
func (s StopError) Unwrap() error { return s.Err }
// Retry attempts to call a given function until it succeeds, or returns a
// StopError value for a certain amount of times. It will delay between calls
// for any errors based on the provided Method. Retry is concurrent safe and
// the zero value does not do anything.
type Retry struct {
Method DelayMethod
Delay time.Duration
MaxDelay time.Duration
Attempts int
}
type repeatFunc func() error
// Do calls fn until it returns nil or a StopError. It delays and retries if
// the fn returns any errors or panics. The value of the returned error, or the
// Err of a StopError, or an error with the panic message will be returned at
// the last cycle.
func (r Retry) Do(fn1 repeatFunc, fns ...repeatFunc) error {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
return r.DoContext(ctx, fn1, fns...)
}
// DoContext calls fn until it returns nil or a StopError. It delays and
// retries if the fn returns any errors or panics. If the context is cancelled,
// it will stop iterating and returns the error reported by ctx.Err() method.
// The value of the returned error, or the Err of a StopError, or an error with
// the panic message will be returned at the last cycle.
func (r Retry) DoContext(ctx context.Context, fn1 repeatFunc, fns ...repeatFunc) error {
method := r.Method
if method == nil {
method = StandardDelay
}
var err error
for i := range r.Attempts {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
err = r.do(ctx, fn1, fns...)
if err == nil {
return nil
}
var e *StopError
if errors.As(err, &e) {
return e.Err
}
time.Sleep(method(i+1, r.Delay))
}
return err
}
var errPanic = errors.New("function caused a panic")
func (r Retry) do(ctx context.Context, fn1 repeatFunc, fns ...repeatFunc) error {
var err error
for _, fn := range append([]repeatFunc{fn1}, fns...) {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
func() {
defer func() {
if e := recover(); e != nil {
switch x := e.(type) {
case error:
err = fmt.Errorf("%w: %w\n%s", errPanic, x, debug.Stack())
default:
err = fmt.Errorf("%w: %s\n%s", errPanic, e, debug.Stack())
}
}
}()
err = fn()
}()
if err != nil {
return err
}
}
return nil
}
// StandardDelay always delays the same amount of time.
func StandardDelay(_ int, delay time.Duration) time.Duration { return delay }
// IncrementalDelay increases the delay between attempts up to a second. It
// adds a jitter to prevent Thundering herd. If the delay is 0, it always
// returns 0.
func IncrementalDelay(attempt int, delay time.Duration) time.Duration {
return IncrementalDelayMax(time.Second)(attempt, delay)
}
// IncrementalDelayMax returns a DelayMethod that increases the delay between
// attempts up to the given maximum duration. It adds a jitter to prevent
// Thundering herd. If the delay is 0, it always returns 0.
func IncrementalDelayMax(maximum time.Duration) func(int, time.Duration) time.Duration {
return func(attempt int, delay time.Duration) time.Duration {
if delay == 0 {
return 0
}
if delay > maximum {
delay = maximum
}
d := int64(delay)
//nolint:gosec // the rand package is used for fast random number generation.
jitter := rand.Int63n(d) / 2
return (delay * time.Duration(attempt)) + time.Duration(jitter)
}
}