-
Notifications
You must be signed in to change notification settings - Fork 1
/
breaker.go
90 lines (76 loc) · 2.05 KB
/
breaker.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
package breaker
import (
"context"
"errors"
"time"
"github.com/goware/superr"
)
var (
ErrFatal = errors.New("breaker: fatal error")
ErrHitMaxRetries = errors.New("breaker: hit max retries")
)
type Breaker struct {
log Logger
backoff time.Duration
factor float64
maxTries int
}
func Default(optLog ...Logger) *Breaker {
var log Logger
if len(optLog) > 0 {
log = optLog[0]
}
return &Breaker{
log: log,
backoff: 1 * time.Second, // backoff for 1 second to start,
factor: 2, // and for each attempt multiply backoff by factor,
maxTries: 15, // until number of maxTries before giving up
}
}
func New(log Logger, backoff time.Duration, factor float64, maxTries int) *Breaker {
return &Breaker{
log: log,
backoff: backoff,
factor: factor,
maxTries: maxTries,
}
}
// Do is an exponential-backoff-retry caller which will wait `backoff*factor**retry` up to `maxTries`
// `maxTries = 1` means retry only once when an error occurs.
func (b *Breaker) Do(ctx context.Context, fn func() error) error {
delay := float64(b.backoff)
try := 0
for {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
err := fn()
if err == nil {
return nil
}
// If we failed for some reason, exp backoff and retry.
// Check if is fatal error and should stop immediately
if errors.Is(err, ErrFatal) {
return err
}
// Move on if we have tried a few times.
if try >= b.maxTries {
if b.log != nil {
b.log.Errorf("breaker: exhausted after max number of retries %d. fail :(", b.maxTries)
}
return superr.New(ErrHitMaxRetries, err)
}
if b.log != nil {
b.log.Warnf("breaker: fn failed: '%v' - backing off for %v and trying again (retry #%d)", err, time.Duration(int64(delay)).String(), try+1)
}
// Sleep and try again.
time.Sleep(time.Duration(int64(delay)))
delay *= b.factor
try++
}
}
func Do(ctx context.Context, fn func() error, log Logger, backoff time.Duration, factor float64, maxTries int) error {
return New(log, backoff, factor, maxTries).Do(ctx, fn)
}