Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat: Add rate limit to Redis client #268

Merged
merged 8 commits into from
Oct 29, 2023
Merged

Feat: Add rate limit to Redis client #268

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
aed8bb3
Add rate limiting bucket to skip Redis if we have errored often
Garbett1 Oct 29, 2023
854d493
Puku format
Garbett1 Oct 29, 2023
e459164
Add x/time to go.mod
Garbett1 Oct 29, 2023
7b307e5
Add limit in more locations
Garbett1 Oct 29, 2023
628d532
Don't mess up the go.mod
Garbett1 Oct 29, 2023
742a8a9
Remove the limit on the write
Garbett1 Oct 29, 2023
d88d714
Avoid any leak of the digest proto
Garbett1 Oct 29, 2023
5d5c7f7
Bump version
Garbett1 Oct 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ChangeLog
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
Version 11.6.0
--------------
* Add rate-limiter to Redis client so we don't fail slowly

Version 11.5.0
--------------
* Update gocloud to v0.34.0 to support setting max batchsize on topics
Expand Down
2 changes: 1 addition & 1 deletion VERSION
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
11.5.0
11.6.0
2 changes: 1 addition & 1 deletion cli/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,6 @@ go_test(
srcs = ["cli_test.go"],
deps = [
":cli",
"///third_party/go/github.com_stretchr_testify//assert",
"//third_party/go:testify",
],
)
7 changes: 2 additions & 5 deletions elan/rpc/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ go_library(
"///third_party/go/github.com_klauspost_compress//zstd",
"///third_party/go/github.com_peterebden_go-cli-init_v4//logging",
"///third_party/go/github.com_prometheus_client_golang//prometheus",
"//proto/purity",
"///third_party/go/gocloud.dev//blob",
"///third_party/go/gocloud.dev//blob/fileblob",
"///third_party/go/gocloud.dev//blob/gcsblob",
Expand All @@ -32,10 +31,9 @@ go_library(
"///third_party/go/google.golang.org_api//googleapi",
"///third_party/go/google.golang.org_genproto_googleapis_bytestream//:bytestream",
"///third_party/go/google.golang.org_genproto_googleapis_rpc//status",
"///third_party/go/google.golang.org_grpc//codes",
"///third_party/go/google.golang.org_grpc//health/grpc_health_v1",
"///third_party/go/google.golang.org_grpc//status",
"//grpcutil",
"//proto/purity",
"//rexclient",
"//third_party/go:grpc",
],
Expand All @@ -48,10 +46,9 @@ go_test(
deps = [
":rpc",
"///third_party/go/github.com_klauspost_compress//zstd",
"///third_party/go/github.com_stretchr_testify//assert",
"///third_party/go/github.com_stretchr_testify//require",
"///third_party/go/google.golang.org_genproto_googleapis_bytestream//:bytestream",
"//grpcutil",
"//third_party/go:testify",
],
)

Expand Down
2 changes: 1 addition & 1 deletion flair/rpc/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@ go_library(
"///third_party/go/github.com_golang_protobuf//proto",
"///third_party/go/github.com_peterebden_go-cli-init_v4//logging",
"///third_party/go/github.com_peterebden_go-sri//:go-sri",
"//proto/purity",
"///third_party/go/golang.org_x_sync//errgroup",
"///third_party/go/google.golang.org_genproto_googleapis_bytestream//:bytestream",
"///third_party/go/google.golang.org_grpc//health/grpc_health_v1",
"///third_party/go/google.golang.org_grpc//metadata",
"//flair/trie",
"//grpcutil",
"//proto/purity",
"//rexclient",
"//third_party/go:grpc",
],
Expand Down
2 changes: 1 addition & 1 deletion flair/trie/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ go_library(
"///third_party/go/github.com_bazelbuild_remote-apis//build/bazel/remote/execution/v2",
"///third_party/go/github.com_hashicorp_go-multierror//:go-multierror",
"///third_party/go/github.com_peterebden_go-cli-init_v4//logging",
"//proto/purity",
"///third_party/go/google.golang.org_genproto_googleapis_bytestream//:bytestream",
"///third_party/go/google.golang.org_grpc//health/grpc_health_v1",
"//proto/purity",
"//third_party/go:grpc",
],
)
Expand Down
5 changes: 3 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ require (
golang.org/x/crypto v0.1.0
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53
golang.org/x/sync v0.1.0
golang.org/x/time v0.3.0
google.golang.org/api v0.114.0
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc
google.golang.org/genproto/googleapis/bytestream v0.0.0-20230530153820-e85fd2cbaebc
Expand All @@ -56,7 +57,6 @@ require (
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/snappy v0.0.3 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/s2a-go v0.1.4 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/google/wire v0.4.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
Expand All @@ -71,9 +71,10 @@ require (
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/xattr v0.4.4 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/google/s2a-go v0.1.4 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/thought-machine/go-flags v1.5.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel v0.20.0 // indirect
Expand Down
209 changes: 209 additions & 0 deletions go.sum
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 1 addition & 3 deletions grpcutil/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,14 @@ go_library(
"///third_party/go/github.com_mostynb_go-grpc-compression//zstd",
"///third_party/go/github.com_peterebden_go-cli-init_v4//logging",
"///third_party/go/github.com_prometheus_client_golang//prometheus",
"///third_party/go/google.golang.org_grpc//:grpc",
"///third_party/go/google.golang.org_grpc//codes",
"///third_party/go/google.golang.org_grpc//credentials",
"///third_party/go/google.golang.org_grpc//encoding/gzip",
"///third_party/go/google.golang.org_grpc//grpclog",
"///third_party/go/google.golang.org_grpc//health",
"///third_party/go/google.golang.org_grpc//health/grpc_health_v1",
"///third_party/go/google.golang.org_grpc//metadata",
"///third_party/go/google.golang.org_grpc//reflection",
"///third_party/go/google.golang.org_grpc//status",
"///third_party/go/gopkg.in_op_go-logging.v1//:go-logging.v1",
"//third_party/go:grpc",
],
)
2 changes: 1 addition & 1 deletion lucidity/rpc/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ go_test(
resources = glob(["static/*.*"]),
deps = [
":rpc",
"///third_party/go/github.com_stretchr_testify//assert",
"//proto/lucidity",
"//third_party/go:testify",
],
)
1 change: 0 additions & 1 deletion mettle/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,3 @@ sh_cmd(
"//sandbox:alt_sandbox",
],
)

11 changes: 3 additions & 8 deletions mettle/api/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,13 @@ go_library(
"///third_party/go/github.com_golang_protobuf//ptypes",
"///third_party/go/github.com_peterebden_go-cli-init_v4//logging",
"///third_party/go/github.com_prometheus_client_golang//prometheus",
"//proto/mettle",
"///third_party/go/gocloud.dev//pubsub",
"///third_party/go/google.golang.org_genproto//googleapis/longrunning",
"///third_party/go/google.golang.org_genproto_googleapis_rpc//status",
"///third_party/go/google.golang.org_grpc//codes",
"///third_party/go/google.golang.org_grpc//metadata",
"///third_party/go/google.golang.org_grpc//status",
"//grpcutil",
"//mettle/common",
"//proto/mettle",
"//rexclient",
"//third_party/go:grpc",
],
Expand All @@ -33,16 +31,13 @@ go_test(
"///third_party/go/github.com_bazelbuild_remote-apis//build/bazel/remote/execution/v2",
"///third_party/go/github.com_golang_protobuf//proto",
"///third_party/go/github.com_golang_protobuf//ptypes",
"///third_party/go/github.com_stretchr_testify//assert",
"///third_party/go/github.com_stretchr_testify//require",
"//proto/mettle",
"///third_party/go/gocloud.dev//pubsub",
"///third_party/go/google.golang.org_genproto//googleapis/longrunning",
"///third_party/go/google.golang.org_genproto_googleapis_bytestream//:bytestream",
"///third_party/go/google.golang.org_grpc//codes",
"///third_party/go/google.golang.org_grpc//status",
"//grpcutil",
"//mettle/common",
"//proto/mettle",
"//third_party/go:grpc",
"//third_party/go:testify",
],
)
5 changes: 2 additions & 3 deletions mettle/common/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,8 @@ go_library(
"///third_party/go/gocloud.dev//pubsub/gcppubsub",
"///third_party/go/google.golang.org_genproto//googleapis/longrunning",
"///third_party/go/google.golang.org_genproto//googleapis/pubsub/v1",
"///third_party/go/google.golang.org_grpc//codes",
"///third_party/go/google.golang.org_grpc//status",
"//mettle/mempubsub",
"//third_party/go:grpc",
],
)

Expand All @@ -22,6 +21,6 @@ go_test(
srcs = ["common_test.go"],
deps = [
":common",
"///third_party/go/github.com_stretchr_testify//assert",
"//third_party/go:testify",
],
)
8 changes: 4 additions & 4 deletions mettle/worker/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,20 @@ go_library(
"///third_party/go/github.com_prometheus_client_golang//prometheus/push",
"///third_party/go/github.com_prometheus_common//expfmt",
"///third_party/go/github.com_shirou_gopsutil//mem",
"//proto/lucidity",
"//third_party/proto/resourceusage",
"///third_party/go/gocloud.dev//pubsub",
"///third_party/go/golang.org_x_sync//errgroup",
"///third_party/go/golang.org_x_time//rate",
"///third_party/go/google.golang.org_genproto//googleapis/pubsub/v1",
"///third_party/go/google.golang.org_genproto_googleapis_rpc//status",
"///third_party/go/google.golang.org_grpc//codes",
"///third_party/go/google.golang.org_grpc//status",
"//cli",
"//elan/rpc",
"//grpcutil",
"//mettle/common",
"//proto/lucidity",
"//rexclient",
"//third_party/go:golang-protobuf",
"//third_party/go:grpc",
"//third_party/proto/cas",
"//third_party/proto/resourceusage",
],
)
40 changes: 28 additions & 12 deletions mettle/worker/redis.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"crypto/tls"
"crypto/x509"
"golang.org/x/time/rate"
"os"
"time"

Expand Down Expand Up @@ -71,8 +72,9 @@ func newRedisClient(client elan.Client, url, readURL, password, caFile string, u
elan: client,
redis: primaryClient,
readRedis: readClient,
timeout: 10 * time.Second,
timeout: 1 * time.Second,
maxSize: 200 * 1012, // 200 Kelly-Bootle standard units
limiter: rate.NewLimiter(rate.Every(time.Second*10), 10),
}
}

Expand All @@ -82,6 +84,7 @@ type redisClient struct {
readRedis *redis.Client
timeout time.Duration
maxSize int64
limiter *rate.Limiter
}

func (r *redisClient) Healthcheck() error {
Expand All @@ -91,24 +94,32 @@ func (r *redisClient) Healthcheck() error {
func (r *redisClient) ReadBlob(dg *pb.Digest) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
defer cancel()
cmd := r.readRedis.Get(ctx, dg.Hash)
if err := cmd.Err(); err == nil {
blob, _ := cmd.Bytes()
return blob, nil
} else if err != redis.Nil {
log.Warning("Failed to read blob from Redis: %s", err)

if r.limiter.Allow() {
cmd := r.readRedis.Get(ctx, dg.Hash)
if err := cmd.Err(); err == nil {
blob, _ := cmd.Bytes()
return blob, nil
} else if err != redis.Nil {
log.Warningf("Failed to read blob from Redis: %s", err)
r.limiter.Reserve()
}
} else {
log.Warningf("limiter error rate exceeded, skipping Redis lookup")
}
// If we get here, the blob didn't exist. Download it then write back to Redis.
blob, err := r.elan.ReadBlob(dg)
if err != nil {
return nil, err
}
if dg.SizeBytes < r.maxSize {
ctx, cancel = context.WithTimeout(context.Background(), r.timeout)
defer cancel()
if cmd := r.redis.Set(ctx, dg.Hash, blob, 0); cmd.Val() != "OK" {
log.Warning("Failed to set blob in Redis: %s", cmd.Err())
}
go func(hash string) {
ctx, cancel = context.WithTimeout(context.Background(), r.timeout)
defer cancel()
if cmd := r.redis.Set(ctx, hash, blob, 0); cmd.Val() != "OK" {
log.Warning("Failed to set blob in Redis: %s", cmd.Err())
}
}(dg.Hash)
}
return blob, nil
}
Expand Down Expand Up @@ -215,11 +226,16 @@ func (r *redisClient) readBlobs(keys []string, metrics bool) [][]byte {
if len(keys) == 0 {
return nil
}
if !r.limiter.Allow() {
// Bail out immediately if Redis has exceeded error limit
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
defer cancel()
resp, err := r.readRedis.MGet(ctx, keys...).Result()
if err != nil {
log.Warning("Failed to check blobs in Redis: %s", err)
r.limiter.Reserve()
return nil
} else if len(resp) != len(keys) {
log.Warning("Length mismatch in Redis response; got %d, expected %d", len(resp), len(keys))
Expand Down
2 changes: 1 addition & 1 deletion purity/gc/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ go_library(
"///third_party/go/github.com_golang_protobuf//proto",
"///third_party/go/github.com_hashicorp_go-multierror//:go-multierror",
"///third_party/go/github.com_peterebden_go-cli-init_v4//logging",
"//proto/purity",
"///third_party/go/golang.org_x_crypto//ssh/terminal",
"//proto/purity",
"//rexclient",
"//third_party/go:grpc",
],
Expand Down
2 changes: 1 addition & 1 deletion rexclient/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,6 @@ go_test(
":rexclient",
"///third_party/go/github.com_bazelbuild_remote-apis-sdks//go/pkg/digest",
"///third_party/go/github.com_bazelbuild_remote-apis//build/bazel/remote/execution/v2",
"///third_party/go/github.com_stretchr_testify//assert",
Garbett1 marked this conversation as resolved.
Show resolved Hide resolved
"//third_party/go:testify",
],
)