Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat: Add rate limit to Redis client #268

Merged
merged 8 commits into from
Oct 29, 2023
Merged

Feat: Add rate limit to Redis client #268

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
aed8bb3
Add rate limiting bucket to skip Redis if we have errored often
Garbett1 Oct 29, 2023
854d493
Puku format
Garbett1 Oct 29, 2023
e459164
Add x/time to go.mod
Garbett1 Oct 29, 2023
7b307e5
Add limit in more locations
Garbett1 Oct 29, 2023
628d532
Don't mess up the go.mod
Garbett1 Oct 29, 2023
742a8a9
Remove the limit on the write
Garbett1 Oct 29, 2023
d88d714
Avoid any leak of the digest proto
Garbett1 Oct 29, 2023
5d5c7f7
Bump version
Garbett1 Oct 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cli/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,6 @@ go_test(
srcs = ["cli_test.go"],
deps = [
":cli",
"///third_party/go/github.com_stretchr_testify//assert",
"//third_party/go:testify",
],
)
7 changes: 2 additions & 5 deletions elan/rpc/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ go_library(
"///third_party/go/github.com_klauspost_compress//zstd",
"///third_party/go/github.com_peterebden_go-cli-init_v4//logging",
"///third_party/go/github.com_prometheus_client_golang//prometheus",
"//proto/purity",
"///third_party/go/gocloud.dev//blob",
"///third_party/go/gocloud.dev//blob/fileblob",
"///third_party/go/gocloud.dev//blob/gcsblob",
Expand All @@ -32,10 +31,9 @@ go_library(
"///third_party/go/google.golang.org_api//googleapi",
"///third_party/go/google.golang.org_genproto_googleapis_bytestream//:bytestream",
"///third_party/go/google.golang.org_genproto_googleapis_rpc//status",
"///third_party/go/google.golang.org_grpc//codes",
"///third_party/go/google.golang.org_grpc//health/grpc_health_v1",
"///third_party/go/google.golang.org_grpc//status",
"//grpcutil",
"//proto/purity",
"//rexclient",
"//third_party/go:grpc",
],
Expand All @@ -48,10 +46,9 @@ go_test(
deps = [
":rpc",
"///third_party/go/github.com_klauspost_compress//zstd",
"///third_party/go/github.com_stretchr_testify//assert",
"///third_party/go/github.com_stretchr_testify//require",
"///third_party/go/google.golang.org_genproto_googleapis_bytestream//:bytestream",
"//grpcutil",
"//third_party/go:testify",
],
)

Expand Down
2 changes: 1 addition & 1 deletion flair/rpc/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@ go_library(
"///third_party/go/github.com_golang_protobuf//proto",
"///third_party/go/github.com_peterebden_go-cli-init_v4//logging",
"///third_party/go/github.com_peterebden_go-sri//:go-sri",
"//proto/purity",
"///third_party/go/golang.org_x_sync//errgroup",
"///third_party/go/google.golang.org_genproto_googleapis_bytestream//:bytestream",
"///third_party/go/google.golang.org_grpc//health/grpc_health_v1",
"///third_party/go/google.golang.org_grpc//metadata",
"//flair/trie",
"//grpcutil",
"//proto/purity",
"//rexclient",
"//third_party/go:grpc",
],
Expand Down
2 changes: 1 addition & 1 deletion flair/trie/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ go_library(
"///third_party/go/github.com_bazelbuild_remote-apis//build/bazel/remote/execution/v2",
"///third_party/go/github.com_hashicorp_go-multierror//:go-multierror",
"///third_party/go/github.com_peterebden_go-cli-init_v4//logging",
"//proto/purity",
"///third_party/go/google.golang.org_genproto_googleapis_bytestream//:bytestream",
"///third_party/go/google.golang.org_grpc//health/grpc_health_v1",
"//proto/purity",
"//third_party/go:grpc",
],
)
Expand Down
4 changes: 1 addition & 3 deletions grpcutil/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,14 @@ go_library(
"///third_party/go/github.com_mostynb_go-grpc-compression//zstd",
"///third_party/go/github.com_peterebden_go-cli-init_v4//logging",
"///third_party/go/github.com_prometheus_client_golang//prometheus",
"///third_party/go/google.golang.org_grpc//:grpc",
"///third_party/go/google.golang.org_grpc//codes",
"///third_party/go/google.golang.org_grpc//credentials",
"///third_party/go/google.golang.org_grpc//encoding/gzip",
"///third_party/go/google.golang.org_grpc//grpclog",
"///third_party/go/google.golang.org_grpc//health",
"///third_party/go/google.golang.org_grpc//health/grpc_health_v1",
"///third_party/go/google.golang.org_grpc//metadata",
"///third_party/go/google.golang.org_grpc//reflection",
"///third_party/go/google.golang.org_grpc//status",
"///third_party/go/gopkg.in_op_go-logging.v1//:go-logging.v1",
"//third_party/go:grpc",
],
)
2 changes: 1 addition & 1 deletion lucidity/rpc/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ go_test(
resources = glob(["static/*.*"]),
deps = [
":rpc",
"///third_party/go/github.com_stretchr_testify//assert",
"//proto/lucidity",
"//third_party/go:testify",
],
)
1 change: 0 additions & 1 deletion mettle/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,3 @@ sh_cmd(
"//sandbox:alt_sandbox",
],
)

11 changes: 3 additions & 8 deletions mettle/api/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,13 @@ go_library(
"///third_party/go/github.com_golang_protobuf//ptypes",
"///third_party/go/github.com_peterebden_go-cli-init_v4//logging",
"///third_party/go/github.com_prometheus_client_golang//prometheus",
"//proto/mettle",
"///third_party/go/gocloud.dev//pubsub",
"///third_party/go/google.golang.org_genproto//googleapis/longrunning",
"///third_party/go/google.golang.org_genproto_googleapis_rpc//status",
"///third_party/go/google.golang.org_grpc//codes",
"///third_party/go/google.golang.org_grpc//metadata",
"///third_party/go/google.golang.org_grpc//status",
"//grpcutil",
"//mettle/common",
"//proto/mettle",
"//rexclient",
"//third_party/go:grpc",
],
Expand All @@ -33,16 +31,13 @@ go_test(
"///third_party/go/github.com_bazelbuild_remote-apis//build/bazel/remote/execution/v2",
"///third_party/go/github.com_golang_protobuf//proto",
"///third_party/go/github.com_golang_protobuf//ptypes",
"///third_party/go/github.com_stretchr_testify//assert",
"///third_party/go/github.com_stretchr_testify//require",
"//proto/mettle",
"///third_party/go/gocloud.dev//pubsub",
"///third_party/go/google.golang.org_genproto//googleapis/longrunning",
"///third_party/go/google.golang.org_genproto_googleapis_bytestream//:bytestream",
"///third_party/go/google.golang.org_grpc//codes",
"///third_party/go/google.golang.org_grpc//status",
"//grpcutil",
"//mettle/common",
"//proto/mettle",
"//third_party/go:grpc",
"//third_party/go:testify",
],
)
5 changes: 2 additions & 3 deletions mettle/common/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,8 @@ go_library(
"///third_party/go/gocloud.dev//pubsub/gcppubsub",
"///third_party/go/google.golang.org_genproto//googleapis/longrunning",
"///third_party/go/google.golang.org_genproto//googleapis/pubsub/v1",
"///third_party/go/google.golang.org_grpc//codes",
"///third_party/go/google.golang.org_grpc//status",
"//mettle/mempubsub",
"//third_party/go:grpc",
],
)

Expand All @@ -22,6 +21,6 @@ go_test(
srcs = ["common_test.go"],
deps = [
":common",
"///third_party/go/github.com_stretchr_testify//assert",
"//third_party/go:testify",
],
)
8 changes: 4 additions & 4 deletions mettle/worker/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,20 @@ go_library(
"///third_party/go/github.com_prometheus_client_golang//prometheus/push",
"///third_party/go/github.com_prometheus_common//expfmt",
"///third_party/go/github.com_shirou_gopsutil//mem",
"//proto/lucidity",
"//third_party/proto/resourceusage",
"///third_party/go/gocloud.dev//pubsub",
"///third_party/go/golang.org_x_sync//errgroup",
"///third_party/go/golang.org_x_time//rate",
"///third_party/go/google.golang.org_genproto//googleapis/pubsub/v1",
"///third_party/go/google.golang.org_genproto_googleapis_rpc//status",
"///third_party/go/google.golang.org_grpc//codes",
"///third_party/go/google.golang.org_grpc//status",
"//cli",
"//elan/rpc",
"//grpcutil",
"//mettle/common",
"//proto/lucidity",
"//rexclient",
"//third_party/go:golang-protobuf",
"//third_party/go:grpc",
"//third_party/proto/cas",
"//third_party/proto/resourceusage",
],
)
35 changes: 24 additions & 11 deletions mettle/worker/redis.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"crypto/tls"
"crypto/x509"
"golang.org/x/time/rate"
"os"
"time"

Expand Down Expand Up @@ -71,8 +72,9 @@ func newRedisClient(client elan.Client, url, readURL, password, caFile string, u
elan: client,
redis: primaryClient,
readRedis: readClient,
timeout: 10 * time.Second,
timeout: 1 * time.Second,
maxSize: 200 * 1012, // 200 Kelly-Bootle standard units
limiter: rate.NewLimiter(rate.Every(time.Second*10), 10),
}
}

Expand All @@ -82,6 +84,7 @@ type redisClient struct {
readRedis *redis.Client
timeout time.Duration
maxSize int64
limiter *rate.Limiter
}

func (r *redisClient) Healthcheck() error {
Expand All @@ -91,23 +94,33 @@ func (r *redisClient) Healthcheck() error {
func (r *redisClient) ReadBlob(dg *pb.Digest) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), r.timeout)
defer cancel()
cmd := r.readRedis.Get(ctx, dg.Hash)
if err := cmd.Err(); err == nil {
blob, _ := cmd.Bytes()
return blob, nil
} else if err != redis.Nil {
log.Warning("Failed to read blob from Redis: %s", err)

if r.limiter.Allow() {
cmd := r.readRedis.Get(ctx, dg.Hash)
if err := cmd.Err(); err == nil {
blob, _ := cmd.Bytes()
return blob, nil
} else if err != redis.Nil {
log.Warningf("Failed to read blob from Redis: %s", err)
r.limiter.Reserve()
}
} else {
log.Warningf("limiter error rate exceeded, skipping Redis lookup")
}
// If we get here, the blob didn't exist. Download it then write back to Redis.
blob, err := r.elan.ReadBlob(dg)
if err != nil {
return nil, err
}
if dg.SizeBytes < r.maxSize {
ctx, cancel = context.WithTimeout(context.Background(), r.timeout)
defer cancel()
if cmd := r.redis.Set(ctx, dg.Hash, blob, 0); cmd.Val() != "OK" {
log.Warning("Failed to set blob in Redis: %s", cmd.Err())
if r.limiter.Allow() {
Garbett1 marked this conversation as resolved.
Show resolved Hide resolved
go func() {
ctx, cancel = context.WithTimeout(context.Background(), r.timeout)
defer cancel()
if cmd := r.redis.Set(ctx, dg.Hash, blob, 0); cmd.Val() != "OK" {
Garbett1 marked this conversation as resolved.
Show resolved Hide resolved
log.Warning("Failed to set blob in Redis: %s", cmd.Err())
}
}()
}
}
return blob, nil
Expand Down
2 changes: 1 addition & 1 deletion purity/gc/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ go_library(
"///third_party/go/github.com_golang_protobuf//proto",
"///third_party/go/github.com_hashicorp_go-multierror//:go-multierror",
"///third_party/go/github.com_peterebden_go-cli-init_v4//logging",
"//proto/purity",
"///third_party/go/golang.org_x_crypto//ssh/terminal",
"//proto/purity",
"//rexclient",
"//third_party/go:grpc",
],
Expand Down
2 changes: 1 addition & 1 deletion rexclient/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,6 @@ go_test(
":rexclient",
"///third_party/go/github.com_bazelbuild_remote-apis-sdks//go/pkg/digest",
"///third_party/go/github.com_bazelbuild_remote-apis//build/bazel/remote/execution/v2",
"///third_party/go/github.com_stretchr_testify//assert",
Garbett1 marked this conversation as resolved.
Show resolved Hide resolved
"//third_party/go:testify",
],
)