Skip to content

Commit

Permalink
fix some bugs; add set lock
Browse files Browse the repository at this point in the history
  • Loading branch information
Luke Lombardi committed Jan 11, 2024
1 parent f73da7a commit 47b3537
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 18 deletions.
35 changes: 25 additions & 10 deletions internal/abstractions/map/map_redis.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,23 +12,33 @@ import (

type RedisMapService struct {
pb.UnimplementedMapServiceServer
lock *common.RedisLock

rdb *common.RedisClient
}

func NewRedisMapService(rdb *common.RedisClient) (MapService, error) {
lock := common.NewRedisLock(rdb)

return &RedisMapService{
rdb: rdb,
rdb: rdb,
lock: lock,
}, nil
}

// Map service implementations
func (m *RedisMapService) MapSet(ctx context.Context, in *pb.MapSetRequest) (*pb.MapSetResponse, error) {
authInfo, _ := auth.AuthInfoFromContext(ctx)

err := m.rdb.Set(context.TODO(), Keys.MapEntry(authInfo.Workspace.Name, in.Name, in.Key), in.Value, 0).Err()
err := m.lock.Acquire(ctx, Keys.MapEntryLock(authInfo.Workspace.Name, in.Name, in.Key), common.RedisLockOptions{TtlS: 10, Retries: 0})
if err != nil {
return &pb.MapSetResponse{Ok: false}, nil
}
defer m.lock.Release(Keys.MapEntryLock(authInfo.Workspace.Name, in.Name, in.Key))

err = m.rdb.Set(ctx, Keys.MapEntry(authInfo.Workspace.Name, in.Name, in.Key), in.Value, 0).Err()
if err != nil {
return &pb.MapSetResponse{Ok: false}, err
return &pb.MapSetResponse{Ok: false}, nil
}

return &pb.MapSetResponse{Ok: true}, nil
Expand All @@ -37,9 +47,9 @@ func (m *RedisMapService) MapSet(ctx context.Context, in *pb.MapSetRequest) (*pb
func (m *RedisMapService) MapGet(ctx context.Context, in *pb.MapGetRequest) (*pb.MapGetResponse, error) {
authInfo, _ := auth.AuthInfoFromContext(ctx)

value, err := m.rdb.Get(context.TODO(), Keys.MapEntry(authInfo.Workspace.Name, in.Name, in.Key)).Bytes()
value, err := m.rdb.Get(ctx, Keys.MapEntry(authInfo.Workspace.Name, in.Name, in.Key)).Bytes()
if err != nil {
return &pb.MapGetResponse{Ok: false, Value: nil}, err
return &pb.MapGetResponse{Ok: false, Value: nil}, nil
}

return &pb.MapGetResponse{Ok: true, Value: value}, nil
Expand All @@ -48,7 +58,7 @@ func (m *RedisMapService) MapGet(ctx context.Context, in *pb.MapGetRequest) (*pb
func (m *RedisMapService) MapDelete(ctx context.Context, in *pb.MapDeleteRequest) (*pb.MapDeleteResponse, error) {
authInfo, _ := auth.AuthInfoFromContext(ctx)

err := m.rdb.Del(context.TODO(), Keys.MapEntry(authInfo.Workspace.Name, in.Name, in.Key)).Err()
err := m.rdb.Del(ctx, Keys.MapEntry(authInfo.Workspace.Name, in.Name, in.Key)).Err()
if err != nil {
return &pb.MapDeleteResponse{Ok: false}, err
}
Expand All @@ -59,7 +69,7 @@ func (m *RedisMapService) MapDelete(ctx context.Context, in *pb.MapDeleteRequest
func (m *RedisMapService) MapCount(ctx context.Context, in *pb.MapCountRequest) (*pb.MapCountResponse, error) {
authInfo, _ := auth.AuthInfoFromContext(ctx)

keys, err := m.rdb.Scan(context.TODO(), Keys.MapEntry(authInfo.Workspace.Name, in.Name, "*"))
keys, err := m.rdb.Scan(ctx, Keys.MapEntry(authInfo.Workspace.Name, in.Name, "*"))
if err != nil {
return &pb.MapCountResponse{Ok: false, Count: 0}, err
}
Expand All @@ -70,7 +80,7 @@ func (m *RedisMapService) MapCount(ctx context.Context, in *pb.MapCountRequest)
func (m *RedisMapService) MapKeys(ctx context.Context, in *pb.MapKeysRequest) (*pb.MapKeysResponse, error) {
authInfo, _ := auth.AuthInfoFromContext(ctx)

keys, err := m.rdb.Scan(context.TODO(), Keys.MapEntry(authInfo.Workspace.Name, in.Name, "*"))
keys, err := m.rdb.Scan(ctx, Keys.MapEntry(authInfo.Workspace.Name, in.Name, "*"))
if err != nil {
return &pb.MapKeysResponse{Ok: false, Keys: []string{}}, err
}
Expand All @@ -85,8 +95,9 @@ func (m *RedisMapService) MapKeys(ctx context.Context, in *pb.MapKeysRequest) (*

// Redis keys
var (
mapPrefix string = "map"
mapEntry string = "map:%s:%s:%s"
mapPrefix string = "map"
mapEntry string = "map:%s:%s:%s"
mapEntryLock string = "map:%s:%s:%s:lock"
)

var Keys = &keys{}
Expand All @@ -100,3 +111,7 @@ func (k *keys) MapPrefix() string {
func (k *keys) MapEntry(workspaceName, name, key string) string {
return fmt.Sprintf(mapEntry, workspaceName, name, key)
}

func (k *keys) MapEntryLock(workspaceName, name, key string) string {
return fmt.Sprintf(mapEntry, workspaceName, name, key)
}
22 changes: 14 additions & 8 deletions sdk/src/beam/abstractions/queue.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,13 @@
import cloudpickle

from beam.abstractions.base import BaseAbstraction
from beam.clients.simplequeue import SimpleQueueServiceStub
from beam.clients.simplequeue import (
SimpleQueueEmptyResponse,
SimpleQueuePeekResponse,
SimpleQueuePopResponse,
SimpleQueuePutResponse,
SimpleQueueServiceStub,
)


class SimpleQueueInternalServerError(Exception):
Expand All @@ -19,22 +25,24 @@ def __init__(self, *, name: str, max_size=100) -> None:
self.max_size: int = max_size

def __len__(self):
r = self.run_sync(self.stub.size(name=self.name))
r = self.run_sync(self.stub.simple_queue_size(name=self.name))
return r.size if r.ok else 0

def __del__(self):
self.channel.close()

def put(self, value: Any) -> bool:
r = self.run_sync(self.stub.put(name=self.name, value=cloudpickle.dumps(value)))
r: SimpleQueuePutResponse = self.run_sync(
self.stub.simple_queue_put(name=self.name, value=cloudpickle.dumps(value))
)

if not r.ok:
raise SimpleQueueInternalServerError

return True

def pop(self) -> Any:
r = self.run_sync(self.stub.pop(name=self.name))
r: SimpleQueuePopResponse = self.run_sync(self.stub.simple_queue_pop(name=self.name))
if not r.ok:
raise SimpleQueueInternalServerError

Expand All @@ -44,16 +52,14 @@ def pop(self) -> Any:
return None

def empty(self) -> bool:
r = self.run_sync(self.stub.empty(name=self.name))

r: SimpleQueueEmptyResponse = self.run_sync(self.stub.simple_queue_empty(name=self.name))
if not r.ok:
raise SimpleQueueInternalServerError

return r.empty if r.ok else True

def peek(self) -> Any:
r = self.run_sync(self.stub.peek(name=self.name))

r: SimpleQueuePeekResponse = self.run_sync(self.stub.simple_queue_peek(name=self.name))
if not r.ok:
raise SimpleQueueInternalServerError

Expand Down

0 comments on commit 47b3537

Please sign in to comment.