Merge pull request #207 from Maple-pro/dev_bloom_filter

Dev bloom filter

go.mod

@@ -207,6 +207,7 @@ require (
 	github.com/sivchari/tenv v1.7.1 // indirect
 	github.com/sonatard/noctx v0.0.2 // indirect
 	github.com/sourcegraph/go-diff v0.7.0 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/afero v1.9.5 // indirect
 	github.com/spf13/cast v1.5.1 // indirect
 	github.com/spf13/cobra v1.7.0 // indirect
@@ -229,6 +230,8 @@ require (
 	github.com/ultraware/funlen v0.1.0 // indirect
 	github.com/ultraware/whitespace v0.0.5 // indirect
 	github.com/uudashr/gocognit v1.0.7 // indirect
+	github.com/willf/bitset v1.1.11 // indirect
+	github.com/willf/bloom v2.0.3+incompatible // indirect
 	github.com/xen0n/gosmopolitan v1.2.1 // indirect
 	github.com/yagipy/maintidx v1.0.0 // indirect
 	github.com/yeya24/promlinter v0.2.0 // indirect

go.sum

@@ -678,6 +678,8 @@ github.com/sonatard/noctx v0.0.2 h1:L7Dz4De2zDQhW8S0t+KUjY0MAQJd6SgVwhzNIc4ok00=
 github.com/sonatard/noctx v0.0.2/go.mod h1:kzFz+CzWSjQ2OzIm46uJZoXuBpa2+0y3T36U18dWqIo=
 github.com/sourcegraph/go-diff v0.7.0 h1:9uLlrd5T46OXs5qpp8L/MTltk0zikUGi0sNNyCpA8G0=
 github.com/sourcegraph/go-diff v0.7.0/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM=
 github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ=
 github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA=
@@ -748,6 +750,10 @@ github.com/ultraware/whitespace v0.0.5 h1:hh+/cpIcopyMYbZNVov9iSxvJU3OYQg78Sfaqz
 github.com/ultraware/whitespace v0.0.5/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
 github.com/uudashr/gocognit v1.0.7 h1:e9aFXgKgUJrQ5+bs61zBigmj7bFJ/5cC6HmMahVzuDo=
 github.com/uudashr/gocognit v1.0.7/go.mod h1:nAIUuVBnYU7pcninia3BHOvQkpQCeO76Uscky5BOwcY=
+github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE=
+github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
+github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA=
+github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8=
 github.com/xen0n/gosmopolitan v1.2.1 h1:3pttnTuFumELBRSh+KQs1zcz4fN6Zy7aB0xlnQSn1Iw=
 github.com/xen0n/gosmopolitan v1.2.1/go.mod h1:JsHq/Brs1o050OOdmzHeOr0N7OtlnKRAGAsElF8xBQA=
 github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM=

src/services/auth/handler.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"github.com/google/uuid"
 	"github.com/sirupsen/logrus"
+	"github.com/willf/bloom"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/crypto/bcrypt"
@@ -35,6 +36,8 @@ var relationClient relation.RelationServiceClient
 var userClient user2.UserServiceClient
 var recommendClient recommend.RecommendServiceClient
 
+var bloomFilter *bloom.BloomFilter
+
 type AuthServiceImpl struct {
 	auth.AuthServiceServer
 }
@@ -46,6 +49,9 @@ func (a AuthServiceImpl) New() {
 	userClient = user2.NewUserServiceClient(userRpcConn)
 	recommendRpcConn := grpc2.Connect(config.RecommendRpcServiceName)
 	recommendClient = recommend.NewRecommendServiceClient(recommendRpcConn)
+
+	// Create a new Bloom filter with a target false positive rate of 0.1%
+	bloomFilter = bloom.NewWithEstimates(10000000, 0.001) // assuming we have 1 million users
 }
 
 func (a AuthServiceImpl) Authenticate(ctx context.Context, request *auth.AuthenticateRequest) (resp *auth.AuthenticateResponse, err error) {
@@ -233,6 +239,7 @@ func (a AuthServiceImpl) Register(ctx context.Context, request *auth.RegisterReq
 	resp.StatusCode = strings.ServiceOKCode
 	resp.StatusMsg = strings.ServiceOK
 
+	bloomFilter.AddString(user.UserName)
 	addMagicUserFriend(ctx, &span, user.ID)
 
 	return
@@ -247,6 +254,19 @@ func (a AuthServiceImpl) Login(ctx context.Context, request *auth.LoginRequest)
 		"username": request.Username,
 	}).Infof("User try to log in.")
 
+	// Check if a username might be in the filter
+	if !bloomFilter.TestString(request.Username) {
+		resp = &auth.LoginResponse{
+			StatusCode: strings.UnableToQueryUserErrorCode,
+			StatusMsg:  strings.UnableToQueryUserError,
+		}
+
+		logger.WithFields(logrus.Fields{
+			"username": request.Username,
+		}).Infof("The user is blocked by Bloom Filter")
+		return
+	}
+
 	resp = &auth.LoginResponse{}
 	user := models.User{
 		UserName: request.Username,

src/services/feed/handler.go

@@ -134,7 +134,7 @@ func (s FeedServiceImpl) ListVideosByRecommend(ctx context.Context, request *fee
 
 	now := time.Now().UnixMilli()
 	latestTime := now
-	if request.LatestTime != nil && *request.LatestTime != "" {
+	if request.LatestTime != nil && *request.LatestTime != "" && *request.LatestTime != "0" {
 		// Check if request.LatestTime is a timestamp
 		t, ok := isUnixMilliTimestamp(*request.LatestTime)
 		if ok {

src/services/message/handler.go

@@ -57,7 +57,7 @@ func (c MessageServiceImpl) New() {
 	var err error
 	conn, err = amqp.Dial(rabbitmq.BuildMQConnAddr())
 	if err != nil {
-		failOnError(err, "Fialed to conenct to RabbitMQ")
+		failOnError(err, "Failed to connect to RabbitMQ")
 	}
 
 	channel, err = conn.Channel()
@@ -317,7 +317,7 @@ func (c MessageServiceImpl) Chat(ctx context.Context, request *chat.ChatRequest)
 			"user_id":      request.UserId,
 			"ActorId":      request.ActorId,
 			"pre_msg_time": request.PreMsgTime,
-		}).Errorf("ChatServiceImpl list chat failed to response when listing message,database err")
+		}).Errorf("ChatServiceImpl list chat failed to response when listing message, database err")
 		logging.SetSpanError(span, err)
 
 		resp = &chat.ChatResponse{