From c55e43f9f7c121cac5e4192f7c9af8ff7de0afd3 Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Fri, 11 Oct 2024 19:07:43 +0530 Subject: [PATCH 01/22] Update config.go --- config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index 08eaeaa..9cc13ba 100644 --- a/config/config.go +++ b/config/config.go @@ -27,7 +27,7 @@ func LoadConfig() *Config { } return &Config{ - DiceDBAddr: getEnv("DICEDB_ADDR", "localhost:7379"), // Default DiceDB address + DiceDBAddr: "localhost:7379", // Default DiceDB address ServerPort: getEnv("SERVER_PORT", ":8080"), // Default server port RequestLimitPerMin: getEnvInt("REQUEST_LIMIT_PER_MIN", 1000), // Default request limit RequestWindowSec: getEnvFloat64("REQUEST_WINDOW_SEC", 60), // Default request window in float64 From 7a0947f7698d48b1f58e3f771f525d85e86c110d Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Fri, 11 Oct 2024 19:10:47 +0530 Subject: [PATCH 02/22] added logs --- main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/main.go b/main.go index 99c98e3..1ebbbf3 100644 --- a/main.go +++ b/main.go @@ -12,6 +12,9 @@ import ( func main() { configValue := config.LoadConfig() + slog.info("aasif!"); + slog.info(configValue); + slog.info("aasif printed!"); diceClient, err := db.InitDiceClient(configValue) if err != nil { slog.Error("Failed to initialize DiceDB client: %v", slog.Any("err", err)) From 947739e5514fded9bb1b08101ce528d22294e61a Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Fri, 11 Oct 2024 19:12:21 +0530 Subject: [PATCH 03/22] fix compilation --- main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index 1ebbbf3..fb47759 100644 --- a/main.go +++ b/main.go @@ -12,9 +12,9 @@ import ( func main() { configValue := config.LoadConfig() - slog.info("aasif!"); - slog.info(configValue); - slog.info("aasif printed!"); + slog.Info("aasif!"); + slog.Info(configValue); + slog.Info("aasif printed!"); diceClient, err := db.InitDiceClient(configValue) if err != nil { slog.Error("Failed to initialize DiceDB client: %v", slog.Any("err", err)) From 285de23e364aa0219c5e8293bbb24d2837a25fcb Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Fri, 11 Oct 2024 19:13:43 +0530 Subject: [PATCH 04/22] fix compilation --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index fb47759..f5d9298 100644 --- a/main.go +++ b/main.go @@ -13,7 +13,7 @@ import ( func main() { configValue := config.LoadConfig() slog.Info("aasif!"); - slog.Info(configValue); + slog.Info("Config loaded:", slog.Any("configValue", configValue)) slog.Info("aasif printed!"); diceClient, err := db.InitDiceClient(configValue) if err != nil { From 78c42b3741c08dbb4110337cd72f96ca649cf1f7 Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Fri, 11 Oct 2024 19:19:00 +0530 Subject: [PATCH 05/22] change IP --- config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index 9cc13ba..6a28046 100644 --- a/config/config.go +++ b/config/config.go @@ -27,7 +27,7 @@ func LoadConfig() *Config { } return &Config{ - DiceDBAddr: "localhost:7379", // Default DiceDB address + DiceDBAddr: "192.168.1.3:7379", // Default DiceDB address ServerPort: getEnv("SERVER_PORT", ":8080"), // Default server port RequestLimitPerMin: getEnvInt("REQUEST_LIMIT_PER_MIN", 1000), // Default request limit RequestWindowSec: getEnvFloat64("REQUEST_WINDOW_SEC", 60), // Default request window in float64 From a2bc2cb539c3b479eeb363926d3f3c7040b300c2 Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Fri, 11 Oct 2024 19:36:31 +0530 Subject: [PATCH 06/22] adding log for diceCmd --- internal/server/http.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/server/http.go b/internal/server/http.go index 876f959..57c9636 100644 --- a/internal/server/http.go +++ b/internal/server/http.go @@ -100,6 +100,7 @@ func (s *HTTPServer) HealthCheck(w http.ResponseWriter, request *http.Request) { func (s *HTTPServer) CliHandler(w http.ResponseWriter, r *http.Request) { diceCmd, err := util.ParseHTTPRequest(r) + slog.Info("aasif diceCmd:", slog.Any("diceCmd", diceCmd)) if err != nil { http.Error(w, errorResponse(err.Error()), http.StatusBadRequest) return From 2e07bc7520d8af669dcbfa5690e04d96af36e1e4 Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Fri, 11 Oct 2024 19:43:20 +0530 Subject: [PATCH 07/22] log request --- internal/server/http.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/server/http.go b/internal/server/http.go index 57c9636..33caec5 100644 --- a/internal/server/http.go +++ b/internal/server/http.go @@ -99,6 +99,7 @@ func (s *HTTPServer) HealthCheck(w http.ResponseWriter, request *http.Request) { } func (s *HTTPServer) CliHandler(w http.ResponseWriter, r *http.Request) { + slog.Info("aasif request:", slog.Any("request=", r)) diceCmd, err := util.ParseHTTPRequest(r) slog.Info("aasif diceCmd:", slog.Any("diceCmd", diceCmd)) if err != nil { From 1a1572c0325cc2acc9929e2be76c296a851a7ee8 Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Tue, 15 Oct 2024 00:37:20 +0530 Subject: [PATCH 08/22] Update ratelimiter.go --- internal/middleware/ratelimiter.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/middleware/ratelimiter.go b/internal/middleware/ratelimiter.go index 66ea05a..c17b4a0 100644 --- a/internal/middleware/ratelimiter.go +++ b/internal/middleware/ratelimiter.go @@ -159,4 +159,7 @@ func addRateLimitHeaders(w http.ResponseWriter, limit, remaining, used, resetTim w.Header().Set("x-ratelimit-remaining", strconv.FormatInt(remaining, 10)) w.Header().Set("x-ratelimit-used", strconv.FormatInt(used, 10)) w.Header().Set("x-ratelimit-reset", strconv.FormatInt(resetTime, 10)) + + // Expose the rate limit headers to the client + w.Header().Set("Access-Control-Expose-Headers", "x-ratelimit-limit, x-ratelimit-remaining, x-ratelimit-used, x-ratelimit-reset") } From 6069eb8d99f69f0f3d6ce650ce5158c7b4bfc62f Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Thu, 17 Oct 2024 00:49:04 +0530 Subject: [PATCH 09/22] change IP --- config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index 6a28046..f006c69 100644 --- a/config/config.go +++ b/config/config.go @@ -27,7 +27,7 @@ func LoadConfig() *Config { } return &Config{ - DiceDBAddr: "192.168.1.3:7379", // Default DiceDB address + DiceDBAddr: "192.168.1.10:7379", // Default DiceDB address ServerPort: getEnv("SERVER_PORT", ":8080"), // Default server port RequestLimitPerMin: getEnvInt("REQUEST_LIMIT_PER_MIN", 1000), // Default request limit RequestWindowSec: getEnvFloat64("REQUEST_WINDOW_SEC", 60), // Default request window in float64 From d1c9ef85c65d10dd2ee6656aad1c5a21ae118470 Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Thu, 17 Oct 2024 15:10:36 +0530 Subject: [PATCH 10/22] changed IP --- config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index f006c69..e688755 100644 --- a/config/config.go +++ b/config/config.go @@ -27,7 +27,7 @@ func LoadConfig() *Config { } return &Config{ - DiceDBAddr: "192.168.1.10:7379", // Default DiceDB address + DiceDBAddr: "192.168.114.28:7379", // Default DiceDB address ServerPort: getEnv("SERVER_PORT", ":8080"), // Default server port RequestLimitPerMin: getEnvInt("REQUEST_LIMIT_PER_MIN", 1000), // Default request limit RequestWindowSec: getEnvFloat64("REQUEST_WINDOW_SEC", 60), // Default request window in float64 From 6ca83c77871ab3f3a8627683745020510aa80788 Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Fri, 25 Oct 2024 12:51:26 +0530 Subject: [PATCH 11/22] start sending x-next-cleanup-time --- internal/middleware/ratelimiter.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/internal/middleware/ratelimiter.go b/internal/middleware/ratelimiter.go index cfc7b3b..db24fb7 100644 --- a/internal/middleware/ratelimiter.go +++ b/internal/middleware/ratelimiter.go @@ -12,12 +12,14 @@ import ( "strconv" "strings" "time" - + "server/config" "github.com/dicedb/dicedb-go" ) // RateLimiter middleware to limit requests based on a specified limit and duration func RateLimiter(client *db.DiceDB, next http.Handler, limit int64, window float64) http.Handler { + configValue := config.LoadConfig() + cronFrequencyInterval := configValue.Server.CronCleanupFrequency return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if handleCors(w, r) { return @@ -92,8 +94,18 @@ func RateLimiter(client *db.DiceDB, next http.Handler, limit int64, window float } } + lastCleanupTime := time.UnixMilli(lastCronCleanupTime) + + // Calculate next cleanup time + nextCleanupTime := lastCleanupTime.Add(cronFrequencyInterval) + + // Get the difference from the current time + timeDifference := nextCleanupTime.Sub(time.Now()) + + secondsDifference := int64(timeDifference.Seconds()) + addRateLimitHeaders(w, limit, limit-(requestCount+1), requestCount+1, currentWindow+int64(window), - lastCronCleanupTime) + secondsDifference) slog.Info("Request processed", "count", requestCount+1) next.ServeHTTP(w, r) @@ -170,12 +182,12 @@ func MockRateLimiter(client *mock.DiceDBMock, next http.Handler, limit int64, wi }) } -func addRateLimitHeaders(w http.ResponseWriter, limit, remaining, used, resetTime, cronLastCleanupTime int64) { +func addRateLimitHeaders(w http.ResponseWriter, limit, remaining, used, resetTime, secondsLeftForCleanup int64) { w.Header().Set("x-ratelimit-limit", strconv.FormatInt(limit, 10)) w.Header().Set("x-ratelimit-remaining", strconv.FormatInt(remaining, 10)) w.Header().Set("x-ratelimit-used", strconv.FormatInt(used, 10)) w.Header().Set("x-ratelimit-reset", strconv.FormatInt(resetTime, 10)) - w.Header().Set("x-last-cleanup-time", strconv.FormatInt(cronLastCleanupTime, 10)) + w.Header().Set("x-next-cleanup-time", strconv.FormatInt(secondsLeftForCleanup, 10)) // Expose the rate limit headers to the client w.Header().Set("Access-Control-Expose-Headers", "x-ratelimit-limit, x-ratelimit-remaining,"+ From d8fbaf21e1c6b53bbc9f932a1128a2a837218ac9 Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Fri, 25 Oct 2024 14:37:13 +0530 Subject: [PATCH 12/22] change IP --- config/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index 1a4558b..90b0c3d 100644 --- a/config/config.go +++ b/config/config.go @@ -49,7 +49,7 @@ func LoadConfig() *Config { Username string Password string }{ - Addr: getEnv("DICEDB_ADMIN_ADDR", "192.168.1.13:7379"), // Default DiceDB Admin address + Addr: getEnv("DICEDB_ADMIN_ADDR", "192.168.1.3:7379"), // Default DiceDB Admin address Username: getEnv("DICEDB_ADMIN_USERNAME", "diceadmin"), // Default DiceDB Admin username Password: getEnv("DICEDB_ADMIN_PASSWORD", ""), // Default DiceDB Admin password }, @@ -58,7 +58,7 @@ func LoadConfig() *Config { Username string Password string }{ - Addr: getEnv("DICEDB_ADDR", "192.168.1.13:7380"), // Default DiceDB address + Addr: getEnv("DICEDB_ADDR", "192.168.1.3:7380"), // Default DiceDB address Username: getEnv("DICEDB_USERNAME", "dice"), // Default username Password: getEnv("DICEDB_PASSWORD", ""), // Default password }, From 8622414212c9396b446194627f4e860d7f6d26d4 Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Fri, 25 Oct 2024 14:39:42 +0530 Subject: [PATCH 13/22] expose x-next-cleanup-time header --- internal/middleware/ratelimiter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/middleware/ratelimiter.go b/internal/middleware/ratelimiter.go index db24fb7..9802b44 100644 --- a/internal/middleware/ratelimiter.go +++ b/internal/middleware/ratelimiter.go @@ -191,5 +191,5 @@ func addRateLimitHeaders(w http.ResponseWriter, limit, remaining, used, resetTim // Expose the rate limit headers to the client w.Header().Set("Access-Control-Expose-Headers", "x-ratelimit-limit, x-ratelimit-remaining,"+ - "x-ratelimit-used, x-ratelimit-reset, x-last-cleanup-time") + "x-ratelimit-used, x-ratelimit-reset, x-next-cleanup-time") } From e9cf16e21dd98f30292c58d36daa9fb2f53e862e Mon Sep 17 00:00:00 2001 From: Aasif Khan Date: Fri, 25 Oct 2024 15:07:12 +0530 Subject: [PATCH 14/22] revert changes --- config/config.go | 4 +- internal/server/http.go | 220 +++++++++++++++++++--------------------- 2 files changed, 107 insertions(+), 117 deletions(-) diff --git a/config/config.go b/config/config.go index 90b0c3d..c24a152 100644 --- a/config/config.go +++ b/config/config.go @@ -49,7 +49,7 @@ func LoadConfig() *Config { Username string Password string }{ - Addr: getEnv("DICEDB_ADMIN_ADDR", "192.168.1.3:7379"), // Default DiceDB Admin address + Addr: getEnv("DICEDB_ADMIN_ADDR", "localhost:7379"), // Default DiceDB Admin address Username: getEnv("DICEDB_ADMIN_USERNAME", "diceadmin"), // Default DiceDB Admin username Password: getEnv("DICEDB_ADMIN_PASSWORD", ""), // Default DiceDB Admin password }, @@ -58,7 +58,7 @@ func LoadConfig() *Config { Username string Password string }{ - Addr: getEnv("DICEDB_ADDR", "192.168.1.3:7380"), // Default DiceDB address + Addr: getEnv("DICEDB_ADDR", "localhost:7380"), // Default DiceDB address Username: getEnv("DICEDB_USERNAME", "dice"), // Default username Password: getEnv("DICEDB_PASSWORD", ""), // Default password }, diff --git a/internal/server/http.go b/internal/server/http.go index 9296fdf..c24a152 100644 --- a/internal/server/http.go +++ b/internal/server/http.go @@ -1,146 +1,136 @@ -package server +package config import ( - "context" - "encoding/json" - "errors" "log/slog" - "net/http" + "os" + "strconv" "strings" "time" - "server/internal/db" - "server/internal/middleware" - util "server/util" + "github.com/joho/godotenv" ) -type HTTPServer struct { - httpServer *http.Server - DiceClient *db.DiceDB -} - -type HandlerMux struct { - mux *http.ServeMux - rateLimiter func(http.ResponseWriter, *http.Request, http.Handler) -} - -type HTTPResponse struct { - Data interface{} `json:"data"` -} - -type HTTPErrorResponse struct { - Error interface{} `json:"error"` +// Config holds the application configuration +type Config struct { + // Config for DiceDBAdmin instance. This instance holds internal keys + // and is separate from DiceDB hosting global key pool i.e. user facing. + DiceDBAdmin struct { + Addr string // Field for the Dice address + Username string // Field for the username + Password string // Field for the password + } + // Config for DiceDB User instance. This instance holds internal keys + // and is separate from DiceDB hosting global key pool i.e. user facing. + DiceDB struct { + Addr string // Field for the Dice address + Username string // Field for the username + Password string // Field for the password + } + Server struct { + Port string // Field for the server port + IsTestEnv bool + RequestLimitPerMin int64 // Field for the request limit + RequestWindowSec float64 // Field for the time window in float64 + AllowedOrigins []string // Field for the allowed origins + CronCleanupFrequency time.Duration // Field for configuring key cleanup cron + } } -func errorResponse(response string) string { - errorMessage := map[string]string{"error": response} - jsonResponse, err := json.Marshal(errorMessage) +// LoadConfig loads the application configuration from environment variables or defaults +func LoadConfig() *Config { + err := godotenv.Load() if err != nil { - slog.Error("Error marshaling response: %v", slog.Any("err", err)) - return `{"error": "internal server error"}` + slog.Debug("Warning: .env file not found, falling back to system environment variables.") } - return string(jsonResponse) -} - -func (cim *HandlerMux) ServeHTTP(w http.ResponseWriter, r *http.Request) { - middleware.TrailingSlashMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - r.URL.Path = strings.ToLower(r.URL.Path) - cim.rateLimiter(w, r, cim.mux) - })).ServeHTTP(w, r) -} - -func NewHTTPServer(addr string, mux *http.ServeMux, diceDBAdminClient *db.DiceDB, diceClient *db.DiceDB, - limit int64, window float64) *HTTPServer { - handlerMux := &HandlerMux{ - mux: mux, - rateLimiter: func(w http.ResponseWriter, r *http.Request, next http.Handler) { - middleware.RateLimiter(diceDBAdminClient, next, limit, window).ServeHTTP(w, r) + return &Config{ + DiceDBAdmin: struct { + Addr string + Username string + Password string + }{ + Addr: getEnv("DICEDB_ADMIN_ADDR", "localhost:7379"), // Default DiceDB Admin address + Username: getEnv("DICEDB_ADMIN_USERNAME", "diceadmin"), // Default DiceDB Admin username + Password: getEnv("DICEDB_ADMIN_PASSWORD", ""), // Default DiceDB Admin password }, - } - - return &HTTPServer{ - httpServer: &http.Server{ - Addr: addr, - Handler: handlerMux, - ReadHeaderTimeout: 5 * time.Second, + DiceDB: struct { + Addr string + Username string + Password string + }{ + Addr: getEnv("DICEDB_ADDR", "localhost:7380"), // Default DiceDB address + Username: getEnv("DICEDB_USERNAME", "dice"), // Default username + Password: getEnv("DICEDB_PASSWORD", ""), // Default password + }, + Server: struct { + Port string + IsTestEnv bool + RequestLimitPerMin int64 + RequestWindowSec float64 + AllowedOrigins []string + CronCleanupFrequency time.Duration + }{ + Port: getEnv("SERVER_PORT", ":8080"), + IsTestEnv: getEnvBool("IS_TEST_ENVIRONMENT", false), // Default server port + RequestLimitPerMin: getEnvInt("REQUEST_LIMIT_PER_MIN", 1000), // Default request limit + RequestWindowSec: getEnvFloat64("REQUEST_WINDOW_SEC", 60), // Default request window in float64 + AllowedOrigins: getEnvArray("ALLOWED_ORIGINS", []string{"http://localhost:3000"}), // Default allowed origins + CronCleanupFrequency: time.Duration(getEnvInt("CRON_CLEANUP_FREQUENCY_MINS", 15)) * time.Minute, // Default cron cleanup frequency }, - DiceClient: diceClient, } } -func (s *HTTPServer) Run(ctx context.Context) error { - var err error - - go func() { - slog.Info("starting HTTP server at", slog.String("addr", s.httpServer.Addr)) - if err = s.httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { - slog.Error("http server error: %v", slog.Any("err", err)) - } - }() - - go func() { - <-ctx.Done() - err = s.Shutdown() - if err != nil { - slog.Error("Failed to gracefully shutdown HTTP server", slog.Any("err", err)) - return - } - }() - - return err -} - -func (s *HTTPServer) Shutdown() error { - if err := s.DiceClient.Client.Close(); err != nil { - slog.Error("failed to close dicedb client: %v", slog.Any("err", err)) +// getEnv retrieves an environment variable or returns a default value +func getEnv(key, fallback string) string { + if value, exists := os.LookupEnv(key); exists { + return value } - - return s.httpServer.Shutdown(context.Background()) -} - -func (s *HTTPServer) HealthCheck(w http.ResponseWriter, request *http.Request) { - util.JSONResponse(w, http.StatusOK, map[string]string{"message": "server is running"}) + return fallback } -func (s *HTTPServer) CliHandler(w http.ResponseWriter, r *http.Request) { - slog.Info("aasif request:", slog.Any("request=", r)) - diceCmd, err := util.ParseHTTPRequest(r) - slog.Info("aasif diceCmd:", slog.Any("diceCmd", diceCmd)) - if err != nil { - http.Error(w, errorResponse(err.Error()), http.StatusBadRequest) - return - } - - resp, err := s.DiceClient.ExecuteCommand(diceCmd) - if err != nil { - slog.Error("error: failure in executing command", "error", slog.Any("err", err)) - http.Error(w, errorResponse(err.Error()), http.StatusBadRequest) - return +// getEnvInt retrieves an environment variable as an integer or returns a default value +func getEnvInt(key string, fallback int) int64 { + if value, exists := os.LookupEnv(key); exists { + if intValue, err := strconv.Atoi(value); err == nil { + return int64(intValue) + } } + return int64(fallback) +} - respStr, ok := resp.(string) - if !ok { - slog.Error("error: response is not a string", "error", slog.Any("err", err)) - http.Error(w, errorResponse("internal server error"), http.StatusInternalServerError) - return +// added for miliseconds request window controls +func getEnvFloat64(key string, fallback float64) float64 { + if value, exists := os.LookupEnv(key); exists { + if floatValue, err := strconv.ParseFloat(value, 64); err == nil { + return floatValue + } } + return fallback +} - httpResponse := HTTPResponse{Data: respStr} - responseJSON, err := json.Marshal(httpResponse) - if err != nil { - slog.Error("error marshaling response to json", "error", slog.Any("err", err)) - http.Error(w, errorResponse("internal server error"), http.StatusInternalServerError) - return +func getEnvArray(key string, fallback []string) []string { + if value, exists := os.LookupEnv(key); exists { + if arrayValue := splitString(value); len(arrayValue) > 0 { + return arrayValue + } } + return fallback +} - _, err = w.Write(responseJSON) - if err != nil { - http.Error(w, errorResponse("internal server error"), http.StatusInternalServerError) - return +func getEnvBool(key string, fallback bool) bool { + if value, exists := os.LookupEnv(key); exists { + if boolValue, err := strconv.ParseBool(value); err == nil { + return boolValue + } } + return fallback } -func (s *HTTPServer) SearchHandler(w http.ResponseWriter, request *http.Request) { - util.JSONResponse(w, http.StatusOK, map[string]string{"message": "search results"}) +// splitString splits a string by comma and returns a slice of strings +func splitString(s string) []string { + var array []string + for _, v := range strings.Split(s, ",") { + array = append(array, strings.TrimSpace(v)) + } + return array } From 81ad533ef6957e394f08489a028e53336c0c4c89 Mon Sep 17 00:00:00 2001 From: Aasif Khan Date: Fri, 25 Oct 2024 15:11:00 +0530 Subject: [PATCH 15/22] revert changes in http.go --- internal/server/http.go | 218 +++++++++++++++++++++------------------- 1 file changed, 113 insertions(+), 105 deletions(-) diff --git a/internal/server/http.go b/internal/server/http.go index c24a152..b5209c2 100644 --- a/internal/server/http.go +++ b/internal/server/http.go @@ -1,136 +1,144 @@ -package config +package server import ( + "context" + "encoding/json" + "errors" "log/slog" - "os" - "strconv" + "net/http" "strings" "time" - "github.com/joho/godotenv" + "server/internal/db" + "server/internal/middleware" + util "server/util" ) -// Config holds the application configuration -type Config struct { - // Config for DiceDBAdmin instance. This instance holds internal keys - // and is separate from DiceDB hosting global key pool i.e. user facing. - DiceDBAdmin struct { - Addr string // Field for the Dice address - Username string // Field for the username - Password string // Field for the password - } - // Config for DiceDB User instance. This instance holds internal keys - // and is separate from DiceDB hosting global key pool i.e. user facing. - DiceDB struct { - Addr string // Field for the Dice address - Username string // Field for the username - Password string // Field for the password - } - Server struct { - Port string // Field for the server port - IsTestEnv bool - RequestLimitPerMin int64 // Field for the request limit - RequestWindowSec float64 // Field for the time window in float64 - AllowedOrigins []string // Field for the allowed origins - CronCleanupFrequency time.Duration // Field for configuring key cleanup cron - } +type HTTPServer struct { + httpServer *http.Server + DiceClient *db.DiceDB +} + +type HandlerMux struct { + mux *http.ServeMux + rateLimiter func(http.ResponseWriter, *http.Request, http.Handler) } -// LoadConfig loads the application configuration from environment variables or defaults -func LoadConfig() *Config { - err := godotenv.Load() +type HTTPResponse struct { + Data interface{} `json:"data"` +} + +type HTTPErrorResponse struct { + Error interface{} `json:"error"` +} + +func errorResponse(response string) string { + errorMessage := map[string]string{"error": response} + jsonResponse, err := json.Marshal(errorMessage) if err != nil { - slog.Debug("Warning: .env file not found, falling back to system environment variables.") + slog.Error("Error marshaling response: %v", slog.Any("err", err)) + return `{"error": "internal server error"}` } - return &Config{ - DiceDBAdmin: struct { - Addr string - Username string - Password string - }{ - Addr: getEnv("DICEDB_ADMIN_ADDR", "localhost:7379"), // Default DiceDB Admin address - Username: getEnv("DICEDB_ADMIN_USERNAME", "diceadmin"), // Default DiceDB Admin username - Password: getEnv("DICEDB_ADMIN_PASSWORD", ""), // Default DiceDB Admin password - }, - DiceDB: struct { - Addr string - Username string - Password string - }{ - Addr: getEnv("DICEDB_ADDR", "localhost:7380"), // Default DiceDB address - Username: getEnv("DICEDB_USERNAME", "dice"), // Default username - Password: getEnv("DICEDB_PASSWORD", ""), // Default password - }, - Server: struct { - Port string - IsTestEnv bool - RequestLimitPerMin int64 - RequestWindowSec float64 - AllowedOrigins []string - CronCleanupFrequency time.Duration - }{ - Port: getEnv("SERVER_PORT", ":8080"), - IsTestEnv: getEnvBool("IS_TEST_ENVIRONMENT", false), // Default server port - RequestLimitPerMin: getEnvInt("REQUEST_LIMIT_PER_MIN", 1000), // Default request limit - RequestWindowSec: getEnvFloat64("REQUEST_WINDOW_SEC", 60), // Default request window in float64 - AllowedOrigins: getEnvArray("ALLOWED_ORIGINS", []string{"http://localhost:3000"}), // Default allowed origins - CronCleanupFrequency: time.Duration(getEnvInt("CRON_CLEANUP_FREQUENCY_MINS", 15)) * time.Minute, // Default cron cleanup frequency + return string(jsonResponse) +} + +func (cim *HandlerMux) ServeHTTP(w http.ResponseWriter, r *http.Request) { + middleware.TrailingSlashMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.URL.Path = strings.ToLower(r.URL.Path) + cim.rateLimiter(w, r, cim.mux) + })).ServeHTTP(w, r) +} + +func NewHTTPServer(addr string, mux *http.ServeMux, diceDBAdminClient *db.DiceDB, diceClient *db.DiceDB, + limit int64, window float64) *HTTPServer { + handlerMux := &HandlerMux{ + mux: mux, + rateLimiter: func(w http.ResponseWriter, r *http.Request, next http.Handler) { + middleware.RateLimiter(diceDBAdminClient, next, limit, window).ServeHTTP(w, r) }, } -} -// getEnv retrieves an environment variable or returns a default value -func getEnv(key, fallback string) string { - if value, exists := os.LookupEnv(key); exists { - return value + return &HTTPServer{ + httpServer: &http.Server{ + Addr: addr, + Handler: handlerMux, + ReadHeaderTimeout: 5 * time.Second, + }, + DiceClient: diceClient, } - return fallback } -// getEnvInt retrieves an environment variable as an integer or returns a default value -func getEnvInt(key string, fallback int) int64 { - if value, exists := os.LookupEnv(key); exists { - if intValue, err := strconv.Atoi(value); err == nil { - return int64(intValue) +func (s *HTTPServer) Run(ctx context.Context) error { + var err error + + go func() { + slog.Info("starting HTTP server at", slog.String("addr", s.httpServer.Addr)) + if err = s.httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + slog.Error("http server error: %v", slog.Any("err", err)) } - } - return int64(fallback) -} + }() -// added for miliseconds request window controls -func getEnvFloat64(key string, fallback float64) float64 { - if value, exists := os.LookupEnv(key); exists { - if floatValue, err := strconv.ParseFloat(value, 64); err == nil { - return floatValue + go func() { + <-ctx.Done() + err = s.Shutdown() + if err != nil { + slog.Error("Failed to gracefully shutdown HTTP server", slog.Any("err", err)) + return } - } - return fallback + }() + + return err } -func getEnvArray(key string, fallback []string) []string { - if value, exists := os.LookupEnv(key); exists { - if arrayValue := splitString(value); len(arrayValue) > 0 { - return arrayValue - } +func (s *HTTPServer) Shutdown() error { + if err := s.DiceClient.Client.Close(); err != nil { + slog.Error("failed to close dicedb client: %v", slog.Any("err", err)) } - return fallback + + return s.httpServer.Shutdown(context.Background()) } -func getEnvBool(key string, fallback bool) bool { - if value, exists := os.LookupEnv(key); exists { - if boolValue, err := strconv.ParseBool(value); err == nil { - return boolValue - } - } - return fallback +func (s *HTTPServer) HealthCheck(w http.ResponseWriter, request *http.Request) { + util.JSONResponse(w, http.StatusOK, map[string]string{"message": "server is running"}) } -// splitString splits a string by comma and returns a slice of strings -func splitString(s string) []string { - var array []string - for _, v := range strings.Split(s, ",") { - array = append(array, strings.TrimSpace(v)) +func (s *HTTPServer) CliHandler(w http.ResponseWriter, r *http.Request) { + diceCmd, err := util.ParseHTTPRequest(r) + if err != nil { + http.Error(w, errorResponse(err.Error()), http.StatusBadRequest) + return + } + + resp, err := s.DiceClient.ExecuteCommand(diceCmd) + if err != nil { + slog.Error("error: failure in executing command", "error", slog.Any("err", err)) + http.Error(w, errorResponse(err.Error()), http.StatusBadRequest) + return + } + + respStr, ok := resp.(string) + if !ok { + slog.Error("error: response is not a string", "error", slog.Any("err", err)) + http.Error(w, errorResponse("internal server error"), http.StatusInternalServerError) + return + } + + httpResponse := HTTPResponse{Data: respStr} + responseJSON, err := json.Marshal(httpResponse) + if err != nil { + slog.Error("error marshaling response to json", "error", slog.Any("err", err)) + http.Error(w, errorResponse("internal server error"), http.StatusInternalServerError) + return + } + + _, err = w.Write(responseJSON) + if err != nil { + http.Error(w, errorResponse("internal server error"), http.StatusInternalServerError) + return } - return array +} + +func (s *HTTPServer) SearchHandler(w http.ResponseWriter, request *http.Request) { + util.JSONResponse(w, http.StatusOK, map[string]string{"message": "search results"}) } From d17db794eee13127becf6364f2663ae1b4b85647 Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Fri, 25 Oct 2024 23:54:33 +0530 Subject: [PATCH 16/22] resolve comment: refactor to put nextCleanupTime in a function --- internal/middleware/ratelimiter.go | 45 +++++++++++++++--------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/internal/middleware/ratelimiter.go b/internal/middleware/ratelimiter.go index 9802b44..92db6a1 100644 --- a/internal/middleware/ratelimiter.go +++ b/internal/middleware/ratelimiter.go @@ -80,30 +80,11 @@ func RateLimiter(client *db.DiceDB, next http.Handler, limit int64, window float } } - // Get the cron last cleanup run time - var lastCronCleanupTime int64 - resp := client.Client.Get(ctx, utils.LastCronCleanupTimeUnixMs) - if resp.Err() != nil && !errors.Is(resp.Err(), dicedb.Nil) { - slog.Error("Failed to get last cron cleanup time for headers", slog.Any("err", resp.Err().Error())) - } - - if resp.Val() != "" { - lastCronCleanupTime, err = strconv.ParseInt(resp.Val(), 10, 64) - if err != nil { - slog.Error("Error converting last cron cleanup time", "error", err) - } + secondsDifference, err := calculateNextCleanupTime(client, ctx, cronFrequencyInterval) + if err != nil { + slog.Error("Error calculating next cleanup time", "error", err) } - lastCleanupTime := time.UnixMilli(lastCronCleanupTime) - - // Calculate next cleanup time - nextCleanupTime := lastCleanupTime.Add(cronFrequencyInterval) - - // Get the difference from the current time - timeDifference := nextCleanupTime.Sub(time.Now()) - - secondsDifference := int64(timeDifference.Seconds()) - addRateLimitHeaders(w, limit, limit-(requestCount+1), requestCount+1, currentWindow+int64(window), secondsDifference) @@ -112,6 +93,26 @@ func RateLimiter(client *db.DiceDB, next http.Handler, limit int64, window float }) } +func calculateNextCleanupTime(client *db.DiceDB, ctx context.Context, cronFrequencyInterval time.Duration) (int64, error) { + var lastCronCleanupTime int64 + resp := client.Client.Get(ctx, utils.LastCronCleanupTimeUnixMs) + if resp.Err() != nil && !errors.Is(resp.Err(), dicedb.Nil) { + return 0, resp.Err() + } + + if resp.Val() != "" { + lastCronCleanupTime, err := strconv.ParseInt(resp.Val(), 10, 64) + if err != nil { + return 0, err + } + } + + lastCleanupTime := time.UnixMilli(lastCronCleanupTime) + nextCleanupTime := lastCleanupTime.Add(cronFrequencyInterval) + timeDifference := nextCleanupTime.Sub(time.Now()) + return int64(timeDifference.Seconds()), nil +} + func MockRateLimiter(client *mock.DiceDBMock, next http.Handler, limit int64, window float64) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if handleCors(w, r) { From 5dfb34a4663ce03a268d03af27b290b1fee53ec5 Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Fri, 25 Oct 2024 23:57:32 +0530 Subject: [PATCH 17/22] temporary IP change --- config/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index c24a152..6773096 100644 --- a/config/config.go +++ b/config/config.go @@ -49,7 +49,7 @@ func LoadConfig() *Config { Username string Password string }{ - Addr: getEnv("DICEDB_ADMIN_ADDR", "localhost:7379"), // Default DiceDB Admin address + Addr: getEnv("DICEDB_ADMIN_ADDR", "192.168.114.28:7379"), // Default DiceDB Admin address Username: getEnv("DICEDB_ADMIN_USERNAME", "diceadmin"), // Default DiceDB Admin username Password: getEnv("DICEDB_ADMIN_PASSWORD", ""), // Default DiceDB Admin password }, @@ -58,7 +58,7 @@ func LoadConfig() *Config { Username string Password string }{ - Addr: getEnv("DICEDB_ADDR", "localhost:7380"), // Default DiceDB address + Addr: getEnv("DICEDB_ADDR", "192.168.114.28:7380"), // Default DiceDB address Username: getEnv("DICEDB_USERNAME", "dice"), // Default username Password: getEnv("DICEDB_PASSWORD", ""), // Default password }, From 70304d2f6228866c36568326ba4c536f5bc5b486 Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Sat, 26 Oct 2024 00:02:45 +0530 Subject: [PATCH 18/22] fix compilation --- internal/middleware/ratelimiter.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/middleware/ratelimiter.go b/internal/middleware/ratelimiter.go index 92db6a1..ff43da8 100644 --- a/internal/middleware/ratelimiter.go +++ b/internal/middleware/ratelimiter.go @@ -101,7 +101,8 @@ func calculateNextCleanupTime(client *db.DiceDB, ctx context.Context, cronFreque } if resp.Val() != "" { - lastCronCleanupTime, err := strconv.ParseInt(resp.Val(), 10, 64) + var err error + lastCronCleanupTime, err = strconv.ParseInt(resp.Val(), 10, 64) // directly assign here if err != nil { return 0, err } @@ -113,6 +114,7 @@ func calculateNextCleanupTime(client *db.DiceDB, ctx context.Context, cronFreque return int64(timeDifference.Seconds()), nil } + func MockRateLimiter(client *mock.DiceDBMock, next http.Handler, limit int64, window float64) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if handleCors(w, r) { From 14b69af73b6daeb6ee7905ed94d8cfcb71f044e9 Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Sat, 26 Oct 2024 00:06:57 +0530 Subject: [PATCH 19/22] revert IP change --- config/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index 6773096..c24a152 100644 --- a/config/config.go +++ b/config/config.go @@ -49,7 +49,7 @@ func LoadConfig() *Config { Username string Password string }{ - Addr: getEnv("DICEDB_ADMIN_ADDR", "192.168.114.28:7379"), // Default DiceDB Admin address + Addr: getEnv("DICEDB_ADMIN_ADDR", "localhost:7379"), // Default DiceDB Admin address Username: getEnv("DICEDB_ADMIN_USERNAME", "diceadmin"), // Default DiceDB Admin username Password: getEnv("DICEDB_ADMIN_PASSWORD", ""), // Default DiceDB Admin password }, @@ -58,7 +58,7 @@ func LoadConfig() *Config { Username string Password string }{ - Addr: getEnv("DICEDB_ADDR", "192.168.114.28:7380"), // Default DiceDB address + Addr: getEnv("DICEDB_ADDR", "localhost:7380"), // Default DiceDB address Username: getEnv("DICEDB_USERNAME", "dice"), // Default username Password: getEnv("DICEDB_PASSWORD", ""), // Default password }, From b6b550f3fa1f32a80c8ad8af70b2f4b415f83b89 Mon Sep 17 00:00:00 2001 From: Aasif Khan Date: Sat, 26 Oct 2024 08:56:23 +0530 Subject: [PATCH 20/22] run linter --- internal/middleware/ratelimiter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/middleware/ratelimiter.go b/internal/middleware/ratelimiter.go index ff43da8..ed909ba 100644 --- a/internal/middleware/ratelimiter.go +++ b/internal/middleware/ratelimiter.go @@ -4,16 +4,16 @@ import ( "context" "errors" "fmt" + "github.com/dicedb/dicedb-go" "log/slog" "net/http" + "server/config" "server/internal/db" "server/internal/server/utils" mock "server/internal/tests/dbmocks" "strconv" "strings" "time" - "server/config" - "github.com/dicedb/dicedb-go" ) // RateLimiter middleware to limit requests based on a specified limit and duration From 3a097bc8943e4bdafacf99d27d70466567d6c87f Mon Sep 17 00:00:00 2001 From: Aasif Khan Date: Sat, 26 Oct 2024 09:24:13 +0530 Subject: [PATCH 21/22] fix linter --- internal/middleware/ratelimiter.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/middleware/ratelimiter.go b/internal/middleware/ratelimiter.go index ed909ba..bf0edfd 100644 --- a/internal/middleware/ratelimiter.go +++ b/internal/middleware/ratelimiter.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "github.com/dicedb/dicedb-go" "log/slog" "net/http" "server/config" @@ -14,6 +13,8 @@ import ( "strconv" "strings" "time" + + "github.com/dicedb/dicedb-go" ) // RateLimiter middleware to limit requests based on a specified limit and duration @@ -80,7 +81,7 @@ func RateLimiter(client *db.DiceDB, next http.Handler, limit int64, window float } } - secondsDifference, err := calculateNextCleanupTime(client, ctx, cronFrequencyInterval) + secondsDifference, err := calculateNextCleanupTime(ctx, client, cronFrequencyInterval) if err != nil { slog.Error("Error calculating next cleanup time", "error", err) } @@ -93,7 +94,7 @@ func RateLimiter(client *db.DiceDB, next http.Handler, limit int64, window float }) } -func calculateNextCleanupTime(client *db.DiceDB, ctx context.Context, cronFrequencyInterval time.Duration) (int64, error) { +func calculateNextCleanupTime(ctx context.Context, client *db.DiceDB, cronFrequencyInterval time.Duration) (int64, error) { var lastCronCleanupTime int64 resp := client.Client.Get(ctx, utils.LastCronCleanupTimeUnixMs) if resp.Err() != nil && !errors.Is(resp.Err(), dicedb.Nil) { @@ -114,7 +115,6 @@ func calculateNextCleanupTime(client *db.DiceDB, ctx context.Context, cronFreque return int64(timeDifference.Seconds()), nil } - func MockRateLimiter(client *mock.DiceDBMock, next http.Handler, limit int64, window float64) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if handleCors(w, r) { From cf1bcf4a96bdff43defca4fbfeacaa1f45c43276 Mon Sep 17 00:00:00 2001 From: aasifkhan7 Date: Sat, 26 Oct 2024 12:48:49 +0530 Subject: [PATCH 22/22] Update ratelimiter.go --- internal/middleware/ratelimiter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/middleware/ratelimiter.go b/internal/middleware/ratelimiter.go index bf0edfd..f8d3c52 100644 --- a/internal/middleware/ratelimiter.go +++ b/internal/middleware/ratelimiter.go @@ -98,14 +98,14 @@ func calculateNextCleanupTime(ctx context.Context, client *db.DiceDB, cronFreque var lastCronCleanupTime int64 resp := client.Client.Get(ctx, utils.LastCronCleanupTimeUnixMs) if resp.Err() != nil && !errors.Is(resp.Err(), dicedb.Nil) { - return 0, resp.Err() + return -1, resp.Err() } if resp.Val() != "" { var err error lastCronCleanupTime, err = strconv.ParseInt(resp.Val(), 10, 64) // directly assign here if err != nil { - return 0, err + return -1, err } }