Implement local cache for crowdsec

.env

@@ -1,2 +1,2 @@
-CROWDSEC_BOUNCER_API_KEY=40796d93c2958f9e58345514e67740e5
+CROWDSEC_BOUNCER_API_KEY=FIXME
 CROWDSEC_AGENT_HOST=localhost:8083

.gitignore

@@ -22,4 +22,7 @@ tmp/
 .idea/
 
 !.idea/runConfigurations/
-!.idea/rest-api.http
+!.idea/rest-api.http
+
+# Dev docker-compose
+docker-compose-dev.yaml

README.md

@@ -11,8 +11,16 @@ A http service to verify request and bounce them according to decisions made by
 
 # Description
 This repository aim to implement a [CrowdSec](https://doc.crowdsec.net/) bouncer for the router [Traefik](https://doc.traefik.io/traefik/) to block malicious IP to access your services.
+
+It can operate with 2 modes:
+- Request
 For this it leverages [Traefik v2 ForwardAuth middleware](https://doc.traefik.io/traefik/middlewares/http/forwardauth/) and query CrowdSec with client IP.
 If the client IP is on ban list, it will get a http code 403 response. Otherwise, request will continue as usual.
+The bouncer can leverage use of a [local cache](https://github.com/patrickmn/go-cache) in order to reduce the number of requests made to the crowdsec local API. It will keep in cache the status for each IP that makes queries.
+
+- Stream
+Streaming mode allows you to keep in the local cache only the Banned IPs, every requests that does not hit the cache is authorized. Every minute, the cache is updated with news from the Local API using [go-cron](https://github.com/robfig/cron) library.  
+It is the recommanded and most perfomant mode, (enabled by default)
 
 # Demo
 ## Prerequisites 
@@ -60,6 +68,10 @@ The webservice configuration is made via environment variables:
 * `CROWDSEC_BOUNCER_LOG_LEVEL`          - Minimum log level for bouncer. Expected value [zerolog levels](https://pkg.go.dev/github.com/rs/zerolog#readme-leveled-logging). Default to 1
 * `CROWDSEC_BOUNCER_BAN_RESPONSE_CODE`  - HTTP code to respond in case of ban. Default to 403
 * `CROWDSEC_BOUNCER_BAN_RESPONSE_MSG`   - HTTP body as message to respond in case of ban. Default to Forbidden
+* `CROWDSEC_BOUNCER_ENABLE_LOCAL_CACHE` - Configure the use of a local cache in memory. Default to false
+* `CROWDSEC_DEFAULT_CACHE_DURATION`     - Configure default duration of the cached data. Default to "15m00s"
+* `CROWDSEC_LAPI_ENABLE_STREAM_MODE`    - Enable streaming mode to pull decisions from the LAPI. Will override CROWDSEC_BOUNCER_ENABLE_LOCAL_CACHE and enable it. Default to "true"
+* `CROWDSEC_LAPI_STREAM_MODE_INTERVAL`  - Define the interval between two calls to LAPI. Default to "1m"
 * `PORT`                                - Change listening port of web server. Default listen on 8080
 * `GIN_MODE`                            - By default, run app in "debug" mode. Set it to "release" in production
 * `TRUSTED_PROXIES`                     - List of trusted proxies IP addresses in CIDR format, delimited by ','. Default of 0.0.0.0/0 should be fine for most use cases, but you HAVE to add them directly in traefik. 
@@ -82,3 +94,5 @@ Any constructive feedback is welcome, fill free to add an issue or a pull reques
 4. In `_test.env` replace `<your_generated_api_key>` with the previously generated key
 5. Adding a banned IP to your crodwsec instance with : `docker exec traefik-crowdsec-bouncer-crowdsec-1 cscli decisions add -i 1.2.3.4`
 6. Run test with `godotenv -f ./_test.env go test -cover`
+
+NB: Be aware that you cannot use network_mode: host with Docker Desktop on Windows. It is used in the docker-compose.yaml file for the traefik container to be able to contact a local instance of the bouncer through localhost

bouncer.go

@@ -1,19 +1,30 @@
 package main
 
 import (
+	"fmt"
 	"os"
+	"time"
+
+	"strings"
 
 	. "github.com/fbonalair/traefik-crowdsec-bouncer/config"
 	"github.com/fbonalair/traefik-crowdsec-bouncer/controler"
 	"github.com/gin-contrib/logger"
 	"github.com/gin-gonic/gin"
+	"github.com/patrickmn/go-cache"
+	"github.com/robfig/cron/v3"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
-	"strings"
 )
 
 var logLevel = OptionalEnv("CROWDSEC_BOUNCER_LOG_LEVEL", "1")
 var trustedProxiesList = strings.Split(OptionalEnv("TRUSTED_PROXIES", "0.0.0.0/0"), ",")
+var crowdsecDefaultCacheDuration = OptionalEnv("CROWDSEC_BOUNCER_DEFAULT_CACHE_DURATION", "15m00s")
+var crowdsecDefaultStreamModeInterval = OptionalEnv("CROWDSEC_LAPI_STREAM_MODE_INTERVAL", "1m")
+var crowdsecEnableLocalCache = OptionalEnv("CROWDSEC_BOUNCER_ENABLE_LOCAL_CACHE", "false")
+var crowdsecEnableStreamMode = OptionalEnv("CROWDSEC_LAPI_ENABLE_STREAM_MODE", "true")
+var cr *cron.Cron
+var lc *cache.Cache
 
 func main() {
 	ValidateEnv()
@@ -31,6 +42,61 @@ func main() {
 
 }
 
+func cacheMiddleware(lc *cache.Cache) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Set("lc", lc)
+		c.Next()
+	}
+}
+
+func cronMiddleware(cr *cron.Cron) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Set("cr", cr)
+		c.Next()
+	}
+}
+
+func setupCacheStream() {
+	// local go-cache and streaming mode
+	if crowdsecEnableLocalCache == "true" || crowdsecEnableStreamMode == "true" {
+		duration, err := time.ParseDuration(crowdsecDefaultCacheDuration)
+		if err != nil {
+			log.Warn().Msg("Duration provided is not valid, defaulting to 15m00s")
+			duration, _ = time.ParseDuration("15m")
+		}
+		lc = cache.New(duration, 5*time.Minute)
+		if crowdsecEnableStreamMode == "true" {
+			duration, err := time.ParseDuration(crowdsecDefaultStreamModeInterval)
+			var strD string
+			if err != nil {
+				log.Warn().Msg("Duration provided is not valid, defaulting to 1m")
+				duration, _ = time.ParseDuration("1m")
+				strD = duration.String()
+				strD = fmt.Sprintf("@every %v", strD)
+			} else {
+				strD = duration.String()
+				strD = fmt.Sprintf("@every %v", strD)
+			}
+			go func() {
+				log.Debug().Msg("Streaming mode enabled")
+				cr = cron.New()
+				cr.Start()
+				cr.AddFunc(strD, func() {
+					controler.CallLAPIStream(lc, false)
+				})
+				log.Debug().Msg("Start polling initial stream")
+				controler.CallLAPIStream(lc, true)
+				log.Debug().Msg("Finish polling initial stream")
+			}()
+		} else {
+			cr = nil
+		}
+
+	} else {
+		lc = nil
+		cr = nil
+	}
+}
 func setupRouter() (*gin.Engine, error) {
 	// logger framework
 	if gin.IsDebugging() {
@@ -49,6 +115,8 @@ func setupRouter() (*gin.Engine, error) {
 	}
 	zerolog.SetGlobalLevel(level)
 
+	setupCacheStream()
+
 	// Web framework
 	router := gin.New()
 	err = router.SetTrustedProxies(trustedProxiesList)
@@ -58,6 +126,8 @@ func setupRouter() (*gin.Engine, error) {
 	router.Use(logger.SetLogger(
 		logger.WithSkipPath([]string{"/api/v1/ping", "/api/v1/healthz"}),
 	))
+	router.Use(cacheMiddleware(lc))
+	router.Use(cronMiddleware(cr))
 	router.GET("/api/v1/ping", controler.Ping)
 	router.GET("/api/v1/healthz", controler.Healthz)
 	router.GET("/api/v1/forwardAuth", controler.ForwardAuth)