Adding deletion for events after retention period

chart/templates/ingestor-configmap.yaml

@@ -10,8 +10,9 @@ data:
       host: {{ .Values.postgres.host }}
       port: {{ .Values.postgres.port }}
       dbname: {{ .Values.postgres.dbname }}
+      retentionDays: {{ .Values.postgres.retentionDays }}
 
     server:
       port: {{ .Values.server.port }}
     healthz:
-      port: 9999
+      port: 9999

chart/values.yaml

@@ -18,6 +18,7 @@ postgres:
   port: 5432
   dbname: falco
   password:
+  retentionDays: 365
 
 server:
   port: 3200
@@ -27,4 +28,4 @@ resources:
     memory: 400Mi
   requests:
     cpu: 10m
-    memory: 32Mi
+    memory: 32Mi

cmd/ingestor/main.go

@@ -51,6 +51,7 @@ func initConfig(configFile string, verificationKey string, postgresPassword stri
 		viper.GetString("postgres.host"),
 		viper.GetInt("postgres.port"),
 		viper.GetString("postgres.dbname"),
+		viper.GetInt("postgres.retentionDays"),
 	)
 	return postgresConfig, validator
 }

go.mod

@@ -7,6 +7,7 @@ toolchain go1.22.3
 require (
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/golang-jwt/jwt/v5 v5.2.1
+	github.com/huandu/go-sqlbuilder v1.30.0
 	github.com/lib/pq v1.10.9
 	github.com/prometheus/client_golang v1.19.1
 	github.com/sirupsen/logrus v1.9.3
@@ -20,6 +21,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/huandu/xstrings v1.4.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.0 // indirect

go.sum

@@ -18,6 +18,12 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/huandu/go-assert v1.1.6 h1:oaAfYxq9KNDi9qswn/6aE0EydfxSa+tWZC1KabNitYs=
+github.com/huandu/go-assert v1.1.6/go.mod h1:JuIfbmYG9ykwvuxoJ3V8TB5QP+3+ajIA54Y44TmkMxs=
+github.com/huandu/go-sqlbuilder v1.30.0 h1:d4QiUZrex3WQjsgyhyKYEsnr9v1msBSQRbQJl4I7mT4=
+github.com/huandu/go-sqlbuilder v1.30.0/go.mod h1:mS0GAtrtW+XL6nM2/gXHRJax2RwSW1TraavWDFAc1JA=
+github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU=
+github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
 github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=

pkg/postgres/postgres.go

@@ -14,6 +14,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/huandu/go-sqlbuilder"
 	_ "github.com/lib/pq"
 	log "github.com/sirupsen/logrus"
 )
@@ -30,13 +31,14 @@ type ClusterIdentity struct {
 }
 
 type PostgresConfig struct {
-	user     string
-	password string
-	host     string
-	port     int
-	dbname   string
-	db       *sql.DB
-	stmt     *sql.Stmt
+	user              string
+	password          string
+	host              string
+	port              int
+	dbname            string
+	db                *sql.DB
+	stmt              *sql.Stmt
+	retentionDuration time.Duration
 }
 
 type EventStruct struct {
@@ -51,10 +53,15 @@ type EventStruct struct {
 	Hostname     string                     `json:"hostname"`
 }
 
-func NewPostgresConfig(user, password, host string, port int, dbname string) *PostgresConfig {
+func NewPostgresConfig(user, password, host string, port int, dbname string, retentionDays int) *PostgresConfig {
 	connStr := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s", host, port, user, password, dbname)
 	log.Infof("Trying connection: host=%s port=%d user=%s password=%s dbname=%s", host, port, user, "******", dbname)
 
+	retentionDuration, err := time.ParseDuration(fmt.Sprintf("%dh", retentionDays*24))
+	if err != nil {
+		log.Fatalf("Could not parse event retention days %v", err)
+	}
+
 	db, err := sql.Open("postgres", connStr)
 	if err != nil {
 		log.Fatal(err)
@@ -75,15 +82,20 @@ func NewPostgresConfig(user, password, host string, port int, dbname string) *Po
 
 	log.Info("Connection to database succeded")
 
-	return &PostgresConfig{
-		user:     user,
-		password: password,
-		host:     host,
-		port:     port,
-		dbname:   dbname,
-		db:       db,
-		stmt:     stmt,
+	postgresConfigInstance := PostgresConfig{
+		user:              user,
+		password:          password,
+		host:              host,
+		port:              port,
+		dbname:            dbname,
+		db:                db,
+		stmt:              stmt,
+		retentionDuration: retentionDuration,
 	}
+
+	go postgresConfigInstance.DeleteLoop(time.Second * 60)
+
+	return &postgresConfigInstance
 }
 
 func (c *PostgresConfig) SetPassword(password string) {
@@ -122,6 +134,23 @@ func prepareInsert(db *sql.DB) (*sql.Stmt, error) {
 	return stmt, nil
 }
 
+func (pgconf *PostgresConfig) DeleteLoop(frequency time.Duration) {
+	for {
+		sql, args := buildDeleteStatement(pgconf.retentionDuration)
+
+		// --------------------------- DO NOT ENABLE YET----------------------------
+		fmt.Println(sql)
+		fmt.Println(args...)
+		// _, err := pgconf.db.Query(sql, args...)
+		// if err != nil {
+		// 	log.Errorf("Delete query failed: %v", err)
+		// }
+		// --------------------------- DO NOT ENABLE YET----------------------------
+
+		time.Sleep(frequency)
+	}
+}
+
 func (pgconf *PostgresConfig) Insert(event *EventStruct) {
 	clusterIdentity, err := parseClusterId(event)
 	if err != nil {
@@ -160,3 +189,14 @@ func (pgconf *PostgresConfig) CheckHealth() error {
 
 	return nil
 }
+
+func buildDeleteStatement(maxAge time.Duration) (string, []interface{}) {
+	maxTime := time.Now().UTC().Add(-maxAge)
+
+	sb := sqlbuilder.PostgreSQL.NewDeleteBuilder()
+	sb.DeleteFrom("falco_events")
+	sb.Where(sb.LessThan("time", maxTime))
+
+	sql, args := sb.Build()
+	return sql, args
+}