From 8049e94b4e2e860988265a914b7673952ee53547 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?= <31509435+aleksasiriski@users.noreply.github.com> Date: Tue, 28 May 2024 20:25:33 +0200 Subject: [PATCH 1/6] fix(cache): switch from CBOR to JSON --- src/cache/badger/badger.go | 6 +++--- src/cache/redis/redis.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/cache/badger/badger.go b/src/cache/badger/badger.go index 6e7921d3..24f304f4 100644 --- a/src/cache/badger/badger.go +++ b/src/cache/badger/badger.go @@ -1,12 +1,12 @@ package badger import ( + "encoding/json" "fmt" "path" "time" "github.com/dgraph-io/badger/v4" - "github.com/fxamacker/cbor/v2" "github.com/hearchco/hearchco/src/anonymize" "github.com/hearchco/hearchco/src/config" "github.com/rs/zerolog/log" @@ -67,7 +67,7 @@ func (db DB) Set(k string, v interface{}, ttl ...time.Duration) error { setTtl = ttl[0] } - if val, err := cbor.Marshal(v); err != nil { + if val, err := json.Marshal(v); err != nil { return fmt.Errorf("badger.Set(): error marshaling value: %w", err) } else if err := db.bdb.Update(func(txn *badger.Txn) error { var e *badger.Entry @@ -116,7 +116,7 @@ func (db DB) Get(k string, o interface{}, hashed ...bool) error { Msg("Found no value in badger") } else if err != nil { return fmt.Errorf("badger.Get(): error getting value from badger for key %v: %w", kInput, err) - } else if err := cbor.Unmarshal(val, o); err != nil { + } else if err := json.Unmarshal(val, o); err != nil { return fmt.Errorf("badger.Get(): failed unmarshaling value from badger for key %v: %w", kInput, err) } diff --git a/src/cache/redis/redis.go b/src/cache/redis/redis.go index 3b102a45..ea87c8cb 100644 --- a/src/cache/redis/redis.go +++ b/src/cache/redis/redis.go @@ -2,10 +2,10 @@ package redis import ( "context" + "encoding/json" "fmt" "time" - "github.com/fxamacker/cbor/v2" "github.com/hearchco/hearchco/src/anonymize" "github.com/hearchco/hearchco/src/config" "github.com/redis/go-redis/v9" @@ -57,7 +57,7 @@ func (db DB) Set(k string, v interface{}, ttl ...time.Duration) error { setTtl = ttl[0] } - if val, err := cbor.Marshal(v); err != nil { + if val, err := json.Marshal(v); err != nil { return fmt.Errorf("redis.Set(): error marshaling value: %w", err) } else if err := db.rdb.Set(db.ctx, anonymize.HashToSHA256B64(k), val, setTtl).Err(); err != nil { return fmt.Errorf("redis.Set(): error setting KV to redis: %w", err) @@ -85,7 +85,7 @@ func (db DB) Get(k string, o interface{}, hashed ...bool) error { Msg("Found no value in redis") } else if err != nil { return fmt.Errorf("redis.Get(): error getting value from redis for key %v: %w", kInput, err) - } else if err := cbor.Unmarshal([]byte(val), o); err != nil { + } else if err := json.Unmarshal([]byte(val), o); err != nil { return fmt.Errorf("redis.Get(): failed unmarshaling value from redis for key %v: %w", kInput, err) } From 4bdd165096b2e6b75724258310d6eecd993a26fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?= <31509435+aleksasiriski@users.noreply.github.com> Date: Tue, 28 May 2024 20:46:52 +0200 Subject: [PATCH 2/6] fix(cache): better interfacing --- src/cache/actions_results.go | 25 +++++++++++++ src/cache/badger/badger.go | 61 ++++++++++++++------------------ src/cache/badger/badger_test.go | 28 +++++++-------- src/cache/interfaces.go | 12 ++++--- src/cache/nocache/nocache.go | 12 +++---- src/cache/redis/redis.go | 56 +++++++++++++---------------- src/cache/redis/redis_test.go | 14 ++++---- src/cache/{new.go => structs.go} | 18 +++++++--- src/cache/utils.go | 15 ++++++++ src/config/structs_server.go | 4 +++ src/search/cache.go | 6 ++-- src/search/search.go | 34 ++++++++---------- 12 files changed, 158 insertions(+), 127 deletions(-) create mode 100644 src/cache/actions_results.go rename src/cache/{new.go => structs.go} (72%) create mode 100644 src/cache/utils.go diff --git a/src/cache/actions_results.go b/src/cache/actions_results.go new file mode 100644 index 00000000..cffbf3b2 --- /dev/null +++ b/src/cache/actions_results.go @@ -0,0 +1,25 @@ +package cache + +import ( + "time" + + "github.com/hearchco/hearchco/src/search/category" + "github.com/hearchco/hearchco/src/search/result" +) + +func (db DB) SetResults(query string, category category.Name, results []result.Result, ttl ...time.Duration) error { + key := combineIntoKey(query, category.String()) + return db.driver.Set(key, results, ttl...) +} + +func (db DB) GetResults(query string, category category.Name) ([]result.Result, error) { + key := combineIntoKey(query, category.String()) + var results []result.Result + err := db.driver.Get(key, &results) + return results, err +} + +func (db DB) GetResultsTTL(query string, category category.Name) (time.Duration, error) { + key := combineIntoKey(query, category.String()) + return db.driver.GetTTL(key) +} diff --git a/src/cache/badger/badger.go b/src/cache/badger/badger.go index 24f304f4..95d0af60 100644 --- a/src/cache/badger/badger.go +++ b/src/cache/badger/badger.go @@ -12,11 +12,12 @@ import ( "github.com/rs/zerolog/log" ) -type DB struct { - bdb *badger.DB +type DRV struct { + keyPrefix string + client *badger.DB } -func New(dataDirPath string, config config.Badger) (DB, error) { +func New(dataDirPath string, keyPrefix string, config config.Badger) (DRV, error) { badgerPath := path.Join(dataDirPath, "database") var opt badger.Options @@ -26,8 +27,7 @@ func New(dataDirPath string, config config.Badger) (DB, error) { opt = badger.DefaultOptions("").WithInMemory(true).WithLoggingLevel(badger.WARNING) } - bdb, err := badger.Open(opt) - + client, err := badger.Open(opt) if err != nil { log.Error(). Err(err). @@ -45,11 +45,11 @@ func New(dataDirPath string, config config.Badger) (DB, error) { Msg("Successfully opened in-memory badger") } - return DB{bdb: bdb}, err + return DRV{keyPrefix, client}, err } -func (db DB) Close() { - if err := db.bdb.Close(); err != nil { +func (drv DRV) Close() { + if err := drv.client.Close(); err != nil { log.Error(). Err(err). Msg("Error closing badger") @@ -58,7 +58,7 @@ func (db DB) Close() { } } -func (db DB) Set(k string, v interface{}, ttl ...time.Duration) error { +func (drv DRV) Set(k string, v any, ttl ...time.Duration) error { log.Debug().Msg("Caching...") cacheTimer := time.Now() @@ -67,14 +67,15 @@ func (db DB) Set(k string, v interface{}, ttl ...time.Duration) error { setTtl = ttl[0] } + key := fmt.Sprintf("%v%v", drv.keyPrefix, k) if val, err := json.Marshal(v); err != nil { return fmt.Errorf("badger.Set(): error marshaling value: %w", err) - } else if err := db.bdb.Update(func(txn *badger.Txn) error { + } else if err := drv.client.Update(func(txn *badger.Txn) error { var e *badger.Entry if setTtl != 0 { - e = badger.NewEntry([]byte(anonymize.HashToSHA256B64(k)), val).WithTTL(ttl[0]) + e = badger.NewEntry([]byte(key), val).WithTTL(ttl[0]) } else { - e = badger.NewEntry([]byte(anonymize.HashToSHA256B64(k)), val) + e = badger.NewEntry([]byte(key), val) } return txn.SetEntry(e) // ^returns error into else if @@ -89,17 +90,12 @@ func (db DB) Set(k string, v interface{}, ttl ...time.Duration) error { return nil } -func (db DB) Get(k string, o interface{}, hashed ...bool) error { - var kInput string - if len(hashed) > 0 && hashed[0] { - kInput = k - } else { - kInput = anonymize.HashToSHA256B64(k) - } +func (drv DRV) Get(k string, o any) error { + key := anonymize.HashToSHA256B64(fmt.Sprintf("%v%v", drv.keyPrefix, k)) var val []byte - err := db.bdb.View(func(txn *badger.Txn) error { - item, err := txn.Get([]byte(kInput)) + err := drv.client.View(func(txn *badger.Txn) error { + item, err := txn.Get([]byte(key)) if err != nil { return err } @@ -112,29 +108,24 @@ func (db DB) Get(k string, o interface{}, hashed ...bool) error { if err == badger.ErrKeyNotFound { log.Trace(). - Str("key", kInput). + Str("key", key). Msg("Found no value in badger") } else if err != nil { - return fmt.Errorf("badger.Get(): error getting value from badger for key %v: %w", kInput, err) + return fmt.Errorf("badger.Get(): error getting value from badger for key %v: %w", key, err) } else if err := json.Unmarshal(val, o); err != nil { - return fmt.Errorf("badger.Get(): failed unmarshaling value from badger for key %v: %w", kInput, err) + return fmt.Errorf("badger.Get(): failed unmarshaling value from badger for key %v: %w", key, err) } return nil } // returns time until the key expires, not the time it will be considered expired -func (db DB) GetTTL(k string, hashed ...bool) (time.Duration, error) { - var kInput string - if len(hashed) > 0 && hashed[0] { - kInput = k - } else { - kInput = anonymize.HashToSHA256B64(k) - } +func (drv DRV) GetTTL(k string) (time.Duration, error) { + key := anonymize.HashToSHA256B64(fmt.Sprintf("%v%v", drv.keyPrefix, k)) var expiresIn time.Duration - err := db.bdb.View(func(txn *badger.Txn) error { - item, err := txn.Get([]byte(kInput)) + err := drv.client.View(func(txn *badger.Txn) error { + item, err := txn.Get([]byte(key)) if err != nil { return err } @@ -152,10 +143,10 @@ func (db DB) GetTTL(k string, hashed ...bool) (time.Duration, error) { if err == badger.ErrKeyNotFound { log.Trace(). - Str("key", kInput). + Str("key", key). Msg("Found no value in badger") } else if err != nil { - return expiresIn, fmt.Errorf("badger.Get(): error getting value from badger for key %v: %w", kInput, err) + return expiresIn, fmt.Errorf("badger.Get(): error getting value from badger for key %v: %w", key, err) } return expiresIn, nil diff --git a/src/cache/badger/badger_test.go b/src/cache/badger/badger_test.go index 1732684b..29a7da22 100644 --- a/src/cache/badger/badger_test.go +++ b/src/cache/badger/badger_test.go @@ -10,7 +10,7 @@ import ( ) func TestNewInMemory(t *testing.T) { - _, err := badger.New("", config.Badger{Persist: false}) + _, err := badger.New("", "", config.Badger{Persist: false}) if err != nil { t.Errorf("error opening in-memory badger: %v", err) } @@ -18,14 +18,14 @@ func TestNewInMemory(t *testing.T) { func TestNewPersistence(t *testing.T) { path := "./testdump/new" - _, err := badger.New(path, config.Badger{Persist: true}) + _, err := badger.New(path, "", config.Badger{Persist: true}) if err != nil { t.Errorf("error opening badger at %v: %v", path, err) } } func TestCloseInMemory(t *testing.T) { - db, err := badger.New("", config.Badger{Persist: false}) + db, err := badger.New("", "", config.Badger{Persist: false}) if err != nil { t.Errorf("error opening in-memory badger: %v", err) } @@ -35,7 +35,7 @@ func TestCloseInMemory(t *testing.T) { func TestClosePersistence(t *testing.T) { path := "./testdump/close" - db, err := badger.New(path, config.Badger{Persist: true}) + db, err := badger.New(path, "", config.Badger{Persist: true}) if err != nil { t.Errorf("error opening badger at %v: %v", path, err) } @@ -44,7 +44,7 @@ func TestClosePersistence(t *testing.T) { } func TestSetInMemory(t *testing.T) { - db, err := badger.New("", config.Badger{Persist: false}) + db, err := badger.New("", "", config.Badger{Persist: false}) if err != nil { t.Errorf("error opening in-memory badger: %v", err) } @@ -59,7 +59,7 @@ func TestSetInMemory(t *testing.T) { func TestSetPersistence(t *testing.T) { path := "./testdump/set" - db, err := badger.New(path, config.Badger{Persist: true}) + db, err := badger.New(path, "", config.Badger{Persist: true}) if err != nil { t.Errorf("error opening badger at %v: %v", path, err) } @@ -73,7 +73,7 @@ func TestSetPersistence(t *testing.T) { } func TestSetTTLInMemory(t *testing.T) { - db, err := badger.New("", config.Badger{Persist: false}) + db, err := badger.New("", "", config.Badger{Persist: false}) if err != nil { t.Errorf("error opening in-memory badger: %v", err) } @@ -88,7 +88,7 @@ func TestSetTTLInMemory(t *testing.T) { func TestSetTTLPersistence(t *testing.T) { path := "./testdump/setttl" - db, err := badger.New(path, config.Badger{Persist: true}) + db, err := badger.New(path, "", config.Badger{Persist: true}) if err != nil { t.Errorf("error opening badger at %v: %v", path, err) } @@ -102,7 +102,7 @@ func TestSetTTLPersistence(t *testing.T) { } func TestGetInMemory(t *testing.T) { - db, err := badger.New("", config.Badger{Persist: false}) + db, err := badger.New("", "", config.Badger{Persist: false}) if err != nil { t.Errorf("error opening in-memory badger: %v", err) } @@ -127,7 +127,7 @@ func TestGetInMemory(t *testing.T) { func TestGetPersistence(t *testing.T) { path := "./testdump/get" - db, err := badger.New(path, config.Badger{Persist: true}) + db, err := badger.New(path, "", config.Badger{Persist: true}) if err != nil { t.Errorf("error opening badger at %v: %v", path, err) } @@ -151,7 +151,7 @@ func TestGetPersistence(t *testing.T) { } func TestGetTTLInMemory(t *testing.T) { - db, err := badger.New("", config.Badger{Persist: false}) + db, err := badger.New("", "", config.Badger{Persist: false}) if err != nil { t.Errorf("error opening in-memory badger: %v", err) } @@ -176,7 +176,7 @@ func TestGetTTLInMemory(t *testing.T) { func TestGetTTLPersistence(t *testing.T) { path := "./testdump/getttl" - db, err := badger.New(path, config.Badger{Persist: true}) + db, err := badger.New(path, "", config.Badger{Persist: true}) if err != nil { t.Errorf("error opening badger at %v: %v", path, err) } @@ -200,7 +200,7 @@ func TestGetTTLPersistence(t *testing.T) { } func TestGetInMemoryExpired(t *testing.T) { - db, err := badger.New("", config.Badger{Persist: false}) + db, err := badger.New("", "", config.Badger{Persist: false}) if err != nil { t.Errorf("error opening in-memory badger: %v", err) } @@ -223,7 +223,7 @@ func TestGetInMemoryExpired(t *testing.T) { func TestGetPersistenceExpired(t *testing.T) { path := "./testdump/getexpired" - db, err := badger.New(path, config.Badger{Persist: true}) + db, err := badger.New(path, "", config.Badger{Persist: true}) if err != nil { t.Errorf("error opening badger at %v: %v", path, err) } diff --git a/src/cache/interfaces.go b/src/cache/interfaces.go index e547ea01..c471fd79 100644 --- a/src/cache/interfaces.go +++ b/src/cache/interfaces.go @@ -1,10 +1,12 @@ package cache -import "time" +import ( + "time" +) -type DB interface { +type Driver interface { Close() - Set(k string, v interface{}, ttl ...time.Duration) error - Get(k string, o interface{}, hashed ...bool) error - GetTTL(k string, hashed ...bool) (time.Duration, error) + Set(k string, v any, ttl ...time.Duration) error + Get(k string, o any) error + GetTTL(k string) (time.Duration, error) } diff --git a/src/cache/nocache/nocache.go b/src/cache/nocache/nocache.go index b59b8a18..aad5b3ea 100644 --- a/src/cache/nocache/nocache.go +++ b/src/cache/nocache/nocache.go @@ -4,14 +4,14 @@ import ( "time" ) -type DB struct{} +type DRV struct{} -func New() (DB, error) { return DB{}, nil } +func New() (DRV, error) { return DRV{}, nil } -func (db DB) Close() {} +func (drv DRV) Close() {} -func (db DB) Set(k string, v interface{}, ttl ...time.Duration) error { return nil } +func (drv DRV) Set(k string, v any, ttl ...time.Duration) error { return nil } -func (db DB) Get(k string, o interface{}, hashed ...bool) error { return nil } +func (drv DRV) Get(k string, o any) error { return nil } -func (db DB) GetTTL(k string, hashed ...bool) (time.Duration, error) { return 0, nil } +func (drv DRV) GetTTL(k string) (time.Duration, error) { return 0, nil } diff --git a/src/cache/redis/redis.go b/src/cache/redis/redis.go index ea87c8cb..f462c961 100644 --- a/src/cache/redis/redis.go +++ b/src/cache/redis/redis.go @@ -12,19 +12,21 @@ import ( "github.com/rs/zerolog/log" ) -type DB struct { - ctx context.Context - rdb *redis.Client +type DRV struct { + ctx context.Context + keyPrefix string + client *redis.Client } -func New(ctx context.Context, config config.Redis) (DB, error) { - rdb := redis.NewClient(&redis.Options{ +func New(ctx context.Context, keyPrefix string, config config.Redis) (DRV, error) { + client := redis.NewClient(&redis.Options{ Addr: fmt.Sprintf("%v:%v", config.Host, config.Port), Password: config.Password, DB: int(config.Database), }) - if err := rdb.Ping(ctx).Err(); err != nil { + err := client.Ping(ctx).Err() + if err != nil { log.Error(). Err(err). Str("address", fmt.Sprintf("%v:%v/%v", config.Host, config.Port, config.Database)). @@ -35,11 +37,11 @@ func New(ctx context.Context, config config.Redis) (DB, error) { Msg("Successfully connected to redis") } - return DB{rdb: rdb, ctx: ctx}, nil + return DRV{ctx, keyPrefix, client}, err } -func (db DB) Close() { - if err := db.rdb.Close(); err != nil { +func (drv DRV) Close() { + if err := drv.client.Close(); err != nil { log.Error(). Err(err). Msg("Error closing connection to redis") @@ -48,7 +50,7 @@ func (db DB) Close() { } } -func (db DB) Set(k string, v interface{}, ttl ...time.Duration) error { +func (drv DRV) Set(k string, v any, ttl ...time.Duration) error { log.Debug().Msg("Caching...") cacheTimer := time.Now() @@ -59,7 +61,7 @@ func (db DB) Set(k string, v interface{}, ttl ...time.Duration) error { if val, err := json.Marshal(v); err != nil { return fmt.Errorf("redis.Set(): error marshaling value: %w", err) - } else if err := db.rdb.Set(db.ctx, anonymize.HashToSHA256B64(k), val, setTtl).Err(); err != nil { + } else if err := drv.client.Set(drv.ctx, anonymize.HashToSHA256B64(k), val, setTtl).Err(); err != nil { return fmt.Errorf("redis.Set(): error setting KV to redis: %w", err) } else { log.Trace(). @@ -70,45 +72,35 @@ func (db DB) Set(k string, v interface{}, ttl ...time.Duration) error { return nil } -func (db DB) Get(k string, o interface{}, hashed ...bool) error { - var kInput string - if len(hashed) > 0 && hashed[0] { - kInput = k - } else { - kInput = anonymize.HashToSHA256B64(k) - } +func (drv DRV) Get(k string, o any) error { + key := anonymize.HashToSHA256B64(k) - val, err := db.rdb.Get(db.ctx, kInput).Result() + val, err := drv.client.Get(drv.ctx, key).Result() if err == redis.Nil { log.Trace(). - Str("key", kInput). + Str("key", key). Msg("Found no value in redis") } else if err != nil { - return fmt.Errorf("redis.Get(): error getting value from redis for key %v: %w", kInput, err) + return fmt.Errorf("redis.Get(): error getting value from redis for key %v: %w", key, err) } else if err := json.Unmarshal([]byte(val), o); err != nil { - return fmt.Errorf("redis.Get(): failed unmarshaling value from redis for key %v: %w", kInput, err) + return fmt.Errorf("redis.Get(): failed unmarshaling value from redis for key %v: %w", key, err) } return nil } // returns time until the key expires, not the time it will be considered expired -func (db DB) GetTTL(k string, hashed ...bool) (time.Duration, error) { - var kInput string - if len(hashed) > 0 && hashed[0] { - kInput = k - } else { - kInput = anonymize.HashToSHA256B64(k) - } +func (drv DRV) GetTTL(k string) (time.Duration, error) { + key := anonymize.HashToSHA256B64(k) // returns time with time.Second precision - expiresIn, err := db.rdb.TTL(db.ctx, kInput).Result() + expiresIn, err := drv.client.TTL(drv.ctx, key).Result() if err == redis.Nil { log.Trace(). - Str("key", kInput). + Str("key", key). Msg("Found no value in redis") } else if err != nil { - return expiresIn, fmt.Errorf("redis.Get(): error getting value from redis for key %v: %w", kInput, err) + return expiresIn, fmt.Errorf("redis.Get(): error getting value from redis for key %v: %w", key, err) } /* diff --git a/src/cache/redis/redis_test.go b/src/cache/redis/redis_test.go index 9a90f8b8..44046f47 100644 --- a/src/cache/redis/redis_test.go +++ b/src/cache/redis/redis_test.go @@ -57,7 +57,7 @@ var redisConf = newRedisConf() func TestNew(t *testing.T) { ctx := context.Background() - _, err := redis.New(ctx, redisConf) + _, err := redis.New(ctx, "", redisConf) if err != nil { t.Errorf("error creating redis: %v", err) } @@ -65,7 +65,7 @@ func TestNew(t *testing.T) { func TestClose(t *testing.T) { ctx := context.Background() - db, err := redis.New(ctx, redisConf) + db, err := redis.New(ctx, "", redisConf) if err != nil { t.Errorf("error creating redis: %v", err) } @@ -75,7 +75,7 @@ func TestClose(t *testing.T) { func TestSet(t *testing.T) { ctx := context.Background() - db, err := redis.New(ctx, redisConf) + db, err := redis.New(ctx, "", redisConf) if err != nil { t.Errorf("error creating redis: %v", err) } @@ -90,7 +90,7 @@ func TestSet(t *testing.T) { func TestSetTTL(t *testing.T) { ctx := context.Background() - db, err := redis.New(ctx, redisConf) + db, err := redis.New(ctx, "", redisConf) if err != nil { t.Errorf("error creating redis: %v", err) } @@ -105,7 +105,7 @@ func TestSetTTL(t *testing.T) { func TestGet(t *testing.T) { ctx := context.Background() - db, err := redis.New(ctx, redisConf) + db, err := redis.New(ctx, "", redisConf) if err != nil { t.Errorf("error creating redis: %v", err) } @@ -130,7 +130,7 @@ func TestGet(t *testing.T) { func TestGetTTL(t *testing.T) { ctx := context.Background() - db, err := redis.New(ctx, redisConf) + db, err := redis.New(ctx, "", redisConf) if err != nil { t.Errorf("error creating redis: %v", err) } @@ -155,7 +155,7 @@ func TestGetTTL(t *testing.T) { func TestGetExpired(t *testing.T) { ctx := context.Background() - db, err := redis.New(ctx, redisConf) + db, err := redis.New(ctx, "", redisConf) if err != nil { t.Errorf("error creating redis: %v", err) } diff --git a/src/cache/new.go b/src/cache/structs.go similarity index 72% rename from src/cache/new.go rename to src/cache/structs.go index 2ad06be3..c072a6ba 100644 --- a/src/cache/new.go +++ b/src/cache/structs.go @@ -11,28 +11,36 @@ import ( "github.com/rs/zerolog/log" ) +type DB struct { + driver Driver +} + func New(ctx context.Context, fileDbPath string, cacheConf config.Cache) (DB, error) { - var db DB + var drv Driver var err error switch cacheConf.Type { case "badger": - db, err = badger.New(fileDbPath, cacheConf.Badger) + drv, err = badger.New(fileDbPath, cacheConf.KeyPrefix, cacheConf.Badger) if err != nil { err = fmt.Errorf("failed creating a badger cache: %w", err) } case "redis": - db, err = redis.New(ctx, cacheConf.Redis) + drv, err = redis.New(ctx, cacheConf.KeyPrefix, cacheConf.Redis) if err != nil { err = fmt.Errorf("failed creating a redis cache: %w", err) } default: - db, err = nocache.New() + drv, err = nocache.New() if err != nil { err = fmt.Errorf("failed creating a nocache: %w", err) } log.Warn().Msg("Running without caching!") } - return db, err + return DB{drv}, err +} + +func (db DB) Close() { + db.driver.Close() } diff --git a/src/cache/utils.go b/src/cache/utils.go new file mode 100644 index 00000000..39993198 --- /dev/null +++ b/src/cache/utils.go @@ -0,0 +1,15 @@ +package cache + +import "fmt" + +func combineIntoKey(s ...string) string { + var key string + for i, v := range s { + if i == 0 { + key = v + } else { + key = fmt.Sprintf("%v_%v", key, v) + } + } + return key +} diff --git a/src/config/structs_server.go b/src/config/structs_server.go index 11944e7d..edb85fa0 100644 --- a/src/config/structs_server.go +++ b/src/config/structs_server.go @@ -33,6 +33,8 @@ type Server struct { type ReaderCache struct { // Can be "none", "badger" or "redis" Type string `koanf:"type"` + // Prefix to use for cache keys + KeyPrefix string `koanf:"keyprefix"` // Has no effect if type is "none" TTL ReaderTTL `koanf:"ttl"` // Badger specific settings @@ -43,6 +45,8 @@ type ReaderCache struct { type Cache struct { // Can be "none", "badger" or "redis" Type string + // Prefix to use for cache keys + KeyPrefix string // Has no effect if type is "none" TTL TTL // Badger specific settings diff --git a/src/search/cache.go b/src/search/cache.go index 1220a386..f1006fd4 100644 --- a/src/search/cache.go +++ b/src/search/cache.go @@ -20,7 +20,7 @@ func CacheAndUpdateResults( Str("queryAnon", anonymize.String(query)). Str("queryHash", anonymize.HashToSHA256B64(query)). Msg("Caching results...") - serr := db.Set(query, results, ttlConf.Time) + serr := db.SetResults(query, options.Category, results, ttlConf.Time) if serr != nil { log.Error(). Caller(). @@ -34,7 +34,7 @@ func CacheAndUpdateResults( Str("queryAnon", anonymize.String(query)). Str("queryHash", anonymize.HashToSHA256B64(query)). Msg("Checking if results need to be updated") - ttl, terr := db.GetTTL(query) + ttl, terr := db.GetResultsTTL(query, options.Category) if terr != nil { log.Error(). Caller(). @@ -48,7 +48,7 @@ func CacheAndUpdateResults( Str("queryHash", anonymize.HashToSHA256B64(query)). Msg("Updating results...") newResults := PerformSearch(query, options, categoryConf, settings, salt) - uerr := db.Set(query, newResults, ttlConf.Time) + uerr := db.SetResults(query, options.Category, newResults, ttlConf.Time) if uerr != nil { // Error in updating cache is not returned, just logged log.Error(). diff --git a/src/search/search.go b/src/search/search.go index 870b73f0..4e47799b 100644 --- a/src/search/search.go +++ b/src/search/search.go @@ -10,38 +10,32 @@ import ( ) func Search(query string, options engines.Options, db cache.DB, categoryConf config.Category, settings map[engines.Name]config.Settings, salt string) ([]result.Result, bool) { - var results []result.Result - var foundInDB bool - gerr := db.Get(query, &results) - if gerr != nil { + if results, err := db.GetResults(query, options.Category); err != nil { // Error in reading cache is not returned, just logged log.Error(). Caller(). - Err(gerr). + Err(err). Str("queryAnon", anonymize.String(query)). Str("queryHash", anonymize.HashToSHA256B64(query)). Msg("Failed accessing cache") } else if results != nil { - foundInDB = true - } else { - foundInDB = false - } - - if foundInDB { log.Debug(). Str("queryAnon", anonymize.String(query)). Str("queryHash", anonymize.HashToSHA256B64(query)). Msg("Found results in cache") - } else { - log.Debug(). - Str("queryAnon", anonymize.String(query)). - Str("queryHash", anonymize.HashToSHA256B64(query)). - Msg("Nothing found in cache, doing a clean search") - // the main line - results = PerformSearch(query, options, categoryConf, settings, salt) - result.Shorten(results, 2500) + return results, true } - return results, foundInDB + // if the cache is inaccesible or the query+category is not in the cache + log.Debug(). + Str("queryAnon", anonymize.String(query)). + Str("queryHash", anonymize.HashToSHA256B64(query)). + Msg("Nothing found in cache, doing a clean search") + + // the main line + results := PerformSearch(query, options, categoryConf, settings, salt) + result.Shorten(results, 2500) + + return results, false } From 3e034ef9a2fdd90d72a7389ccd59ba35094ae17f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?= <31509435+aleksasiriski@users.noreply.github.com> Date: Tue, 28 May 2024 20:47:56 +0200 Subject: [PATCH 3/6] fix: go.mod --- go.mod | 20 +++++++++----------- go.sum | 54 +++++++++++++++++++++++++----------------------------- 2 files changed, 34 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index 91a2e54e..98fd168c 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/aws/aws-lambda-go v1.47.0 github.com/awslabs/aws-lambda-go-api-proxy v0.16.2 github.com/dgraph-io/badger/v4 v4.2.0 - github.com/fxamacker/cbor/v2 v2.6.0 github.com/go-chi/chi/v5 v5.0.12 github.com/go-chi/cors v1.2.1 github.com/gocolly/colly/v2 v2.1.1-0.20231020184023-3c987f1982ed @@ -27,7 +26,7 @@ require ( require ( github.com/bits-and-blooms/bitset v1.13.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -36,21 +35,20 @@ require ( github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/glog v1.2.0 // indirect + github.com/golang/glog v1.2.1 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v24.3.25+incompatible // indirect - github.com/google/pprof v0.0.0-20240402174815-29b9bb013b0f // indirect - github.com/klauspost/compress v1.17.7 // indirect + github.com/google/pprof v0.0.0-20240528025155-186aa0362fba // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/nlnwa/whatwg-url v0.4.0 // indirect + github.com/nlnwa/whatwg-url v0.4.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/rs/xid v1.5.0 // indirect github.com/stretchr/testify v1.8.4 // indirect - github.com/x448/float16 v0.8.4 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/sync v0.7.0 // indirect @@ -61,9 +59,9 @@ require ( require ( github.com/PuerkitoBio/goquery v1.9.2 github.com/andybalholm/cascadia v1.3.2 // indirect - github.com/antchfx/htmlquery v1.3.0 // indirect - github.com/antchfx/xmlquery v1.3.18 // indirect - github.com/antchfx/xpath v1.2.5 // indirect + github.com/antchfx/htmlquery v1.3.1 // indirect + github.com/antchfx/xmlquery v1.4.0 // indirect + github.com/antchfx/xpath v1.3.0 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect @@ -76,5 +74,5 @@ require ( golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect ) diff --git a/go.sum b/go.sum index c8cd306c..f11990c4 100644 --- a/go.sum +++ b/go.sum @@ -16,23 +16,20 @@ github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEq github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2if0BLYa3V0= -github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E= -github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8= +github.com/antchfx/htmlquery v1.3.1 h1:wm0LxjLMsZhRHfQKKZscDf2COyH4vDYA3wyH+qZ+Ylc= +github.com/antchfx/htmlquery v1.3.1/go.mod h1:PTj+f1V2zksPlwNt7uVvZPsxpKNa7mlVliCRxLX6Nx8= github.com/antchfx/xmlquery v1.3.4/go.mod h1:64w0Xesg2sTaawIdNqMB+7qaW/bSqkQm+ssPaCMWNnc= -github.com/antchfx/xmlquery v1.3.18 h1:FSQ3wMuphnPPGJOFhvc+cRQ2CT/rUj4cyQXkJcjOwz0= -github.com/antchfx/xmlquery v1.3.18/go.mod h1:Afkq4JIeXut75taLSuI31ISJ/zeq+3jG7TunF7noreA= +github.com/antchfx/xmlquery v1.4.0 h1:xg2HkfcRK2TeTbdb0m1jxCYnvsPaGY/oeZWTGqX/0hA= +github.com/antchfx/xmlquery v1.4.0/go.mod h1:Ax2aeaeDjfIw3CwXKDQ0GkwZ6QlxoChlIBP+mGnDFjI= github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/antchfx/xpath v1.1.10/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= -github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= -github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= -github.com/antchfx/xpath v1.2.5 h1:hqZ+wtQ+KIOV/S3bGZcIhpgYC26um2bZYP2KVGcR7VY= -github.com/antchfx/xpath v1.2.5/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/antchfx/xpath v1.3.0 h1:nTMlzGAK3IJ0bPpME2urTuFL76o4A96iYvoKFHRXJgc= +github.com/antchfx/xpath v1.3.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI= github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= github.com/awslabs/aws-lambda-go-api-proxy v0.16.2 h1:CJyGEyO1CIwOnXTU40urf0mchf6t3voxpvUDikOU9LY= github.com/awslabs/aws-lambda-go-api-proxy v0.16.2/go.mod h1:vxxjwBHe/KbgFeNlAP/Tvp4SsVRL3WQamcWRxqVh0z0= github.com/bits-and-blooms/bitset v1.2.2-0.20220111210104-dfa3e347c392/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= @@ -41,8 +38,8 @@ github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= @@ -81,8 +78,6 @@ github.com/felixge/fgprof v0.9.4 h1:ocDNwMFlnA0NU0zSB3I52xkO4sFXk80VK9lXjLClu88= github.com/felixge/fgprof v0.9.4/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/fxamacker/cbor/v2 v2.6.0 h1:sU6J2usfADwWlYDAFhZBQ6TnLFBHxgesMrQfQgk1tWA= -github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4= @@ -100,8 +95,8 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= -github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4= +github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -136,8 +131,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= -github.com/google/pprof v0.0.0-20240402174815-29b9bb013b0f h1:f00RU+zOX+B3rLAmMMkzHUF2h1z4DeYR9tTCvEq2REY= -github.com/google/pprof v0.0.0-20240402174815-29b9bb013b0f/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/pprof v0.0.0-20240528025155-186aa0362fba h1:ql1qNgCyOB7iAEk8JTNM+zJrgIbnyCKX/wdlyPufP5g= +github.com/google/pprof v0.0.0-20240528025155-186aa0362fba/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= @@ -149,8 +144,8 @@ github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8Nz github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= -github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= github.com/knadh/koanf/parsers/yaml v0.1.0 h1:ZZ8/iGfRLvKSaMEECEBPM1HQslrZADk8fP1XFUxVI5w= @@ -183,8 +178,8 @@ github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HK github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/nlnwa/whatwg-url v0.1.2/go.mod h1:b0r+dEyM/KztLMDSVY6ApcO9Fmzgq+e9+Ugq20UBYck= -github.com/nlnwa/whatwg-url v0.4.0 h1:B3kFb5EL7KILeBkhrlQvFi41Ex0p4ropVA9brt5ungI= -github.com/nlnwa/whatwg-url v0.4.0/go.mod h1:pLzpJjFPtA+n7RCLvp0GBxvDHa/2ckNCBK9mfEeNOMQ= +github.com/nlnwa/whatwg-url v0.4.1 h1:m0+XWylS9IuCPd5GMW2lzmSI9ssSwynT3nug0p3bUIo= +github.com/nlnwa/whatwg-url v0.4.1/go.mod h1:X/ejnFFVbaOWdSul+cnlsSHviCzGZJdvPkgc9zD8IY8= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= @@ -226,8 +221,6 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= -github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= -github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -238,6 +231,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -266,12 +261,12 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -298,32 +293,33 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -367,8 +363,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 1cc5785824164ebfcf5387719c521ce32f525068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?= <31509435+aleksasiriski@users.noreply.github.com> Date: Tue, 28 May 2024 20:52:36 +0200 Subject: [PATCH 4/6] fix(config): default for cache key prefix --- src/config/defaults.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/config/defaults.go b/src/config/defaults.go index 241ea422..afa5199e 100644 --- a/src/config/defaults.go +++ b/src/config/defaults.go @@ -143,7 +143,8 @@ func New() Config { Port: 3030, FrontendUrls: []string{"http://localhost:5173"}, Cache: Cache{ - Type: "badger", + Type: "badger", + KeyPrefix: "HEARCHCO_", TTL: TTL{ Time: moretime.Week, RefreshTime: 3 * moretime.Day, From b50e97f207af7ca07408df46f55992c34b281e52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?= <31509435+aleksasiriski@users.noreply.github.com> Date: Tue, 28 May 2024 20:53:19 +0200 Subject: [PATCH 5/6] fix(config): default to no cache --- src/config/defaults.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/defaults.go b/src/config/defaults.go index afa5199e..d4790f4e 100644 --- a/src/config/defaults.go +++ b/src/config/defaults.go @@ -143,7 +143,7 @@ func New() Config { Port: 3030, FrontendUrls: []string{"http://localhost:5173"}, Cache: Cache{ - Type: "badger", + Type: "none", KeyPrefix: "HEARCHCO_", TTL: TTL{ Time: moretime.Week, From ab5032cfda49a5f959d2268a68d8edde01124178 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?= <31509435+aleksasiriski@users.noreply.github.com> Date: Tue, 28 May 2024 21:14:04 +0200 Subject: [PATCH 6/6] fix(cache): combine all params into a key --- src/cache/actions_results.go | 18 +++++++++++------- src/cache/utils.go | 4 ++-- src/search/cache.go | 18 +++++++++--------- src/search/search.go | 2 +- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/cache/actions_results.go b/src/cache/actions_results.go index cffbf3b2..408494bb 100644 --- a/src/cache/actions_results.go +++ b/src/cache/actions_results.go @@ -3,23 +3,27 @@ package cache import ( "time" - "github.com/hearchco/hearchco/src/search/category" + "github.com/hearchco/hearchco/src/search/engines" "github.com/hearchco/hearchco/src/search/result" ) -func (db DB) SetResults(query string, category category.Name, results []result.Result, ttl ...time.Duration) error { - key := combineIntoKey(query, category.String()) +func (db DB) SetResults(query string, options engines.Options, results []result.Result, ttl ...time.Duration) error { + key := combineQueryWithOptions(query, options) return db.driver.Set(key, results, ttl...) } -func (db DB) GetResults(query string, category category.Name) ([]result.Result, error) { - key := combineIntoKey(query, category.String()) +func (db DB) GetResults(query string, options engines.Options) ([]result.Result, error) { + key := combineQueryWithOptions(query, options) var results []result.Result err := db.driver.Get(key, &results) return results, err } -func (db DB) GetResultsTTL(query string, category category.Name) (time.Duration, error) { - key := combineIntoKey(query, category.String()) +func (db DB) GetResultsTTL(query string, options engines.Options) (time.Duration, error) { + key := combineQueryWithOptions(query, options) return db.driver.GetTTL(key) } + +func combineQueryWithOptions(query string, options engines.Options) string { + return combineIntoKey(query, options.VisitPages, options.SafeSearch, options.Pages.Start, options.Pages.Max, options.Locale, options.Category.String()) +} diff --git a/src/cache/utils.go b/src/cache/utils.go index 39993198..b231e06b 100644 --- a/src/cache/utils.go +++ b/src/cache/utils.go @@ -2,11 +2,11 @@ package cache import "fmt" -func combineIntoKey(s ...string) string { +func combineIntoKey(s ...any) string { var key string for i, v := range s { if i == 0 { - key = v + key = fmt.Sprintf("%v", v) } else { key = fmt.Sprintf("%v_%v", key, v) } diff --git a/src/search/cache.go b/src/search/cache.go index f1006fd4..cead354b 100644 --- a/src/search/cache.go +++ b/src/search/cache.go @@ -20,11 +20,11 @@ func CacheAndUpdateResults( Str("queryAnon", anonymize.String(query)). Str("queryHash", anonymize.HashToSHA256B64(query)). Msg("Caching results...") - serr := db.SetResults(query, options.Category, results, ttlConf.Time) - if serr != nil { + err := db.SetResults(query, options, results, ttlConf.Time) + if err != nil { log.Error(). Caller(). - Err(serr). + Err(err). Str("queryAnon", anonymize.String(query)). Str("queryHash", anonymize.HashToSHA256B64(query)). Msg("Error updating database with search results") @@ -34,11 +34,11 @@ func CacheAndUpdateResults( Str("queryAnon", anonymize.String(query)). Str("queryHash", anonymize.HashToSHA256B64(query)). Msg("Checking if results need to be updated") - ttl, terr := db.GetResultsTTL(query, options.Category) - if terr != nil { + ttl, err := db.GetResultsTTL(query, options) + if err != nil { log.Error(). Caller(). - Err(terr). + Err(err). Str("queryAnon", anonymize.String(query)). Str("queryHash", anonymize.HashToSHA256B64(query)). Msg("Error getting TTL from database") @@ -48,12 +48,12 @@ func CacheAndUpdateResults( Str("queryHash", anonymize.HashToSHA256B64(query)). Msg("Updating results...") newResults := PerformSearch(query, options, categoryConf, settings, salt) - uerr := db.SetResults(query, options.Category, newResults, ttlConf.Time) - if uerr != nil { + err := db.SetResults(query, options, newResults, ttlConf.Time) + if err != nil { // Error in updating cache is not returned, just logged log.Error(). Caller(). - Err(uerr). + Err(err). Str("queryAnon", anonymize.String(query)). Str("queryHash", anonymize.HashToSHA256B64(query)). Msg("Error replacing old results while updating database") diff --git a/src/search/search.go b/src/search/search.go index 4e47799b..94553292 100644 --- a/src/search/search.go +++ b/src/search/search.go @@ -10,7 +10,7 @@ import ( ) func Search(query string, options engines.Options, db cache.DB, categoryConf config.Category, settings map[engines.Name]config.Settings, salt string) ([]result.Result, bool) { - if results, err := db.GetResults(query, options.Category); err != nil { + if results, err := db.GetResults(query, options); err != nil { // Error in reading cache is not returned, just logged log.Error(). Caller().