Merge pull request #320 from hearchco/as/fix/cache-drivers

fix(cache): buggy key in drivers
hearchco · May 28, 2024 · 3a3c97c · 3a3c97c
2 parents 92fdbbb + f8615a5
commit 3a3c97c
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 27 deletions.
diff --git a/src/cache/badger/badger.go b/src/cache/badger/badger.go
@@ -67,7 +67,7 @@ func (drv DRV) Set(k string, v any, ttl ...time.Duration) error {
 		setTtl = ttl[0]
 	}
 
-	key := fmt.Sprintf("%v%v", drv.keyPrefix, k)
+	key := anonymize.HashToSHA256B64(fmt.Sprintf("%v%v", drv.keyPrefix, k))
 	if val, err := json.Marshal(v); err != nil {
 		return fmt.Errorf("badger.Set(): error marshaling value: %w", err)
 	} else if err := drv.client.Update(func(txn *badger.Txn) error {

diff --git a/src/cache/badger/badger_test.go b/src/cache/badger/badger_test.go
@@ -10,22 +10,22 @@ import (
 )
 
 func TestNewInMemory(t *testing.T) {
-	_, err := badger.New("", "", config.Badger{Persist: false})
+	_, err := badger.New("", "TEST_", config.Badger{Persist: false})
 	if err != nil {
 		t.Errorf("error opening in-memory badger: %v", err)
 	}
 }
 
 func TestNewPersistence(t *testing.T) {
 	path := "./testdump/new"
-	_, err := badger.New(path, "", config.Badger{Persist: true})
+	_, err := badger.New(path, "TEST_", config.Badger{Persist: true})
 	if err != nil {
 		t.Errorf("error opening badger at %v: %v", path, err)
 	}
 }
 
 func TestCloseInMemory(t *testing.T) {
-	db, err := badger.New("", "", config.Badger{Persist: false})
+	db, err := badger.New("", "TEST_", config.Badger{Persist: false})
 	if err != nil {
 		t.Errorf("error opening in-memory badger: %v", err)
 	}
@@ -35,7 +35,7 @@ func TestCloseInMemory(t *testing.T) {
 
 func TestClosePersistence(t *testing.T) {
 	path := "./testdump/close"
-	db, err := badger.New(path, "", config.Badger{Persist: true})
+	db, err := badger.New(path, "TEST_", config.Badger{Persist: true})
 	if err != nil {
 		t.Errorf("error opening badger at %v: %v", path, err)
 	}
@@ -44,7 +44,7 @@ func TestClosePersistence(t *testing.T) {
 }
 
 func TestSetInMemory(t *testing.T) {
-	db, err := badger.New("", "", config.Badger{Persist: false})
+	db, err := badger.New("", "TEST_", config.Badger{Persist: false})
 	if err != nil {
 		t.Errorf("error opening in-memory badger: %v", err)
 	}
@@ -59,7 +59,7 @@ func TestSetInMemory(t *testing.T) {
 
 func TestSetPersistence(t *testing.T) {
 	path := "./testdump/set"
-	db, err := badger.New(path, "", config.Badger{Persist: true})
+	db, err := badger.New(path, "TEST_", config.Badger{Persist: true})
 	if err != nil {
 		t.Errorf("error opening badger at %v: %v", path, err)
 	}
@@ -73,7 +73,7 @@ func TestSetPersistence(t *testing.T) {
 }
 
 func TestSetTTLInMemory(t *testing.T) {
-	db, err := badger.New("", "", config.Badger{Persist: false})
+	db, err := badger.New("", "TEST_", config.Badger{Persist: false})
 	if err != nil {
 		t.Errorf("error opening in-memory badger: %v", err)
 	}
@@ -88,7 +88,7 @@ func TestSetTTLInMemory(t *testing.T) {
 
 func TestSetTTLPersistence(t *testing.T) {
 	path := "./testdump/setttl"
-	db, err := badger.New(path, "", config.Badger{Persist: true})
+	db, err := badger.New(path, "TEST_", config.Badger{Persist: true})
 	if err != nil {
 		t.Errorf("error opening badger at %v: %v", path, err)
 	}
@@ -102,7 +102,7 @@ func TestSetTTLPersistence(t *testing.T) {
 }
 
 func TestGetInMemory(t *testing.T) {
-	db, err := badger.New("", "", config.Badger{Persist: false})
+	db, err := badger.New("", "TEST_", config.Badger{Persist: false})
 	if err != nil {
 		t.Errorf("error opening in-memory badger: %v", err)
 	}
@@ -127,7 +127,7 @@ func TestGetInMemory(t *testing.T) {
 
 func TestGetPersistence(t *testing.T) {
 	path := "./testdump/get"
-	db, err := badger.New(path, "", config.Badger{Persist: true})
+	db, err := badger.New(path, "TEST_", config.Badger{Persist: true})
 	if err != nil {
 		t.Errorf("error opening badger at %v: %v", path, err)
 	}
@@ -151,7 +151,7 @@ func TestGetPersistence(t *testing.T) {
 }
 
 func TestGetTTLInMemory(t *testing.T) {
-	db, err := badger.New("", "", config.Badger{Persist: false})
+	db, err := badger.New("", "TEST_", config.Badger{Persist: false})
 	if err != nil {
 		t.Errorf("error opening in-memory badger: %v", err)
 	}
@@ -176,7 +176,7 @@ func TestGetTTLInMemory(t *testing.T) {
 
 func TestGetTTLPersistence(t *testing.T) {
 	path := "./testdump/getttl"
-	db, err := badger.New(path, "", config.Badger{Persist: true})
+	db, err := badger.New(path, "TEST_", config.Badger{Persist: true})
 	if err != nil {
 		t.Errorf("error opening badger at %v: %v", path, err)
 	}
@@ -200,7 +200,7 @@ func TestGetTTLPersistence(t *testing.T) {
 }
 
 func TestGetInMemoryExpired(t *testing.T) {
-	db, err := badger.New("", "", config.Badger{Persist: false})
+	db, err := badger.New("", "TEST_", config.Badger{Persist: false})
 	if err != nil {
 		t.Errorf("error opening in-memory badger: %v", err)
 	}
@@ -223,7 +223,7 @@ func TestGetInMemoryExpired(t *testing.T) {
 
 func TestGetPersistenceExpired(t *testing.T) {
 	path := "./testdump/getexpired"
-	db, err := badger.New(path, "", config.Badger{Persist: true})
+	db, err := badger.New(path, "TEST_", config.Badger{Persist: true})
 	if err != nil {
 		t.Errorf("error opening badger at %v: %v", path, err)
 	}

diff --git a/src/cache/redis/redis.go b/src/cache/redis/redis.go
@@ -59,9 +59,10 @@ func (drv DRV) Set(k string, v any, ttl ...time.Duration) error {
 		setTtl = ttl[0]
 	}
 
+	key := anonymize.HashToSHA256B64(fmt.Sprintf("%v%v", drv.keyPrefix, k))
 	if val, err := json.Marshal(v); err != nil {
 		return fmt.Errorf("redis.Set(): error marshaling value: %w", err)
-	} else if err := drv.client.Set(drv.ctx, anonymize.HashToSHA256B64(k), val, setTtl).Err(); err != nil {
+	} else if err := drv.client.Set(drv.ctx, key, val, setTtl).Err(); err != nil {
 		return fmt.Errorf("redis.Set(): error setting KV to redis: %w", err)
 	} else {
 		log.Trace().
@@ -73,7 +74,7 @@ func (drv DRV) Set(k string, v any, ttl ...time.Duration) error {
 }
 
 func (drv DRV) Get(k string, o any) error {
-	key := anonymize.HashToSHA256B64(k)
+	key := anonymize.HashToSHA256B64(fmt.Sprintf("%v%v", drv.keyPrefix, k))
 
 	val, err := drv.client.Get(drv.ctx, key).Result()
 	if err == redis.Nil {
@@ -91,7 +92,7 @@ func (drv DRV) Get(k string, o any) error {
 
 // returns time until the key expires, not the time it will be considered expired
 func (drv DRV) GetTTL(k string) (time.Duration, error) {
-	key := anonymize.HashToSHA256B64(k)
+	key := anonymize.HashToSHA256B64(fmt.Sprintf("%v%v", drv.keyPrefix, k))
 
 	// returns time with time.Second precision
 	expiresIn, err := drv.client.TTL(drv.ctx, key).Result()

diff --git a/src/cache/redis/redis_test.go b/src/cache/redis/redis_test.go
@@ -57,15 +57,15 @@ var redisConf = newRedisConf()
 
 func TestNew(t *testing.T) {
 	ctx := context.Background()
-	_, err := redis.New(ctx, "", redisConf)
+	_, err := redis.New(ctx, "TEST_", redisConf)
 	if err != nil {
 		t.Errorf("error creating redis: %v", err)
 	}
 }
 
 func TestClose(t *testing.T) {
 	ctx := context.Background()
-	db, err := redis.New(ctx, "", redisConf)
+	db, err := redis.New(ctx, "TEST_", redisConf)
 	if err != nil {
 		t.Errorf("error creating redis: %v", err)
 	}
@@ -75,7 +75,7 @@ func TestClose(t *testing.T) {
 
 func TestSet(t *testing.T) {
 	ctx := context.Background()
-	db, err := redis.New(ctx, "", redisConf)
+	db, err := redis.New(ctx, "TEST_", redisConf)
 	if err != nil {
 		t.Errorf("error creating redis: %v", err)
 	}
@@ -90,7 +90,7 @@ func TestSet(t *testing.T) {
 
 func TestSetTTL(t *testing.T) {
 	ctx := context.Background()
-	db, err := redis.New(ctx, "", redisConf)
+	db, err := redis.New(ctx, "TEST_", redisConf)
 	if err != nil {
 		t.Errorf("error creating redis: %v", err)
 	}
@@ -105,7 +105,7 @@ func TestSetTTL(t *testing.T) {
 
 func TestGet(t *testing.T) {
 	ctx := context.Background()
-	db, err := redis.New(ctx, "", redisConf)
+	db, err := redis.New(ctx, "TEST_", redisConf)
 	if err != nil {
 		t.Errorf("error creating redis: %v", err)
 	}
@@ -130,7 +130,7 @@ func TestGet(t *testing.T) {
 
 func TestGetTTL(t *testing.T) {
 	ctx := context.Background()
-	db, err := redis.New(ctx, "", redisConf)
+	db, err := redis.New(ctx, "TEST_", redisConf)
 	if err != nil {
 		t.Errorf("error creating redis: %v", err)
 	}
@@ -155,7 +155,7 @@ func TestGetTTL(t *testing.T) {
 
 func TestGetExpired(t *testing.T) {
 	ctx := context.Background()
-	db, err := redis.New(ctx, "", redisConf)
+	db, err := redis.New(ctx, "TEST_", redisConf)
 	if err != nil {
 		t.Errorf("error creating redis: %v", err)
 	}

diff --git a/src/search/engines/_sedefaults/init.go b/src/search/engines/_sedefaults/init.go
@@ -24,7 +24,15 @@ func InitializeCollectors(ctx context.Context, engineName engines.Name, options
 		colly.UserAgent(userAgent),
 		colly.IgnoreRobotsTxt(),
 		colly.Headers(map[string]string{
-			"Sec-Ch-Ua": secChUa,
+			"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+			// "Accept-Encoding":    "gzip, deflate, br", // Chromium-based browsers have "zstd" but that isn't supported by Firefox nor Safari
+			"Accept-Language":    "en-US,en;q=0.9",
+			"Sec-Ch-Ua":          secChUa, // "Google Chrome";v="119", "Chromium";v="119", "Not=A?Brand";v="24"
+			"Sec-Ch-Ua-Mobile":   "?0",
+			"Sec-Ch-Ua-Platform": "\"Windows\"",
+			"Sec-Fetch-Dest":     "document",
+			"Sec-Fetch-Mode":     "navigate",
+			"Sec-Fetch-Site":     "none",
 		}),
 	)
 	pagesCol := colly.NewCollector(
@@ -33,7 +41,15 @@ func InitializeCollectors(ctx context.Context, engineName engines.Name, options
 		colly.UserAgent(userAgent),
 		colly.IgnoreRobotsTxt(),
 		colly.Headers(map[string]string{
-			"Sec-Ch-Ua": secChUa,
+			"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+			// "Accept-Encoding":    "gzip, deflate, br", // Chromium-based browsers have "zstd" but that isn't supported by Firefox nor Safari
+			"Accept-Language":    "en-US,en;q=0.9",
+			"Sec-Ch-Ua":          secChUa, // "Google Chrome";v="119", "Chromium";v="119", "Not=A?Brand";v="24"
+			"Sec-Ch-Ua-Mobile":   "?0",
+			"Sec-Ch-Ua-Platform": "\"Windows\"",
+			"Sec-Fetch-Dest":     "document",
+			"Sec-Fetch-Mode":     "navigate",
+			"Sec-Fetch-Site":     "none",
 		}),
 	)