From 392e859b743d7f6973c5999ddbe6c7a048394a1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?=
<31509435+aleksasiriski@users.noreply.github.com>
Date: Tue, 28 May 2024 14:57:28 +0200
Subject: [PATCH 1/2] fix!: removed bangs, since they can be handled by
frontend
---
src/cli/climode.go | 11 +++-
src/cli/setup.go | 2 +-
src/router/search.go | 27 ++++----
src/search/bang.go | 75 ----------------------
src/search/cache.go | 5 +-
src/search/category/category.go | 42 ++++--------
src/search/engines/_engines_test/tester.go | 6 +-
src/search/perform.go | 26 +++-----
src/search/search.go | 5 +-
9 files changed, 49 insertions(+), 150 deletions(-)
delete mode 100644 src/search/bang.go
diff --git a/src/cli/climode.go b/src/cli/climode.go
index 58215783..e564da82 100644
--- a/src/cli/climode.go
+++ b/src/cli/climode.go
@@ -61,13 +61,18 @@ func Run(flags Flags, db cache.DB, conf config.Config) {
Bool("visit", flags.Visit).
Msg("Started hearching")
+ categoryName, err := category.FromString(flags.Category)
+ if err != nil {
+ log.Fatal().Err(err).Msg("Invalid category")
+ }
+
options := engines.Options{
Pages: engines.Pages{
Start: flags.StartPage,
Max: flags.MaxPages,
},
VisitPages: flags.Visit,
- Category: category.FromString[flags.Category],
+ Category: categoryName,
UserAgent: flags.UserAgent,
Locale: flags.Locale,
SafeSearch: flags.SafeSearch,
@@ -76,7 +81,7 @@ func Run(flags Flags, db cache.DB, conf config.Config) {
start := time.Now()
- results, foundInDB := search.Search(flags.Query, options, db, conf.Settings, conf.Categories, conf.Server.Proxy.Salt)
+ results, foundInDB := search.Search(flags.Query, options, db, conf.Categories[options.Category], conf.Settings, conf.Server.Proxy.Salt)
if !flags.Silent {
printResults(results)
@@ -87,5 +92,5 @@ func Run(flags Flags, db cache.DB, conf config.Config) {
Dur("duration", time.Since(start)).
Msg("Found results")
- search.CacheAndUpdateResults(flags.Query, options, db, conf.Server.Cache.TTL, conf.Settings, conf.Categories, results, foundInDB, conf.Server.Proxy.Salt)
+ search.CacheAndUpdateResults(flags.Query, options, db, conf.Server.Cache.TTL, conf.Categories[options.Category], conf.Settings, results, foundInDB, conf.Server.Proxy.Salt)
}
diff --git a/src/cli/setup.go b/src/cli/setup.go
index 297af758..347be328 100644
--- a/src/cli/setup.go
+++ b/src/cli/setup.go
@@ -36,7 +36,7 @@ func Setup() Flags {
// ^FATAL
}
- if category.SafeFromString(cli.Category) == category.UNDEFINED {
+ if _, err := category.FromString(cli.Category); err != nil {
log.Fatal().Msg("cli.Setup(): invalid category flag")
// ^FATAL
}
diff --git a/src/router/search.go b/src/router/search.go
index 76b79bdc..5823603b 100644
--- a/src/router/search.go
+++ b/src/router/search.go
@@ -32,7 +32,7 @@ func Search(w http.ResponseWriter, r *http.Request, db cache.DB, ttlConf config.
params := r.Form
- query := getParamOrDefault(params, "q")
+ query := strings.TrimSpace(getParamOrDefault(params, "q"))
pagesStartS := getParamOrDefault(params, "start", "1")
pagesMaxS := getParamOrDefault(params, "pages", "1")
visitPagesS := getParamOrDefault(params, "deep", "false")
@@ -42,10 +42,12 @@ func Search(w http.ResponseWriter, r *http.Request, db cache.DB, ttlConf config.
safeSearchS := getParamOrDefault(params, "safesearch", "false")
mobileS := getParamOrDefault(params, "mobile", "false")
- queryWithoutSpaces := strings.TrimSpace(query)
- if queryWithoutSpaces == "" || "!"+string(category.FromQuery(query)) == queryWithoutSpaces {
- // return "[]" JSON when the query is empty or contains only category name
- return writeResponseJSON(w, http.StatusOK, []struct{}{})
+ if query == "" {
+ // user error
+ return writeResponseJSON(w, http.StatusBadRequest, ErrorResponse{
+ Message: "query cannot be empty or whitespace",
+ Value: "empty query",
+ })
}
pagesMax, err := strconv.Atoi(pagesMaxS)
@@ -106,12 +108,12 @@ func Search(w http.ResponseWriter, r *http.Request, db cache.DB, ttlConf config.
})
}
- categoryName := category.SafeFromString(categoryS)
- if categoryName == category.UNDEFINED {
+ categoryName, err := category.FromString(categoryS)
+ if err != nil {
// user error
return writeResponseJSON(w, http.StatusBadRequest, ErrorResponse{
Message: "invalid category value",
- Value: fmt.Sprintf("%v", category.UNDEFINED),
+ Value: fmt.Sprintf("%v", categoryName),
})
}
@@ -147,13 +149,10 @@ func Search(w http.ResponseWriter, r *http.Request, db cache.DB, ttlConf config.
}
// search for results in db and web, afterwards return JSON
- results, foundInDB := search.Search(query, options, db, settings, categories, salt)
-
- // get category from query or options
- cat := category.FromQueryWithFallback(query, options.Category)
+ results, foundInDB := search.Search(query, options, db, categories[options.Category], settings, salt)
// send response as soon as possible
- if cat == category.IMAGES {
+ if categoryName == category.IMAGES {
resultsOutput := result.ConvertToImageOutput(results)
err = writeResponseJSON(w, http.StatusOK, resultsOutput)
} else {
@@ -162,7 +161,7 @@ func Search(w http.ResponseWriter, r *http.Request, db cache.DB, ttlConf config.
}
// don't return immediately, we want to cache results and update them if necessary
- search.CacheAndUpdateResults(query, options, db, ttlConf, settings, categories, results, foundInDB, salt)
+ search.CacheAndUpdateResults(query, options, db, ttlConf, categories[options.Category], settings, results, foundInDB, salt)
// if writing response failed, return the error
return err
diff --git a/src/search/bang.go b/src/search/bang.go
deleted file mode 100644
index 7a8e1f0b..00000000
--- a/src/search/bang.go
+++ /dev/null
@@ -1,75 +0,0 @@
-package search
-
-import (
- "strings"
-
- "github.com/hearchco/hearchco/src/anonymize"
- "github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/search/category"
- "github.com/hearchco/hearchco/src/search/engines"
- "github.com/rs/zerolog/log"
-)
-
-func procBang(query string, setCategory category.Name, settings map[engines.Name]config.Settings, categories map[category.Name]config.Category) (string, category.Name, config.CategoryTimings, []engines.Name) {
- useSpec, specEng := procSpecificEngine(query, settings)
- goodCat, cat := procCategory(query, setCategory)
- if !goodCat && !useSpec && (query != "" && query[0] == '!') {
- // cat is set to GENERAL
- log.Debug().
- Str("queryAnon", anonymize.String(query)).
- Str("queryHash", anonymize.HashToSHA256B64(query)).
- Msg("search.procBang(): invalid bang (not category or engine shortcut)")
- }
-
- query = trimBang(query)
-
- if useSpec {
- return query, category.GENERAL, categories[category.GENERAL].Timings, []engines.Name{specEng}
- } else {
- return query, cat, categories[cat].Timings, categories[cat].Engines
- }
-}
-
-// takes the bang out of the query performs TrimSpace
-func trimBang(query string) string {
- query = strings.TrimSpace(query)
-
- if query == "" || query[0] != '!' {
- return query
- }
-
- sp := strings.SplitN(query, " ", 2)
- if len(sp) == 1 {
- // only the bang is present
- return ""
- }
-
- return strings.TrimSpace(sp[1])
-}
-
-func procSpecificEngine(query string, settings map[engines.Name]config.Settings) (bool, engines.Name) {
- if query == "" || query[0] != '!' {
- return false, engines.UNDEFINED
- }
- sp := strings.SplitN(query, " ", 2)
- bangWord := sp[0][1:]
- for key, val := range settings {
- if strings.EqualFold(bangWord, val.Shortcut) || strings.EqualFold(bangWord, key.String()) {
- return true, key
- }
- }
-
- return false, engines.UNDEFINED
-}
-
-// returns category in the query if a valid category is present
-func procCategory(query string, setCategory category.Name) (bool, category.Name) {
- cat := category.FromQuery(query)
- if cat != "" {
- return true, cat
- } else if setCategory == "" {
- return false, category.GENERAL
- } else {
- return false, setCategory
- }
-}
diff --git a/src/search/cache.go b/src/search/cache.go
index fbb11f58..325f2b70 100644
--- a/src/search/cache.go
+++ b/src/search/cache.go
@@ -4,7 +4,6 @@ import (
"github.com/hearchco/hearchco/src/anonymize"
"github.com/hearchco/hearchco/src/cache"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/search/category"
"github.com/hearchco/hearchco/src/search/engines"
"github.com/hearchco/hearchco/src/search/result"
"github.com/rs/zerolog/log"
@@ -12,7 +11,7 @@ import (
func CacheAndUpdateResults(
query string, options engines.Options, db cache.DB,
- ttlConf config.TTL, settings map[engines.Name]config.Settings, categories map[category.Name]config.Category,
+ ttlConf config.TTL, categoryConf config.Category, settings map[engines.Name]config.Settings,
results []result.Result, foundInDB bool,
salt string,
) {
@@ -46,7 +45,7 @@ func CacheAndUpdateResults(
Str("queryAnon", anonymize.String(query)).
Str("queryHash", anonymize.HashToSHA256B64(query)).
Msg("Updating results...")
- newResults := PerformSearch(query, options, settings, categories, salt)
+ newResults := PerformSearch(query, options, categoryConf, settings, salt)
uerr := db.Set(query, newResults, ttlConf.Time)
if uerr != nil {
// Error in updating cache is not returned, just logged
diff --git a/src/search/category/category.go b/src/search/category/category.go
index 2ea0520a..3fe47457 100644
--- a/src/search/category/category.go
+++ b/src/search/category/category.go
@@ -1,10 +1,8 @@
package category
-import (
- "strings"
-)
+import "fmt"
-var FromString = map[string]Name{
+var catMap = map[string]Name{
"general": GENERAL,
"images": IMAGES,
"science": SCIENCE,
@@ -13,36 +11,18 @@ var FromString = map[string]Name{
"slow": THOROUGH,
}
-// returns category
-func FromQuery(query string) Name {
- if query == "" || query[0] != '!' {
- return ""
- }
- cat := strings.SplitN(query, " ", 2)[0][1:]
- if val, ok := FromString[cat]; ok {
- return val
- }
- return ""
-}
-
-func SafeFromString(cat string) Name {
+// converts a string to a category name if it exists
+// if the string is empty, then GENERAL is returned
+// otherwise returns UNDEFINED
+func FromString(cat string) (Name, error) {
if cat == "" {
- return ""
+ return GENERAL, nil
}
- ret, ok := FromString[cat]
+
+ catName, ok := catMap[cat]
if !ok {
- return UNDEFINED
+ return UNDEFINED, fmt.Errorf("category %q is not defined", cat)
}
- return ret
-}
-func FromQueryWithFallback(query string, fallback Name) Name {
- cat := FromQuery(query)
- if cat != "" {
- return cat
- } else if fallback != "" {
- return fallback
- } else {
- return GENERAL
- }
+ return catName, nil
}
diff --git a/src/search/engines/_engines_test/tester.go b/src/search/engines/_engines_test/tester.go
index b367ccea..1cb64947 100644
--- a/src/search/engines/_engines_test/tester.go
+++ b/src/search/engines/_engines_test/tester.go
@@ -13,14 +13,14 @@ func CheckTestCases(tchar []TestCaseHasAnyResults, tccr []TestCaseContainsResult
// TestCaseHasAnyResults
for _, tc := range tchar {
- if results := search.PerformSearch(tc.Query, tc.Options, conf.Settings, conf.Categories, ""); len(results) == 0 {
+ if results := search.PerformSearch(tc.Query, tc.Options, conf.Categories[tc.Options.Category], conf.Settings, ""); len(results) == 0 {
defer t.Errorf("Got no results for %q", tc.Query)
}
}
// TestCaseContainsResults
for _, tc := range tccr {
- results := search.PerformSearch(tc.Query, tc.Options, conf.Settings, conf.Categories, "")
+ results := search.PerformSearch(tc.Query, tc.Options, conf.Categories[tc.Options.Category], conf.Settings, "")
if len(results) == 0 {
defer t.Errorf("Got no results for %q", tc.Query)
} else {
@@ -43,7 +43,7 @@ func CheckTestCases(tchar []TestCaseHasAnyResults, tccr []TestCaseContainsResult
// TestCaseRankedResults
for _, tc := range tcrr {
- results := search.PerformSearch(tc.Query, tc.Options, conf.Settings, conf.Categories, "")
+ results := search.PerformSearch(tc.Query, tc.Options, conf.Categories[tc.Options.Category], conf.Settings, "")
if len(results) == 0 {
defer t.Errorf("Got no results for %q", tc.Query)
} else if len(results) < len(tc.ResultURL) {
diff --git a/src/search/perform.go b/src/search/perform.go
index fbfd3f5f..552ba1a9 100644
--- a/src/search/perform.go
+++ b/src/search/perform.go
@@ -9,55 +9,47 @@ import (
"github.com/hearchco/hearchco/src/anonymize"
"github.com/hearchco/hearchco/src/config"
"github.com/hearchco/hearchco/src/search/bucket"
- "github.com/hearchco/hearchco/src/search/category"
"github.com/hearchco/hearchco/src/search/engines"
"github.com/hearchco/hearchco/src/search/rank"
"github.com/hearchco/hearchco/src/search/result"
"github.com/rs/zerolog/log"
)
-func PerformSearch(query string, options engines.Options, settings map[engines.Name]config.Settings, categories map[category.Name]config.Category, salt string) []result.Result {
+func PerformSearch(query string, options engines.Options, categoryConf config.Category, settings map[engines.Name]config.Settings, salt string) []result.Result {
+ // check for empty query
if query == "" {
log.Trace().Msg("Empty search query.")
return []result.Result{}
}
+ // start searching
searchTimer := time.Now()
-
- query, cat, timings, enginesToRun := procBang(query, options.Category, settings, categories)
- // set the new category only within the scope of this function
- options.Category = cat
- query = url.QueryEscape(query)
-
- // check again after the bang is taken out
- if query == "" {
- log.Trace().Msg("Empty search query (with bang present).")
- return []result.Result{}
- }
-
log.Debug().
Str("queryAnon", anonymize.String(query)).
Str("queryHash", anonymize.HashToSHA256B64(query)).
- Msg("Searching")
+ Msg("Searching...")
+ // getting results from engines
resTimer := time.Now()
log.Debug().Msg("Waiting for results from engines...")
- resultMap := runEngines(enginesToRun, query, options, settings, timings, salt)
+ resultMap := runEngines(categoryConf.Engines, url.QueryEscape(query), options, settings, categoryConf.Timings, salt)
log.Debug().
Dur("duration", time.Since(resTimer)).
Msg("Got results")
+ // ranking results
rankTimer := time.Now()
log.Debug().Msg("Ranking...")
- results := rank.Rank(resultMap, categories[options.Category].Ranking)
+ results := rank.Rank(resultMap, categoryConf.Ranking)
log.Debug().
Dur("duration", time.Since(rankTimer)).
Msg("Finished ranking")
+ // finish searching
log.Debug().
Dur("duration", time.Since(searchTimer)).
Msg("Found results")
diff --git a/src/search/search.go b/src/search/search.go
index 3e0cf24c..0865ece7 100644
--- a/src/search/search.go
+++ b/src/search/search.go
@@ -4,13 +4,12 @@ import (
"github.com/hearchco/hearchco/src/anonymize"
"github.com/hearchco/hearchco/src/cache"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/search/category"
"github.com/hearchco/hearchco/src/search/engines"
"github.com/hearchco/hearchco/src/search/result"
"github.com/rs/zerolog/log"
)
-func Search(query string, options engines.Options, db cache.DB, settings map[engines.Name]config.Settings, categories map[category.Name]config.Category, salt string) ([]result.Result, bool) {
+func Search(query string, options engines.Options, db cache.DB, categoryConf config.Category, settings map[engines.Name]config.Settings, salt string) ([]result.Result, bool) {
var results []result.Result
var foundInDB bool
gerr := db.Get(query, &results)
@@ -39,7 +38,7 @@ func Search(query string, options engines.Options, db cache.DB, settings map[eng
Msg("Nothing found in cache, doing a clean search")
// the main line
- results = PerformSearch(query, options, settings, categories, salt)
+ results = PerformSearch(query, options, categoryConf, settings, salt)
result.Shorten(results, 2500)
}
From 7795e348730a159e25a4603a56e873a3b6befc27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?=
<31509435+aleksasiriski@users.noreply.github.com>
Date: Tue, 28 May 2024 15:24:48 +0200
Subject: [PATCH 2/2] chore(Makefile): fix run
---
Makefile | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/Makefile b/Makefile
index b66fb2cd..cdc75745 100644
--- a/Makefile
+++ b/Makefile
@@ -1,17 +1,17 @@
run:
- air --pretty
+ air -- --pretty
run-cli:
go run ./src --cli --pretty
debug:
- air -- -v --pretty
+ air -- --pretty -v
debug-cli:
- go run ./srv -v --cli --pretty
+ go run ./srv --cli --pretty -v
trace:
- air -- -vv --pretty
+ air -- --pretty -vv
trace-cli:
- go run ./src -vv --cli --pretty
+ go run ./src --cli --pretty -vv
install:
go get ./...