From c4316cf5058431b4d79dd9d04c253cd3c2e294c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?=
<31509435+aleksasiriski@users.noreply.github.com>
Date: Thu, 8 Feb 2024 13:58:12 +0100
Subject: [PATCH 01/11] refactor(search): moved all search related pkgs into
search folder
---
generate/searcher/searcher.go | 2 +-
src/cli/climode.go | 6 +--
src/cli/setup.go | 4 +-
src/config/defaults.go | 4 +-
src/config/load.go | 4 +-
src/config/structs.go | 4 +-
src/router/search.go | 6 +--
src/{ => search}/bucket/bucket.go | 14 +++----
src/search/cache.go | 4 +-
src/{ => search}/category/category.go | 0
src/{ => search}/category/name.go | 0
.../engines/_engines_test/structs.go | 4 +-
.../engines/_engines_test/tester.go | 0
.../engines/_sedefaults}/sedefaults.go | 36 +++++++++---------
src/{ => search}/engines/bing/bing.go | 26 ++++++-------
src/{ => search}/engines/bing/bing.md | 0
src/{ => search}/engines/bing/bing_test.go | 4 +-
src/{ => search}/engines/bing/options.go | 4 +-
src/{ => search}/engines/brave/brave.go | 26 ++++++-------
src/{ => search}/engines/brave/brave_test.go | 4 +-
src/{ => search}/engines/brave/options.go | 4 +-
src/{ => search}/engines/duckduckgo/ddg.md | 0
.../engines/duckduckgo/duckduckgo.go | 24 ++++++------
.../engines/duckduckgo/duckduckgo_test.go | 4 +-
.../engines/duckduckgo/options.go | 4 +-
src/{ => search}/engines/etools/captcha.png | Bin
src/{ => search}/engines/etools/etools.go | 26 ++++++-------
src/{ => search}/engines/etools/etools.md | 0
.../engines/etools/etools_test.go | 4 +-
src/{ => search}/engines/etools/options.go | 4 +-
src/{ => search}/engines/google/google.go | 26 ++++++-------
.../engines/google/google_test.go | 4 +-
src/{ => search}/engines/google/options.go | 2 +-
.../engines/googlescholar/googlescholar.go | 26 ++++++-------
.../googlescholar/googlescholar_test.go | 4 +-
.../engines/googlescholar/options.go | 4 +-
src/{ => search}/engines/mojeek/mojeek.go | 26 ++++++-------
.../engines/mojeek/mojeek_test.go | 4 +-
src/{ => search}/engines/mojeek/options.go | 2 +-
src/{ => search}/engines/name.go | 2 +-
.../engines/presearch/json_response.go | 0
src/{ => search}/engines/presearch/options.go | 2 +-
.../engines/presearch/presearch.go | 24 ++++++------
.../engines/presearch/presearch.md | 0
.../engines/presearch/presearch_test.go | 4 +-
.../engines/qwant/json_response.go | 0
src/{ => search}/engines/qwant/options.go | 4 +-
src/{ => search}/engines/qwant/qwant.go | 22 +++++------
src/{ => search}/engines/qwant/qwant.md | 0
src/{ => search}/engines/qwant/qwant_test.go | 4 +-
.../engines/startpage/image-1.png | Bin
.../engines/startpage/image-2.png | Bin
src/{ => search}/engines/startpage/image.png | Bin
src/{ => search}/engines/startpage/options.go | 2 +-
.../engines/startpage/startpage.go | 26 ++++++-------
.../engines/startpage/startpage.md | 0
.../engines/startpage/startpage_test.go | 4 +-
src/{ => search}/engines/structs.go | 19 +--------
.../engines/swisscows/authenticator.go | 0
.../engines/swisscows/image-1.png | Bin
src/{ => search}/engines/swisscows/image.png | Bin
.../engines/swisscows/json_response.go | 0
src/{ => search}/engines/swisscows/options.go | 4 +-
.../engines/swisscows/swisscows.go | 22 +++++------
.../engines/swisscows/swisscows.md | 0
.../engines/swisscows/swisscows_test.go | 4 +-
src/{ => search}/engines/timeout.go | 0
src/{ => search}/engines/yahoo/options.go | 4 +-
src/{ => search}/engines/yahoo/yahoo.go | 26 ++++++-------
src/{ => search}/engines/yahoo/yahoo_test.go | 4 +-
src/{ => search}/engines/yep/json_response.go | 0
src/{ => search}/engines/yep/options.go | 2 +-
src/{ => search}/engines/yep/yep.go | 22 +++++------
src/{ => search}/engines/yep/yep.md | 0
src/{ => search}/engines/yep/yep_test.go | 4 +-
src/search/perform.go | 10 ++---
src/{ => search}/rank/math.go | 0
src/{ => search}/rank/rank.go | 6 +--
src/{ => search}/rank/sorting.go | 2 +-
src/{bucket => search}/result/result.go | 3 +-
src/search/result/retrieved.go | 20 ++++++++++
src/search/search.go | 4 +-
82 files changed, 294 insertions(+), 276 deletions(-)
rename src/{ => search}/bucket/bucket.go (83%)
rename src/{ => search}/category/category.go (100%)
rename src/{ => search}/category/name.go (100%)
rename src/{ => search}/engines/_engines_test/structs.go (88%)
rename src/{ => search}/engines/_engines_test/tester.go (100%)
rename src/{sedefaults => search/engines/_sedefaults}/sedefaults.go (84%)
rename src/{ => search}/engines/bing/bing.go (78%)
rename src/{ => search}/engines/bing/bing.md (100%)
rename src/{ => search}/engines/bing/bing_test.go (87%)
rename src/{ => search}/engines/bing/options.go (87%)
rename src/{ => search}/engines/brave/brave.go (75%)
rename src/{ => search}/engines/brave/brave_test.go (87%)
rename src/{ => search}/engines/brave/options.go (88%)
rename src/{ => search}/engines/duckduckgo/ddg.md (100%)
rename src/{ => search}/engines/duckduckgo/duckduckgo.go (74%)
rename src/{ => search}/engines/duckduckgo/duckduckgo_test.go (87%)
rename src/{ => search}/engines/duckduckgo/options.go (89%)
rename src/{ => search}/engines/etools/captcha.png (100%)
rename src/{ => search}/engines/etools/etools.go (71%)
rename src/{ => search}/engines/etools/etools.md (100%)
rename src/{ => search}/engines/etools/etools_test.go (87%)
rename src/{ => search}/engines/etools/options.go (89%)
rename src/{ => search}/engines/google/google.go (66%)
rename src/{ => search}/engines/google/google_test.go (87%)
rename src/{ => search}/engines/google/options.go (89%)
rename src/{ => search}/engines/googlescholar/googlescholar.go (73%)
rename src/{ => search}/engines/googlescholar/googlescholar_test.go (89%)
rename src/{ => search}/engines/googlescholar/options.go (88%)
rename src/{ => search}/engines/mojeek/mojeek.go (72%)
rename src/{ => search}/engines/mojeek/mojeek_test.go (87%)
rename src/{ => search}/engines/mojeek/options.go (88%)
rename src/{ => search}/engines/name.go (87%)
rename src/{ => search}/engines/presearch/json_response.go (100%)
rename src/{ => search}/engines/presearch/options.go (91%)
rename src/{ => search}/engines/presearch/presearch.go (80%)
rename src/{ => search}/engines/presearch/presearch.md (100%)
rename src/{ => search}/engines/presearch/presearch_test.go (87%)
rename src/{ => search}/engines/qwant/json_response.go (100%)
rename src/{ => search}/engines/qwant/options.go (86%)
rename src/{ => search}/engines/qwant/qwant.go (86%)
rename src/{ => search}/engines/qwant/qwant.md (100%)
rename src/{ => search}/engines/qwant/qwant_test.go (87%)
rename src/{ => search}/engines/startpage/image-1.png (100%)
rename src/{ => search}/engines/startpage/image-2.png (100%)
rename src/{ => search}/engines/startpage/image.png (100%)
rename src/{ => search}/engines/startpage/options.go (89%)
rename src/{ => search}/engines/startpage/startpage.go (76%)
rename src/{ => search}/engines/startpage/startpage.md (100%)
rename src/{ => search}/engines/startpage/startpage_test.go (89%)
rename src/{ => search}/engines/structs.go (76%)
rename src/{ => search}/engines/swisscows/authenticator.go (100%)
rename src/{ => search}/engines/swisscows/image-1.png (100%)
rename src/{ => search}/engines/swisscows/image.png (100%)
rename src/{ => search}/engines/swisscows/json_response.go (100%)
rename src/{ => search}/engines/swisscows/options.go (82%)
rename src/{ => search}/engines/swisscows/swisscows.go (85%)
rename src/{ => search}/engines/swisscows/swisscows.md (100%)
rename src/{ => search}/engines/swisscows/swisscows_test.go (87%)
rename src/{ => search}/engines/timeout.go (100%)
rename src/{ => search}/engines/yahoo/options.go (91%)
rename src/{ => search}/engines/yahoo/yahoo.go (75%)
rename src/{ => search}/engines/yahoo/yahoo_test.go (87%)
rename src/{ => search}/engines/yep/json_response.go (100%)
rename src/{ => search}/engines/yep/options.go (92%)
rename src/{ => search}/engines/yep/yep.go (77%)
rename src/{ => search}/engines/yep/yep.md (100%)
rename src/{ => search}/engines/yep/yep_test.go (87%)
rename src/{ => search}/rank/math.go (100%)
rename src/{ => search}/rank/rank.go (93%)
rename src/{ => search}/rank/sorting.go (94%)
rename src/{bucket => search}/result/result.go (90%)
create mode 100644 src/search/result/retrieved.go
diff --git a/generate/searcher/searcher.go b/generate/searcher/searcher.go
index 24d30b68..11f3ee37 100644
--- a/generate/searcher/searcher.go
+++ b/generate/searcher/searcher.go
@@ -22,7 +22,7 @@ var (
trimprefix = flag.String("trimprefix", "", "trim the `prefix` from the generated constant names")
buildTags = flag.String("tags", "", "comma-separated list of build tags to apply")
packageName = flag.String("packagename", "", "name of the package for generated code; default current package")
- enginesImport = flag.String("enginesimport", "github.com/hearchco/hearchco/src/engines", "source of the engines import, which is prefixed to imports for consts; default github.com/hearchco/hearchco/src/engines")
+ enginesImport = flag.String("enginesimport", "github.com/hearchco/hearchco/src/search/engines", "source of the engines import, which is prefixed to imports for consts; default github.com/hearchco/hearchco/src/search/engines")
linecomment = flag.Bool("linecomment", false, "use line comment text as printed text when present")
)
diff --git a/src/cli/climode.go b/src/cli/climode.go
index 0d9a4572..6f692048 100644
--- a/src/cli/climode.go
+++ b/src/cli/climode.go
@@ -5,12 +5,12 @@ import (
"time"
"github.com/hearchco/hearchco/src/anonymize"
- "github.com/hearchco/hearchco/src/bucket/result"
"github.com/hearchco/hearchco/src/cache"
- "github.com/hearchco/hearchco/src/category"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
"github.com/hearchco/hearchco/src/search"
+ "github.com/hearchco/hearchco/src/search/category"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/result"
"github.com/rs/zerolog/log"
)
diff --git a/src/cli/setup.go b/src/cli/setup.go
index 049592cb..513e44db 100644
--- a/src/cli/setup.go
+++ b/src/cli/setup.go
@@ -4,8 +4,8 @@ import (
"fmt"
"github.com/alecthomas/kong"
- "github.com/hearchco/hearchco/src/category"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/category"
+ "github.com/hearchco/hearchco/src/search/engines"
"github.com/rs/zerolog/log"
)
diff --git a/src/config/defaults.go b/src/config/defaults.go
index 56a10612..ecd5b60a 100644
--- a/src/config/defaults.go
+++ b/src/config/defaults.go
@@ -4,9 +4,9 @@ import (
"log"
"time"
- "github.com/hearchco/hearchco/src/category"
- "github.com/hearchco/hearchco/src/engines"
"github.com/hearchco/hearchco/src/moretime"
+ "github.com/hearchco/hearchco/src/search/category"
+ "github.com/hearchco/hearchco/src/search/engines"
)
const DefaultLocale string = "en_US"
diff --git a/src/config/load.go b/src/config/load.go
index f82d6bbe..563f74b3 100644
--- a/src/config/load.go
+++ b/src/config/load.go
@@ -6,9 +6,9 @@ import (
"strings"
"time"
- "github.com/hearchco/hearchco/src/category"
- "github.com/hearchco/hearchco/src/engines"
"github.com/hearchco/hearchco/src/moretime"
+ "github.com/hearchco/hearchco/src/search/category"
+ "github.com/hearchco/hearchco/src/search/engines"
"github.com/knadh/koanf/parsers/yaml"
"github.com/knadh/koanf/providers/env"
"github.com/knadh/koanf/providers/file"
diff --git a/src/config/structs.go b/src/config/structs.go
index 6ee590f5..5a8bd818 100644
--- a/src/config/structs.go
+++ b/src/config/structs.go
@@ -3,8 +3,8 @@ package config
import (
"time"
- "github.com/hearchco/hearchco/src/category"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/category"
+ "github.com/hearchco/hearchco/src/search/engines"
)
type EngineRanking struct {
diff --git a/src/router/search.go b/src/router/search.go
index ac9ad080..d88e8e56 100644
--- a/src/router/search.go
+++ b/src/router/search.go
@@ -8,12 +8,12 @@ import (
"github.com/gin-gonic/gin"
"github.com/goccy/go-json"
- "github.com/hearchco/hearchco/src/bucket/result"
"github.com/hearchco/hearchco/src/cache"
- "github.com/hearchco/hearchco/src/category"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
"github.com/hearchco/hearchco/src/search"
+ "github.com/hearchco/hearchco/src/search/category"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/result"
)
func Search(c *gin.Context, conf *config.Config, db cache.DB) error {
diff --git a/src/bucket/bucket.go b/src/search/bucket/bucket.go
similarity index 83%
rename from src/bucket/bucket.go
rename to src/search/bucket/bucket.go
index fd8a8463..307efff2 100644
--- a/src/bucket/bucket.go
+++ b/src/search/bucket/bucket.go
@@ -5,9 +5,9 @@ import (
"sync"
"github.com/gocolly/colly/v2"
- "github.com/hearchco/hearchco/src/bucket/result"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/result"
"github.com/rs/zerolog/log"
)
@@ -16,7 +16,7 @@ type Relay struct {
Mutex sync.RWMutex
}
-func AddSEResult(seResult *engines.RetrievedResult, seName engines.Name, relay *Relay, options *engines.Options, pagesCol *colly.Collector) {
+func AddSEResult(seResult *result.RetrievedResult, seName engines.Name, relay *Relay, options *engines.Options, pagesCol *colly.Collector) {
log.Trace().
Str("engine", seName.String()).
Str("title", seResult.Title).
@@ -28,7 +28,7 @@ func AddSEResult(seResult *engines.RetrievedResult, seName engines.Name, relay *
relay.Mutex.RUnlock()
if !exists {
- engineRanks := make([]engines.RetrievedRank, len(config.EnabledEngines))
+ engineRanks := make([]result.RetrievedRank, len(config.EnabledEngines))
engineRanks[0] = seResult.Rank
result := result.Result{
URL: seResult.URL,
@@ -98,14 +98,14 @@ func SetResultResponse(link string, response *colly.Response, relay *Relay, seNa
return nil
}
-func MakeSEResult(urll string, title string, description string, searchEngineName engines.Name, sePage int, seOnPageRank int) *engines.RetrievedResult {
- ser := engines.RetrievedRank{
+func MakeSEResult(urll string, title string, description string, searchEngineName engines.Name, sePage int, seOnPageRank int) *result.RetrievedResult {
+ ser := result.RetrievedRank{
SearchEngine: searchEngineName,
Rank: 0,
Page: uint(sePage),
OnPageRank: uint(seOnPageRank),
}
- res := engines.RetrievedResult{
+ res := result.RetrievedResult{
URL: urll,
Title: title,
Description: description,
diff --git a/src/search/cache.go b/src/search/cache.go
index ed393c17..007f6989 100644
--- a/src/search/cache.go
+++ b/src/search/cache.go
@@ -2,10 +2,10 @@ package search
import (
"github.com/hearchco/hearchco/src/anonymize"
- "github.com/hearchco/hearchco/src/bucket/result"
"github.com/hearchco/hearchco/src/cache"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/result"
"github.com/rs/zerolog/log"
)
diff --git a/src/category/category.go b/src/search/category/category.go
similarity index 100%
rename from src/category/category.go
rename to src/search/category/category.go
diff --git a/src/category/name.go b/src/search/category/name.go
similarity index 100%
rename from src/category/name.go
rename to src/search/category/name.go
diff --git a/src/engines/_engines_test/structs.go b/src/search/engines/_engines_test/structs.go
similarity index 88%
rename from src/engines/_engines_test/structs.go
rename to src/search/engines/_engines_test/structs.go
index 63eed460..24ec18bd 100644
--- a/src/engines/_engines_test/structs.go
+++ b/src/search/engines/_engines_test/structs.go
@@ -3,9 +3,9 @@ package _engines_test
import (
"time"
- "github.com/hearchco/hearchco/src/category"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/category"
+ "github.com/hearchco/hearchco/src/search/engines"
)
type TestCaseHasAnyResults struct {
diff --git a/src/engines/_engines_test/tester.go b/src/search/engines/_engines_test/tester.go
similarity index 100%
rename from src/engines/_engines_test/tester.go
rename to src/search/engines/_engines_test/tester.go
diff --git a/src/sedefaults/sedefaults.go b/src/search/engines/_sedefaults/sedefaults.go
similarity index 84%
rename from src/sedefaults/sedefaults.go
rename to src/search/engines/_sedefaults/sedefaults.go
index 57d8b0f0..6ceecf52 100644
--- a/src/sedefaults/sedefaults.go
+++ b/src/search/engines/_sedefaults/sedefaults.go
@@ -1,4 +1,4 @@
-package sedefaults
+package _sedefaults
import (
"context"
@@ -11,9 +11,9 @@ import (
"github.com/gocolly/colly/v2"
"github.com/gocolly/colly/v2/proxy"
- "github.com/hearchco/hearchco/src/bucket"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/bucket"
+ "github.com/hearchco/hearchco/src/search/engines"
"github.com/hearchco/hearchco/src/search/useragent"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
@@ -26,12 +26,12 @@ func PagesColRequest(seName engines.Name, pagesCol *colly.Collector, ctx context
log.Trace().
Err(err).
Str("engine", seName.String()).
- Msg("sedefaults.PagesColRequest() -> pagesCol.OnRequest(): context timeout error")
+ Msg("_sedefaults.PagesColRequest() -> pagesCol.OnRequest(): context timeout error")
} else {
log.Error().
Err(err).
Str("engine", seName.String()).
- Msg("sedefaults.PagesColRequest() -> pagesCol.OnRequest(): context error")
+ Msg("_sedefaults.PagesColRequest() -> pagesCol.OnRequest(): context error")
}
r.Abort()
return
@@ -48,14 +48,14 @@ func PagesColError(seName engines.Name, pagesCol *colly.Collector) {
Err(err).
Str("engine", seName.String()).
Str("url", urll).
- Msg("sedefaults.PagesColError() -> pagesCol.OnError(): request timeout error for url")
+ Msg("_sedefaults.PagesColError() -> pagesCol.OnError(): request timeout error for url")
} else {
log.Trace().
Err(err).
Str("engine", seName.String()).
Str("url", urll).
Str("response", string(r.Body)).
- Msg("sedefaults.PagesColError() -> pagesCol.OnError(): request error for url")
+ Msg("_sedefaults.PagesColError() -> pagesCol.OnError(): request error for url")
}
})
}
@@ -65,7 +65,7 @@ func PagesColResponse(seName engines.Name, pagesCol *colly.Collector, relay *buc
urll := r.Ctx.Get("originalURL")
err := bucket.SetResultResponse(urll, r, relay, seName)
if err != nil {
- log.Error().Err(err).Msg("sedefaults.PagesColResponse(): error setting result")
+ log.Error().Err(err).Msg("_sedefaults.PagesColResponse(): error setting result")
}
})
}
@@ -77,12 +77,12 @@ func ColRequest(seName engines.Name, col *colly.Collector, ctx context.Context)
log.Trace().
Err(err).
Str("engine", seName.String()).
- Msg("sedefaults.ColRequest() -> col.OnRequest(): context timeout error")
+ Msg("_sedefaults.ColRequest() -> col.OnRequest(): context timeout error")
} else {
log.Error().
Err(err).
Str("engine", seName.String()).
- Msg("sedefaults.ColRequest() -> col.OnRequest(): context error")
+ Msg("_sedefaults.ColRequest() -> col.OnRequest(): context error")
}
r.Abort()
return
@@ -97,7 +97,7 @@ func ColError(seName engines.Name, col *colly.Collector) {
// Err(err). // timeout error produces Get "url" error with the query
Str("engine", seName.String()).
// Str("url", urll). // can't reliably anonymize it (because it's engine dependent and query isn't passed to this function)
- Msg("sedefaults.ColError() -> col.OnError(): request timeout error for url")
+ Msg("_sedefaults.ColError() -> col.OnError(): request timeout error for url")
} else {
log.Error().
Err(err).
@@ -105,7 +105,7 @@ func ColError(seName engines.Name, col *colly.Collector) {
// Str("url", urll). // can't reliably anonymize it (because it's engine dependent and query isn't passed to this function)
Int("statusCode", r.StatusCode).
Str("response", string(r.Body)). // query can be present, depending on the response from the engine (Google has the query in 3 places)
- Msg("sedefaults.ColError() -> col.OnError(): request error for url")
+ Msg("_sedefaults.ColError() -> col.OnError(): request error for url")
dumpPath := fmt.Sprintf("%v%v_col.log.html", config.LogDumpLocation, seName.String())
log.Debug().
@@ -117,10 +117,10 @@ func ColError(seName engines.Name, col *colly.Collector) {
log.Error().
Err(bodyWriteErr).
Str("engine", seName.String()).
- Msg("sedefaults.ColError() -> col.OnError(): error writing html response body to file")
+ Msg("_sedefaults.ColError() -> col.OnError(): error writing html response body to file")
}
}).
- Msg("sedefaults.ColError() -> col.OnError(): html response written")
+ Msg("_sedefaults.ColError() -> col.OnError(): html response written")
}
})
}
@@ -143,7 +143,7 @@ func Prepare(seName engines.Name, options *engines.Options, settings *config.Set
log.Panic().
Str("engine", seName.String()).
Int("requestedResultsPerPage", settings.RequestedResultsPerPage).
- Msg("sedefaults.Prepare(): setting not supported by engine")
+ Msg("_sedefaults.Prepare(): setting not supported by engine")
// ^PANIC
}
if settings.RequestedResultsPerPage == 0 && support.RequestedResultsPerPage {
@@ -198,7 +198,7 @@ func InitializeCollectors(colPtr **colly.Collector, pagesColPtr **colly.Collecto
log.Error().
Err(err).
Str("limitRule", fmt.Sprintf("%v", limitRule)).
- Msg("sedefaults.InitializeCollectors(): failed adding new limit rule")
+ Msg("_sedefaults.InitializeCollectors(): failed adding new limit rule")
}
if timings.Timeout != 0 {
@@ -221,7 +221,7 @@ func InitializeCollectors(colPtr **colly.Collector, pagesColPtr **colly.Collecto
log.Fatal().
Err(err).
Strs("proxies", settings.Proxies).
- Msg("sedefaults.InitializeCollectors(): failed creating proxy switcher")
+ Msg("_sedefaults.InitializeCollectors(): failed creating proxy switcher")
}
(*colPtr).SetProxyFunc(rp)
@@ -267,7 +267,7 @@ func PageFromContext(ctx *colly.Context, seName engines.Name) int {
Err(converr).
Str("engine", seName.String()).
Str("page", pageStr).
- Msg("sedefaults.PageFromContext(): failed to convert page number to int")
+ Msg("_sedefaults.PageFromContext(): failed to convert page number to int")
// ^PANIC
}
return page
diff --git a/src/engines/bing/bing.go b/src/search/engines/bing/bing.go
similarity index 78%
rename from src/engines/bing/bing.go
rename to src/search/engines/bing/bing.go
index f0dc2454..e4cd7380 100644
--- a/src/engines/bing/bing.go
+++ b/src/search/engines/bing/bing.go
@@ -9,16 +9,16 @@ import (
"github.com/gocolly/colly/v2"
"github.com/hearchco/hearchco/src/anonymize"
- "github.com/hearchco/hearchco/src/bucket"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/bucket"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_sedefaults"
"github.com/hearchco/hearchco/src/search/parse"
- "github.com/hearchco/hearchco/src/sedefaults"
"github.com/rs/zerolog/log"
)
func Search(ctx context.Context, query string, relay *bucket.Relay, options engines.Options, settings config.Settings, timings config.Timings) error {
- if err := sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
+ if err := _sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
return err
}
@@ -26,14 +26,14 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
+ _sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
- sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
- sedefaults.PagesColError(Info.Name, pagesCol)
- sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
+ _sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
+ _sedefaults.PagesColError(Info.Name, pagesCol)
+ _sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
- sedefaults.ColRequest(Info.Name, col, ctx)
- sedefaults.ColError(Info.Name, col)
+ _sedefaults.ColRequest(Info.Name, col, ctx)
+ _sedefaults.ColError(Info.Name, col)
var pageRankCounter []int = make([]int, options.MaxPages*Info.ResultsPerPage)
@@ -55,7 +55,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
descText = strings.Split(descText, "Web")[1]
}
- page := sedefaults.PageFromContext(e.Request.Ctx, Info.Name)
+ page := _sedefaults.PageFromContext(e.Request.Ctx, Info.Name)
res := bucket.MakeSEResult(linkText, titleText, descText, Info.Name, page, pageRankCounter[page]+1)
bucket.AddSEResult(res, Info.Name, relay, &options, pagesCol)
@@ -77,7 +77,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + query + localeParam
anonUrll := Info.URL + anonymize.String(query) + localeParam
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
for i := 1; i < options.MaxPages; i++ {
colCtx = colly.NewContext()
@@ -85,7 +85,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + query + "&first=" + strconv.Itoa(i*10+1) + localeParam
anonUrll := Info.URL + anonymize.String(query) + "&first=" + strconv.Itoa(i*10+1) + localeParam
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
}
col.Wait()
diff --git a/src/engines/bing/bing.md b/src/search/engines/bing/bing.md
similarity index 100%
rename from src/engines/bing/bing.md
rename to src/search/engines/bing/bing.md
diff --git a/src/engines/bing/bing_test.go b/src/search/engines/bing/bing_test.go
similarity index 87%
rename from src/engines/bing/bing_test.go
rename to src/search/engines/bing/bing_test.go
index b8158557..feed8ec4 100644
--- a/src/engines/bing/bing_test.go
+++ b/src/search/engines/bing/bing_test.go
@@ -3,8 +3,8 @@ package bing_test
import (
"testing"
- "github.com/hearchco/hearchco/src/engines"
- "github.com/hearchco/hearchco/src/engines/_engines_test"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_engines_test"
)
func TestSearch(t *testing.T) {
diff --git a/src/engines/bing/options.go b/src/search/engines/bing/options.go
similarity index 87%
rename from src/engines/bing/options.go
rename to src/search/engines/bing/options.go
index 723c0cfc..b495c8e3 100644
--- a/src/engines/bing/options.go
+++ b/src/search/engines/bing/options.go
@@ -1,6 +1,8 @@
package bing
-import "github.com/hearchco/hearchco/src/engines"
+import (
+ "github.com/hearchco/hearchco/src/search/engines"
+)
var Info engines.Info = engines.Info{
Domain: "www.bing.com",
diff --git a/src/engines/brave/brave.go b/src/search/engines/brave/brave.go
similarity index 75%
rename from src/engines/brave/brave.go
rename to src/search/engines/brave/brave.go
index 58196f8d..e0289954 100644
--- a/src/engines/brave/brave.go
+++ b/src/search/engines/brave/brave.go
@@ -7,15 +7,15 @@ import (
"github.com/gocolly/colly/v2"
"github.com/hearchco/hearchco/src/anonymize"
- "github.com/hearchco/hearchco/src/bucket"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/bucket"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_sedefaults"
"github.com/hearchco/hearchco/src/search/parse"
- "github.com/hearchco/hearchco/src/sedefaults"
)
func Search(ctx context.Context, query string, relay *bucket.Relay, options engines.Options, settings config.Settings, timings config.Timings) error {
- if err := sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
+ if err := _sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
return err
}
@@ -23,14 +23,14 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
+ _sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
- sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
- sedefaults.PagesColError(Info.Name, pagesCol)
- sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
+ _sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
+ _sedefaults.PagesColError(Info.Name, pagesCol)
+ _sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
- sedefaults.ColRequest(Info.Name, col, ctx)
- sedefaults.ColError(Info.Name, col)
+ _sedefaults.ColRequest(Info.Name, col, ctx)
+ _sedefaults.ColError(Info.Name, col)
var pageRankCounter []int = make([]int, options.MaxPages*Info.ResultsPerPage)
@@ -58,7 +58,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
descText = strings.TrimSpace(dom.Find("p.snippet-description").Text())
}
- page := sedefaults.PageFromContext(e.Request.Ctx, Info.Name)
+ page := _sedefaults.PageFromContext(e.Request.Ctx, Info.Name)
res := bucket.MakeSEResult(linkText, titleText, descText, Info.Name, page, pageRankCounter[page]+1)
bucket.AddSEResult(res, Info.Name, relay, &options, pagesCol)
@@ -71,7 +71,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + query + "&source=web"
anonUrll := Info.URL + anonymize.String(query) + "&source=web"
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
for i := 1; i < options.MaxPages; i++ {
colCtx = colly.NewContext()
@@ -79,7 +79,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + query + "&spellcheck=0&offset=" + strconv.Itoa(i)
anonUrll := Info.URL + anonymize.String(query) + "&spellcheck=0&offset=" + strconv.Itoa(i)
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
}
col.Wait()
diff --git a/src/engines/brave/brave_test.go b/src/search/engines/brave/brave_test.go
similarity index 87%
rename from src/engines/brave/brave_test.go
rename to src/search/engines/brave/brave_test.go
index b528a4cc..9157745e 100644
--- a/src/engines/brave/brave_test.go
+++ b/src/search/engines/brave/brave_test.go
@@ -3,8 +3,8 @@ package brave_test
import (
"testing"
- "github.com/hearchco/hearchco/src/engines"
- "github.com/hearchco/hearchco/src/engines/_engines_test"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_engines_test"
)
func TestSearch(t *testing.T) {
diff --git a/src/engines/brave/options.go b/src/search/engines/brave/options.go
similarity index 88%
rename from src/engines/brave/options.go
rename to src/search/engines/brave/options.go
index 7dabd01f..b7af318a 100644
--- a/src/engines/brave/options.go
+++ b/src/search/engines/brave/options.go
@@ -1,6 +1,8 @@
package brave
-import "github.com/hearchco/hearchco/src/engines"
+import (
+ "github.com/hearchco/hearchco/src/search/engines"
+)
var Info engines.Info = engines.Info{
Domain: "search.brave.com",
diff --git a/src/engines/duckduckgo/ddg.md b/src/search/engines/duckduckgo/ddg.md
similarity index 100%
rename from src/engines/duckduckgo/ddg.md
rename to src/search/engines/duckduckgo/ddg.md
diff --git a/src/engines/duckduckgo/duckduckgo.go b/src/search/engines/duckduckgo/duckduckgo.go
similarity index 74%
rename from src/engines/duckduckgo/duckduckgo.go
rename to src/search/engines/duckduckgo/duckduckgo.go
index 9b1c6c78..5d10f714 100644
--- a/src/engines/duckduckgo/duckduckgo.go
+++ b/src/search/engines/duckduckgo/duckduckgo.go
@@ -9,15 +9,15 @@ import (
"github.com/PuerkitoBio/goquery"
"github.com/gocolly/colly/v2"
"github.com/hearchco/hearchco/src/anonymize"
- "github.com/hearchco/hearchco/src/bucket"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/bucket"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_sedefaults"
"github.com/hearchco/hearchco/src/search/parse"
- "github.com/hearchco/hearchco/src/sedefaults"
)
func Search(ctx context.Context, query string, relay *bucket.Relay, options engines.Options, settings config.Settings, timings config.Timings) error {
- if err := sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
+ if err := _sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
return err
}
@@ -25,14 +25,14 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
+ _sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
- sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
- sedefaults.PagesColError(Info.Name, pagesCol)
- sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
+ _sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
+ _sedefaults.PagesColError(Info.Name, pagesCol)
+ _sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
- sedefaults.ColRequest(Info.Name, col, ctx)
- sedefaults.ColError(Info.Name, col)
+ _sedefaults.ColRequest(Info.Name, col, ctx)
+ _sedefaults.ColError(Info.Name, col)
localeCookie := getLocale(&options)
@@ -80,13 +80,13 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + "?q=" + query
anonUrll := Info.URL + "?q=" + anonymize.String(query)
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
for i := 1; i < options.MaxPages; i++ {
colCtx = colly.NewContext()
colCtx.Put("page", strconv.Itoa(i+1))
- sedefaults.DoPostRequest(Info.URL, strings.NewReader("q="+query+"&dc="+strconv.Itoa(i*20)), colCtx, col, Info.Name, &retError)
+ _sedefaults.DoPostRequest(Info.URL, strings.NewReader("q="+query+"&dc="+strconv.Itoa(i*20)), colCtx, col, Info.Name, &retError)
}
col.Wait()
diff --git a/src/engines/duckduckgo/duckduckgo_test.go b/src/search/engines/duckduckgo/duckduckgo_test.go
similarity index 87%
rename from src/engines/duckduckgo/duckduckgo_test.go
rename to src/search/engines/duckduckgo/duckduckgo_test.go
index 061e9037..1f78a3dc 100644
--- a/src/engines/duckduckgo/duckduckgo_test.go
+++ b/src/search/engines/duckduckgo/duckduckgo_test.go
@@ -3,8 +3,8 @@ package duckduckgo_test
import (
"testing"
- "github.com/hearchco/hearchco/src/engines"
- "github.com/hearchco/hearchco/src/engines/_engines_test"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_engines_test"
)
func TestSearch(t *testing.T) {
diff --git a/src/engines/duckduckgo/options.go b/src/search/engines/duckduckgo/options.go
similarity index 89%
rename from src/engines/duckduckgo/options.go
rename to src/search/engines/duckduckgo/options.go
index bf044edc..af33d20c 100644
--- a/src/engines/duckduckgo/options.go
+++ b/src/search/engines/duckduckgo/options.go
@@ -1,6 +1,8 @@
package duckduckgo
-import "github.com/hearchco/hearchco/src/engines"
+import (
+ "github.com/hearchco/hearchco/src/search/engines"
+)
var Info engines.Info = engines.Info{
Domain: "lite.duckduckgo.com",
diff --git a/src/engines/etools/captcha.png b/src/search/engines/etools/captcha.png
similarity index 100%
rename from src/engines/etools/captcha.png
rename to src/search/engines/etools/captcha.png
diff --git a/src/engines/etools/etools.go b/src/search/engines/etools/etools.go
similarity index 71%
rename from src/engines/etools/etools.go
rename to src/search/engines/etools/etools.go
index 411cfed9..517ae878 100644
--- a/src/engines/etools/etools.go
+++ b/src/search/engines/etools/etools.go
@@ -6,16 +6,16 @@ import (
"strings"
"github.com/gocolly/colly/v2"
- "github.com/hearchco/hearchco/src/bucket"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/bucket"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_sedefaults"
"github.com/hearchco/hearchco/src/search/parse"
- "github.com/hearchco/hearchco/src/sedefaults"
"github.com/rs/zerolog/log"
)
func Search(ctx context.Context, query string, relay *bucket.Relay, options engines.Options, settings config.Settings, timings config.Timings) error {
- if err := sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
+ if err := _sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
return err
}
@@ -23,14 +23,14 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
+ _sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
- sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
- sedefaults.PagesColError(Info.Name, pagesCol)
- sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
+ _sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
+ _sedefaults.PagesColError(Info.Name, pagesCol)
+ _sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
- sedefaults.ColRequest(Info.Name, col, ctx)
- sedefaults.ColError(Info.Name, col)
+ _sedefaults.ColRequest(Info.Name, col, ctx)
+ _sedefaults.ColError(Info.Name, col)
var pageRankCounter []int = make([]int, options.MaxPages*Info.ResultsPerPage)
@@ -53,7 +53,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
descText := strings.TrimSpace(dom.Find(dompaths.Description).Text())
if hrefExists && linkText != "" && linkText != "#" && titleText != "" {
- page := sedefaults.PageFromContext(e.Request.Ctx, Info.Name)
+ page := _sedefaults.PageFromContext(e.Request.Ctx, Info.Name)
res := bucket.MakeSEResult(linkText, titleText, descText, Info.Name, page, pageRankCounter[page]+1)
bucket.AddSEResult(res, Info.Name, relay, &options, pagesCol)
@@ -74,7 +74,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
colCtx := colly.NewContext()
colCtx.Put("page", strconv.Itoa(1))
- sedefaults.DoPostRequest(Info.URL, strings.NewReader("query="+query+"&country=web&language=all"+safeSearchParam), colCtx, col, Info.Name, &retError)
+ _sedefaults.DoPostRequest(Info.URL, strings.NewReader("query="+query+"&country=web&language=all"+safeSearchParam), colCtx, col, Info.Name, &retError)
col.Wait() // wait so I can get the JSESSION cookie back
for i := 1; i < options.MaxPages; i++ {
@@ -83,7 +83,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
colCtx.Put("page", pageStr)
// query not needed as its saved in the session
- sedefaults.DoGetRequest(pageURL+pageStr, pageURL+pageStr, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(pageURL+pageStr, pageURL+pageStr, colCtx, col, Info.Name, &retError)
}
col.Wait()
diff --git a/src/engines/etools/etools.md b/src/search/engines/etools/etools.md
similarity index 100%
rename from src/engines/etools/etools.md
rename to src/search/engines/etools/etools.md
diff --git a/src/engines/etools/etools_test.go b/src/search/engines/etools/etools_test.go
similarity index 87%
rename from src/engines/etools/etools_test.go
rename to src/search/engines/etools/etools_test.go
index 7e1f4013..5c60d541 100644
--- a/src/engines/etools/etools_test.go
+++ b/src/search/engines/etools/etools_test.go
@@ -3,8 +3,8 @@ package etools_test
import (
"testing"
- "github.com/hearchco/hearchco/src/engines"
- "github.com/hearchco/hearchco/src/engines/_engines_test"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_engines_test"
)
func TestSearch(t *testing.T) {
diff --git a/src/engines/etools/options.go b/src/search/engines/etools/options.go
similarity index 89%
rename from src/engines/etools/options.go
rename to src/search/engines/etools/options.go
index 48b7219b..b334e3e1 100644
--- a/src/engines/etools/options.go
+++ b/src/search/engines/etools/options.go
@@ -1,6 +1,8 @@
package etools
-import "github.com/hearchco/hearchco/src/engines"
+import (
+ "github.com/hearchco/hearchco/src/search/engines"
+)
const pageURL string = "https://www.etools.ch/search.do?page="
diff --git a/src/engines/google/google.go b/src/search/engines/google/google.go
similarity index 66%
rename from src/engines/google/google.go
rename to src/search/engines/google/google.go
index c4991710..b34c304e 100644
--- a/src/engines/google/google.go
+++ b/src/search/engines/google/google.go
@@ -7,15 +7,15 @@ import (
"github.com/gocolly/colly/v2"
"github.com/hearchco/hearchco/src/anonymize"
- "github.com/hearchco/hearchco/src/bucket"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/bucket"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_sedefaults"
"github.com/hearchco/hearchco/src/search/parse"
- "github.com/hearchco/hearchco/src/sedefaults"
)
func Search(ctx context.Context, query string, relay *bucket.Relay, options engines.Options, settings config.Settings, timings config.Timings) error {
- if err := sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
+ if err := _sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
return err
}
@@ -23,14 +23,14 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
+ _sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
- sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
- sedefaults.PagesColError(Info.Name, pagesCol)
- sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
+ _sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
+ _sedefaults.PagesColError(Info.Name, pagesCol)
+ _sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
- sedefaults.ColRequest(Info.Name, col, ctx)
- sedefaults.ColError(Info.Name, col)
+ _sedefaults.ColRequest(Info.Name, col, ctx)
+ _sedefaults.ColError(Info.Name, col)
var pageRankCounter []int = make([]int, options.MaxPages*Info.ResultsPerPage)
@@ -43,7 +43,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
descText := strings.TrimSpace(dom.Find(dompaths.Description).Text())
if hrefExists && linkText != "" && linkText != "#" && titleText != "" {
- page := sedefaults.PageFromContext(e.Request.Ctx, Info.Name)
+ page := _sedefaults.PageFromContext(e.Request.Ctx, Info.Name)
res := bucket.MakeSEResult(linkText, titleText, descText, Info.Name, page, pageRankCounter[page]+1)
bucket.AddSEResult(res, Info.Name, relay, &options, pagesCol)
@@ -56,7 +56,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + query
anonUrll := Info.URL + anonymize.String(query)
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
for i := 1; i < options.MaxPages; i++ {
colCtx = colly.NewContext()
@@ -64,7 +64,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + query + "&start=" + strconv.Itoa(i*10)
anonUrll := Info.URL + anonymize.String(query) + "&start=" + strconv.Itoa(i*10)
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
}
col.Wait()
diff --git a/src/engines/google/google_test.go b/src/search/engines/google/google_test.go
similarity index 87%
rename from src/engines/google/google_test.go
rename to src/search/engines/google/google_test.go
index 5c466684..e19bde40 100644
--- a/src/engines/google/google_test.go
+++ b/src/search/engines/google/google_test.go
@@ -3,8 +3,8 @@ package google_test
import (
"testing"
- "github.com/hearchco/hearchco/src/engines"
- "github.com/hearchco/hearchco/src/engines/_engines_test"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_engines_test"
)
func TestSearch(t *testing.T) {
diff --git a/src/engines/google/options.go b/src/search/engines/google/options.go
similarity index 89%
rename from src/engines/google/options.go
rename to src/search/engines/google/options.go
index 58d8d376..b7fa92a1 100644
--- a/src/engines/google/options.go
+++ b/src/search/engines/google/options.go
@@ -1,7 +1,7 @@
package google
import (
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/engines"
)
var Info engines.Info = engines.Info{
diff --git a/src/engines/googlescholar/googlescholar.go b/src/search/engines/googlescholar/googlescholar.go
similarity index 73%
rename from src/engines/googlescholar/googlescholar.go
rename to src/search/engines/googlescholar/googlescholar.go
index 1d8e2600..5476526c 100644
--- a/src/engines/googlescholar/googlescholar.go
+++ b/src/search/engines/googlescholar/googlescholar.go
@@ -8,16 +8,16 @@ import (
"github.com/gocolly/colly/v2"
"github.com/hearchco/hearchco/src/anonymize"
- "github.com/hearchco/hearchco/src/bucket"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/bucket"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_sedefaults"
"github.com/hearchco/hearchco/src/search/parse"
- "github.com/hearchco/hearchco/src/sedefaults"
"github.com/rs/zerolog/log"
)
func Search(ctx context.Context, query string, relay *bucket.Relay, options engines.Options, settings config.Settings, timings config.Timings) error {
- if err := sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
+ if err := _sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
return err
}
@@ -25,14 +25,14 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
+ _sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
- sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
- sedefaults.PagesColError(Info.Name, pagesCol)
- sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
+ _sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
+ _sedefaults.PagesColError(Info.Name, pagesCol)
+ _sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
- sedefaults.ColRequest(Info.Name, col, ctx)
- sedefaults.ColError(Info.Name, col)
+ _sedefaults.ColRequest(Info.Name, col, ctx)
+ _sedefaults.ColError(Info.Name, col)
var pageRankCounter []int = make([]int, options.MaxPages*Info.ResultsPerPage)
@@ -48,7 +48,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
descText = citeInfo + " || " + descText
if hrefExists && linkText != "" && linkText != "#" && titleText != "" {
- page := sedefaults.PageFromContext(e.Request.Ctx, Info.Name)
+ page := _sedefaults.PageFromContext(e.Request.Ctx, Info.Name)
res := bucket.MakeSEResult(linkText, titleText, descText, Info.Name, page, pageRankCounter[page]+1)
bucket.AddSEResult(res, Info.Name, relay, &options, pagesCol)
@@ -61,7 +61,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + query
anonUrll := Info.URL + anonymize.String(query)
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
for i := 1; i < options.MaxPages; i++ {
colCtx = colly.NewContext()
@@ -69,7 +69,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + query + "&start=" + strconv.Itoa(i*10)
anonUrll := Info.URL + anonymize.String(query) + "&start=" + strconv.Itoa(i*10)
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
}
col.Wait()
diff --git a/src/engines/googlescholar/googlescholar_test.go b/src/search/engines/googlescholar/googlescholar_test.go
similarity index 89%
rename from src/engines/googlescholar/googlescholar_test.go
rename to src/search/engines/googlescholar/googlescholar_test.go
index 5d385b93..170303fd 100644
--- a/src/engines/googlescholar/googlescholar_test.go
+++ b/src/search/engines/googlescholar/googlescholar_test.go
@@ -3,8 +3,8 @@ package googlescholar_test
import (
"testing"
- "github.com/hearchco/hearchco/src/engines"
- "github.com/hearchco/hearchco/src/engines/_engines_test"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_engines_test"
)
func TestSearch(t *testing.T) {
diff --git a/src/engines/googlescholar/options.go b/src/search/engines/googlescholar/options.go
similarity index 88%
rename from src/engines/googlescholar/options.go
rename to src/search/engines/googlescholar/options.go
index 68cb9474..14f4fcba 100644
--- a/src/engines/googlescholar/options.go
+++ b/src/search/engines/googlescholar/options.go
@@ -1,6 +1,8 @@
package googlescholar
-import "github.com/hearchco/hearchco/src/engines"
+import (
+ "github.com/hearchco/hearchco/src/search/engines"
+)
var Info engines.Info = engines.Info{
Domain: "scholar.google.com",
diff --git a/src/engines/mojeek/mojeek.go b/src/search/engines/mojeek/mojeek.go
similarity index 72%
rename from src/engines/mojeek/mojeek.go
rename to src/search/engines/mojeek/mojeek.go
index 8c42c573..7986d681 100644
--- a/src/engines/mojeek/mojeek.go
+++ b/src/search/engines/mojeek/mojeek.go
@@ -7,15 +7,15 @@ import (
"github.com/gocolly/colly/v2"
"github.com/hearchco/hearchco/src/anonymize"
- "github.com/hearchco/hearchco/src/bucket"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/bucket"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_sedefaults"
"github.com/hearchco/hearchco/src/search/parse"
- "github.com/hearchco/hearchco/src/sedefaults"
)
func Search(ctx context.Context, query string, relay *bucket.Relay, options engines.Options, settings config.Settings, timings config.Timings) error {
- if err := sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
+ if err := _sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
return err
}
@@ -23,14 +23,14 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
+ _sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
- sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
- sedefaults.PagesColError(Info.Name, pagesCol)
- sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
+ _sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
+ _sedefaults.PagesColError(Info.Name, pagesCol)
+ _sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
- sedefaults.ColRequest(Info.Name, col, ctx)
- sedefaults.ColError(Info.Name, col)
+ _sedefaults.ColRequest(Info.Name, col, ctx)
+ _sedefaults.ColError(Info.Name, col)
var pageRankCounter []int = make([]int, options.MaxPages*Info.ResultsPerPage)
@@ -44,7 +44,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
descText := strings.TrimSpace(dom.Find(dompaths.Description).Text())
if hrefExists && linkText != "" && linkText != "#" && titleText != "" {
- page := sedefaults.PageFromContext(e.Request.Ctx, Info.Name)
+ page := _sedefaults.PageFromContext(e.Request.Ctx, Info.Name)
res := bucket.MakeSEResult(linkText, titleText, descText, Info.Name, page, pageRankCounter[page]+1)
bucket.AddSEResult(res, Info.Name, relay, &options, pagesCol)
@@ -60,7 +60,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + query + localeParam + safeSearchParam
anonUrll := Info.URL + anonymize.String(query) + localeParam + safeSearchParam
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
for i := 1; i < options.MaxPages; i++ {
colCtx = colly.NewContext()
@@ -68,7 +68,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + query + "&s=" + strconv.Itoa(i*10+1) + localeParam + safeSearchParam
anonUrll := Info.URL + anonymize.String(query) + "&s=" + strconv.Itoa(i*10+1) + localeParam + safeSearchParam
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
}
col.Wait()
diff --git a/src/engines/mojeek/mojeek_test.go b/src/search/engines/mojeek/mojeek_test.go
similarity index 87%
rename from src/engines/mojeek/mojeek_test.go
rename to src/search/engines/mojeek/mojeek_test.go
index c26845ff..51504d8f 100644
--- a/src/engines/mojeek/mojeek_test.go
+++ b/src/search/engines/mojeek/mojeek_test.go
@@ -3,8 +3,8 @@ package mojeek_test
import (
"testing"
- "github.com/hearchco/hearchco/src/engines"
- "github.com/hearchco/hearchco/src/engines/_engines_test"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_engines_test"
)
func TestSearch(t *testing.T) {
diff --git a/src/engines/mojeek/options.go b/src/search/engines/mojeek/options.go
similarity index 88%
rename from src/engines/mojeek/options.go
rename to src/search/engines/mojeek/options.go
index 57bcd621..5e16d75b 100644
--- a/src/engines/mojeek/options.go
+++ b/src/search/engines/mojeek/options.go
@@ -1,6 +1,6 @@
package mojeek
-import "github.com/hearchco/hearchco/src/engines"
+import "github.com/hearchco/hearchco/src/search/engines"
var Info engines.Info = engines.Info{
Domain: "www.mojeek.com",
diff --git a/src/engines/name.go b/src/search/engines/name.go
similarity index 87%
rename from src/engines/name.go
rename to src/search/engines/name.go
index 98c0b029..d90edd88 100644
--- a/src/engines/name.go
+++ b/src/search/engines/name.go
@@ -5,7 +5,7 @@ import "strings"
type Name uint8
//go:generate enumer -type=Name -json -text -yaml -sql
-//go:generate go run github.com/hearchco/hearchco/generate/searcher -type=Name -packagename search -output ../search/engine_searcher.go
+//go:generate go run github.com/hearchco/hearchco/generate/searcher -type=Name -packagename search -output ../engine_searcher.go
const (
UNDEFINED Name = iota
BING
diff --git a/src/engines/presearch/json_response.go b/src/search/engines/presearch/json_response.go
similarity index 100%
rename from src/engines/presearch/json_response.go
rename to src/search/engines/presearch/json_response.go
diff --git a/src/engines/presearch/options.go b/src/search/engines/presearch/options.go
similarity index 91%
rename from src/engines/presearch/options.go
rename to src/search/engines/presearch/options.go
index 48f0ec76..d39ebdf8 100644
--- a/src/engines/presearch/options.go
+++ b/src/search/engines/presearch/options.go
@@ -1,6 +1,6 @@
package presearch
-import "github.com/hearchco/hearchco/src/engines"
+import "github.com/hearchco/hearchco/src/search/engines"
var Info engines.Info = engines.Info{
Domain: "presearch.com",
diff --git a/src/engines/presearch/presearch.go b/src/search/engines/presearch/presearch.go
similarity index 80%
rename from src/engines/presearch/presearch.go
rename to src/search/engines/presearch/presearch.go
index 5bdff10d..8e1448e3 100644
--- a/src/engines/presearch/presearch.go
+++ b/src/search/engines/presearch/presearch.go
@@ -8,16 +8,16 @@ import (
"github.com/gocolly/colly/v2"
"github.com/hearchco/hearchco/src/anonymize"
- "github.com/hearchco/hearchco/src/bucket"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/bucket"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_sedefaults"
"github.com/hearchco/hearchco/src/search/parse"
- "github.com/hearchco/hearchco/src/sedefaults"
"github.com/rs/zerolog/log"
)
func Search(ctx context.Context, query string, relay *bucket.Relay, options engines.Options, settings config.Settings, timings config.Timings) error {
- if err := sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
+ if err := _sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
return err
}
@@ -25,14 +25,14 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
+ _sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
- sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
- sedefaults.PagesColError(Info.Name, pagesCol)
- sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
+ _sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
+ _sedefaults.PagesColError(Info.Name, pagesCol)
+ _sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
- sedefaults.ColRequest(Info.Name, col, ctx)
- sedefaults.ColError(Info.Name, col)
+ _sedefaults.ColRequest(Info.Name, col, ctx)
+ _sedefaults.ColError(Info.Name, col)
safeSearch := getSafeSearch(options.SafeSearch)
@@ -100,7 +100,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + query
anonUrll := Info.URL + anonymize.String(query)
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
for i := 1; i < options.MaxPages; i++ {
colCtx = colly.NewContext()
@@ -109,7 +109,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + query + "&page=" + strconv.Itoa(i+1)
anonUrll := Info.URL + anonymize.String(query) + "&page=" + strconv.Itoa(i+1)
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
}
col.Wait()
diff --git a/src/engines/presearch/presearch.md b/src/search/engines/presearch/presearch.md
similarity index 100%
rename from src/engines/presearch/presearch.md
rename to src/search/engines/presearch/presearch.md
diff --git a/src/engines/presearch/presearch_test.go b/src/search/engines/presearch/presearch_test.go
similarity index 87%
rename from src/engines/presearch/presearch_test.go
rename to src/search/engines/presearch/presearch_test.go
index 7c026283..81cfb966 100644
--- a/src/engines/presearch/presearch_test.go
+++ b/src/search/engines/presearch/presearch_test.go
@@ -3,8 +3,8 @@ package presearch_test
import (
"testing"
- "github.com/hearchco/hearchco/src/engines"
- "github.com/hearchco/hearchco/src/engines/_engines_test"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_engines_test"
)
func TestSearch(t *testing.T) {
diff --git a/src/engines/qwant/json_response.go b/src/search/engines/qwant/json_response.go
similarity index 100%
rename from src/engines/qwant/json_response.go
rename to src/search/engines/qwant/json_response.go
diff --git a/src/engines/qwant/options.go b/src/search/engines/qwant/options.go
similarity index 86%
rename from src/engines/qwant/options.go
rename to src/search/engines/qwant/options.go
index b16b4eb7..da1c0ba2 100644
--- a/src/engines/qwant/options.go
+++ b/src/search/engines/qwant/options.go
@@ -1,6 +1,8 @@
package qwant
-import "github.com/hearchco/hearchco/src/engines"
+import (
+ "github.com/hearchco/hearchco/src/search/engines"
+)
var Info engines.Info = engines.Info{
Domain: "www.qwant.com",
diff --git a/src/engines/qwant/qwant.go b/src/search/engines/qwant/qwant.go
similarity index 86%
rename from src/engines/qwant/qwant.go
rename to src/search/engines/qwant/qwant.go
index a30b0933..5e6fe019 100644
--- a/src/engines/qwant/qwant.go
+++ b/src/search/engines/qwant/qwant.go
@@ -8,16 +8,16 @@ import (
"github.com/gocolly/colly/v2"
"github.com/hearchco/hearchco/src/anonymize"
- "github.com/hearchco/hearchco/src/bucket"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/bucket"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_sedefaults"
"github.com/hearchco/hearchco/src/search/parse"
- "github.com/hearchco/hearchco/src/sedefaults"
"github.com/rs/zerolog/log"
)
func Search(ctx context.Context, query string, relay *bucket.Relay, options engines.Options, settings config.Settings, timings config.Timings) error {
- if err := sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
+ if err := _sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
return err
}
@@ -25,14 +25,14 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
+ _sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
- sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
- sedefaults.PagesColError(Info.Name, pagesCol)
- sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
+ _sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
+ _sedefaults.PagesColError(Info.Name, pagesCol)
+ _sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
- sedefaults.ColRequest(Info.Name, col, ctx)
- sedefaults.ColError(Info.Name, col)
+ _sedefaults.ColRequest(Info.Name, col, ctx)
+ _sedefaults.ColError(Info.Name, col)
col.OnResponse(func(r *colly.Response) {
var pageStr string = r.Ctx.Get("page")
@@ -80,7 +80,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + query + "&count=" + strconv.Itoa(nRequested) + localeParam + "&offset=" + strconv.Itoa(i*nRequested) + deviceParam + safeSearchParam
anonUrll := Info.URL + anonymize.String(query) + "&count=" + strconv.Itoa(nRequested) + localeParam + "&offset=" + strconv.Itoa(i*nRequested) + deviceParam + safeSearchParam
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
}
col.Wait()
diff --git a/src/engines/qwant/qwant.md b/src/search/engines/qwant/qwant.md
similarity index 100%
rename from src/engines/qwant/qwant.md
rename to src/search/engines/qwant/qwant.md
diff --git a/src/engines/qwant/qwant_test.go b/src/search/engines/qwant/qwant_test.go
similarity index 87%
rename from src/engines/qwant/qwant_test.go
rename to src/search/engines/qwant/qwant_test.go
index 9fab81ef..09dfdfae 100644
--- a/src/engines/qwant/qwant_test.go
+++ b/src/search/engines/qwant/qwant_test.go
@@ -3,8 +3,8 @@ package qwant_test
import (
"testing"
- "github.com/hearchco/hearchco/src/engines"
- "github.com/hearchco/hearchco/src/engines/_engines_test"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_engines_test"
)
func TestSearch(t *testing.T) {
diff --git a/src/engines/startpage/image-1.png b/src/search/engines/startpage/image-1.png
similarity index 100%
rename from src/engines/startpage/image-1.png
rename to src/search/engines/startpage/image-1.png
diff --git a/src/engines/startpage/image-2.png b/src/search/engines/startpage/image-2.png
similarity index 100%
rename from src/engines/startpage/image-2.png
rename to src/search/engines/startpage/image-2.png
diff --git a/src/engines/startpage/image.png b/src/search/engines/startpage/image.png
similarity index 100%
rename from src/engines/startpage/image.png
rename to src/search/engines/startpage/image.png
diff --git a/src/engines/startpage/options.go b/src/search/engines/startpage/options.go
similarity index 89%
rename from src/engines/startpage/options.go
rename to src/search/engines/startpage/options.go
index 823a54bf..d64a3428 100644
--- a/src/engines/startpage/options.go
+++ b/src/search/engines/startpage/options.go
@@ -1,6 +1,6 @@
package startpage
-import "github.com/hearchco/hearchco/src/engines"
+import "github.com/hearchco/hearchco/src/search/engines"
var Info engines.Info = engines.Info{
Domain: "www.startpage.com",
diff --git a/src/engines/startpage/startpage.go b/src/search/engines/startpage/startpage.go
similarity index 76%
rename from src/engines/startpage/startpage.go
rename to src/search/engines/startpage/startpage.go
index 2dac3f26..7126f85d 100644
--- a/src/engines/startpage/startpage.go
+++ b/src/search/engines/startpage/startpage.go
@@ -7,16 +7,16 @@ import (
"github.com/gocolly/colly/v2"
"github.com/hearchco/hearchco/src/anonymize"
- "github.com/hearchco/hearchco/src/bucket"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/bucket"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_sedefaults"
"github.com/hearchco/hearchco/src/search/parse"
- "github.com/hearchco/hearchco/src/sedefaults"
"github.com/rs/zerolog/log"
)
func Search(ctx context.Context, query string, relay *bucket.Relay, options engines.Options, settings config.Settings, timings config.Timings) error {
- if err := sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
+ if err := _sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
return err
}
@@ -24,14 +24,14 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
+ _sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
- sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
- sedefaults.PagesColError(Info.Name, pagesCol)
- sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
+ _sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
+ _sedefaults.PagesColError(Info.Name, pagesCol)
+ _sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
- sedefaults.ColRequest(Info.Name, col, ctx)
- sedefaults.ColError(Info.Name, col)
+ _sedefaults.ColRequest(Info.Name, col, ctx)
+ _sedefaults.ColError(Info.Name, col)
var pageRankCounter []int = make([]int, options.MaxPages*Info.ResultsPerPage)
@@ -44,7 +44,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
descText := strings.TrimSpace(dom.Find(dompaths.Description).Text())
if hrefExists && linkText != "" && linkText != "#" && titleText != "" {
- page := sedefaults.PageFromContext(e.Request.Ctx, Info.Name)
+ page := _sedefaults.PageFromContext(e.Request.Ctx, Info.Name)
res := bucket.MakeSEResult(linkText, titleText, descText, Info.Name, page, pageRankCounter[page]+1)
bucket.AddSEResult(res, Info.Name, relay, &options, pagesCol)
@@ -78,7 +78,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + query + safeSearch
anonUrll := Info.URL + anonymize.String(query) + safeSearch
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
for i := 1; i < options.MaxPages; i++ {
colCtx = colly.NewContext()
@@ -86,7 +86,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + query + "&page=" + strconv.Itoa(i+1) + safeSearch
anonUrll := Info.URL + anonymize.String(query) + "&page=" + strconv.Itoa(i+1) + safeSearch
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
}
col.Wait()
diff --git a/src/engines/startpage/startpage.md b/src/search/engines/startpage/startpage.md
similarity index 100%
rename from src/engines/startpage/startpage.md
rename to src/search/engines/startpage/startpage.md
diff --git a/src/engines/startpage/startpage_test.go b/src/search/engines/startpage/startpage_test.go
similarity index 89%
rename from src/engines/startpage/startpage_test.go
rename to src/search/engines/startpage/startpage_test.go
index 49db12f1..134d348e 100644
--- a/src/engines/startpage/startpage_test.go
+++ b/src/search/engines/startpage/startpage_test.go
@@ -3,8 +3,8 @@ package startpage_test
import (
"testing"
- "github.com/hearchco/hearchco/src/engines"
- "github.com/hearchco/hearchco/src/engines/_engines_test"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_engines_test"
)
func TestSearch(t *testing.T) {
diff --git a/src/engines/structs.go b/src/search/engines/structs.go
similarity index 76%
rename from src/engines/structs.go
rename to src/search/engines/structs.go
index 5d64b1be..229e1144 100644
--- a/src/engines/structs.go
+++ b/src/search/engines/structs.go
@@ -3,26 +3,9 @@ package engines
import (
"fmt"
- "github.com/hearchco/hearchco/src/category"
+ "github.com/hearchco/hearchco/src/search/category"
)
-// variables are 1-indexed
-// Information about what Rank a result was on some Search Engine
-type RetrievedRank struct {
- SearchEngine Name
- Rank uint
- Page uint
- OnPageRank uint
-}
-
-// The info a Search Engine returned about some Result
-type RetrievedResult struct {
- URL string
- Title string
- Description string
- Rank RetrievedRank
-}
-
type SupportedSettings struct {
Locale bool
SafeSearch bool
diff --git a/src/engines/swisscows/authenticator.go b/src/search/engines/swisscows/authenticator.go
similarity index 100%
rename from src/engines/swisscows/authenticator.go
rename to src/search/engines/swisscows/authenticator.go
diff --git a/src/engines/swisscows/image-1.png b/src/search/engines/swisscows/image-1.png
similarity index 100%
rename from src/engines/swisscows/image-1.png
rename to src/search/engines/swisscows/image-1.png
diff --git a/src/engines/swisscows/image.png b/src/search/engines/swisscows/image.png
similarity index 100%
rename from src/engines/swisscows/image.png
rename to src/search/engines/swisscows/image.png
diff --git a/src/engines/swisscows/json_response.go b/src/search/engines/swisscows/json_response.go
similarity index 100%
rename from src/engines/swisscows/json_response.go
rename to src/search/engines/swisscows/json_response.go
diff --git a/src/engines/swisscows/options.go b/src/search/engines/swisscows/options.go
similarity index 82%
rename from src/engines/swisscows/options.go
rename to src/search/engines/swisscows/options.go
index cde85d26..d6ed4710 100644
--- a/src/engines/swisscows/options.go
+++ b/src/search/engines/swisscows/options.go
@@ -1,6 +1,8 @@
package swisscows
-import "github.com/hearchco/hearchco/src/engines"
+import (
+ "github.com/hearchco/hearchco/src/search/engines"
+)
var Info engines.Info = engines.Info{
Domain: "swisscows.com",
diff --git a/src/engines/swisscows/swisscows.go b/src/search/engines/swisscows/swisscows.go
similarity index 85%
rename from src/engines/swisscows/swisscows.go
rename to src/search/engines/swisscows/swisscows.go
index eda92be9..a6db7e16 100644
--- a/src/engines/swisscows/swisscows.go
+++ b/src/search/engines/swisscows/swisscows.go
@@ -8,16 +8,16 @@ import (
"github.com/gocolly/colly/v2"
"github.com/hearchco/hearchco/src/anonymize"
- "github.com/hearchco/hearchco/src/bucket"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/bucket"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_sedefaults"
"github.com/hearchco/hearchco/src/search/parse"
- "github.com/hearchco/hearchco/src/sedefaults"
"github.com/rs/zerolog/log"
)
func Search(ctx context.Context, query string, relay *bucket.Relay, options engines.Options, settings config.Settings, timings config.Timings) error {
- if err := sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
+ if err := _sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
return err
}
@@ -25,14 +25,14 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
+ _sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
- sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
- sedefaults.PagesColError(Info.Name, pagesCol)
- sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
+ _sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
+ _sedefaults.PagesColError(Info.Name, pagesCol)
+ _sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
- sedefaults.ColRequest(Info.Name, col, ctx)
- sedefaults.ColError(Info.Name, col)
+ _sedefaults.ColRequest(Info.Name, col, ctx)
+ _sedefaults.ColError(Info.Name, col)
col.OnRequest(func(r *colly.Request) {
if r.Method == "OPTIONS" {
@@ -99,7 +99,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + "freshness=All&itemsCount=" + strconv.Itoa(settings.RequestedResultsPerPage) + "&offset=" + strconv.Itoa(i*10) + "&query=" + query + localeParam
anonUrll := Info.URL + "freshness=All&itemsCount=" + strconv.Itoa(settings.RequestedResultsPerPage) + "&offset=" + strconv.Itoa(i*10) + "&query=" + anonymize.String(query) + localeParam
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
}
col.Wait()
diff --git a/src/engines/swisscows/swisscows.md b/src/search/engines/swisscows/swisscows.md
similarity index 100%
rename from src/engines/swisscows/swisscows.md
rename to src/search/engines/swisscows/swisscows.md
diff --git a/src/engines/swisscows/swisscows_test.go b/src/search/engines/swisscows/swisscows_test.go
similarity index 87%
rename from src/engines/swisscows/swisscows_test.go
rename to src/search/engines/swisscows/swisscows_test.go
index 53644114..8c529e40 100644
--- a/src/engines/swisscows/swisscows_test.go
+++ b/src/search/engines/swisscows/swisscows_test.go
@@ -3,8 +3,8 @@ package swisscows_test
import (
"testing"
- "github.com/hearchco/hearchco/src/engines"
- "github.com/hearchco/hearchco/src/engines/_engines_test"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_engines_test"
)
func TestSearch(t *testing.T) {
diff --git a/src/engines/timeout.go b/src/search/engines/timeout.go
similarity index 100%
rename from src/engines/timeout.go
rename to src/search/engines/timeout.go
diff --git a/src/engines/yahoo/options.go b/src/search/engines/yahoo/options.go
similarity index 91%
rename from src/engines/yahoo/options.go
rename to src/search/engines/yahoo/options.go
index d1cb26df..ac68e32f 100644
--- a/src/engines/yahoo/options.go
+++ b/src/search/engines/yahoo/options.go
@@ -1,6 +1,8 @@
package yahoo
-import "github.com/hearchco/hearchco/src/engines"
+import (
+ "github.com/hearchco/hearchco/src/search/engines"
+)
// doesn't catch the yt videos
// the title cathes the link - e.g.: teentitans.fandom.com › wiki › Nya-NyaNya-Nya | Teen Titans Wiki | Fandom
diff --git a/src/engines/yahoo/yahoo.go b/src/search/engines/yahoo/yahoo.go
similarity index 75%
rename from src/engines/yahoo/yahoo.go
rename to src/search/engines/yahoo/yahoo.go
index e2c3ebb8..2210c8b0 100644
--- a/src/engines/yahoo/yahoo.go
+++ b/src/search/engines/yahoo/yahoo.go
@@ -8,16 +8,16 @@ import (
"github.com/gocolly/colly/v2"
"github.com/hearchco/hearchco/src/anonymize"
- "github.com/hearchco/hearchco/src/bucket"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/bucket"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_sedefaults"
"github.com/hearchco/hearchco/src/search/parse"
- "github.com/hearchco/hearchco/src/sedefaults"
"github.com/rs/zerolog/log"
)
func Search(ctx context.Context, query string, relay *bucket.Relay, options engines.Options, settings config.Settings, timings config.Timings) error {
- if err := sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
+ if err := _sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
return err
}
@@ -25,14 +25,14 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
+ _sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
- sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
- sedefaults.PagesColError(Info.Name, pagesCol)
- sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
+ _sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
+ _sedefaults.PagesColError(Info.Name, pagesCol)
+ _sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
- sedefaults.ColRequest(Info.Name, col, ctx)
- sedefaults.ColError(Info.Name, col)
+ _sedefaults.ColRequest(Info.Name, col, ctx)
+ _sedefaults.ColError(Info.Name, col)
var pageRankCounter []int = make([]int, options.MaxPages*Info.ResultsPerPage)
@@ -53,7 +53,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
descText := strings.TrimSpace(dom.Find(dompaths.Description).Text())
if labelExists && hrefExists && linkText != "" && linkText != "#" && titleText != "" {
- page := sedefaults.PageFromContext(e.Request.Ctx, Info.Name)
+ page := _sedefaults.PageFromContext(e.Request.Ctx, Info.Name)
res := bucket.MakeSEResult(linkText, titleText, descText, Info.Name, page, pageRankCounter[page]+1)
bucket.AddSEResult(res, Info.Name, relay, &options, pagesCol)
@@ -66,7 +66,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + query
anonUrll := Info.URL + anonymize.String(query)
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
for i := 1; i < options.MaxPages; i++ {
colCtx = colly.NewContext()
@@ -74,7 +74,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
urll := Info.URL + query + "&b=" + strconv.Itoa((i+1)*10)
anonUrll := Info.URL + anonymize.String(query) + "&b=" + strconv.Itoa((i+1)*10)
- sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, colCtx, col, Info.Name, &retError)
}
col.Wait()
diff --git a/src/engines/yahoo/yahoo_test.go b/src/search/engines/yahoo/yahoo_test.go
similarity index 87%
rename from src/engines/yahoo/yahoo_test.go
rename to src/search/engines/yahoo/yahoo_test.go
index dab5f7af..be5bb184 100644
--- a/src/engines/yahoo/yahoo_test.go
+++ b/src/search/engines/yahoo/yahoo_test.go
@@ -3,8 +3,8 @@ package yahoo_test
import (
"testing"
- "github.com/hearchco/hearchco/src/engines"
- "github.com/hearchco/hearchco/src/engines/_engines_test"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_engines_test"
)
func TestSearch(t *testing.T) {
diff --git a/src/engines/yep/json_response.go b/src/search/engines/yep/json_response.go
similarity index 100%
rename from src/engines/yep/json_response.go
rename to src/search/engines/yep/json_response.go
diff --git a/src/engines/yep/options.go b/src/search/engines/yep/options.go
similarity index 92%
rename from src/engines/yep/options.go
rename to src/search/engines/yep/options.go
index 645d0f02..cf8c469a 100644
--- a/src/engines/yep/options.go
+++ b/src/search/engines/yep/options.go
@@ -1,7 +1,7 @@
package yep
import (
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/engines"
)
var Info engines.Info = engines.Info{
diff --git a/src/engines/yep/yep.go b/src/search/engines/yep/yep.go
similarity index 77%
rename from src/engines/yep/yep.go
rename to src/search/engines/yep/yep.go
index 17e57361..d43fe885 100644
--- a/src/engines/yep/yep.go
+++ b/src/search/engines/yep/yep.go
@@ -7,15 +7,15 @@ import (
"github.com/gocolly/colly/v2"
"github.com/hearchco/hearchco/src/anonymize"
- "github.com/hearchco/hearchco/src/bucket"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/bucket"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_sedefaults"
"github.com/hearchco/hearchco/src/search/parse"
- "github.com/hearchco/hearchco/src/sedefaults"
)
func Search(ctx context.Context, query string, relay *bucket.Relay, options engines.Options, settings config.Settings, timings config.Timings) error {
- if err := sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
+ if err := _sedefaults.Prepare(Info.Name, &options, &settings, &Support, &Info, &ctx); err != nil {
return err
}
@@ -23,14 +23,14 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
+ _sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
- sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
- sedefaults.PagesColError(Info.Name, pagesCol)
- sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
+ _sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
+ _sedefaults.PagesColError(Info.Name, pagesCol)
+ _sedefaults.PagesColResponse(Info.Name, pagesCol, relay)
- sedefaults.ColRequest(Info.Name, col, ctx)
- sedefaults.ColError(Info.Name, col)
+ _sedefaults.ColRequest(Info.Name, col, ctx)
+ _sedefaults.ColError(Info.Name, col)
col.OnRequest(func(r *colly.Request) {
r.Headers.Del("Accept")
@@ -72,7 +72,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
anonUrll = Info.URL + "client=web" + localeParam + "&limit=" + strconv.Itoa(nRequested) + "&no_correct=false&q=" + anonymize.String(query) + safeSearchParam + "&type=web"
}
- sedefaults.DoGetRequest(urll, anonUrll, nil, col, Info.Name, &retError)
+ _sedefaults.DoGetRequest(urll, anonUrll, nil, col, Info.Name, &retError)
col.Wait()
pagesCol.Wait()
diff --git a/src/engines/yep/yep.md b/src/search/engines/yep/yep.md
similarity index 100%
rename from src/engines/yep/yep.md
rename to src/search/engines/yep/yep.md
diff --git a/src/engines/yep/yep_test.go b/src/search/engines/yep/yep_test.go
similarity index 87%
rename from src/engines/yep/yep_test.go
rename to src/search/engines/yep/yep_test.go
index c23f66e2..ccf82a2f 100644
--- a/src/engines/yep/yep_test.go
+++ b/src/search/engines/yep/yep_test.go
@@ -3,8 +3,8 @@ package yep_test
import (
"testing"
- "github.com/hearchco/hearchco/src/engines"
- "github.com/hearchco/hearchco/src/engines/_engines_test"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/engines/_engines_test"
)
func TestSearch(t *testing.T) {
diff --git a/src/search/perform.go b/src/search/perform.go
index 26478c68..35d5800a 100644
--- a/src/search/perform.go
+++ b/src/search/perform.go
@@ -8,12 +8,12 @@ import (
"time"
"github.com/hearchco/hearchco/src/anonymize"
- "github.com/hearchco/hearchco/src/bucket"
- "github.com/hearchco/hearchco/src/bucket/result"
- "github.com/hearchco/hearchco/src/category"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
- "github.com/hearchco/hearchco/src/rank"
+ "github.com/hearchco/hearchco/src/search/bucket"
+ "github.com/hearchco/hearchco/src/search/category"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/rank"
+ "github.com/hearchco/hearchco/src/search/result"
"github.com/rs/zerolog/log"
"github.com/sourcegraph/conc"
)
diff --git a/src/rank/math.go b/src/search/rank/math.go
similarity index 100%
rename from src/rank/math.go
rename to src/search/rank/math.go
diff --git a/src/rank/rank.go b/src/search/rank/rank.go
similarity index 93%
rename from src/rank/rank.go
rename to src/search/rank/rank.go
index 456fa051..9ba0dddf 100644
--- a/src/rank/rank.go
+++ b/src/search/rank/rank.go
@@ -4,9 +4,9 @@ import (
"math"
"sort"
- "github.com/hearchco/hearchco/src/bucket/result"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/result"
)
func GetScore(result *result.Result, rconf *config.Ranking) float64 {
@@ -46,7 +46,7 @@ func Rank(resMap map[string]*result.Result, rconf config.Ranking) []result.Resul
type RankFiller struct {
ArrInd int
- RetRank engines.RetrievedRank
+ RetRank result.RetrievedRank
RRInd int
}
diff --git a/src/rank/sorting.go b/src/search/rank/sorting.go
similarity index 94%
rename from src/rank/sorting.go
rename to src/search/rank/sorting.go
index d35fb238..dbd85aec 100644
--- a/src/rank/sorting.go
+++ b/src/search/rank/sorting.go
@@ -3,7 +3,7 @@ package rank
import (
"fmt"
- "github.com/hearchco/hearchco/src/bucket/result"
+ "github.com/hearchco/hearchco/src/search/result"
"github.com/rs/zerolog/log"
)
diff --git a/src/bucket/result/result.go b/src/search/result/result.go
similarity index 90%
rename from src/bucket/result/result.go
rename to src/search/result/result.go
index ee60ee4f..36905f34 100644
--- a/src/bucket/result/result.go
+++ b/src/search/result/result.go
@@ -2,7 +2,6 @@ package result
import (
"github.com/gocolly/colly/v2"
- "github.com/hearchco/hearchco/src/engines"
)
// Everything about some Result, calculated and compiled from multiple search engines
@@ -13,7 +12,7 @@ type Result struct {
Score float64
Title string
Description string
- EngineRanks []engines.RetrievedRank
+ EngineRanks []RetrievedRank
TimesReturned uint8
Response *colly.Response
}
diff --git a/src/search/result/retrieved.go b/src/search/result/retrieved.go
new file mode 100644
index 00000000..093814b1
--- /dev/null
+++ b/src/search/result/retrieved.go
@@ -0,0 +1,20 @@
+package result
+
+import "github.com/hearchco/hearchco/src/search/engines"
+
+// variables are 1-indexed
+// Information about what Rank a result was on some Search Engine
+type RetrievedRank struct {
+ SearchEngine engines.Name
+ Rank uint
+ Page uint
+ OnPageRank uint
+}
+
+// The info a Search Engine returned about some Result
+type RetrievedResult struct {
+ URL string
+ Title string
+ Description string
+ Rank RetrievedRank
+}
diff --git a/src/search/search.go b/src/search/search.go
index 2bf519aa..d723702c 100644
--- a/src/search/search.go
+++ b/src/search/search.go
@@ -2,10 +2,10 @@ package search
import (
"github.com/hearchco/hearchco/src/anonymize"
- "github.com/hearchco/hearchco/src/bucket/result"
"github.com/hearchco/hearchco/src/cache"
"github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/engines"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/result"
"github.com/rs/zerolog/log"
)
From 8711d03ae95032bb52cb01d8b94fe804b2a2b35f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?=
<31509435+aleksasiriski@users.noreply.github.com>
Date: Thu, 8 Feb 2024 14:03:44 +0100
Subject: [PATCH 02/11] chore(makefile): more runnables
---
Makefile | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/Makefile b/Makefile
index a51efab5..eb0b7edd 100644
--- a/Makefile
+++ b/Makefile
@@ -1,34 +1,40 @@
run:
go run ./src
+run-cli:
+ go run ./src --cli
debug:
go run ./src -v
+debug-cli:
+ go run ./srv -v --cli
trace:
go run ./src -vv
+trace-cli:
+ go run ./src -vv --cli
setup:
go get ./...
go install github.com/dmarkham/enumer@latest
go generate ./...
+install: setup
build:
go build ./...
test:
sh ./scripts/test.sh
-
test-engines:
sh ./scripts/test-engines.sh
-
test-redis:
sh ./scripts/test-redis.sh
-
test-redis-podman:
sh ./scripts/test-redis-podman.sh
-
test-redis-docker:
sh ./scripts/test-redis-docker.sh
+test-all: test test-engines test-redis
+test-all-podman: test test-engines test-redis-podman
+test-all-docker: test test-engines test-redis-docker
update:
go get -u ./...
From 447811b6105daad1780ec6c8f27520543ac83974 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?=
<31509435+aleksasiriski@users.noreply.github.com>
Date: Thu, 8 Feb 2024 14:04:01 +0100
Subject: [PATCH 03/11] refactor(bucket): separate files
---
src/search/bucket/{bucket.go => addresult.go} | 47 -------------------
src/search/bucket/makeresult.go | 22 +++++++++
src/search/bucket/relay.go | 12 +++++
src/search/bucket/setresponse.go | 32 +++++++++++++
4 files changed, 66 insertions(+), 47 deletions(-)
rename src/search/bucket/{bucket.go => addresult.go} (61%)
create mode 100644 src/search/bucket/makeresult.go
create mode 100644 src/search/bucket/relay.go
create mode 100644 src/search/bucket/setresponse.go
diff --git a/src/search/bucket/bucket.go b/src/search/bucket/addresult.go
similarity index 61%
rename from src/search/bucket/bucket.go
rename to src/search/bucket/addresult.go
index 307efff2..442ee88b 100644
--- a/src/search/bucket/bucket.go
+++ b/src/search/bucket/addresult.go
@@ -1,9 +1,6 @@
package bucket
import (
- "fmt"
- "sync"
-
"github.com/gocolly/colly/v2"
"github.com/hearchco/hearchco/src/config"
"github.com/hearchco/hearchco/src/search/engines"
@@ -11,11 +8,6 @@ import (
"github.com/rs/zerolog/log"
)
-type Relay struct {
- ResultMap map[string]*result.Result
- Mutex sync.RWMutex
-}
-
func AddSEResult(seResult *result.RetrievedResult, seName engines.Name, relay *Relay, options *engines.Options, pagesCol *colly.Collector) {
log.Trace().
Str("engine", seName.String()).
@@ -74,42 +66,3 @@ func AddSEResult(seResult *result.RetrievedResult, seName engines.Name, relay *R
}
}
}
-
-func SetResultResponse(link string, response *colly.Response, relay *Relay, seName engines.Name) error {
- log.Trace().
- Str("engine", seName.String()).
- Str("link", link).
- Msg("Got response")
-
- relay.Mutex.Lock()
- mapRes, exists := relay.ResultMap[link]
-
- if !exists {
- relay.Mutex.Unlock()
- relay.Mutex.RLock()
- err := fmt.Errorf("bucket.SetResultResponse(): URL not in map when adding response, should not be possible. URL: %v.\nRelay: %v", link, relay)
- relay.Mutex.RUnlock()
- return err
- } else {
- mapRes.Response = response
- relay.Mutex.Unlock()
- }
-
- return nil
-}
-
-func MakeSEResult(urll string, title string, description string, searchEngineName engines.Name, sePage int, seOnPageRank int) *result.RetrievedResult {
- ser := result.RetrievedRank{
- SearchEngine: searchEngineName,
- Rank: 0,
- Page: uint(sePage),
- OnPageRank: uint(seOnPageRank),
- }
- res := result.RetrievedResult{
- URL: urll,
- Title: title,
- Description: description,
- Rank: ser,
- }
- return &res
-}
diff --git a/src/search/bucket/makeresult.go b/src/search/bucket/makeresult.go
new file mode 100644
index 00000000..ceca5038
--- /dev/null
+++ b/src/search/bucket/makeresult.go
@@ -0,0 +1,22 @@
+package bucket
+
+import (
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/result"
+)
+
+func MakeSEResult(urll string, title string, description string, searchEngineName engines.Name, sePage int, seOnPageRank int) *result.RetrievedResult {
+ ser := result.RetrievedRank{
+ SearchEngine: searchEngineName,
+ Rank: 0,
+ Page: uint(sePage),
+ OnPageRank: uint(seOnPageRank),
+ }
+ res := result.RetrievedResult{
+ URL: urll,
+ Title: title,
+ Description: description,
+ Rank: ser,
+ }
+ return &res
+}
diff --git a/src/search/bucket/relay.go b/src/search/bucket/relay.go
new file mode 100644
index 00000000..0d69d74a
--- /dev/null
+++ b/src/search/bucket/relay.go
@@ -0,0 +1,12 @@
+package bucket
+
+import (
+ "sync"
+
+ "github.com/hearchco/hearchco/src/search/result"
+)
+
+type Relay struct {
+ ResultMap map[string]*result.Result
+ Mutex sync.RWMutex
+}
diff --git a/src/search/bucket/setresponse.go b/src/search/bucket/setresponse.go
new file mode 100644
index 00000000..64c703e3
--- /dev/null
+++ b/src/search/bucket/setresponse.go
@@ -0,0 +1,32 @@
+package bucket
+
+import (
+ "fmt"
+
+ "github.com/gocolly/colly/v2"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/rs/zerolog/log"
+)
+
+func SetResultResponse(link string, response *colly.Response, relay *Relay, seName engines.Name) error {
+ log.Trace().
+ Str("engine", seName.String()).
+ Str("link", link).
+ Msg("Got response")
+
+ relay.Mutex.Lock()
+ mapRes, exists := relay.ResultMap[link]
+
+ if !exists {
+ relay.Mutex.Unlock()
+ relay.Mutex.RLock()
+ err := fmt.Errorf("bucket.SetResultResponse(): URL not in map when adding response, should not be possible. URL: %v.\nRelay: %v", link, relay)
+ relay.Mutex.RUnlock()
+ return err
+ } else {
+ mapRes.Response = response
+ relay.Mutex.Unlock()
+ }
+
+ return nil
+}
From 620dffebb4a8f8465c4303dc34db8af8fa0c03f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?=
<31509435+aleksasiriski@users.noreply.github.com>
Date: Thu, 8 Feb 2024 14:07:14 +0100
Subject: [PATCH 04/11] refactor(_sedefaults): separate files
---
src/search/engines/_sedefaults/colly.go | 68 +++++
src/search/engines/_sedefaults/collypages.go | 61 ++++
src/search/engines/_sedefaults/init.go | 68 +++++
src/search/engines/_sedefaults/pagecontext.go | 23 ++
src/search/engines/_sedefaults/prepare.go | 67 +++++
src/search/engines/_sedefaults/requests.go | 32 ++
src/search/engines/_sedefaults/sedefaults.go | 274 ------------------
7 files changed, 319 insertions(+), 274 deletions(-)
create mode 100644 src/search/engines/_sedefaults/colly.go
create mode 100644 src/search/engines/_sedefaults/collypages.go
create mode 100644 src/search/engines/_sedefaults/init.go
create mode 100644 src/search/engines/_sedefaults/pagecontext.go
create mode 100644 src/search/engines/_sedefaults/prepare.go
create mode 100644 src/search/engines/_sedefaults/requests.go
delete mode 100644 src/search/engines/_sedefaults/sedefaults.go
diff --git a/src/search/engines/_sedefaults/colly.go b/src/search/engines/_sedefaults/colly.go
new file mode 100644
index 00000000..439397fb
--- /dev/null
+++ b/src/search/engines/_sedefaults/colly.go
@@ -0,0 +1,68 @@
+package _sedefaults
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ "github.com/gocolly/colly/v2"
+ "github.com/hearchco/hearchco/src/config"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/rs/zerolog"
+ "github.com/rs/zerolog/log"
+)
+
+func ColRequest(seName engines.Name, col *colly.Collector, ctx context.Context) {
+ col.OnRequest(func(r *colly.Request) {
+ if err := ctx.Err(); err != nil {
+ if engines.IsTimeoutError(err) {
+ log.Trace().
+ Err(err).
+ Str("engine", seName.String()).
+ Msg("_sedefaults.ColRequest() -> col.OnRequest(): context timeout error")
+ } else {
+ log.Error().
+ Err(err).
+ Str("engine", seName.String()).
+ Msg("_sedefaults.ColRequest() -> col.OnRequest(): context error")
+ }
+ r.Abort()
+ return
+ }
+ })
+}
+
+func ColError(seName engines.Name, col *colly.Collector) {
+ col.OnError(func(r *colly.Response, err error) {
+ if engines.IsTimeoutError(err) {
+ log.Trace().
+ // Err(err). // timeout error produces Get "url" error with the query
+ Str("engine", seName.String()).
+ // Str("url", urll). // can't reliably anonymize it (because it's engine dependent and query isn't passed to this function)
+ Msg("_sedefaults.ColError() -> col.OnError(): request timeout error for url")
+ } else {
+ log.Error().
+ Err(err).
+ Str("engine", seName.String()).
+ // Str("url", urll). // can't reliably anonymize it (because it's engine dependent and query isn't passed to this function)
+ Int("statusCode", r.StatusCode).
+ Str("response", string(r.Body)). // query can be present, depending on the response from the engine (Google has the query in 3 places)
+ Msg("_sedefaults.ColError() -> col.OnError(): request error for url")
+
+ dumpPath := fmt.Sprintf("%v%v_col.log.html", config.LogDumpLocation, seName.String())
+ log.Debug().
+ Str("engine", seName.String()).
+ Str("responsePath", dumpPath).
+ Func(func(e *zerolog.Event) {
+ bodyWriteErr := os.WriteFile(dumpPath, r.Body, 0644)
+ if bodyWriteErr != nil {
+ log.Error().
+ Err(bodyWriteErr).
+ Str("engine", seName.String()).
+ Msg("_sedefaults.ColError() -> col.OnError(): error writing html response body to file")
+ }
+ }).
+ Msg("_sedefaults.ColError() -> col.OnError(): html response written")
+ }
+ })
+}
diff --git a/src/search/engines/_sedefaults/collypages.go b/src/search/engines/_sedefaults/collypages.go
new file mode 100644
index 00000000..177cea98
--- /dev/null
+++ b/src/search/engines/_sedefaults/collypages.go
@@ -0,0 +1,61 @@
+package _sedefaults
+
+import (
+ "context"
+
+ "github.com/gocolly/colly/v2"
+ "github.com/hearchco/hearchco/src/search/bucket"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/rs/zerolog/log"
+)
+
+func PagesColRequest(seName engines.Name, pagesCol *colly.Collector, ctx context.Context) {
+ pagesCol.OnRequest(func(r *colly.Request) {
+ if err := ctx.Err(); err != nil {
+ if engines.IsTimeoutError(err) {
+ log.Trace().
+ Err(err).
+ Str("engine", seName.String()).
+ Msg("_sedefaults.PagesColRequest() -> pagesCol.OnRequest(): context timeout error")
+ } else {
+ log.Error().
+ Err(err).
+ Str("engine", seName.String()).
+ Msg("_sedefaults.PagesColRequest() -> pagesCol.OnRequest(): context error")
+ }
+ r.Abort()
+ return
+ }
+ r.Ctx.Put("originalURL", r.URL.String())
+ })
+}
+
+func PagesColError(seName engines.Name, pagesCol *colly.Collector) {
+ pagesCol.OnError(func(r *colly.Response, err error) {
+ urll := r.Ctx.Get("originalURL")
+ if engines.IsTimeoutError(err) {
+ log.Trace().
+ Err(err).
+ Str("engine", seName.String()).
+ Str("url", urll).
+ Msg("_sedefaults.PagesColError() -> pagesCol.OnError(): request timeout error for url")
+ } else {
+ log.Trace().
+ Err(err).
+ Str("engine", seName.String()).
+ Str("url", urll).
+ Str("response", string(r.Body)).
+ Msg("_sedefaults.PagesColError() -> pagesCol.OnError(): request error for url")
+ }
+ })
+}
+
+func PagesColResponse(seName engines.Name, pagesCol *colly.Collector, relay *bucket.Relay) {
+ pagesCol.OnResponse(func(r *colly.Response) {
+ urll := r.Ctx.Get("originalURL")
+ err := bucket.SetResultResponse(urll, r, relay, seName)
+ if err != nil {
+ log.Error().Err(err).Msg("_sedefaults.PagesColResponse(): error setting result")
+ }
+ })
+}
diff --git a/src/search/engines/_sedefaults/init.go b/src/search/engines/_sedefaults/init.go
new file mode 100644
index 00000000..a8c21d04
--- /dev/null
+++ b/src/search/engines/_sedefaults/init.go
@@ -0,0 +1,68 @@
+package _sedefaults
+
+import (
+ "crypto/tls"
+ "fmt"
+ "net/http"
+
+ "github.com/gocolly/colly/v2"
+ "github.com/gocolly/colly/v2/proxy"
+ "github.com/hearchco/hearchco/src/config"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/rs/zerolog/log"
+)
+
+func InitializeCollectors(colPtr **colly.Collector, pagesColPtr **colly.Collector, settings *config.Settings, options *engines.Options, timings *config.Timings) {
+ *colPtr = colly.NewCollector(colly.MaxDepth(1), colly.UserAgent(options.UserAgent), colly.Async())
+ *pagesColPtr = colly.NewCollector(colly.MaxDepth(1), colly.UserAgent(options.UserAgent), colly.Async())
+
+ if timings != nil {
+ var limitRule *colly.LimitRule = &colly.LimitRule{
+ DomainGlob: "*",
+ Delay: timings.Delay,
+ RandomDelay: timings.RandomDelay,
+ Parallelism: timings.Parallelism,
+ }
+
+ if err := (*colPtr).Limit(limitRule); err != nil {
+ log.Error().
+ Err(err).
+ Str("limitRule", fmt.Sprintf("%v", limitRule)).
+ Msg("_sedefaults.InitializeCollectors(): failed adding new limit rule")
+ }
+
+ if timings.Timeout != 0 {
+ (*colPtr).SetRequestTimeout(timings.Timeout)
+ }
+
+ if timings.PageTimeout != 0 {
+ (*pagesColPtr).SetRequestTimeout(timings.PageTimeout)
+ }
+ }
+
+ if settings.Proxies != nil {
+ log.Debug().
+ Strs("proxies", settings.Proxies).
+ Msg("Using proxies")
+
+ // Rotate proxies
+ rp, err := proxy.RoundRobinProxySwitcher(settings.Proxies...)
+ if err != nil {
+ log.Fatal().
+ Err(err).
+ Strs("proxies", settings.Proxies).
+ Msg("_sedefaults.InitializeCollectors(): failed creating proxy switcher")
+ }
+
+ (*colPtr).SetProxyFunc(rp)
+ (*pagesColPtr).SetProxyFunc(rp)
+ }
+
+ if settings.InsecureSkipVerify {
+ tp := &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+ }
+ (*colPtr).WithTransport(tp)
+ (*pagesColPtr).WithTransport(tp)
+ }
+}
diff --git a/src/search/engines/_sedefaults/pagecontext.go b/src/search/engines/_sedefaults/pagecontext.go
new file mode 100644
index 00000000..8370c898
--- /dev/null
+++ b/src/search/engines/_sedefaults/pagecontext.go
@@ -0,0 +1,23 @@
+package _sedefaults
+
+import (
+ "strconv"
+
+ "github.com/gocolly/colly/v2"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/rs/zerolog/log"
+)
+
+func PageFromContext(ctx *colly.Context, seName engines.Name) int {
+ var pageStr string = ctx.Get("page")
+ page, converr := strconv.Atoi(pageStr)
+ if converr != nil {
+ log.Panic().
+ Err(converr).
+ Str("engine", seName.String()).
+ Str("page", pageStr).
+ Msg("_sedefaults.PageFromContext(): failed to convert page number to int")
+ // ^PANIC
+ }
+ return page
+}
diff --git a/src/search/engines/_sedefaults/prepare.go b/src/search/engines/_sedefaults/prepare.go
new file mode 100644
index 00000000..2662cb02
--- /dev/null
+++ b/src/search/engines/_sedefaults/prepare.go
@@ -0,0 +1,67 @@
+package _sedefaults
+
+import (
+ "context"
+
+ "github.com/hearchco/hearchco/src/config"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/hearchco/hearchco/src/search/useragent"
+ "github.com/rs/zerolog/log"
+)
+
+func Prepare(seName engines.Name, options *engines.Options, settings *config.Settings, support *engines.SupportedSettings, info *engines.Info, ctx *context.Context) error {
+ if ctx == nil {
+ *ctx = context.Background()
+ }
+
+ if options.UserAgent == "" {
+ options.UserAgent = useragent.RandomUserAgent()
+ }
+ log.Trace().
+ Str("engine", seName.String()).
+ Str("userAgent", options.UserAgent).
+ Msg("Prepare")
+
+ // TODO: move to config.SetupConfig
+ if settings.RequestedResultsPerPage != 0 && !support.RequestedResultsPerPage {
+ log.Panic().
+ Str("engine", seName.String()).
+ Int("requestedResultsPerPage", settings.RequestedResultsPerPage).
+ Msg("_sedefaults.Prepare(): setting not supported by engine")
+ // ^PANIC
+ }
+ if settings.RequestedResultsPerPage == 0 && support.RequestedResultsPerPage {
+ // if its used in the code but not set, give it the default value
+ settings.RequestedResultsPerPage = info.ResultsPerPage
+ }
+
+ if options.Mobile && !support.Mobile {
+ options.Mobile = false // this line shouldn't matter [1]
+ log.Debug().
+ Str("engine", seName.String()).
+ Bool("mobile", options.Mobile).
+ Msg("Mobile set but not supported")
+ }
+
+ if options.Locale != "" && !support.Locale {
+ options.Locale = config.DefaultLocale // [1]
+ log.Debug().
+ Str("engine", seName.String()).
+ Str("locale", options.Locale).
+ Msg("Locale set but not supported")
+ }
+
+ if options.Locale == "" && support.Locale {
+ options.Locale = config.DefaultLocale
+ }
+
+ if options.SafeSearch && !support.SafeSearch {
+ options.SafeSearch = false // [1]
+ log.Debug().
+ Str("engine", seName.String()).
+ Bool("safeSearch", options.SafeSearch).
+ Msg("SafeSearch set but not supported")
+ }
+
+ return nil
+}
diff --git a/src/search/engines/_sedefaults/requests.go b/src/search/engines/_sedefaults/requests.go
new file mode 100644
index 00000000..c15d2b87
--- /dev/null
+++ b/src/search/engines/_sedefaults/requests.go
@@ -0,0 +1,32 @@
+package _sedefaults
+
+import (
+ "fmt"
+ "io"
+
+ "github.com/gocolly/colly/v2"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/rs/zerolog/log"
+)
+
+func DoGetRequest(urll string, anonurll string, colCtx *colly.Context, collector *colly.Collector, packageName engines.Name, retError *error) {
+ log.Trace().
+ Str("engine", packageName.String()).
+ Str("url", anonurll).
+ Msg("GET")
+ err := collector.Request("GET", urll, nil, colCtx, nil)
+ if err != nil {
+ *retError = fmt.Errorf("%v.Search(): failed GET request to %v with %w", packageName.ToLower(), urll, err)
+ }
+}
+
+func DoPostRequest(urll string, requestData io.Reader, colCtx *colly.Context, collector *colly.Collector, packageName engines.Name, retError *error) {
+ log.Trace().
+ Str("engine", packageName.String()).
+ Str("url", urll).
+ Msg("POST")
+ err := collector.Request("POST", urll, requestData, colCtx, nil)
+ if err != nil {
+ *retError = fmt.Errorf("%v.Search(): failed POST request to %v and body %v. error %w", packageName.ToLower(), requestData, urll, err)
+ }
+}
diff --git a/src/search/engines/_sedefaults/sedefaults.go b/src/search/engines/_sedefaults/sedefaults.go
deleted file mode 100644
index 6ceecf52..00000000
--- a/src/search/engines/_sedefaults/sedefaults.go
+++ /dev/null
@@ -1,274 +0,0 @@
-package _sedefaults
-
-import (
- "context"
- "crypto/tls"
- "fmt"
- "io"
- "net/http"
- "os"
- "strconv"
-
- "github.com/gocolly/colly/v2"
- "github.com/gocolly/colly/v2/proxy"
- "github.com/hearchco/hearchco/src/config"
- "github.com/hearchco/hearchco/src/search/bucket"
- "github.com/hearchco/hearchco/src/search/engines"
- "github.com/hearchco/hearchco/src/search/useragent"
- "github.com/rs/zerolog"
- "github.com/rs/zerolog/log"
-)
-
-func PagesColRequest(seName engines.Name, pagesCol *colly.Collector, ctx context.Context) {
- pagesCol.OnRequest(func(r *colly.Request) {
- if err := ctx.Err(); err != nil {
- if engines.IsTimeoutError(err) {
- log.Trace().
- Err(err).
- Str("engine", seName.String()).
- Msg("_sedefaults.PagesColRequest() -> pagesCol.OnRequest(): context timeout error")
- } else {
- log.Error().
- Err(err).
- Str("engine", seName.String()).
- Msg("_sedefaults.PagesColRequest() -> pagesCol.OnRequest(): context error")
- }
- r.Abort()
- return
- }
- r.Ctx.Put("originalURL", r.URL.String())
- })
-}
-
-func PagesColError(seName engines.Name, pagesCol *colly.Collector) {
- pagesCol.OnError(func(r *colly.Response, err error) {
- urll := r.Ctx.Get("originalURL")
- if engines.IsTimeoutError(err) {
- log.Trace().
- Err(err).
- Str("engine", seName.String()).
- Str("url", urll).
- Msg("_sedefaults.PagesColError() -> pagesCol.OnError(): request timeout error for url")
- } else {
- log.Trace().
- Err(err).
- Str("engine", seName.String()).
- Str("url", urll).
- Str("response", string(r.Body)).
- Msg("_sedefaults.PagesColError() -> pagesCol.OnError(): request error for url")
- }
- })
-}
-
-func PagesColResponse(seName engines.Name, pagesCol *colly.Collector, relay *bucket.Relay) {
- pagesCol.OnResponse(func(r *colly.Response) {
- urll := r.Ctx.Get("originalURL")
- err := bucket.SetResultResponse(urll, r, relay, seName)
- if err != nil {
- log.Error().Err(err).Msg("_sedefaults.PagesColResponse(): error setting result")
- }
- })
-}
-
-func ColRequest(seName engines.Name, col *colly.Collector, ctx context.Context) {
- col.OnRequest(func(r *colly.Request) {
- if err := ctx.Err(); err != nil {
- if engines.IsTimeoutError(err) {
- log.Trace().
- Err(err).
- Str("engine", seName.String()).
- Msg("_sedefaults.ColRequest() -> col.OnRequest(): context timeout error")
- } else {
- log.Error().
- Err(err).
- Str("engine", seName.String()).
- Msg("_sedefaults.ColRequest() -> col.OnRequest(): context error")
- }
- r.Abort()
- return
- }
- })
-}
-
-func ColError(seName engines.Name, col *colly.Collector) {
- col.OnError(func(r *colly.Response, err error) {
- if engines.IsTimeoutError(err) {
- log.Trace().
- // Err(err). // timeout error produces Get "url" error with the query
- Str("engine", seName.String()).
- // Str("url", urll). // can't reliably anonymize it (because it's engine dependent and query isn't passed to this function)
- Msg("_sedefaults.ColError() -> col.OnError(): request timeout error for url")
- } else {
- log.Error().
- Err(err).
- Str("engine", seName.String()).
- // Str("url", urll). // can't reliably anonymize it (because it's engine dependent and query isn't passed to this function)
- Int("statusCode", r.StatusCode).
- Str("response", string(r.Body)). // query can be present, depending on the response from the engine (Google has the query in 3 places)
- Msg("_sedefaults.ColError() -> col.OnError(): request error for url")
-
- dumpPath := fmt.Sprintf("%v%v_col.log.html", config.LogDumpLocation, seName.String())
- log.Debug().
- Str("engine", seName.String()).
- Str("responsePath", dumpPath).
- Func(func(e *zerolog.Event) {
- bodyWriteErr := os.WriteFile(dumpPath, r.Body, 0644)
- if bodyWriteErr != nil {
- log.Error().
- Err(bodyWriteErr).
- Str("engine", seName.String()).
- Msg("_sedefaults.ColError() -> col.OnError(): error writing html response body to file")
- }
- }).
- Msg("_sedefaults.ColError() -> col.OnError(): html response written")
- }
- })
-}
-
-func Prepare(seName engines.Name, options *engines.Options, settings *config.Settings, support *engines.SupportedSettings, info *engines.Info, ctx *context.Context) error {
- if ctx == nil {
- *ctx = context.Background()
- }
-
- if options.UserAgent == "" {
- options.UserAgent = useragent.RandomUserAgent()
- }
- log.Trace().
- Str("engine", seName.String()).
- Str("userAgent", options.UserAgent).
- Msg("Prepare")
-
- // TODO: move to config.SetupConfig
- if settings.RequestedResultsPerPage != 0 && !support.RequestedResultsPerPage {
- log.Panic().
- Str("engine", seName.String()).
- Int("requestedResultsPerPage", settings.RequestedResultsPerPage).
- Msg("_sedefaults.Prepare(): setting not supported by engine")
- // ^PANIC
- }
- if settings.RequestedResultsPerPage == 0 && support.RequestedResultsPerPage {
- // if its used in the code but not set, give it the default value
- settings.RequestedResultsPerPage = info.ResultsPerPage
- }
-
- if options.Mobile && !support.Mobile {
- options.Mobile = false // this line shouldn't matter [1]
- log.Debug().
- Str("engine", seName.String()).
- Bool("mobile", options.Mobile).
- Msg("Mobile set but not supported")
- }
-
- if options.Locale != "" && !support.Locale {
- options.Locale = config.DefaultLocale // [1]
- log.Debug().
- Str("engine", seName.String()).
- Str("locale", options.Locale).
- Msg("Locale set but not supported")
- }
-
- if options.Locale == "" && support.Locale {
- options.Locale = config.DefaultLocale
- }
-
- if options.SafeSearch && !support.SafeSearch {
- options.SafeSearch = false // [1]
- log.Debug().
- Str("engine", seName.String()).
- Bool("safeSearch", options.SafeSearch).
- Msg("SafeSearch set but not supported")
- }
-
- return nil
-}
-
-func InitializeCollectors(colPtr **colly.Collector, pagesColPtr **colly.Collector, settings *config.Settings, options *engines.Options, timings *config.Timings) {
- *colPtr = colly.NewCollector(colly.MaxDepth(1), colly.UserAgent(options.UserAgent), colly.Async())
- *pagesColPtr = colly.NewCollector(colly.MaxDepth(1), colly.UserAgent(options.UserAgent), colly.Async())
-
- if timings != nil {
- var limitRule *colly.LimitRule = &colly.LimitRule{
- DomainGlob: "*",
- Delay: timings.Delay,
- RandomDelay: timings.RandomDelay,
- Parallelism: timings.Parallelism,
- }
-
- if err := (*colPtr).Limit(limitRule); err != nil {
- log.Error().
- Err(err).
- Str("limitRule", fmt.Sprintf("%v", limitRule)).
- Msg("_sedefaults.InitializeCollectors(): failed adding new limit rule")
- }
-
- if timings.Timeout != 0 {
- (*colPtr).SetRequestTimeout(timings.Timeout)
- }
-
- if timings.PageTimeout != 0 {
- (*pagesColPtr).SetRequestTimeout(timings.PageTimeout)
- }
- }
-
- if settings.Proxies != nil {
- log.Debug().
- Strs("proxies", settings.Proxies).
- Msg("Using proxies")
-
- // Rotate proxies
- rp, err := proxy.RoundRobinProxySwitcher(settings.Proxies...)
- if err != nil {
- log.Fatal().
- Err(err).
- Strs("proxies", settings.Proxies).
- Msg("_sedefaults.InitializeCollectors(): failed creating proxy switcher")
- }
-
- (*colPtr).SetProxyFunc(rp)
- (*pagesColPtr).SetProxyFunc(rp)
- }
-
- if settings.InsecureSkipVerify {
- tp := &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
- }
- (*colPtr).WithTransport(tp)
- (*pagesColPtr).WithTransport(tp)
- }
-}
-
-func DoGetRequest(urll string, anonurll string, colCtx *colly.Context, collector *colly.Collector, packageName engines.Name, retError *error) {
- log.Trace().
- Str("engine", packageName.String()).
- Str("url", anonurll).
- Msg("GET")
- err := collector.Request("GET", urll, nil, colCtx, nil)
- if err != nil {
- *retError = fmt.Errorf("%v.Search(): failed GET request to %v with %w", packageName.ToLower(), urll, err)
- }
-}
-
-func DoPostRequest(urll string, requestData io.Reader, colCtx *colly.Context, collector *colly.Collector, packageName engines.Name, retError *error) {
- log.Trace().
- Str("engine", packageName.String()).
- Str("url", urll).
- Msg("POST")
- err := collector.Request("POST", urll, requestData, colCtx, nil)
- if err != nil {
- *retError = fmt.Errorf("%v.Search(): failed POST request to %v and body %v. error %w", packageName.ToLower(), requestData, urll, err)
- }
-}
-
-func PageFromContext(ctx *colly.Context, seName engines.Name) int {
- var pageStr string = ctx.Get("page")
- page, converr := strconv.Atoi(pageStr)
- if converr != nil {
- log.Panic().
- Err(converr).
- Str("engine", seName.String()).
- Str("page", pageStr).
- Msg("_sedefaults.PageFromContext(): failed to convert page number to int")
- // ^PANIC
- }
- return page
-}
From 07619ada1fbbeb2819bf4c6e381b43bc56564f99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?=
<31509435+aleksasiriski@users.noreply.github.com>
Date: Thu, 8 Feb 2024 14:09:16 +0100
Subject: [PATCH 05/11] chore(makefile): reorded make-all tests
---
Makefile | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/Makefile b/Makefile
index eb0b7edd..487e2698 100644
--- a/Makefile
+++ b/Makefile
@@ -32,9 +32,9 @@ test-redis-podman:
sh ./scripts/test-redis-podman.sh
test-redis-docker:
sh ./scripts/test-redis-docker.sh
-test-all: test test-engines test-redis
-test-all-podman: test test-engines test-redis-podman
-test-all-docker: test test-engines test-redis-docker
+test-all: test test-redis test-engines
+test-all-podman: test test-redis-podman test-engines
+test-all-docker: test test-redis-docker test-engines
update:
go get -u ./...
From cc45b54fb75aca63b8f087f5d860e72ea1df98ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?=
<31509435+aleksasiriski@users.noreply.github.com>
Date: Thu, 8 Feb 2024 14:10:57 +0100
Subject: [PATCH 06/11] refactor(search): moved helper funcs to utils.go
---
src/search/perform.go | 55 -------------------------------------
src/search/utils.go | 64 +++++++++++++++++++++++++++++++++++++++++++
2 files changed, 64 insertions(+), 55 deletions(-)
create mode 100644 src/search/utils.go
diff --git a/src/search/perform.go b/src/search/perform.go
index 35d5800a..0d1990ef 100644
--- a/src/search/perform.go
+++ b/src/search/perform.go
@@ -4,13 +4,11 @@ import (
"context"
"fmt"
"net/url"
- "strings"
"time"
"github.com/hearchco/hearchco/src/anonymize"
"github.com/hearchco/hearchco/src/config"
"github.com/hearchco/hearchco/src/search/bucket"
- "github.com/hearchco/hearchco/src/search/category"
"github.com/hearchco/hearchco/src/search/engines"
"github.com/hearchco/hearchco/src/search/rank"
"github.com/hearchco/hearchco/src/search/result"
@@ -84,56 +82,3 @@ func runEngines(engs []engines.Name, timings config.Timings, settings map[engine
})
}
}
-
-func procBang(query string, options *engines.Options, conf *config.Config) (string, config.Timings, []engines.Name) {
- useSpec, specEng := procSpecificEngine(query, options, conf)
- goodCat := procCategory(query, options)
- if !goodCat && !useSpec && query[0] == '!' {
- // options.category is set to GENERAL
- log.Debug().
- Str("queryAnon", anonymize.String(query)).
- Str("queryHash", anonymize.HashToSHA256B64(query)).
- Msg("search.procBang(): invalid bang (not category or engine shortcut)")
- }
-
- query = trimBang(query)
-
- if useSpec {
- return query, conf.Categories[category.GENERAL].Timings, []engines.Name{specEng}
- } else {
- return query, conf.Categories[options.Category].Timings, conf.Categories[options.Category].Engines
- }
-}
-
-func trimBang(query string) string {
- if (query)[0] == '!' {
- return strings.SplitN(query, " ", 2)[1]
- }
- return query
-}
-
-func procSpecificEngine(query string, options *engines.Options, conf *config.Config) (bool, engines.Name) {
- if query[0] != '!' {
- return false, engines.UNDEFINED
- }
- sp := strings.SplitN(query, " ", 2)
- bangWord := sp[0][1:]
- for key, val := range conf.Settings {
- if strings.EqualFold(bangWord, val.Shortcut) || strings.EqualFold(bangWord, key.String()) {
- return true, key
- }
- }
-
- return false, engines.UNDEFINED
-}
-
-func procCategory(query string, options *engines.Options) bool {
- cat := category.FromQuery(query)
- if cat != "" {
- options.Category = cat
- }
- if options.Category == "" {
- options.Category = category.GENERAL
- }
- return cat != ""
-}
diff --git a/src/search/utils.go b/src/search/utils.go
new file mode 100644
index 00000000..bbe6d01a
--- /dev/null
+++ b/src/search/utils.go
@@ -0,0 +1,64 @@
+package search
+
+import (
+ "strings"
+
+ "github.com/hearchco/hearchco/src/anonymize"
+ "github.com/hearchco/hearchco/src/config"
+ "github.com/hearchco/hearchco/src/search/category"
+ "github.com/hearchco/hearchco/src/search/engines"
+ "github.com/rs/zerolog/log"
+)
+
+func procBang(query string, options *engines.Options, conf *config.Config) (string, config.Timings, []engines.Name) {
+ useSpec, specEng := procSpecificEngine(query, options, conf)
+ goodCat := procCategory(query, options)
+ if !goodCat && !useSpec && query[0] == '!' {
+ // options.category is set to GENERAL
+ log.Debug().
+ Str("queryAnon", anonymize.String(query)).
+ Str("queryHash", anonymize.HashToSHA256B64(query)).
+ Msg("search.procBang(): invalid bang (not category or engine shortcut)")
+ }
+
+ query = trimBang(query)
+
+ if useSpec {
+ return query, conf.Categories[category.GENERAL].Timings, []engines.Name{specEng}
+ } else {
+ return query, conf.Categories[options.Category].Timings, conf.Categories[options.Category].Engines
+ }
+}
+
+func trimBang(query string) string {
+ if (query)[0] == '!' {
+ return strings.SplitN(query, " ", 2)[1]
+ }
+ return query
+}
+
+func procSpecificEngine(query string, options *engines.Options, conf *config.Config) (bool, engines.Name) {
+ if query[0] != '!' {
+ return false, engines.UNDEFINED
+ }
+ sp := strings.SplitN(query, " ", 2)
+ bangWord := sp[0][1:]
+ for key, val := range conf.Settings {
+ if strings.EqualFold(bangWord, val.Shortcut) || strings.EqualFold(bangWord, key.String()) {
+ return true, key
+ }
+ }
+
+ return false, engines.UNDEFINED
+}
+
+func procCategory(query string, options *engines.Options) bool {
+ cat := category.FromQuery(query)
+ if cat != "" {
+ options.Category = cat
+ }
+ if options.Category == "" {
+ options.Category = category.GENERAL
+ }
+ return cat != ""
+}
From d343881ac4d3c582dd1374a1cfc99342f46fc904 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?=
<31509435+aleksasiriski@users.noreply.github.com>
Date: Thu, 8 Feb 2024 14:12:12 +0100
Subject: [PATCH 07/11] chore(search): move EngineSearch and comment to top
---
src/search/perform.go | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/search/perform.go b/src/search/perform.go
index 0d1990ef..a0d8ea16 100644
--- a/src/search/perform.go
+++ b/src/search/perform.go
@@ -16,6 +16,9 @@ import (
"github.com/sourcegraph/conc"
)
+// engine_searcher -> NewEngineStarter() uses this
+type EngineSearch func(context.Context, string, *bucket.Relay, engines.Options, config.Settings, config.Timings) error
+
func PerformSearch(query string, options engines.Options, conf *config.Config) []result.Result {
searchTimer := time.Now()
@@ -56,9 +59,6 @@ func PerformSearch(query string, options engines.Options, conf *config.Config) [
return results
}
-// engine_searcher, NewEngineStarter() use this.
-type EngineSearch func(context.Context, string, *bucket.Relay, engines.Options, config.Settings, config.Timings) error
-
func runEngines(engs []engines.Name, timings config.Timings, settings map[engines.Name]config.Settings, query string, worker *conc.WaitGroup, relay *bucket.Relay, options engines.Options) {
config.EnabledEngines = engs
log.Info().
From 91992d5fdb1b52bc44464f947f7021378e190353 Mon Sep 17 00:00:00 2001
From: hiddenMedic <124312252+hiddenMedic@users.noreply.github.com>
Date: Thu, 8 Feb 2024 23:36:46 +0100
Subject: [PATCH 08/11] fix: updated .gitignore to reflect refactor
---
.gitignore | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/.gitignore b/.gitignore
index fb8e7cee..ca329bdc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,8 +27,9 @@ hearchco.*
.vscode/*
test.go
-src/engines/*/site/*
-!src/engines/_engines_test
+src/search/engines/*/site/*
+!src/search/engines/_engines_test
+!src/search/engines/_sedefaults
log/
database/
From 1055bb6a1a1583e8694b6c42846e79e3ac34dfb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?=
<31509435+aleksasiriski@users.noreply.github.com>
Date: Fri, 9 Feb 2024 00:00:51 +0100
Subject: [PATCH 09/11] refactor(search): return relay and use stdlib waitgroup
concurrency
---
go.mod | 1 -
go.sum | 2 --
src/search/perform.go | 31 ++++++++++++++++++-------------
3 files changed, 18 insertions(+), 16 deletions(-)
diff --git a/go.mod b/go.mod
index f2a4ee9d..6fb6e16b 100644
--- a/go.mod
+++ b/go.mod
@@ -90,7 +90,6 @@ require (
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
- github.com/sourcegraph/conc v0.3.0
github.com/temoto/robotstxt v1.1.2 // indirect
golang.org/x/net v0.21.0
golang.org/x/sys v0.17.0 // indirect
diff --git a/go.sum b/go.sum
index 2c7ca4ef..03b703a9 100644
--- a/go.sum
+++ b/go.sum
@@ -247,8 +247,6 @@ github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWR
github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
-github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
-github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
diff --git a/src/search/perform.go b/src/search/perform.go
index a0d8ea16..11c73231 100644
--- a/src/search/perform.go
+++ b/src/search/perform.go
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"net/url"
+ "sync"
"time"
"github.com/hearchco/hearchco/src/anonymize"
@@ -13,7 +14,6 @@ import (
"github.com/hearchco/hearchco/src/search/rank"
"github.com/hearchco/hearchco/src/search/result"
"github.com/rs/zerolog/log"
- "github.com/sourcegraph/conc"
)
// engine_searcher -> NewEngineStarter() uses this
@@ -22,11 +22,7 @@ type EngineSearch func(context.Context, string, *bucket.Relay, engines.Options,
func PerformSearch(query string, options engines.Options, conf *config.Config) []result.Result {
searchTimer := time.Now()
- relay := bucket.Relay{
- ResultMap: make(map[string]*result.Result),
- }
-
- query, timings, toRun := procBang(query, &options, conf)
+ query, timings, enginesToRun := procBang(query, &options, conf)
query = url.QueryEscape(query)
log.Debug().
@@ -36,9 +32,7 @@ func PerformSearch(query string, options engines.Options, conf *config.Config) [
resTimer := time.Now()
log.Debug().Msg("Waiting for results from engines...")
- var worker conc.WaitGroup
- runEngines(toRun, timings, conf.Settings, query, &worker, &relay, options)
- worker.Wait()
+ relay := runEngines(enginesToRun, query, options, conf.Settings, timings)
log.Debug().
Int64("ms", time.Since(resTimer).Milliseconds()).
Msg("Got results")
@@ -59,26 +53,37 @@ func PerformSearch(query string, options engines.Options, conf *config.Config) [
return results
}
-func runEngines(engs []engines.Name, timings config.Timings, settings map[engines.Name]config.Settings, query string, worker *conc.WaitGroup, relay *bucket.Relay, options engines.Options) {
+func runEngines(engs []engines.Name, query string, options engines.Options, settings map[engines.Name]config.Settings, timings config.Timings) *bucket.Relay {
config.EnabledEngines = engs
log.Info().
Int("number", len(config.EnabledEngines)).
Str("engines", fmt.Sprintf("%v", config.EnabledEngines)).
Msg("Enabled engines")
+ relay := bucket.Relay{
+ ResultMap: make(map[string]*result.Result),
+ }
+
+ var wg sync.WaitGroup
engineStarter := NewEngineStarter()
+
for i := range engs {
+ wg.Add(1)
eng := engs[i] // dont change for to `for _, eng := range engs {`, eng retains the same address throughout the whole loop
- worker.Go(func() {
+ go func() {
+ defer wg.Done()
// if an error can be handled inside, it wont be returned
// runs the Search function in the engine package
- err := engineStarter[eng](context.Background(), query, relay, options, settings[eng], timings)
+ err := engineStarter[eng](context.Background(), query, &relay, options, settings[eng], timings)
if err != nil {
log.Error().
Err(err).
Str("engine", eng.String()).
Msg("search.runEngines(): error while searching")
}
- })
+ }()
}
+
+ wg.Wait()
+ return &relay
}
From d57e9f7a4eb9d59d76287df95f0745a44bf3dbae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?=
<31509435+aleksasiriski@users.noreply.github.com>
Date: Fri, 9 Feb 2024 00:08:27 +0100
Subject: [PATCH 10/11] refactor(search): made relay pointer from the start
---
src/search/perform.go | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/search/perform.go b/src/search/perform.go
index 11c73231..d8f0158b 100644
--- a/src/search/perform.go
+++ b/src/search/perform.go
@@ -60,7 +60,7 @@ func runEngines(engs []engines.Name, query string, options engines.Options, sett
Str("engines", fmt.Sprintf("%v", config.EnabledEngines)).
Msg("Enabled engines")
- relay := bucket.Relay{
+ relay := &bucket.Relay{
ResultMap: make(map[string]*result.Result),
}
@@ -74,7 +74,7 @@ func runEngines(engs []engines.Name, query string, options engines.Options, sett
defer wg.Done()
// if an error can be handled inside, it wont be returned
// runs the Search function in the engine package
- err := engineStarter[eng](context.Background(), query, &relay, options, settings[eng], timings)
+ err := engineStarter[eng](context.Background(), query, relay, options, settings[eng], timings)
if err != nil {
log.Error().
Err(err).
@@ -85,5 +85,5 @@ func runEngines(engs []engines.Name, query string, options engines.Options, sett
}
wg.Wait()
- return &relay
+ return relay
}
From 70560872f9a66caad96624a38d5a8df044e71583 Mon Sep 17 00:00:00 2001
From: hiddenMedic <124312252+hiddenMedic@users.noreply.github.com>
Date: Fri, 9 Feb 2024 23:49:52 +0100
Subject: [PATCH 11/11] fix: return resultMap instead of relay
---
src/search/perform.go | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)
diff --git a/src/search/perform.go b/src/search/perform.go
index d8f0158b..22a6ee6f 100644
--- a/src/search/perform.go
+++ b/src/search/perform.go
@@ -23,8 +23,8 @@ func PerformSearch(query string, options engines.Options, conf *config.Config) [
searchTimer := time.Now()
query, timings, enginesToRun := procBang(query, &options, conf)
-
query = url.QueryEscape(query)
+
log.Debug().
Str("queryAnon", anonymize.String(query)).
Str("queryHash", anonymize.HashToSHA256B64(query)).
@@ -32,14 +32,18 @@ func PerformSearch(query string, options engines.Options, conf *config.Config) [
resTimer := time.Now()
log.Debug().Msg("Waiting for results from engines...")
- relay := runEngines(enginesToRun, query, options, conf.Settings, timings)
+
+ resultMap := runEngines(enginesToRun, query, options, conf.Settings, timings)
+
log.Debug().
Int64("ms", time.Since(resTimer).Milliseconds()).
Msg("Got results")
rankTimer := time.Now()
log.Debug().Msg("Ranking...")
- results := rank.Rank(relay.ResultMap, conf.Categories[options.Category].Ranking) // have to make copy, since its a map value
+
+ results := rank.Rank(resultMap, conf.Categories[options.Category].Ranking)
+
rankTimeSince := time.Since(rankTimer)
log.Debug().
Int64("ms", rankTimeSince.Milliseconds()).
@@ -53,14 +57,14 @@ func PerformSearch(query string, options engines.Options, conf *config.Config) [
return results
}
-func runEngines(engs []engines.Name, query string, options engines.Options, settings map[engines.Name]config.Settings, timings config.Timings) *bucket.Relay {
+func runEngines(engs []engines.Name, query string, options engines.Options, settings map[engines.Name]config.Settings, timings config.Timings) map[string]*result.Result {
config.EnabledEngines = engs
log.Info().
Int("number", len(config.EnabledEngines)).
Str("engines", fmt.Sprintf("%v", config.EnabledEngines)).
Msg("Enabled engines")
- relay := &bucket.Relay{
+ relay := bucket.Relay{
ResultMap: make(map[string]*result.Result),
}
@@ -74,7 +78,7 @@ func runEngines(engs []engines.Name, query string, options engines.Options, sett
defer wg.Done()
// if an error can be handled inside, it wont be returned
// runs the Search function in the engine package
- err := engineStarter[eng](context.Background(), query, relay, options, settings[eng], timings)
+ err := engineStarter[eng](context.Background(), query, &relay, options, settings[eng], timings)
if err != nil {
log.Error().
Err(err).
@@ -85,5 +89,5 @@ func runEngines(engs []engines.Name, query string, options engines.Options, sett
}
wg.Wait()
- return relay
+ return relay.ResultMap
}