From 76acff058a0ed8036e5e73355760f6f808948245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?= <31509435+aleksasiriski@users.noreply.github.com> Date: Tue, 28 May 2024 20:17:43 +0200 Subject: [PATCH] fix(logging): remove function names from logs, instead use zerolog Caller --- src/cache/badger/badger.go | 6 ++- src/cache/redis/redis.go | 8 ++-- src/cli/climode.go | 7 ++- src/cli/setup.go | 22 +++++++--- src/config/defaults.go | 3 +- src/config/load.go | 43 ++++++++++++++----- src/main.go | 3 +- src/moretime/fancy.go | 3 +- src/profiling.go | 4 +- src/router/proxy.go | 1 + src/router/router.go | 2 + src/search/bucket/addresult.go | 9 ++-- src/search/cache.go | 9 ++-- src/search/engines/_sedefaults/colly.go | 15 ++++--- src/search/engines/_sedefaults/fields.go | 3 +- src/search/engines/_sedefaults/init.go | 6 ++- src/search/engines/_sedefaults/pagecontext.go | 3 +- src/search/engines/bing/bing.go | 11 ++++- src/search/engines/bingimages/bingimages.go | 32 +++++++++----- src/search/engines/duckduckgo/duckduckgo.go | 3 +- src/search/engines/etools/etools.go | 3 +- .../engines/googleimages/googleimages.go | 12 ++++-- .../engines/googlescholar/googlescholar.go | 3 +- src/search/engines/presearch/presearch.go | 7 +-- src/search/engines/qwant/qwant.go | 10 +++-- src/search/engines/startpage/startpage.go | 4 +- src/search/engines/swisscows/swisscows.go | 13 ++++-- src/search/engines/yahoo/yahoo.go | 13 ++++-- src/search/engines/yep/yep.go | 8 ++-- src/search/parse/parse.go | 15 ++++--- src/search/perform.go | 7 ++- src/search/rank/sorting.go | 5 ++- src/search/search.go | 3 +- 33 files changed, 207 insertions(+), 89 deletions(-) diff --git a/src/cache/badger/badger.go b/src/cache/badger/badger.go index 6a25113a..6e7921d3 100644 --- a/src/cache/badger/badger.go +++ b/src/cache/badger/badger.go @@ -33,7 +33,7 @@ func New(dataDirPath string, config config.Badger) (DB, error) { Err(err). Bool("persistence", config.Persist). Str("path", badgerPath). - Msg("badger.New(): error opening badger") + Msg("Error opening badger") } else if config.Persist { log.Info(). Bool("persistence", config.Persist). @@ -50,7 +50,9 @@ func New(dataDirPath string, config config.Badger) (DB, error) { func (db DB) Close() { if err := db.bdb.Close(); err != nil { - log.Error().Err(err).Msg("badger.Close(): error closing badger") + log.Error(). + Err(err). + Msg("Error closing badger") } else { log.Debug().Msg("Successfully closed badger") } diff --git a/src/cache/redis/redis.go b/src/cache/redis/redis.go index b2904fcf..3b102a45 100644 --- a/src/cache/redis/redis.go +++ b/src/cache/redis/redis.go @@ -28,11 +28,11 @@ func New(ctx context.Context, config config.Redis) (DB, error) { log.Error(). Err(err). Str("address", fmt.Sprintf("%v:%v/%v", config.Host, config.Port, config.Database)). - Msg("redis.New(): error connecting to redis") + Msg("Error creating new connection to redis") } else { log.Info(). Str("address", fmt.Sprintf("%v:%v/%v", config.Host, config.Port, config.Database)). - Msg("Successful connection to redis") + Msg("Successfully connected to redis") } return DB{rdb: rdb, ctx: ctx}, nil @@ -40,7 +40,9 @@ func New(ctx context.Context, config config.Redis) (DB, error) { func (db DB) Close() { if err := db.rdb.Close(); err != nil { - log.Error().Err(err).Msg("redis.Close(): error disconnecting from redis") + log.Error(). + Err(err). + Msg("Error closing connection to redis") } else { log.Debug().Msg("Successfully disconnected from redis") } diff --git a/src/cli/climode.go b/src/cli/climode.go index 15f6a025..2ee0952d 100644 --- a/src/cli/climode.go +++ b/src/cli/climode.go @@ -59,11 +59,14 @@ func Run(flags Flags, db cache.DB, conf config.Config) { Str("queryHash", anonymize.HashToSHA256B64(flags.Query)). Int("maxPages", flags.PagesMax). Bool("visit", flags.Visit). - Msg("Started hearching") + Msg("Started searching...") categoryName, err := category.FromString(flags.Category) if err != nil { - log.Fatal().Err(err).Msg("Invalid category") + log.Fatal(). + Caller(). + Err(err). + Msg("Invalid category") } // all of these have default values set and are validated beforehand diff --git a/src/cli/setup.go b/src/cli/setup.go index 938f7841..d8c4426e 100644 --- a/src/cli/setup.go +++ b/src/cli/setup.go @@ -28,12 +28,17 @@ func Setup() Flags { ) if err := ctx.Validate(); err != nil { - log.Panic().Caller().Err(err).Msg("failed parsing cli") // panic is also run inside the library. when does this happen? + log.Panic(). + Caller(). + Err(err). + Msg("Failed parsing cli") // ^PANIC } if cli.Query == "" { - log.Fatal().Caller().Msg("query cannot be empty or whitespace") + log.Fatal(). + Caller(). + Msg("Query cannot be empty or whitespace") // ^FATAL } @@ -45,7 +50,7 @@ func Setup() Flags { Int("pages", cli.PagesMax). Int("min", 1). Int("max", pagesMaxUpperLimit). - Msg("pages value out of range") + Msg("Pages value out of range") // ^FATAL } @@ -55,7 +60,7 @@ func Setup() Flags { Int("start", cli.PagesStart). Int("min", 1). Int("max", gotypelimits.MaxInt-pagesMaxUpperLimit). - Msg("start value out of range") + Msg("Start value out of range") // ^FATAL } else { // since it's >=1, we decrement it to match the 0-based index @@ -63,12 +68,17 @@ func Setup() Flags { } if err := engines.ValidateLocale(cli.Locale); err != nil { - log.Fatal().Caller().Err(err).Msg("invalid locale flag") + log.Fatal(). + Caller(). + Err(err). + Msg("Invalid locale flag") // ^FATAL } if _, err := category.FromString(cli.Category); err != nil { - log.Fatal().Caller().Msg("invalid category flag") + log.Fatal(). + Caller(). + Msg("Invalid category flag") // ^FATAL } diff --git a/src/config/defaults.go b/src/config/defaults.go index 2001a73f..241ea422 100644 --- a/src/config/defaults.go +++ b/src/config/defaults.go @@ -92,8 +92,9 @@ func NewSettings() map[engines.Name]Settings { for _, eng := range engines.Names() { if _, ok := mp[eng]; !ok { log.Fatal(). + Caller(). Str("engine", eng.String()). - Msg("config.NewSettings(): no shortcut set") + Msg("No shortcut set") // ^FATAL } } diff --git a/src/config/load.go b/src/config/load.go index c0fd3abe..7993115c 100644 --- a/src/config/load.go +++ b/src/config/load.go @@ -19,7 +19,9 @@ import ( // passed as pointer since config is modified func (c *Config) fromReader(rc ReaderConfig) { if rc.Server.Proxy.Salt == "" { - log.Fatal().Msg("config.fromReader(): proxy salt is empty") + log.Fatal(). + Caller(). + Msg("Image proxy salt is empty") } nc := Config{ @@ -53,9 +55,10 @@ func (c *Config) fromReader(rc ReaderConfig) { keyName, err := engines.NameString(key) if err != nil { log.Panic(). + Caller(). Err(err). Str("engine", key). - Msg("config.fromReader(): invalid engine name") + Msg("Invalid engine name") // ^PANIC } nc.Settings[keyName] = val @@ -67,7 +70,10 @@ func (c *Config) fromReader(rc ReaderConfig) { if eng.Enabled { engineName, nameErr := engines.NameString(name) if nameErr != nil { - log.Panic().Err(nameErr).Msg("failed converting string to engine name") + log.Panic(). + Caller(). + Err(nameErr). + Msg("Failed converting string to engine name") // ^PANIC } @@ -164,7 +170,10 @@ func (c *Config) Load(dataDirPath string) { // We provide a struct along with the struct tag `koanf` to the // provider. if err := k.Load(structs.Provider(&rc, "koanf"), nil); err != nil { - log.Panic().Err(err).Msg("config.Load(): failed loading default values") + log.Panic(). + Caller(). + Err(err). + Msg("Failed loading default values") // ^PANIC } @@ -172,19 +181,27 @@ func (c *Config) Load(dataDirPath string) { yamlPath := path.Join(dataDirPath, "hearchco.yaml") if _, err := os.Stat(yamlPath); err != nil { log.Trace(). + Caller(). Str("path", yamlPath). - Msg("config.Load(): no yaml config found, looking for .yml") + Msg("No yaml (.yaml) config found, looking for .yml") yamlPath = path.Join(dataDirPath, "hearchco.yml") if _, errr := os.Stat(yamlPath); errr != nil { log.Trace(). + Caller(). Str("path", yamlPath). - Msg("config.Load(): no yaml config found") + Msg("No yaml (.yml) config found") } else if errr := k.Load(file.Provider(yamlPath), yaml.Parser()); errr != nil { - log.Panic().Err(err).Msg("config.Load(): error loading yaml config") + log.Panic(). + Caller(). + Err(err). + Msg("Error loading yaml config") // ^PANIC } } else if err := k.Load(file.Provider(yamlPath), yaml.Parser()); err != nil { - log.Panic().Err(err).Msg("config.Load(): error loading yaml config") + log.Panic(). + Caller(). + Err(err). + Msg("Error loading yaml config") // ^PANIC } @@ -192,13 +209,19 @@ func (c *Config) Load(dataDirPath string) { if err := k.Load(env.Provider("HEARCHCO_", ".", func(s string) string { return strings.Replace(strings.ToLower(strings.TrimPrefix(s, "HEARCHCO_")), "_", ".", -1) }), nil); err != nil { - log.Panic().Err(err).Msg("config.Load(): error loading env config") + log.Panic(). + Caller(). + Err(err). + Msg("Error loading env config") // ^PANIC } // Unmarshal config into struct if err := k.Unmarshal("", &rc); err != nil { - log.Panic().Err(err).Msg("config.Load(): failed unmarshaling koanf config") + log.Panic(). + Caller(). + Err(err). + Msg("Failed unmarshaling koanf config") // ^PANIC } diff --git a/src/main.go b/src/main.go index 136bcada..c63acf79 100644 --- a/src/main.go +++ b/src/main.go @@ -40,8 +40,9 @@ func main() { db, err := cache.New(ctx, cliFlags.DataDirPath, conf.Server.Cache) if err != nil { log.Fatal(). + Caller(). Err(err). - Msg("main.main(): failed creating a new db") + Msg("Failed creating a new db") // ^FATAL } diff --git a/src/moretime/fancy.go b/src/moretime/fancy.go index 478b1bc2..f7db38c5 100644 --- a/src/moretime/fancy.go +++ b/src/moretime/fancy.go @@ -11,8 +11,9 @@ func handleAtoi(s string) int64 { i, err := strconv.Atoi(s) if err != nil { log.Panic(). + Caller(). Err(err). - Msg("failed converting string to int") + Msg("Failed converting string to int") // ^PANIC } return int64(i) diff --git a/src/profiling.go b/src/profiling.go index 1adc46ed..b1729502 100644 --- a/src/profiling.go +++ b/src/profiling.go @@ -53,7 +53,9 @@ func runProfiler(cliFlags cli.Flags) (bool, func()) { profilerToRun := profiler{enabled: false} for _, p := range profilers { if profilerToRun.enabled && p.enabled { - log.Fatal().Msg("main.runProfiler(): only one profiler can be run at a time") + log.Fatal(). + Caller(). + Msg("Only one profiler can be run at a time") // ^FATAL } else if p.enabled { profilerToRun = p diff --git a/src/router/proxy.go b/src/router/proxy.go index 12b72865..428b46f9 100644 --- a/src/router/proxy.go +++ b/src/router/proxy.go @@ -86,6 +86,7 @@ func Proxy(w http.ResponseWriter, r *http.Request, salt string, timeouts config. } log.Trace(). + Caller(). Str("request", fmt.Sprint(nr)). Msg("Created new request") diff --git a/src/router/router.go b/src/router/router.go index 5ced0ec5..bcd358e2 100644 --- a/src/router/router.go +++ b/src/router/router.go @@ -54,6 +54,7 @@ func (rw RouterWrapper) Start(ctx context.Context) { err := srv.Shutdown(timeout) if err != nil { log.Error(). + Caller(). Err(err). Msg("Server shut down failed") } else { @@ -66,6 +67,7 @@ func (rw RouterWrapper) Start(ctx context.Context) { err := srv.ListenAndServe() if err != nil && err != http.ErrServerClosed { log.Fatal(). + Caller(). Err(err). Msg("Failed to start server") } diff --git a/src/search/bucket/addresult.go b/src/search/bucket/addresult.go index 80070d37..d0f404dc 100644 --- a/src/search/bucket/addresult.go +++ b/src/search/bucket/addresult.go @@ -12,19 +12,21 @@ import ( func AddSEResult(seResult *result.RetrievedResult, seName engines.Name, relay *Relay, options engines.Options, pagesCol *colly.Collector, nEnabledEngines int) bool { if seResult == nil { log.Error(). + Caller(). Str("engine", seName.String()). - Msg("bucket.AddSEResult(): nil result") + Msg("Result is nil") return false } // TODO: add a check if image result is valid if seResult.URL == "" || seResult.Title == "" { log.Error(). + Caller(). Str("engine", seName.String()). Str("url", seResult.URL). Str("title", seResult.Title). Str("description", seResult.Description). - Msg("bucket.AddSEResult(): invalid result, some fields are empty") + Msg("Invalid result, some fields are empty") return false } @@ -90,9 +92,10 @@ func AddSEResult(seResult *result.RetrievedResult, seName engines.Name, relay *R if !exists && options.VisitPages { if err := pagesCol.Visit(seResult.URL); err != nil { log.Error(). + Caller(). Err(err). Str("url", seResult.URL). - Msg("bucket.AddSEResult(): failed visiting") + Msg("Failed visiting page") } } diff --git a/src/search/cache.go b/src/search/cache.go index 325f2b70..1220a386 100644 --- a/src/search/cache.go +++ b/src/search/cache.go @@ -23,10 +23,11 @@ func CacheAndUpdateResults( serr := db.Set(query, results, ttlConf.Time) if serr != nil { log.Error(). + Caller(). Err(serr). Str("queryAnon", anonymize.String(query)). Str("queryHash", anonymize.HashToSHA256B64(query)). - Msg("cli.Run(): error updating database with search results") + Msg("Error updating database with search results") } } else { log.Debug(). @@ -36,10 +37,11 @@ func CacheAndUpdateResults( ttl, terr := db.GetTTL(query) if terr != nil { log.Error(). + Caller(). Err(terr). Str("queryAnon", anonymize.String(query)). Str("queryHash", anonymize.HashToSHA256B64(query)). - Msg("cli.Run(): error getting TTL from database") + Msg("Error getting TTL from database") } else if ttl < ttlConf.RefreshTime { log.Info(). Str("queryAnon", anonymize.String(query)). @@ -50,10 +52,11 @@ func CacheAndUpdateResults( if uerr != nil { // Error in updating cache is not returned, just logged log.Error(). + Caller(). Err(uerr). Str("queryAnon", anonymize.String(query)). Str("queryHash", anonymize.HashToSHA256B64(query)). - Msg("cli.Run(): error replacing old results while updating database") + Msg("Error replacing old results while updating database") } } } diff --git a/src/search/engines/_sedefaults/colly.go b/src/search/engines/_sedefaults/colly.go index 77245ad1..66b7adc2 100644 --- a/src/search/engines/_sedefaults/colly.go +++ b/src/search/engines/_sedefaults/colly.go @@ -14,14 +14,16 @@ func colRequest(col *colly.Collector, ctx context.Context, seName engines.Name, if err := ctx.Err(); err != nil { if engines.IsTimeoutError(err) { log.Trace(). + Caller(). Err(err). Str("engine", seName.String()). - Msg("_sedefaults.colRequest(): context timeout error") + Msg("Context timeout error") } else { log.Error(). + Caller(). Err(err). Str("engine", seName.String()). - Msg("_sedefaults.colRequest(): context error") + Msg("Context error") } r.Abort() return @@ -46,12 +48,13 @@ func colError(col *colly.Collector, seName engines.Name, visiting bool) { event = log.Trace() } event. + Caller(). Err(err). Str("engine", seName.String()). // Str("url", urll). // can't reliably anonymize it (because it's engine dependent) Int("statusCode", r.StatusCode). Bytes("response", r.Body). // WARN: query can be present, depending on the response from the engine (example: google has the query in 3 places) - Msg("_sedefaults.colError(): request error for url") + Msg("Request error for url") } }) } @@ -61,15 +64,17 @@ func pagesColResponse(pagesCol *colly.Collector, seName engines.Name, relay *buc urll := r.Ctx.Get("originalURL") if urll == "" { log.Error(). - Msg("_sedefaults.pagesColResponse(): error getting original url") + Caller(). + Msg("Error getting original url") return } err := bucket.SetResultResponse(urll, r, relay, seName) if err != nil { log.Error(). + Caller(). Err(err). - Msg("_sedefaults.pagesColResponse(): error setting result") + Msg("Error setting result") } }) } diff --git a/src/search/engines/_sedefaults/fields.go b/src/search/engines/_sedefaults/fields.go index 2227e8b3..57b16f90 100644 --- a/src/search/engines/_sedefaults/fields.go +++ b/src/search/engines/_sedefaults/fields.go @@ -27,11 +27,12 @@ func RawFieldsFromDOM(dom *goquery.Selection, dompaths engines.DOMPaths, seName if !hrefExists { log.Error(). + Caller(). Str("engine", seName.String()). Str("url", linkText). Str("title", titleText). Str("description", descText). - Msgf("_sedefaults.RawFieldsFromDOM(): href attribute doesn't exist on matched URL element (%v)", dompaths.Link) + Msgf("Href attribute doesn't exist on matched URL element (%v)", dompaths.Link) return "", "", "" } diff --git a/src/search/engines/_sedefaults/init.go b/src/search/engines/_sedefaults/init.go index 7241ab4b..cb6bb0d4 100644 --- a/src/search/engines/_sedefaults/init.go +++ b/src/search/engines/_sedefaults/init.go @@ -46,9 +46,10 @@ func InitializeCollectors(ctx context.Context, engineName engines.Name, options } if err := col.Limit(&limitRule); err != nil { log.Error(). + Caller(). Err(err). Str("limitRule", fmt.Sprintf("%v", limitRule)). - Msg("_sedefaults.InitializeCollectors(): failed adding new limit rule") + Msg("Failed adding new limit rule") } // set collector proxies @@ -61,9 +62,10 @@ func InitializeCollectors(ctx context.Context, engineName engines.Name, options rp, err := proxy.RoundRobinProxySwitcher(settings.Proxies...) if err != nil { log.Fatal(). + Caller(). Err(err). Strs("proxies", settings.Proxies). - Msg("_sedefaults.InitializeCollectors(): failed creating proxy switcher") + Msg("Failed creating proxy switcher") } col.SetProxyFunc(rp) diff --git a/src/search/engines/_sedefaults/pagecontext.go b/src/search/engines/_sedefaults/pagecontext.go index 8370c898..8be1e9c6 100644 --- a/src/search/engines/_sedefaults/pagecontext.go +++ b/src/search/engines/_sedefaults/pagecontext.go @@ -13,10 +13,11 @@ func PageFromContext(ctx *colly.Context, seName engines.Name) int { page, converr := strconv.Atoi(pageStr) if converr != nil { log.Panic(). + Caller(). Err(converr). Str("engine", seName.String()). Str("page", pageStr). - Msg("_sedefaults.PageFromContext(): failed to convert page number to int") + Msg("Failed to convert page number to int") // ^PANIC } return page diff --git a/src/search/engines/bing/bing.go b/src/search/engines/bing/bing.go index 1147feda..a82d483a 100644 --- a/src/search/engines/bing/bing.go +++ b/src/search/engines/bing/bing.go @@ -90,7 +90,11 @@ func removeTelemetry(link string) string { if strings.HasPrefix(link, "https://www.bing.com/ck/a?") { parsedUrl, err := url.Parse(link) if err != nil { - log.Error().Err(err).Str("url", link).Msg("bing.removeTelemetry(): error parsing url") + log.Error(). + Caller(). + Err(err). + Str("url", link). + Msg("Error parsing url") return "" } @@ -99,7 +103,10 @@ func removeTelemetry(link string) string { cleanUrl, err := base64.RawURLEncoding.DecodeString(encodedUrl) if err != nil { - log.Error().Err(err).Msg("bing.removeTelemetry(): failed decoding string from base64") + log.Error(). + Caller(). + Err(err). + Msg("Failed decoding string from base64") } return string(cleanUrl) } diff --git a/src/search/engines/bingimages/bingimages.go b/src/search/engines/bingimages/bingimages.go index 203eb5ba..84177d9a 100644 --- a/src/search/engines/bingimages/bingimages.go +++ b/src/search/engines/bingimages/bingimages.go @@ -45,20 +45,22 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o if err := json.Unmarshal([]byte(metadataS), &jsonMetadata); err != nil { log.Error(). + Caller(). Err(err). Str("jsonMetadata", metadataS). - Msg("bingimages.Search() -> onHTML: failed to unmarshal metadata") + Msg("Failed to unmarshal metadata") return } if jsonMetadata.ImageURL == "" || jsonMetadata.PageURL == "" || jsonMetadata.ThumbnailURL == "" { log.Error(). + Caller(). Str("engine", Info.Name.String()). Str("jsonMetadata", metadataS). Str("url", jsonMetadata.PageURL). Str("original", jsonMetadata.ImageURL). Str("thumbnail", jsonMetadata.ThumbnailURL). - Msg("bingimages.Search() -> onHTML: Couldn't find image, thumbnail, or page URL") + Msg("Couldn't find image, thumbnail, or page URL") return } @@ -66,9 +68,10 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o if titleText == "" { // could also use the json data ("t" field), it seems to include weird/erroneous characters though (particularly '\ue000' and '\ue001') log.Error(). + Caller(). Str("engine", Info.Name.String()). Str("jsonMetadata", metadataS). - Msg("bingimages.Search() -> onHTML: Couldn't find title") + Msg("Couldn't find title") return } @@ -76,10 +79,11 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o imgFormatS := strings.TrimSpace(dom.Find(dompaths.ImgFormatStr).Text()) if imgFormatS == "" { log.Trace(). + Caller(). Str("engine", Info.Name.String()). Str("jsonMetadata", metadataS). Str("title", titleText). - Msg("bingimages.Search() -> onHTML: Couldn't find image format (probably a video)") + Msg("Couldn't find image format (probably a video)") return } @@ -93,26 +97,28 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o imgH, err := strconv.Atoi(imgFormat[0]) if err != nil { log.Error(). + Caller(). Err(err). Str("engine", Info.Name.String()). Str("height", imgFormat[0]). Str("jsonMetadata", metadataS). Str("title", titleText). Str("imgFormatS", imgFormatS). - Msg("bingimages.Search() -> onHTML: Failed to convert original height to int") + Msg("Failed to convert original height to int") return } imgW, err := strconv.Atoi(imgFormat[1]) if err != nil { log.Error(). + Caller(). Err(err). Str("engine", Info.Name.String()). Str("width", imgFormat[1]). Str("jsonMetadata", metadataS). Str("title", titleText). Str("imgFormatS", imgFormatS). - Msg("bingimages.Search() -> onHTML: Failed to convert original width to int") + Msg("Failed to convert original width to int") return } @@ -130,46 +136,50 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o if !found { log.Error(). + Caller(). Str("engine", Info.Name.String()). Str("jsonMetadata", metadataS). Str("title", titleText). Str("height", thmbHS). Str("width", thmbWS). - Msg("bingimages.Search() -> onHTML: Couldn't find thumbnail format") + Msg("Couldn't find thumbnail format") return } thmbH, err := strconv.Atoi(thmbHS) if err != nil { log.Error(). + Caller(). Err(err). Str("engine", Info.Name.String()). Str("height", thmbHS). Str("jsonMetadata", metadataS). Str("title", titleText). - Msg("bingimages.Search() -> onHTML: Failed to convert thumbnail height to int") + Msg("Failed to convert thumbnail height to int") return } thmbW, err := strconv.Atoi(thmbWS) if err != nil { log.Error(). + Caller(). Err(err). Str("engine", Info.Name.String()). Str("width", thmbWS). Str("jsonMetadata", metadataS). Str("title", titleText). - Msg("bingimages.Search() -> onHTML: Failed to convert thumbnail width to int") + Msg("Failed to convert thumbnail width to int") return } source := strings.TrimSpace(dom.Find(dompaths.Source).Text()) if source == "" { log.Error(). + Caller(). Str("engine", Info.Name.String()). Str("jsonMetadata", metadataS). Str("title", titleText). - Msg("bingimages.Search() -> onHTML: Couldn't find source") + Msg("Couldn't find source") return } @@ -191,7 +201,7 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o col.OnResponse(func(r *colly.Response) { if len(r.Body) == 0 { - log.Trace(). + log.Error(). Str("engine", Info.Name.String()). Msg("Got empty response, probably too many requests") } diff --git a/src/search/engines/duckduckgo/duckduckgo.go b/src/search/engines/duckduckgo/duckduckgo.go index 90be3be9..31c624ae 100644 --- a/src/search/engines/duckduckgo/duckduckgo.go +++ b/src/search/engines/duckduckgo/duckduckgo.go @@ -61,12 +61,13 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o case 3: if !hrefExists { log.Error(). + Caller(). Str("engine", Info.Name.String()). Str("url", linkText). Str("title", titleText). Str("description", descText). Str("link selector", dompaths.Link). - Msg("duckduckgo.Search(): href attribute doesn't exist on matched URL element") + Msg("Href attribute doesn't exist on matched URL element") return } diff --git a/src/search/engines/etools/etools.go b/src/search/engines/etools/etools.go index f0ed87b8..a56025e9 100644 --- a/src/search/engines/etools/etools.go +++ b/src/search/engines/etools/etools.go @@ -35,9 +35,10 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o // Need to perform this check here so the check below (linkText[0] != 'h') doesn't panic if linkText == "" { log.Error(). + Caller(). Str("title", titleText). Str("description", descText). - Msg("etools.Search(): invalid result, url is empty.") + Msg("Invalid result, url is empty") return } diff --git a/src/search/engines/googleimages/googleimages.go b/src/search/engines/googleimages/googleimages.go index 7785a01e..e4ace963 100644 --- a/src/search/engines/googleimages/googleimages.go +++ b/src/search/engines/googleimages/googleimages.go @@ -41,8 +41,10 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o if index == -1 { log.Error(). + Caller(). + Str("engine", Info.Name.String()). Str("body", body). - Msg("googleimages.Search() -> col.OnResponse: failed parsing response: failed finding start of JSON") + Msg("Failed parsing response, couldn't find the start of JSON") return } @@ -50,8 +52,11 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o var jsonResponse JsonResponse if err := json.Unmarshal([]byte(body), &jsonResponse); err != nil { log.Error(). + Caller(). + Err(err). + Str("engine", Info.Name.String()). Str("body", body). - Msg("googleimages.Search() -> col.OnResponse: failed parsing response: failed unmarshalling JSON") + Msg("Failed parsing response, couldn't unmarshal JSON") return } @@ -83,12 +88,13 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o } } else { log.Error(). + Caller(). Str("engine", Info.Name.String()). Str("jsonMetadata", fmt.Sprintf("%v", metadata)). Str("url", resultJson.ReferrerUrl). Str("original", origImg.Url). Str("thumbnail", thmbImg.Url). - Msg("googleimages.Search() -> col.OnResponse: Couldn't find image URL") + Msg("Couldn't find image URL") } } }) diff --git a/src/search/engines/googlescholar/googlescholar.go b/src/search/engines/googlescholar/googlescholar.go index 8e08707c..425c7277 100644 --- a/src/search/engines/googlescholar/googlescholar.go +++ b/src/search/engines/googlescholar/googlescholar.go @@ -79,9 +79,10 @@ func removeTelemetry(link string) string { parsedURL, err := url.Parse(link) if err != nil { log.Error(). + Caller(). Err(err). Str("link", link). - Msg("error parsing link") + Msg("Error parsing link") return link } diff --git a/src/search/engines/presearch/presearch.go b/src/search/engines/presearch/presearch.go index 666bd995..b6815d04 100644 --- a/src/search/engines/presearch/presearch.go +++ b/src/search/engines/presearch/presearch.go @@ -50,10 +50,11 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o err := json.Unmarshal(r.Body, &pr) if err != nil { log.Error(). + Caller(). Err(err). Str("engine", Info.Name.String()). Bytes("body", r.Body). - Msg("Failed body unmarshall to json") + Msg("Failed to parse response, couldn't unmarshal JSON") } counter := 1 @@ -79,12 +80,12 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o log.Trace(). Err(err). Str("engine", Info.Name.String()). - Msg("failed requesting with API") + Msg("Failed requesting with API due to timeout") } else if err != nil { log.Error(). Err(err). Str("engine", Info.Name.String()). - Msg("failed requesting with API") + Msg("Failed requesting with API") } } }) diff --git a/src/search/engines/qwant/qwant.go b/src/search/engines/qwant/qwant.go index aaa0f2cf..9d0de3c5 100644 --- a/src/search/engines/qwant/qwant.go +++ b/src/search/engines/qwant/qwant.go @@ -40,13 +40,13 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o page := pageIndex + options.Pages.Start + 1 var parsedResponse QwantResponse - err := json.Unmarshal(r.Body, &parsedResponse) - if err != nil { + if err := json.Unmarshal(r.Body, &parsedResponse); err != nil { log.Error(). + Caller(). Err(err). Str("engine", Info.Name.String()). Bytes("body", r.Body). - Msg("Failed body unmarshall to json") + Msg("Failed to parse response, couldn't unmarshal JSON") } mainline := parsedResponse.Data.Res.Items.Mainline @@ -107,10 +107,12 @@ func getLocale(options engines.Options) string { return "&locale=" + locale } } + log.Warn(). + Caller(). Str("locale", options.Locale). Strs("validLocales", validLocales[:]). - Msg("qwant.getLocale(): Invalid qwant locale supplied. Falling back to en_US. Qwant supports these (disregard specific formatting)") + Msg("Invalid locale supplied, falling back to en_US") return "&locale=" + strings.ToLower(config.DefaultLocale) } diff --git a/src/search/engines/startpage/startpage.go b/src/search/engines/startpage/startpage.go index d0a247dd..353738b2 100644 --- a/src/search/engines/startpage/startpage.go +++ b/src/search/engines/startpage/startpage.go @@ -47,11 +47,11 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o if strings.Contains(string(r.Body), "to prevent possible abuse of our service") { log.Error(). Str("engine", Info.Name.String()). - Msg("Request blocked by engine due to scraping") + Msg("Request blocked due to scraping") } else if strings.Contains(string(r.Body), "This page cannot function without javascript") { log.Error(). Str("engine", Info.Name.String()). - Msg("Engine couldn't load requests, needs javascript") + Msg("Couldn't load requests, needs javascript") } }) diff --git a/src/search/engines/swisscows/swisscows.go b/src/search/engines/swisscows/swisscows.go index af587d51..d56d090a 100644 --- a/src/search/engines/swisscows/swisscows.go +++ b/src/search/engines/swisscows/swisscows.go @@ -37,7 +37,10 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o var qry string = "?" + r.URL.RawQuery nonce, sig, err := generateAuth(qry) if err != nil { - log.Error().Err(err).Msg("swisscows.Search() -> col.OnRequest: failed building request: failed generating auth") + log.Error(). + Caller(). + Err(err). + Msg("Failed building request, couldn't generate auth") return } @@ -51,10 +54,11 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o urll := r.Request.URL.String() anonUrll := anonymize.Substring(urll, query) log.Trace(). + Str("engine", Info.Name.String()). Str("url", anonUrll). Str("nonce", r.Request.Headers.Get("X-Request-Nonce")). Str("signature", r.Request.Headers.Get("X-Request-Signature")). - Msg("swisscows.Search() -> col.OnResponse()") + Msg("Got response") pageIndex := _sedefaults.PageFromContext(r.Request.Ctx, Info.Name) page := pageIndex + options.Pages.Start + 1 @@ -63,9 +67,10 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o err := json.Unmarshal(r.Body, &parsedResponse) if err != nil { log.Error(). + Caller(). Err(err). Bytes("body", r.Body). - Msg("swisscows.Search() -> col.OnResponse(): failed body unmarshall to json") + Msg("Failed to parse response, couldn't unmarshal JSON") return } @@ -140,7 +145,7 @@ col.OnHTML("div.web-results > article.item-web", func(e *colly.HTMLElement) { Str("url", linkText). Str("title", titleText). Str("description", descText). - Msg("Matched Result, but couldn't retrieve data") + Msg("Matched result, but couldn't retrieve data") } }) */ diff --git a/src/search/engines/yahoo/yahoo.go b/src/search/engines/yahoo/yahoo.go index 17655cd0..25a1744c 100644 --- a/src/search/engines/yahoo/yahoo.go +++ b/src/search/engines/yahoo/yahoo.go @@ -53,24 +53,26 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o if !hrefExists { log.Error(). + Caller(). Str("engine", Info.Name.String()). Str("url", linkText). Str("title", titleText). Str("description", descText). Str("link selector", dompaths.Link). - Msg("yahoo.Search(): href attribute doesn't exist on matched URL element") + Msg("Href attribute doesn't exist on matched URL element") return } if !labelExists { log.Error(). + Caller(). Str("engine", Info.Name.String()). Str("url", linkText). Str("title", titleText). Str("description", descText). Str("title selector", dompaths.Title). - Msg("yahoo.Search(): aria attribute doesn't exist on matched title element") + Msg("Aria attribute doesn't exist on matched title element") return } @@ -118,16 +120,21 @@ func removeTelemetry(link string) string { if !strings.Contains(link, "://r.search.yahoo.com/") { return link } + suff := strings.SplitAfterN(link, "/RU=http", 2)[1] link = "http" + strings.SplitN(suff, "/RK=", 2)[0] + newLink, err := url.QueryUnescape(link) if err != nil { log.Error(). + Caller(). Err(err). Str("url", link). - Msg("yahoo.removeTelemetry(): couldn't parse url, url.QueryUnescape() failed") + Msg("Couldn't parse url, url.QueryUnescape() failed") + return "" } + return newLink } diff --git a/src/search/engines/yep/yep.go b/src/search/engines/yep/yep.go index 6c92f1a3..2d70495a 100644 --- a/src/search/engines/yep/yep.go +++ b/src/search/engines/yep/yep.go @@ -43,11 +43,12 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o if index != 0 || body[len(body)-1] != byte(end) { log.Error(). + Caller(). + Str("engine", Info.Name.String()). Str("body", body). Str("start", start). Str("end", string(end)). - Str("engine", Info.Name.String()). - Msg("failed parsing response: failed finding start and/or end of JSON") + Msg("Failed parsing response, couldn't find start/end of JSON") return } @@ -57,11 +58,12 @@ func (e Engine) Search(ctx context.Context, query string, relay *bucket.Relay, o var content JsonResponse if err := json.Unmarshal([]byte(resultsJson), &content); err != nil { log.Error(). + Caller(). Err(err). Str("engine", Info.Name.String()). Str("body", body). Str("content", resultsJson). - Msg("Failed unmarshalling content") + Msg("Failed parsing response, couldn't unmarshal JSON") return } diff --git a/src/search/parse/parse.go b/src/search/parse/parse.go index 140dc2e1..66b010c9 100644 --- a/src/search/parse/parse.go +++ b/src/search/parse/parse.go @@ -14,9 +14,10 @@ func ParseURL(rawURL string) string { urll, err := parseURL(rawURL) if err != nil { log.Error(). + Caller(). Err(err). Str("url", urll). - Msg("parse.ParseURL(): couldn't parse url") + Msg("Couldn't parse url") return rawURL } return urll @@ -42,9 +43,10 @@ func ParseTextWithHTML(rawHTML string) string { text, err := parseTextWithHTML(rawHTML) if err != nil { log.Error(). + Caller(). Err(err). Str("html", rawHTML). - Msg("parse.ParseTextWithHTML(): failed parsing text with html") + Msg("Failed parsing text with html") return rawHTML } return text @@ -52,13 +54,16 @@ func ParseTextWithHTML(rawHTML string) string { func parseTextWithHTML(rawHTML string) (string, error) { var result string = "" - htmlNode, perr := html.ParseFragment(strings.NewReader(rawHTML), nil) - if perr != nil { - return "", fmt.Errorf("parse.parseTextWithHTML(): failed html.ParseFragment on %v. error: %w", rawHTML, perr) + + htmlNode, err := html.ParseFragment(strings.NewReader(rawHTML), nil) + if err != nil { + return "", fmt.Errorf("Failed html.ParseFragment on %v: %w", rawHTML, err) } + for _, el := range htmlNode { sel := goquery.NewDocumentFromNode(el) result += sel.Text() } + return result, nil } diff --git a/src/search/perform.go b/src/search/perform.go index 552ba1a9..b9e4a316 100644 --- a/src/search/perform.go +++ b/src/search/perform.go @@ -18,7 +18,9 @@ import ( func PerformSearch(query string, options engines.Options, categoryConf config.Category, settings map[engines.Name]config.Settings, salt string) []result.Result { // check for empty query if query == "" { - log.Trace().Msg("Empty search query.") + log.Trace(). + Caller(). + Msg("Empty search query.") return []result.Result{} } @@ -93,9 +95,10 @@ func runEngines(engs []engines.Name, query string, options engines.Options, sett errs := engineStarter[eng].Search(context.Background(), query, &relay, options, settings[eng], timings, salt, len(engs)) if len(errs) > 0 { log.Error(). + Caller(). Errs("errors", errs). Str("engine", eng.String()). - Msg("search.runEngines(): error while searching") + Msg("Error(s) while searching") } }() } diff --git a/src/search/rank/sorting.go b/src/search/rank/sorting.go index dbd85aec..891215d4 100644 --- a/src/search/rank/sorting.go +++ b/src/search/rank/sorting.go @@ -21,13 +21,16 @@ func (r ByRetrievedRank) Less(i, j int) bool { if r[i].RetRank.Page != r[j].RetRank.Page { return r[i].RetRank.Page < r[j].RetRank.Page } + if r[i].RetRank.OnPageRank != r[j].RetRank.OnPageRank { return r[i].RetRank.OnPageRank < r[j].RetRank.OnPageRank } log.Error(). + Caller(). Str("comparableA", fmt.Sprintf("%v", r[i])). Str("comparableB", fmt.Sprintf("%v", r[j])). - Msg("rank.(r ByRetrievedRank)Less(): failed at ranking") + Msg("Failed at ranking") + return true } diff --git a/src/search/search.go b/src/search/search.go index 0865ece7..870b73f0 100644 --- a/src/search/search.go +++ b/src/search/search.go @@ -16,10 +16,11 @@ func Search(query string, options engines.Options, db cache.DB, categoryConf con if gerr != nil { // Error in reading cache is not returned, just logged log.Error(). + Caller(). Err(gerr). Str("queryAnon", anonymize.String(query)). Str("queryHash", anonymize.HashToSHA256B64(query)). - Msg("cli.Run(): failed accessing cache") + Msg("Failed accessing cache") } else if results != nil { foundInDB = true } else {