feat: added some flags to cli, added support for locale in some engines

src/cli/climode.go

@@ -38,6 +38,10 @@ func Run(flags Flags, db cache.DB, conf *config.Config) {
 		MaxPages:   flags.MaxPages,
 		VisitPages: flags.Visit,
 		Category:   category.FromString[flags.Category],
+		UserAgent:  flags.UserAgent,
+		Locale:     flags.Locale,
+		SafeSearch: flags.SafeSearch,
+		Mobile:     flags.Mobile,
 	}
 
 	start := time.Now()

src/cli/structs.go

@@ -11,14 +11,19 @@ type Flags struct {
 
 	// flags
 	Query       string `type:"string" default:"${query_string}" env:"HEARCHCO_QUERY" help:"Query string used for search"`
-	MaxPages    int    `type:"counter" default:"1" env:"HEARCHCO_MAX_PAGES" help:"Number of pages to search"`
 	Cli         bool   `type:"bool" default:"false" env:"HEARCHCO_CLI" help:"Use CLI mode"`
-	Visit       bool   `type:"bool" default:"false" env:"HEARCHCO_VISIT" help:"Should results be visited"`
 	Silent      bool   `type:"bool" default:"false" short:"s" env:"HEARCHCO_SILENT" help:"Should results be printed"`
 	DataDirPath string `type:"path" default:"${data_folder}" env:"HEARCHCO_DATA_DIR" help:"Data folder path"`
 	LogDirPath  string `type:"path" default:"${log_folder}" env:"HEARCHCO_LOG_DIR" help:"Log folder path"`
 	Verbosity   int8   `type:"counter" default:"0" short:"v" env:"HEARCHCO_VERBOSITY" help:"Log level verbosity"`
-	Category    string `type:"string" default:"" short:"c" env:"HEARCHCO_CATEGORY" help:"Search result category. Can also be supplied through the query (e.g. \"!info smartphone\"). Supported values: info[/wiki], science[/sci], news, blog, surf, newnews[/nnews]"`
+	// options
+	MaxPages   int    `type:"counter" default:"1" env:"HEARCHCO_MAX_PAGES" help:"Number of pages to search"`
+	Visit      bool   `type:"bool" default:"false" env:"HEARCHCO_VISIT" help:"Should results be visited"`
+	Category   string `type:"string" default:"" short:"c" env:"HEARCHCO_CATEGORY" help:"Search result category. Can also be supplied through the query (e.g. \"!info smartphone\"). Supported values: info[/wiki], science[/sci], news, blog, surf, newnews[/nnews]"`
+	UserAgent  string `type:"string" default:"" env:"HEARCHCO_USER_AGENT" help:"The user agent"`
+	Locale     string `type:"string" default:"" env:"HEARCHCO_LOCALE" help:"Locale string specifying result language and region preference. The format is en_US"`
+	SafeSearch bool   `type:"bool" default:"false" env:"HEARCHCO_SAFE_SEARCH" help:"Whether to use safe search"`
+	Mobile     bool   `type:"bool" default:"false" env:"HEARCHCO_MOBILE" help:"Whether to gear results towards mobile"`
 	// profiler
 	CPUProfile    bool `type:"bool" default:"false" env:"HEARCHCO_CPUPROFILE" help:"Use cpu profiling"`
 	HeapProfile   bool `type:"bool" default:"false" env:"HEARCHCO_HEAPPROFILE" help:"Use heap profiling"`

src/config/defaults.go

@@ -7,7 +7,7 @@ import (
 	"github.com/hearchco/hearchco/src/engines"
 )
 
-const DefaultLocale string = "en-US"
+const DefaultLocale string = "en_US"
 
 func NewRanking() Ranking {
 	return Ranking{

src/engines/bing/bing.go

@@ -104,6 +104,6 @@ func removeTelemetry(link string) string {
 }
 
 func getLocale(options *engines.Options) string {
-	spl := strings.SplitN(strings.ToLower(options.Locale), "-", 2)
+	spl := strings.SplitN(strings.ToLower(options.Locale), "_", 2)
 	return "&setlang=" + spl[0] + "&cc=" + spl[1]
 }

src/engines/brave/brave.go

@@ -33,6 +33,14 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
 
 	var pageRankCounter []int = make([]int, options.MaxPages*Info.ResultsPerPage)
 
+	localeCookie := getLocale(&options)
+	safeSearchCookie := getSafeSearch(&options)
+
+	col.OnRequest(func(r *colly.Request) {
+		r.Headers.Add("Cookie", localeCookie)
+		r.Headers.Add("Cookie", safeSearchCookie)
+	})
+
 	col.OnHTML(dompaths.Result, func(e *colly.HTMLElement) {
 		dom := e.DOM
 
@@ -74,3 +82,15 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
 
 	return retError
 }
+
+func getLocale(options *engines.Options) string {
+	region := strings.SplitN(strings.ToLower(options.Locale), "_", 2)[1]
+	return "country=" + region
+}
+
+func getSafeSearch(options *engines.Options) string {
+	if options.SafeSearch {
+		return "safesearch=strict"
+	}
+	return "safesearch=off"
+}

src/engines/brave/options.go

@@ -17,4 +17,7 @@ var dompaths engines.DOMPaths = engines.DOMPaths{
 	Description: "div.snippet-description",
 }
 
-var Support engines.SupportedSettings = engines.SupportedSettings{}
+var Support engines.SupportedSettings = engines.SupportedSettings{
+	Locale:     true,
+	SafeSearch: true,
+}

src/engines/mojeek/mojeek.go

@@ -73,7 +73,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
 }
 
 func getLocale(options *engines.Options) string {
-	spl := strings.SplitN(strings.ToLower(options.Locale), "-", 2)
+	spl := strings.SplitN(strings.ToLower(options.Locale), "_", 2)
 	return "&lb=" + spl[0] + "&arc=" + spl[1]
 }
 

src/engines/qwant/qwant.go

@@ -83,11 +83,18 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
 	return retError
 }
 
+// qwant returns this array when an invalid locale is supplied
+var validLocales = [...]string{"bg_bg", "br_fr", "ca_ad", "ca_es", "ca_fr", "co_fr", "cs_cz", "cy_gb", "da_dk", "de_at", "de_ch", "de_de", "ec_ca", "el_gr", "en_au", "en_ca", "en_gb", "en_ie", "en_my", "en_nz", "en_us", "es_ad", "es_ar", "es_cl", "es_co", "es_es", "es_mx", "es_pe", "et_ee", "eu_es", "eu_fr", "fc_ca", "fi_fi", "fr_ad", "fr_be", "fr_ca", "fr_ch", "fr_fr", "gd_gb", "he_il", "hu_hu", "it_ch", "it_it", "ko_kr", "nb_no", "nl_be", "nl_nl", "pl_pl", "pt_ad", "pt_pt", "ro_ro", "sv_se", "th_th", "zh_cn", "zh_hk"}
+
 func getLocale(options *engines.Options) string {
-	locale := options.Locale
-	locale = strings.ToLower(locale)
-	locale = strings.ReplaceAll(locale, "-", "_")
-	return "&locale=" + locale
+	locale := strings.ToLower(options.Locale)
+	for _, vl := range validLocales {
+		if locale == vl {
+			return "&locale=" + locale
+		}
+	}
+	log.Warn().Msgf("qwant.getLocale(): Invalid qwant locale (%v) supplied. Defaulting to en_US. Qwant supports these (disregard specific formatting): %v", options.Locale, validLocales)
+	return "&locale=" + strings.ToLower(config.DefaultLocale)
 }
 
 func getDevice(options *engines.Options) string {

src/engines/structs.go

@@ -43,13 +43,14 @@ type DOMPaths struct {
 }
 
 type Options struct {
-	UserAgent     string
-	MaxPages      int
+	MaxPages   int
+	VisitPages bool
+	Category   category.Name
+	UserAgent  string
+	Locale     string //format: en_US
+	SafeSearch bool
+	Mobile     bool
+
 	ProxyAddr     string
 	JustFirstPage bool
-	VisitPages    bool
-	Locale        string //format: en-US
-	SafeSearch    bool
-	Mobile        bool
-	Category      category.Name
 }

src/engines/swisscows/swisscows.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"strconv"
+	"strings"
 
 	"github.com/gocolly/colly/v2"
 	"github.com/hearchco/hearchco/src/bucket"
@@ -97,7 +98,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
 }
 
 func getLocale(options *engines.Options) string {
-	return "&region=" + options.Locale
+	return "&region=" + strings.Replace(options.Locale, "_", "-", 1)
 }
 
 /*

src/engines/yep/yep.go

@@ -74,7 +74,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
 }
 
 func getLocale(options *engines.Options) string {
-	locale := strings.Split(options.Locale, "-")[1]
+	locale := strings.Split(options.Locale, "_")[1]
 	return "&gl=" + locale
 }