Merge pull request #203 from hearchco/refactor-search

refactor(search,bucket,sedefaults): better project structure

.gitignore

@@ -27,8 +27,9 @@ hearchco.*
 .vscode/*
 test.go
 
-src/engines/*/site/*
-!src/engines/_engines_test
+src/search/engines/*/site/*
+!src/search/engines/_engines_test
+!src/search/engines/_sedefaults
 
 log/
 database/

Makefile

@@ -1,34 +1,40 @@
 run:
 	go run ./src
+run-cli:
+	go run ./src --cli
 
 debug:
 	go run ./src -v
+debug-cli:
+	go run ./srv -v --cli
 
 trace:
 	go run ./src -vv
+trace-cli:
+	go run ./src -vv --cli
 
 setup:
 	go get ./...
 	go install github.com/dmarkham/enumer@latest
 	go generate ./...
+install: setup
 
 build:
 	go build ./...
 
 test:
 	sh ./scripts/test.sh
-
 test-engines:
 	sh ./scripts/test-engines.sh
-
 test-redis:
 	sh ./scripts/test-redis.sh
-
 test-redis-podman:
 	sh ./scripts/test-redis-podman.sh
-
 test-redis-docker:
 	sh ./scripts/test-redis-docker.sh
+test-all: test test-redis test-engines
+test-all-podman: test test-redis-podman test-engines
+test-all-docker: test test-redis-docker test-engines
 
 update:
 	go get -u ./...

generate/searcher/searcher.go

@@ -22,7 +22,7 @@ var (
 	trimprefix    = flag.String("trimprefix", "", "trim the `prefix` from the generated constant names")
 	buildTags     = flag.String("tags", "", "comma-separated list of build tags to apply")
 	packageName   = flag.String("packagename", "", "name of the package for generated code; default current package")
-	enginesImport = flag.String("enginesimport", "github.com/hearchco/hearchco/src/engines", "source of the engines import, which is prefixed to imports for consts; default github.com/hearchco/hearchco/src/engines")
+	enginesImport = flag.String("enginesimport", "github.com/hearchco/hearchco/src/search/engines", "source of the engines import, which is prefixed to imports for consts; default github.com/hearchco/hearchco/src/search/engines")
 	linecomment   = flag.Bool("linecomment", false, "use line comment text as printed text when present")
 )
 

go.mod

@@ -90,7 +90,6 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
-	github.com/sourcegraph/conc v0.3.0
 	github.com/temoto/robotstxt v1.1.2 // indirect
 	golang.org/x/net v0.21.0
 	golang.org/x/sys v0.17.0 // indirect

go.sum

@@ -247,8 +247,6 @@ github.com/rs/zerolog v1.32.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWR
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
-github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
-github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=

src/cli/climode.go

@@ -5,12 +5,12 @@ import (
 	"time"
 
 	"github.com/hearchco/hearchco/src/anonymize"
-	"github.com/hearchco/hearchco/src/bucket/result"
 	"github.com/hearchco/hearchco/src/cache"
-	"github.com/hearchco/hearchco/src/category"
 	"github.com/hearchco/hearchco/src/config"
-	"github.com/hearchco/hearchco/src/engines"
 	"github.com/hearchco/hearchco/src/search"
+	"github.com/hearchco/hearchco/src/search/category"
+	"github.com/hearchco/hearchco/src/search/engines"
+	"github.com/hearchco/hearchco/src/search/result"
 	"github.com/rs/zerolog/log"
 )
 

src/cli/setup.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 
 	"github.com/alecthomas/kong"
-	"github.com/hearchco/hearchco/src/category"
-	"github.com/hearchco/hearchco/src/engines"
+	"github.com/hearchco/hearchco/src/search/category"
+	"github.com/hearchco/hearchco/src/search/engines"
 	"github.com/rs/zerolog/log"
 )
 

src/config/defaults.go

@@ -4,9 +4,9 @@ import (
 	"log"
 	"time"
 
-	"github.com/hearchco/hearchco/src/category"
-	"github.com/hearchco/hearchco/src/engines"
 	"github.com/hearchco/hearchco/src/moretime"
+	"github.com/hearchco/hearchco/src/search/category"
+	"github.com/hearchco/hearchco/src/search/engines"
 )
 
 const DefaultLocale string = "en_US"

src/config/load.go

@@ -6,9 +6,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/hearchco/hearchco/src/category"
-	"github.com/hearchco/hearchco/src/engines"
 	"github.com/hearchco/hearchco/src/moretime"
+	"github.com/hearchco/hearchco/src/search/category"
+	"github.com/hearchco/hearchco/src/search/engines"
 	"github.com/knadh/koanf/parsers/yaml"
 	"github.com/knadh/koanf/providers/env"
 	"github.com/knadh/koanf/providers/file"

src/config/structs.go

@@ -3,8 +3,8 @@ package config
 import (
 	"time"
 
-	"github.com/hearchco/hearchco/src/category"
-	"github.com/hearchco/hearchco/src/engines"
+	"github.com/hearchco/hearchco/src/search/category"
+	"github.com/hearchco/hearchco/src/search/engines"
 )
 
 type EngineRanking struct {

src/router/search.go

@@ -8,12 +8,12 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/goccy/go-json"
 
-	"github.com/hearchco/hearchco/src/bucket/result"
 	"github.com/hearchco/hearchco/src/cache"
-	"github.com/hearchco/hearchco/src/category"
 	"github.com/hearchco/hearchco/src/config"
-	"github.com/hearchco/hearchco/src/engines"
 	"github.com/hearchco/hearchco/src/search"
+	"github.com/hearchco/hearchco/src/search/category"
+	"github.com/hearchco/hearchco/src/search/engines"
+	"github.com/hearchco/hearchco/src/search/result"
 )
 
 func Search(c *gin.Context, conf *config.Config, db cache.DB) error {

src/bucket/bucket.go → src/search/bucket/addresult.go

@@ -1,22 +1,14 @@
 package bucket
 
 import (
-	"fmt"
-	"sync"
-
 	"github.com/gocolly/colly/v2"
-	"github.com/hearchco/hearchco/src/bucket/result"
 	"github.com/hearchco/hearchco/src/config"
-	"github.com/hearchco/hearchco/src/engines"
+	"github.com/hearchco/hearchco/src/search/engines"
+	"github.com/hearchco/hearchco/src/search/result"
 	"github.com/rs/zerolog/log"
 )
 
-type Relay struct {
-	ResultMap map[string]*result.Result
-	Mutex     sync.RWMutex
-}
-
-func AddSEResult(seResult *engines.RetrievedResult, seName engines.Name, relay *Relay, options *engines.Options, pagesCol *colly.Collector) {
+func AddSEResult(seResult *result.RetrievedResult, seName engines.Name, relay *Relay, options *engines.Options, pagesCol *colly.Collector) {
 	log.Trace().
 		Str("engine", seName.String()).
 		Str("title", seResult.Title).
@@ -28,7 +20,7 @@ func AddSEResult(seResult *engines.RetrievedResult, seName engines.Name, relay *
 	relay.Mutex.RUnlock()
 
 	if !exists {
-		engineRanks := make([]engines.RetrievedRank, len(config.EnabledEngines))
+		engineRanks := make([]result.RetrievedRank, len(config.EnabledEngines))
 		engineRanks[0] = seResult.Rank
 		result := result.Result{
 			URL:           seResult.URL,
@@ -74,42 +66,3 @@ func AddSEResult(seResult *engines.RetrievedResult, seName engines.Name, relay *
 		}
 	}
 }
-
-func SetResultResponse(link string, response *colly.Response, relay *Relay, seName engines.Name) error {
-	log.Trace().
-		Str("engine", seName.String()).
-		Str("link", link).
-		Msg("Got response")
-
-	relay.Mutex.Lock()
-	mapRes, exists := relay.ResultMap[link]
-
-	if !exists {
-		relay.Mutex.Unlock()
-		relay.Mutex.RLock()
-		err := fmt.Errorf("bucket.SetResultResponse(): URL not in map when adding response, should not be possible. URL: %v.\nRelay: %v", link, relay)
-		relay.Mutex.RUnlock()
-		return err
-	} else {
-		mapRes.Response = response
-		relay.Mutex.Unlock()
-	}
-
-	return nil
-}
-
-func MakeSEResult(urll string, title string, description string, searchEngineName engines.Name, sePage int, seOnPageRank int) *engines.RetrievedResult {
-	ser := engines.RetrievedRank{
-		SearchEngine: searchEngineName,
-		Rank:         0,
-		Page:         uint(sePage),
-		OnPageRank:   uint(seOnPageRank),
-	}
-	res := engines.RetrievedResult{
-		URL:         urll,
-		Title:       title,
-		Description: description,
-		Rank:        ser,
-	}
-	return &res
-}

src/search/bucket/makeresult.go

@@ -0,0 +1,22 @@
+package bucket
+
+import (
+	"github.com/hearchco/hearchco/src/search/engines"
+	"github.com/hearchco/hearchco/src/search/result"
+)
+
+func MakeSEResult(urll string, title string, description string, searchEngineName engines.Name, sePage int, seOnPageRank int) *result.RetrievedResult {
+	ser := result.RetrievedRank{
+		SearchEngine: searchEngineName,
+		Rank:         0,
+		Page:         uint(sePage),
+		OnPageRank:   uint(seOnPageRank),
+	}
+	res := result.RetrievedResult{
+		URL:         urll,
+		Title:       title,
+		Description: description,
+		Rank:        ser,
+	}
+	return &res
+}

src/search/bucket/relay.go

@@ -0,0 +1,12 @@
+package bucket
+
+import (
+	"sync"
+
+	"github.com/hearchco/hearchco/src/search/result"
+)
+
+type Relay struct {
+	ResultMap map[string]*result.Result
+	Mutex     sync.RWMutex
+}

src/search/bucket/setresponse.go

@@ -0,0 +1,32 @@
+package bucket
+
+import (
+	"fmt"
+
+	"github.com/gocolly/colly/v2"
+	"github.com/hearchco/hearchco/src/search/engines"
+	"github.com/rs/zerolog/log"
+)
+
+func SetResultResponse(link string, response *colly.Response, relay *Relay, seName engines.Name) error {
+	log.Trace().
+		Str("engine", seName.String()).
+		Str("link", link).
+		Msg("Got response")
+
+	relay.Mutex.Lock()
+	mapRes, exists := relay.ResultMap[link]
+
+	if !exists {
+		relay.Mutex.Unlock()
+		relay.Mutex.RLock()
+		err := fmt.Errorf("bucket.SetResultResponse(): URL not in map when adding response, should not be possible. URL: %v.\nRelay: %v", link, relay)
+		relay.Mutex.RUnlock()
+		return err
+	} else {
+		mapRes.Response = response
+		relay.Mutex.Unlock()
+	}
+
+	return nil
+}

src/search/cache.go

@@ -2,10 +2,10 @@ package search
 
 import (
 	"github.com/hearchco/hearchco/src/anonymize"
-	"github.com/hearchco/hearchco/src/bucket/result"
 	"github.com/hearchco/hearchco/src/cache"
 	"github.com/hearchco/hearchco/src/config"
-	"github.com/hearchco/hearchco/src/engines"
+	"github.com/hearchco/hearchco/src/search/engines"
+	"github.com/hearchco/hearchco/src/search/result"
 	"github.com/rs/zerolog/log"
 )
 

src/engines/_engines_test/structs.go → src/search/engines/_engines_test/structs.go

@@ -3,9 +3,9 @@ package _engines_test
 import (
 	"time"
 
-	"github.com/hearchco/hearchco/src/category"
 	"github.com/hearchco/hearchco/src/config"
-	"github.com/hearchco/hearchco/src/engines"
+	"github.com/hearchco/hearchco/src/search/category"
+	"github.com/hearchco/hearchco/src/search/engines"
 )
 
 type TestCaseHasAnyResults struct {