From a0d96620170042fe0aa1cd52266bf367bcf28fa3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?=
<31509435+aleksasiriski@users.noreply.github.com>
Date: Fri, 26 Jan 2024 18:01:23 +0100
Subject: [PATCH 1/4] fix(proxy)!: added config for multiple rotating proxies
and tls verification with CI integration
---
.github/workflows/dockerci.yml | 3 +++
.github/workflows/testingci.yml | 3 +++
src/cli/climode.go | 16 +++++++++-------
src/config/structs.go | 8 +++++---
src/engines/structs.go | 20 ++++++++++----------
src/router/search.go | 16 +++++++++-------
src/sedefaults/sedefaults.go | 29 +++++++++++++++++++++++++++++
7 files changed, 68 insertions(+), 27 deletions(-)
diff --git a/.github/workflows/dockerci.yml b/.github/workflows/dockerci.yml
index 7c137491..8009f012 100644
--- a/.github/workflows/dockerci.yml
+++ b/.github/workflows/dockerci.yml
@@ -66,3 +66,6 @@ jobs:
# go:test
- name: Test units
run: make test
+ env:
+ HEARCHCO_SERVER_INSECURESKIPVERIFY: true
+ HEARCHCO_SERVER_PROXIES: ${{ secrets.HEARCHCO_SERVER_PROXIES}}
diff --git a/.github/workflows/testingci.yml b/.github/workflows/testingci.yml
index 97906466..1bdc0ef6 100644
--- a/.github/workflows/testingci.yml
+++ b/.github/workflows/testingci.yml
@@ -36,6 +36,9 @@ jobs:
# go:test
- name: Test units
run: make test
+ env:
+ HEARCHCO_SERVER_INSECURESKIPVERIFY: true
+ HEARCHCO_SERVER_PROXIES: ${{ secrets.HEARCHCO_SERVER_PROXIES}}
lint:
runs-on: ubuntu-latest
diff --git a/src/cli/climode.go b/src/cli/climode.go
index 0b9f24d4..ab15dcfa 100644
--- a/src/cli/climode.go
+++ b/src/cli/climode.go
@@ -35,13 +35,15 @@ func Run(flags Flags, db cache.DB, conf *config.Config) {
Msg("Started hearching")
options := engines.Options{
- MaxPages: flags.MaxPages,
- VisitPages: flags.Visit,
- Category: category.FromString[flags.Category],
- UserAgent: flags.UserAgent,
- Locale: flags.Locale,
- SafeSearch: flags.SafeSearch,
- Mobile: flags.Mobile,
+ MaxPages: flags.MaxPages,
+ VisitPages: flags.Visit,
+ Category: category.FromString[flags.Category],
+ UserAgent: flags.UserAgent,
+ Locale: flags.Locale,
+ SafeSearch: flags.SafeSearch,
+ Mobile: flags.Mobile,
+ Proxies: conf.Server.Proxies,
+ InsecureSkipVerify: conf.Server.InsecureSkipVerify,
}
start := time.Now()
diff --git a/src/config/structs.go b/src/config/structs.go
index 7330386a..bff88345 100644
--- a/src/config/structs.go
+++ b/src/config/structs.go
@@ -43,9 +43,11 @@ type Cache struct {
}
type Server struct {
- Port int `koanf:"port"`
- FrontendUrl string `koanf:"frontendurl"`
- Cache Cache `koanf:"cache"`
+ Port int `koanf:"port"`
+ FrontendUrl string `koanf:"frontendurl"`
+ Proxies []string `koanf:"proxies"`
+ InsecureSkipVerify bool `koanf:"insecureskipverify"`
+ Cache Cache `koanf:"cache"`
}
type ReaderEngine struct {
diff --git a/src/engines/structs.go b/src/engines/structs.go
index 156dfa2a..1afe548f 100644
--- a/src/engines/structs.go
+++ b/src/engines/structs.go
@@ -47,16 +47,16 @@ type DOMPaths struct {
}
type Options struct {
- MaxPages int
- VisitPages bool
- Category category.Name
- UserAgent string
- Locale string //format: en_US
- SafeSearch bool
- Mobile bool
-
- ProxyAddr string
- JustFirstPage bool
+ MaxPages int
+ VisitPages bool
+ Category category.Name
+ UserAgent string
+ Locale string //format: en_US
+ SafeSearch bool
+ Mobile bool
+ Proxies []string
+ InsecureSkipVerify bool
+ JustFirstPage bool
}
func ValidateLocale(locale string) error {
diff --git a/src/router/search.go b/src/router/search.go
index e72afc42..d8861391 100644
--- a/src/router/search.go
+++ b/src/router/search.go
@@ -85,13 +85,15 @@ func Search(c *gin.Context, conf *config.Config, db cache.DB) error {
}
options := engines.Options{
- MaxPages: maxPages,
- VisitPages: visitPages,
- Category: ccateg,
- UserAgent: useragent,
- Locale: locale,
- SafeSearch: safeSearchB,
- Mobile: isMobile,
+ MaxPages: maxPages,
+ VisitPages: visitPages,
+ Category: ccateg,
+ UserAgent: useragent,
+ Locale: locale,
+ SafeSearch: safeSearchB,
+ Mobile: isMobile,
+ Proxies: conf.Server.Proxies,
+ InsecureSkipVerify: conf.Server.InsecureSkipVerify,
}
var results []result.Result
diff --git a/src/sedefaults/sedefaults.go b/src/sedefaults/sedefaults.go
index bc29ebb6..13f64aa9 100644
--- a/src/sedefaults/sedefaults.go
+++ b/src/sedefaults/sedefaults.go
@@ -2,12 +2,15 @@ package sedefaults
import (
"context"
+ "crypto/tls"
"fmt"
"io"
+ "net/http"
"os"
"strconv"
"github.com/gocolly/colly/v2"
+ "github.com/gocolly/colly/v2/proxy"
"github.com/hearchco/hearchco/src/bucket"
"github.com/hearchco/hearchco/src/config"
"github.com/hearchco/hearchco/src/engines"
@@ -207,6 +210,32 @@ func InitializeCollectors(colPtr **colly.Collector, pagesColPtr **colly.Collecto
(*pagesColPtr).SetRequestTimeout(timings.PageTimeout)
}
}
+
+ if options.Proxies != nil {
+ log.Debug().
+ Strs("proxies", options.Proxies).
+ Msg("Using proxies")
+
+ // Rotate proxies
+ rp, err := proxy.RoundRobinProxySwitcher(options.Proxies...)
+ if err != nil {
+ log.Fatal().
+ Err(err).
+ Strs("proxies", options.Proxies).
+ Msg("sedefaults.InitializeCollectors(): failed creating proxy switcher")
+ }
+
+ (*colPtr).SetProxyFunc(rp)
+ (*pagesColPtr).SetProxyFunc(rp)
+ }
+
+ if options.InsecureSkipVerify {
+ tp := &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+ }
+ (*colPtr).WithTransport(tp)
+ (*pagesColPtr).WithTransport(tp)
+ }
}
func DoGetRequest(urll string, colCtx *colly.Context, collector *colly.Collector, packageName engines.Name, retError *error) {
From 99eda327660af78d87694024f5016c94d060fb4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?=
<31509435+aleksasiriski@users.noreply.github.com>
Date: Wed, 7 Feb 2024 19:02:15 +0100
Subject: [PATCH 2/4] fix: moved proxy config to Settings, reverting a lot of
changes
---
src/cli/climode.go | 16 +++++++---------
src/config/structs.go | 14 +++++++-------
src/engines/bing/bing.go | 2 +-
src/engines/brave/brave.go | 2 +-
src/engines/duckduckgo/duckduckgo.go | 2 +-
src/engines/etools/etools.go | 2 +-
src/engines/google/google.go | 2 +-
src/engines/googlescholar/googlescholar.go | 2 +-
src/engines/mojeek/mojeek.go | 2 +-
src/engines/presearch/presearch.go | 2 +-
src/engines/qwant/qwant.go | 2 +-
src/engines/startpage/startpage.go | 2 +-
src/engines/structs.go | 18 ++++++++----------
src/engines/swisscows/swisscows.go | 2 +-
src/engines/yahoo/yahoo.go | 2 +-
src/engines/yep/yep.go | 2 +-
src/router/search.go | 16 +++++++---------
src/sedefaults/sedefaults.go | 12 ++++++------
18 files changed, 48 insertions(+), 54 deletions(-)
diff --git a/src/cli/climode.go b/src/cli/climode.go
index ab15dcfa..0b9f24d4 100644
--- a/src/cli/climode.go
+++ b/src/cli/climode.go
@@ -35,15 +35,13 @@ func Run(flags Flags, db cache.DB, conf *config.Config) {
Msg("Started hearching")
options := engines.Options{
- MaxPages: flags.MaxPages,
- VisitPages: flags.Visit,
- Category: category.FromString[flags.Category],
- UserAgent: flags.UserAgent,
- Locale: flags.Locale,
- SafeSearch: flags.SafeSearch,
- Mobile: flags.Mobile,
- Proxies: conf.Server.Proxies,
- InsecureSkipVerify: conf.Server.InsecureSkipVerify,
+ MaxPages: flags.MaxPages,
+ VisitPages: flags.Visit,
+ Category: category.FromString[flags.Category],
+ UserAgent: flags.UserAgent,
+ Locale: flags.Locale,
+ SafeSearch: flags.SafeSearch,
+ Mobile: flags.Mobile,
}
start := time.Now()
diff --git a/src/config/structs.go b/src/config/structs.go
index bff88345..6e0d31c7 100644
--- a/src/config/structs.go
+++ b/src/config/structs.go
@@ -26,8 +26,10 @@ type Ranking struct {
}
type Settings struct {
- RequestedResultsPerPage int `koanf:"requestedresults"`
- Shortcut string `koanf:"shortcut"`
+ RequestedResultsPerPage int `koanf:"requestedresults"`
+ Shortcut string `koanf:"shortcut"`
+ Proxies []string `koanf:"proxies"`
+ InsecureSkipVerify bool `koanf:"insecureskipverify"`
}
type Redis struct {
@@ -43,11 +45,9 @@ type Cache struct {
}
type Server struct {
- Port int `koanf:"port"`
- FrontendUrl string `koanf:"frontendurl"`
- Proxies []string `koanf:"proxies"`
- InsecureSkipVerify bool `koanf:"insecureskipverify"`
- Cache Cache `koanf:"cache"`
+ Port int `koanf:"port"`
+ FrontendUrl string `koanf:"frontendurl"`
+ Cache Cache `koanf:"cache"`
}
type ReaderEngine struct {
diff --git a/src/engines/bing/bing.go b/src/engines/bing/bing.go
index 2cc3a850..06aa6a96 100644
--- a/src/engines/bing/bing.go
+++ b/src/engines/bing/bing.go
@@ -25,7 +25,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &options, &timings)
+ sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
sedefaults.PagesColError(Info.Name, pagesCol)
diff --git a/src/engines/brave/brave.go b/src/engines/brave/brave.go
index 312a269d..49d56160 100644
--- a/src/engines/brave/brave.go
+++ b/src/engines/brave/brave.go
@@ -22,7 +22,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &options, &timings)
+ sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
sedefaults.PagesColError(Info.Name, pagesCol)
diff --git a/src/engines/duckduckgo/duckduckgo.go b/src/engines/duckduckgo/duckduckgo.go
index be59d48c..a536205d 100644
--- a/src/engines/duckduckgo/duckduckgo.go
+++ b/src/engines/duckduckgo/duckduckgo.go
@@ -24,7 +24,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &options, &timings)
+ sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
sedefaults.PagesColError(Info.Name, pagesCol)
diff --git a/src/engines/etools/etools.go b/src/engines/etools/etools.go
index 0a1df7fa..01e731a8 100644
--- a/src/engines/etools/etools.go
+++ b/src/engines/etools/etools.go
@@ -23,7 +23,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &options, &timings)
+ sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
sedefaults.PagesColError(Info.Name, pagesCol)
diff --git a/src/engines/google/google.go b/src/engines/google/google.go
index 38ab7820..d5fdba03 100644
--- a/src/engines/google/google.go
+++ b/src/engines/google/google.go
@@ -22,7 +22,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &options, &timings)
+ sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
sedefaults.PagesColError(Info.Name, pagesCol)
diff --git a/src/engines/googlescholar/googlescholar.go b/src/engines/googlescholar/googlescholar.go
index 2133cb91..5d141402 100644
--- a/src/engines/googlescholar/googlescholar.go
+++ b/src/engines/googlescholar/googlescholar.go
@@ -24,7 +24,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &options, &timings)
+ sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
sedefaults.PagesColError(Info.Name, pagesCol)
diff --git a/src/engines/mojeek/mojeek.go b/src/engines/mojeek/mojeek.go
index 3442257e..32eba946 100644
--- a/src/engines/mojeek/mojeek.go
+++ b/src/engines/mojeek/mojeek.go
@@ -22,7 +22,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &options, &timings)
+ sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
sedefaults.PagesColError(Info.Name, pagesCol)
diff --git a/src/engines/presearch/presearch.go b/src/engines/presearch/presearch.go
index 6fbc4abb..7b82218c 100644
--- a/src/engines/presearch/presearch.go
+++ b/src/engines/presearch/presearch.go
@@ -24,7 +24,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &options, &timings)
+ sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
sedefaults.PagesColError(Info.Name, pagesCol)
diff --git a/src/engines/qwant/qwant.go b/src/engines/qwant/qwant.go
index 3bf2d763..1b9fc446 100644
--- a/src/engines/qwant/qwant.go
+++ b/src/engines/qwant/qwant.go
@@ -24,7 +24,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &options, &timings)
+ sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
sedefaults.PagesColError(Info.Name, pagesCol)
diff --git a/src/engines/startpage/startpage.go b/src/engines/startpage/startpage.go
index 88a4c034..ea2c66e6 100644
--- a/src/engines/startpage/startpage.go
+++ b/src/engines/startpage/startpage.go
@@ -23,7 +23,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &options, &timings)
+ sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
sedefaults.PagesColError(Info.Name, pagesCol)
diff --git a/src/engines/structs.go b/src/engines/structs.go
index 1afe548f..8c027dbc 100644
--- a/src/engines/structs.go
+++ b/src/engines/structs.go
@@ -47,16 +47,14 @@ type DOMPaths struct {
}
type Options struct {
- MaxPages int
- VisitPages bool
- Category category.Name
- UserAgent string
- Locale string //format: en_US
- SafeSearch bool
- Mobile bool
- Proxies []string
- InsecureSkipVerify bool
- JustFirstPage bool
+ MaxPages int
+ VisitPages bool
+ Category category.Name
+ UserAgent string
+ Locale string //format: en_US
+ SafeSearch bool
+ Mobile bool
+ JustFirstPage bool
}
func ValidateLocale(locale string) error {
diff --git a/src/engines/swisscows/swisscows.go b/src/engines/swisscows/swisscows.go
index b319be5e..e594405f 100644
--- a/src/engines/swisscows/swisscows.go
+++ b/src/engines/swisscows/swisscows.go
@@ -24,7 +24,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &options, &timings)
+ sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
sedefaults.PagesColError(Info.Name, pagesCol)
diff --git a/src/engines/yahoo/yahoo.go b/src/engines/yahoo/yahoo.go
index 8b177b4b..a0808d2d 100644
--- a/src/engines/yahoo/yahoo.go
+++ b/src/engines/yahoo/yahoo.go
@@ -24,7 +24,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &options, &timings)
+ sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
sedefaults.PagesColError(Info.Name, pagesCol)
diff --git a/src/engines/yep/yep.go b/src/engines/yep/yep.go
index af48855a..6b69bbbd 100644
--- a/src/engines/yep/yep.go
+++ b/src/engines/yep/yep.go
@@ -22,7 +22,7 @@ func Search(ctx context.Context, query string, relay *bucket.Relay, options engi
var pagesCol *colly.Collector
var retError error
- sedefaults.InitializeCollectors(&col, &pagesCol, &options, &timings)
+ sedefaults.InitializeCollectors(&col, &pagesCol, &settings, &options, &timings)
sedefaults.PagesColRequest(Info.Name, pagesCol, ctx)
sedefaults.PagesColError(Info.Name, pagesCol)
diff --git a/src/router/search.go b/src/router/search.go
index d8861391..e72afc42 100644
--- a/src/router/search.go
+++ b/src/router/search.go
@@ -85,15 +85,13 @@ func Search(c *gin.Context, conf *config.Config, db cache.DB) error {
}
options := engines.Options{
- MaxPages: maxPages,
- VisitPages: visitPages,
- Category: ccateg,
- UserAgent: useragent,
- Locale: locale,
- SafeSearch: safeSearchB,
- Mobile: isMobile,
- Proxies: conf.Server.Proxies,
- InsecureSkipVerify: conf.Server.InsecureSkipVerify,
+ MaxPages: maxPages,
+ VisitPages: visitPages,
+ Category: ccateg,
+ UserAgent: useragent,
+ Locale: locale,
+ SafeSearch: safeSearchB,
+ Mobile: isMobile,
}
var results []result.Result
diff --git a/src/sedefaults/sedefaults.go b/src/sedefaults/sedefaults.go
index 13f64aa9..49b4a08f 100644
--- a/src/sedefaults/sedefaults.go
+++ b/src/sedefaults/sedefaults.go
@@ -183,7 +183,7 @@ func Prepare(seName engines.Name, options *engines.Options, settings *config.Set
return nil
}
-func InitializeCollectors(colPtr **colly.Collector, pagesColPtr **colly.Collector, options *engines.Options, timings *config.Timings) {
+func InitializeCollectors(colPtr **colly.Collector, pagesColPtr **colly.Collector, settings *config.Settings, options *engines.Options, timings *config.Timings) {
*colPtr = colly.NewCollector(colly.MaxDepth(1), colly.UserAgent(options.UserAgent), colly.Async())
*pagesColPtr = colly.NewCollector(colly.MaxDepth(1), colly.UserAgent(options.UserAgent), colly.Async())
@@ -211,17 +211,17 @@ func InitializeCollectors(colPtr **colly.Collector, pagesColPtr **colly.Collecto
}
}
- if options.Proxies != nil {
+ if settings.Proxies != nil {
log.Debug().
- Strs("proxies", options.Proxies).
+ Strs("proxies", settings.Proxies).
Msg("Using proxies")
// Rotate proxies
- rp, err := proxy.RoundRobinProxySwitcher(options.Proxies...)
+ rp, err := proxy.RoundRobinProxySwitcher(settings.Proxies...)
if err != nil {
log.Fatal().
Err(err).
- Strs("proxies", options.Proxies).
+ Strs("proxies", settings.Proxies).
Msg("sedefaults.InitializeCollectors(): failed creating proxy switcher")
}
@@ -229,7 +229,7 @@ func InitializeCollectors(colPtr **colly.Collector, pagesColPtr **colly.Collecto
(*pagesColPtr).SetProxyFunc(rp)
}
- if options.InsecureSkipVerify {
+ if settings.InsecureSkipVerify {
tp := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
From 52f10682b7559261f2eea22d8e6c238808a5488c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?=
<31509435+aleksasiriski@users.noreply.github.com>
Date: Wed, 7 Feb 2024 19:05:43 +0100
Subject: [PATCH 3/4] revert: usage of rot proxy in CI
---
.github/workflows/dockerci.yml | 3 ---
.github/workflows/testingci.yml | 3 ---
2 files changed, 6 deletions(-)
diff --git a/.github/workflows/dockerci.yml b/.github/workflows/dockerci.yml
index 8009f012..7c137491 100644
--- a/.github/workflows/dockerci.yml
+++ b/.github/workflows/dockerci.yml
@@ -66,6 +66,3 @@ jobs:
# go:test
- name: Test units
run: make test
- env:
- HEARCHCO_SERVER_INSECURESKIPVERIFY: true
- HEARCHCO_SERVER_PROXIES: ${{ secrets.HEARCHCO_SERVER_PROXIES}}
diff --git a/.github/workflows/testingci.yml b/.github/workflows/testingci.yml
index 453376c6..3066a9f5 100644
--- a/.github/workflows/testingci.yml
+++ b/.github/workflows/testingci.yml
@@ -43,9 +43,6 @@ jobs:
# test
- name: Test units (w/o engines)
run: make test
- env:
- HEARCHCO_SERVER_INSECURESKIPVERIFY: true
- HEARCHCO_SERVER_PROXIES: ${{ secrets.HEARCHCO_SERVER_PROXIES}}
test-engines:
runs-on: ubuntu-latest
From 4f39d4bc4347bb304cc49e0d29f9e247399429db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleksa=20Siri=C5=A1ki?=
<31509435+aleksasiriski@users.noreply.github.com>
Date: Fri, 9 Feb 2024 00:13:42 +0100
Subject: [PATCH 4/4] revert(rotproxy): insecure skip verify
---
src/config/structs.go | 1 -
src/sedefaults/sedefaults.go | 10 ----------
2 files changed, 11 deletions(-)
diff --git a/src/config/structs.go b/src/config/structs.go
index 6ee590f5..de30ea71 100644
--- a/src/config/structs.go
+++ b/src/config/structs.go
@@ -29,7 +29,6 @@ type Settings struct {
RequestedResultsPerPage int `koanf:"requestedresults"`
Shortcut string `koanf:"shortcut"`
Proxies []string `koanf:"proxies"`
- InsecureSkipVerify bool `koanf:"insecureskipverify"`
}
// ReaderTTL is format in which the config is read from the config file
diff --git a/src/sedefaults/sedefaults.go b/src/sedefaults/sedefaults.go
index 57d8b0f0..b273a198 100644
--- a/src/sedefaults/sedefaults.go
+++ b/src/sedefaults/sedefaults.go
@@ -2,10 +2,8 @@ package sedefaults
import (
"context"
- "crypto/tls"
"fmt"
"io"
- "net/http"
"os"
"strconv"
@@ -227,14 +225,6 @@ func InitializeCollectors(colPtr **colly.Collector, pagesColPtr **colly.Collecto
(*colPtr).SetProxyFunc(rp)
(*pagesColPtr).SetProxyFunc(rp)
}
-
- if settings.InsecureSkipVerify {
- tp := &http.Transport{
- TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
- }
- (*colPtr).WithTransport(tp)
- (*pagesColPtr).WithTransport(tp)
- }
}
func DoGetRequest(urll string, anonurll string, colCtx *colly.Context, collector *colly.Collector, packageName engines.Name, retError *error) {