diff --git a/src/router/proxy.go b/src/router/proxy.go index e0eb2eec..428b46f9 100644 --- a/src/router/proxy.go +++ b/src/router/proxy.go @@ -69,7 +69,7 @@ func Proxy(w http.ResponseWriter, r *http.Request, salt string, timeouts config. ProtoMinor: 1, Header: map[string][]string{ "Accept": {"image/avif", "image/webp", "image/apng", "image/svg+xml", "image/*", "*/*;q=0.8"}, - "Accept-Encoding": {"gzip", "deflate", "br", "zstd"}, // Chromium-based browsers have "zstd" but that isn't supported by Firefox nor Safari + "Accept-Encoding": {"gzip", "deflate", "br"}, // Google Chrome also has "zstd" but that isn't supported by Firefox and Safari "Accept-Language": {"en-US,en;q=0.9"}, // "Connection": {"keep-alive"}, // commented since it's not present by default in Google Chrome // "DNT": {"1"}, // do not track, commented since it's not present by default in Google Chrome diff --git a/src/search/engines/_sedefaults/init.go b/src/search/engines/_sedefaults/init.go index ca1d12b0..cb6bb0d4 100644 --- a/src/search/engines/_sedefaults/init.go +++ b/src/search/engines/_sedefaults/init.go @@ -24,15 +24,7 @@ func InitializeCollectors(ctx context.Context, engineName engines.Name, options colly.UserAgent(userAgent), colly.IgnoreRobotsTxt(), colly.Headers(map[string]string{ - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", - "Accept-Encoding": "gzip, deflate, br, zstd", // Chromium-based browsers have "zstd" but that isn't supported by Firefox nor Safari - "Accept-Language": "en-US,en;q=0.9", - "Sec-Ch-Ua": secChUa, // "Google Chrome";v="119", "Chromium";v="119", "Not=A?Brand";v="24" - "Sec-Ch-Ua-Mobile": "?0", - "Sec-Ch-Ua-Platform": "\"Windows\"", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "none", + "Sec-Ch-Ua": secChUa, }), ) pagesCol := colly.NewCollector( @@ -41,15 +33,7 @@ func InitializeCollectors(ctx context.Context, engineName engines.Name, options colly.UserAgent(userAgent), colly.IgnoreRobotsTxt(), colly.Headers(map[string]string{ - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", - "Accept-Encoding": "gzip, deflate, br, zstd", // Chromium-based browsers have "zstd" but that isn't supported by Firefox nor Safari - "Accept-Language": "en-US,en;q=0.9", - "Sec-Ch-Ua": secChUa, // "Google Chrome";v="119", "Chromium";v="119", "Not=A?Brand";v="24" - "Sec-Ch-Ua-Mobile": "?0", - "Sec-Ch-Ua-Platform": "\"Windows\"", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "none", + "Sec-Ch-Ua": secChUa, }), )