feat(*): add custom url

Author: Benoist Bouteiller

src/modules/duckduckgo.js

@@ -46,11 +46,8 @@ class DuckduckgoScraper extends Scraper {
 
     async load_start_page() {
         debug('load_start_page');
-        let startUrl = 'https://duckduckgo.com/';
-
-        this.last_response = await this.page.goto(startUrl);
+        this.last_response = await this.page.goto(this.startUrl);
         await this.page.waitForSelector('input[name="q"]', { timeout: this.STANDARD_TIMEOUT });
-        
         return true;
     }
 

src/modules/google.js

@@ -213,26 +213,8 @@ class GoogleScraper extends Scraper {
     }
 
     async load_start_page() {
-        let startUrl = 'https://www.google.com';
-
-        if (this.config.google_settings) {
-            startUrl = `https://www.${this.config.google_settings.google_domain}/search?q=`;
-            if (this.config.google_settings.google_domain) {
-                startUrl = `https://www.${this.config.google_settings.google_domain}/search?`;
-            } else {
-                startUrl = `https://www.google.com/search?`;
-            }
-
-            for (var key in this.config.google_settings) {
-                if (key !== 'google_domain') {
-                    startUrl += `${key}=${this.config.google_settings[key]}&`
-                }
-            }
-        }
-
-        this.logger.info('Using startUrl: ' + startUrl);
-
-        this.last_response = await this.page.goto(startUrl);
+        this.logger.info('Using startUrl: ' + this.startUrl);
+        this.last_response = await this.page.goto(this.startUrl);
 
         await this.page.waitForSelector('input[name="q"]', { timeout: this.STANDARD_TIMEOUT });
 

src/modules/infospace.js

@@ -41,11 +41,8 @@ class InfospaceScraper extends Scraper {
     }
 
     async load_start_page() {
-
-        let startUrl = this.build_start_url('http://search.infospace.com/search/web?') || 'http://infospace.com/index.html';
-
         try {
-            this.last_response = await this.page.goto(startUrl);
+            this.last_response = await this.page.goto(this.this.startUrl);
             await this.page.waitForSelector('input[name="q"]', { timeout: 5000 });
         } catch (e) {
             return false;

src/modules/yandex.js

@@ -71,11 +71,10 @@ class YandexScraper extends Scraper {
     }
 
     async load_start_page() {
-        let startUrl = 'https://yandex.com';
+        
+        this.logger.info('Using startUrl: ' + this.startUrl);
 
-        this.logger.info('Using startUrl: ' + startUrl);
-
-        this.last_response = await this.page.goto(startUrl);
+        this.last_response = await this.page.goto(this.startUrl);
 
         await this.page.waitForSelector('input[name="text"]', { timeout: this.STANDARD_TIMEOUT });