Merge pull request #3862 from GeotrekCE/impr_parsers_user_agent

💫 [IMPR] Allow configuring headers in parsers requests (refs #3861)
GeotrekCE · Dec 19, 2023 · 0f4da76 · 0f4da76
2 parents e3a2f8c + fdbbaa2
commit 0f4da76
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 9 deletions.
diff --git a/docs/changelog.rst b/docs/changelog.rst
@@ -13,6 +13,7 @@ CHANGELOG
 
 - Add rules fixture on sensitive area (#3470)
 - Change condition on signage & blade to select many of them (#3847)
+- Allow to set headers in requests from Parsers (#3861)
 
 **Documentation**
 

diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py
@@ -73,6 +73,7 @@ class Parser:
     """
     provider: Allow to differentiate multiple Parser for the same model
     default_language: Allow to define which language this parser will populate by default
+    headers: Allow to configure headers on parser requests
     """
     label = None
     model = None
@@ -97,6 +98,7 @@ class Parser:
     natural_keys = {}
     field_options = {}
     default_language = None
+    headers = {"User-Agent": "Geotrek-Admin"}
 
     def __init__(self, progress_cb=None, user=None, encoding='utf8'):
         self.warnings = {}
@@ -558,7 +560,7 @@ def request_or_retry(self, url, verb='get', **kwargs):
         assert try_get > 0
         while try_get:
             action = getattr(requests, verb)
-            response = action(url, allow_redirects=True, **kwargs)
+            response = action(url, headers=self.headers, allow_redirects=True, **kwargs)
             if response.status_code in settings.PARSER_RETRY_HTTP_STATUS:
                 logger.info("Failed to fetch url {}. Retrying ...".format(url))
                 sleep(settings.PARSER_RETRY_SLEEP_TIME)

diff --git a/geotrek/sensitivity/tests/test_parsers.py b/geotrek/sensitivity/tests/test_parsers.py
@@ -136,7 +136,7 @@ class BiodivParserTests(TranslationResetMixin, TestCase):
     def test_create(self, mocked):
         self.page = 1
 
-        def side_effect(url, allow_redirects, params=None):
+        def side_effect(url, allow_redirects, headers, params=None):
             response = requests.Response()
             response.status_code = 200
 
@@ -178,7 +178,7 @@ def side_effect(url, allow_redirects, params=None):
     def test_create_with_practice(self, mocked):
         self.page = 1
 
-        def side_effect(url, allow_redirects, params=None):
+        def side_effect(url, allow_redirects, headers, params=None):
             response = requests.Response()
             response.status_code = 200
             if 'sportpractice' in url:
@@ -196,7 +196,7 @@ def side_effect(url, allow_redirects, params=None):
 
     @mock.patch('requests.get')
     def test_status_code_404(self, mocked):
-        def side_effect(url, allow_redirects, params=None):
+        def side_effect(url, allow_redirects, headers, params=None):
             response = requests.Response()
             response.status_code = 404
             response.url = url
@@ -207,7 +207,7 @@ def side_effect(url, allow_redirects, params=None):
 
     @mock.patch('requests.get')
     def test_status_code_404_practice(self, mocked):
-        def side_effect(url, allow_redirects, params=None):
+        def side_effect(url, allow_redirects, headers, params=None):
             response = requests.Response()
             if params and params.get('in_bbox', None):
                 response.status_code = 404
@@ -224,7 +224,7 @@ def side_effect(url, allow_redirects, params=None):
     def test_create_no_id(self, mocked):
         self.page = 1
 
-        def side_effect(url, allow_redirects, params=None):
+        def side_effect(url, allow_redirects, headers, params=None):
             response = requests.Response()
             response.status_code = 200
             if 'sportpractice' in url:
@@ -255,7 +255,7 @@ def side_effect(url, allow_redirects, params=None):
     def test_create_species_url(self, mocked):
         self.page = 1
 
-        def side_effect(url, allow_redirects, params=None):
+        def side_effect(url, allow_redirects, headers, params=None):
             response = requests.Response()
             response.status_code = 200
             if 'sportpractice' in url:
@@ -278,7 +278,7 @@ def side_effect(url, allow_redirects, params=None):
     def test_create_species_radius(self, mocked):
         self.page = 1
 
-        def side_effect(url, allow_redirects, params=None):
+        def side_effect(url, allow_redirects, headers, params=None):
             response = requests.Response()
             response.status_code = 200
             if 'sportpractice' in url:

diff --git a/geotrek/tourism/tests/test_parsers.py b/geotrek/tourism/tests/test_parsers.py
@@ -305,7 +305,7 @@ def mocked_json():
             with open(filename, 'r') as f:
                 return json.load(f)
 
-        def side_effect(url, allow_redirects, params):
+        def side_effect(url, allow_redirects, headers, params):
             response = requests.Response()
             response.status_code = 503
             response.url = url