diff --git a/docs/changelog.rst b/docs/changelog.rst index 2e36756133..988ee0aa48 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,6 +13,7 @@ CHANGELOG - Add rules fixture on sensitive area (#3470) - Change condition on signage & blade to select many of them (#3847) +- Allow to set headers in requests from Parsers (#3861) **Documentation** diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 506c45e7bf..d7e6545c00 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -73,6 +73,7 @@ class Parser: """ provider: Allow to differentiate multiple Parser for the same model default_language: Allow to define which language this parser will populate by default + headers: Allow to configure headers on parser requests """ label = None model = None @@ -97,6 +98,7 @@ class Parser: natural_keys = {} field_options = {} default_language = None + headers = {"User-Agent": "Geotrek-Admin"} def __init__(self, progress_cb=None, user=None, encoding='utf8'): self.warnings = {} @@ -558,7 +560,7 @@ def request_or_retry(self, url, verb='get', **kwargs): assert try_get > 0 while try_get: action = getattr(requests, verb) - response = action(url, allow_redirects=True, **kwargs) + response = action(url, headers=self.headers, allow_redirects=True, **kwargs) if response.status_code in settings.PARSER_RETRY_HTTP_STATUS: logger.info("Failed to fetch url {}. Retrying ...".format(url)) sleep(settings.PARSER_RETRY_SLEEP_TIME) diff --git a/geotrek/sensitivity/tests/test_parsers.py b/geotrek/sensitivity/tests/test_parsers.py index 8c01d15bc8..769e2c4ccb 100644 --- a/geotrek/sensitivity/tests/test_parsers.py +++ b/geotrek/sensitivity/tests/test_parsers.py @@ -136,7 +136,7 @@ class BiodivParserTests(TranslationResetMixin, TestCase): def test_create(self, mocked): self.page = 1 - def side_effect(url, allow_redirects, params=None): + def side_effect(url, allow_redirects, headers, params=None): response = requests.Response() response.status_code = 200 @@ -178,7 +178,7 @@ def side_effect(url, allow_redirects, params=None): def test_create_with_practice(self, mocked): self.page = 1 - def side_effect(url, allow_redirects, params=None): + def side_effect(url, allow_redirects, headers, params=None): response = requests.Response() response.status_code = 200 if 'sportpractice' in url: @@ -196,7 +196,7 @@ def side_effect(url, allow_redirects, params=None): @mock.patch('requests.get') def test_status_code_404(self, mocked): - def side_effect(url, allow_redirects, params=None): + def side_effect(url, allow_redirects, headers, params=None): response = requests.Response() response.status_code = 404 response.url = url @@ -207,7 +207,7 @@ def side_effect(url, allow_redirects, params=None): @mock.patch('requests.get') def test_status_code_404_practice(self, mocked): - def side_effect(url, allow_redirects, params=None): + def side_effect(url, allow_redirects, headers, params=None): response = requests.Response() if params and params.get('in_bbox', None): response.status_code = 404 @@ -224,7 +224,7 @@ def side_effect(url, allow_redirects, params=None): def test_create_no_id(self, mocked): self.page = 1 - def side_effect(url, allow_redirects, params=None): + def side_effect(url, allow_redirects, headers, params=None): response = requests.Response() response.status_code = 200 if 'sportpractice' in url: @@ -255,7 +255,7 @@ def side_effect(url, allow_redirects, params=None): def test_create_species_url(self, mocked): self.page = 1 - def side_effect(url, allow_redirects, params=None): + def side_effect(url, allow_redirects, headers, params=None): response = requests.Response() response.status_code = 200 if 'sportpractice' in url: @@ -278,7 +278,7 @@ def side_effect(url, allow_redirects, params=None): def test_create_species_radius(self, mocked): self.page = 1 - def side_effect(url, allow_redirects, params=None): + def side_effect(url, allow_redirects, headers, params=None): response = requests.Response() response.status_code = 200 if 'sportpractice' in url: diff --git a/geotrek/tourism/tests/test_parsers.py b/geotrek/tourism/tests/test_parsers.py index 172d9db209..5966c44a49 100644 --- a/geotrek/tourism/tests/test_parsers.py +++ b/geotrek/tourism/tests/test_parsers.py @@ -305,7 +305,7 @@ def mocked_json(): with open(filename, 'r') as f: return json.load(f) - def side_effect(url, allow_redirects, params): + def side_effect(url, allow_redirects, headers, params): response = requests.Response() response.status_code = 503 response.url = url