From fdbbaa26f481279c98508c21e326abf7de190345 Mon Sep 17 00:00:00 2001 From: Chatewgne Date: Tue, 19 Dec 2023 16:50:26 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=92=AB=20[IMPR]=20Allow=20to=20configure?= =?UTF-8?q?=20headers=20in=20parsers=20requests=20(refs=20#3861)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/changelog.rst | 2 +- geotrek/common/parsers.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4e866cd2b7..988ee0aa48 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -13,7 +13,7 @@ CHANGELOG - Add rules fixture on sensitive area (#3470) - Change condition on signage & blade to select many of them (#3847) -- Set User-Agent header in requests from Parsers (#3861) +- Allow to set headers in requests from Parsers (#3861) **Documentation** diff --git a/geotrek/common/parsers.py b/geotrek/common/parsers.py index 02a298630d..d7e6545c00 100644 --- a/geotrek/common/parsers.py +++ b/geotrek/common/parsers.py @@ -73,6 +73,7 @@ class Parser: """ provider: Allow to differentiate multiple Parser for the same model default_language: Allow to define which language this parser will populate by default + headers: Allow to configure headers on parser requests """ label = None model = None @@ -97,7 +98,7 @@ class Parser: natural_keys = {} field_options = {} default_language = None - user_agent = "Geotrek" + headers = {"User-Agent": "Geotrek-Admin"} def __init__(self, progress_cb=None, user=None, encoding='utf8'): self.warnings = {} @@ -559,8 +560,7 @@ def request_or_retry(self, url, verb='get', **kwargs): assert try_get > 0 while try_get: action = getattr(requests, verb) - headers = {"User-Agent": self.user_agent} - response = action(url, headers=headers, allow_redirects=True, **kwargs) + response = action(url, headers=self.headers, allow_redirects=True, **kwargs) if response.status_code in settings.PARSER_RETRY_HTTP_STATUS: logger.info("Failed to fetch url {}. Retrying ...".format(url)) sleep(settings.PARSER_RETRY_SLEEP_TIME)