Merge pull request #648 from maykinmedia/feature/594-add-retry-mechanism

[#496] Add retry mechanism
maykinmedia · Jan 27, 2025 · 0670532 · 0670532
2 parents c5dca5f + 5157a99
commit 0670532
Show file tree

Hide file tree

Showing 6 changed files with 228 additions and 4 deletions.
diff --git a/backend/src/openarchiefbeheer/conf/base.py b/backend/src/openarchiefbeheer/conf/base.py
@@ -407,6 +407,13 @@
 
 E2E_SERVE_FRONTEND = False
 
+RETRY_TOTAL = config("RETRY_TOTAL", 5)
+RETRY_BACKOFF_FACTOR = config("RETRY_BACKOFF_FACTOR", 5)
+RETRY_STATUS_FORCELIST = config(
+    "RETRY_STATUS_FORCELIST", "502,503,504", split=True, csv_cast=int
+)
+
+
 ##############################
 #                            #
 # 3RD PARTY LIBRARY SETTINGS #

diff --git a/backend/src/openarchiefbeheer/conf/utils.py b/backend/src/openarchiefbeheer/conf/utils.py
@@ -18,7 +18,10 @@ def config(option: str, default=undefined, *args, **kwargs):
     """
     if "split" in kwargs:
         kwargs.pop("split")
-        kwargs["cast"] = Csv()
+        csv_kwargs = {}
+        if csv_cast := kwargs.pop("csv_cast", None):
+            csv_kwargs.update({"cast": csv_cast})
+        kwargs["cast"] = Csv(**csv_kwargs)
         if default == []:
             default = ""
 

diff --git a/backend/src/openarchiefbeheer/zaken/tasks.py b/backend/src/openarchiefbeheer/zaken/tasks.py
@@ -5,6 +5,8 @@
 from django.db import transaction
 from django.db.models import Max
 
+from ape_pie import APIClient
+from requests.adapters import HTTPAdapter, Retry
 from zgw_consumers.client import build_client
 from zgw_consumers.constants import APITypes
 from zgw_consumers.models import Service
@@ -22,9 +24,21 @@
 logger = logging.getLogger(__name__)
 
 
+def configure_retry(client: APIClient) -> APIClient:
+    retries = Retry(
+        total=settings.RETRY_TOTAL,
+        backoff_factor=settings.RETRY_BACKOFF_FACTOR,
+        status_forcelist=settings.RETRY_STATUS_FORCELIST,
+    )
+    client.adapters["http://"] = HTTPAdapter(max_retries=retries)
+    client.adapters["https://"] = HTTPAdapter(max_retries=retries)
+    return client
+
+
 def retrieve_and_cache_zaken(is_full_resync=False):
     zrc_service = Service.objects.get(api_type=APITypes.zrc)
     zrc_client = build_client(zrc_service)
+    zrc_client = configure_retry(zrc_client)
 
     config = APIConfig.get_solo()
     service = config.selectielijst_api_service

diff --git a/...heer/zaken/tests/vcr/cassettes/RecachingZakenTests.test_resync_zaken_retry_mechanism.yaml b/...heer/zaken/tests/vcr/cassettes/RecachingZakenTests.test_resync_zaken_retry_mechanism.yaml
@@ -0,0 +1,176 @@
+interactions:
+- request:
+    body: null
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Crs:
+      - EPSG:4326
+      Accept-Encoding:
+      - gzip, deflate, br
+      Authorization:
+      - Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJ0ZXN0LXZjciIsImlhdCI6MTczNzk3NjYwMSwiY2xpZW50X2lkIjoidGVzdC12Y3IiLCJ1c2VyX2lkIjoiIiwidXNlcl9yZXByZXNlbnRhdGlvbiI6IiJ9.1LYlUWSglBqFMRp4uh3wFBwBmR9midapufWeCga_k10
+      Connection:
+      - keep-alive
+      User-Agent:
+      - python-requests/2.32.3
+    method: GET
+    uri: http://localhost:8003/zaken/api/v1/zaken?expand=resultaat%2Cresultaat.resultaattype%2Czaaktype%2Crollen&archiefnominatie=vernietigen&einddatum__isnull=False&einddatum__lt=2025-01-27
+  response:
+    body:
+      string: ''
+    headers:
+      API-version:
+      - 1.5.1
+      Allow:
+      - GET, POST, HEAD, OPTIONS
+      Content-Crs:
+      - EPSG:4326
+      Content-Length:
+      - '0'
+      Content-Security-Policy:
+      - 'img-src ''self'' data: cdn.redoc.ly cdnjs.cloudflare.com *.tile.openstreetmap.org;
+        style-src ''self'' ''unsafe-inline'' fonts.googleapis.com cdnjs.cloudflare.com;
+        connect-src ''self'' raw.githubusercontent.com; base-uri ''self''; frame-src
+        ''self''; form-action ''self''; worker-src ''self'' blob:; font-src ''self''
+        fonts.gstatic.com; default-src ''self''; script-src ''self'' ''unsafe-inline''
+        cdnjs.cloudflare.com; frame-ancestors ''none''; object-src ''none'''
+      Cross-Origin-Opener-Policy:
+      - same-origin
+      Date:
+      - Mon, 27 Jan 2025 11:16:41 GMT
+      Referrer-Policy:
+      - same-origin
+      Server:
+      - WSGIServer/0.2 CPython/3.10.14
+      Server-Timing:
+      - TimerPanel_utime;dur=173.88300000001777;desc="User CPU time", TimerPanel_stime;dur=9.13500000000056;desc="System
+        CPU time", TimerPanel_total;dur=183.01800000001833;desc="Total CPU time",
+        TimerPanel_total_time;dur=190.99712371826172;desc="Elapsed time", SQLPanel_sql_time;dur=3.4432411193847656;desc="SQL
+        3 queries", CachePanel_total_time;dur=0;desc="Cache 0 Calls"
+      Vary:
+      - origin, Cookie
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - DENY
+    status:
+      code: 504
+      message: Gateway Timeout
+- request:
+    body: null
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Crs:
+      - EPSG:4326
+      Accept-Encoding:
+      - gzip, deflate, br
+      Authorization:
+      - Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJ0ZXN0LXZjciIsImlhdCI6MTczNzk3NjYwMSwiY2xpZW50X2lkIjoidGVzdC12Y3IiLCJ1c2VyX2lkIjoiIiwidXNlcl9yZXByZXNlbnRhdGlvbiI6IiJ9.1LYlUWSglBqFMRp4uh3wFBwBmR9midapufWeCga_k10
+      Connection:
+      - keep-alive
+      User-Agent:
+      - python-requests/2.32.3
+    method: GET
+    uri: http://localhost:8003/zaken/api/v1/zaken?expand=resultaat%2Cresultaat.resultaattype%2Czaaktype%2Crollen&archiefnominatie=vernietigen&einddatum__isnull=False&einddatum__lt=2025-01-27
+  response:
+    body:
+      string: ''
+    headers:
+      API-version:
+      - 1.5.1
+      Allow:
+      - GET, POST, HEAD, OPTIONS
+      Content-Crs:
+      - EPSG:4326
+      Content-Length:
+      - '0'
+      Content-Security-Policy:
+      - 'img-src ''self'' data: cdn.redoc.ly cdnjs.cloudflare.com *.tile.openstreetmap.org;
+        style-src ''self'' ''unsafe-inline'' fonts.googleapis.com cdnjs.cloudflare.com;
+        connect-src ''self'' raw.githubusercontent.com; base-uri ''self''; frame-src
+        ''self''; form-action ''self''; worker-src ''self'' blob:; font-src ''self''
+        fonts.gstatic.com; default-src ''self''; script-src ''self'' ''unsafe-inline''
+        cdnjs.cloudflare.com; frame-ancestors ''none''; object-src ''none'''
+      Cross-Origin-Opener-Policy:
+      - same-origin
+      Date:
+      - Mon, 27 Jan 2025 11:16:57 GMT
+      Referrer-Policy:
+      - same-origin
+      Server:
+      - WSGIServer/0.2 CPython/3.10.14
+      Server-Timing:
+      - TimerPanel_utime;dur=288.60300000002326;desc="User CPU time", TimerPanel_stime;dur=30.381999999995912;desc="System
+        CPU time", TimerPanel_total;dur=318.98500000001917;desc="Total CPU time",
+        TimerPanel_total_time;dur=329.4947147369385;desc="Elapsed time", SQLPanel_sql_time;dur=5.710124969482422;desc="SQL
+        3 queries", CachePanel_total_time;dur=0;desc="Cache 0 Calls"
+      Vary:
+      - origin, Cookie
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - DENY
+    status:
+      code: 504
+      message: Gateway Timeout
+- request:
+    body: null
+    headers:
+      Accept:
+      - '*/*'
+      Accept-Crs:
+      - EPSG:4326
+      Accept-Encoding:
+      - gzip, deflate, br
+      Authorization:
+      - Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJ0ZXN0LXZjciIsImlhdCI6MTczNzk3NjYwMSwiY2xpZW50X2lkIjoidGVzdC12Y3IiLCJ1c2VyX2lkIjoiIiwidXNlcl9yZXByZXNlbnRhdGlvbiI6IiJ9.1LYlUWSglBqFMRp4uh3wFBwBmR9midapufWeCga_k10
+      Connection:
+      - keep-alive
+      User-Agent:
+      - python-requests/2.32.3
+    method: GET
+    uri: http://localhost:8003/zaken/api/v1/zaken?expand=resultaat%2Cresultaat.resultaattype%2Czaaktype%2Crollen&archiefnominatie=vernietigen&einddatum__isnull=False&einddatum__lt=2025-01-27
+  response:
+    body:
+      string: ''
+    headers:
+      API-version:
+      - 1.5.1
+      Allow:
+      - GET, POST, HEAD, OPTIONS
+      Content-Crs:
+      - EPSG:4326
+      Content-Length:
+      - '0'
+      Content-Security-Policy:
+      - 'img-src ''self'' data: cdn.redoc.ly cdnjs.cloudflare.com *.tile.openstreetmap.org;
+        style-src ''self'' ''unsafe-inline'' fonts.googleapis.com cdnjs.cloudflare.com;
+        connect-src ''self'' raw.githubusercontent.com; base-uri ''self''; frame-src
+        ''self''; form-action ''self''; worker-src ''self'' blob:; font-src ''self''
+        fonts.gstatic.com; default-src ''self''; script-src ''self'' ''unsafe-inline''
+        cdnjs.cloudflare.com; frame-ancestors ''none''; object-src ''none'''
+      Cross-Origin-Opener-Policy:
+      - same-origin
+      Date:
+      - Mon, 27 Jan 2025 11:17:02 GMT
+      Referrer-Policy:
+      - same-origin
+      Server:
+      - WSGIServer/0.2 CPython/3.10.14
+      Server-Timing:
+      - TimerPanel_utime;dur=286.20599999999286;desc="User CPU time", TimerPanel_stime;dur=31.54599999999874;desc="System
+        CPU time", TimerPanel_total;dur=317.7519999999916;desc="Total CPU time", TimerPanel_total_time;dur=327.64530181884766;desc="Elapsed
+        time", SQLPanel_sql_time;dur=4.678249359130859;desc="SQL 3 queries", CachePanel_total_time;dur=0;desc="Cache
+        0 Calls"
+      Vary:
+      - origin, Cookie
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - DENY
+    status:
+      code: 504
+      message: Gateway Timeout
+version: 1
diff --git a/backend/src/openarchiefbeheer/zaken/tests/vcr/test_tasks.py b/backend/src/openarchiefbeheer/zaken/tests/vcr/test_tasks.py
@@ -1,15 +1,19 @@
-from django.test import TestCase, tag
+from unittest.mock import patch
 
+from django.test import TestCase, override_settings, tag
+
+import requests
 from freezegun import freeze_time
 from requests_mock import Mocker
 from vcr.unittest import VCRMixin
 from zgw_consumers.constants import APITypes
 from zgw_consumers.test.factories import ServiceFactory
 
+from openarchiefbeheer.config.models import APIConfig
 from openarchiefbeheer.utils.utils_decorators import reload_openzaak_fixtures
 from openarchiefbeheer.zaken.models import Zaak
 
-from ...tasks import retrieve_and_cache_zaken_from_openzaak
+from ...tasks import resync_zaken, retrieve_and_cache_zaken_from_openzaak
 
 
 @tag("vcr")
@@ -24,6 +28,10 @@ def setUpClass(cls) -> None:
             client_id="test-vcr",
             secret="test-vcr",
         )
+        cls.selectielijst_service = ServiceFactory.create(
+            api_type=APITypes.orc,
+            api_root="https://selectielijst.openzaak.nl/zaken/api/v1",
+        )
 
     @tag("gh-298")
     @reload_openzaak_fixtures()
@@ -58,3 +66,18 @@ def test_recaching_zaken_correct_eindatum(self):
         zaken_in_db = Zaak.objects.all()
 
         self.assertEqual(zaken_in_db.count(), 103)
+
+    @override_settings(
+        RETRY_TOTAL=2, RETRY_BACKOFF_FACTOR=1, RETRY_STATUS_FORCELIST=[504]
+    )
+    def test_resync_zaken_retry_mechanism(self):
+        with (
+            self.assertRaises(requests.exceptions.RetryError),
+            patch(
+                "openarchiefbeheer.zaken.utils.APIConfig.get_solo",
+                return_value=APIConfig(
+                    selectielijst_api_service=self.selectielijst_service
+                ),
+            ),
+        ):
+            resync_zaken()
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -50,6 +50,7 @@ services:
       - REACT_APP_API_URL=http://localhost:8000
       - REACT_APP_API_PATH=/api/v1
       - REACT_APP_ZAAK_URL_TEMPLATE=https://www.example.com/zaken/{identificatie}
+      - REQUESTS_READ_TIMEOUT=5000
     ports:
       - 8000:8000
     depends_on:
@@ -66,7 +67,7 @@ services:
     environment: *web_env
     command: /setup_configuration.sh
     volumes:
-      - ./backend/src/openarchiefbeheer/config/setup-configuration/fixtures:/app/setup_configuration
+      - ./backend/src/openarchiefbeheer/config/setup_configuration/fixtures:/app/setup_configuration
     depends_on:
       - db
       - redis