extend healthcheck to fail when a single mapping is in a broken state

docs/conf.py

@@ -48,7 +48,7 @@ def linkcode_resolve(domain, info):
             pass
         try:
             lines, first_line = inspect.getsourcelines(item)
-            lineno = "#L%d-L%s" % (first_line, first_line + len(lines) - 1)
+            lineno = f"#L{first_line:d}-L{first_line + len(lines) - 1}"
         except (OSError, TypeError):
             pass
     return (

heroku_connect/contrib/heroku_connect_health_check/backends.py

@@ -46,3 +46,20 @@ def check_status(self):
                         )
                     )
                 )
+
+            connection_state = utils.get_connection(connection["id"], deep=True)
+
+            for mapping in connection_state["mappings"]:
+                object_name = mapping["object_name"]
+                state = mapping["state"]
+
+                if (
+                    state in utils.ERROR_MAPPING_STATES
+                    or state in utils.TEMPORARY_ERROR_MAPPING_STATES
+                ):
+                    self.add_error(
+                        ServiceUnavailable(
+                            f"mapping {object_name} on connection {connection['name']} "
+                            f"is in state {state}"
+                        )
+                    )

heroku_connect/utils.py

@@ -1,7 +1,7 @@
 """Utility methods for Django Heroku Connect."""
 
 import os
-from enum import Enum
+from enum import Enum, unique
 from functools import lru_cache
 
 import requests
@@ -32,6 +32,70 @@ class ConnectionStates:
     )
 
 
+@unique
+class MappingState(str, Enum):
+    # https://devcenter.heroku.com/articles/mapping-states-reference#list-of-mapping-states
+    DATA_SYNCED = "DATA_SYNCED"
+    INITIAL = "INITIAL"
+    RESYNC = "RESYNC"
+    RELOAD_TABLE = "RELOAD_TABLE"
+    SCHEMA_CHANGED = "SCHEMA_CHANGED"
+    POLLING_SF_CHANGES = "POLLING_SF_CHANGES"
+    POLLING_SF_BULK = "POLLING_SF_BULK"
+    POLLING_SF_FOR_DELETES = "POLLING_SF_FOR_DELETES"
+    WAIT_BULK_LOAD = "WAIT_BULK_LOAD"
+    LOADING_BULK_JOB = "LOADING_BULK_JOB"
+    RESOLVE_EXTERNAL_IDS = "RESOLVE_EXTERNAL_IDS"
+    BULK_LOAD_ERROR = "BULK_LOAD_ERROR"
+    WAIT_BULK_UPDATE = "WAIT_BULK_UPDATE"
+    APPLYING_BULK_UPDATE = "APPLYING_BULK_UPDATE"
+    POLL_EXTERNAL_IDS = "POLL_EXTERNAL_IDS"
+    RECOVERY = "RECOVERY"
+    ABORTED = "ABORTED"
+    SYSTEM_ERROR = "SYSTEM_ERROR"
+    DB_UNAVAILABLE = "DB_UNAVAILABLE"
+    BAD_CONFIG = "BAD_CONFIG"
+    SYNC_REMOVED = "SYNC_REMOVED"
+    INACTIVE_ORG = "INACTIVE_ORG"
+    UNAUTHORIZED = "UNAUTHORIZED"
+
+
+OK_MAPPING_STATES = (
+    MappingState.DATA_SYNCED,
+    MappingState.RESYNC,
+    MappingState.RELOAD_TABLE,
+    MappingState.POLLING_SF_CHANGES,
+    MappingState.POLLING_SF_BULK,
+    MappingState.POLLING_SF_FOR_DELETES,
+    MappingState.WAIT_BULK_LOAD,
+    MappingState.LOADING_BULK_JOB,
+    MappingState.RESOLVE_EXTERNAL_IDS,
+    MappingState.WAIT_BULK_UPDATE,
+    MappingState.APPLYING_BULK_UPDATE,
+    MappingState.POLL_EXTERNAL_IDS,
+    MappingState.ABORTED,
+)
+
+# mapping states that mean that the sync is not working, but will typically resolve
+# itself
+TEMPORARY_ERROR_MAPPING_STATES = (
+    MappingState.INITIAL,
+    MappingState.SCHEMA_CHANGED,
+    MappingState.RECOVERY,
+)
+
+# mapping states that are permanent errors
+ERROR_MAPPING_STATES = (
+    MappingState.BULK_LOAD_ERROR,
+    MappingState.SYSTEM_ERROR,
+    MappingState.DB_UNAVAILABLE,
+    MappingState.BAD_CONFIG,
+    MappingState.SYNC_REMOVED,
+    MappingState.INACTIVE_ORG,
+    MappingState.UNAUTHORIZED,
+)
+
+
 class WriteAlgorithm(Enum):
     MERGE_WRITES = 1
     ORDERED_WRITES = 2

tests/contrib/test_healthcheck.py

@@ -1,3 +1,4 @@
+import copy
 import json
 import secrets
 
@@ -11,6 +12,46 @@
 from tests import fixtures
 
 
+@httpretty.activate
+def test_check_status_mapping():
+    httpretty.register_uri(
+        httpretty.GET,
+        "https://connect-eu.heroku.com/api/v3/connections",
+        body=json.dumps(fixtures.connections),
+        status=200,
+        content_type="application/json",
+    )
+    httpretty.register_uri(
+        httpretty.GET,
+        "https://connect-eu.heroku.com/api/v3/connections/1?deep=true",
+        body=json.dumps(fixtures.connection),
+        status=200,
+        content_type="application/json",
+    )
+    hc = HerokuConnectHealthCheck()
+    hc.check_status()
+    assert not hc.errors
+
+    failed_connection = copy.deepcopy(fixtures.connection)
+    failed_connection["mappings"][0]["state"] = "BAD_CONFIG"
+
+    httpretty.register_uri(
+        httpretty.GET,
+        "https://connect-eu.heroku.com/api/v3/connections/1?deep=true",
+        body=json.dumps(failed_connection),
+        status=200,
+        content_type="application/json",
+    )
+
+    hc = HerokuConnectHealthCheck()
+    hc.check_status()
+    assert hc.errors
+    assert (
+        hc.errors[0].message
+        == "mapping Account on connection sample name is in state BAD_CONFIG"
+    )
+
+
 @httpretty.activate
 def test_check_status():
     httpretty.register_uri(
@@ -20,6 +61,13 @@ def test_check_status():
         status=200,
         content_type="application/json",
     )
+    httpretty.register_uri(
+        httpretty.GET,
+        "https://connect-eu.heroku.com/api/v3/connections/1?deep=true",
+        body=json.dumps(fixtures.connection),
+        status=200,
+        content_type="application/json",
+    )
     hc = HerokuConnectHealthCheck()
     hc.check_status()
     assert not hc.errors
@@ -79,6 +127,13 @@ def test_health_check_url(client):
         status=200,
         content_type="application/json",
     )
+    httpretty.register_uri(
+        httpretty.GET,
+        "https://connect-eu.heroku.com/api/v3/connections/1?deep=true",
+        body=json.dumps(fixtures.connection),
+        status=200,
+        content_type="application/json",
+    )
     response = client.get("/ht/")
     assert response.status_code == 200
     assert b"<td>Heroku Connect</td>" in response.content

tests/fixtures.py

@@ -5,7 +5,7 @@
     "schema_name": "salesforce",
     "db_key": "DATABASE_URL",
     "state": "IDLE",
-    "mappings": [{"id": "XYZ", "object_name": "Account", "state": "SCHEMA_CHANGED"}],
+    "mappings": [{"id": "XYZ", "object_name": "Account", "state": "DATA_SYNCED"}],
 }
 
 connections = {"count": 1, "results": [connection]}