Added OpenTelemetry, remove OpenCensus.

Also changed the log formatting to use an actual JSON-formatter. The old
solution generated JSON-like log strings by injecting the strings into
an existing JSON-formatted string. This doesn't escape the '"'
character, thus breaks with some django server messages.

Fixed requirements file (regenerating it broke, sorting the requirements
out fixed it).

src/dso_api/dynamic_api/models.py

@@ -23,7 +23,6 @@ def refresh_from_db(self, using=None, fields=None):
         the queryset/manager and queryset iterator.
         """
         if fields and fields[0] not in self.__dict__:
-
             message = (
                 f"Deferred attribute access: field '{fields[0]}' "
                 f"was excluded by .only() but was still accessed."

src/dso_api/settings.py

@@ -1,4 +1,3 @@
-import json
 import logging
 import os
 import sys
@@ -9,8 +8,7 @@
 import sentry_sdk
 import sentry_sdk.utils
 from corsheaders.defaults import default_headers
-from django.core.exceptions import ImproperlyConfigured
-from opencensus.trace import config_integration
+from pythonjsonlogger import jsonlogger
 from sentry_sdk.integrations.django import DjangoIntegration
 from sentry_sdk.integrations.logging import LoggingIntegration
 
@@ -54,15 +52,6 @@
 OAUTH_DEFAULT_SCOPE = env.str("OAUTH_DEFAULT_SCOPE", None)
 OAUTH_CLIENT_ID = os.getenv("OAUTH_CLIENT_ID", "dso-api-open-api")
 
-# -- Azure specific settings
-
-# Microsoft recommended abbreviation for Application Insights is `APPI`
-AZURE_APPI_CONNECTION_STRING: str | None = env.str("AZURE_APPI_CONNECTION_STRING", None)
-AZURE_APPI_AUDIT_CONNECTION_STRING: str | None = env.str(
-    "AZURE_APPI_AUDIT_CONNECTION_STRING", None
-)
-
-MAX_REPLICA_COUNT = env.int("MAX_REPLICA_COUNT", 5)
 
 # -- Security
 
@@ -178,6 +167,7 @@
 
     # Support up to 5 replicas configured with environment variables using
     # PGHOST_REPLICA_1 to PGHOST_REPLICA_5
+    MAX_REPLICA_COUNT = env.int("MAX_REPLICA_COUNT", 5)
     for replica_count in range(1, MAX_REPLICA_COUNT + 1):
         if env.str(f"PGHOST_REPLICA_{replica_count}", False):
             DATABASES.update(
@@ -260,35 +250,66 @@
     )
     sentry_sdk.utils.MAX_STRING_LENGTH = 2048  # for WFS FILTER exceptions
 
-base_log_fmt = {"time": "%(asctime)s", "name": "%(name)s", "level": "%(levelname)s"}
-log_fmt = base_log_fmt.copy()
-log_fmt["message"] = "%(message)s"
+# -- Logging
+
+
+class CustomJsonFormatter(jsonlogger.JsonFormatter):
+    def __init__(self, *args, **kwargs):
+        # Make sure some 'extra' fields are not included:
+        super().__init__(*args, **kwargs)
+        self._skip_fields.update({"request": "request", "taskName": "taskName"})
+
+    def add_fields(self, log_record: dict, record, message_dict: dict):
+        # The 'rename_fields' logic fails when fields are missing, this is easier:
+        super().add_fields(log_record, record, message_dict)
+        # An in-place reordering, sotime/level appear first (easier for docker log scrolling)
+        ordered_dict = {
+            "time": log_record.pop("asctime", record.asctime),
+            "level": log_record.pop("levelname", record.levelname),
+            **log_record,
+        }
+        log_record.clear()
+        log_record.update(ordered_dict)
 
-audit_log_fmt = {"audit": True}
-audit_log_fmt.update(log_fmt)
+
+_json_log_formatter = {
+    "()": CustomJsonFormatter,
+    "format": "%(asctime)s $(levelname)s %(name)s %(message)s",  # parsed as a fields list.
+}
+
+DJANGO_LOG_LEVEL = env.str("DJANGO_LOG_LEVEL", "INFO")
+LOG_LEVEL = env.str("LOG_LEVEL", "DEBUG" if DEBUG else "INFO")
+AUDIT_LOG_LEVEL = env.str("AUDIT_LOG_LEVEL", "INFO")
 
 LOGGING = {
     "version": 1,
     "disable_existing_loggers": True,
     "formatters": {
-        "json": {"format": json.dumps(log_fmt)},
-        "audit_json": {"format": json.dumps(audit_log_fmt)},
+        "json": _json_log_formatter,
+        "audit_json": _json_log_formatter | {"static_fields": {"audit": True}},
     },
     "handlers": {
         "console": {
             "level": "DEBUG",
             "class": "logging.StreamHandler",
             "formatter": "json",
         },
+        "console_print": {
+            "level": "DEBUG",
+            "class": "logging.StreamHandler",
+        },
         "audit_console": {
+            # For azure, this is replaced below.
             "level": "DEBUG",
             "class": "logging.StreamHandler",
             "formatter": "audit_json",
         },
     },
-    "root": {"level": "INFO", "handlers": ["console"]},
+    "root": {
+        "level": DJANGO_LOG_LEVEL,
+        "handlers": ["console"],
+    },
     "loggers": {
-        "opencensus": {"handlers": ["console"], "level": DJANGO_LOG_LEVEL, "propagate": False},
         "django": {"handlers": ["console"], "level": DJANGO_LOG_LEVEL, "propagate": False},
         "django.utils.autoreload": {"handlers": ["console"], "level": "INFO", "propagate": False},
         "dso_api": {"handlers": ["console"], "level": DSO_API_LOG_LEVEL, "propagate": False},
@@ -311,54 +332,85 @@
     },
 }
 
-if CLOUD_ENV.lower().startswith("azure"):
-    if AZURE_APPI_CONNECTION_STRING is None:
-        raise ImproperlyConfigured(
-            "Please specify the 'AZURE_APPI_CONNECTION_STRING' environment variable."
+if DEBUG:
+    # Print tracebacks without JSON formatting.
+    LOGGING["loggers"]["django.request"] = {
+        "handlers": ["console_print"],
+        "level": "ERROR",
+        "propagate": False,
+    }
+
+# -- Azure specific settings
+if CLOUD_ENV.startswith("azure"):
+    from azure.monitor.opentelemetry import configure_azure_monitor
+    from opentelemetry.instrumentation.django import DjangoInstrumentor
+    from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor
+    from opentelemetry.sdk.resources import Resource
+    from opentelemetry.semconv.resource import ResourceAttributes
+
+    # Microsoft recommended abbreviation for Application Insights is `APPI`
+    AZURE_APPI_CONNECTION_STRING = env.str("AZURE_APPI_CONNECTION_STRING")
+    AZURE_APPI_AUDIT_CONNECTION_STRING: str | None = env.str(
+        "AZURE_APPI_AUDIT_CONNECTION_STRING", None
+    )
+
+    # Configure OpenTelemetry to use Azure Monitor with the specified connection string
+    if AZURE_APPI_CONNECTION_STRING is not None:
+        configure_azure_monitor(
+            connection_string=AZURE_APPI_CONNECTION_STRING,
+            logger_name="root",
+            instrumentation_options={
+                "azure_sdk": {"enabled": False},
+                "django": {"enabled": False},  # Manually done
+                "fastapi": {"enabled": False},
+                "flask": {"enabled": False},
+                "psycopg2": {"enabled": False},  # Manually done
+                "requests": {"enabled": True},
+                "urllib": {"enabled": True},
+                "urllib3": {"enabled": True},
+            },
+            resource=Resource.create({ResourceAttributes.SERVICE_NAME: "haal-centraal-proxy"}),
         )
-    if AZURE_APPI_AUDIT_CONNECTION_STRING is None:
-        logging.warning(
-            "Using AZURE_APPI_CONNECTION_STRING as AZURE_APPI_AUDIT_CONNECTION_STRING."
+        print("OpenTelemetry has been enabled")
+
+        def response_hook(span, request, response):
+            if (
+                span.is_recording()
+                and hasattr(request, "get_token_claims")
+                and (email := request.get_token_claims.get("email", request.get_token_subject))
+            ):
+                span.set_attribute("user.AuthenticatedId", email)
+
+        DjangoInstrumentor().instrument(response_hook=response_hook)
+        print("Django instrumentor enabled")
+
+        Psycopg2Instrumentor().instrument(enable_commenter=True, commenter_options={})
+        print("Psycopg instrumentor enabled")
+
+    if AZURE_APPI_AUDIT_CONNECTION_STRING is not None:
+        # Configure audit logging to an extra log (not telemetry data).
+        from azure.monitor.opentelemetry.exporter import AzureMonitorLogExporter
+        from opentelemetry.sdk._logs import LoggerProvider
+        from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+
+        audit_logger_provider = LoggerProvider()
+        audit_logger_provider.add_log_record_processor(
+            BatchLogRecordProcessor(
+                AzureMonitorLogExporter(connection_string=AZURE_APPI_AUDIT_CONNECTION_STRING)
+            )
         )
 
-    MIDDLEWARE.append("opencensus.ext.django.middleware.OpencensusMiddleware")
-    OPENCENSUS = {
-        "TRACE": {
-            "SAMPLER": "opencensus.trace.samplers.ProbabilitySampler(rate=1)",
-            "EXPORTER": f"""opencensus.ext.azure.trace_exporter.AzureExporter(
-                connection_string='{AZURE_APPI_CONNECTION_STRING}',
-                service_name='dso-api'
-            )""",  # noqa: E202
-            "EXCLUDELIST_PATHS": [],
+        LOGGING["handlers"]["audit_console"] = {
+            # This does: handler = LoggingHandler(logger_provider=audit_logger_provider)
+            "level": "DEBUG",
+            "class": "opentelemetry.sdk._logs.LoggingHandler",
+            "logger_provider": audit_logger_provider,
+            "formatter": "audit_json",
         }
-    }
-    config_integration.trace_integrations(["logging"])
-    azure_json = base_log_fmt.copy()
-    azure_json.update({"message": "%(message)s"})
-    audit_azure_json = {"audit": True}
-    audit_azure_json.update(azure_json)
-    LOGGING["formatters"]["azure"] = {"format": json.dumps(azure_json)}
-    LOGGING["formatters"]["audit_azure"] = {"format": json.dumps(audit_azure_json)}
-    LOGGING["handlers"]["azure"] = {
-        "level": "DEBUG",
-        "class": "opencensus.ext.azure.log_exporter.AzureLogHandler",
-        "connection_string": AZURE_APPI_CONNECTION_STRING,
-        "formatter": "azure",
-    }
-    LOGGING["handlers"]["audit_azure"] = {
-        "level": "DEBUG",
-        "class": "opencensus.ext.azure.log_exporter.AzureLogHandler",
-        "connection_string": AZURE_APPI_AUDIT_CONNECTION_STRING,
-        "formatter": "audit_azure",
-    }
+        for logger_name, logger_details in LOGGING["loggers"].items():
+            if "audit_console" in logger_details["handlers"]:
+                LOGGING["loggers"][logger_name]["handlers"] = ["audit_console", "console"]
 
-    LOGGING["root"]["handlers"] = ["azure"]
-    LOGGING["root"]["level"] = DJANGO_LOG_LEVEL
-    for logger_name, logger_details in LOGGING["loggers"].items():
-        if "audit_console" in logger_details["handlers"]:
-            LOGGING["loggers"][logger_name]["handlers"] = ["audit_azure", "console"]
-        else:
-            LOGGING["loggers"][logger_name]["handlers"] = ["azure", "console"]
 
 # -- Third party app settings
 

src/requirements.in

@@ -10,22 +10,21 @@ djangorestframework == 3.15.2
 djangorestframework-csv == 3.0.2
 djangorestframework-gis == 1.1
 amsterdam-schema-tools[django] == 6.1.1
+azure-identity == 1.17.1
+azure-monitor-opentelemetry == 1.6.2
+cachetools == 5.5.0
 datadiensten-apikeyclient == 0.6.0
 datapunt-authorization-django==1.3.3
 drf-spectacular == 0.27.2
 Geoalchemy2 == 0.15.2
-importlib-metadata == 8.5.0
-importlib-resources == 6.4.5
 jsonschema == 4.23.0
 lru_dict == 1.3.0
 Markdown == 3.7
 more-ds == 0.0.6
 more-itertools == 10.5.0
 openapi-spec-validator == 0.7.1
-opencensus-ext-azure >= 1.1.13
-opencensus-ext-django >= 0.8
-opencensus-ext-logging >= 0.1.1
 orjson == 3.10.7
+python-json-logger==2.0.7
 python-string-utils == 1.0.0
 requests == 2.32.3
 sentry-sdk == 2.14.0