chore: Add structured logging for DataDog integration (M2-8041)

.dockerignore

@@ -7,3 +7,5 @@ docker-compose.yaml
 *.md
 *.rst
 venv*/
+allure-results
+.git

.env.default

@@ -94,4 +94,9 @@ OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=
 # OTEL_TRACES_EXPORTER
 # OTEL_EXPORTER_OTLP_CERTIFICATE
 
-MULTI_INFORMANT__TEMP_RELATION_EXPIRY_SECS=86400
+MULTI_INFORMANT__TEMP_RELATION_EXPIRY_SECS=86400
+
+# False for local dev
+DD_TRACE_ENABLED=false
+DD_LOGS_ENABLED=false
+LOG_JSON_FORMAT=false

.github/workflows/pr-open.yml

@@ -49,6 +49,7 @@ jobs:
     permissions:
       issues: write
       pull-requests: write
+      discussions: write
     needs: [ run-unit-tests ]
     steps:
       - name: Post the link to the report

.github/workflows/run_build_deploy.yaml

@@ -2,8 +2,8 @@
 name: Build and Deploy
 on: 
   push:
-#    branches:
-#      - develop
+    branches:
+      - develop
   workflow_dispatch: {}
 
 

.gitignore

@@ -62,3 +62,4 @@ venv/
 
 # MacOS
 .DS_Store
+

Pipfile

@@ -51,8 +51,14 @@ taskiq = { extras = ["reload"], version = "==0.11.7" }
 taskiq-aio-pika = "==0.4.1"
 taskiq-fastapi = "==0.3.2"
 taskiq-redis = "==1.0.2"
-typer = "==0.15.1"
-uvicorn = { extras = ["standard"], version = "==0.32.1" }
+typer = "==0.12.5"
+uvicorn = { extras = ["standard"], version = "==0.32.0" }
+pyjwt = "==2.9.0"
+ddtrace = "==2.17.2"
+bytecode = "==0.16.0"
+structlog = "==24.4.0"
+asgi-correlation-id = "==4.3.4"
+
 
 [dev-packages]
 allure-pytest = "==2.13.5"

compose/fastapi/Dockerfile

@@ -37,6 +37,11 @@ RUN sed -i 's/\r$//g' /fastapi-entrypoint && chmod +x /fastapi-entrypoint
 COPY --chown=code:code ./compose/fastapi/start /fastapi-start
 RUN sed -i 's/\r$//g' /fastapi-start && chmod +x /fastapi-start
 
+COPY --chown=code:code ./compose/fastapi/start-datadog /fastapi-start-datadog
+RUN sed -i 's/\r$//g' /fastapi-start-datadog && chmod +x /fastapi-start-datadog
+
+
+
 COPY --chown=code:code ./compose/fastapi/migrate /fastapi-migrate
 RUN sed -i 's/\r$//g' /fastapi-migrate && chmod +x /fastapi-migrate
 

compose/fastapi/ecs-start-feature

@@ -4,4 +4,4 @@ set -eo pipefail
 set -o nounset
 
 ENV=testing /fastapi-migrate
-/fastapi-start
+/fastapi-start-datadog

compose/fastapi/start

@@ -8,6 +8,9 @@ set -o nounset
 export UVICORN_HOST="0.0.0.0"
 export UVICORN_PORT=80
 
-opentelemetry-instrument \
-    --logs_exporter otlp \
-    uvicorn src.main:app --reload --host ${UVICORN_HOST} --port ${UVICORN_PORT} --proxy-headers
+#opentelemetry-instrument \
+#    --logs_exporter otlp \
+#    uvicorn src.main:app --reload --host ${UVICORN_HOST} --port ${UVICORN_PORT} --proxy-headers
+
+
+uvicorn src.main:app --reload --host ${UVICORN_HOST} --port ${UVICORN_PORT} --proxy-headers

compose/fastapi/start-datadog

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+# https://www.uvicorn.org/settings/
+export UVICORN_HOST="0.0.0.0"
+export UVICORN_PORT=80
+
+
+LOG_JSON_FORMAT=true DD_TRACE_ENABLED=true uvicorn main:app \
+  --host ${UVICORN_HOST} --port ${UVICORN_PORT} \
+  --reload --proxy-headers \
+  --log-config uvicorn_disable_logging.json

copilot/mindlogger-backend/addons/svc-role.yaml

@@ -39,6 +39,13 @@ Resources:
               - "arn:aws:s3:::cmiml-feature-answer/*"
               - "arn:aws:s3:::cmiml-feature-operations/*"
 
+          - Sid: DataDogAgent
+            Effect: Allow
+            Action:
+              - "ecs:ListClusters"
+              - "ecs:ListContainerInstances"
+              - "ecs:DescribeContainerInstances"
+            Resource: ["*"]
 
 
 Outputs:

copilot/mindlogger-backend/manifest.yml

@@ -56,7 +56,19 @@ storage:
       efs: false
       path: /app/uploads
       read_only: false
-
+    # For Datadog
+#    sock:
+#      efs: false
+#      path: '/var/run/docker.sock'
+#      read_only: true
+#    proc:
+#      efs: false
+#      path: '/proc/'
+#      read_only: true
+#    cgroup:
+#      efs: false
+#      path: '/sys/fs/cgroup/'
+#      read_only: true
 
 variables:
   # Python configurations
@@ -119,6 +131,10 @@ variables:
 #  CDN__LEGACY_ACCESS_KEY:
   CDN__TTL_SIGNED_URLS: 3600
 
+  DD_ENV: feature
+  DD_SERVICE: backend-api
+  DD_VERSION: ${COPILOT_ENVIRONMENT_NAME}
+
   # jsonld converter
   JSONLD_CONVERTER__PROTOCOL_PASSWORD:
 
@@ -158,4 +174,56 @@ taskdef_overrides:
     value:
       Name: "nofile"
       SoftLimit: 1048576
-      HardLimit: 1048576
+      HardLimit: 1048576
+
+logging:
+  destination:
+    Name: "datadog"
+    Host: "http-intake.logs.datadoghq.com"
+    dd_service: "backend-api"
+    dd_source: "backend"
+    dd_message_key: "log"
+    dd_tags: "project:fluentbit"
+    TLS: "on"
+    provider: "ecs"
+  configFilePath: "/fluent-bit/configs/parse-json.conf"
+  enableMetadata: true
+  secretOptions:
+    apikey:
+      secretsmanager: "cmiml-feature-${COPILOT_ENVIRONMENT_NAME}:DD_API_KEY::"
+
+
+sidecars:
+  datadog:
+    image: public.ecr.aws/datadog/agent:7
+    variables:
+      ECS_FARGATE: true
+      DD_SITE: us5.datadoghq.com
+      DD_ENV: ${COPILOT_ENVIRONMENT_NAME}
+      DD_LOGS_ENABLED: true
+      DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL: true
+      DD_CONTAINER_EXCLUDE: name:datadog-agent
+      DD_APM_ENABLED: true
+    secrets:
+      DD_API_KEY:
+        secretsmanager: "cmiml-feature-${COPILOT_ENVIRONMENT_NAME}:DD_API_KEY::"
+
+        #/copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/DATADOG_API_KEY
+#    mount_points:
+#      - source_volume: sock
+#        path: '/var/run/docker.sock'
+#      - source_volume: proc
+#        path: '/proc/'
+#      - source_volume: cgroup
+#        path: '/sys/fs/cgroup/'
+
+#  firelens:
+#    essential: true
+#    image: amazon/aws-for-fluent-bit:stable
+#    name: log_router
+#    firelensConfiguration:
+#      type: fluentbit
+#      options:
+#        enable-ecs-log-metadata: 'true'
+#        config-file-type: file
+#        config-file-value: "/fluent-bit/configs/parse-json.conf"

docker-compose.yaml

@@ -44,7 +44,7 @@ services:
       args:
         - PIPENV_EXTRA_ARGS=--dev
     entrypoint: /fastapi-entrypoint
-    command: /fastapi-start
+    command: /fastapi-start-datadog
     env_file: .env
     depends_on:
       - postgres

src/infrastructure/app.py

@@ -1,6 +1,7 @@
 from typing import Iterable, Type
 
 import sentry_sdk
+from asgi_correlation_id import CorrelationIdMiddleware
 from fastapi import FastAPI
 from fastapi.exceptions import RequestValidationError
 from fastapi.routing import APIRouter
@@ -28,6 +29,7 @@
 import middlewares as middlewares_
 from apps.shared.exception import BaseError
 from config import settings
+from infrastructure.datadog import StructuredLoggingMiddleware
 from infrastructure.http.execeptions import (
     custom_base_errors_handler,
     pydantic_validation_errors_handler,
@@ -77,6 +79,8 @@
     ),
     (middlewares_.InternalizationMiddleware, {}),
     (middlewares_.CORSMiddleware, middlewares_.cors_options),
+    (StructuredLoggingMiddleware, {}),
+    (CorrelationIdMiddleware, {}),
 )
 
 
@@ -108,4 +112,21 @@ def create_app():
     # https://github.com/Tufin/oasdiff/issues/52
     app.openapi_version = "3.0.3"
 
+    # UGLY HACK
+    # Datadog's `TraceMiddleware` is applied as the very first middleware
+    # in the list, by patching `FastAPI` constructor.
+    # Unfortunately that means that it is the innermost middleware, so the trace/span are
+    # created last in the middleware
+    # chain. Because we want to add the trace_id/span_id in the access log,
+    # we need to extract it from the middleware list,
+    # put it back as the outermost middleware, and rebuild the middleware stack.
+    # tracing_middleware = next(
+    #     (m for m in app.user_middleware if m.cls == TraceMiddleware), None
+    # )
+    # if tracing_middleware is not None:
+    #     app.user_middleware = [m for m in app.user_middleware if m.cls != TraceMiddleware]
+    #
+    #     app.user_middleware.insert(0, tracing_middleware)
+    #     app.middleware_stack = app.build_middleware_stack()
+
     return app