Merge pull request #226 from ParkenDD/loki-sending-in-celery

fix loki in celery, ruff mocked service

.github/workflows/lint.yml

@@ -34,9 +34,3 @@ jobs:
       # https://beta.ruff.rs/docs/usage/#github-action
       # uses: chartboost/ruff-action@v1
       run: ruff check --output-format github ./webapp ./tests ./migrations
-
-    - name: format using ruff formatter
-      # We could also use the official GitHub Actions integration.
-      # https://beta.ruff.rs/docs/usage/#github-action
-      # uses: chartboost/ruff-action@v1
-      run: ruff check ./webapp ./tests ./migrations

CHANGELOG.md

@@ -1,9 +1,20 @@
 # Changelog
 
+## 0.16.3
+
+Released 2024-12-26
+
+### Fixes
+
+* [Fixes an issue with logs not sent to Loki in Celery](https://github.com/ParkenDD/park-api-v3/pull/226)
+
+
 ## 0.16.2
 
 Released 2024-11-28
 
+### Fixes
+
 * [Fixes an issue with Celery Tasks sometimes preventing static data imports](https://github.com/ParkenDD/park-api-v3/pull/224)
 
 

dev/test_services/mocked_loki/app.py

@@ -0,0 +1,25 @@
+"""
+Copyright 2024 binary butterfly GmbH
+Use of this source code is governed by an MIT-style license that can be found in the LICENSE.txt.
+"""
+
+from flask import Flask, make_response, request
+
+
+def empty_json_response():
+    response = make_response('')
+    response.mimetype = 'application/json'
+    return response
+
+
+app = Flask('mocked_loki')
+
+
+@app.post('/loki/api/v1/push')
+def loki_push_logs():
+    print(f'<< {request.data}')
+    return empty_json_response(), 204
+
+
+if __name__ == '__main__':
+    app.run(debug=True, host='0.0.0.0')

docker-compose.yml

@@ -26,6 +26,11 @@ x-flask-defaults: &flask-defaults
     rabbitmq:
       condition: service_healthy
 
+# Define reusable defaults for mocked services
+x-mocked-service-defaults: &mocked-service-defaults
+  image: park-api:local-dev
+  command: ["python", "/app/app.py"]
+  user: *local-user
 
 # Define actual services (containers)
 services:
@@ -76,6 +81,10 @@ services:
       timeout: 1s
       retries: 20
 
+  mocked-loki:
+    <<: *mocked-service-defaults
+    volumes:
+      - ./dev/test_services/mocked_loki:/app
 
 volumes:
   postgresql:

pyproject.toml

@@ -32,6 +32,7 @@ filterwarnings = [
 
 
 [tool.ruff]
+
 lint.select = [
     "F", # pyflakes
     "I", # isort
@@ -77,6 +78,7 @@ exclude = [
     "dist",
     "node_modules",
     "venv",
+    "dev/*"
 ]
 
 line-length = 120

webapp/common/logging/logger.py

@@ -21,6 +21,7 @@ class Logger:
     config_helper: ConfigHelper
     context_helper: ContextHelper
     logger: PythonLogger
+    local_handler: Optional[LocalFileHandler] = None
     local_file_handler: Optional[LocalFileHandler] = None
     stdout_handler: Optional[StdoutHandler] = None
     loki_handler: Optional[LokiQueueHandler] = None
@@ -32,11 +33,11 @@ def __init__(self, config_helper: ConfigHelper, context_helper: ContextHelper):
         self.logger = logging.getLogger('app')
         self.logger.setLevel(logging.INFO)
 
-        self.local_handler = LocalFileHandler(config_helper=config_helper)
+        self.local_handler = LocalFileHandler(config_helper=self.config_helper)
         self.logger.addHandler(self.local_handler)
 
         if self.config_helper.get('LOKI_ENABLED'):
-            self.loki_handler = LokiQueueHandler(config_helper=config_helper)
+            self.loki_handler = LokiQueueHandler(config_helper=self.config_helper)
             self.logger.addHandler(self.loki_handler)
 
         if self.config_helper.get('STDOUT_LOGGING_ENABLED'):

webapp/common/logging/loki_handler.py

@@ -3,10 +3,11 @@
 Use of this source code is governed by an MIT-style license that can be found in the LICENSE.txt.
 """
 
+import logging
 import os
 from logging import Handler, LogRecord
 from logging.handlers import QueueHandler, QueueListener
-from queue import Queue
+from multiprocessing import Queue
 from typing import Optional
 
 from webapp.common.config import ConfigHelper
@@ -60,6 +61,7 @@ def __init__(self, config_helper: ConfigHelper):
 
         self.queue = Queue()
         super().__init__(self.queue)
+        self.setLevel(logging.INFO)
 
         if config_helper.get('LOKI_USER') and config_helper.get('LOKI_PASSWORD'):
             auth = BasicAuth(config_helper.get('LOKI_USER'), config_helper.get('LOKI_PASSWORD'))