Merge pull request #4 from maykinmedia/feature/log-request-response-body

[#2] Add option to log request response body

.github/workflows/ci.yml

@@ -15,11 +15,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: ["3.7", "3.8", "3.9", "3.10"]
+        python: ["3.8", "3.9", "3.10"]
         django: ["3.2", "4.1"]
-        exclude:
-          - python: "3.7"
-            django: "4.1"
 
     name: Run the test suite (Python ${{ matrix.python }}, Django ${{ matrix.django }})
 

README.rst

@@ -97,6 +97,17 @@ To use this with your project you need to follow these steps:
         }
 
         LOG_OUTGOING_REQUESTS_DB_SAVE = True # save logs enabled/disabled based on the boolean value
+        LOG_OUTGOING_REQUESTS_DB_SAVE_BODY = True # save request/response body
+        LOG_OUTGOING_REQUESTS_EMIT_BODY = True # log request/response body
+        LOG_OUTGOING_REQUESTS_CONTENT_TYPES = [
+                "text/*",
+                "application/json",
+                "application/xml",
+                "application/soap+xml",
+        ] # save request/response bodies with matching content type
+        LOG_OUTGOING_REQUESTS_MAX_CONTENT_LENGTH = 524_288  # maximal size (in bytes) for the request/response body
+        LOG_OUTGOING_REQUESTS_LOG_BODY_TO_STDOUT = True
+
 
 #.  Run the migrations
 
@@ -112,8 +123,9 @@ To use this with your project you need to follow these steps:
         res = requests.get("https://httpbin.org/json")
         print(res.json())
 
-#.  Check stdout for the printable output, and navigate to ``/admin/log_outgoing_requests/outgoingrequestslog/`` to see 
-    the saved log records
+#.  Check stdout for the printable output, and navigate to ``Admin > Miscellaneous > Outgoing Requests Logs``
+    to see the saved log records. In order to override the settings for saving logs, navigate to
+    ``Admin > Miscellaneous > Outgoing Requests Log Configuration``.
 
 
 Local development

docs/quickstart.rst

@@ -58,6 +58,17 @@ Installation
         }
 
         LOG_OUTGOING_REQUESTS_DB_SAVE = True # save logs enabled/disabled based on the boolean value
+        LOG_OUTGOING_REQUESTS_DB_SAVE_BODY = True # save request/response body
+        LOG_OUTGOING_REQUESTS_EMIT_BODY = True # log request/response body
+        LOG_OUTGOING_REQUESTS_CONTENT_TYPES = [
+                "text/*",
+                "application/json",
+                "application/xml",
+                "application/soap+xml",
+        ] # save request/response bodies with matching content type
+        LOG_OUTGOING_REQUESTS_MAX_CONTENT_LENGTH = 524_288  # maximal size (in bytes) for the request/response body
+        LOG_OUTGOING_REQUESTS_LOG_BODY_TO_STDOUT = True
+
 
 #.  Run ``python manage.py migrate`` to create the necessary database tables.
 

log_outgoing_requests/admin.py

@@ -1,7 +1,11 @@
+from django import forms
+from django.conf import settings
 from django.contrib import admin
 from django.utils.translation import gettext as _
 
-from .models import OutgoingRequestsLog
+from solo.admin import SingletonModelAdmin
+
+from .models import OutgoingRequestsLog, OutgoingRequestsLogConfig
 
 
 @admin.register(OutgoingRequestsLog)
@@ -31,18 +35,43 @@ class OutgoingRequestsLogAdmin(admin.ModelAdmin):
         "response_ms",
         "timestamp",
     )
-    list_filter = ("method", "status_code", "hostname")
+    list_filter = ("method", "timestamp", "status_code", "hostname")
     search_fields = ("url", "params", "hostname")
     date_hierarchy = "timestamp"
     show_full_result_count = False
+    change_form_template = "log_outgoing_requests/change_form.html"
+
+    class Media:
+        css = {
+            "all": ("log_outgoing_requests/css/admin.css",),
+        }
 
     def has_add_permission(self, request):
         return False
 
     def has_change_permission(self, request, obj=None):
         return False
 
+    @admin.display(description=_("Query parameters"))
     def query_params(self, obj):
         return obj.query_params
 
-    query_params.short_description = _("Query parameters")
+
+class ConfigAdminForm(forms.ModelForm):
+    class Meta:
+        model = OutgoingRequestsLogConfig
+        fields = "__all__"
+        help_texts = {
+            "save_to_db": _(
+                "Whether request logs should be saved to the database (default: {default})."
+            ).format(default=settings.LOG_OUTGOING_REQUESTS_DB_SAVE),
+            "save_body": _(
+                "Whether the body of the request and response should be logged (default: "
+                "{default})."
+            ).format(default=settings.LOG_OUTGOING_REQUESTS_DB_SAVE_BODY),
+        }
+
+
+@admin.register(OutgoingRequestsLogConfig)
+class OutgoingRequestsLogConfigAdmin(SingletonModelAdmin):
+    form = ConfigAdminForm

log_outgoing_requests/compat.py

@@ -0,0 +1,60 @@
+import django
+
+# Taken from djangorestframework, see
+# https://github.com/encode/django-rest-framework/blob/376a5cbbba3f8df9c9db8c03a7c8fa2a6e6c05f4/rest_framework/compat.py#LL156C1-L177C10
+#
+# License:
+#
+# Copyright © 2011-present, [Encode OSS Ltd](https://www.encode.io/).
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# * Neither the name of the copyright holder nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+if django.VERSION >= (4, 2):
+    # Django 4.2+: use the stock parse_header_parameters function
+    # Note: Django 4.1 also has an implementation of parse_header_parameters
+    #       which is slightly different from the one in 4.2, it needs
+    #       the compatibility shim as well.
+    from django.utils.http import parse_header_parameters  # type: ignore
+else:
+    # Django <= 4.1: create a compatibility shim for parse_header_parameters
+    from django.http.multipartparser import parse_header
+
+    def parse_header_parameters(line):
+        # parse_header works with bytes, but parse_header_parameters
+        # works with strings. Call encode to convert the line to bytes.
+        main_value_pair, params = parse_header(line.encode())
+        return main_value_pair, {
+            # parse_header will convert *some* values to string.
+            # parse_header_parameters converts *all* values to string.
+            # Make sure all values are converted by calling decode on
+            # any remaining non-string values.
+            k: v if isinstance(v, str) else v.decode()
+            for k, v in params.items()
+        }
+
+
+__all__ = ["parse_header_parameters"]

log_outgoing_requests/constants.py

@@ -0,0 +1,8 @@
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+
+
+class SaveLogsChoice(models.TextChoices):
+    use_default = "use_default", _("Use default")
+    yes = "yes", _("Yes")
+    no = "no", _("No")

log_outgoing_requests/datastructures.py

@@ -0,0 +1,27 @@
+"""
+Datastructure(s) for use in settings.py
+
+Note: do not place any Django-specific imports in this file, as
+it must be imported in settings.py.
+"""
+
+from dataclasses import dataclass
+from typing import Union
+
+
+@dataclass
+class ContentType:
+    """
+    Data class for keeping track of content types and associated default encodings
+    """
+
+    pattern: str
+    default_encoding: str
+
+
+@dataclass
+class ProcessedBody:
+    allow_saving_to_db: bool
+    content: Union[bytes, str]
+    content_type: str
+    encoding: str

log_outgoing_requests/formatters.py

@@ -1,30 +1,42 @@
 import logging
 import textwrap
 
+from django.conf import settings
+
 
 class HttpFormatter(logging.Formatter):
     def _formatHeaders(self, d):
         return "\n".join(f"{k}: {v}" for k, v in d.items())
 
+    def _formatBody(self, content: str, request_or_response: str) -> str:
+        if settings.LOG_OUTGOING_REQUESTS_EMIT_BODY:
+            return f"\n{request_or_response} body:\n{content}"
+        return ""
+
     def formatMessage(self, record):
         result = super().formatMessage(record)
-        if record.name == "requests":
-            result += textwrap.dedent(
-                """
-                ---------------- request ----------------
-                {req.method} {req.url}
-                {reqhdrs}
 
-                ---------------- response ----------------
-                {res.status_code} {res.reason} {res.url}
-                {reshdrs}
+        if record.name != "requests":
+            return result
 
+        result += textwrap.dedent(
             """
-            ).format(
-                req=record.req,
-                res=record.res,
-                reqhdrs=self._formatHeaders(record.req.headers),
-                reshdrs=self._formatHeaders(record.res.headers),
-            )
+            ---------------- request ----------------
+            {req.method} {req.url}
+            {reqhdrs} {request_body}
+
+            ---------------- response ----------------
+            {res.status_code} {res.reason} {res.url}
+            {reshdrs} {response_body}
+
+        """
+        ).format(
+            req=record.req,
+            res=record.res,
+            reqhdrs=self._formatHeaders(record.req.headers),
+            reshdrs=self._formatHeaders(record.res.headers),
+            request_body=self._formatBody(record.req.body, "Request"),
+            response_body=self._formatBody(record.res.content, "Response"),
+        )
 
         return result