From 8b6931352f9b5a0b5d00957cb70a2be5a0ae3dca Mon Sep 17 00:00:00 2001 From: Nick Doornekamp Date: Fri, 21 Jul 2023 16:44:41 +0200 Subject: [PATCH 1/4] Move exception stack trace to ECS-compliant field --- ecs_logging/_structlog.py | 8 ++++++++ tests/test_structlog_formatter.py | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/ecs_logging/_structlog.py b/ecs_logging/_structlog.py index a34e2c9..70a83ca 100644 --- a/ecs_logging/_structlog.py +++ b/ecs_logging/_structlog.py @@ -47,5 +47,13 @@ def format_to_ecs(self, event_dict): )[:-3] + "Z" ) + + if "exception" in event_dict: + stack_trace = event_dict.pop("exception") + if "error" in event_dict: + event_dict["error"]["stack_trace"] = stack_trace + else: + event_dict["error"] = {"stack_trace": stack_trace} + event_dict.setdefault("ecs", {}).setdefault("version", ECS_VERSION) return event_dict diff --git a/tests/test_structlog_formatter.py b/tests/test_structlog_formatter.py index 75b6f22..a6311c6 100644 --- a/tests/test_structlog_formatter.py +++ b/tests/test_structlog_formatter.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import json import ecs_logging import structlog from unittest import mock @@ -37,6 +38,15 @@ def make_event_dict(): } +def event_dict_with_exception(): + return { + "event": "test message", + "log.logger": "logger-name", + "foo": "bar", + "exception": "", + } + + def test_conflicting_event_dict(): formatter = ecs_logging.StructlogFormatter() event_dict = make_event_dict() @@ -80,3 +90,16 @@ def test_can_be_set_as_processor(time, spec_validator): '"message":"test message","custom":"key","dot":{"ted":1},' '"ecs":{"version":"1.6.0"}}\n' ) + + +@mock.patch("time.time") +def test_exception_log_is_ecs_compliant_when_used_with_format_exc_info(time): + time.return_value = 1584720997.187709 + + formatter = ecs_logging.StructlogFormatter() + formatted_event_dict = json.loads(formatter(None, "debug", event_dict_with_exception())) + + assert "exception" not in formatted_event_dict, "The key 'exception' at the root of a log is not ECS-compliant" + assert "error" in formatted_event_dict + assert "stack_trace" in formatted_event_dict["error"] + assert "" in formatted_event_dict["error"]["stack_trace"] From 752216593a4a4815a0beabe177ca3d30394bf00d Mon Sep 17 00:00:00 2001 From: Nick Doornekamp Date: Fri, 21 Jul 2023 16:48:50 +0200 Subject: [PATCH 2/4] refactor: patching time is not needed --- tests/test_structlog_formatter.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_structlog_formatter.py b/tests/test_structlog_formatter.py index a6311c6..d9251bb 100644 --- a/tests/test_structlog_formatter.py +++ b/tests/test_structlog_formatter.py @@ -92,10 +92,7 @@ def test_can_be_set_as_processor(time, spec_validator): ) -@mock.patch("time.time") -def test_exception_log_is_ecs_compliant_when_used_with_format_exc_info(time): - time.return_value = 1584720997.187709 - +def test_exception_log_is_ecs_compliant_when_used_with_format_exc_info(): formatter = ecs_logging.StructlogFormatter() formatted_event_dict = json.loads(formatter(None, "debug", event_dict_with_exception())) From 0077e44e482413679fe8afce223fd33cbe653ab4 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Mon, 24 Jul 2023 11:57:53 -0600 Subject: [PATCH 3/4] Move event dicts to fixtures --- tests/test_structlog_formatter.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/tests/test_structlog_formatter.py b/tests/test_structlog_formatter.py index d9251bb..3338a4c 100644 --- a/tests/test_structlog_formatter.py +++ b/tests/test_structlog_formatter.py @@ -16,12 +16,13 @@ # under the License. import json -import ecs_logging -import structlog -from unittest import mock from io import StringIO +from unittest import mock import pytest +import structlog + +import ecs_logging class NotSerializable: @@ -29,7 +30,8 @@ def __repr__(self): return "" -def make_event_dict(): +@pytest.fixture +def event_dict(): return { "event": "test message", "log.logger": "logger-name", @@ -38,6 +40,7 @@ def make_event_dict(): } +@pytest.fixture def event_dict_with_exception(): return { "event": "test message", @@ -47,20 +50,19 @@ def event_dict_with_exception(): } -def test_conflicting_event_dict(): +def test_conflicting_event_dict(event_dict): formatter = ecs_logging.StructlogFormatter() - event_dict = make_event_dict() event_dict["foo.bar"] = "baz" with pytest.raises(TypeError): formatter(None, "debug", event_dict) @mock.patch("time.time") -def test_event_dict_formatted(time, spec_validator): +def test_event_dict_formatted(time, spec_validator, event_dict): time.return_value = 1584720997.187709 formatter = ecs_logging.StructlogFormatter() - assert spec_validator(formatter(None, "debug", make_event_dict())) == ( + assert spec_validator(formatter(None, "debug", event_dict)) == ( '{"@timestamp":"2020-03-20T16:16:37.187Z","log.level":"debug",' '"message":"test message",' '"baz":"",' @@ -92,11 +94,17 @@ def test_can_be_set_as_processor(time, spec_validator): ) -def test_exception_log_is_ecs_compliant_when_used_with_format_exc_info(): +def test_exception_log_is_ecs_compliant_when_used_with_format_exc_info( + event_dict_with_exception, +): formatter = ecs_logging.StructlogFormatter() - formatted_event_dict = json.loads(formatter(None, "debug", event_dict_with_exception())) + formatted_event_dict = json.loads( + formatter(None, "debug", event_dict_with_exception) + ) - assert "exception" not in formatted_event_dict, "The key 'exception' at the root of a log is not ECS-compliant" + assert ( + "exception" not in formatted_event_dict + ), "The key 'exception' at the root of a log is not ECS-compliant" assert "error" in formatted_event_dict assert "stack_trace" in formatted_event_dict["error"] assert "" in formatted_event_dict["error"]["stack_trace"] From 9804698a30a0eed377089df653aa75437c88ff08 Mon Sep 17 00:00:00 2001 From: Colton Myers Date: Mon, 24 Jul 2023 11:59:39 -0600 Subject: [PATCH 4/4] CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9348d7f..e700f85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 2.1.0 (unreleased) - Add support for `service.environment` from APM log correlation ([#96](https://github.com/elastic/ecs-logging-python/pull/96)) +- Fix stack trace handling in StructLog for ECS compliance ([#97](https://github.com/elastic/ecs-logging-python/pull/97)) ## 2.0.2 (2023-05-17)