From c4b411f72d17cf8e357f4784bc86f316e3d2e195 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 11 Sep 2024 16:27:42 -0400 Subject: [PATCH 1/8] changelog --- .changes/unreleased/Under the Hood-20240911-162730.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20240911-162730.yaml diff --git a/.changes/unreleased/Under the Hood-20240911-162730.yaml b/.changes/unreleased/Under the Hood-20240911-162730.yaml new file mode 100644 index 00000000000..0d35aeb5262 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240911-162730.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add Snowplow tracking for behavior flag deprecations +time: 2024-09-11T16:27:30.293832-04:00 +custom: + Author: mikealfare + Issue: "10552" From 407001cf6682adea15a4cc41c8a9e4acfc34f243 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Thu, 12 Sep 2024 10:06:47 -0400 Subject: [PATCH 2/8] add behavior deprecation snowplow callback --- core/dbt/tracking.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/core/dbt/tracking.py b/core/dbt/tracking.py index 880243e4d6e..711f5d03551 100644 --- a/core/dbt/tracking.py +++ b/core/dbt/tracking.py @@ -25,6 +25,7 @@ SendingEvent, TrackingInitializeFailure, ) +from dbt_common.events.base_types import EventMsg from dbt_common.events.functions import fire_event, get_invocation_id from dbt_common.exceptions import NotImplementedError @@ -36,6 +37,7 @@ ADAPTER_INFO_SPEC = "iglu:com.dbt/adapter_info/jsonschema/1-0-1" DEPRECATION_WARN_SPEC = "iglu:com.dbt/deprecation_warn/jsonschema/1-0-0" +BEHAVIOR_DEPRECATION_WARN_SPEC = "iglu:com.dbt/behavior_deprecation_warn/jsonschema/1-0-0" EXPERIMENTAL_PARSER = "iglu:com.dbt/experimental_parser/jsonschema/1-0-0" INVOCATION_ENV_SPEC = "iglu:com.dbt/invocation_env/jsonschema/1-0-0" INVOCATION_SPEC = "iglu:com.dbt/invocation/jsonschema/1-0-2" @@ -364,6 +366,34 @@ def track_deprecation_warn(options): ) +def track_behavior_deprecation_warn(msg: EventMsg) -> None: + if msg.info.name != "BehaviorDeprecationEvent": + return + + if active_user is None: + # cannot track deprecation warnings when active user is None + return + + data = { + "flag_name": getattr(msg.data, "flag_name"), + "flag_source": getattr(msg.data, "flag_source"), + "deprecation_version": getattr(msg.data, "deprecation_version", ""), + "deprecation_message": getattr(msg.data, "deprecation_message", ""), + "docs_url": getattr(msg.data, "docs_url", ""), + } + + context = [SelfDescribingJson(BEHAVIOR_DEPRECATION_WARN_SPEC, data)] + + track( + active_user, + category="dbt", + action="behavior_deprecation", + label=get_invocation_id(), + property_=getattr(msg.data, "flag_name"), + context=context, + ) + + def track_invocation_end(invocation_context, result_type=None): data = {"progress": "end", "result_type": result_type, "result": None} data.update(invocation_context) From b3bf1a129a83febf08cc2c61d0d6eca319aef6de Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 16 Sep 2024 14:08:27 -0400 Subject: [PATCH 3/8] add behavior deprecation snowplow callback --- core/dbt/events/logging.py | 4 ++++ core/dbt/tracking.py | 13 ++----------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/core/dbt/events/logging.py b/core/dbt/events/logging.py index f0bef3ae497..8c0a5119c22 100644 --- a/core/dbt/events/logging.py +++ b/core/dbt/events/logging.py @@ -2,8 +2,10 @@ from functools import partial from typing import Callable, List +from dbt.tracking import track_behavior_deprecation_warn from dbt_common.events.base_types import EventLevel, EventMsg from dbt_common.events.event_manager_client import ( + add_callback_to_manager, add_logger_to_manager, cleanup_event_logger, get_event_manager, @@ -89,6 +91,8 @@ def setup_event_logger(flags, callbacks: List[Callable[[EventMsg], None]] = []) console_config.output_stream = get_capture_stream() add_logger_to_manager(console_config) + add_callback_to_manager(track_behavior_deprecation_warn) + if flags.LOG_LEVEL_FILE != "none": # create and add the file logger to the event manager log_file = os.path.join(flags.LOG_PATH, "dbt.log") diff --git a/core/dbt/tracking.py b/core/dbt/tracking.py index 711f5d03551..1bfa9036bb0 100644 --- a/core/dbt/tracking.py +++ b/core/dbt/tracking.py @@ -26,7 +26,7 @@ TrackingInitializeFailure, ) from dbt_common.events.base_types import EventMsg -from dbt_common.events.functions import fire_event, get_invocation_id +from dbt_common.events.functions import fire_event, get_invocation_id, msg_to_dict from dbt_common.exceptions import NotImplementedError sp_logger.setLevel(100) @@ -374,22 +374,13 @@ def track_behavior_deprecation_warn(msg: EventMsg) -> None: # cannot track deprecation warnings when active user is None return - data = { - "flag_name": getattr(msg.data, "flag_name"), - "flag_source": getattr(msg.data, "flag_source"), - "deprecation_version": getattr(msg.data, "deprecation_version", ""), - "deprecation_message": getattr(msg.data, "deprecation_message", ""), - "docs_url": getattr(msg.data, "docs_url", ""), - } - - context = [SelfDescribingJson(BEHAVIOR_DEPRECATION_WARN_SPEC, data)] + context = [SelfDescribingJson(BEHAVIOR_DEPRECATION_WARN_SPEC, msg_to_dict(msg))] track( active_user, category="dbt", action="behavior_deprecation", label=get_invocation_id(), - property_=getattr(msg.data, "flag_name"), context=context, ) From ec049a92183187e42b6ed220f7f26ece9858fc20 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 16 Sep 2024 19:17:07 -0400 Subject: [PATCH 4/8] add testing --- core/dbt/tracking.py | 2 +- tests/unit/test_behavior_flags.py | 59 +++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_behavior_flags.py diff --git a/core/dbt/tracking.py b/core/dbt/tracking.py index 1bfa9036bb0..fcae7d86028 100644 --- a/core/dbt/tracking.py +++ b/core/dbt/tracking.py @@ -379,7 +379,7 @@ def track_behavior_deprecation_warn(msg: EventMsg) -> None: track( active_user, category="dbt", - action="behavior_deprecation", + action=msg.info.name, label=get_invocation_id(), context=context, ) diff --git a/tests/unit/test_behavior_flags.py b/tests/unit/test_behavior_flags.py new file mode 100644 index 00000000000..0188480ce02 --- /dev/null +++ b/tests/unit/test_behavior_flags.py @@ -0,0 +1,59 @@ +import pytest + +from dbt.tracking import ( + disable_tracking, + initialize_from_flags, + track_behavior_deprecation_warn, +) +from dbt_common.behavior_flags import Behavior +from dbt_common.events.event_manager_client import ( + add_callback_to_manager, + cleanup_event_logger, +) + + +@pytest.fixture +def snowplow_tracker(mocker): + # initialize `active_user` without writing the cookie to disk + initialize_from_flags(True, "") + mocker.patch("dbt.tracking.User.set_cookie").return_value = {"id": 42} + + # add the relevant callback to the event manager + add_callback_to_manager(track_behavior_deprecation_warn) + + # don't make a call, catch the request + snowplow_tracker = mocker.patch("dbt.tracking.tracker.track_struct_event") + + yield snowplow_tracker + + # teardown + cleanup_event_logger() + disable_tracking() + + +def test_false_evaluation_triggers_snowplow_tracking(snowplow_tracker): + behavior = Behavior([{"name": "my_flag", "default": False}], {}) + if behavior.my_flag: + # trigger a False evaluation + assert False, "This flag should evaluate to false and skip this line" + assert snowplow_tracker.called + + +def test_true_evaluation_does_not_trigger_snowplow_tracking(snowplow_tracker): + behavior = Behavior([{"name": "my_flag", "default": True}], {}) + if behavior.my_flag: + pass + else: + # trigger a True evaluation + assert False, "This flag should evaluate to false and skip this line" + assert not snowplow_tracker.called + + +def test_false_evaluation_does_not_trigger_snowplow_tracking_when_disabled(snowplow_tracker): + disable_tracking() + + behavior = Behavior([{"name": "my_flag", "default": False}], {}) + if behavior.my_flag: + # trigger a False evaluation + assert False, "This flag should evaluate to false and skip this line" + assert not snowplow_tracker.called From f49ae26c15bf97578e55e0490406e3ffd778e06c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 16 Sep 2024 19:46:15 -0400 Subject: [PATCH 5/8] add testing --- core/dbt/events/logging.py | 3 +-- core/dbt/tracking.py | 7 +------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/core/dbt/events/logging.py b/core/dbt/events/logging.py index 8c0a5119c22..e12ff2d5e4e 100644 --- a/core/dbt/events/logging.py +++ b/core/dbt/events/logging.py @@ -70,6 +70,7 @@ def setup_event_logger(flags, callbacks: List[Callable[[EventMsg], None]] = []) make_log_dir_if_missing(flags.LOG_PATH) event_manager = get_event_manager() event_manager.callbacks = callbacks.copy() + add_callback_to_manager(track_behavior_deprecation_warn) if flags.LOG_LEVEL != "none": line_format = _line_format_from_str(flags.LOG_FORMAT, LineFormat.PlainText) @@ -91,8 +92,6 @@ def setup_event_logger(flags, callbacks: List[Callable[[EventMsg], None]] = []) console_config.output_stream = get_capture_stream() add_logger_to_manager(console_config) - add_callback_to_manager(track_behavior_deprecation_warn) - if flags.LOG_LEVEL_FILE != "none": # create and add the file logger to the event manager log_file = os.path.join(flags.LOG_PATH, "dbt.log") diff --git a/core/dbt/tracking.py b/core/dbt/tracking.py index fcae7d86028..c2a53bd4b46 100644 --- a/core/dbt/tracking.py +++ b/core/dbt/tracking.py @@ -367,15 +367,10 @@ def track_deprecation_warn(options): def track_behavior_deprecation_warn(msg: EventMsg) -> None: - if msg.info.name != "BehaviorDeprecationEvent": - return - - if active_user is None: - # cannot track deprecation warnings when active user is None + if msg.info.name != "BehaviorDeprecationEvent" or active_user is None: return context = [SelfDescribingJson(BEHAVIOR_DEPRECATION_WARN_SPEC, msg_to_dict(msg))] - track( active_user, category="dbt", From 12f6436df24c135da48bb000989c16ba41c9c218 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Mon, 16 Sep 2024 20:10:35 -0400 Subject: [PATCH 6/8] update tests for new callback --- tests/unit/events/test_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/events/test_logging.py b/tests/unit/events/test_logging.py index 00284ecab78..b2077b64793 100644 --- a/tests/unit/events/test_logging.py +++ b/tests/unit/events/test_logging.py @@ -23,7 +23,7 @@ def test_clears_preexisting_event_manager_state(self) -> None: setup_event_logger(get_flags()) assert len(manager.loggers) == 0 - assert len(manager.callbacks) == 0 + assert len(manager.callbacks) == 1 # snowplow tracker for behavior flags def test_specify_max_bytes( self, From 267808774f2ae22a7ea2e3fe23cce9e6e79f13fb Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 18 Sep 2024 12:56:20 -0400 Subject: [PATCH 7/8] update names to reflect the new event name in dbt-common --- core/dbt/events/logging.py | 4 ++-- core/dbt/tracking.py | 8 ++++---- tests/unit/test_behavior_flags.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/dbt/events/logging.py b/core/dbt/events/logging.py index e12ff2d5e4e..b0db1003e72 100644 --- a/core/dbt/events/logging.py +++ b/core/dbt/events/logging.py @@ -2,7 +2,7 @@ from functools import partial from typing import Callable, List -from dbt.tracking import track_behavior_deprecation_warn +from dbt.tracking import track_behavior_change_warn from dbt_common.events.base_types import EventLevel, EventMsg from dbt_common.events.event_manager_client import ( add_callback_to_manager, @@ -70,7 +70,7 @@ def setup_event_logger(flags, callbacks: List[Callable[[EventMsg], None]] = []) make_log_dir_if_missing(flags.LOG_PATH) event_manager = get_event_manager() event_manager.callbacks = callbacks.copy() - add_callback_to_manager(track_behavior_deprecation_warn) + add_callback_to_manager(track_behavior_change_warn) if flags.LOG_LEVEL != "none": line_format = _line_format_from_str(flags.LOG_FORMAT, LineFormat.PlainText) diff --git a/core/dbt/tracking.py b/core/dbt/tracking.py index c2a53bd4b46..0358b29f020 100644 --- a/core/dbt/tracking.py +++ b/core/dbt/tracking.py @@ -37,7 +37,7 @@ ADAPTER_INFO_SPEC = "iglu:com.dbt/adapter_info/jsonschema/1-0-1" DEPRECATION_WARN_SPEC = "iglu:com.dbt/deprecation_warn/jsonschema/1-0-0" -BEHAVIOR_DEPRECATION_WARN_SPEC = "iglu:com.dbt/behavior_deprecation_warn/jsonschema/1-0-0" +BEHAVIOR_CHANGE_WARN_SPEC = "iglu:com.dbt/behavior_change_warn/jsonschema/1-0-0" EXPERIMENTAL_PARSER = "iglu:com.dbt/experimental_parser/jsonschema/1-0-0" INVOCATION_ENV_SPEC = "iglu:com.dbt/invocation_env/jsonschema/1-0-0" INVOCATION_SPEC = "iglu:com.dbt/invocation/jsonschema/1-0-2" @@ -366,11 +366,11 @@ def track_deprecation_warn(options): ) -def track_behavior_deprecation_warn(msg: EventMsg) -> None: - if msg.info.name != "BehaviorDeprecationEvent" or active_user is None: +def track_behavior_change_warn(msg: EventMsg) -> None: + if msg.info.name != "BehaviorChangeEvent" or active_user is None: return - context = [SelfDescribingJson(BEHAVIOR_DEPRECATION_WARN_SPEC, msg_to_dict(msg))] + context = [SelfDescribingJson(BEHAVIOR_CHANGE_WARN_SPEC, msg_to_dict(msg))] track( active_user, category="dbt", diff --git a/tests/unit/test_behavior_flags.py b/tests/unit/test_behavior_flags.py index 0188480ce02..c84ff4e4540 100644 --- a/tests/unit/test_behavior_flags.py +++ b/tests/unit/test_behavior_flags.py @@ -3,7 +3,7 @@ from dbt.tracking import ( disable_tracking, initialize_from_flags, - track_behavior_deprecation_warn, + track_behavior_change_warn, ) from dbt_common.behavior_flags import Behavior from dbt_common.events.event_manager_client import ( @@ -19,7 +19,7 @@ def snowplow_tracker(mocker): mocker.patch("dbt.tracking.User.set_cookie").return_value = {"id": 42} # add the relevant callback to the event manager - add_callback_to_manager(track_behavior_deprecation_warn) + add_callback_to_manager(track_behavior_change_warn) # don't make a call, catch the request snowplow_tracker = mocker.patch("dbt.tracking.tracker.track_struct_event") From 1e20bd8c027bded2baa335001288e644042a7e64 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 18 Sep 2024 14:20:48 -0400 Subject: [PATCH 8/8] update test input with the new required field --- tests/unit/test_behavior_flags.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_behavior_flags.py b/tests/unit/test_behavior_flags.py index c84ff4e4540..6f4746c438c 100644 --- a/tests/unit/test_behavior_flags.py +++ b/tests/unit/test_behavior_flags.py @@ -32,7 +32,9 @@ def snowplow_tracker(mocker): def test_false_evaluation_triggers_snowplow_tracking(snowplow_tracker): - behavior = Behavior([{"name": "my_flag", "default": False}], {}) + behavior = Behavior( + [{"name": "my_flag", "default": False, "description": "This is a false flag."}], {} + ) if behavior.my_flag: # trigger a False evaluation assert False, "This flag should evaluate to false and skip this line" @@ -40,7 +42,9 @@ def test_false_evaluation_triggers_snowplow_tracking(snowplow_tracker): def test_true_evaluation_does_not_trigger_snowplow_tracking(snowplow_tracker): - behavior = Behavior([{"name": "my_flag", "default": True}], {}) + behavior = Behavior( + [{"name": "my_flag", "default": True, "description": "This is a true flag."}], {} + ) if behavior.my_flag: pass else: @@ -52,7 +56,9 @@ def test_true_evaluation_does_not_trigger_snowplow_tracking(snowplow_tracker): def test_false_evaluation_does_not_trigger_snowplow_tracking_when_disabled(snowplow_tracker): disable_tracking() - behavior = Behavior([{"name": "my_flag", "default": False}], {}) + behavior = Behavior( + [{"name": "my_flag", "default": False, "description": "This is a false flag."}], {} + ) if behavior.my_flag: # trigger a False evaluation assert False, "This flag should evaluate to false and skip this line"