From 03e22b2fff84ccc794b6dbce0d04369a262cd5f6 Mon Sep 17 00:00:00 2001 From: CHT2110 Date: Wed, 24 Jan 2024 16:40:28 +0100 Subject: [PATCH 01/14] adding dotenv to normal dependency block --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9929f31..eb48805 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ requests = "^2.25.1" singer-python = "^5.9.1" sendgrid = "^6.7.0" boto3 = "^1.17.79" +python-dotenv = "^0.15.0" [tool.poetry.dev-dependencies] coverage = "^5.4" @@ -26,7 +27,6 @@ pytest-cov = "^2.11.1" coveralls = "^3.0.0" pytest-mock = "^3.5.1" pytest-vcr = "^1.0.2" -python-dotenv = "^0.15.0" vcrpy = "^4.1.1" black = "^21.5b1" moto = {extras = ["s3"], version = "^2.0.8"} From 3941ef52d561d604e0f9d2b2cbb778a4623941de Mon Sep 17 00:00:00 2001 From: CHT2110 Date: Thu, 25 Jan 2024 12:44:00 +0100 Subject: [PATCH 02/14] group dependency --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index eb48805..a381b7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,9 +17,8 @@ requests = "^2.25.1" singer-python = "^5.9.1" sendgrid = "^6.7.0" boto3 = "^1.17.79" -python-dotenv = "^0.15.0" -[tool.poetry.dev-dependencies] +[tool.poetry.group.dev-dependencies] coverage = "^5.4" pylint = "^2.6.0" pytest = "^6.2.2" @@ -27,6 +26,7 @@ pytest-cov = "^2.11.1" coveralls = "^3.0.0" pytest-mock = "^3.5.1" pytest-vcr = "^1.0.2" +python-dotenv = "^0.15.0" vcrpy = "^4.1.1" black = "^21.5b1" moto = {extras = ["s3"], version = "^2.0.8"} From 120273719d3b1ee688ce8b420ce1c7594c3ed2d7 Mon Sep 17 00:00:00 2001 From: CHT2110 Date: Tue, 6 Feb 2024 12:52:06 +0100 Subject: [PATCH 03/14] remove dotenv from tap --- pyproject.toml | 1 - tap_ask_nicely/utils.py | 3 --- tests/conftest.py | 3 --- 3 files changed, 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index eb48805..963f765 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,6 @@ requests = "^2.25.1" singer-python = "^5.9.1" sendgrid = "^6.7.0" boto3 = "^1.17.79" -python-dotenv = "^0.15.0" [tool.poetry.dev-dependencies] coverage = "^5.4" diff --git a/tap_ask_nicely/utils.py b/tap_ask_nicely/utils.py index 947a702..4591ad7 100644 --- a/tap_ask_nicely/utils.py +++ b/tap_ask_nicely/utils.py @@ -13,9 +13,6 @@ import os import json from datetime import datetime -from dotenv import load_dotenv - -load_dotenv() class SlackMessenger: diff --git a/tests/conftest.py b/tests/conftest.py index cbc795d..833a9b4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,9 +5,6 @@ from tap_ask_nicely.storage import StorageHandler import os -from dotenv import load_dotenv - -load_dotenv() @pytest.fixture From 84a001982908103c8af51cd8e5ba02724dca427c Mon Sep 17 00:00:00 2001 From: CHT2110 Date: Tue, 6 Feb 2024 13:45:10 +0100 Subject: [PATCH 04/14] fixed dev dependency --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index eb48805..bef4f3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ sendgrid = "^6.7.0" boto3 = "^1.17.79" python-dotenv = "^0.15.0" -[tool.poetry.dev-dependencies] +[tool.poetry.dev.dependencies] coverage = "^5.4" pylint = "^2.6.0" pytest = "^6.2.2" From 8118c3662f2882f9b2fcbc7b095cf6b5f810a725 Mon Sep 17 00:00:00 2001 From: CHT2110 Date: Tue, 6 Feb 2024 14:33:45 +0100 Subject: [PATCH 05/14] sync fix .get() --- tap_ask_nicely/sync.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tap_ask_nicely/sync.py b/tap_ask_nicely/sync.py index 0396061..ef25000 100644 --- a/tap_ask_nicely/sync.py +++ b/tap_ask_nicely/sync.py @@ -99,7 +99,7 @@ def sync(config, state, catalog): } # Comment out for local runs - if config["slack_notifications"] == True: + if config.get("slack_notifications"): SlackMessenger.send_message( run_id=run_id, start_time=pipeline_start, @@ -108,6 +108,6 @@ def sync(config, state, catalog): comments='\n'.join(stream_comments), ) - if config["email_notifications"] == True: + if config.get("email_notifications"): sg = SendgridMessenger(notification_data) sg.send_message() From 950343d7f35cee7b1cdf724537789409c8a930c7 Mon Sep 17 00:00:00 2001 From: CHT2110 Date: Tue, 6 Feb 2024 15:47:50 +0100 Subject: [PATCH 06/14] additional fixes --- tap_ask_nicely/schemas/historical_stats.json | 8 +------- tap_ask_nicely/streams.py | 8 ++++---- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/tap_ask_nicely/schemas/historical_stats.json b/tap_ask_nicely/schemas/historical_stats.json index c3c22b1..11465e6 100644 --- a/tap_ask_nicely/schemas/historical_stats.json +++ b/tap_ask_nicely/schemas/historical_stats.json @@ -2,13 +2,7 @@ "type": ["null", "object"], "additionalProperties": false, "properties": { - "year": { - "type": ["null", "string"] - }, - "month": { - "type": ["null", "string"] - }, - "day": { + "date": { "type": ["null", "string"] }, "weekday": { diff --git a/tap_ask_nicely/streams.py b/tap_ask_nicely/streams.py index a8613c1..f2d2123 100644 --- a/tap_ask_nicely/streams.py +++ b/tap_ask_nicely/streams.py @@ -120,7 +120,7 @@ def sync(self) -> Generator[dict, None, None]: class HistoricalStats(Stream): tap_stream_id = "historical_stats" - key_properties = [] + key_properties = ["date"] replication_key = "" object_type = "HISTORICAL_STATS" replication_method = "INCREMENTAL" @@ -140,9 +140,9 @@ def sync(self) -> Generator[dict, None, None]: while start_from != datetime.strftime(datetime.now(), "%Y-%m-%d"): response = self.client.fetch_historical_stats(date=start_from) sent_stats = response["data"] - if sent_stats != []: - for stat in sent_stats: - yield stat + for stat in sent_stats: + stat["date"] = datetime(int(stat.pop("year")), int(stat.pop("month")), int(stat.pop("day"))).strftime("%Y-%m-%d") + yield stat start_from = increment_date_by_day(start_from) singer.write_bookmark( From 5bf06a4b96a6998750851a22652b5b804f236fc1 Mon Sep 17 00:00:00 2001 From: CHT2110 Date: Wed, 7 Feb 2024 11:29:10 +0100 Subject: [PATCH 07/14] fixing unix timestamps --- tap_ask_nicely/streams.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tap_ask_nicely/streams.py b/tap_ask_nicely/streams.py index f2d2123..f939577 100644 --- a/tap_ask_nicely/streams.py +++ b/tap_ask_nicely/streams.py @@ -71,6 +71,11 @@ def sync(self, **kwargs) -> Generator[dict, None, None]: ) records = res.get("data", []) for record in records: + record["sent"] = datetime.fromtimestamp(record["sent"]) + record["opened"] = datetime.fromtimestamp(record["opened"]) + record["responded"] = datetime.fromtimestamp(record["responded"]) + record["lastemailed"] = datetime.fromtimestamp(record["lastemailed"]) + record["created"] = datetime.fromtimestamp(record["created"]) yield record contact_ids.add(record["contact_id"]) page = page + 1 From 9b784e7e0dc483d4e605f2630f451b54692bc5aa Mon Sep 17 00:00:00 2001 From: CHT2110 Date: Wed, 7 Feb 2024 12:53:26 +0100 Subject: [PATCH 08/14] fixing string error --- tap_ask_nicely/streams.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tap_ask_nicely/streams.py b/tap_ask_nicely/streams.py index f939577..602003f 100644 --- a/tap_ask_nicely/streams.py +++ b/tap_ask_nicely/streams.py @@ -71,11 +71,11 @@ def sync(self, **kwargs) -> Generator[dict, None, None]: ) records = res.get("data", []) for record in records: - record["sent"] = datetime.fromtimestamp(record["sent"]) - record["opened"] = datetime.fromtimestamp(record["opened"]) - record["responded"] = datetime.fromtimestamp(record["responded"]) - record["lastemailed"] = datetime.fromtimestamp(record["lastemailed"]) - record["created"] = datetime.fromtimestamp(record["created"]) + record["sent"] = datetime.fromtimestamp(int(record["sent"])) + record["opened"] = datetime.fromtimestamp(int(record["opened"])) + record["responded"] = datetime.fromtimestamp(int(record["responded"])) + record["lastemailed"] = datetime.fromtimestamp(int(record["lastemailed"])) + record["created"] = datetime.fromtimestamp(int(record["created"])) yield record contact_ids.add(record["contact_id"]) page = page + 1 From 430c51a16380836c81273f203e08550d07edceef Mon Sep 17 00:00:00 2001 From: CHT2110 Date: Sun, 18 Feb 2024 22:18:48 +0100 Subject: [PATCH 09/14] adding primary key --- tap_ask_nicely/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tap_ask_nicely/utils.py b/tap_ask_nicely/utils.py index 4591ad7..41196cb 100644 --- a/tap_ask_nicely/utils.py +++ b/tap_ask_nicely/utils.py @@ -67,6 +67,7 @@ def audit_schema() -> dict: "run_time": {"type": ["null", "number"]}, "comments": {"type": ["null", "string"]}, }, + "primary_key": ["run_id"] } return schema_base From 6b8af2d906c3aa3db2bb847e4b1940c5971b9feb Mon Sep 17 00:00:00 2001 From: CHT2110 Date: Mon, 19 Feb 2024 19:17:29 +0100 Subject: [PATCH 10/14] required key --- tap_ask_nicely/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_ask_nicely/utils.py b/tap_ask_nicely/utils.py index 41196cb..1928da1 100644 --- a/tap_ask_nicely/utils.py +++ b/tap_ask_nicely/utils.py @@ -67,7 +67,7 @@ def audit_schema() -> dict: "run_time": {"type": ["null", "number"]}, "comments": {"type": ["null", "string"]}, }, - "primary_key": ["run_id"] + "required": ["run_id"] } return schema_base From 235c669e18c870b44484c8bd38a787543b8abe8d Mon Sep 17 00:00:00 2001 From: CHT2110 Date: Mon, 19 Feb 2024 20:00:28 +0100 Subject: [PATCH 11/14] adding key-properties --- tap_ask_nicely/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tap_ask_nicely/utils.py b/tap_ask_nicely/utils.py index 1928da1..7889fbd 100644 --- a/tap_ask_nicely/utils.py +++ b/tap_ask_nicely/utils.py @@ -109,6 +109,12 @@ def schema_metadata(): "breadcrumb": ["properties", "comments"], "metadata": {"inclusion": "available"}, }, + { + "breadcrumb": [], + "metadata": { + "table-key-properties": ["run_id"] + } + } ] return metadata From 65b04d8f0ebe52a80be7482348e90209e80c60c0 Mon Sep 17 00:00:00 2001 From: CHT2110 Date: Mon, 19 Feb 2024 20:12:29 +0100 Subject: [PATCH 12/14] fixing typo --- tap_ask_nicely/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_ask_nicely/utils.py b/tap_ask_nicely/utils.py index 7889fbd..6ed5b19 100644 --- a/tap_ask_nicely/utils.py +++ b/tap_ask_nicely/utils.py @@ -86,7 +86,7 @@ def schema_metadata(): "metadata": {"inclusion": "available"}, }, { - "breadcrumb": ["properties", "stream_stream"], + "breadcrumb": ["properties", "stream_name"], "metadata": {"inclusion": "available"}, }, { From ca8550a89397920aa346a8d1341726c7c7594ade Mon Sep 17 00:00:00 2001 From: CHT2110 Date: Mon, 19 Feb 2024 20:41:19 +0100 Subject: [PATCH 13/14] fixing key properteis --- tap_ask_nicely/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_ask_nicely/utils.py b/tap_ask_nicely/utils.py index 6ed5b19..ec890a4 100644 --- a/tap_ask_nicely/utils.py +++ b/tap_ask_nicely/utils.py @@ -112,7 +112,7 @@ def schema_metadata(): { "breadcrumb": [], "metadata": { - "table-key-properties": ["run_id"] + "key-properties": ["run_id"] } } ] From 553b1271edb32954db4170cb6627cd22db6a3c95 Mon Sep 17 00:00:00 2001 From: CHT2110 Date: Tue, 20 Feb 2024 13:54:54 +0100 Subject: [PATCH 14/14] removing auditlog --- tap_ask_nicely/sync.py | 19 +----- tap_ask_nicely/utils.py | 141 ++++++---------------------------------- 2 files changed, 20 insertions(+), 140 deletions(-) diff --git a/tap_ask_nicely/sync.py b/tap_ask_nicely/sync.py index ef25000..e9104ae 100644 --- a/tap_ask_nicely/sync.py +++ b/tap_ask_nicely/sync.py @@ -4,7 +4,7 @@ from singer import Transformer, metadata from tap_ask_nicely.client import AskNicelyClient from tap_ask_nicely.streams import STREAMS -from tap_ask_nicely.utils import AuditLogs, SendgridMessenger, SlackMessenger, GmailMessenger +from tap_ask_nicely.utils import SendgridMessenger, SlackMessenger, GmailMessenger from datetime import date, datetime import time @@ -63,27 +63,10 @@ def sync(config, state, catalog): singer.write_state(state, tap_stream_id) batch_stop = datetime.now().strftime("%Y-%m-%d, %H:%M:%S") - AuditLogs.write_audit_log( - run_id=run_id, - stream_name=tap_stream_id, - batch_start=batch_start, - batch_end=batch_stop, - records_synced=record_count, - run_time=(time.perf_counter() - start_time), - ) except Exception as e: stream_comments.append(f"{tap_stream_id.upper}: {e}") batch_stop = datetime.now().strftime("%Y-%m-%d, %H:%M:%S") - AuditLogs.write_audit_log( - run_id=run_id, - stream_name=tap_stream_id, - batch_start=batch_start, - batch_end=batch_stop, - records_synced=record_count, - run_time=(time.perf_counter() - start_time), - comments=e, - ) state = singer.set_currently_syncing(state, None) singer.write_state(state) diff --git a/tap_ask_nicely/utils.py b/tap_ask_nicely/utils.py index ec890a4..4594bdf 100644 --- a/tap_ask_nicely/utils.py +++ b/tap_ask_nicely/utils.py @@ -52,128 +52,6 @@ def send_message( def build_url() -> str: return f'{SlackMessenger.slack_base}/{os.getenv("SLACK_WEBHOOK_ADDRESS")}' - -class AuditLogs: - def audit_schema() -> dict: - schema_base = { - "type": ["null", "object"], - "additionalProperties": False, - "properties": { - "run_id": {"type": ["null", "integer"]}, - "stream_name": {"type": ["null", "string"]}, - "batch_start": {"type": ["null", "string"], "format": "date-time"}, - "batch_end": {"type": ["null", "string"], "format": "date-time"}, - "records_synced": {"type": ["null", "integer"]}, - "run_time": {"type": ["null", "number"]}, - "comments": {"type": ["null", "string"]}, - }, - "required": ["run_id"] - } - return schema_base - - def schema_metadata(): - metadata = [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [], - "forced-replication-method": "FULL_TABLE", - "inclusion": "available", - }, - }, - { - "breadcrumb": ["properties", "run_id"], - "metadata": {"inclusion": "available"}, - }, - { - "breadcrumb": ["properties", "stream_name"], - "metadata": {"inclusion": "available"}, - }, - { - "breadcrumb": ["properties", "batch_start"], - "metadata": {"inclusion": "available"}, - }, - { - "breadcrumb": ["properties", "batch_end"], - "metadata": {"inclusion": "available"}, - }, - { - "breadcrumb": ["properties", "records_synced"], - "metadata": {"inclusion": "available"}, - }, - { - "breadcrumb": ["properties", "run_time"], - "metadata": {"inclusion": "available"}, - }, - { - "breadcrumb": ["properties", "comments"], - "metadata": {"inclusion": "available"}, - }, - { - "breadcrumb": [], - "metadata": { - "key-properties": ["run_id"] - } - } - ] - - return metadata - - def audit_record( - run_id: int, - stream_name: str, - batch_start: str, - run_time: int, - batch_end: str = datetime.now(), - records_synced: int = 0, - comments: str = "", - ) -> dict: - - audit = { - "run_id": run_id, - "stream_name": stream_name, - "batch_start": batch_start, - "batch_end": batch_end, - "records_synced": records_synced, - "run_time": run_time, - "comments": comments, - } - return audit - - def write_audit_log( - run_id: int, - stream_name: str, - batch_start: str, - batch_end: str = datetime.now(), - records_synced: int = 0, - run_time=int, - comments: str = "", - ): - singer.write_schema( - "audit_log", - AuditLogs.audit_schema(), - [], - "", - ) - audit_log = Transformer().transform( - AuditLogs.audit_record( - run_id=run_id, - stream_name=stream_name, - batch_start=batch_start, - batch_end=batch_end, - records_synced=records_synced, - run_time=run_time, - comments=comments, - ), - AuditLogs.audit_schema(), - metadata.to_map(AuditLogs.schema_metadata()), - ) - singer.write_record( - "audit_log", - audit_log, - ) - - class EmailMessenger: def __init__(self, sync_data: dict) -> None: self.data = sync_data @@ -310,3 +188,22 @@ def send_message(self): LOGGER.error(f"There was an issue sending the email: {error}") return f"There was an issue sending the email: {error}" + +def write_audit_log(run_id: int, stream_name: str, batch_start: str, + batch_end: str = datetime.now(), + records_synced: int = 0, run_time=int, + comments: str = ""): + with Counter('audit_log_written') as counter: + with Timer('audit_log_write_time'): + audit_data = { + "run_id": run_id, + "stream_name": stream_name, + "batch_start": batch_start, + "batch_end": batch_end, + "records_synced": records_synced, + "run_time": run_time, + "comments": comments, + } + email_messenger = SendgridMessenger(audit_data) + email_messenger.send_message() + counter.increment()