From 50941729ae29ae340800d3494abca7e084a3f612 Mon Sep 17 00:00:00 2001
From: marcos314 <marcoswesley314@gmail.com>
Date: Thu, 19 Oct 2023 09:45:17 -0300
Subject: [PATCH] chore(tap_suiteql): Add filter to MonthlyRecurringRevenue

---
 meltano.yml                             |  2 ++
 tap_suiteql/client.py                   |  1 +
 tap_suiteql/query_builder.py            |  7 ++++-
 tap_suiteql/streams.py                  |  1 +
 tap_suiteql/tests/test_query_builder.py | 35 +++++++++++++++++++++++++
 5 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/meltano.yml b/meltano.yml
index b51838f..47d38d5 100644
--- a/meltano.yml
+++ b/meltano.yml
@@ -30,3 +30,5 @@ plugins:
   - name: target-jsonl
     variant: andyh1203
     pip_url: target-jsonl
+    config:
+      destination_path: ''
diff --git a/tap_suiteql/client.py b/tap_suiteql/client.py
index b0de044..c38964f 100644
--- a/tap_suiteql/client.py
+++ b/tap_suiteql/client.py
@@ -26,6 +26,7 @@ def __init__(
     rest_method = "POST"
     stream_type = ""
     entity_name = ""
+    year_date_field = ""
 
     @property
     def url_base(self) -> str:
diff --git a/tap_suiteql/query_builder.py b/tap_suiteql/query_builder.py
index 190536e..5de58e3 100644
--- a/tap_suiteql/query_builder.py
+++ b/tap_suiteql/query_builder.py
@@ -36,7 +36,12 @@ def _build_from_statement(self):
     def _build_where_statement(self):
         where_clauses = ["1=1"]
         where_statement = "where "
-
+        
+        if self.stream.year_date_field: # Filter added due to the limit of 100 thousand records that the Sensedata API returns 
+            where_clauses.append(
+                f"TO_CHAR({self.stream.year_date_field}, 'YYYY') >= 2023 and {self.stream.year_date_field} < ADD_MONTHS(SYSDATE, 3)" #noqa:E501
+            )            
+            
         if self.stream.replication_key:
             where_clauses.append(
                 f"{self.stream.replication_key} >= TO_DATE(:{self.stream.replication_key}, 'YYYY-MM-DD\"T\"HH24:MI:SS')"  # noqa:E501
diff --git a/tap_suiteql/streams.py b/tap_suiteql/streams.py
index 031a4c5..6cc885f 100644
--- a/tap_suiteql/streams.py
+++ b/tap_suiteql/streams.py
@@ -264,6 +264,7 @@ class MonthlyRecurringRevenueStream(suiteqlStream):
     name = "MonthlyRecurringRevenue"
     path = "/query/v1/suiteql"
     primary_keys = ["id"]
+    year_date_field = "yearmonth"
     schema = th.PropertiesList(
         th.Property("id", th.StringType),
         th.Property("newchurnrevenue", th.StringType),
diff --git a/tap_suiteql/tests/test_query_builder.py b/tap_suiteql/tests/test_query_builder.py
index 10631e0..1007a41 100644
--- a/tap_suiteql/tests/test_query_builder.py
+++ b/tap_suiteql/tests/test_query_builder.py
@@ -4,6 +4,7 @@
 class DummyStream:
     name = "dummy"
     entity_name = ""
+    year_date_field = None
     primary_keys = ["col_id"]
     replication_key = "replication_key_col"
     skip_attributes = []
@@ -23,6 +24,7 @@ class DummyStream:
 class DummyStreamWithoutReplicationKey:
     name = "dummy_without_replication_key"
     entity_name = ""
+    year_date_field = None
     primary_keys = ["col_id"]
     skip_attributes = []
     replication_key = None
@@ -41,6 +43,7 @@ class DummyStreamWithoutReplicationKey:
 class DummyStreamWithoutPrimaryKeys:
     name = "dummy_without_primary_keys"
     entity_name = ""
+    year_date_field = None
     skip_attributes = []
     primary_keys = None
     replication_key = None
@@ -58,6 +61,7 @@ class DummyStreamWithoutPrimaryKeys:
 class DummyStreamTransaction:
     name = "dummy"
     entity_name = "dummy_transaction"
+    year_date_field = None
     primary_keys = ["col_id"]
     replication_key = "replication_key_col"
     stream_type = "CustDummy"
@@ -74,6 +78,26 @@ class DummyStreamTransaction:
     }
 
 
+class DummyStreamWithFilter:
+    name = "dummy_with_stream"
+    entity_name = ""
+    primary_keys = ["col_id"]
+    year_date_field = "year_date_field"
+    replication_key = None
+    skip_attributes = []
+    stream_type = None
+    schema = {
+        "type": "object",
+        "properties": {
+            "col_id": {},
+            "col_1": {},
+            "col_2": {},
+            "year_date_field": {"format": "date-time"},            
+        },
+    }
+
+
+
 def test_sql_builder_with_replication_key():
     expected = """select col_id,col_1,col_2,TO_CHAR(date_col, 'YYYY-MM-DD\"T\"HH24:MI:SS') date_col,TO_CHAR(replication_key_col, 'YYYY-MM-DD\"T\"HH24:MI:SS') replication_key_col
             from dummy
@@ -111,3 +135,14 @@ def test_sql_builder_from_transaction():
     query = QueryBuilder(DummyStreamTransaction).query()
 
     assert expected == query
+
+
+def test_sql_builder_with_filter():
+    expected = """select col_id,col_1,col_2,TO_CHAR(year_date_field, 'YYYY-MM-DD\"T\"HH24:MI:SS') year_date_field
+            from dummy_with_stream
+            where 1=1 and TO_CHAR(year_date_field, 'YYYY') >= 2023 and year_date_field < ADD_MONTHS(SYSDATE, 3)
+            order by col_id"""  # noqa:E501
+
+    query = QueryBuilder(DummyStreamWithFilter).query()
+    print(query)
+    assert expected == query