diff --git a/.streamlit_config b/.streamlit_config deleted file mode 100644 index 1a6c4fc..0000000 --- a/.streamlit_config +++ /dev/null @@ -1,2 +0,0 @@ -export primaryColor="#86acff" -export STREAMLIT_SERVER_PORT=80 diff --git a/app/ETL/EL.py b/app/ETL/EL.py index a7d29c9..35fc4b5 100644 --- a/app/ETL/EL.py +++ b/app/ETL/EL.py @@ -9,9 +9,11 @@ import duckdb import pandas as pd from helper.source_env import dotenv_path,raw_path -# import requests -# requests_session = requests.session() +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s' +) cache_path=os.path.join(dotenv_path,'.token-oauth') @@ -21,37 +23,54 @@ password=environ.get('password') redirect_uri=environ.get('redirect_uri') - tasks_file_path = os.path.join(raw_path,'tasks.json') lists_file_path = os.path.join(raw_path,'lists.json') folders_file_path = os.path.join(raw_path,'folders.json') - - -# import pdb; pdb.set_trace() - -auth_client = OAuth2(client_id=client_id, - client_secret=client_secret, - redirect_uri=redirect_uri, - cache_path=cache_path - ) - - - -client = TickTickClient( - username, password, - auth_client) - - - default_start = datetime(2022, 7, 23,tzinfo=timezone.utc) date_format = '%Y-%m-%dT%H:%M:%S.%f%z' -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s' -) +class TickTickClientWrapper: + """ + wrapper so that the ticktick client class wont be triggered by arbitrary server events like dagster webserver checks. + goal : create a single client instance that all funcs subscribes to. + """ + _instance = None + + + # this __new__ func is called whenever this wrapper class is called. + def __new__(cls,client_id,client_secret,username,password,redirect_uri,cache_path): + if not cls._instance: + cls._instance = super(TickTickClientWrapper, cls).__new__(cls) + cls._instance.client_id = client_id + cls._instance.client_secret = client_secret + cls._instance.username = username + cls._instance.password = password + cls._instance.redirect_uri = redirect_uri + cls._instance.cache_path = cache_path + cls._instance.client = None + return cls._instance + # this makes sure only a single instance of the class is used across the module whichever func calls this class. + # the _instance attribute now holds all the suceeding function's output. + + + def _initialize_client(self): + auth_client = OAuth2(client_id=self.client_id, + client_secret=self.client_secret, + redirect_uri=self.redirect_uri, + cache_path=self.cache_path + ) + self.client = TickTickClient(self.username, self.password, auth_client) + + def get_client(self): + if self.client is None: + self._initialize_client() + return self.client + +# one time call the client instance. +wrapper = TickTickClientWrapper(client_id,client_secret,username,password,redirect_uri,cache_path) + def deduplicate(source) -> list: """ @@ -82,6 +101,8 @@ def _get_completed_tasks(start=None, end=datetime.now(timezone.utc), full_load=T completed_tasks=[] logging.info('start loading tasks') + client=wrapper.get_client() + if full_load: current_date=default_start elif not full_load: @@ -128,6 +149,7 @@ def get_completed_task() -> list: def get_new_tasks() -> list: + client = wrapper.get_client() new_tasks=client.state['tasks'] return new_tasks @@ -143,11 +165,13 @@ def get_all_tasks(context: AssetExecutionContext) -> list: @asset(compute_kind='Python') def get_lists(): + client = wrapper.get_client() lists = client.state['projects'] return lists @asset(compute_kind='Python') def get_folders(): + client = wrapper.get_client() folders = client.state['project_folders'] return folders diff --git a/app/ETL/raw/tasks.json b/app/ETL/raw/tasks.json index 3c7f789..9bab9d1 100644 --- a/app/ETL/raw/tasks.json +++ b/app/ETL/raw/tasks.json @@ -1904,6 +1904,247 @@ ], "kind": "TEXT" }, + { + "id": "65d216d15250520f0bd97110", + "projectId": "inbox120295392", + "sortOrder": 6917693059109748488, + "title": "learn nesting in pandas like json", + "timeZone": "Asia/Ho_Chi_Minh", + "isFloating": false, + "isAllDay": false, + "reminder": "", + "reminders": [], + "exDate": [], + "repeatTaskId": "65d216d15250520f0bd97110", + "priority": 0, + "status": 0, + "items": [], + "modifiedTime": "2024-02-18T14:40:17.000+0000", + "etag": "bve4l1be", + "deleted": 0, + "createdTime": "2024-02-18T14:40:17.000+0000", + "creator": 120295392, + "repeatFrom": "2", + "tags": [ + "clarifyme" + ], + "attachments": [], + "commentCount": 0, + "focusSummaries": [], + "columnId": "64c626b19b3279687b62fd86", + "kind": "TEXT" + }, + { + "id": "65d25aafbc36518b442ca9f4", + "projectId": "inbox120295392", + "sortOrder": 6917693059110797064, + "title": "be real good to run the recurring report now", + "content": "", + "desc": "", + "timeZone": "America/Guayaquil", + "isFloating": false, + "isAllDay": false, + "reminder": "", + "reminders": [], + "exDate": [], + "priority": 0, + "status": 0, + "items": [], + "progress": 0, + "modifiedTime": "2024-02-18T19:29:51.000+0000", + "etag": "ydgex3n5", + "deleted": 0, + "createdTime": "2024-02-18T19:29:51.000+0000", + "creator": 120295392, + "tags": [ + "clarifyme" + ], + "attachments": [], + "commentCount": 0, + "focusSummaries": [], + "columnId": "64c626b19b3279687b62fd86", + "kind": "TEXT" + }, + { + "id": "65d2e2025250520f0bd971ef", + "projectId": "inbox120295392", + "sortOrder": 6917693059110862600, + "title": "cool what the dagster class lifecycle", + "timeZone": "Asia/Ho_Chi_Minh", + "isFloating": false, + "isAllDay": false, + "reminder": "", + "reminders": [], + "exDate": [], + "repeatTaskId": "65d2e2025250520f0bd971ef", + "priority": 0, + "status": 0, + "items": [], + "modifiedTime": "2024-02-19T05:07:14.000+0000", + "etag": "1vzawtwz", + "deleted": 0, + "createdTime": "2024-02-19T05:07:14.000+0000", + "creator": 120295392, + "repeatFrom": "2", + "tags": [ + "clarifyme" + ], + "attachments": [], + "commentCount": 0, + "focusSummaries": [], + "columnId": "64c626b19b3279687b62fd86", + "kind": "TEXT" + }, + { + "id": "65d2e4bd5250520f0bd97241", + "projectId": "inbox120295392", + "sortOrder": 6917693059110928136, + "title": "learn how hot meege rebase work then next action on conflict", + "timeZone": "Asia/Ho_Chi_Minh", + "isFloating": false, + "isAllDay": false, + "reminder": "", + "reminders": [], + "exDate": [], + "repeatTaskId": "65d2e4bd5250520f0bd97241", + "priority": 0, + "status": 0, + "items": [], + "modifiedTime": "2024-02-19T05:18:53.000+0000", + "etag": "aeqq5oe3", + "deleted": 0, + "createdTime": "2024-02-19T05:18:53.000+0000", + "creator": 120295392, + "repeatFrom": "2", + "tags": [ + "clarifyme" + ], + "attachments": [], + "commentCount": 0, + "focusSummaries": [], + "columnId": "64c626b19b3279687b62fd86", + "kind": "TEXT" + }, + { + "id": "65d2ea158f08b8edcc076872", + "projectId": "inbox120295392", + "sortOrder": -9152126610516316168, + "title": " implement a sign out func(ClarifyToday)", + "content": "", + "desc": "", + "timeZone": "America/Guayaquil", + "isFloating": false, + "isAllDay": false, + "reminder": "", + "reminders": [], + "exDate": [], + "priority": 0, + "status": 0, + "items": [], + "progress": 0, + "modifiedTime": "2024-02-19T05:44:25.000+0000", + "etag": "2b8rtgkf", + "deleted": 0, + "createdTime": "2024-02-19T05:41:41.247+0000", + "creator": 120295392, + "tags": [ + "clarifyme" + ], + "focusSummaries": [], + "kind": "TEXT" + }, + { + "id": "65d2ea135250520f0bd97253", + "projectId": "inbox120295392", + "sortOrder": 6917693059110993672, + "title": "a pie chart for task to be clarified target", + "timeZone": "Asia/Ho_Chi_Minh", + "isFloating": false, + "isAllDay": false, + "reminder": "", + "reminders": [], + "exDate": [], + "repeatTaskId": "65d2ea135250520f0bd97253", + "priority": 0, + "status": 0, + "items": [], + "modifiedTime": "2024-02-19T05:41:39.000+0000", + "etag": "8qegyq7o", + "deleted": 0, + "createdTime": "2024-02-19T05:41:39.000+0000", + "creator": 120295392, + "repeatFrom": "2", + "tags": [ + "clarifyme" + ], + "attachments": [], + "commentCount": 0, + "focusSummaries": [], + "columnId": "64c626b19b3279687b62fd86", + "kind": "TEXT" + }, + { + "id": "65d2eb284c1c518b442cb216", + "projectId": "inbox120295392", + "sortOrder": 6917693059112042248, + "title": "first target of business get analytics up to serve the weekly review", + "content": "", + "desc": "", + "timeZone": "America/Guayaquil", + "isFloating": false, + "isAllDay": false, + "reminder": "", + "reminders": [], + "exDate": [], + "priority": 0, + "status": 0, + "items": [], + "progress": 0, + "modifiedTime": "2024-02-19T05:46:16.000+0000", + "etag": "639v9iil", + "deleted": 0, + "createdTime": "2024-02-19T05:46:16.000+0000", + "creator": 120295392, + "tags": [ + "clarifyme" + ], + "attachments": [], + "commentCount": 0, + "focusSummaries": [], + "columnId": "64c626b19b3279687b62fd86", + "kind": "TEXT" + }, + { + "id": "65d2eb9241ded18b442cb254", + "projectId": "inbox120295392", + "sortOrder": 6917693059113090824, + "title": "recurringtask to clear out the duckweed", + "content": "", + "desc": "", + "timeZone": "America/Guayaquil", + "isFloating": false, + "isAllDay": false, + "reminder": "", + "reminders": [], + "exDate": [], + "priority": 0, + "status": 0, + "items": [], + "progress": 0, + "modifiedTime": "2024-02-19T05:48:02.000+0000", + "etag": "5sx8i0b3", + "deleted": 0, + "createdTime": "2024-02-19T05:48:02.000+0000", + "creator": 120295392, + "tags": [ + "clarifyme" + ], + "attachments": [], + "commentCount": 0, + "focusSummaries": [], + "columnId": "64c626b19b3279687b62fd86", + "kind": "TEXT" + }, { "id": "653e339c5d1d510f732da047", "projectId": "f524451ba47155f3049e9ae8", @@ -13720,7 +13961,7 @@ "projectId": "6485239952503fd25a10f40c", "sortOrder": -3023931506688, "title": "loops from offline blocks that should be looked up -asdfsdasd\u00a0", - "content": "\n\ncheckout what the BA replied on the tableau source decomission\n\ncaveat the run folder will get corrupted with owners dbt run\n\ndang the env wont last for different vs code terminal sesh\n\u00a0\n\nadd a note : the WOR reason will use deafult main branch non R2 release item \n\nidea tsql to show log of my command to help script the developments\n\n\nidea to automate dev : when create new branch update the conn to dev\n\n\nnext up tick py now integrate the dbt dagster\n\n\n\n\nanother assumption for the BR change : once the WOR value 1k / 2k assigned that IS THE ONLY INSTANCE added aka no future WOR value assigned tothat \n\n\nping the folks when will the expected value gets in WOR reason 1khrs \n\n\nwe have a bug on dagster run overwriting the ownership of dbt power user \n\n\n\nwhats the diff func sql and func execute\n\nnew use case : to download the snapshot main app\n\nfor the prod machine in future do change the secrest to another folde\nnext up add some numbers to count my dash\n\nweird why the noip not routing my ports out\n\n\ngotta debug why the progress atin getting updates is that because there's a lag in time\n\n\nnope just because i got a bug in tags wrangling \n\n\n\n\n\n\n\n\n\n\n\n\n\n\nwait whens the next monthly parkling lot ? add items to that \n\n\nspin up a vultr machine to get my daily loads of ticktick gtd and setup ci cd\n\nalso put code to coalsece the defaults for the obt, and handle data types properly for analysis \n\n\n\n\ngotta clean the ignore without removing it accidentally\n\n cools there's this free datapoint \nhttp://data.un.org/Explorer.aspx\n\n\n\ncools look at morepandas manip with streamlit demo project df\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n", + "content": "\n\ncheckout what the BA replied on the tableau source decomission\n\ncaveat the run folder will get corrupted with owners dbt run\n\ndang the env wont last for different vs code terminal sesh\n\u00a0\n\nadd a note : the WOR reason will use deafult main branch non R2 release item \n\nidea tsql to show log of my command to help script the developments\n\n\nidea to automate dev : when create new branch update the conn to dev\n\n\nnext up tick py now integrate the dbt dagster\n\n\n\n\nanother assumption for the BR change : once the WOR value 1k / 2k assigned that IS THE ONLY INSTANCE added aka no future WOR value assigned tothat \n\n\nping the folks when will the expected value gets in WOR reason 1khrs \n\n\nwe have a bug on dagster run overwriting the ownership of dbt power user \n\n\n\nwhats the diff func sql and func execute\n\nnew use case : to download the snapshot main app\n\nfor the prod machine in future do change the secrest to another folde\nnext up add some numbers to count my dash\n\nweird why the noip not routing my ports out\n\n\ngotta debug why the progress atin getting updates is that because there's a lag in time\n\n\nnope just because i got a bug in tags wrangling \n\n\n\n\n\n\n\n\n\n\n\n\n\n\nwait whens the next monthly parkling lot ? add items to that \n\n\nspin up a vultr machine to get my daily loads of ticktick gtd and setup ci cd\n\nalso put code to coalsece the defaults for the obt, and handle data types properly for analysis \n\n\n\n\ngotta clean the ignore without removing it accidentally\n\n cools there's this free datapoint \nhttp://data.un.org/Explorer.aspx\n\n\n\ncools look at morepandas manip with streamlit demo project df\n\n\n\nnext up get some counts going for the top level dash\n\nand, move my stuff to another dev branch\n\n\n\n\nweird behaviour : whenever the dagster server is triggered theclient sign in will kick in ? at least learn how to mitigate that now that i dont need the source part on every devyou\n\n\n\n\n\n\n\ncool what worked with my gtd branch now with CD : \ncheckout to dev\nmerge from main\nsync changes\n\n\n\ncools learn the way the new ticktick client is designed \n\n\n\n\n\n", "desc": "", "timeZone": "America/Guayaquil", "isFloating": false, @@ -13733,8 +13974,8 @@ "status": 0, "items": [], "progress": 0, - "modifiedTime": "2024-02-18T13:53:31.000+0000", - "etag": "ld406h2a", + "modifiedTime": "2024-02-19T06:11:11.000+0000", + "etag": "foid3w4j", "deleted": 0, "createdTime": "2023-08-21T10:06:36.596+0000", "creator": 120295392, @@ -42272,6 +42513,32 @@ "focusSummaries": [], "kind": "TEXT" }, + { + "id": "65d2dfb78f08b8edcc0600da", + "projectId": "4b3f4cf882b91b978a5bdb35", + "sortOrder": 23110680576, + "title": " learn to wrap the tick code like gpt ", + "content": "next action\nskim back the chat describe what that does\n---\ninit \nyep what i need is exactly that : create a class sothat only whichever dag call will init it; not other arbitrary events like webserver up, daemon checks etc. \n\n\ncool i amearning how to create a single instance \n\n????\n> The _instance variable is a class variable used to store the single instance of the class.\n", + "desc": "", + "startDate": "2024-02-19T05:00:00.000+0000", + "dueDate": "2024-02-19T05:00:00.000+0000", + "timeZone": "America/Guayaquil", + "isFloating": false, + "isAllDay": true, + "reminders": [], + "exDate": [], + "priority": 0, + "status": 0, + "items": [], + "progress": 0, + "modifiedTime": "2024-02-19T06:07:44.000+0000", + "etag": "rmwahu5q", + "deleted": 0, + "createdTime": "2024-02-19T04:57:27.759+0000", + "creator": 120295392, + "focusSummaries": [], + "kind": "TEXT" + }, { "id": "65d02b199d74d1e32fc78285", "projectId": "4b3f4cf882b91b978a5bdb35", @@ -47315,6 +47582,73 @@ "kind": "TEXT", "imgMode": 0 }, + { + "id": "65d05df652509b5a6b7d3c06", + "projectId": "4b3f4cf882b91b978a5bdb35", + "sortOrder": 23108583424, + "title": "put my focus on getting the dash out checkout bi as code", + "content": "next action\n\n---\ninit\ntry streamlit cause rill is trash. it wont let me reuse the motherduck db\n\n\n\n\nwhat dash am i looking to put into \nhow to host multiple dash ? \n\ntry to reload the ticktick client now \n\nnow config the streamlit to make it more intuitive to actioanable \n\nnow do CD with it \n\n\n\n\n\n\n\n\n\n\n", + "desc": "", + "startDate": "2024-02-18T05:00:00.000+0000", + "dueDate": "2024-02-18T05:00:00.000+0000", + "timeZone": "America/Guayaquil", + "isFloating": false, + "isAllDay": true, + "reminders": [], + "exDate": [], + "completedTime": "2024-02-18T14:29:56.000+0000", + "completedUserId": 120295392, + "repeatTaskId": "65d05df652509b5a6b7d3c06", + "priority": 0, + "status": 2, + "items": [], + "progress": 0, + "modifiedTime": "2024-02-18T14:29:56.000+0000", + "etag": "p5p077dt", + "deleted": 0, + "createdTime": "2024-02-17T07:19:18.000+0000", + "creator": 120295392, + "repeatFrom": "2", + "attachments": [], + "commentCount": 0, + "focusSummaries": [], + "columnId": "64c626b19b3279687b62fd86", + "kind": "TEXT" + }, + { + "id": "65d0edc3c4c951e32fc7bfdc", + "projectId": "62904287ae5ef5c26fefefc2", + "sortOrder": 279723376640, + "title": "schedule in advance 3 next actions for sat and sunday", + "content": "next action\r\nselect my pro / personal folder, group them by tags and skim all the way down. then pick 3 intersting things to do in the weekend \r\n\r\n---\rog \r\none way to help commit clear next actions in the weekend : schedule them in advaance, dont wait till weekend to actually line up tasks\r\n\r\n---\r# tmp\r\n", + "desc": "", + "startDate": "2024-02-17T05:00:00.000+0000", + "dueDate": "2024-02-17T05:00:00.000+0000", + "timeZone": "America/Port-au-Prince", + "isFloating": false, + "isAllDay": true, + "reminders": [], + "exDate": [], + "completedTime": "2024-02-17T17:32:51.000+0000", + "completedUserId": 120295392, + "repeatTaskId": "5493472b8e8c81ed5b1397db", + "priority": 0, + "status": 2, + "items": [], + "progress": 0, + "modifiedTime": "2024-02-17T08:07:53.000+0000", + "etag": "z76lg7b5", + "deleted": 0, + "createdTime": "2023-12-18T17:48:09.000+0000", + "creator": 120295392, + "repeatFrom": "2", + "tags": [], + "attachments": [], + "commentCount": 0, + "focusSummaries": [], + "columnId": "", + "kind": "TEXT" + }, { "id": "94404f83b8916b50f0b4e002", "projectId": "inbox120295392", @@ -48041,40 +48375,6 @@ "focusSummaries": [], "kind": "TEXT" }, - { - "id": "65d0edc3c4c951e32fc7bfdc", - "projectId": "62904287ae5ef5c26fefefc2", - "sortOrder": 279723376640, - "title": "schedule in advance 3 next actions for sat and sunday", - "content": "next action\r\nselect my pro / personal folder, group them by tags and skim all the way down. then pick 3 intersting things to do in the weekend \r\n\r\n---\rog \r\none way to help commit clear next actions in the weekend : schedule them in advaance, dont wait till weekend to actually line up tasks\r\n\r\n---\r# tmp\r\n", - "desc": "", - "startDate": "2024-02-17T05:00:00.000+0000", - "dueDate": "2024-02-17T05:00:00.000+0000", - "timeZone": "America/Port-au-Prince", - "isFloating": false, - "isAllDay": true, - "reminders": [], - "exDate": [], - "completedTime": "2024-02-17T17:32:51.000+0000", - "completedUserId": 120295392, - "repeatTaskId": "5493472b8e8c81ed5b1397db", - "priority": 0, - "status": 2, - "items": [], - "progress": 0, - "modifiedTime": "2024-02-17T08:07:53.000+0000", - "etag": "z76lg7b5", - "deleted": 0, - "createdTime": "2023-12-18T17:48:09.000+0000", - "creator": 120295392, - "repeatFrom": "2", - "tags": [], - "attachments": [], - "commentCount": 0, - "focusSummaries": [], - "columnId": "", - "kind": "TEXT" - }, { "id": "4add41dcaf0ee5bb855189dc", "projectId": "27784e39af4bcdf0d60fd71d", diff --git a/app/env/.secrets b/app/env/.secrets index e2c7499..4ef5cef 100644 --- a/app/env/.secrets +++ b/app/env/.secrets @@ -1,7 +1,7 @@ client_id='17XH05ysiYjB1Y30Um' client_secret='V79oT+8*1KnL$H%4Z_VJ*_jJl5Hh7fYk' username='luutuankiet.ftu2@gmail.com' -password='11 fish 8 red 3 black 1 died' +password='Boyluu96@' redirect_uri='https://127.0.0.1:8080' access_token_dict={"access_token": "30c9101e-3ea3-4fdc-a5f4-38df67dd3226", "token_type": "bearer", "expires_in": 15551999, "scope": "tasks:read tasks:write", "expire_time": 1722757907, "readable_expire_time": "Sun Aug 4 07:51:47 2024"} motherduck_token='eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzZXNzaW9uIjoibHV1dHVhbmtpZXQuZnR1Mi5nbWFpbC5jb20iLCJlbWFpbCI6Imx1dXR1YW5raWV0LmZ0dTJAZ21haWwuY29tIiwidXNlcklkIjoiMjRkOTQxYzktMDg2OC00ZThmLWIwNjMtNDFmNjE2MDYyMjMxIiwiaWF0IjoxNzA3MTU2NzY2LCJleHAiOjE3Mzg3MTQzNjZ9.z-FjZKJ8oAewQKJ2X3_4emnJXszt1q_VaB0ZVOsQEUM' \ No newline at end of file diff --git a/app/helper/motherduck_helper.py b/app/helper/motherduck_helper.py index 98ea90e..e2546d2 100644 --- a/app/helper/motherduck_helper.py +++ b/app/helper/motherduck_helper.py @@ -13,3 +13,4 @@ +# TODO : kill this script. \ No newline at end of file