diff --git a/app/charts/main.ipynb b/app/charts/main.ipynb index 227d1ea..647e888 100644 --- a/app/charts/main.ipynb +++ b/app/charts/main.ipynb @@ -1,14 +1,32 @@ { "cells": [ + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env: PYTHONPATH=app\n" + ] + } + ], + "source": [ + "%env PYTHONPATH=app" + ] + }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ - "from click import style\n", + "from helper.source_env import dbt_models_path\n", + "import os\n", "import streamlit as st\n", - "import pandas as pd\n", + "\n", "import duckdb\n", "import os\n", "\n", @@ -16,86 +34,19 @@ "con = duckdb.connect(f'md:ticktick_gtd?motherduck_token={motherduck_token}')\n", "cur = con.cursor()\n", "\n", - "def highlight_cell(val):\n", - " if val < 20:\n", - " color = 'yellow' \n", - " # elif val < 80:\n", - " # color = 'yellow'\n", - " # elif val < 75:\n", - " # color = 'orange'\n", - " elif val == 100:\n", - " color = 'blue'\n", - " else: \n", - " color = ''\n", - " return f'background-color: {color}'\n", - "\n", - "def highlight_text(val):\n", - " if val == 100:\n", - " return 'color: #86acff; font-weight: bold;'\n", - " elif 0 <= val < 100:\n", - " # Calculate the RGB values for a color between red and green based on the progress\n", - " r = int(255 * (1 - val/100))\n", - " g = int(255 * (val/100))\n", - " b = 0\n", - " return f'color: rgb({r}, {g}, {b}); font-weight: bold;'\n", - " else:\n", - " return ''\n", - "\n", - "def highlight_row(row):\n", - " if any(row.astype(str).str.count('-') >= 4):\n", - " return ['background-color: gray; font-weight: bold;'] * len(row)\n", - " else:\n", - " return [''] * len(row)\n", - "\n", - "st.set_page_config(page_title=\"MY GTD DASHBOARD\", layout=\"wide\", initial_sidebar_state=\"collapsed\", menu_items=None)\n", - "# [theme]\n", - "# base=\"light\"\n", - "# primaryColor=\"#86acff\"\n", - "\n", - "\n", - "\n", - "# df=lvl1_lvl2_progress\n", - "# df=df.style.map(highlight_low_val,subset=['done_progress','clarify_progress'])\n", - "obt=cur.sql(\"select * from obt\").df() \n", - "\n", - "\n", - "# with st.sidebar:\n", - "# folders = obt['fld_folder_name'].drop_duplicates().to_list()\n", - "# filter_folder = st.multiselect('folders',folders,default=folders)\n", "\n", + "analytics_path = os.path.join(dbt_models_path,'analytics')\n", + "target = os.path.join(analytics_path,'active_tags_count.sql')\n", "\n", - "# st.write(\"# lvl1-lvl2 analytics\")\n", - "# st.write(\"## progress summary\")\n", - "lvl1_lvl2_progress = cur.query(\"select * from lvl1_lvl2_progress\").df()\n", "\n", - "filtered_lvl1_lvl2_progress = lvl1_lvl2_progress[lvl1_lvl2_progress['fld_folder_name'].isin(filter_folder)]\n", - "colored_lvl1_lvl2_progress = filtered_lvl1_lvl2_progress.style.map(\n", - " highlight_text,subset=['done_progress','clarify_progress']\n", - ").apply(\n", - " highlight_row\n", - ")\n", - "final_lvl1_lvl2_progress = colored_lvl1_lvl2_progress\n", "\n", - "# st.dataframe(\n", - "# final_lvl1_lvl2_progress,\n", - "# column_config={\n", - "# \"done_progress\": st.column_config.ProgressColumn(\n", - "# \"done_progress\",\n", - "# format=\"%f\",\n", - "# min_value=0,\n", - "# max_value=100\n", - "# ),\n", - "# \"clarify_progress\": st.column_config.ProgressColumn(\n", - "# \"clarify_progress\",\n", - "# format=\"%f\",\n", - "# min_value=0,\n", - "# max_value=100\n", - "# )\n", - "# },\n", + "with open(target,'r') as f:\n", + " query = f.read()\n", + " # print(query)\n", + " df = cur.sql(query).df()\n", + " \n", "\n", - "# hide_index=True,\n", - "# use_container_width=True\n", - "# )\n" + "\n" ] }, { @@ -106,674 +57,47 @@ { "data": { "text/html": [ - "\n", - "\n", + "
\n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - "
 fld_folder_namel_list_namedone_progressclarify_progress
cnt_clarifymecnt_nonecnt_contextcnt_somedaycnt_waiting_for
0▶️lvl2 - Innovature-----------------------60.41000091.390000
1▶️lvl2 - Innovature1H 2024 comm pi SOP dev handbook33.330000100.000000
2▶️lvl2 - Innovature1H 2024 commitment doc mngmnt system0.000000100.000000
3▶️lvl2 - InnovatureUA add new field JP on off shore type40.0000000.000000
4▶️lvl2 - InnovatureUA bug incorrect financial SP amounts100.000000100.000000
5▶️lvl2 - InnovatureUA implement job family mapping75.000000100.000000
6▶️lvl2 - InnovatureUA re position the field in contractor table Pre id80.000000100.000000
7▶️lvl2 - InnovatureUA split TS in daily ETL50.000000100.000000
8▶️lvl2 - InnovatureUA update new cost center schema25.000000100.000000
9▶️lvl2 - InnovatureUAL re point correct pay rate values95.450000100.000000
10▶️lvl2 - InnovatureUALdebug multi cost center associated to time and invoice amount90.630000100.000000
11▶️lvl2 - InnovatureUnited collaborated UAT testing96.510000100.000000
12▶️lvl2 - Innovatureaddress integrity issues in inno dw22.220000100.000000
13▶️lvl2 - Innovatureapply recursive query to inno hierrachy needs66.670000100.000000
14▶️lvl2 - Innovaturebeef up pi scripting for deployment0.000000100.000000
15▶️lvl2 - Innovaturebeef up testing united (h2 comm sub list)83.330000100.000000
16▶️lvl2 - Innovaturebiweekly notes98.590000100.000000
17▶️lvl2 - Innovaturebuild logic for applicants and requests activity inno dw50.000000100.000000
18▶️lvl2 - Innovatureetl and dag design inno DW56.250000100.000000
19▶️lvl2 - Innovaturefix workflow cycle time UA84.21000033.330000
20▶️lvl2 - Innovatureinno capturing historical employee hierrachy33.330000100.000000
21▶️lvl2 - Innovaturephase 1 UAL WO 1k and 2k hours change flag73.68000060.000000
22▶️lvl2 - Innovaturepython for ad hoc csv64.710000100.000000
23▶️lvl2 - Innovaturereverse engineer & learn Pi etl design47.370000100.000000
24▶️lvl2 - Innovature▶️BAU NEXT ACTION83.440000100.000000
25🏚lvl2 - Personal-----------------------38.52000093.750000
26🏚lvl2 - Personaladjust my weight to a comfortable state.75.000000100.000000
27🏚lvl2 - Personalcaress for my body and soul under soltitude17.650000100.000000
28🏚lvl2 - Personaldai hoc sis mission control10.0000000.000000
29🏚lvl2 - Personalemergency plan for phone lost situations31.250000100.000000
30🏚lvl2 - Personalkeeping healthy habits in check89.470000100.000000
31🏚lvl2 - Personalmoderate stress to void dizziness50.000000100.000000
32🏚lvl2 - Personalnew identity no PMO25.000000100.000000
33🏚lvl2 - Personalpeace of mind for the reduncany0.000000100.000000
34🏚lvl2 - Personalpeople observation20.000000100.000000
35🏚lvl2 - Personalpersonal GTD dashboard44.740000100.000000
36🏚lvl2 - Personalroom org87.500000100.000000
37🏚lvl2 - Personalsetup a remote troubleshooting tool for her40.740000100.000000
38🏚lvl2 - Personalsetup photo studio for mom0.000000100.000000
39🏚lvl2 - Personalsis career and schooling path insights50.000000100.000000
40🏚lvl2 - Personalticktick gtd v2 with dbt, dagster and postgres48.280000100.000000
41🏚lvl2 - Personal⚓breaking a bad habit pmo26.670000100.000000
42💤on hold lists-----------------------75.000000100.000000
43💤on hold listsUAL phase 2 BR change flag75.000000100.000000
44💵lvl2 - Finance-----------------------65.380000100.000000
45💵lvl2 - Financefinancing for next 3 months65.380000100.000000
46💻lvl2 - Professional-----------------------30.430000100.000000
47💻lvl2 - ProfessionalETL ticktick data with tickpy API51.430000100.000000
48💻lvl2 - ProfessionalGTD implementation85.860000100.000000
49💻lvl2 - Professionaldeep implement quota to shallow work endeavor in the month0.000000100.000000
50💻lvl2 - Professionaldeep work implementations37.500000100.000000
51💻lvl2 - Professionalfirst take with duckdb for cloud DE12.500000100.000000
52💻lvl2 - Professionalgtd lvl3 quarterly review31.580000100.000000
53💻lvl2 - Professionalgtd weekly review recurring tasks0.000000100.000000
54💻lvl2 - Professionaljob hoppin - getting ready42.860000100.000000
55💻lvl2 - Professionalmove from lastpass100.000000100.000000
56💻lvl2 - Professionalpgsql cli 10150.000000100.000000
57💻lvl2 - Professionalpractice wiki google scraping0.000000100.000000
58💻lvl2 - Professionalscrape tictick habit data for data driven next action16.670000100.000000
59💻lvl2 - Professionalset up learning a cert0.000000100.000000
60💻lvl2 - Professionalsetup lvl2 gtd review process88.890000100.000000
61💻lvl2 - Professionaltaking up cloud DE courses and material0.000000100.000000
62💻lvl2 - Professionalup my documentation skills0.000000100.000000
63💻lvl2 - Professionalwildly important goal and any benefit mindset0.000000100.000000
64📥lvl2 - organized inbox-----------------------64.650000100.000000
65📥lvl2 - organized inbox⏭NEXT64.650000100.000000035.021.0204.0232.03.0
\n" + "\n", + "" ], "text/plain": [ - "" + " cnt_clarifyme cnt_none cnt_context cnt_someday cnt_waiting_for\n", + "0 35.0 21.0 204.0 232.0 3.0" ] }, "execution_count": 15, @@ -782,16 +106,7 @@ } ], "source": [ - "final_lvl1_lvl2_progress\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "folders" + "df" ] }, { diff --git a/app/charts/main.py b/app/charts/main.py index fd19d25..001df9e 100644 --- a/app/charts/main.py +++ b/app/charts/main.py @@ -3,6 +3,7 @@ import pandas as pd import duckdb import os +from helper.source_env import dbt_models_path motherduck_token = os.environ.get("motherduck_token") con = duckdb.connect(f'md:ticktick_gtd?motherduck_token={motherduck_token}') @@ -40,14 +41,7 @@ def highlight_row(row): return [''] * len(row) st.set_page_config(page_title="MY GTD DASHBOARD", layout="wide", initial_sidebar_state="collapsed", menu_items=None) -# [theme] -# base="light" -# primaryColor="#86acff" - - -# df=lvl1_lvl2_progress -# df=df.style.map(highlight_low_val,subset=['done_progress','clarify_progress']) obt=cur.sql("select * from obt").df() @@ -56,6 +50,42 @@ def highlight_row(row): filter_folder = st.multiselect('folders',folders,default=folders) +st.write("# at a glance") +st.write("## count of clarified and next action") + + +analytics_path = os.path.join(dbt_models_path,'analytics') +tags_count_path = os.path.join(analytics_path,'active_tags_count.sql') + + +with open(tags_count_path,'r') as f: + query = f.read() + tags_count = cur.sql(query).df() + + st.dataframe( + tags_count, + column_config={ + "weight_clarifyme": st.column_config.ProgressColumn( + "weight_clarifyme", + format="%f", + min_value=0, + max_value=100 + ), + "weight_next_action": st.column_config.ProgressColumn( + "weight_next_action", + format="%f", + min_value=0, + max_value=100 + ) + }, + + hide_index=True, + use_container_width=True + ) + + + + st.write("# lvl1-lvl2 analytics") st.write("## progress summary") lvl1_lvl2_progress = cur.query("select * from lvl1_lvl2_progress").df() diff --git a/app/helper/source_env.py b/app/helper/source_env.py index 57f238c..0412563 100644 --- a/app/helper/source_env.py +++ b/app/helper/source_env.py @@ -1,9 +1,25 @@ import os from dotenv import load_dotenv + +# from app.ETL.constants import DBT_PROJECT_DIR + + + # setup paths current_dir=os.path.dirname(os.path.abspath(__file__)) + +# source .env from project root to construct dbt paths +project_dotenv_path=os.path.join(current_dir,'..','..','.env') +load_dotenv(project_dotenv_path) + raw_path = os.path.join(current_dir,'..','ETL','raw') dotenv_path=os.path.join(current_dir,'..','env') secrets_path = os.path.join(dotenv_path,'.secrets') -project_dotenv_path=os.path.join(current_dir,'..','..','.env') + + +dbt_project_dir = os.environ.get('DBT_PROJECT_DIR') +dbt_models_path=os.path.join(dbt_project_dir,'models') + + + load_dotenv(secrets_path) \ No newline at end of file diff --git a/dbt_project/models/analytics/active_tags_count.sql b/dbt_project/models/analytics/active_tags_count.sql new file mode 100644 index 0000000..6f3005e --- /dev/null +++ b/dbt_project/models/analytics/active_tags_count.sql @@ -0,0 +1,57 @@ +with source as ( +SELECT + SUM( + CASE + WHEN td_tags LIKE '%clarifyme%' THEN 1 + ELSE 0 + END + ) AS cnt_clarifyme, + SUM( + CASE + WHEN td_tags = '[]' THEN 1 + ELSE 0 + END + ) AS cnt_none, + SUM( + CASE + WHEN td_tags LIKE '%@%' THEN 1 + ELSE 0 + END + ) AS cnt_context, + SUM( + CASE + WHEN td_tags LIKE '%someday%' THEN 1 + ELSE 0 + END + ) AS cnt_someday, + SUM( + CASE + WHEN td_tags LIKE '%waiting_for%' THEN 1 + ELSE 0 + END + ) AS cnt_waiting_for +FROM + obt +WHERE + td_kind = 'TEXT' + AND ss_desc = 'undone' + AND ( + fld_folder_name NOT IN ( + '🚀SOMEDAY lists', + '🛩Horizon of focus', + '💤on hold lists' + ) + OR fld_folder_name IS NULL + ) +) + +select +cast( + (cnt_clarifyme* 100 / (cnt_clarifyme + cnt_none + cnt_context)) as decimal(10,2) + ) as weight_clarifyme, +cnt_clarifyme, +cast(( + (cnt_none+cnt_context)* 100 / (cnt_clarifyme + cnt_none + cnt_context)) as decimal(10,2)) as weight_next_action, +(cnt_none+cnt_context) as cnt_next_action, + +from source \ No newline at end of file