From 382558eef25d55208ca38232df7398ce1b569c60 Mon Sep 17 00:00:00 2001 From: luutuankiet Date: Mon, 19 Feb 2024 16:34:59 +0700 Subject: [PATCH 1/3] Remove .ipynb files from tracking --- app/charts/main.ipynb | 141 -------------------------------------- app/get_all_tasks.ipynb | 147 ---------------------------------------- 2 files changed, 288 deletions(-) delete mode 100644 app/charts/main.ipynb delete mode 100644 app/get_all_tasks.ipynb diff --git a/app/charts/main.ipynb b/app/charts/main.ipynb deleted file mode 100644 index 647e888..0000000 --- a/app/charts/main.ipynb +++ /dev/null @@ -1,141 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "env: PYTHONPATH=app\n" - ] - } - ], - "source": [ - "%env PYTHONPATH=app" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "from helper.source_env import dbt_models_path\n", - "import os\n", - "import streamlit as st\n", - "\n", - "import duckdb\n", - "import os\n", - "\n", - "motherduck_token = os.environ.get(\"motherduck_token\")\n", - "con = duckdb.connect(f'md:ticktick_gtd?motherduck_token={motherduck_token}')\n", - "cur = con.cursor()\n", - "\n", - "\n", - "analytics_path = os.path.join(dbt_models_path,'analytics')\n", - "target = os.path.join(analytics_path,'active_tags_count.sql')\n", - "\n", - "\n", - "\n", - "with open(target,'r') as f:\n", - " query = f.read()\n", - " # print(query)\n", - " df = cur.sql(query).df()\n", - " \n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
cnt_clarifymecnt_nonecnt_contextcnt_somedaycnt_waiting_for
035.021.0204.0232.03.0
\n", - "
" - ], - "text/plain": [ - " cnt_clarifyme cnt_none cnt_context cnt_someday cnt_waiting_for\n", - "0 35.0 21.0 204.0 232.0 3.0" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.6" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/app/get_all_tasks.ipynb b/app/get_all_tasks.ipynb deleted file mode 100644 index 87719c2..0000000 --- a/app/get_all_tasks.ipynb +++ /dev/null @@ -1,147 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "import json\n", - "import pandas as pd\n", - "tasks_path='ETL/raw/all_tasks.json'\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "task_df = pd.read_json(tasks_path)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "with open(tasks_path,'r') as f:\n", - " task_json=json.load(f)\n", - "\n", - "sample='sample.csv'\n", - "\n", - "sample_df=pd.read_csv(sample,\n", - " skiprows=3,\n", - " header=0,\n", - " dtype={i: str for i in range(100)})" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['id', 'projectId', 'sortOrder', 'title', 'content', 'desc', 'timeZone',\n", - " 'isFloating', 'isAllDay', 'reminder', 'reminders', 'exDate', 'priority',\n", - " 'status', 'items', 'progress', 'modifiedTime', 'etag', 'deleted',\n", - " 'createdTime', 'creator', 'focusSummaries', 'columnId', 'kind',\n", - " 'imgMode', 'tags', 'repeatFrom', 'attachments', 'repeatTaskId',\n", - " 'commentCount', 'completedTime', 'completedUserId', 'repeatFlag',\n", - " 'pinnedTime', 'startDate', 'dueDate', 'childIds', 'deletedTime',\n", - " 'repeatFirstDate', 'pomodoroSummaries', 'parentId', 'annoyingAlert',\n", - " 'remindTime'],\n", - " dtype='object')" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "task_df.columns" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['Folder Name', 'List Name', 'Title', 'Kind', 'Tags', 'Content',\n", - " 'Is Check list', 'Start Date', 'Due Date', 'Reminder', 'Repeat',\n", - " 'Priority', 'Status', 'Created Time', 'Completed Time', 'Order',\n", - " 'Timezone', 'Is All Day', 'Is Floating', 'Column Name', 'Column Order',\n", - " 'View Mode', 'taskId', 'parentId'],\n", - " dtype='object')" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sample_df.columns" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# current schema\n", - "Index(['Folder Name', 'List Name', 'Title', 'Kind', 'Tags', 'Content',\n", - " 'Is Check list', 'Start Date', 'Due Date', 'Reminder', 'Repeat',\n", - " 'Priority', 'Status', 'Created Time', 'Completed Time', 'Order',\n", - " 'Timezone', 'Is All Day', 'Is Floating', 'Column Name', 'Column Order',\n", - " 'View Mode', 'taskId', 'parentId'],\n", - " dtype='object')\n", - "\n", - "# tickpy schema (target)\n", - "Index(['id', 'projectId', 'sortOrder', 'title', 'content', 'desc', 'timeZone',\n", - " 'isFloating', 'isAllDay', 'reminder', 'reminders', 'exDate', 'priority',\n", - " 'status', 'items', 'progress', 'modifiedTime', 'etag', 'deleted',\n", - " 'createdTime', 'creator', 'focusSummaries', 'columnId', 'kind',\n", - " 'imgMode', 'tags', 'repeatFrom', 'attachments', 'repeatTaskId',\n", - " 'commentCount', 'completedTime', 'completedUserId', 'repeatFlag',\n", - " 'pinnedTime', 'startDate', 'dueDate', 'childIds', 'deletedTime',\n", - " 'repeatFirstDate', 'pomodoroSummaries', 'parentId', 'annoyingAlert',\n", - " 'remindTime'],\n", - " dtype='object')\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.6" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 1d9bf9522d1acb2f4693c7294703661c4a0f6db7 Mon Sep 17 00:00:00 2001 From: luutuankiet Date: Mon, 19 Feb 2024 16:35:25 +0700 Subject: [PATCH 2/3] gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 48edd66..dc57fc9 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,4 @@ __pycache__/ *.pyc *.pyo *.pyd - +*.ipynb \ No newline at end of file From 6178d87121c3e59bca9cb778eca125d7625b37f8 Mon Sep 17 00:00:00 2001 From: luutuankiet Date: Mon, 19 Feb 2024 17:46:25 +0700 Subject: [PATCH 3/3] fix weird bug that duplicated the models folder --- app/charts/main.py | 45 ++--- .../active_tags_count.sql | 12 +- .../lvl3_helper_list_extract.sql | 0 models/_source.yml | 28 --- models/analytics/lvl3_helper_list_extract.sql | 23 --- models/marts/core/dim_dates.sql | 9 - models/marts/core/dim_folders.sql | 9 - models/marts/core/dim_lists.sql | 9 - models/marts/core/dim_statuses.sql | 9 - models/marts/core/fact_todos.sql | 74 ------- models/marts/core/obt.sql | 114 ----------- models/marts/metrics/lvl1_lvl2_progress.sql | 182 ------------------ models/marts/metrics/lvl3_progress.sql | 23 --- models/marts/metrics/lvl3_sumarize.sql | 6 - models/marts/metrics/lvl3_unmapped.sql | 15 -- models/staging/init_duckdb__lvl3.sql | 16 -- models/staging/stg_duckdb__lvl3.sql | 22 --- models/staging/stg_pg__dates.sql | 14 -- models/staging/stg_pg__folders.sql | 35 ---- models/staging/stg_pg__lists.sql | 46 ----- models/staging/stg_pg__statuses.sql | 33 ---- models/staging/stg_pg__todos.sql | 80 -------- 22 files changed, 25 insertions(+), 779 deletions(-) rename dbt_project/{models/analytics => analyses}/active_tags_count.sql (75%) rename dbt_project/{models/analytics => analyses}/lvl3_helper_list_extract.sql (100%) delete mode 100644 models/_source.yml delete mode 100644 models/analytics/lvl3_helper_list_extract.sql delete mode 100644 models/marts/core/dim_dates.sql delete mode 100644 models/marts/core/dim_folders.sql delete mode 100644 models/marts/core/dim_lists.sql delete mode 100644 models/marts/core/dim_statuses.sql delete mode 100644 models/marts/core/fact_todos.sql delete mode 100644 models/marts/core/obt.sql delete mode 100644 models/marts/metrics/lvl1_lvl2_progress.sql delete mode 100644 models/marts/metrics/lvl3_progress.sql delete mode 100644 models/marts/metrics/lvl3_sumarize.sql delete mode 100644 models/marts/metrics/lvl3_unmapped.sql delete mode 100644 models/staging/init_duckdb__lvl3.sql delete mode 100644 models/staging/stg_duckdb__lvl3.sql delete mode 100644 models/staging/stg_pg__dates.sql delete mode 100644 models/staging/stg_pg__folders.sql delete mode 100644 models/staging/stg_pg__lists.sql delete mode 100644 models/staging/stg_pg__statuses.sql delete mode 100644 models/staging/stg_pg__todos.sql diff --git a/app/charts/main.py b/app/charts/main.py index 001df9e..bf3dc2f 100644 --- a/app/charts/main.py +++ b/app/charts/main.py @@ -3,7 +3,7 @@ import pandas as pd import duckdb import os -from helper.source_env import dbt_models_path +from helper.source_env import dbt_project_dir motherduck_token = os.environ.get("motherduck_token") con = duckdb.connect(f'md:ticktick_gtd?motherduck_token={motherduck_token}') @@ -54,34 +54,29 @@ def highlight_row(row): st.write("## count of clarified and next action") -analytics_path = os.path.join(dbt_models_path,'analytics') +analytics_path = os.path.join(dbt_project_dir,'analyses') tags_count_path = os.path.join(analytics_path,'active_tags_count.sql') -with open(tags_count_path,'r') as f: - query = f.read() - tags_count = cur.sql(query).df() - - st.dataframe( - tags_count, - column_config={ - "weight_clarifyme": st.column_config.ProgressColumn( - "weight_clarifyme", - format="%f", - min_value=0, - max_value=100 - ), - "weight_next_action": st.column_config.ProgressColumn( - "weight_next_action", - format="%f", - min_value=0, - max_value=100 - ) - }, +query = lambda: open(tags_count_path, 'r').read() +tags_count = cur.sql(query()).df() +colored_tags_count = tags_count.style.map(highlight_text,subset=['clarification_progress']) +tags_count_final = colored_tags_count + +st.dataframe( + tags_count_final, + column_config={ + "clarification_progress": st.column_config.ProgressColumn( + "clarification_progress", + format="%f", + min_value=0, + max_value=100 + ), + }, - hide_index=True, - use_container_width=True - ) + hide_index=True, + use_container_width=True + ) diff --git a/dbt_project/models/analytics/active_tags_count.sql b/dbt_project/analyses/active_tags_count.sql similarity index 75% rename from dbt_project/models/analytics/active_tags_count.sql rename to dbt_project/analyses/active_tags_count.sql index 6f3005e..a64c4a3 100644 --- a/dbt_project/models/analytics/active_tags_count.sql +++ b/dbt_project/analyses/active_tags_count.sql @@ -45,13 +45,11 @@ WHERE ) ) -select +select cast( - (cnt_clarifyme* 100 / (cnt_clarifyme + cnt_none + cnt_context)) as decimal(10,2) - ) as weight_clarifyme, -cnt_clarifyme, -cast(( - (cnt_none+cnt_context)* 100 / (cnt_clarifyme + cnt_none + cnt_context)) as decimal(10,2)) as weight_next_action, -(cnt_none+cnt_context) as cnt_next_action, + 100 - (cnt_clarifyme* 100 / (cnt_clarifyme + cnt_none + cnt_context)) as decimal(10,2) + ) as clarification_progress, +cnt_clarifyme::int, +(cnt_none+cnt_context)::int as cnt_next_action, from source \ No newline at end of file diff --git a/dbt_project/models/analytics/lvl3_helper_list_extract.sql b/dbt_project/analyses/lvl3_helper_list_extract.sql similarity index 100% rename from dbt_project/models/analytics/lvl3_helper_list_extract.sql rename to dbt_project/analyses/lvl3_helper_list_extract.sql diff --git a/models/_source.yml b/models/_source.yml deleted file mode 100644 index f322a33..0000000 --- a/models/_source.yml +++ /dev/null @@ -1,28 +0,0 @@ -version: 2 - -sources: - - name: raw_data - description: A replica of the extracted ticktick data. - database: ticktick_gtd - schema: main - tables: - - name: tasks_raw - description: raw tasks data from tickpy - meta: - dagster: - asset_key: ["raw_data"] - - name: lists_raw - description: raw lists data from tickpy - meta: - dagster: - asset_key: ["raw_data"] - - name: folders_raw - description: raw folders data from tickpy - meta: - dagster: - asset_key: ["raw_data"] - - name: date_seed - description: pre-generated dates - meta: - dagster: - asset_key: ["raw_data"] diff --git a/models/analytics/lvl3_helper_list_extract.sql b/models/analytics/lvl3_helper_list_extract.sql deleted file mode 100644 index ff90729..0000000 --- a/models/analytics/lvl3_helper_list_extract.sql +++ /dev/null @@ -1,23 +0,0 @@ -with source as ( - select * from {{ ref('lvl1_lvl2_progress') }} -), - -lvl3_goals as ( - select * from {{ ref('init_duckdb__lvl3') }} -) - -select - fld_folder_name, - l_list_name, - '' as goal_ids -from source -union all -( -select -'goal', -'id', -'' as goal_ids - -union all - -select *,'' as goal_ids from lvl3_goals) diff --git a/models/marts/core/dim_dates.sql b/models/marts/core/dim_dates.sql deleted file mode 100644 index 7b84a17..0000000 --- a/models/marts/core/dim_dates.sql +++ /dev/null @@ -1,9 +0,0 @@ -WITH source AS ( - SELECT * - FROM - {{ ref('stg_pg__dates') }} -) - -SELECT * -FROM - source diff --git a/models/marts/core/dim_folders.sql b/models/marts/core/dim_folders.sql deleted file mode 100644 index a01499d..0000000 --- a/models/marts/core/dim_folders.sql +++ /dev/null @@ -1,9 +0,0 @@ -WITH stg_folders AS ( - SELECT * - FROM - {{ ref('stg_pg__folders') }} -) - -SELECT * -FROM - stg_folders diff --git a/models/marts/core/dim_lists.sql b/models/marts/core/dim_lists.sql deleted file mode 100644 index fa170fb..0000000 --- a/models/marts/core/dim_lists.sql +++ /dev/null @@ -1,9 +0,0 @@ -WITH stg_lists AS ( - SELECT * - FROM - {{ ref ('stg_pg__lists') }} -) - -SELECT * -FROM - stg_lists diff --git a/models/marts/core/dim_statuses.sql b/models/marts/core/dim_statuses.sql deleted file mode 100644 index af9874d..0000000 --- a/models/marts/core/dim_statuses.sql +++ /dev/null @@ -1,9 +0,0 @@ -WITH stg_statuses AS ( - SELECT * - FROM - {{ ref('stg_pg__statuses') }} -) - -SELECT * -FROM - stg_statuses diff --git a/models/marts/core/fact_todos.sql b/models/marts/core/fact_todos.sql deleted file mode 100644 index def2596..0000000 --- a/models/marts/core/fact_todos.sql +++ /dev/null @@ -1,74 +0,0 @@ -WITH todos AS ( - SELECT * - FROM - {{ ref ('stg_pg__todos') }} -), - -lists AS ( - SELECT * - FROM - {{ ref ('stg_pg__lists') }} -), - -folders AS ( - SELECT * - FROM - {{ ref('stg_pg__folders') }} -), - -statuses AS ( - SELECT * - FROM - {{ ref('stg_pg__statuses') }} -), - -dates AS ( - SELECT * - FROM - {{ ref('stg_pg__dates') }} -), - -facts_todo AS ( - SELECT - td.todo_id, - {{ dbt_utils.generate_surrogate_key(['td.todo_id']) }} AS todo_key, - {{ dbt_utils.generate_surrogate_key(['l.list_id']) }} AS list_key, - {{ dbt_utils.generate_surrogate_key(['fld.folder_id']) }} AS folder_key, - {{ dbt_utils.generate_surrogate_key(['ss.status_id']) }} AS status_key, - {{ dbt_utils.generate_surrogate_key(['dds.date_id']) }} AS date_start_key, - {{ dbt_utils.generate_surrogate_key(['ddd.date_id']) }} AS date_due_key, - {{ dbt_utils.generate_surrogate_key(['ddcm.date_id']) }} AS date_completed_key, - {{ dbt_utils.generate_surrogate_key(['ddc.date_id']) }} AS date_created_key, - title, - kind, - tags, - reminder, - repeat_, - priority, - timezone, - isallday, - isfloating - FROM - todos AS td - LEFT JOIN -- folder - folders AS fld - ON td.folder_name = fld.folder_name - LEFT JOIN -- list - lists AS l - ON td.list_name = l.list_name - LEFT JOIN -- status - statuses AS ss - ON cast(ss.status_id AS text) = cast(td.status_id AS text) -- role play dates for start, due, create, complete - LEFT JOIN dates AS dds - ON cast(dds.date_id AS date) = cast(td.start_date AS date) - LEFT JOIN dates AS ddd - ON cast(ddd.date_id AS date) = cast(td.due_date AS date) - LEFT JOIN dates AS ddc - ON cast(ddc.date_id AS date) = cast(td.created_time AS date) - LEFT JOIN dates AS ddcm - ON cast(ddcm.date_id AS date) = cast(td.completed_time AS date) -) - -SELECT * -FROM - facts_todo diff --git a/models/marts/core/obt.sql b/models/marts/core/obt.sql deleted file mode 100644 index 9900ca8..0000000 --- a/models/marts/core/obt.sql +++ /dev/null @@ -1,114 +0,0 @@ -WITH f_todos AS ( - SELECT * - FROM - {{ ref('fact_todos') }} -), - -d_lists AS ( - SELECT * - FROM - {{ ref('dim_lists') }} -), - -d_folders AS ( - SELECT * - FROM - {{ ref('dim_folders') }} -), - -d_statuses AS ( - SELECT * - FROM - {{ ref('dim_statuses') }} -), - -d_start_dates AS ( - SELECT * - FROM - {{ ref('dim_dates') }} -), - -d_due_dates AS ( - SELECT * - FROM - {{ ref('dim_dates') }} -), - -d_created_dates AS ( - SELECT * - FROM - {{ ref('dim_dates') }} -), - -d_completed_dates AS ( - SELECT * - FROM - {{ ref('dim_dates') }} -) - -SELECT - {{ dbt_utils.star( - from = ref('fact_todos'), - relation_alias = 'f_todos', - prefix = 'td_', - except = [ 'list_key', 'folder_key','status_key','todo_key','date_start_key','date_due_key','date_completed_key', 'date_created_key' ] - ) }}, - {{ dbt_utils.star( - from = ref('dim_lists'), - relation_alias = 'd_lists', - prefix = 'l_', - except = [ 'list_key' ] - ) }}, - {{ dbt_utils.star( - from = ref('dim_folders'), - relation_alias = 'd_folders', - prefix = 'fld_', - except = [ 'folder_key' ] - ) }}, - {{ dbt_utils.star( - from = ref('dim_statuses'), - relation_alias = 'd_statuses', - prefix = 'ss_', - except = [ 'status_key' ] - ) }}, - -- dates roleplay - {{ dbt_utils.star( - from = ref('dim_dates'), - relation_alias = 'd_start_dates', - prefix = 'start_', - except = [ 'date_key' ] - ) }}, - {{ dbt_utils.star( - from = ref('dim_dates'), - relation_alias = 'd_due_dates', - prefix = 'due_', - except = [ 'date_key' ] - ) }}, - {{ dbt_utils.star( - from = ref('dim_dates'), - relation_alias = 'd_completed_dates', - prefix = 'completed_', - except = [ 'date_key' ] - ) }}, - {{ dbt_utils.star( - from = ref('dim_dates'), - relation_alias = 'd_created_dates', - prefix = 'created_', - except = [ 'date_key' ] - ) }} -FROM - f_todos -LEFT JOIN d_lists - ON f_todos.list_key = d_lists.list_key -LEFT JOIN d_folders - ON f_todos.folder_key = d_folders.folder_key -LEFT JOIN d_statuses - ON f_todos.status_key = d_statuses.status_key -LEFT JOIN d_start_dates - ON f_todos.date_start_key = d_start_dates.date_key -LEFT JOIN d_due_dates - ON f_todos.date_due_key = d_due_dates.date_key -LEFT JOIN d_completed_dates - ON f_todos.date_completed_key = d_completed_dates.date_key -LEFT JOIN d_created_dates - ON f_todos.date_created_key = d_created_dates.date_key diff --git a/models/marts/metrics/lvl1_lvl2_progress.sql b/models/marts/metrics/lvl1_lvl2_progress.sql deleted file mode 100644 index 22c3b96..0000000 --- a/models/marts/metrics/lvl1_lvl2_progress.sql +++ /dev/null @@ -1,182 +0,0 @@ --- pool -WITH pool AS ( - SELECT - *, - CASE - WHEN td_tags LIKE '%ClarifyMe%' THEN 'not_clarified' - WHEN td_tags NOT LIKE '%ClarifyMe%' THEN 'clarified' - END AS progress_type - FROM - {{ ref('obt') }} - WHERE - --ss_desc ='undone' - td_tags NOT LIKE '%SOMEDAY%' - AND td_tags NOT LIKE '%WAITING_FOR%' - AND td_tags NOT LIKE '%tickler%' - AND td_kind = 'TEXT' - AND l_is_active = TRUE - AND fld_folder_name NOT IN ( - '🚀SOMEDAY lists', - '🛩Horizon of focus' - ) - AND l_list_name NOT IN ('🧳SOMEDAY') -), - -done_progress AS ( - SELECT - fld_folder_name, - l_list_name, - SUM( - CASE - WHEN ss_desc = 'done' THEN 1 - ELSE 0 - END - ) AS cnt_done, - SUM( - CASE - WHEN ss_desc = 'undone' THEN 1 - ELSE 0 - END - ) AS cnt_not_done, - SUM( - CASE - WHEN ss_desc = 'done' THEN 1 - ELSE 0 - END - ) * 100.0 / ( - SUM( - CASE - WHEN ss_desc = 'done' THEN 1 - ELSE 0 - END - ) + SUM( - CASE - WHEN ss_desc = 'undone' THEN 1 - ELSE 0 - END - ) - ) AS done_progress - FROM - pool - GROUP BY - fld_folder_name, - l_list_name -), - -clarify_progress AS ( - SELECT - fld_folder_name, - l_list_name, - SUM( - CASE - WHEN - progress_type = 'clarified' - AND ss_desc = 'undone' THEN 1 - ELSE 0 - END - ) AS cnt_clarified, - SUM( - CASE - WHEN - progress_type = 'not_clarified' - AND ss_desc = 'undone' THEN 1 - ELSE 0 - END - ) AS cnt_not_clarified, - COALESCE( - ( - SUM( - CASE - WHEN - progress_type = 'clarified' - AND ss_desc = 'undone' THEN 1 - ELSE 0 - END - ) * 100.0 / ( - SUM( - CASE - WHEN - progress_type = 'clarified' - AND ss_desc = 'undone' THEN 1 - ELSE 0 - END - ) + SUM( - CASE - WHEN - progress_type = 'not_clarified' - AND ss_desc = 'undone' THEN 1 - ELSE 0 - END - ) - ) - ), - 100.0 - ) AS clarify_progress - FROM - pool - GROUP BY - fld_folder_name, - l_list_name -), - -lists_progress AS ( - SELECT - clarify_progress.fld_folder_name, - clarify_progress.l_list_name, - cnt_done, - cnt_not_done, - done_progress.done_progress, - cnt_clarified, - cnt_not_clarified, - clarify_progress - FROM - done_progress - INNER JOIN clarify_progress - ON - done_progress.l_list_name = clarify_progress.l_list_name - AND done_progress.fld_folder_name = clarify_progress.fld_folder_name -), - -folder_progress AS ( - -- aggregate folder progresss - SELECT - fld_folder_name, - '-----------------------' AS l_list_name, - 100 AS cnt_done, - 100 AS cnt_not_done, - folder_progress.folder_progress AS done_progress, - 100 AS cnt_clarified, - 100 AS cnt_not_clarified, - list_progress AS clarify_progress - FROM - ( - SELECT - fld_folder_name, - AVG(done_progress) AS folder_progress, - AVG(clarify_progress) AS list_progress - FROM - lists_progress - GROUP BY - fld_folder_name - ) AS folder_progress - UNION ALL - SELECT * - FROM - lists_progress -), - -staging AS ( - SELECT - fld_folder_name, - l_list_name, - done_progress, - clarify_progress - - - FROM folder_progress -) - -SELECT * -FROM - staging -ORDER BY 1, 2, 3, 4 diff --git a/models/marts/metrics/lvl3_progress.sql b/models/marts/metrics/lvl3_progress.sql deleted file mode 100644 index 5145ebc..0000000 --- a/models/marts/metrics/lvl3_progress.sql +++ /dev/null @@ -1,23 +0,0 @@ -with source as ( - select * from {{ ref('stg_duckdb__lvl3') }} - where goal_name is not null -), - -progress as ( - select * from {{ ref('lvl1_lvl2_progress') }} -) - -select - goal_id, - goal_name as lvl3_goal, - avg(done_progress) over (partition by goal_name) as lvl3_done_progress, - avg(clarify_progress) over (partition by goal_name) as lvl3_clarify_progress, - source.l_list_name, - progress.done_progress as l_done_progress, - progress.clarify_progress as l_clarify_progress - - -from source -left join progress - on source.l_list_name = progress.l_list_name -order by 1 diff --git a/models/marts/metrics/lvl3_sumarize.sql b/models/marts/metrics/lvl3_sumarize.sql deleted file mode 100644 index 7ffecb5..0000000 --- a/models/marts/metrics/lvl3_sumarize.sql +++ /dev/null @@ -1,6 +0,0 @@ -select distinct - goal_id, - lvl3_goal, - lvl3_done_progress -from {{ ref('lvl3_progress') }} -order by 1 diff --git a/models/marts/metrics/lvl3_unmapped.sql b/models/marts/metrics/lvl3_unmapped.sql deleted file mode 100644 index ed1a387..0000000 --- a/models/marts/metrics/lvl3_unmapped.sql +++ /dev/null @@ -1,15 +0,0 @@ -with source as ( - select * from {{ ref('lvl1_lvl2_progress') }} -), - -map as ( - select * from {{ ref('lvl3_progress') }} -) - -select source.* - - -from source left join map on source.l_list_name = map.l_list_name -where - map.l_list_name is null - and source.l_list_name not like '%------%' diff --git a/models/staging/init_duckdb__lvl3.sql b/models/staging/init_duckdb__lvl3.sql deleted file mode 100644 index 313c8ed..0000000 --- a/models/staging/init_duckdb__lvl3.sql +++ /dev/null @@ -1,16 +0,0 @@ -with source as ( - select - title as goal_name, - ROW_NUMBER() over (order by title) as goal_id - - from - {{ source('raw_data', 'raw_data') }} where "Folder Name" = '🛩Horizon of focus' - - and "List Name" like '%lvl3%' - and kind = 'TEXT' - - -) - -select * -from source diff --git a/models/staging/stg_duckdb__lvl3.sql b/models/staging/stg_duckdb__lvl3.sql deleted file mode 100644 index 8c2a5c0..0000000 --- a/models/staging/stg_duckdb__lvl3.sql +++ /dev/null @@ -1,22 +0,0 @@ -with map as ( - select * from {{ ref('list_goal_mapping') }} - -), - -goals as ( - select * from {{ ref('init_duckdb__lvl3') }} -), - -joined as ( - - select - goals.*, - map.fld_folder_name, - map.l_list_name - from - map full outer join - goals on position(',' || goals.goal_id || ',' in ',' || map.goal_ids || ',') > 0 - {# to allow for multi link rows #} -) - -select * from joined diff --git a/models/staging/stg_pg__dates.sql b/models/staging/stg_pg__dates.sql deleted file mode 100644 index e86f68f..0000000 --- a/models/staging/stg_pg__dates.sql +++ /dev/null @@ -1,14 +0,0 @@ -WITH source AS ( - SELECT * - FROM - {{ source( - 'raw_data', - 'date_seed' - ) }} -) - -SELECT - {{ dbt_utils.generate_surrogate_key(['date_id']) }} AS date_key, - * -FROM - source diff --git a/models/staging/stg_pg__folders.sql b/models/staging/stg_pg__folders.sql deleted file mode 100644 index feeb2af..0000000 --- a/models/staging/stg_pg__folders.sql +++ /dev/null @@ -1,35 +0,0 @@ -WITH source AS ( - SELECT * - FROM - {{ source ( - 'raw_data', - 'raw_data' - ) }} -), - -stg_folders AS ( - SELECT DISTINCT - s2.rn AS folder_id, - s1."Folder Name" AS folder_name - FROM - source AS s1 - INNER JOIN ( - SELECT - "Folder Name", - ROW_NUMBER() OVER ( - ORDER BY - "Folder Name" - ) AS rn - FROM - source - GROUP BY - "Folder Name" - ) AS s2 - ON s1."Folder Name" = s2."Folder Name" -) - -SELECT - {{ dbt_utils.generate_surrogate_key(['folder_id']) }} AS folder_key, - * -FROM - stg_folders diff --git a/models/staging/stg_pg__lists.sql b/models/staging/stg_pg__lists.sql deleted file mode 100644 index 3e6fdc6..0000000 --- a/models/staging/stg_pg__lists.sql +++ /dev/null @@ -1,46 +0,0 @@ -WITH source AS ( - SELECT * - FROM - {{ source ( - 'raw_data', - 'raw_data' - ) }} -), - -stg_lists AS ( - SELECT DISTINCT - list_id, - list_name::text AS list_name, - isactive::boolean AS is_active, - created_time::timestamp AS created_time - FROM - ( - SELECT - "List Name" AS list_name, - created_time, - ROW_NUMBER() OVER ( - ORDER BY - created_time ASC - ) AS list_id, - CASE - WHEN "List Name" LIKE '!%' THEN 0 - ELSE 1 - END AS isactive - FROM - ( - SELECT - "List Name", - MIN("Created Time") AS created_time - FROM - source - GROUP BY - "List Name" - ) AS p1 - ) AS p2 -) - -SELECT - {{ dbt_utils.generate_surrogate_key(['list_id']) }} AS list_key, - * -FROM - stg_lists diff --git a/models/staging/stg_pg__statuses.sql b/models/staging/stg_pg__statuses.sql deleted file mode 100644 index 822ca21..0000000 --- a/models/staging/stg_pg__statuses.sql +++ /dev/null @@ -1,33 +0,0 @@ -WITH source AS ( - SELECT * - FROM - {{ source ( - 'raw_data', - 'raw_data' - ) }} -) -, -stg_statuses AS ( - SELECT DISTINCT - CAST( - status AS INT - ) AS status_id, - CASE - WHEN status = '-1' THEN 'wont do' - WHEN status = '2' THEN 'done' - WHEN status = '0' THEN 'undone' - END AS "desc", - CASE - WHEN status = '-1' THEN 'regardless of archival' - WHEN status = '2' THEN 'regardless of archival' - WHEN status = '0' THEN 'regardless of archival' - END AS status_comments - FROM - source -) - -SELECT - {{ dbt_utils.generate_surrogate_key(['status_id']) }} AS status_key, - * -FROM - stg_statuses diff --git a/models/staging/stg_pg__todos.sql b/models/staging/stg_pg__todos.sql deleted file mode 100644 index d0b8970..0000000 --- a/models/staging/stg_pg__todos.sql +++ /dev/null @@ -1,80 +0,0 @@ -WITH source AS ( - SELECT * - FROM - {{ source ( - 'raw_data', - 'raw_data' - ) }} -), - -stg_fact_todo AS ( - SELECT - taskid::INT AS todo_id, - COALESCE( - "Folder Name", - 'Default' - )::TEXT AS folder_name, - COALESCE( - "List Name", - 'Default' - )::TEXT AS list_name, - status::INT AS status_id, - -- sl.list_id::int as list_id, - -- fld.folder_id::int as folder_id, - -- ss.status_id::int as status_id, - -- dd.date_id::text as date_id, - COALESCE( - "Start Date", - '1900-01-01' - )::TIMESTAMP AS start_date, - COALESCE( - "Due Date", - '1900-01-01' - )::TIMESTAMP AS due_date, - priority::INT AS priority, - "Created Time"::TIMESTAMP AS created_time, - COALESCE( - "Completed Time", - '1900-01-01' - )::TIMESTAMP AS completed_time, - COALESCE( - timezone, - 'Default' - )::TEXT AS timezone, - COALESCE( - "Is All Day", - 'Default' - )::TEXT AS isallday, - COALESCE( - "Is Floating", - 'Default' - )::TEXT AS isfloating, - COALESCE( - title, - 'Default' - ) AS title, - COALESCE( - kind, - 'Default' - ) AS kind, - COALESCE( - tags, - 'Default' - ) AS tags, - COALESCE( - reminder, - 'Default' - ) AS reminder, - COALESCE( - repeat, - 'Default' - ) AS repeat_ - FROM - source -) - -SELECT - {{ dbt_utils.generate_surrogate_key(['todo_id']) }} AS todo_key, - * -FROM - stg_fact_todo