From 6240e76cdc867ec6166535689b106a37a1b40ee2 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Mon, 27 May 2024 13:44:41 -0500 Subject: [PATCH] chore: automatically format sql files --- Makefile | 2 + pyproject.toml | 2 + requirements/dev.in | 1 + requirements/dev.txt | 12 +- .../queries/at_risk_learner_filter.sql | 9 +- .../queries/at_risk_problem_results.sql | 8 +- .../openedx-assets/queries/common_filters.sql | 16 +- .../queries/dim_at_risk_learners.sql | 2 +- .../queries/dim_course_problems.sql | 16 +- .../queries/dim_course_videos.sql | 16 +- .../fact_at_risk_navigation_completion.sql | 12 +- .../fact_at_risk_pageview_engagement.sql | 9 +- .../fact_at_risk_problem_engagement.sql | 17 +- .../queries/fact_at_risk_video_engagement.sql | 11 +- .../queries/fact_at_risk_video_plays.sql | 10 +- .../queries/fact_at_risk_video_watches.sql | 10 +- .../fact_at_risk_watched_video_segments.sql | 10 +- .../queries/fact_course_grades.sql | 56 ++-- .../queries/fact_enrollment_pii.sql | 29 +- .../queries/fact_enrollments.sql | 13 +- .../queries/fact_enrollments_by_day.sql | 107 ++++---- .../queries/fact_forum_interactions.sql | 3 +- .../fact_learner_problem_course_summary.sql | 250 ++++++++---------- .../queries/fact_learner_problem_summary.sql | 221 ++++++++-------- .../queries/fact_learner_summary.sql | 16 +- .../queries/fact_navigation_completion.sql | 5 +- .../queries/fact_page_engagement.sql | 6 +- .../queries/fact_problem_engagement.sql | 11 +- .../queries/fact_problem_grades.sql | 61 ++--- .../queries/fact_problem_responses.sql | 27 +- .../queries/fact_transcript_usage.sql | 29 +- .../queries/fact_video_engagement.sql | 11 +- .../queries/fact_video_plays.sql | 32 ++- .../queries/fact_video_watches.sql | 15 +- .../queries/fact_watched_video_segments.sql | 6 +- .../queries/hints_per_success.sql | 8 +- .../openedx-assets/queries/indexed_events.sql | 48 ++-- .../queries/int_problem_responses.sql | 15 +- .../queries/int_problem_results.sql | 19 +- .../openedx-assets/queries/posts_per_user.sql | 28 +- .../queries/slowest_clickhouse_queries.sql | 16 +- .../queries/superset_action_log.sql | 79 ++---- .../openedx-assets/queries/user_filters.sql | 34 +-- 43 files changed, 607 insertions(+), 701 deletions(-) create mode 100644 pyproject.toml diff --git a/Makefile b/Makefile index 8df6c583a..a58b04cb3 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,7 @@ test: dev-requirements test-lint test-format test-pythonpackage ## Run all tests test-format: ## Run code formatting tests black --check --diff $(BLACK_OPTS) + sqlfmt tutoraspects/templates/openedx-assets/queries --check test-lint: ## Run code linting tests pylint ${SOURCES} @@ -66,6 +67,7 @@ test-pythonpackage: build-pythonpackage ## Test that package can be uploaded to format: ## Format code automatically black $(BLACK_OPTS) + sqlfmt tutoraspects/templates/openedx-assets/queries ###### Deployment diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..eea6167a5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.sqlfmt] +dialect="clickhouse" diff --git a/requirements/dev.in b/requirements/dev.in index 985b9c3dd..f8bc1917c 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -4,3 +4,4 @@ black pyinstaller pylint twine +shandy-sqlfmt[jinjafmt]==0.21.2 diff --git a/requirements/dev.txt b/requirements/dev.txt index be8836bbe..a61da5017 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -21,7 +21,9 @@ backports-tarfile==1.1.1 bcrypt==4.1.3 # via -r requirements/base.txt black==24.4.2 - # via -r requirements/dev.in + # via + # -r requirements/dev.in + # shandy-sqlfmt cachetools==5.3.3 # via # -r requirements/base.txt @@ -41,6 +43,7 @@ click==8.1.7 # via # -r requirements/base.txt # black + # shandy-sqlfmt # transifex-python # tutor cryptography==42.0.7 @@ -89,6 +92,7 @@ jeepney==0.8.0 jinja2==3.1.4 # via # -r requirements/base.txt + # shandy-sqlfmt # tutor keyring==25.2.1 # via twine @@ -145,6 +149,7 @@ platformdirs==4.2.2 # via # black # pylint + # shandy-sqlfmt pyasn1==0.6.0 # via # -r requirements/base.txt @@ -219,6 +224,8 @@ ruamel-yaml-clib==0.2.8 # ruamel-yaml secretstorage==3.3.3 # via keyring +shandy-sqlfmt[jinjafmt]==0.21.2 + # via -r requirements/dev.in six==1.16.0 # via # -r requirements/base.txt @@ -231,12 +238,15 @@ tomli==2.0.1 # black # mypy # pylint + # shandy-sqlfmt tomlkit==0.12.5 # via pylint toolz==0.12.1 # via # -r requirements/base.txt # pyseeyou +tqdm==4.66.4 + # via shandy-sqlfmt transifex-python==3.5.0 # via -r requirements/base.txt tutor==17.0.5 diff --git a/tutoraspects/templates/openedx-assets/queries/at_risk_learner_filter.sql b/tutoraspects/templates/openedx-assets/queries/at_risk_learner_filter.sql index 04f3df425..09db324af 100644 --- a/tutoraspects/templates/openedx-assets/queries/at_risk_learner_filter.sql +++ b/tutoraspects/templates/openedx-assets/queries/at_risk_learner_filter.sql @@ -3,16 +3,13 @@ with select org, course_key, actor_id, max(emission_time) as last_visited from {{ ASPECTS_XAPI_DATABASE }}.fact_learner_last_course_visit where - 1=1 + 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} and emission_time < subtractDays(now(), 7) group by org, course_key, actor_id ) -select - org, course_key, learners.actor_id as actor_id +select org, course_key, learners.actor_id as actor_id from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_student_status learners join page_visits using (org, course_key, actor_id) -where - approving_state = 'failed' - and enrollment_status = 'registered' +where approving_state = 'failed' and enrollment_status = 'registered' diff --git a/tutoraspects/templates/openedx-assets/queries/at_risk_problem_results.sql b/tutoraspects/templates/openedx-assets/queries/at_risk_problem_results.sql index 04ddb5e79..a8bd54b33 100644 --- a/tutoraspects/templates/openedx-assets/queries/at_risk_problem_results.sql +++ b/tutoraspects/templates/openedx-assets/queries/at_risk_problem_results.sql @@ -1,6 +1,6 @@ select results.* from {{ DBT_PROFILE_TARGET_DATABASE }}.int_problem_results results -join ( - {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %} -) as at_risk_learners -using (org, course_key, actor_id) +join + ( + {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %} + ) as at_risk_learners using (org, course_key, actor_id) diff --git a/tutoraspects/templates/openedx-assets/queries/common_filters.sql b/tutoraspects/templates/openedx-assets/queries/common_filters.sql index b57b646eb..a2467c646 100644 --- a/tutoraspects/templates/openedx-assets/queries/common_filters.sql +++ b/tutoraspects/templates/openedx-assets/queries/common_filters.sql @@ -1,13 +1,13 @@ {% raw -%} -{% if filter_values('org') != [] %} -and org in {{ filter_values('org') | where_in }} +{% if filter_values("org") != [] %} + and org in {{ filter_values("org") | where_in }} {% endif %} -{% if filter_values('course_name') != [] %} -and course_key in ( - select course_key - from {% endraw -%}{{ ASPECTS_EVENT_SINK_DATABASE }}.course_names{%- raw %} - where course_name in {{ filter_values('course_name') | where_in }} -) +{% if filter_values("course_name") != [] %} + and course_key in ( + select course_key + from {% endraw -%} {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names{%- raw %} + where course_name in {{ filter_values("course_name") | where_in }} + ) {% endif %} {%- endraw %} diff --git a/tutoraspects/templates/openedx-assets/queries/dim_at_risk_learners.sql b/tutoraspects/templates/openedx-assets/queries/dim_at_risk_learners.sql index 863aeca5e..94a43a71f 100644 --- a/tutoraspects/templates/openedx-assets/queries/dim_at_risk_learners.sql +++ b/tutoraspects/templates/openedx-assets/queries/dim_at_risk_learners.sql @@ -3,7 +3,7 @@ with select org, course_key, actor_id, max(emission_time) as last_visited from {{ ASPECTS_XAPI_DATABASE }}.fact_learner_last_course_visit where - 1=1 + 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} and emission_time < subtractDays(now(), 7) group by org, course_key, actor_id diff --git a/tutoraspects/templates/openedx-assets/queries/dim_course_problems.sql b/tutoraspects/templates/openedx-assets/queries/dim_course_problems.sql index 1745cedb3..1d014dba5 100644 --- a/tutoraspects/templates/openedx-assets/queries/dim_course_problems.sql +++ b/tutoraspects/templates/openedx-assets/queries/dim_course_problems.sql @@ -6,15 +6,15 @@ select block_id as problem_id, block_name as problem_name, display_name_with_location as problem_name_with_location -from - {{ DBT_PROFILE_TARGET_DATABASE }}.dim_course_blocks +from {{ DBT_PROFILE_TARGET_DATABASE }}.dim_course_blocks where problem_id like '%problem+block%' -{% raw -%} - {% if filter_values('org') != [] %} - and org in {{ filter_values('org') | where_in }} + {% raw -%} + {% if filter_values("org") != [] %} + and org in {{ filter_values("org") | where_in }} {% endif %} - {% if filter_values('problem_name_with_location') != [] %} - and problem_name_with_location in {{ filter_values('problem_name_with_location') | where_in }} + {% if filter_values("problem_name_with_location") != [] %} + and problem_name_with_location + in {{ filter_values("problem_name_with_location") | where_in }} {% endif %} -{%- endraw %} + {%- endraw %} diff --git a/tutoraspects/templates/openedx-assets/queries/dim_course_videos.sql b/tutoraspects/templates/openedx-assets/queries/dim_course_videos.sql index 373b4dd19..16a12a10a 100644 --- a/tutoraspects/templates/openedx-assets/queries/dim_course_videos.sql +++ b/tutoraspects/templates/openedx-assets/queries/dim_course_videos.sql @@ -6,15 +6,15 @@ select block_id as video_id, block_name as video_name, display_name_with_location as video_name_with_location -from - {{ DBT_PROFILE_TARGET_DATABASE }}.dim_course_blocks +from {{ DBT_PROFILE_TARGET_DATABASE }}.dim_course_blocks where video_id like '%video+block%' -{% raw -%} - {% if filter_values('org') != [] %} - and org in {{ filter_values('org') | where_in }} + {% raw -%} + {% if filter_values("org") != [] %} + and org in {{ filter_values("org") | where_in }} {% endif %} - {% if filter_values('video_name_with_location') != [] %} - and video_name_with_location in {{ filter_values('video_name_with_location') | where_in }} + {% if filter_values("video_name_with_location") != [] %} + and video_name_with_location + in {{ filter_values("video_name_with_location") | where_in }} {% endif %} -{%- endraw %} + {%- endraw %} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_at_risk_navigation_completion.sql b/tutoraspects/templates/openedx-assets/queries/fact_at_risk_navigation_completion.sql index dc8d2abe2..ea2894572 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_at_risk_navigation_completion.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_at_risk_navigation_completion.sql @@ -1,8 +1,6 @@ -select - fnc.* -from - {{ DBT_PROFILE_TARGET_DATABASE }}.fact_navigation_completion fnc - join ( +select fnc.* +from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_navigation_completion fnc +join + ( {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %} - ) as at_risk_learners - using (org, course_key, actor_id) + ) as at_risk_learners using (org, course_key, actor_id) diff --git a/tutoraspects/templates/openedx-assets/queries/fact_at_risk_pageview_engagement.sql b/tutoraspects/templates/openedx-assets/queries/fact_at_risk_pageview_engagement.sql index 79abc290a..c65b6828b 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_at_risk_pageview_engagement.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_at_risk_pageview_engagement.sql @@ -1,7 +1,6 @@ -select - fact_pageview_engagement.* +select fact_pageview_engagement.* from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_pageview_engagement - join ( +join + ( {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %} - ) as at_risk_learners - using (org, course_key, actor_id) + ) as at_risk_learners using (org, course_key, actor_id) diff --git a/tutoraspects/templates/openedx-assets/queries/fact_at_risk_problem_engagement.sql b/tutoraspects/templates/openedx-assets/queries/fact_at_risk_problem_engagement.sql index fc9cca4b4..bc1630cda 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_at_risk_problem_engagement.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_at_risk_problem_engagement.sql @@ -1,9 +1,10 @@ {% include 'openedx-assets/queries/fact_problem_engagement.sql' %} -join ( - {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %} -) as at_risk_learners -on ( - pe.org = at_risk_learners.org - and pe.course_key = at_risk_learners.course_key - and pe.actor_id = at_risk_learners.actor_id -) +join + ( + {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %} + ) as at_risk_learners + on ( + pe.org = at_risk_learners.org + and pe.course_key = at_risk_learners.course_key + and pe.actor_id = at_risk_learners.actor_id + ) diff --git a/tutoraspects/templates/openedx-assets/queries/fact_at_risk_video_engagement.sql b/tutoraspects/templates/openedx-assets/queries/fact_at_risk_video_engagement.sql index 82ed79f9b..b5a9b793a 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_at_risk_video_engagement.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_at_risk_video_engagement.sql @@ -1,7 +1,6 @@ -select - fact_video_engagement.* +select fact_video_engagement.* from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_video_engagement -join ( - {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %} -) as at_risk_learners -using (org, course_key, actor_id) +join + ( + {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %} + ) as at_risk_learners using (org, course_key, actor_id) diff --git a/tutoraspects/templates/openedx-assets/queries/fact_at_risk_video_plays.sql b/tutoraspects/templates/openedx-assets/queries/fact_at_risk_video_plays.sql index 60a192b00..b22e43c4b 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_at_risk_video_plays.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_at_risk_video_plays.sql @@ -1,10 +1,8 @@ -with video_plays as ( - {% include 'openedx-assets/queries/fact_video_plays.sql' %} -) +with video_plays as ({% include 'openedx-assets/queries/fact_video_plays.sql' %}) select video_plays.* from video_plays - join ( +join + ( {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %} - ) as at_risk_learners - using (org, course_key, actor_id) + ) as at_risk_learners using (org, course_key, actor_id) diff --git a/tutoraspects/templates/openedx-assets/queries/fact_at_risk_video_watches.sql b/tutoraspects/templates/openedx-assets/queries/fact_at_risk_video_watches.sql index b0de479ad..86efff57d 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_at_risk_video_watches.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_at_risk_video_watches.sql @@ -1,10 +1,8 @@ -with watches as ( - {% include 'openedx-assets/queries/fact_video_watches.sql' %} -) +with watches as ({% include 'openedx-assets/queries/fact_video_watches.sql' %}) select watches.* from watches - join ( +join + ( {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %} - ) as at_risk_learners - using (org, course_key, actor_id) + ) as at_risk_learners using (org, course_key, actor_id) diff --git a/tutoraspects/templates/openedx-assets/queries/fact_at_risk_watched_video_segments.sql b/tutoraspects/templates/openedx-assets/queries/fact_at_risk_watched_video_segments.sql index b270585e6..df054f3a9 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_at_risk_watched_video_segments.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_at_risk_watched_video_segments.sql @@ -1,10 +1,8 @@ -with watches as ( - {% include 'openedx-assets/queries/fact_watched_video_segments.sql' %} -) +with watches as ({% include 'openedx-assets/queries/fact_watched_video_segments.sql' %}) select watches.* from watches - join ( +join + ( {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %} - ) as at_risk_learners - using (org, course_key, actor_id) + ) as at_risk_learners using (org, course_key, actor_id) diff --git a/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql b/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql index 6341e83c1..08c489444 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql @@ -1,32 +1,24 @@ -with grades as ( - select * - from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_grades - where grade_type = 'course' - {% raw %} - {% if get_filters('course_name', remove_filter=True) == [] %} - {% elif filter_values('course_name') != [] %} - and entity_name in {{ filter_values('course_name', remove_filter=True) | where_in }} - {% else %} - and 1=0 - {% endif %} - {% endraw %} - {% include 'openedx-assets/queries/common_filters.sql' %} -), -most_recent_grades as ( - select - org, - course_key, - entity_id, - actor_id, - max(emission_time) as emission_time - from - grades - group by - org, - course_key, - entity_id, - actor_id -) +with + grades as ( + select * + from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_grades + where + grade_type = 'course' + {% raw %} + {% if get_filters("course_name", remove_filter=True) == [] %} + {% elif filter_values("course_name") != [] %} + and entity_name + in {{ filter_values("course_name", remove_filter=True) | where_in }} + {% else %} and 1 = 0 + {% endif %} + {% endraw %} + {% include 'openedx-assets/queries/common_filters.sql' %} + ), + most_recent_grades as ( + select org, course_key, entity_id, actor_id, max(emission_time) as emission_time + from grades + group by org, course_key, entity_id, actor_id + ) select grades.emission_time as emission_time, @@ -39,7 +31,5 @@ select grades.grade_type as grade_type, grades.scaled_score as scaled_score, grades.grade_bucket as grade_bucket -from - grades - join most_recent_grades - using (org, course_key, entity_id, actor_id, emission_time) +from grades +join most_recent_grades using (org, course_key, entity_id, actor_id, emission_time) diff --git a/tutoraspects/templates/openedx-assets/queries/fact_enrollment_pii.sql b/tutoraspects/templates/openedx-assets/queries/fact_enrollment_pii.sql index 1e09a106e..52c0a860b 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_enrollment_pii.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_enrollment_pii.sql @@ -1,16 +1,16 @@ -with enrollments as ( - select - enrollment.actor_id, - enrollment.course_key, - enrollment.org, - course_names.course_name as course_name - from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_enrollment_status enrollment - inner join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names as course_names - on course_names.course_key = enrollment.course_key - where - 1=1 - {% include 'openedx-assets/queries/common_filters.sql' %} -) +with + enrollments as ( + select + enrollment.actor_id, + enrollment.course_key, + enrollment.org, + course_names.course_name as course_name + from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_enrollment_status enrollment + inner join + {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names as course_names + on course_names.course_key = enrollment.course_key + where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} + ) select users.username as username, @@ -20,5 +20,6 @@ select enrollments.course_key as course_key, enrollments.course_name as course_name from enrollments -inner join {{ DBT_PROFILE_TARGET_DATABASE }}.dim_user_pii as users +inner join + {{ DBT_PROFILE_TARGET_DATABASE }}.dim_user_pii as users on enrollments.actor_id = users.external_user_id::String diff --git a/tutoraspects/templates/openedx-assets/queries/fact_enrollments.sql b/tutoraspects/templates/openedx-assets/queries/fact_enrollments.sql index 091d2f566..528540135 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_enrollments.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_enrollments.sql @@ -1,10 +1,9 @@ -with enrollments_base as ( -select * -from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_enrollments -where - 1=1 - {% include 'openedx-assets/queries/common_filters.sql' %} -) +with + enrollments_base as ( + select * + from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_enrollments + where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} + ) select emission_time, diff --git a/tutoraspects/templates/openedx-assets/queries/fact_enrollments_by_day.sql b/tutoraspects/templates/openedx-assets/queries/fact_enrollments_by_day.sql index da5b5a176..c41f9e9d2 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_enrollments_by_day.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_enrollments_by_day.sql @@ -1,56 +1,63 @@ -with enrollments as ( - {% include 'openedx-assets/queries/fact_enrollments.sql' %} -), enrollments_ranked as ( - select - emission_time, - org, - course_key, - course_name, - course_run, - actor_id, - enrollment_mode, - enrollment_status, - rank() over (partition by date(emission_time), org, course_name, course_run, actor_id order by emission_time desc) as event_rank - from - enrollments -), enrollment_windows as ( - select - org, - course_key, - course_name, - course_run, - actor_id, - enrollment_status, - enrollment_mode, - emission_time as window_start_at, - lagInFrame(emission_time, 1, now() + interval '1' day) over (partition by org, course_name, course_run, actor_id order by emission_time desc) as window_end_at - from - enrollments_ranked - where - event_rank = 1 -), enrollment_window_dates as ( - select - org, - course_key, - course_name, - course_run, - actor_id, - enrollment_status, - enrollment_mode, - date_trunc('day', window_start_at) as window_start_date, - date_trunc('day', window_end_at) as window_end_date - from enrollment_windows -) +with + enrollments as ({% include 'openedx-assets/queries/fact_enrollments.sql' %}), + enrollments_ranked as ( + select + emission_time, + org, + course_key, + course_name, + course_run, + actor_id, + enrollment_mode, + enrollment_status, + rank() over ( + partition by date(emission_time), org, course_name, course_run, actor_id + order by emission_time desc + ) as event_rank + from enrollments + ), + enrollment_windows as ( + select + org, + course_key, + course_name, + course_run, + actor_id, + enrollment_status, + enrollment_mode, + emission_time as window_start_at, + lagInFrame(emission_time, 1, now() + interval '1' day) over ( + partition by org, course_name, course_run, actor_id + order by emission_time desc + ) as window_end_at + from enrollments_ranked + where event_rank = 1 + ), + enrollment_window_dates as ( + select + org, + course_key, + course_name, + course_run, + actor_id, + enrollment_status, + enrollment_mode, + date_trunc('day', window_start_at) as window_start_date, + date_trunc('day', window_end_at) as window_end_date + from enrollment_windows + ) select - date(fromUnixTimestamp( - arrayJoin( - range( - toUnixTimestamp(window_start_date), - toUnixTimestamp(window_end_date), - 86400 + date( + fromUnixTimestamp( + arrayJoin( + range( + toUnixTimestamp(window_start_date), + toUnixTimestamp(window_end_date), + 86400 + ) ) ) - )) as enrollment_status_date, + ) as enrollment_status_date, org, course_key, course_name, diff --git a/tutoraspects/templates/openedx-assets/queries/fact_forum_interactions.sql b/tutoraspects/templates/openedx-assets/queries/fact_forum_interactions.sql index 1126c18bc..4570c0aa0 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_forum_interactions.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_forum_interactions.sql @@ -1,4 +1,3 @@ select * from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_forum_interactions -where 1=1 - {% include 'openedx-assets/queries/common_filters.sql' %} +where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql index 0046b1e91..29ff1a79c 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_course_summary.sql @@ -1,125 +1,111 @@ -WITH problem_responses AS ( +with + problem_responses as ( {% include 'openedx-assets/queries/int_problem_responses.sql' %} -), outcomes AS ( - SELECT - emission_time, - org, - course_key, - problem_id, - actor_id, - success, - first_value(success) OVER (PARTITION BY course_key, problem_id, actor_id ORDER BY success DESC) AS was_successful - FROM problem_responses -), successful_responses AS ( - SELECT - org, - course_key, - problem_id, - actor_id, - min(emission_time) AS first_success_at - FROM outcomes - WHERE was_successful = true and success = true - GROUP BY - org, - course_key, - problem_id, - actor_id -), unsuccessful_responses AS ( - SELECT - org, - course_key, - problem_id, - actor_id, - max(emission_time) AS last_response_at - FROM outcomes - WHERE was_successful = false - GROUP BY - org, - course_key, - problem_id, - actor_id -), final_responses AS ( - SELECT - org, - course_key, - problem_id, - actor_id, - first_success_at AS emission_time - FROM successful_responses - UNION ALL - SELECT - org, - course_key, - problem_id, - actor_id, - last_response_at AS emission_time - FROM unsuccessful_responses -), int_problem_results AS ( - SELECT - emission_time, - org, - course_key, - course_name, - course_run, - problem_id, - problem_name, - problem_name_with_location, - actor_id, - responses, - success, - attempts - FROM problem_responses - INNER JOIN final_responses USING (org, course_key, problem_id, actor_id, emission_time) -), summary AS ( - SELECT - org, - course_key, - course_name, - course_run, - problem_name, - problem_name_with_location, - actor_id, - success, - attempts, - 0 AS num_hints_displayed, - 0 AS num_answers_displayed - FROM int_problem_results - WHERE 1=1 - {% raw %} - {% if from_dttm %} - and emission_time > '{{ from_dttm }}' - {% endif %} - {% if to_dttm %} - and emission_time < '{{ to_dttm }}' - {% endif %} - {% endraw %} - UNION ALL - SELECT - org, - course_key, - course_name, - course_run, - problem_name, - problem_name_with_location, - actor_id, - NULL AS success, - NULL AS attempts, - caseWithExpression(help_type, 'hint', 1, 0) AS num_hints_displayed, - caseWithExpression(help_type, 'answer', 1, 0) AS num_answers_displayed - FROM {{ DBT_PROFILE_TARGET_DATABASE }}.int_problem_hints - WHERE 1=1 - {% raw %} - {% if from_dttm %} - and emission_time > '{{ from_dttm }}' - {% endif %} - {% if to_dttm %} - and emission_time < '{{ to_dttm }}' - {% endif %} - {% endraw %} - {% include 'openedx-assets/queries/common_filters.sql' %} -) + ), + outcomes as ( + select + emission_time, + org, + course_key, + problem_id, + actor_id, + success, + first_value(success) over ( + partition by course_key, problem_id, actor_id order by success DESC + ) as was_successful + from problem_responses + ), + successful_responses as ( + select + org, + course_key, + problem_id, + actor_id, + min(emission_time) as first_success_at + from outcomes + where was_successful = true and success = true + group by org, course_key, problem_id, actor_id + ), + unsuccessful_responses as ( + select + org, + course_key, + problem_id, + actor_id, + max(emission_time) as last_response_at + from outcomes + where was_successful = false + group by org, course_key, problem_id, actor_id + ), + final_responses as ( + select org, course_key, problem_id, actor_id, first_success_at as emission_time + from successful_responses + union all + select org, course_key, problem_id, actor_id, last_response_at as emission_time + from unsuccessful_responses + ), + int_problem_results as ( + select + emission_time, + org, + course_key, + course_name, + course_run, + problem_id, + problem_name, + problem_name_with_location, + actor_id, + responses, + success, + attempts + from problem_responses + inner join + final_responses using (org, course_key, problem_id, actor_id, emission_time) + ), + summary as ( + select + org, + course_key, + course_name, + course_run, + problem_name, + problem_name_with_location, + actor_id, + success, + attempts, + 0 as num_hints_displayed, + 0 as num_answers_displayed + from int_problem_results + where + 1 = 1 + {% raw %} + {% if from_dttm %} and emission_time > '{{ from_dttm }}' {% endif %} + {% if to_dttm %} and emission_time < '{{ to_dttm }}' {% endif %} + {% endraw %} + union all + select + org, + course_key, + course_name, + course_run, + problem_name, + problem_name_with_location, + actor_id, + NULL as success, + NULL as attempts, + caseWithExpression(help_type, 'hint', 1, 0) as num_hints_displayed, + caseWithExpression(help_type, 'answer', 1, 0) as num_answers_displayed + from {{ DBT_PROFILE_TARGET_DATABASE }}.int_problem_hints + where + 1 = 1 + {% raw %} + {% if from_dttm %} and emission_time > '{{ from_dttm }}' {% endif %} + {% if to_dttm %} and emission_time < '{{ to_dttm }}' {% endif %} + {% endraw %} + {% include 'openedx-assets/queries/common_filters.sql' %} + ) -SELECT +select org, course_key, course_name, @@ -127,22 +113,20 @@ SELECT problem_name, problem_name_with_location, actor_id, - coalesce(any(success), false) AS success, - coalesce(any(attempts), 0) AS attempts, - sum(num_hints_displayed) AS num_hints_displayed, - sum(num_answers_displayed) AS num_answers_displayed -FROM summary -WHERE + coalesce(any(success), false) as success, + coalesce(any(attempts), 0) as attempts, + sum(num_hints_displayed) as num_hints_displayed, + sum(num_answers_displayed) as num_answers_displayed +from summary +where {% raw %} - {% if get_filters('course_name', remove_filter=True) == [] %} - 1=1 - {% elif filter_values('course_name') != [] %} - course_name in {{ filter_values('course_name') | where_in }} - {% else %} - 1=0 + {% if get_filters("course_name", remove_filter=True) == [] %} 1 = 1 + {% elif filter_values("course_name") != [] %} + course_name in {{ filter_values("course_name") | where_in }} + {% else %} 1 = 0 {% endif %} {% endraw %} -GROUP BY +group by org, course_key, course_name, diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql index 81dcbded4..e08113378 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql @@ -1,109 +1,99 @@ -WITH problem_responses AS ( +with + problem_responses as ( {% include 'openedx-assets/queries/int_problem_responses.sql' %} -), outcomes AS ( - SELECT - emission_time, - org, - course_key, - problem_id, - actor_id, - success, - first_value(success) OVER (PARTITION BY course_key, problem_id, actor_id ORDER BY success ASC) AS was_successful - FROM problem_responses -), successful_responses AS ( - SELECT - org, - course_key, - problem_id, - actor_id, - min(emission_time) AS first_success_at - FROM outcomes - WHERE was_successful = true and success = true - GROUP BY - org, - course_key, - problem_id, - actor_id -), unsuccessful_responses AS ( - SELECT - org, - course_key, - problem_id, - actor_id, - max(emission_time) AS last_response_at - FROM outcomes - WHERE was_successful = false - GROUP BY - org, - course_key, - problem_id, - actor_id -), final_responses AS ( - SELECT - org, - course_key, - problem_id, - actor_id, - first_success_at AS emission_time - FROM successful_responses - UNION ALL - SELECT - org, - course_key, - problem_id, - actor_id, - last_response_at AS emission_time - FROM unsuccessful_responses -), int_problem_results AS ( - SELECT - emission_time, - org, - course_key, - course_name, - course_run, - problem_id, - problem_name, - problem_name_with_location, - actor_id, - responses, - success, - attempts - FROM problem_responses - INNER JOIN final_responses USING (org, course_key, problem_id, actor_id, emission_time) -), summary_base AS ( - SELECT - org, - course_key, - course_name, - course_run, - problem_name, - problem_name_with_location, - actor_id, - success, - attempts, - 0 AS num_hints_displayed, - 0 AS num_answers_displayed - FROM int_problem_results - UNION ALL - SELECT - org, - course_key, - course_name, - course_run, - problem_name, - problem_name_with_location, - actor_id, - NULL AS success, - NULL AS attempts, - caseWithExpression(help_type, 'hint', 1, 0) AS num_hints_displayed, - caseWithExpression(help_type, 'answer', 1, 0) AS num_answers_displayed - FROM {{ DBT_PROFILE_TARGET_DATABASE }}.int_problem_hints - WHERE - 1=1 - {% include 'openedx-assets/queries/common_filters.sql' %} -) + ), + outcomes as ( + select + emission_time, + org, + course_key, + problem_id, + actor_id, + success, + first_value(success) over ( + partition by course_key, problem_id, actor_id order by success ASC + ) as was_successful + from problem_responses + ), + successful_responses as ( + select + org, + course_key, + problem_id, + actor_id, + min(emission_time) as first_success_at + from outcomes + where was_successful = true and success = true + group by org, course_key, problem_id, actor_id + ), + unsuccessful_responses as ( + select + org, + course_key, + problem_id, + actor_id, + max(emission_time) as last_response_at + from outcomes + where was_successful = false + group by org, course_key, problem_id, actor_id + ), + final_responses as ( + select org, course_key, problem_id, actor_id, first_success_at as emission_time + from successful_responses + union all + select org, course_key, problem_id, actor_id, last_response_at as emission_time + from unsuccessful_responses + ), + int_problem_results as ( + select + emission_time, + org, + course_key, + course_name, + course_run, + problem_id, + problem_name, + problem_name_with_location, + actor_id, + responses, + success, + attempts + from problem_responses + inner join + final_responses using (org, course_key, problem_id, actor_id, emission_time) + ), + summary_base as ( + select + org, + course_key, + course_name, + course_run, + problem_name, + problem_name_with_location, + actor_id, + success, + attempts, + 0 as num_hints_displayed, + 0 as num_answers_displayed + from int_problem_results + union all + select + org, + course_key, + course_name, + course_run, + problem_name, + problem_name_with_location, + actor_id, + NULL as success, + NULL as attempts, + caseWithExpression(help_type, 'hint', 1, 0) as num_hints_displayed, + caseWithExpression(help_type, 'answer', 1, 0) as num_answers_displayed + from {{ DBT_PROFILE_TARGET_DATABASE }}.int_problem_hints + where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} + ) -SELECT +select org, course_key, course_name, @@ -111,22 +101,21 @@ SELECT problem_name, problem_name_with_location, actor_id, - coalesce(any(success), false) AS success, - coalesce(any(attempts), 0) AS attempts, - sum(num_hints_displayed) AS num_hints_displayed, - sum(num_answers_displayed) AS num_answers_displayed -FROM summary_base + coalesce(any(success), false) as success, + coalesce(any(attempts), 0) as attempts, + sum(num_hints_displayed) as num_hints_displayed, + sum(num_answers_displayed) as num_answers_displayed +from summary_base where {% raw %} - {% if get_filters('problem_name_with_location', remove_filter=True) == [] %} - 1=1 - {% elif filter_values('problem_name_with_location') != [] %} - problem_name_with_location in {{ filter_values('problem_name_with_location') | where_in }} - {% else %} - 1=0 + {% if get_filters("problem_name_with_location", remove_filter=True) == [] %} 1 = 1 + {% elif filter_values("problem_name_with_location") != [] %} + problem_name_with_location + in {{ filter_values("problem_name_with_location") | where_in }} + {% else %} 1 = 0 {% endif %} {% endraw %} -GROUP BY +group by org, course_key, course_name, diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_summary.sql index 5ea649cc4..bd4af6c02 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_learner_summary.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_learner_summary.sql @@ -1,18 +1,18 @@ with latest_emission_time as ( select course_key, actor_id, MAX(emission_time) as last_visited - from {{DBT_PROFILE_TARGET_DATABASE}}.fact_navigation + from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_navigation where - 1=1 + 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} {% include 'openedx-assets/queries/user_filters.sql' %} group by course_key, actor_id ), enrollment_status as ( select course_key, actor_id, MAX(emission_time) as max_emission_time - from {{DBT_PROFILE_TARGET_DATABASE}}.fact_enrollment_status + from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_enrollment_status where - 1=1 + 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} {% include 'openedx-assets/queries/user_filters.sql' %} group by course_key, actor_id @@ -33,11 +33,11 @@ with fss.name as name, fss.email as email, fss.enrolled_at as enrolled_at - from {{DBT_PROFILE_TARGET_DATABASE}}.fact_student_status fss + from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_student_status fss where - 1=1 - {% include 'openedx-assets/queries/common_filters.sql' %} - {% include 'openedx-assets/queries/user_filters.sql' %} + 1 = 1 + {% include 'openedx-assets/queries/common_filters.sql' %} + {% include 'openedx-assets/queries/user_filters.sql' %} ) select fss.org as org, diff --git a/tutoraspects/templates/openedx-assets/queries/fact_navigation_completion.sql b/tutoraspects/templates/openedx-assets/queries/fact_navigation_completion.sql index e01632c92..ea52e96ef 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_navigation_completion.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_navigation_completion.sql @@ -1,4 +1,3 @@ select * -from {{DBT_PROFILE_TARGET_DATABASE}}.fact_navigation_completion -where 1=1 -{% include 'openedx-assets/queries/common_filters.sql' %} +from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_navigation_completion +where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_page_engagement.sql b/tutoraspects/templates/openedx-assets/queries/fact_page_engagement.sql index d25342040..ae3a4b63d 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_page_engagement.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_page_engagement.sql @@ -8,8 +8,7 @@ with subsection_block_id as block_id, engagement_level as section_subsection_page_engagement from {{ ASPECTS_XAPI_DATABASE }}.subsection_page_engagement - where 1=1 - {% include 'openedx-assets/queries/common_filters.sql' %} + where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} ), section_engagement as ( select @@ -49,4 +48,5 @@ join and pv.block_id = course_blocks.block_id ) left outer join - {{ DBT_PROFILE_TARGET_DATABASE }}.dim_user_pii users on toUUID(pv.actor_id) = users.external_user_id + {{ DBT_PROFILE_TARGET_DATABASE }}.dim_user_pii users + on toUUID(pv.actor_id) = users.external_user_id diff --git a/tutoraspects/templates/openedx-assets/queries/fact_problem_engagement.sql b/tutoraspects/templates/openedx-assets/queries/fact_problem_engagement.sql index f716058d6..ebe8d97f0 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_problem_engagement.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_problem_engagement.sql @@ -8,9 +8,7 @@ with subsection_block_id as block_id, engagement_level as section_subsection_problem_engagement from {{ ASPECTS_XAPI_DATABASE }}.subsection_problem_engagement - where - 1=1 - {% include 'openedx-assets/queries/common_filters.sql' %} + where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} ), section_engagement as ( @@ -22,9 +20,7 @@ with section_block_id as block_id, engagement_level as section_subsection_problem_engagement from {{ ASPECTS_XAPI_DATABASE }}.section_problem_engagement - where - 1=1 - {% include 'openedx-assets/queries/common_filters.sql' %} + where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} ), problem_engagement as ( select * @@ -53,4 +49,5 @@ join and pe.block_id = course_blocks.block_id ) left outer join - {{ DBT_PROFILE_TARGET_DATABASE }}.dim_user_pii users on toUUID(pe.actor_id) = users.external_user_id + {{ DBT_PROFILE_TARGET_DATABASE }}.dim_user_pii users + on toUUID(pe.actor_id) = users.external_user_id diff --git a/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql b/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql index d9be4306d..64dae4984 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_problem_grades.sql @@ -1,35 +1,30 @@ -with grades as ( - select * - from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_grades - where - grade_type = 'problem' +with + grades as ( + select * + from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_grades + where + grade_type = 'problem' - {% raw %} - {% if get_filters('problem_name_with_location', remove_filter=True) == [] %} - {% elif filter_values('problem_name_with_location') != [] %} - and entity_name_with_location in {{ filter_values('problem_name_with_location', remove_filter=True) | where_in }} - {% else %} - and 1=0 - {% endif %} - {% endraw %} + {% raw %} + {% if get_filters("problem_name_with_location", remove_filter=True) == [] %} + {% elif filter_values("problem_name_with_location") != [] %} + and entity_name_with_location + in {{ + filter_values( + "problem_name_with_location", remove_filter=True + ) | where_in + }} + {% else %} and 1 = 0 + {% endif %} + {% endraw %} - {% include 'openedx-assets/queries/common_filters.sql' %} -), -most_recent_grades as ( - select - org, - course_key, - entity_id, - actor_id, - max(emission_time) as emission_time - from - grades - group by - org, - course_key, - entity_id, - actor_id -) + {% include 'openedx-assets/queries/common_filters.sql' %} + ), + most_recent_grades as ( + select org, course_key, entity_id, actor_id, max(emission_time) as emission_time + from grades + group by org, course_key, entity_id, actor_id + ) select grades.emission_time as emission_time, @@ -43,7 +38,5 @@ select grades.grade_type as grade_type, grades.scaled_score as scaled_score, grades.grade_bucket as grade_bucket -from - grades - join most_recent_grades - using (org, course_key, entity_id, actor_id, emission_time) +from grades +join most_recent_grades using (org, course_key, entity_id, actor_id, emission_time) diff --git a/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql b/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql index e320b21db..fe273177b 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_problem_responses.sql @@ -1,6 +1,7 @@ -with problem_responses as ( - {% include 'openedx-assets/queries/int_problem_responses.sql' %} -) +with + problem_responses as ( + {% include 'openedx-assets/queries/int_problem_responses.sql' %} + ) select emission_time, @@ -15,21 +16,15 @@ select attempts, success, arrayJoin( - if( - JSONArrayLength(responses) > 0, - JSONExtractArrayRaw(responses), - [responses] - ) + if(JSONArrayLength(responses) > 0, JSONExtractArrayRaw(responses), [responses]) ) as responses -from - problem_responses +from problem_responses where {% raw %} - {% if get_filters('problem_name_with_location', remove_filter=True) == [] %} - 1=1 - {% elif filter_values('problem_name_with_location') != [] %} - problem_name_with_location in {{ filter_values('problem_name_with_location') | where_in }} - {% else %} - 1=0 + {% if get_filters("problem_name_with_location", remove_filter=True) == [] %} 1 = 1 + {% elif filter_values("problem_name_with_location") != [] %} + problem_name_with_location + in {{ filter_values("problem_name_with_location") | where_in }} + {% else %} 1 = 0 {% endif %} {% endraw %} diff --git a/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql b/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql index 3059cc779..57a2ccbea 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_transcript_usage.sql @@ -1,18 +1,17 @@ -with transcripts as ( -select * -from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_transcript_usage -where - {% raw %} - {% if get_filters('course_name', remove_filter=True) == [] %} - 1=1 - {% elif filter_values('course_name') != [] %} - course_name in {{ filter_values('course_name') | where_in }} - {% else %} - 1=0 - {% endif %} - {% endraw %} - {% include 'openedx-assets/queries/common_filters.sql' %} -) +with + transcripts as ( + select * + from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_transcript_usage + where + {% raw %} + {% if get_filters("course_name", remove_filter=True) == [] %} 1 = 1 + {% elif filter_values("course_name") != [] %} + course_name in {{ filter_values("course_name") | where_in }} + {% else %} 1 = 0 + {% endif %} + {% endraw %} + {% include 'openedx-assets/queries/common_filters.sql' %} + ) select emission_time, diff --git a/tutoraspects/templates/openedx-assets/queries/fact_video_engagement.sql b/tutoraspects/templates/openedx-assets/queries/fact_video_engagement.sql index 6670c5fff..300d30e55 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_video_engagement.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_video_engagement.sql @@ -8,9 +8,7 @@ with subsection_block_id as block_id, engagement_level as section_subsection_video_engagement from {{ ASPECTS_XAPI_DATABASE }}.subsection_video_engagement - where - 1=1 - {% include 'openedx-assets/queries/common_filters.sql' %} + where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} ), section_engagement as ( select @@ -21,9 +19,7 @@ with section_block_id as block_id, engagement_level as section_subsection_video_engagement from {{ ASPECTS_XAPI_DATABASE }}.section_video_engagement - where - 1=1 - {% include 'openedx-assets/queries/common_filters.sql' %} + where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} ), video_engagement as ( select * @@ -52,4 +48,5 @@ join and ve.block_id = course_blocks.block_id ) left outer join - {{ DBT_PROFILE_TARGET_DATABASE }}.dim_user_pii users on toUUID(ve.actor_id) = users.external_user_id + {{ DBT_PROFILE_TARGET_DATABASE }}.dim_user_pii users + on toUUID(ve.actor_id) = users.external_user_id diff --git a/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql b/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql index a516366ab..34977539b 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_video_plays.sql @@ -1,19 +1,17 @@ -with plays as ( -select * -from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_video_plays -where - {% raw %} - {% if get_filters('course_name', remove_filter=True) == [] %} - 1=1 - {% elif filter_values('course_name') != [] %} - course_name in {{ filter_values('course_name') | where_in }} - {% else %} - 1=0 - {% endif %} - {% endraw %} - {% include 'openedx-assets/queries/common_filters.sql' %} -) +with + plays as ( + select * + from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_video_plays + where + {% raw %} + {% if get_filters("course_name", remove_filter=True) == [] %} 1 = 1 + {% elif filter_values("course_name") != [] %} + course_name in {{ filter_values("course_name") | where_in }} + {% else %} 1 = 0 + {% endif %} + {% endraw %} + {% include 'openedx-assets/queries/common_filters.sql' %} + ) -select - * +select * from plays diff --git a/tutoraspects/templates/openedx-assets/queries/fact_video_watches.sql b/tutoraspects/templates/openedx-assets/queries/fact_video_watches.sql index b7cfa1a26..49ca3d914 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_video_watches.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_video_watches.sql @@ -1,6 +1,7 @@ -with watched_segments as ( - {% include 'openedx-assets/queries/fact_watched_video_segments.sql' %} -) +with + watched_segments as ( + {% include 'openedx-assets/queries/fact_watched_video_segments.sql' %} + ) select org, @@ -25,12 +26,8 @@ where {% if filter_values("Subsection Name") != [] %} and subsection_with_name in {{ filter_values("Subsection Name") | where_in }} {% endif %} - {% if from_dttm %} - and started_at > '{{ from_dttm }}' - {% endif %} - {% if to_dttm %} - and started_at < '{{ to_dttm }}' - {% endif %} + {% if from_dttm %} and started_at > '{{ from_dttm }}' {% endif %} + {% if to_dttm %} and started_at < '{{ to_dttm }}' {% endif %} {% endraw %} group by org, diff --git a/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql b/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql index 4aecb2945..7ee9fc769 100644 --- a/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql +++ b/tutoraspects/templates/openedx-assets/queries/fact_watched_video_segments.sql @@ -10,8 +10,7 @@ with video_position, video_duration from {{ ASPECTS_XAPI_DATABASE }}.video_playback_events - where 1=1 - {% include 'openedx-assets/queries/common_filters.sql' %} + where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} ), starts as ( select * @@ -96,5 +95,6 @@ select email from enriched_segments left outer join - {{ DBT_PROFILE_TARGET_DATABASE }}.dim_user_pii users on toUUID(actor_id) = users.external_user_id + {{ DBT_PROFILE_TARGET_DATABASE }}.dim_user_pii users + on toUUID(actor_id) = users.external_user_id order by segment_start diff --git a/tutoraspects/templates/openedx-assets/queries/hints_per_success.sql b/tutoraspects/templates/openedx-assets/queries/hints_per_success.sql index 65d7253ee..cf797bbfd 100644 --- a/tutoraspects/templates/openedx-assets/queries/hints_per_success.sql +++ b/tutoraspects/templates/openedx-assets/queries/hints_per_success.sql @@ -1,6 +1,5 @@ -with summary as ( - {% include 'openedx-assets/queries/fact_learner_problem_summary.sql' %} -) +with + summary as ({% include 'openedx-assets/queries/fact_learner_problem_summary.sql' %}) select org, @@ -11,8 +10,7 @@ select problem_name_with_location, actor_id, sum(num_hints_displayed) + sum(num_answers_displayed) as total_hints -from - summary +from summary where success = 1 group by org, diff --git a/tutoraspects/templates/openedx-assets/queries/indexed_events.sql b/tutoraspects/templates/openedx-assets/queries/indexed_events.sql index 682b68534..c188925cf 100644 --- a/tutoraspects/templates/openedx-assets/queries/indexed_events.sql +++ b/tutoraspects/templates/openedx-assets/queries/indexed_events.sql @@ -1,27 +1,27 @@ -with events as ( - SELECT +with + events as ( + select + event_id, + CAST(emission_time, 'DateTime') as emission_time, + actor_id, + object_id, + splitByString('/', course_id)[-1] as course_key, + org, + verb_id + from {{ ASPECTS_XAPI_DATABASE }}.xapi_events_all_parsed + ) + +select + courses.course_name as course_name, + courses.course_run as course_run, event_id, - CAST(emission_time, 'DateTime') AS emission_time, actor_id, object_id, - splitByString('/', course_id)[-1] AS course_key, - org, - verb_id - FROM {{ ASPECTS_XAPI_DATABASE }}.xapi_events_all_parsed -) - -SELECT - courses.course_name as course_name, - courses.course_run as course_run, - event_id, - actor_id, - object_id, - events.course_key, - events.org, - events.verb_id, - emission_time -FROM events -JOIN - {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names courses -ON - (events.course_key = courses.course_key) + events.course_key, + events.org, + events.verb_id, + emission_time +from events +join + {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names courses + on (events.course_key = courses.course_key) diff --git a/tutoraspects/templates/openedx-assets/queries/int_problem_responses.sql b/tutoraspects/templates/openedx-assets/queries/int_problem_responses.sql index 75e38e67a..f0e3c9e48 100644 --- a/tutoraspects/templates/openedx-assets/queries/int_problem_responses.sql +++ b/tutoraspects/templates/openedx-assets/queries/int_problem_responses.sql @@ -1,9 +1,9 @@ -with problem_responses_base as ( -select * -from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_problem_responses -where 1=1 - {% include 'openedx-assets/queries/common_filters.sql' %} -) +with + problem_responses_base as ( + select * + from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_problem_responses + where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %} + ) select emission_time, @@ -18,5 +18,4 @@ select attempts, success, responses -from - problem_responses_base +from problem_responses_base diff --git a/tutoraspects/templates/openedx-assets/queries/int_problem_results.sql b/tutoraspects/templates/openedx-assets/queries/int_problem_results.sql index 422031814..fd7773b4a 100644 --- a/tutoraspects/templates/openedx-assets/queries/int_problem_results.sql +++ b/tutoraspects/templates/openedx-assets/queries/int_problem_results.sql @@ -5,10 +5,12 @@ -- this will be used to pick the xAPI event corresponding to that submission with successful_responses as ( - select org, course_key, problem_id, actor_id::String as actor_id, first_success_at + select + org, course_key, problem_id, actor_id::String as actor_id, first_success_at from {{ ASPECTS_XAPI_DATABASE }}.responses - where isNotNull(first_success_at) - {% include 'openedx-assets/queries/common_filters.sql' %} + where + isNotNull(first_success_at) + {% include 'openedx-assets/queries/common_filters.sql' %} ), -- for all learners who did not submit a successful response, -- find the timestamp of the most recent unsuccessful response @@ -20,8 +22,9 @@ with actor_id::String as actor_id, max(last_attempt_at) as last_attempt_at from {{ ASPECTS_XAPI_DATABASE }}.responses - where actor_id not in (select distinct actor_id from successful_responses) - {% include 'openedx-assets/queries/common_filters.sql' %} + where + actor_id not in (select distinct actor_id from successful_responses) + {% include 'openedx-assets/queries/common_filters.sql' %} group by org, course_key, problem_id, actor_id ), -- combine result sets for successful and unsuccessful problem submissions @@ -59,7 +62,11 @@ select blocks.display_name_with_location as problem_name_with_location, blocks.course_order as course_order, concat( - '', blocks.block_name, '' + '', + blocks.block_name, + '' ) as problem_link, full_responses.actor_id as actor_id, full_responses.responses as responses, diff --git a/tutoraspects/templates/openedx-assets/queries/posts_per_user.sql b/tutoraspects/templates/openedx-assets/queries/posts_per_user.sql index 6c879934e..548b724b0 100644 --- a/tutoraspects/templates/openedx-assets/queries/posts_per_user.sql +++ b/tutoraspects/templates/openedx-assets/queries/posts_per_user.sql @@ -1,25 +1,9 @@ -select - org, - course_key, - course_name, - course_run, - actor_id, - count(*) as num_posts -from - {{ DBT_PROFILE_TARGET_DATABASE }}.fact_forum_interactions +select org, course_key, course_name, course_run, actor_id, count(*) as num_posts +from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_forum_interactions where - 1=1 + 1 = 1 {% raw %} - {% if from_dttm %} - and emission_time > '{{ from_dttm }}' - {% endif %} - {% if to_dttm %} - and emission_time < '{{ to_dttm }}' - {% endif %} + {% if from_dttm %} and emission_time > '{{ from_dttm }}' {% endif %} + {% if to_dttm %} and emission_time < '{{ to_dttm }}' {% endif %} {% endraw %} -group by - org, - course_key, - course_name, - course_run, - actor_id +group by org, course_key, course_name, course_run, actor_id diff --git a/tutoraspects/templates/openedx-assets/queries/slowest_clickhouse_queries.sql b/tutoraspects/templates/openedx-assets/queries/slowest_clickhouse_queries.sql index 4a450f40d..ea3037355 100644 --- a/tutoraspects/templates/openedx-assets/queries/slowest_clickhouse_queries.sql +++ b/tutoraspects/templates/openedx-assets/queries/slowest_clickhouse_queries.sql @@ -1,12 +1,10 @@ -SELECT +select event_time, - query_duration_ms / 1000 AS duration_secs, + query_duration_ms / 1000 as duration_secs, read_rows, - memory_usage / 1024 AS memory_usage_kb, + memory_usage / 1024 as memory_usage_kb, query -FROM - system.query_log -WHERE - type = 'QueryFinish' -ORDER BY - query_duration_ms DESC; +from system.query_log +where type = 'QueryFinish' +order by query_duration_ms DESC +; diff --git a/tutoraspects/templates/openedx-assets/queries/superset_action_log.sql b/tutoraspects/templates/openedx-assets/queries/superset_action_log.sql index 777880076..777f2fa15 100644 --- a/tutoraspects/templates/openedx-assets/queries/superset_action_log.sql +++ b/tutoraspects/templates/openedx-assets/queries/superset_action_log.sql @@ -1,62 +1,33 @@ -SELECT - l.dttm AS action_date, - CASE - WHEN LOWER( - l.action - ) = 'queries' THEN 'query from sqllab' - WHEN LOWER( - l.action - ) = 'chartrestapi.data' THEN 'query from charts' - WHEN LOWER( - l.action - ) = 'count' THEN 'dashboard view' - WHEN LOWER( - l.action - ) LIKE 'annotation%' THEN 'annotations' - WHEN LOWER( - l.action - ) LIKE 'css%' THEN 'CSS' - ELSE LOWER( - l.action - ) - END AS action, +select + l.dttm as action_date, + case + when LOWER(l.action) = 'queries' + then 'query from sqllab' + when LOWER(l.action) = 'chartrestapi.data' + then 'query from charts' + when LOWER(l.action) = 'count' + then 'dashboard view' + when LOWER(l.action) like 'annotation%' + then 'annotations' + when LOWER(l.action) like 'css%' + then 'CSS' + else LOWER(l.action) + end as action, l.user_id, - u.username AS user_name, - u.created_on AS user_registration_date, + u.username as user_name, + u.created_on as user_registration_date, l.dashboard_id, d.dashboard_title, - CASE - WHEN d.published = TRUE THEN 'published' - ELSE 'draft' - END AS dashboard_status, + case when d.published = TRUE then 'published' else 'draft' end as dashboard_status, l.slice_id, s.slice_name, s.datasource_type, s.datasource_name, s.datasource_id, - COUNT(1) AS action_count -FROM - logs AS l - LEFT JOIN ab_user AS u - ON u.id = l.user_id - LEFT JOIN dashboards AS d - ON d.id = l.dashboard_id - LEFT JOIN slices AS s - ON s.id = l.slice_id -WHERE - 1 = 1 - AND l.action != 'log' -GROUP BY - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13 + COUNT(1) as action_count +from logs as l +left join ab_user as u on u.id = l.user_id +left join dashboards as d on d.id = l.dashboard_id +left join slices as s on s.id = l.slice_id +where 1 = 1 and l.action != 'log' +group by 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 diff --git a/tutoraspects/templates/openedx-assets/queries/user_filters.sql b/tutoraspects/templates/openedx-assets/queries/user_filters.sql index 5412cdfa2..81a3e585a 100644 --- a/tutoraspects/templates/openedx-assets/queries/user_filters.sql +++ b/tutoraspects/templates/openedx-assets/queries/user_filters.sql @@ -1,21 +1,23 @@ {% raw -%} -{% if filter_values('username') != [] or filter_values('name') != [] or filter_values('email') != [] %} -and actor_id in ( - select external_user_id::String - from {% endraw -%}{{ ASPECTS_EVENT_SINK_DATABASE }}.user_pii{%- raw %} - where - 1=1 - {% if filter_values('username') != [] %} - and username in {{ filter_values('username') | where_in }} - {% endif %} - {% if filter_values('name') != [] %} - and name in {{ filter_values('name') | where_in }} - {% endif %} - {% if filter_values('email') != [] %} - and email in {{ filter_values('email') | where_in }} - {% endif %} -) +{% if filter_values("username") != [] or filter_values( + "name" +) != [] or filter_values("email") != [] %} + and actor_id in ( + select external_user_id::String + from {% endraw -%} {{ ASPECTS_EVENT_SINK_DATABASE }}.user_pii{%- raw %} + where + 1 = 1 + {% if filter_values("username") != [] %} + and username in {{ filter_values("username") | where_in }} + {% endif %} + {% if filter_values("name") != [] %} + and name in {{ filter_values("name") | where_in }} + {% endif %} + {% if filter_values("email") != [] %} + and email in {{ filter_values("email") | where_in }} + {% endif %} + ) {% endif %} {%- endraw %}