chore: automatically format sql files

Makefile

@@ -54,6 +54,7 @@ test: dev-requirements test-lint test-format test-pythonpackage ## Run all tests
 
 test-format: ## Run code formatting tests
 	black --check --diff $(BLACK_OPTS)
+	sqlfmt tutoraspects/templates/openedx-assets/queries --check
 
 test-lint: ## Run code linting tests
 	pylint ${SOURCES}
@@ -66,6 +67,7 @@ test-pythonpackage: build-pythonpackage ## Test that package can be uploaded to
 
 format: ## Format code automatically
 	black $(BLACK_OPTS)
+	sqlfmt tutoraspects/templates/openedx-assets/queries
 
 ###### Deployment
 

pyproject.toml

@@ -0,0 +1,2 @@
+[tool.sqlfmt]
+dialect="clickhouse"

requirements/dev.in

@@ -4,3 +4,4 @@ black
 pyinstaller
 pylint
 twine
+shandy-sqlfmt[jinjafmt]==0.21.2

requirements/dev.txt

@@ -21,7 +21,9 @@ backports-tarfile==1.1.1
 bcrypt==4.1.3
     # via -r requirements/base.txt
 black==24.4.2
-    # via -r requirements/dev.in
+    # via
+    #   -r requirements/dev.in
+    #   shandy-sqlfmt
 cachetools==5.3.3
     # via
     #   -r requirements/base.txt
@@ -41,6 +43,7 @@ click==8.1.7
     # via
     #   -r requirements/base.txt
     #   black
+    #   shandy-sqlfmt
     #   transifex-python
     #   tutor
 cryptography==42.0.7
@@ -89,6 +92,7 @@ jeepney==0.8.0
 jinja2==3.1.4
     # via
     #   -r requirements/base.txt
+    #   shandy-sqlfmt
     #   tutor
 keyring==25.2.1
     # via twine
@@ -145,6 +149,7 @@ platformdirs==4.2.2
     # via
     #   black
     #   pylint
+    #   shandy-sqlfmt
 pyasn1==0.6.0
     # via
     #   -r requirements/base.txt
@@ -219,6 +224,8 @@ ruamel-yaml-clib==0.2.8
     #   ruamel-yaml
 secretstorage==3.3.3
     # via keyring
+shandy-sqlfmt[jinjafmt]==0.21.2
+    # via -r requirements/dev.in
 six==1.16.0
     # via
     #   -r requirements/base.txt
@@ -231,12 +238,15 @@ tomli==2.0.1
     #   black
     #   mypy
     #   pylint
+    #   shandy-sqlfmt
 tomlkit==0.12.5
     # via pylint
 toolz==0.12.1
     # via
     #   -r requirements/base.txt
     #   pyseeyou
+tqdm==4.66.4
+    # via shandy-sqlfmt
 transifex-python==3.5.0
     # via -r requirements/base.txt
 tutor==17.0.5

tutoraspects/templates/openedx-assets/queries/at_risk_learner_filter.sql

@@ -3,16 +3,13 @@ with
         select org, course_key, actor_id, max(emission_time) as last_visited
         from {{ ASPECTS_XAPI_DATABASE }}.fact_learner_last_course_visit
         where
-            1=1
+            1 = 1
             {% include 'openedx-assets/queries/common_filters.sql' %}
             and emission_time < subtractDays(now(), 7)
         group by org, course_key, actor_id
     )
 
-select
-    org, course_key, learners.actor_id as actor_id
+select org, course_key, learners.actor_id as actor_id
 from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_student_status learners
 join page_visits using (org, course_key, actor_id)
-where
-    approving_state = 'failed'
-    and enrollment_status = 'registered'
+where approving_state = 'failed' and enrollment_status = 'registered'

tutoraspects/templates/openedx-assets/queries/at_risk_problem_results.sql

@@ -1,6 +1,6 @@
 select results.*
 from {{ DBT_PROFILE_TARGET_DATABASE }}.int_problem_results results
-join (
-    {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %}
-) as at_risk_learners
-using (org, course_key, actor_id)
+join
+    (
+        {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %}
+    ) as at_risk_learners using (org, course_key, actor_id)

tutoraspects/templates/openedx-assets/queries/common_filters.sql

@@ -1,13 +1,13 @@
 {% raw -%}
-{% if filter_values('org') != [] %}
-and org in {{ filter_values('org') | where_in }}
+{% if filter_values("org") != [] %}
+    and org in {{ filter_values("org") | where_in }}
 {% endif %}
 
-{% if filter_values('course_name') != [] %}
-and course_key in (
-    select course_key
-    from {% endraw -%}{{ ASPECTS_EVENT_SINK_DATABASE }}.course_names{%- raw %}
-    where course_name in {{ filter_values('course_name') | where_in }}
-)
+{% if filter_values("course_name") != [] %}
+    and course_key in (
+        select course_key
+        from {% endraw -%} {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names{%- raw %}
+        where course_name in {{ filter_values("course_name") | where_in }}
+    )
 {% endif %}
 {%- endraw %}

tutoraspects/templates/openedx-assets/queries/dim_at_risk_learners.sql

@@ -3,7 +3,7 @@ with
         select org, course_key, actor_id, max(emission_time) as last_visited
         from {{ ASPECTS_XAPI_DATABASE }}.fact_learner_last_course_visit
         where
-            1=1
+            1 = 1
             {% include 'openedx-assets/queries/common_filters.sql' %}
             and emission_time < subtractDays(now(), 7)
         group by org, course_key, actor_id

tutoraspects/templates/openedx-assets/queries/dim_course_problems.sql

@@ -6,15 +6,15 @@ select
     block_id as problem_id,
     block_name as problem_name,
     display_name_with_location as problem_name_with_location
-from
-    {{ DBT_PROFILE_TARGET_DATABASE }}.dim_course_blocks
+from {{ DBT_PROFILE_TARGET_DATABASE }}.dim_course_blocks
 where
     problem_id like '%problem+block%'
-{% raw -%}
-    {% if filter_values('org') != [] %}
-    and org in {{ filter_values('org') | where_in }}
+    {% raw -%}
+    {% if filter_values("org") != [] %}
+        and org in {{ filter_values("org") | where_in }}
     {% endif %}
-    {% if filter_values('problem_name_with_location') != [] %}
-    and problem_name_with_location in {{ filter_values('problem_name_with_location') | where_in }}
+    {% if filter_values("problem_name_with_location") != [] %}
+        and problem_name_with_location
+        in {{ filter_values("problem_name_with_location") | where_in }}
     {% endif %}
-{%- endraw %}
+    {%- endraw %}

tutoraspects/templates/openedx-assets/queries/dim_course_videos.sql

@@ -6,15 +6,15 @@ select
     block_id as video_id,
     block_name as video_name,
     display_name_with_location as video_name_with_location
-from
-    {{ DBT_PROFILE_TARGET_DATABASE }}.dim_course_blocks
+from {{ DBT_PROFILE_TARGET_DATABASE }}.dim_course_blocks
 where
     video_id like '%video+block%'
-{% raw -%}
-    {% if filter_values('org') != [] %}
-    and org in {{ filter_values('org') | where_in }}
+    {% raw -%}
+    {% if filter_values("org") != [] %}
+        and org in {{ filter_values("org") | where_in }}
     {% endif %}
-    {% if filter_values('video_name_with_location') != [] %}
-    and video_name_with_location in {{ filter_values('video_name_with_location') | where_in }}
+    {% if filter_values("video_name_with_location") != [] %}
+        and video_name_with_location
+        in {{ filter_values("video_name_with_location") | where_in }}
     {% endif %}
-{%- endraw %}
+    {%- endraw %}

tutoraspects/templates/openedx-assets/queries/fact_at_risk_navigation_completion.sql

@@ -1,8 +1,6 @@
-select
-    fnc.*
-from
-    {{ DBT_PROFILE_TARGET_DATABASE }}.fact_navigation_completion fnc
-    join (
+select fnc.*
+from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_navigation_completion fnc
+join
+    (
         {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %}
-    ) as at_risk_learners
-    using (org, course_key, actor_id)
+    ) as at_risk_learners using (org, course_key, actor_id)

tutoraspects/templates/openedx-assets/queries/fact_at_risk_pageview_engagement.sql

@@ -1,7 +1,6 @@
-select
-    fact_pageview_engagement.*
+select fact_pageview_engagement.*
 from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_pageview_engagement
-    join (
+join
+    (
         {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %}
-    ) as at_risk_learners
-    using (org, course_key, actor_id)
+    ) as at_risk_learners using (org, course_key, actor_id)

tutoraspects/templates/openedx-assets/queries/fact_at_risk_problem_engagement.sql

@@ -1,9 +1,10 @@
 {% include 'openedx-assets/queries/fact_problem_engagement.sql' %}
-join (
-    {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %}
-) as at_risk_learners
-on (
-    pe.org = at_risk_learners.org
-    and pe.course_key = at_risk_learners.course_key
-    and pe.actor_id = at_risk_learners.actor_id
-)
+join
+    (
+        {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %}
+    ) as at_risk_learners
+    on (
+        pe.org = at_risk_learners.org
+        and pe.course_key = at_risk_learners.course_key
+        and pe.actor_id = at_risk_learners.actor_id
+    )

tutoraspects/templates/openedx-assets/queries/fact_at_risk_video_engagement.sql

@@ -1,7 +1,6 @@
-select
-    fact_video_engagement.*
+select fact_video_engagement.*
 from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_video_engagement
-join (
-    {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %}
-) as at_risk_learners
-using (org, course_key, actor_id)
+join
+    (
+        {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %}
+    ) as at_risk_learners using (org, course_key, actor_id)

tutoraspects/templates/openedx-assets/queries/fact_at_risk_video_plays.sql

@@ -1,10 +1,8 @@
-with video_plays as (
-  {% include 'openedx-assets/queries/fact_video_plays.sql' %}
-)
+with video_plays as ({% include 'openedx-assets/queries/fact_video_plays.sql' %})
 
 select video_plays.*
 from video_plays
-    join (
+join
+    (
         {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %}
-    ) as at_risk_learners
-    using (org, course_key, actor_id)
+    ) as at_risk_learners using (org, course_key, actor_id)

tutoraspects/templates/openedx-assets/queries/fact_at_risk_video_watches.sql

@@ -1,10 +1,8 @@
-with watches as (
-  {% include 'openedx-assets/queries/fact_video_watches.sql' %}
-)
+with watches as ({% include 'openedx-assets/queries/fact_video_watches.sql' %})
 
 select watches.*
 from watches
-    join (
+join
+    (
         {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %}
-    ) as at_risk_learners
-    using (org, course_key, actor_id)
+    ) as at_risk_learners using (org, course_key, actor_id)

tutoraspects/templates/openedx-assets/queries/fact_at_risk_watched_video_segments.sql

@@ -1,10 +1,8 @@
-with watches as (
-  {% include 'openedx-assets/queries/fact_watched_video_segments.sql' %}
-)
+with watches as ({% include 'openedx-assets/queries/fact_watched_video_segments.sql' %})
 
 select watches.*
 from watches
-    join (
+join
+    (
         {% include 'openedx-assets/queries/at_risk_learner_filter.sql' %}
-    ) as at_risk_learners
-    using (org, course_key, actor_id)
+    ) as at_risk_learners using (org, course_key, actor_id)

tutoraspects/templates/openedx-assets/queries/fact_course_grades.sql

@@ -1,32 +1,24 @@
-with grades as (
-    select *
-    from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_grades
-    where grade_type = 'course'
-    {% raw %}
-    {% if get_filters('course_name', remove_filter=True) == [] %}
-    {% elif filter_values('course_name') != [] %}
-    and entity_name in {{ filter_values('course_name', remove_filter=True) | where_in }}
-    {% else %}
-    and 1=0
-    {% endif %}
-    {% endraw %}
-    {% include 'openedx-assets/queries/common_filters.sql' %}
-),
-most_recent_grades as (
-    select
-        org,
-        course_key,
-        entity_id,
-        actor_id,
-        max(emission_time) as emission_time
-    from
-        grades
-    group by
-        org,
-        course_key,
-        entity_id,
-        actor_id
-)
+with
+    grades as (
+        select *
+        from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_grades
+        where
+            grade_type = 'course'
+            {% raw %}
+            {% if get_filters("course_name", remove_filter=True) == [] %}
+            {% elif filter_values("course_name") != [] %}
+                and entity_name
+                in {{ filter_values("course_name", remove_filter=True) | where_in }}
+            {% else %} and 1 = 0
+            {% endif %}
+            {% endraw %}
+            {% include 'openedx-assets/queries/common_filters.sql' %}
+    ),
+    most_recent_grades as (
+        select org, course_key, entity_id, actor_id, max(emission_time) as emission_time
+        from grades
+        group by org, course_key, entity_id, actor_id
+    )
 
 select
     grades.emission_time as emission_time,
@@ -39,7 +31,5 @@ select
     grades.grade_type as grade_type,
     grades.scaled_score as scaled_score,
     grades.grade_bucket as grade_bucket
-from
-    grades
-    join most_recent_grades
-        using (org, course_key, entity_id, actor_id, emission_time)
+from grades
+join most_recent_grades using (org, course_key, entity_id, actor_id, emission_time)

tutoraspects/templates/openedx-assets/queries/fact_enrollment_pii.sql

@@ -1,16 +1,16 @@
-with enrollments as (
-    select
-        enrollment.actor_id,
-        enrollment.course_key,
-        enrollment.org,
-        course_names.course_name as course_name
-    from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_enrollment_status enrollment
-    inner join {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names as course_names
-        on course_names.course_key = enrollment.course_key
-    where
-        1=1
-        {% include 'openedx-assets/queries/common_filters.sql' %}
-)
+with
+    enrollments as (
+        select
+            enrollment.actor_id,
+            enrollment.course_key,
+            enrollment.org,
+            course_names.course_name as course_name
+        from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_enrollment_status enrollment
+        inner join
+            {{ ASPECTS_EVENT_SINK_DATABASE }}.course_names as course_names
+            on course_names.course_key = enrollment.course_key
+        where 1 = 1 {% include 'openedx-assets/queries/common_filters.sql' %}
+    )
 
 select
     users.username as username,
@@ -20,5 +20,6 @@ select
     enrollments.course_key as course_key,
     enrollments.course_name as course_name
 from enrollments
-inner join {{ DBT_PROFILE_TARGET_DATABASE }}.dim_user_pii as users
+inner join
+    {{ DBT_PROFILE_TARGET_DATABASE }}.dim_user_pii as users
     on enrollments.actor_id = users.external_user_id::String