Merge pull request #314 from SoryRawyer/rds/filter-on-course-key

refactor: use course_key index in dashboard queries (FC-0033)
openedx · Aug 24, 2023 · 461fcb7 · 461fcb7
2 parents a290d8c + 01f7d28
commit 461fcb7
Show file tree

Hide file tree

Showing 17 changed files with 320 additions and 29 deletions.
diff --git a/tutoraspects/templates/openedx-assets/assets/datasets/fact_enrollments.yaml b/tutoraspects/templates/openedx-assets/assets/datasets/fact_enrollments.yaml
@@ -37,6 +37,18 @@ columns:
   python_date_format: null
   type: String
   verbose_name: null
+- advanced_data_type: null
+  column_name: course_key
+  description: null
+  expression: null
+  extra: null
+  filterable: true
+  groupby: true
+  is_active: true
+  is_dttm: false
+  python_date_format: null
+  type: String
+  verbose_name: null
 - advanced_data_type: null
   column_name: course_name
   description: null
@@ -103,8 +115,8 @@ metrics:
   warning_text: null
 offset: 0
 params: null
-schema: {{ DBT_PROFILE_TARGET_DATABASE }}
-sql: null
+schema: null
+sql: "{% include 'openedx-assets/queries/fact_enrollments.sql' %}"
 table_name: fact_enrollments
 template_params: {}
 uuid: a234545d-08ff-480d-8361-961c3d15f14f

diff --git a/tutoraspects/templates/openedx-assets/assets/datasets/fact_enrollments_by_day.yaml b/tutoraspects/templates/openedx-assets/assets/datasets/fact_enrollments_by_day.yaml
@@ -37,6 +37,18 @@ columns:
   python_date_format: null
   type: String
   verbose_name: null
+- advanced_data_type: null
+  column_name: course_key
+  description: null
+  expression: null
+  extra: null
+  filterable: true
+  groupby: true
+  is_active: true
+  is_dttm: false
+  python_date_format: null
+  type: String
+  verbose_name: null
 - advanced_data_type: null
   column_name: course_name
   description: null

diff --git a/tutoraspects/templates/openedx-assets/assets/datasets/fact_learner_problem_summary.yaml b/tutoraspects/templates/openedx-assets/assets/datasets/fact_learner_problem_summary.yaml
@@ -73,6 +73,18 @@ columns:
   python_date_format: null
   type: String
   verbose_name: null
+- advanced_data_type: null
+  column_name: course_key
+  description: null
+  expression: null
+  extra: null
+  filterable: true
+  groupby: true
+  is_active: true
+  is_dttm: false
+  python_date_format: null
+  type: String
+  verbose_name: null
 - advanced_data_type: null
   column_name: course_name
   description: null
@@ -127,8 +139,8 @@ metrics:
   warning_text: null
 offset: 0
 params: null
-schema: {{ DBT_PROFILE_TARGET_DATABASE }}
-sql: null
+schema: null
+sql: "{% include 'openedx-assets/queries/fact_learner_problem_summary.sql' %}"
 table_name: fact_learner_problem_summary
 template_params: null
 uuid: 9362354c-1541-43c2-b769-da9818236298

diff --git a/tutoraspects/templates/openedx-assets/assets/datasets/fact_problem_responses.yaml b/tutoraspects/templates/openedx-assets/assets/datasets/fact_problem_responses.yaml
@@ -127,8 +127,8 @@ metrics:
   warning_text: null
 offset: 0
 params: null
-schema: {{ DBT_PROFILE_TARGET_DATABASE }}
-sql: null
+schema: null
+sql: "{% include 'openedx-assets/queries/fact_problem_responses.sql' %}"
 table_name: fact_problem_responses
 template_params: null
 uuid: 511dd5f1-3925-4e9e-ad8a-a227b5680047

diff --git a/tutoraspects/templates/openedx-assets/assets/datasets/fact_transcript_usage.yaml b/tutoraspects/templates/openedx-assets/assets/datasets/fact_transcript_usage.yaml
@@ -25,6 +25,18 @@ columns:
   python_date_format: null
   type: String
   verbose_name: null
+- advanced_data_type: null
+  column_name: course_key
+  description: null
+  expression: null
+  extra: null
+  filterable: true
+  groupby: true
+  is_active: true
+  is_dttm: false
+  python_date_format: null
+  type: String
+  verbose_name: null
 - advanced_data_type: null
   column_name: course_name
   description: null
@@ -92,8 +104,8 @@ metrics:
   warning_text: null
 offset: 0
 params: null
-schema: {{ DBT_PROFILE_TARGET_DATABASE }}
-sql: null
+schema: null
+sql: "{% include 'openedx-assets/queries/fact_transcript_usage.sql' %}"
 table_name: fact_transcript_usage
 template_params: {}
 uuid: a96a4b13-a429-442d-83ca-5d6f94010909

diff --git a/tutoraspects/templates/openedx-assets/assets/datasets/fact_video_plays.yaml b/tutoraspects/templates/openedx-assets/assets/datasets/fact_video_plays.yaml
@@ -25,6 +25,18 @@ columns:
   python_date_format: null
   type: String
   verbose_name: null
+- advanced_data_type: null
+  column_name: course_key
+  description: null
+  expression: null
+  extra: null
+  filterable: true
+  groupby: true
+  is_active: true
+  is_dttm: false
+  python_date_format: null
+  type: String
+  verbose_name: null
 - advanced_data_type: null
   column_name: course_name
   description: null
@@ -92,8 +104,8 @@ metrics:
   warning_text: null
 offset: 0
 params: null
-schema: {{ DBT_PROFILE_TARGET_DATABASE }}
-sql: null
+schema: null
+sql: "{% include 'openedx-assets/queries/fact_video_plays.sql' %}"
 table_name: fact_video_plays
 template_params: {}
 uuid: 6ec360a5-e247-42e7-b301-fa8275fc93f9

diff --git a/tutoraspects/templates/openedx-assets/assets/datasets/fact_watched_video_segments.yaml b/tutoraspects/templates/openedx-assets/assets/datasets/fact_watched_video_segments.yaml
@@ -37,6 +37,18 @@ columns:
   python_date_format: null
   type: String
   verbose_name: null
+- advanced_data_type: null
+  column_name: course_key
+  description: null
+  expression: null
+  extra: null
+  filterable: true
+  groupby: true
+  is_active: true
+  is_dttm: false
+  python_date_format: null
+  type: String
+  verbose_name: null
 - advanced_data_type: null
   column_name: course_name
   description: null

diff --git a/tutoraspects/templates/openedx-assets/assets/datasets/hints_per_success.yaml b/tutoraspects/templates/openedx-assets/assets/datasets/hints_per_success.yaml
@@ -37,6 +37,18 @@ columns:
   python_date_format: null
   type: String
   verbose_name: null
+- advanced_data_type: null
+  column_name: course_key
+  description: null
+  expression: null
+  extra: null
+  filterable: true
+  groupby: true
+  is_active: true
+  is_dttm: false
+  python_date_format: null
+  type: String
+  verbose_name: null
 - advanced_data_type: null
   column_name: course_name
   description: null

diff --git a/tutoraspects/templates/openedx-assets/queries/common_filters.sql b/tutoraspects/templates/openedx-assets/queries/common_filters.sql
@@ -0,0 +1,13 @@
+{% raw -%}
+{% if filter_values('org') != [] %}
+and org in {{ filter_values('org') | where_in }}
+{% endif %}
+
+{% if filter_values('course_name') != [] %}
+and course_key in (
+    select course_key
+    from {% endraw -%}{{ ASPECTS_EVENT_SINK_DATABASE }}.course_names{%- raw %}
+    where course_name in {{ filter_values('course_name') | where_in }}
+)
+{% endif %}
+{%- endraw %}
diff --git a/tutoraspects/templates/openedx-assets/queries/fact_enrollments.sql b/tutoraspects/templates/openedx-assets/queries/fact_enrollments.sql
@@ -0,0 +1,18 @@
+with enrollments as (
+select *
+from {{ DBT_PROFILE_TARGET_DATABASE }}.fact_enrollments
+where
+    1=1
+    {% include 'openedx-assets/queries/common_filters.sql' %}
+)
+
+select
+    emission_time,
+    org,
+    course_key,
+    course_name,
+    course_run,
+    actor_id,
+    enrollment_mode,
+    enrollment_status
+from enrollments
diff --git a/tutoraspects/templates/openedx-assets/queries/fact_enrollments_by_day.sql b/tutoraspects/templates/openedx-assets/queries/fact_enrollments_by_day.sql
@@ -1,24 +1,22 @@
-with enrollments_ranked as (
+with enrollments as (
+    {% include 'openedx-assets/queries/fact_enrollments.sql' %}
+), enrollments_ranked as (
   select
     emission_time,
     org,
+    course_key,
     course_name,
     course_run,
     actor_id,
     enrollment_mode,
     enrollment_status,
     rank() over (partition by date(emission_time), org, course_name, course_run, actor_id order by emission_time desc) as event_rank
   from
-    {{ DBT_PROFILE_TARGET_DATABASE }}.fact_enrollments
-  {% raw -%}
-  {% if filter_values('org') != [] %}
-    where
-      org in {{ filter_values('org') | where_in }}
-  {% endif %}
-  {%- endraw %}
+    enrollments
 ), enrollment_windows as (
   select
     org,
+    course_key,
     course_name,
     course_run,
     actor_id,
@@ -33,6 +31,7 @@ with enrollments_ranked as (
 ), enrollment_window_dates as (
     select
         org,
+        course_key,
         course_name,
         course_run,
         actor_id,
@@ -53,6 +52,7 @@ select
         )
     )) as enrollment_status_date,
     org,
+    course_key,
     course_name,
     course_run,
     actor_id,

diff --git a/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql b/tutoraspects/templates/openedx-assets/queries/fact_learner_problem_summary.sql
@@ -0,0 +1,120 @@
+WITH problem_responses AS (
+        {% include 'openedx-assets/queries/fact_problem_responses.sql' %}
+), outcomes AS (
+    SELECT
+        emission_time,
+        org,
+        course_key,
+        problem_id,
+        actor_id,
+        success,
+        first_value(success) OVER (PARTITION BY course_key, problem_id, actor_id ORDER BY success DESC) AS was_successful
+    FROM problem_responses
+), successful_responses AS (
+    SELECT
+        org,
+        course_key,
+        problem_id,
+        actor_id,
+        min(emission_time) AS first_success_at
+    FROM outcomes
+    WHERE was_successful = true and success = true
+    GROUP BY
+        org,
+        course_key,
+        problem_id,
+        actor_id
+), unsuccessful_responses AS (
+    SELECT
+        org,
+        course_key,
+        problem_id,
+        actor_id,
+        max(emission_time) AS last_response_at
+    FROM outcomes
+    WHERE was_successful = false
+    GROUP BY
+        org,
+        course_key,
+        problem_id,
+        actor_id
+), final_responses AS (
+    SELECT
+        org,
+        course_key,
+        problem_id,
+        actor_id,
+        first_success_at AS emission_time
+    FROM successful_responses
+    UNION ALL
+    SELECT
+        org,
+        course_key,
+        problem_id,
+        actor_id,
+        last_response_at AS emission_time
+    FROM unsuccessful_responses
+), int_problem_results AS (
+    SELECT
+        emission_time,
+        org,
+        course_key,
+        course_name,
+        course_run,
+        problem_id,
+        problem_name,
+        actor_id,
+        responses,
+        success,
+        attempts
+    FROM problem_responses
+    INNER JOIN final_responses USING (org, course_key, problem_id, actor_id, emission_time)
+), summary AS (
+    SELECT
+        org,
+        course_key,
+        course_name,
+        course_run,
+        problem_name,
+        actor_id,
+        success,
+        attempts,
+        0 AS num_hints_displayed,
+        0 AS num_answers_displayed
+    FROM int_problem_results
+    UNION ALL
+    SELECT
+        org,
+        course_key,
+        course_name,
+        course_run,
+        problem_name,
+        actor_id,
+        NULL AS success,
+        NULL AS attempts,
+        caseWithExpression(help_type, 'hint', 1, 0) AS num_hints_displayed,
+        caseWithExpression(help_type, 'answer', 1, 0) AS num_answers_displayed
+    FROM {{ DBT_PROFILE_TARGET_DATABASE }}.int_problem_hints
+    WHERE 1=1
+    {% include 'openedx-assets/queries/common_filters.sql' %}
+)
+
+SELECT
+    org,
+    course_key,
+    course_name,
+    course_run,
+    problem_name,
+    actor_id,
+    coalesce(any(success), false) AS success,
+    coalesce(any(attempts), 0) AS attempts,
+    sum(num_hints_displayed) AS num_hints_displayed,
+    sum(num_answers_displayed) AS num_answers_displayed
+FROM summary
+GROUP BY
+    org,
+    course_key,
+    course_name,
+    course_run,
+    problem_name,
+    actor_id