Skip to content

Commit

Permalink
fix: use proper dataset columns (#785)
Browse files Browse the repository at this point in the history
* fix: only omit template vars for dictionaries

* feat: upgrade dbt

* fix: add missing verbose name for datasets

* fix: update fields to latest dbt changes

* fix: only select fact_video_engagement fields for at risk datasets

* chore: update engagement datasets columns

* chore: Bump aspects_dbt to 3.24.0

* feat: import new individual learner dashboard (#786)
  • Loading branch information
Ian2012 authored May 9, 2024
1 parent 7d77975 commit 1113805
Show file tree
Hide file tree
Showing 39 changed files with 2,204 additions and 553 deletions.
2 changes: 1 addition & 1 deletion tutoraspects/asset_command_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ def omit_templated_vars(self, content: dict, existing: dict):
Omit templated variables from the content if they are not present in
the existing file content.
"""
if not content or not existing:
if not isinstance(content, dict) or not isinstance(existing, dict):
return

for key in content.keys():
Expand Down
2 changes: 1 addition & 1 deletion tutoraspects/plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@
# For now we are pulling this from github, which should allow maximum
# flexibility for forking, running branches, specific versions, etc.
("DBT_REPOSITORY", "https://github.com/openedx/aspects-dbt"),
("DBT_BRANCH", "v3.23.0"),
("DBT_BRANCH", "v3.24.0"),
("DBT_SSH_KEY", ""),
("DBT_STATE_DIR", "/app/aspects/dbt_state/"),
# This is the name of the database dbt will write to
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
_file_name: Learner_Summary.yaml
cache_timeout: null
certification_details: null
certified_by: null
dataset_uuid: da75efef-09ba-4ed0-8b6b-bf11c5c26008
description: null
params:
adhoc_filters:
- clause: WHERE
comparator: No filter
expressionType: SIMPLE
operator: TEMPORAL_RANGE
subject: emission_time
all_columns: []
color_pn: true
conditional_formatting:
- colorScheme: '#ACE1C4'
column: course_grade
operator: "\u2265"
targetValue: 0.7
- colorScheme: '#FDE380'
column: course_grade
operator: "\u2264 x \u2264"
targetValueLeft: '0.5'
targetValueRight: '0.7'
- colorScheme: '#EFA1AA'
column: course_grade
operator: <
targetValue: 0.5
extra_form_data: {}
groupby:
- username
- name
- email
- course_name
- course_run
- expressionType: SQL
label: Enrollment Date
sqlExpression: emission_time
- last_visited
- expressionType: SQL
label: Passed/Failed
sqlExpression: approving_state
- expressionType: SQL
label: Enrollment Track
sqlExpression: enrollment_mode
- enrollment_status
- expressionType: SQL
label: Grade Range
sqlExpression: grade_bucket
- course_grade
order_by_cols: []
order_desc: true
percent_metrics: []
query_mode: aggregate
row_limit: 1000
server_page_length: 100
server_pagination: true
show_cell_bars: true
show_totals: true
table_timestamp_format: smart_date
temporal_columns_lookup:
emission_time: true
last_visited: true
time_grain_sqla: P1D
viz_type: table
query_context: "{\"datasource\":{\"id\":105,\"type\":\"table\"},\"force\":false,\"\
queries\":[{\"filters\":[{\"col\":\"emission_time\",\"op\":\"TEMPORAL_RANGE\",\"\
val\":\"No filter\"}],\"extras\":{\"time_grain_sqla\":\"P1D\",\"having\":\"\",\"\
where\":\"\"},\"applied_time_extras\":{},\"columns\":[\"username\",\"name\",\"email\"\
,\"course_name\",\"course_run\",{\"expressionType\":\"SQL\",\"label\":\"Enrollment\
\ Date\",\"sqlExpression\":\"emission_time\"},{\"timeGrain\":\"P1D\",\"columnType\"\
:\"BASE_AXIS\",\"sqlExpression\":\"last_visited\",\"label\":\"last_visited\",\"\
expressionType\":\"SQL\"},{\"expressionType\":\"SQL\",\"label\":\"Passed/Failed\"\
,\"sqlExpression\":\"approving_state\"},{\"label\":\"Enrollment Track\",\"sqlExpression\"\
:\"enrollment_mode\",\"expressionType\":\"SQL\"},\"enrollment_status\",{\"label\"\
:\"Grade Range\",\"sqlExpression\":\"grade_bucket\",\"expressionType\":\"SQL\"},\"\
course_grade\"],\"metrics\":[],\"orderby\":[],\"annotation_layers\":[],\"row_limit\"\
:100,\"row_offset\":0,\"series_limit\":0,\"order_desc\":true,\"url_params\":{},\"\
custom_params\":{},\"custom_form_data\":{},\"post_processing\":[]},{\"filters\"\
:[{\"col\":\"emission_time\",\"op\":\"TEMPORAL_RANGE\",\"val\":\"No filter\"}],\"\
extras\":{\"time_grain_sqla\":\"P1D\",\"having\":\"\",\"where\":\"\"},\"applied_time_extras\"\
:{},\"columns\":[\"username\",\"name\",\"email\",\"course_name\",\"course_run\"\
,{\"expressionType\":\"SQL\",\"label\":\"Enrollment Date\",\"sqlExpression\":\"\
emission_time\"},{\"timeGrain\":\"P1D\",\"columnType\":\"BASE_AXIS\",\"sqlExpression\"\
:\"last_visited\",\"label\":\"last_visited\",\"expressionType\":\"SQL\"},{\"expressionType\"\
:\"SQL\",\"label\":\"Passed/Failed\",\"sqlExpression\":\"approving_state\"},{\"\
label\":\"Enrollment Track\",\"sqlExpression\":\"enrollment_mode\",\"expressionType\"\
:\"SQL\"},\"enrollment_status\",{\"label\":\"Grade Range\",\"sqlExpression\":\"\
grade_bucket\",\"expressionType\":\"SQL\"},\"course_grade\"],\"metrics\":[],\"orderby\"\
:[],\"annotation_layers\":[],\"row_limit\":0,\"row_offset\":0,\"series_limit\":0,\"\
order_desc\":true,\"url_params\":{},\"custom_params\":{},\"custom_form_data\":{},\"\
post_processing\":[],\"is_rowcount\":true}],\"form_data\":{\"datasource\":\"105__table\"\
,\"viz_type\":\"table\",\"slice_id\":193,\"query_mode\":\"aggregate\",\"groupby\"\
:[\"username\",\"name\",\"email\",\"course_name\",\"course_run\",{\"expressionType\"\
:\"SQL\",\"label\":\"Enrollment Date\",\"sqlExpression\":\"emission_time\"},\"last_visited\"\
,{\"expressionType\":\"SQL\",\"label\":\"Passed/Failed\",\"sqlExpression\":\"approving_state\"\
},{\"label\":\"Enrollment Track\",\"sqlExpression\":\"enrollment_mode\",\"expressionType\"\
:\"SQL\"},\"enrollment_status\",{\"label\":\"Grade Range\",\"sqlExpression\":\"\
grade_bucket\",\"expressionType\":\"SQL\"},\"course_grade\"],\"time_grain_sqla\"\
:\"P1D\",\"temporal_columns_lookup\":{\"emission_time\":true,\"last_visited\":true},\"\
all_columns\":[],\"percent_metrics\":[],\"adhoc_filters\":[{\"clause\":\"WHERE\"\
,\"comparator\":\"No filter\",\"expressionType\":\"SIMPLE\",\"operator\":\"TEMPORAL_RANGE\"\
,\"subject\":\"emission_time\"}],\"order_by_cols\":[],\"server_pagination\":true,\"\
row_limit\":1000,\"server_page_length\":100,\"order_desc\":true,\"show_totals\"\
:true,\"table_timestamp_format\":\"smart_date\",\"show_cell_bars\":true,\"color_pn\"\
:true,\"conditional_formatting\":[{\"column\":\"course_grade\",\"colorScheme\":\"\
#ACE1C4\",\"operator\":\"\u2265\",\"targetValue\":0.7},{\"column\":\"course_grade\"\
,\"colorScheme\":\"#FDE380\",\"targetValueLeft\":\"0.5\",\"operator\":\"\u2264 x\
\ \u2264\",\"targetValueRight\":\"0.7\"},{\"column\":\"course_grade\",\"colorScheme\"\
:\"#EFA1AA\",\"operator\":\"<\",\"targetValue\":0.5}],\"extra_form_data\":{},\"\
dashboards\":[42],\"force\":false,\"result_format\":\"json\",\"result_type\":\"\
full\"},\"result_format\":\"json\",\"result_type\":\"full\"}"
slice_name: Learner Summary
uuid: 6da50964-000e-4342-a931-3881feb02c13
version: 1.0.0
viz_type: table
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
_file_name: Page_views_per_section.yaml
cache_timeout: null
certification_details: null
certified_by: null
dataset_uuid: 9febd6be-5102-4dbf-86b9-45ebd3cbbc45
description: null
params:
adhoc_filters:
- clause: WHERE
comparator: No filter
datasourceWarning: false
expressionType: SIMPLE
filterOptionName: filter_s8d5o5h20q_2r3i3mfdrkw
isExtra: false
isNew: false
operator: TEMPORAL_RANGE
sqlExpression: null
subject: visited_on
all_columns: []
color_pn: true
conditional_formatting:
- colorScheme: '#ACE1C4'
column: all_pages_viewed
operator: '>'
targetValue: 0
- colorScheme: '#FDE380'
column: at_leat_one_page_viewed
operator: '>'
targetValue: 0
extra_form_data: {}
groupby:
- section_with_name
metrics:
- all_pages_viewed
- at_leat_one_page_viewed
order_by_cols: []
order_desc: true
percent_metrics: []
query_mode: aggregate
row_limit: 10000
server_page_length: 10
show_cell_bars: true
table_timestamp_format: smart_date
temporal_columns_lookup: {}
time_grain_sqla: P1M
viz_type: table
query_context: '{"datasource":{"id":99,"type":"table"},"force":false,"queries":[{"filters":[{"col":"visited_on","op":"TEMPORAL_RANGE","val":"No
filter"}],"extras":{"time_grain_sqla":"P1M","having":"","where":""},"applied_time_extras":{},"columns":["section_with_name"],"metrics":["all_pages_viewed","at_leat_one_page_viewed"],"orderby":[["all_pages_viewed",false]],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"post_processing":[]}],"form_data":{"datasource":"99__table","viz_type":"table","slice_id":192,"query_mode":"aggregate","groupby":["section_with_name"],"time_grain_sqla":"P1M","temporal_columns_lookup":{},"metrics":["all_pages_viewed","at_leat_one_page_viewed"],"all_columns":[],"percent_metrics":[],"adhoc_filters":[{"clause":"WHERE","comparator":"No
filter","datasourceWarning":false,"expressionType":"SIMPLE","filterOptionName":"filter_s8d5o5h20q_2r3i3mfdrkw","isExtra":false,"isNew":false,"operator":"TEMPORAL_RANGE","sqlExpression":null,"subject":"visited_on"}],"order_by_cols":[],"row_limit":10000,"server_page_length":10,"order_desc":true,"table_timestamp_format":"smart_date","show_cell_bars":true,"color_pn":true,"conditional_formatting":[{"colorScheme":"#ACE1C4","column":"all_pages_viewed","operator":">","targetValue":0},{"colorScheme":"#FDE380","column":"at_leat_one_page_viewed","operator":">","targetValue":0}],"extra_form_data":{},"dashboards":[42],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
slice_name: Page views per section
uuid: 834ec7c4-ead8-4dea-b227-1c8be5fbb1d3
version: 1.0.0
viz_type: table
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,15 @@ params:
row_limit: 10000
show_empty_columns: true
show_legend: true
show_value: true
show_value: false
sort_series_type: sum
time_grain_sqla: P1M
tooltipTimeFormat: smart_date
truncateXAxis: true
truncate_metric: true
viz_type: echarts_timeseries_bar
xAxisLabelRotation: 0
x_axis: section/subsection name
x_axis: section_subsection_name
x_axis_sort_asc: true
x_axis_sort_series: name
x_axis_sort_series_ascending: true
Expand All @@ -52,13 +52,10 @@ params:
y_axis_title_margin: 30
y_axis_title_position: Left
zoomable: true
query_context: '{"datasource":{"id":381,"type":"table"},"force":false,"queries":[{"filters":[{"col":"visited_on","op":"TEMPORAL_RANGE","val":"No
filter"}],"extras":{"time_grain_sqla":"P1M","having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1M","columnType":"BASE_AXIS","sqlExpression":"section/subsection
name","label":"section/subsection name","expressionType":"SQL"}],"metrics":["all_pages_viewed","at_leat_one_page_viewed"],"orderby":[["all_pages_viewed",false]],"annotation_layers":[],"row_limit":10000,"series_columns":[],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["section/subsection
name"],"columns":[],"aggregates":{"all_pages_viewed":{"operator":"mean"},"at_leat_one_page_viewed":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"flatten"}]}],"form_data":{"datasource":"381__table","viz_type":"echarts_timeseries_bar","slice_id":729,"x_axis":"section/subsection
name","time_grain_sqla":"P1M","x_axis_sort_asc":true,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":["all_pages_viewed","at_leat_one_page_viewed"],"groupby":[],"adhoc_filters":[{"clause":"WHERE","comparator":"No
query_context: '{"datasource":{"id":99,"type":"table"},"force":false,"queries":[{"filters":[{"col":"visited_on","op":"TEMPORAL_RANGE","val":"No
filter"}],"extras":{"time_grain_sqla":"P1M","having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1M","columnType":"BASE_AXIS","sqlExpression":"section_subsection_name","label":"section_subsection_name","expressionType":"SQL"}],"metrics":["all_pages_viewed","at_leat_one_page_viewed"],"orderby":[["all_pages_viewed",false]],"annotation_layers":[],"row_limit":10000,"series_columns":[],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["section_subsection_name"],"columns":[],"aggregates":{"all_pages_viewed":{"operator":"mean"},"at_leat_one_page_viewed":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"flatten"}]}],"form_data":{"datasource":"99__table","viz_type":"echarts_timeseries_bar","slice_id":189,"x_axis":"section_subsection_name","time_grain_sqla":"P1M","x_axis_sort_asc":true,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":["all_pages_viewed","at_leat_one_page_viewed"],"groupby":[],"adhoc_filters":[{"clause":"WHERE","comparator":"No
filter","expressionType":"SIMPLE","operator":"TEMPORAL_RANGE","subject":"visited_on"}],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"orientation":"vertical","x_axis_title_margin":75,"y_axis_title":"Number
of learners","y_axis_title_margin":30,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","show_value":true,"only_total":true,"zoomable":true,"show_legend":true,"legendType":"scroll","legendOrientation":"top","x_axis_time_format":"smart_date","xAxisLabelRotation":0,"y_axis_format":",d","truncateXAxis":true,"y_axis_bounds":[null,null],"rich_tooltip":true,"tooltipTimeFormat":"smart_date","extra_form_data":{},"dashboards":[629],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
of learners","y_axis_title_margin":30,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","show_value":false,"only_total":true,"zoomable":true,"show_legend":true,"legendType":"scroll","legendOrientation":"top","x_axis_time_format":"smart_date","xAxisLabelRotation":0,"y_axis_format":",d","truncateXAxis":true,"y_axis_bounds":[null,null],"rich_tooltip":true,"tooltipTimeFormat":"smart_date","extra_form_data":{},"dashboards":[43,42],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
slice_name: Page views per section/subsection
uuid: bf2d6219-d633-48c1-a9b5-742eac6a4c0a
version: 1.0.0
Expand Down
Loading

0 comments on commit 1113805

Please sign in to comment.