diff --git a/synapse_data_warehouse/synapse/dynamic_tables/V2.29.4__certifiedquizquestion_latest.sql b/synapse_data_warehouse/synapse/dynamic_tables/V2.29.4__certifiedquizquestion_latest.sql new file mode 100644 index 00000000..c22fa0d5 --- /dev/null +++ b/synapse_data_warehouse/synapse/dynamic_tables/V2.29.4__certifiedquizquestion_latest.sql @@ -0,0 +1,24 @@ +USE SCHEMA {{database_name}}.synapse; --noqa: JJ01,PRS,TMP + +CREATE DYNAMIC TABLE IF NOT EXISTS CERTIFIEDQUIZQUESTION_LATEST + TARGET_LAG = '1 day' + WAREHOUSE = compute_xsmall + AS + WITH latest_unique_rows AS ( + SELECT + * + FROM + {{database_name}}.synapse_raw.certifiedquizquestionsnapshots --noqa: TMP + WHERE + SNAPSHOT_TIMESTAMP >= CURRENT_TIMESTAMP - INTERVAL '14 DAYS' + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY response_id, question_index + ORDER BY change_timestamp DESC, snapshot_timestamp DESC + ) = 1 + ) + SELECT + * + FROM + latest_unique_rows + ORDER BY + latest_unique_rows.response_id ASC; diff --git a/synapse_data_warehouse/synapse/dynamic_tables/V2.29.5__add_comments_certifiedquizquestion_latest.sql b/synapse_data_warehouse/synapse/dynamic_tables/V2.29.5__add_comments_certifiedquizquestion_latest.sql new file mode 100644 index 00000000..46bd59fc --- /dev/null +++ b/synapse_data_warehouse/synapse/dynamic_tables/V2.29.5__add_comments_certifiedquizquestion_latest.sql @@ -0,0 +1,16 @@ +-- Configure environment +USE SCHEMA {{database_name}}.synapse; --noqa: JJ01,PRS,TMP,CP02 + +-- Add the table and column comments +ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST SET COMMENT = 'This table contains snapshots of the questions of the certification quiz taken within the last 14 days. With each entry representing a question answered by the user during the quiz.'; + +ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN change_type COMMENT 'The change type is always as CREATE since each instance of a user submitting a quiz results in a new submission of the quiz.'; +ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN change_timestamp COMMENT 'The time when the user submitted the quiz.'; +ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN change_user_id COMMENT 'The unique identifier of the user that submitted the quiz.'; +ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN response_id COMMENT 'The unique identifier of a response wherein a user submitted a set of answers while participating in the quiz.'; +ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN snapshot_timestamp COMMENT 'The time when the snapshot was taken (It is usually after the change happened).'; +ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN question_index COMMENT 'The position of the question within the quiz.'; +ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN is_correct COMMENT 'If true, the answer to the question was correct.'; +ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN stack COMMENT 'The stack (prod, dev) on which the quiz question record was processed.'; +ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN instance COMMENT 'The version of the stack that processed the quiz question record.'; +ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN snapshot_date COMMENT 'The data is partitioned for fast and cost effective queries. The snapshot_timestamp field is converted into a date and stored in the snapshot_date field for partitioning. The date should be used as a condition (WHERE CLAUSE) in the queries.'; diff --git a/synapse_data_warehouse/synapse/tables/V2.29.0__add_certifiedquizquestions_latest_backup.sql b/synapse_data_warehouse/synapse/tables/V2.29.0__add_certifiedquizquestions_latest_backup.sql new file mode 100644 index 00000000..32d3ab51 --- /dev/null +++ b/synapse_data_warehouse/synapse/tables/V2.29.0__add_certifiedquizquestions_latest_backup.sql @@ -0,0 +1,5 @@ +USE SCHEMA {{database_name}}.synapse; --noqa: JJ01,PRS,TMP + +-- Clone the CERTIFIEDQUIZQUESTIONS_LATEST table to ``CERTIFIEDQUIZQUESTIONS_LATEST_BACKUP`` +-- This will begin the process of converting ``CERTIFIEDQUIZQUESTIONS_LATEST`` table into a dynamic table +CREATE OR REPLACE TABLE CERTIFIEDQUIZQUESTION_LATEST_BACKUP CLONE CERTIFIEDQUIZQUESTION_LATEST; diff --git a/synapse_data_warehouse/synapse/tables/V2.29.3__drop_certifiedquizquestion_latest.sql b/synapse_data_warehouse/synapse/tables/V2.29.3__drop_certifiedquizquestion_latest.sql new file mode 100644 index 00000000..0f174903 --- /dev/null +++ b/synapse_data_warehouse/synapse/tables/V2.29.3__drop_certifiedquizquestion_latest.sql @@ -0,0 +1,4 @@ +USE SCHEMA {{database_name}}.synapse; --noqa: JJ01,PRS,TMP + +-- Drop the ``CERTIFIEDQUIZQUESTION_LATEST`` table so that a new dynamic table with the same name can be created in its place +DROP TABLE CERTIFIEDQUIZQUESTION_LATEST; diff --git a/synapse_data_warehouse/synapse_raw/streams/V2.29.2__drop_certifiedquizquestionsnapshots_stream.sql b/synapse_data_warehouse/synapse_raw/streams/V2.29.2__drop_certifiedquizquestionsnapshots_stream.sql new file mode 100644 index 00000000..ad67aab0 --- /dev/null +++ b/synapse_data_warehouse/synapse_raw/streams/V2.29.2__drop_certifiedquizquestionsnapshots_stream.sql @@ -0,0 +1,3 @@ +USE SCHEMA {{database_name}}.synapse_raw; --noqa: JJ01,PRS,TMP + +DROP STREAM CERTIFIEDQUIZQUESTIONSNAPSHOT_STREAM; diff --git a/synapse_data_warehouse/synapse_raw/tasks/V2.29.1__drop_certifiedquizquestion_latest_task.sql b/synapse_data_warehouse/synapse_raw/tasks/V2.29.1__drop_certifiedquizquestion_latest_task.sql new file mode 100644 index 00000000..0629b6b6 --- /dev/null +++ b/synapse_data_warehouse/synapse_raw/tasks/V2.29.1__drop_certifiedquizquestion_latest_task.sql @@ -0,0 +1,10 @@ +USE SCHEMA {{database_name}}.synapse_raw; --noqa: JJ01,PRS,TMP + +-- Momentarily suspend the ROOT task so we can remove the child tasks +ALTER TASK REFRESH_SYNAPSE_WAREHOUSE_S3_STAGE_TASK SUSPEND; + +-- Remove the tasks in question +DROP TASK UPSERT_TO_CERTIFIEDQUIZQUESTION_LATEST_TASK; + +-- Resume the ROOT task and its child tasks +SELECT SYSTEM$TASK_DEPENDENTS_ENABLE( 'REFRESH_SYNAPSE_WAREHOUSE_S3_STAGE_TASK' );