[SNOW-171] Convert certifiedquizquestion_latest table -> dynamic ta…

…ble (#99)

* new dynamic table

* fix

* Add table/col comments

* Update to deduplication method

* Revert dedup logic

* 14 days

* Update table comment

* Rename to V2.29.X

synapse_data_warehouse/synapse/dynamic_tables/V2.29.4__certifiedquizquestion_latest.sql

@@ -0,0 +1,24 @@
+USE SCHEMA {{database_name}}.synapse; --noqa: JJ01,PRS,TMP
+
+CREATE DYNAMIC TABLE IF NOT EXISTS CERTIFIEDQUIZQUESTION_LATEST
+    TARGET_LAG = '1 day'
+    WAREHOUSE = compute_xsmall
+    AS
+        WITH latest_unique_rows AS (
+            SELECT
+                *
+            FROM
+                {{database_name}}.synapse_raw.certifiedquizquestionsnapshots --noqa: TMP
+            WHERE
+                SNAPSHOT_TIMESTAMP >= CURRENT_TIMESTAMP - INTERVAL '14 DAYS'
+            QUALIFY ROW_NUMBER() OVER (
+                    PARTITION BY response_id, question_index
+                    ORDER BY change_timestamp DESC, snapshot_timestamp DESC
+                ) = 1
+        )
+        SELECT
+            *
+        FROM
+            latest_unique_rows
+        ORDER BY
+            latest_unique_rows.response_id ASC;

synapse_data_warehouse/synapse/dynamic_tables/V2.29.5__add_comments_certifiedquizquestion_latest.sql

@@ -0,0 +1,16 @@
+-- Configure environment
+USE SCHEMA {{database_name}}.synapse; --noqa: JJ01,PRS,TMP,CP02
+
+-- Add the table and column comments
+ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST SET COMMENT = 'This table contains snapshots of the questions of the certification quiz taken within the last 14 days. With each entry representing a question answered by the user during the quiz.';
+
+ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN change_type COMMENT 'The change type is always as CREATE since each instance of a user submitting a quiz results in a new submission of the quiz.';
+ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN change_timestamp COMMENT 'The time when the user submitted the quiz.';
+ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN change_user_id COMMENT 'The unique identifier of the user that submitted the quiz.';
+ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN response_id COMMENT 'The unique identifier of a response wherein a user submitted a set of answers while participating in the quiz.';
+ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN snapshot_timestamp COMMENT 'The time when the snapshot was taken (It is usually after the change happened).';
+ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN question_index COMMENT 'The position of the question within the quiz.';
+ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN is_correct COMMENT 'If true, the answer to the question was correct.';
+ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN stack COMMENT 'The stack (prod, dev) on which the quiz question record was processed.';
+ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN instance COMMENT 'The version of the stack that processed the quiz question record.';
+ALTER DYNAMIC TABLE CERTIFIEDQUIZQUESTION_LATEST ALTER COLUMN snapshot_date COMMENT 'The data is partitioned for fast and cost effective queries. The snapshot_timestamp field is converted into a date and stored in the snapshot_date field for partitioning. The date should be used as a condition (WHERE CLAUSE) in the queries.';

synapse_data_warehouse/synapse/tables/V2.29.0__add_certifiedquizquestions_latest_backup.sql

@@ -0,0 +1,5 @@
+USE SCHEMA {{database_name}}.synapse; --noqa: JJ01,PRS,TMP
+
+-- Clone the CERTIFIEDQUIZQUESTIONS_LATEST table to ``CERTIFIEDQUIZQUESTIONS_LATEST_BACKUP``
+-- This will begin the process of converting ``CERTIFIEDQUIZQUESTIONS_LATEST`` table into a dynamic table
+CREATE OR REPLACE TABLE CERTIFIEDQUIZQUESTION_LATEST_BACKUP CLONE CERTIFIEDQUIZQUESTION_LATEST;

synapse_data_warehouse/synapse/tables/V2.29.3__drop_certifiedquizquestion_latest.sql

@@ -0,0 +1,4 @@
+USE SCHEMA {{database_name}}.synapse; --noqa: JJ01,PRS,TMP
+
+-- Drop the ``CERTIFIEDQUIZQUESTION_LATEST`` table so that a new dynamic table with the same name can be created in its place
+DROP TABLE CERTIFIEDQUIZQUESTION_LATEST;

synapse_data_warehouse/synapse_raw/streams/V2.29.2__drop_certifiedquizquestionsnapshots_stream.sql

@@ -0,0 +1,3 @@
+USE SCHEMA {{database_name}}.synapse_raw; --noqa: JJ01,PRS,TMP
+
+DROP STREAM CERTIFIEDQUIZQUESTIONSNAPSHOT_STREAM;

synapse_data_warehouse/synapse_raw/tasks/V2.29.1__drop_certifiedquizquestion_latest_task.sql

@@ -0,0 +1,10 @@
+USE SCHEMA {{database_name}}.synapse_raw; --noqa: JJ01,PRS,TMP
+
+-- Momentarily suspend the ROOT task so we can remove the child tasks
+ALTER TASK REFRESH_SYNAPSE_WAREHOUSE_S3_STAGE_TASK SUSPEND;
+
+-- Remove the tasks in question
+DROP TASK UPSERT_TO_CERTIFIEDQUIZQUESTION_LATEST_TASK;
+
+-- Resume the ROOT task and its child tasks
+SELECT SYSTEM$TASK_DEPENDENTS_ENABLE( 'REFRESH_SYNAPSE_WAREHOUSE_S3_STAGE_TASK' );