clean ci DB changed models parents to enforce deferal

.github/workflows/reusable_dbt_workflow.yml

@@ -91,13 +91,21 @@ jobs:
       #     ENV_SHORT_NAME: ${{ inputs.ENV_SHORT_NAME }}
       #     TARGET_BRANCH: ${{ github.base_ref }}
       #   run: ./scripts/sqlfluff_lint.sh $TARGET_BRANCH
-      - name: "compile dbt project"
+      - name: "Compile dbt project"
         working-directory: "./orchestration/dags/data_gcp_dbt/"
         env:
           DBT_TARGET_PATH: "target/"
           DBT_PROFILES_DIR: "."
           ENV_SHORT_NAME: ${{ inputs.ENV_SHORT_NAME }}
         run: dbt compile --profile CI --target $ENV_SHORT_NAME --vars "{'ENV_SHORT_NAME':'$ENV_SHORT_NAME'}"
+      - name: "Cleaning parent models from database's ci schema"
+        working-directory: "."
+        env:
+          TARGET_BRANCH: ${{ github.base_ref }}
+          ENV_SHORT_NAME: ${{ inputs.ENV_SHORT_NAME }}
+          DBT_TARGET_PATH: "target/"
+          DBT_PROFILES_DIR: "."
+        run: .github/workflows/scripts/clean_ci_db.sh ${{ github.base_ref }} orchestration/dags/data_gcp_dbt
       - name: "Run modified models & 1st order childrens"
         working-directory: .
         env:
@@ -118,7 +126,7 @@ jobs:
           dbt test --store-failures -s "state:modified" \
           --exclude test_name:relationships config.materialized:incremental \
           --profile CI --target $ENV_SHORT_NAME --defer \
-          --state env-run-artifacts \
+          --state env-run-artifacts --favor-state \
           --vars "{'CI_MATERIALIZATION':'view','ENV_SHORT_NAME':'$ENV_SHORT_NAME'}"
       - name: "Post to a Slack channel"
         if: ${{ inputs.NOTIF_CHANNEL_ID != '' && failure() }}

.github/workflows/scripts/clean_ci_db.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Check if TARGET_BRANCH is provided
+if [ -z "$1" ]; then
+  echo "TARGET_BRANCH is required as the first argument"
+  exit 1
+fi
+
+# Check if WORKING_DIR is provided
+if [ -z "$2" ]; then
+  echo "WORKING_DIR is required as the second argument"
+  exit 1
+fi
+
+TARGET_BRANCH=$1
+WORKING_DIR=$2
+parents="1+"
+
+# Change to the working directory
+cd $WORKING_DIR || { echo "Failed to change directory to $WORKING_DIR"; exit 1; }
+
+# Function to delete CI parent views
+dbt_run_op_delete_ci_parents_views() {
+  git fetch origin $TARGET_BRANCH
+
+  # Get modified models
+  models=$(git diff origin/$TARGET_BRANCH HEAD --name-only | grep 'orchestration/dags/data_gcp_dbt/models/' | grep '\.sql$' | awk -F '/' '{ print $NF }' | sed "s/\.sql$/${children}/g" | tr '\n' ' ')
+
+  if [ -z "$models" ]; then
+    echo "No models were modified"
+  else
+    echo "Cleaning CI views for ${models} ${parents} parents"
+    # Generate the list of parent models
+    model_list=$(dbt list -s "${parents}${models}" --exclude "${models}" --exclude-resource-type source test | grep data_gcp_dbt | awk -F '.' '{ print $NF }' | tr '\n' ',' | sed "s/,$//g")
+
+    # Format the model list for JSON
+    model_list="['$(echo $model_list | sed "s/,/','/g")']"
+
+    echo "Parents list to delete: $model_list"
+
+    # Run the dbt operation to delete the views
+    dbt run-operation delete_views --args "{\"models\": $model_list}" --profile CI
+  fi
+}
+
+# Run the function to delete CI parent views
+dbt_run_op_delete_ci_parents_views

.github/workflows/scripts/run_dbt_changed_models.sh

@@ -25,7 +25,7 @@ dbt_run_changed_models() {
     echo "no models were modified"
   else
     echo "Running models: ${models}"
-    dbt run --model $models --profile CI --target $ENV_SHORT_NAME --defer --state env-run-artifacts --vars "{'CI_MATERIALIZATION':'view','ENV_SHORT_NAME':'$ENV_SHORT_NAME'}" --exclude tag:failing_ci
+    dbt run --model $models --profile CI --target $ENV_SHORT_NAME --defer --state env-run-artifacts --favor-state --vars "{'CI_MATERIALIZATION':'view','ENV_SHORT_NAME':'$ENV_SHORT_NAME'}" --exclude tag:failing_ci
   fi
 }
 

orchestration/dags/data_gcp_dbt/macros/clean_ci_views.sql

@@ -0,0 +1,9 @@
+
+{% macro delete_views(models) %}
+    {% if target.profile_name == 'CI' %}
+        {% for model in models %}
+            {% set model_ref = ref(model) %}
+            DROP VIEW IF EXISTS {{ model_ref }} CASCADE;
+        {% endfor %}
+    {% endif %}
+{% endmacro %}

orchestration/dags/data_gcp_dbt/models/analytics/diversification/diversification_booking.sql

@@ -16,6 +16,7 @@ select
     user.user_civility,
     booking.booking_intermediary_amount as booking_amount,
     user.first_deposit_creation_date,
+    null as ci_test_field,
     COALESCE(
         IF(booking.physical_goods = True, 'physical', Null),
         IF(booking.digital_goods = True, 'digital', Null),