MM-56793: add EU log ingestion

mattermost · Mar 20, 2024 · 58af111 · 58af111
1 parent 47fbf73
commit 58af111
Showing 1 changed file with 28 additions and 1 deletion.
diff --git a/airflow/dags/mattermost_dags/extract/push_proxy_new.py b/airflow/dags/mattermost_dags/extract/push_proxy_new.py
@@ -55,6 +55,33 @@ def push_proxy_loader():
         ],
     )
 
+    extract_hpns_eu_logs = KubernetesPodOperator(
+        **pod_defaults,
+        image=MATTERMOST_DATAWAREHOUSE_IMAGE,  # Uses latest build from master
+        task_id="task-push-proxy-us",
+        name=f"push-proxy-us",
+        secrets=[
+            SNOWFLAKE_LOAD_USER,
+            SNOWFLAKE_LOAD_PASSWORD,
+            SNOWFLAKE_ACCOUNT,
+            SNOWFLAKE_LOAD_DATABASE,
+            SNOWFLAKE_LOAD_WAREHOUSE,
+            SNOWFLAKE_LOAD_ROLE,
+        ],
+        env_vars={},
+        arguments=[
+            "push_proxy PUSH_PROXY_LOGS_EU_NEW LOGS_EU_NEW "
+            " --prefix /AWSLogs/{{ var.value.push_proxy_aws_account_id }}/elasticloadbalancing/{{ var.value.push_proxy_aws_region }}"
+            " -s {{ var.value.push_proxy_target_schema }}"
+            " -a ${SNOWFLAKE_ACCOUNT}"
+            " -d ${SNOWFLAKE_LOAD_DATABASE}"
+            " -w ${SNOWFLAKE_LOAD_WAREHOUSE}"
+            " -r ${SNOWFLAKE_LOAD_ROLE}"
+            " -u ${SNOWFLAKE_LOAD_USER}"
+            " -p ${SNOWFLAKE_LOAD_PASSWORD}"
+        ],
+    )
+
     extract_tpns_logs = KubernetesPodOperator(
         **pod_defaults,
         image=MATTERMOST_DATAWAREHOUSE_IMAGE,  # Uses latest build from master
@@ -82,7 +109,7 @@ def push_proxy_loader():
         ],
     )
 
-    extract_hpns_us_logs >> extract_tpns_logs
+    extract_hpns_us_logs >> extract_hpns_eu_logs >> extract_tpns_logs
 
 
 dag = push_proxy_loader()