AN-5278/test-fsc-evm-v2 (do not merge)

.github/workflows/dbt_run_scheduled_curated.yml

@@ -43,4 +43,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run -m "ethereum_models,tag:curated"
+          dbt run -m "ethereum_models,tag:curated" "fsc_evm,tag:curated"

.github/workflows/dbt_run_scheduled_non_realtime.yml

@@ -43,4 +43,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:non_realtime" "ethereum_models,tag:realtime" "ethereum_models,tag:streamline_decoded_logs_realtime" "ethereum_models,tag:streamline_decoded_logs_complete" "ethereum_models,tag:streamline_balances_realtime" "ethereum_models,tag:streamline_balances_complete" "ethereum_models,tag:streamline_decoded_traces_complete" "ethereum_models,tag:streamline_decoded_traces_realtime"       
+          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:non_realtime" "ethereum_models,tag:realtime" "fsc_evm,tag:silver_core" "fsc_evm,tag:gold_core" "fsc_evm,tag:streamline_decoded_logs_realtime" "fsc_evm,tag:streamline_decoded_logs_complete" "fsc_evm,tag:streamline_balances_realtime" "fsc_evm,tag:streamline_balances_complete" "fsc_evm,tag:streamline_decoded_traces_complete" "fsc_evm,tag:streamline_decoded_traces_realtime"       

.github/workflows/dbt_run_streamline_abis.yml

@@ -44,4 +44,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_abis_realtime" "ethereum_models,tag:streamline_abis_complete"
+          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_abis_realtime" "fsc_evm,tag:streamline_abis_complete"

.github/workflows/dbt_run_streamline_chainhead.yml

@@ -43,4 +43,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_core_realtime" "ethereum_models,tag:streamline_core_complete"
+          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:streamline_core_realtime_receipts" "fsc_evm,tag:streamline_core_realtime_confirm_blocks" "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_complete_confirm_blocks"

.github/workflows/dbt_run_streamline_decoder_history_range_0.yml

@@ -44,4 +44,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120,"row_limit":2000000}' -m "ethereum_models,tag:streamline_decoded_logs_history_range_0" "ethereum_models,tag:streamline_decoded_logs_complete"
+          dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120,"row_limit":2000000}' -m "ethereum_models,tag:streamline_decoded_logs_history_range_0" "fsc_evm,tag:streamline_decoded_logs_complete"

.github/workflows/dbt_run_streamline_decoder_history_range_1.yml

@@ -44,4 +44,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120,"row_limit":2000000}' -m "ethereum_models,tag:streamline_decoded_logs_history_range_1" "ethereum_models,tag:streamline_decoded_logs_complete"
+          dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120,"row_limit":2000000}' -m "ethereum_models,tag:streamline_decoded_logs_history_range_1" "fsc_evm,tag:streamline_decoded_logs_complete"

.github/workflows/dbt_run_streamline_history_adhoc.yml

@@ -29,12 +29,12 @@ on:
         description: DBT Run Command
         required: true
         options:
-          - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_core_history" "ethereum_models,tag:streamline_core_complete"
-          - dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120,"row_limit":2000000}' -m "ethereum_models,tag:streamline_decoded_logs_history" "ethereum_models,tag:streamline_decoded_logs_complete"
+          - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_core_history" "fsc_evm,tag:streamline_core_history_receipts" "fsc_evm,tag:streamline_core_history_confirm_blocks" "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_complete_receipts" "fsc_evm,tag:streamline_core_complete_confirm_blocks"
+          - dbt run --threads 8 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":120,"row_limit":2000000}' -m "ethereum_models,tag:streamline_decoded_logs_history" "fsc_evm,tag:streamline_decoded_logs_complete"
           - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_beacon_history" "ethereum_models,tag:streamline_beacon_complete"
-          - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_balances_history" "ethereum_models,tag:streamline_balances_complete"
-          - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_abis_history" "ethereum_models,tag:streamline_abis_complete"
-          - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_reads_history" "ethereum_models,tag:streamline_reads_complete tag:streamline_reads_curated"
+          - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_balances_history" "fsc_evm,tag:streamline_balances_complete"
+          - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_abis_history" "fsc_evm,tag:streamline_abis_complete"
+          - dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "fsc_evm,tag:streamline_reads_history" "fsc_evm,tag:streamline_reads_complete" "ethereum_models,tag:streamline_reads_curated"
 env:
   DBT_PROFILES_DIR: ./
 

.github/workflows/dbt_run_streamline_reads.yml

@@ -44,4 +44,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_reads_curated" "ethereum_models,tag:streamline_reads_realtime" "ethereum_models,tag:streamline_reads_complete"
+          dbt run --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -m "ethereum_models,tag:streamline_reads_curated" "fsc_evm,tag:streamline_reads_realtime" "fsc_evm,tag:streamline_reads_complete"

.github/workflows/dbt_run_streamline_traces_decoder_history_range_0.yml

@@ -44,4 +44,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run --threads 6 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":150,"row_limit":10000000}' -m "ethereum_models,tag:streamline_decoded_traces_history_range_0" "ethereum_models,tag:streamline_decoded_traces_complete"
+          dbt run --threads 6 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":150,"row_limit":10000000}' -m "ethereum_models,tag:streamline_decoded_traces_history_range_0" "fsc_evm,tag:streamline_decoded_traces_complete"

.github/workflows/dbt_run_streamline_traces_decoder_history_range_1.yml

@@ -44,4 +44,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run --threads 6 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":150,"row_limit":10000000}' -m "ethereum_models,tag:streamline_decoded_traces_history_range_1" "ethereum_models,tag:streamline_decoded_traces_complete"
+          dbt run --threads 6 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":150,"row_limit":10000000}' -m "ethereum_models,tag:streamline_decoded_traces_history_range_1" "fsc_evm,tag:streamline_decoded_traces_complete"

.github/workflows/dbt_run_streamline_traces_decoder_history_range_2.yml

@@ -44,4 +44,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run --threads 6 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":150,"row_limit":10000000}' -m "ethereum_models,tag:streamline_decoded_traces_history_range_2" "ethereum_models,tag:streamline_decoded_traces_complete"
+          dbt run --threads 6 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":150,"row_limit":10000000}' -m "ethereum_models,tag:streamline_decoded_traces_history_range_2" "fsc_evm,tag:streamline_decoded_traces_complete"

.github/workflows/dbt_run_streamline_traces_decoder_history_range_3.yml

@@ -44,4 +44,4 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt run --threads 6 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":150,"row_limit":10000000}' -m "ethereum_models,tag:streamline_decoded_traces_history_range_3" "ethereum_models,tag:streamline_decoded_traces_complete"
+          dbt run --threads 6 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":150,"row_limit":10000000}' -m "ethereum_models,tag:streamline_decoded_traces_history_range_3" "fsc_evm,tag:streamline_decoded_traces_complete"

.github/workflows/dbt_test_daily.yml

@@ -44,7 +44,7 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt test --exclude "ethereum_models,tag:full_test" "ethereum_models,tag:recent_test" "ethereum_models,tag:beacon" "ethereum_models,tag:gha_tasks" livequery_models
+          dbt test --exclude "ethereum_models,tag:full_test" "ethereum_models,tag:recent_test" "ethereum_models,tag:beacon" "fsc_evm,tag:gha_tasks" livequery_models
 
 
 

.github/workflows/dbt_test_tasks.yml

@@ -24,6 +24,24 @@ concurrency:
 
 
 jobs:
-  called_workflow_template:
-    uses: FlipsideCrypto/analytics-workflow-templates/.github/workflows/dbt_test_tasks.yml@AN-4374/upgrade-dbt-1.7
-    secrets: inherit
+  run_dbt_jobs:
+    runs-on: ubuntu-latest
+    environment: 
+      name: workflow_prod
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+
+      - name: install dependencies
+        run: |
+          pip install -r requirements.txt
+          dbt deps
+
+      - name: Run DBT Jobs
+        run: |
+          dbt test -m "fsc_evm,tag:gha_tasks"

.gitignore

@@ -16,4 +16,6 @@ logs/
 .vscode/
 .env
 .user.yml
-dbt-env/
+dbt-env/
+
+package-lock.yml

Makefile

@@ -1,6 +1,46 @@
-SHELL := /bin/bash
+DBT_TARGET ?= dev
 
-dbt-console: 
-	docker-compose run dbt_console
+deploy_streamline_functions:
+	rm -f package-lock.yml && dbt clean && dbt deps
+	dbt run -s livequery_models.deploy.core --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET)
+	dbt run-operation fsc_utils.create_evm_streamline_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET)
 
-.PHONY: dbt-console
+cleanup_time:
+	rm -f package-lock.yml && dbt clean && dbt deps
+
+deploy_streamline_tables:
+	rm -f package-lock.yml && dbt clean && dbt deps
+ifeq ($(findstring dev,$(DBT_TARGET)),dev)
+	dbt run -m "fsc_evm,tag:bronze_external" --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":True}' -t $(DBT_TARGET)
+else
+	dbt run -m "fsc_evm,tag:bronze_external" -t $(DBT_TARGET)
+endif
+	dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" "fsc_evm,tag:utils" --full-refresh -t $(DBT_TARGET)
+
+deploy_streamline_requests:
+	rm -f package-lock.yml && dbt clean && dbt deps
+	dbt run -m "fsc_evm,tag:streamline_core_complete" "fsc_evm,tag:streamline_core_realtime" --vars '{"STREAMLINE_INVOKE_STREAMS":True}' -t $(DBT_TARGET)
+
+deploy_github_actions:
+	dbt run -s livequery_models.deploy.marketplace.github --vars '{"UPDATE_UDFS_AND_SPS":True}' -t $(DBT_TARGET)
+	dbt seed -s github_actions__workflows -t $(DBT_TARGET)
+	dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET)
+ifeq ($(findstring dev,$(DBT_TARGET)),dev)
+	dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET)
+else
+	dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":True}' -t $(DBT_TARGET)
+endif
+
+deploy_new_github_action:
+	dbt seed -s github_actions__workflows -t $(DBT_TARGET)
+	dbt run -m "fsc_evm,tag:gha_tasks" --full-refresh -t $(DBT_TARGET)
+ifeq ($(findstring dev,$(DBT_TARGET)),dev)
+	dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":False}' -t $(DBT_TARGET)
+else
+	dbt run-operation fsc_utils.create_gha_tasks --vars '{"START_GHA_TASKS":True}' -t $(DBT_TARGET)
+endif
+
+regular_incremental:
+	dbt run -m "fsc_evm,tag:core" -t $(DBT_TARGET)
+
+.PHONY: deploy_streamline_functions deploy_streamline_tables deploy_streamline_requests deploy_github_actions cleanup_time regular_incremental deploy_new_github_action