AN5386/new-decoded-traces & traces history macro (#976)

* add * traces * add macros * delete comments * delete test * add improvement * change to gold traces * delete * packagelock
FlipsideCrypto · Nov 13, 2024 · d10f34d · d10f34d
1 parent 896c983
commit d10f34d
Show file tree

Hide file tree

Showing 13 changed files with 557 additions and 442 deletions.
diff --git a/...amline_traces_decoder_history_range_0.yml → ...run_streamline_decoded_traces_history.yml b/...amline_traces_decoder_history_range_0.yml → ...run_streamline_decoded_traces_history.yml
@@ -1,12 +1,12 @@
-name: dbt_run_streamline_traces_decoder_history_range_0
-run-name: dbt_run_streamline_traces_decoder_history_range_0
+name: dbt_run_streamline_decoded_traces_history
+run-name: dbt_run_streamline_decoded_traces_history
 
 on:
   workflow_dispatch:
   schedule:
-    # Runs “At 01:18.” (see https://crontab.guru)
-    - cron: '18 1 * * *'
-    
+    # Runs “At 22:05 every Saturday.” (see https://crontab.guru)
+    - cron: '5 22 * * 6'
+
 env:
   DBT_PROFILES_DIR: ./
 
@@ -22,12 +22,10 @@ env:
 concurrency:
   group: ${{ github.workflow }}
 
-
-
 jobs:
   run_dbt_jobs:
     runs-on: ubuntu-latest
-    environment: 
+    environment:
       name: workflow_prod
 
     steps:
@@ -42,6 +40,11 @@ jobs:
         run: |
           pip install -r requirements.txt
           dbt deps
-      - name: Run DBT Jobs
+      - name: Update complete table
         run: |
-          dbt run --threads 6 --vars '{"STREAMLINE_INVOKE_STREAMS":True,"WAIT":150,"row_limit":10000000}' -m "ethereum_models,tag:streamline_decoded_traces_history_range_0" "ethereum_models,tag:streamline_decoded_traces_complete"
+          dbt run -m "fsc_evm,tag:streamline_decoded_traces_complete"
+      - name: Decode historical traces
+        run: |
+          dbt run-operation decoded_traces_history --args '{"backfill_mode": false}' --vars '{"STREAMLINE_INVOKE_STREAMS":True}'
+
+
diff --git a/.github/workflows/dbt_run_streamline_traces_decoder_history_range_1.yml b/.github/workflows/dbt_run_streamline_traces_decoder_history_range_1.yml
diff --git a/.github/workflows/dbt_run_streamline_traces_decoder_history_range_2.yml b/.github/workflows/dbt_run_streamline_traces_decoder_history_range_2.yml
diff --git a/.github/workflows/dbt_run_streamline_traces_decoder_history_range_3.yml b/.github/workflows/dbt_run_streamline_traces_decoder_history_range_3.yml
diff --git a/.gitignore b/.gitignore
@@ -16,4 +16,5 @@ logs/
 .vscode/
 .env
 .user.yml
-dbt-env/
+dbt-env/
+package-lock.yml
diff --git a/Makefile b/Makefile
@@ -3,4 +3,47 @@ SHELL := /bin/bash
 dbt-console: 
 	docker-compose run dbt_console
 
-.PHONY: dbt-console
+refresh_package:
+	rm -f package-lock.yml
+	dbt clean
+	dbt deps
+	dbt run-operation fsc_utils.create_evm_streamline_udfs --vars '{UPDATE_UDFS_AND_SPS: true}' --target dev-admin
+
+realtime:
+	dbt run -m models/streamline/silver/decoder/realtime/streamline__decode_traces_realtime.sql --vars '{"STREAMLINE_INVOKE_STREAMS":True,"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":True"}' --target dev-admin
+	dbt run -m models/streamline/bronze/decoder/bronze__streamline_decoded_traces.sql --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":True}' --target dev-admin
+	dbt run -m models/silver/core/silver__decoded_traces.sql
+
+realtime_logs:
+	dbt run -m models/streamline/silver/decoder/realtime/streamline__decode_logs_realtime.sql --vars '{"STREAMLINE_INVOKE_STREAMS":True,"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":True"}' --target dev-admin
+	dbt run -m models/streamline/bronze/decoder/bronze__streamline_decoded_logs.sql --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":True}' --target dev-admin
+	dbt run -m models/silver/core/silver__decoded_logs.sql
+
+history:
+	dbt run -m models/streamline/silver/decoder/history/traces/range_1/streamline__decode_traces_history_011667449_011706397.sql --vars '{"STREAMLINE_INVOKE_STREAMS":True,"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":True"}' --target dev-admin
+	dbt run -m models/streamline/bronze/decoder/bronze__streamline_decoded_traces.sql --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":True}' --target dev-admin
+	dbt run -m models/silver/core/silver__decoded_traces.sql
+
+history_logs:
+	dbt run -m models/streamline/silver/decoder/history/event_logs/range_0/streamline__decode_logs_history_016532020_016560020.sql --vars '{"STREAMLINE_INVOKE_STREAMS":True,"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":True"}' --target dev-admin
+	dbt run -m models/streamline/bronze/decoder/bronze__streamline_decoded_logs.sql --full-refresh --vars '{"STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES":True}' --target dev-admin
+	dbt run -m models/silver/core/silver__decoded_logs.sql
+
+load_new: 
+	dbt run -m models/silver/core/silver__blocks.sql 
+	dbt run -m models/silver/core/silver__transactions.sql 
+	dbt run -m models/silver/core/silver__receipts.sql 
+	dbt run -m models/silver/core/silver__logs.sql 
+	dbt run -m models/silver/core/silver__traces.sql	
+
+load_abi: 
+	dbt run -m models/silver/core/silver__relevant_contracts.sql
+	dbt run -m models/silver/core/silver__created_contracts.sql
+	dbt run -m models/silver/abis --exclude models/silver/abis/event_logs
+
+load_new_and_abi:
+	make load_new
+	make load_abi
+
+.PHONY: dbt-console refresh_package
+