finalize udf_get_block_data | update tf_fact_block macros

Makefile

@@ -10,7 +10,7 @@ rm_logs:
 deploy_near_mainnet_lv: rm_logs
 	dbt run \
 	-s livequery_models.deploy.near.near__mainnet \
-	--vars '{UPDATE_UDFS_AND_SPS: true}' \
+	--vars '{LQ_UPDATE_UDFS_AND_SPS: true, UPDATE_UDFS_AND_SPS: false}' \
 	--profiles-dir ~/.dbt \
 	--profile livequery \
 	--target dev
@@ -22,10 +22,10 @@ compile_near_mainnet: rm_logs
 	--profile livequery \
 	--target dev
 
-deploy_fact_blocks_poc: rm_logs
+deploy_fact_blocks: rm_logs
 	dbt run \
-	-s livequery_models.deploy.near.silver.streamline.near_mainnet__fact_blocks_poc \
-	--vars '{UPDATE_UDFS_AND_SPS: true}' \
+	-s livequery_models.deploy.near.silver.streamline.near_mainnet__fact_blocks \
+	--vars '{LQ_UPDATE_UDFS_AND_SPS: true}' \
 	--profiles-dir ~/.dbt \
 	--profile livequery \
 	--target dev

README.md

@@ -462,20 +462,20 @@ livequery:
 
 To control the creation of UDF or SP macros with dbt run:
 
-* UPDATE_UDFS_AND_SPS
+* LQ_UPDATE_UDFS_AND_SPS
 When True, executes all macros included in the on-run-start hooks within dbt_project.yml on model run as normal
 When False, none of the on-run-start macros are executed on model run
 
 Default values are False
 
 * Usage:
-`dbt run --var '{"UPDATE_UDFS_AND_SPS":True}' -m ...`
+`dbt run --var '{"LQ_UPDATE_UDFS_AND_SPS":True}' -m ...`
 
 Dropping and creating udfs can also be done without running a model:
 
 ```sh
-dbt run-operation create_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' --args '{"drop_":false}'
-dbt run-operation create_udfs --vars '{"UPDATE_UDFS_AND_SPS":True}' --args '{"drop_":true}'
+dbt run-operation create_udfs --vars '{"LQ_UPDATE_UDFS_AND_SPS":True}' --args '{"drop_":false}'
+dbt run-operation create_udfs --vars '{"LQ_UPDATE_UDFS_AND_SPS":True}' --args '{"drop_":true}'
 ```
 
 ## Resources

dbt_project.yml

@@ -81,7 +81,7 @@ vars:
     UPDATE_UDFS_AND_SPS: false
 
   "dbt_date:time_zone": GMT
-  UPDATE_UDFS_AND_SPS: false
+  LQ_UPDATE_UDFS_AND_SPS: false
   DROP_UDFS_AND_SPS: false
   UPDATE_SNOWFLAKE_TAGS: true
   STREAMLINE_INVOKE_STREAMS: False

docs/manifest.json

@@ -189,7 +189,7 @@
       "path": "macros/create_sps.sql",
       "original_file_path": "macros/create_sps.sql",
       "name": "create_sps",
-      "macro_sql": "{% macro create_sps() %}\n    {% if var(\"UPDATE_UDFS_AND_SPS\") %}\n        {% if target.database == 'LIVEQUERY' %}\n            CREATE schema IF NOT EXISTS _internal;\n    {{ sp_create_prod_clone('_internal') }};\n        {% endif %}\n    {% endif %}\n{% endmacro %}",
+      "macro_sql": "{% macro create_sps() %}\n    {% if var(\"LQ_UPDATE_UDFS_AND_SPS\") %}\n        {% if target.database == 'LIVEQUERY' %}\n            CREATE schema IF NOT EXISTS _internal;\n    {{ sp_create_prod_clone('_internal') }};\n        {% endif %}\n    {% endif %}\n{% endmacro %}",
       "resource_type": "macro",
       "tags": [],
       "depends_on": {
@@ -267,7 +267,7 @@
       "path": "macros/create_udfs.sql",
       "original_file_path": "macros/create_udfs.sql",
       "name": "create_udfs",
-      "macro_sql": "{% macro create_udfs() %}\n--     {% if var(\"UPDATE_UDFS_AND_SPS\") %}\n--         {% set sql %}\n--         CREATE schema if NOT EXISTS silver;\n--         {% endset %}\n--         {% do run_query(sql) %}\n--     {% endif %}\n-- {% endmacro %}",
+      "macro_sql": "{% macro create_udfs() %}\n--     {% if var(\"LQ_UPDATE_UDFS_AND_SPS\") %}\n--         {% set sql %}\n--         CREATE schema if NOT EXISTS silver;\n--         {% endset %}\n--         {% do run_query(sql) %}\n--     {% endif %}\n-- {% endmacro %}",
       "resource_type": "macro",
       "tags": [],
       "depends_on": { "macros": ["macro.dbt.run_query"] },

macros/create_sps.sql

@@ -1,5 +1,5 @@
 {% macro create_sps() %}
-    {% if var("UPDATE_UDFS_AND_SPS") %}
+    {% if var("LQ_UPDATE_UDFS_AND_SPS") %}
         {% if target.database == 'LIVEQUERY' %}
             CREATE schema IF NOT EXISTS _internal;
     {{ sp_create_prod_clone('_internal') }};

macros/livequery/manage_udfs.sql

@@ -152,7 +152,7 @@
     This macro is used to deploy functions using ephemeral models.
     It should only be used within an ephemeral model.
  #}
-    {% if execute and (var("UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %}
+    {% if execute and (var("LQ_UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %}
         {% set sql %}
             {{- crud_udfs(config, this.schema, var("DROP_UDFS_AND_SPS")) -}}
         {%- endset -%}
@@ -174,7 +174,7 @@
     {%- set blockchain = this.schema -%}
     {%- set network = this.identifier -%}
     {% set schema = blockchain ~ "_" ~ network %}
-    {% if execute and (var("UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %}
+    {% if execute and (var("LQ_UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %}
         {% set sql %}
             {% for config in configs %}
                 {{- crud_udfs_by_chain(config, blockchain, network, var("DROP_UDFS_AND_SPS")) -}}
@@ -197,7 +197,7 @@
  #}
     {%- set schema = this.schema -%}
     {%- set utility_schema = this.identifier -%}
-    {% if execute and (var("UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %}
+    {% if execute and (var("LQ_UPDATE_UDFS_AND_SPS") or var("DROP_UDFS_AND_SPS")) and model.unique_id in selected_resources %}
         {% set sql %}
             {% for config in configs %}
                 {{- crud_udfs_by_marketplace(config, schema, utility_schema, var("DROP_UDFS_AND_SPS")) -}}

macros/near/near.yaml.sql

@@ -9,7 +9,7 @@
   signature:
     - [file_url, STRING, File URL created using BUILD_SCOPED_FILE_URL() snowflake internal function]
   return_type:
-    - VARIANT
+    - STRING
   options: |
     LANGUAGE PYTHON
     RUNTIME_VERSION = '3.11'
@@ -19,20 +19,6 @@
   sql: |
     {{ near_live_view_udf_get_block_data() | indent(4) -}}
 
-- name: {{ schema -}}.tf_get_block_data
-  signature:
-    - [file_urls, ARRAY, List of stage file URLs created using BUILD_SCOPED_FILE_URL() snowflake internal function]
-  return_type:
-    - "TABLE(block_data VARIANT)"
-  options: |
-    LANGUAGE PYTHON
-    RUNTIME_VERSION = '3.11'
-    PACKAGES = ('snowflake-snowpark-python')
-    HANDLER = 'GetBlockData'
-    COMMENT = $$A UDTF to retrieve NEAR block data stored in files from the Near Lake Snowflake External Stage.$$
-  sql: |
-    {{ near_live_view_get_block_data() | indent(4) -}}
-
 - name: {{ schema -}}.tf_fact_blocks
   signature:
     - [block_id, INTEGER, The start block height to get the blocks from]
@@ -47,19 +33,5 @@
   sql: |
     {{ near_live_view_fact_blocks(schema, blockchain, network) | indent(4) -}}
 
-- name: {{ schema -}}.tf_fact_blocks_poc
-  signature:
-    - [block_id, INTEGER, The start block height to get the blocks from]
-    - [to_latest, BOOLEAN, Whether to continue fetching blocks until the latest block or not]
-  return_type:
-    - "TABLE(block_id NUMBER, block_timestamp TIMESTAMP_NTZ, block_hash STRING, tx_count STRING, block_author STRING, header VARIANT, block_challenges_result ARRAY, block_challenges_root STRING, chunk_headers_root STRING, chunk_tx_root STRING, chunk_mask ARRAY, chunk_receipts_root STRING, chunks VARIANT, chunks_included NUMBER, epoch_id STRING, epoch_sync_data_hash STRING, events VARIANT, gas_price NUMBER, last_ds_final_block STRING, last_final_block STRING, latest_protocol_version NUMBER, next_bp_hash STRING, next_epoch_id STRING, outcome_root STRING, prev_hash STRING, prev_height NUMBER, prev_state_root STRING, random_value STRING, rent_paid FLOAT, signature STRING, total_supply NUMBER, validator_proposals VARIANT, validator_reward NUMBER, fact_blocks_id STRING, inserted_timestamp TIMESTAMP_NTZ, modified_timestamp TIMESTAMP_NTZ)"
-  options: |
-    NOT NULL
-    RETURNS NULL ON NULL INPUT
-    VOLATILE
-    COMMENT = $$Returns the block data for a given block height. If to_latest is true, it will continue fetching blocks until the latest block. Otherwise, it will fetch blocks until the block_id height is reached.$$
-  sql: |
-    {{ near_live_view_fact_blocks_poc(schema, blockchain, network) | indent(4) -}}
-
 {%- endmacro -%}