vault secrets (#133)

* vault secrets * batching
FlipsideCrypto · Feb 28, 2024 · 35014e8 · 35014e8
1 parent a662e6f
commit 35014e8
Show file tree

Hide file tree

Showing 9 changed files with 477 additions and 368 deletions.
diff --git a/models/bronze/api_udf/bronze_api__contract_abis.sql b/models/bronze/api_udf/bronze_api__contract_abis.sql
@@ -46,7 +46,7 @@ row_nos AS (
 ),
 batched AS ({% for item in range(101) %}
 SELECT
-    rn.contract_address, live.udf_api('GET', CONCAT('https://api.gnosisscan.io/api?module=contract&action=getabi&address=', contract_address, '&apikey={gnosis_key}'),{ 'User-Agent': 'FlipsideStreamline' },{},'EXPLORER') AS abi_data, SYSDATE() AS _inserted_timestamp
+    rn.contract_address, live.udf_api('GET', CONCAT('https://api.gnosisscan.io/api?module=contract&action=getabi&address=', contract_address, '&apikey={key}'),{ 'User-Agent': 'FlipsideStreamline' },{}, 'Vault/prod/block_explorers/gnosis_scan') AS abi_data, SYSDATE() AS _inserted_timestamp
 FROM
     row_nos rn
 WHERE

diff --git a/models/bronze/api_udf/bronze_api__token_reads.sql b/models/bronze/api_udf/bronze_api__token_reads.sql
@@ -50,39 +50,45 @@ ready_reads AS (
         contract_address,
         latest_block,
         function_sig,
-        CONCAT(
-            '[\'',
-            contract_address,
-            '\',',
-            latest_block,
-            ',\'',
+        RPAD(
             function_sig,
-            '\',\'\']'
-        ) AS read_input
+            64,
+            '0'
+        ) AS input,
+        utils.udf_json_rpc_call(
+            'eth_call',
+            [{'to': contract_address, 'from': null, 'data': input}, utils.udf_int_to_hex(latest_block)],
+            concat_ws(
+                '-',
+                contract_address,
+                input,
+                latest_block
+            )
+        ) AS rpc_request
     FROM
         all_reads
 ),
 batch_reads AS (
     SELECT
-        CONCAT('[', LISTAGG(read_input, ','), ']') AS batch_read
+        ARRAY_AGG(rpc_request) AS batch_rpc_request
     FROM
         ready_reads
 ),
-results AS (
+node_call AS (
     SELECT
-        ethereum.streamline.udf_json_rpc_read_calls(
-            node_url,
-            headers,
-            PARSE_JSON(batch_read)
-        ) AS read_output
+        *,
+        live.udf_api(
+            'POST',
+            CONCAT(
+                '{service}',
+                '/',
+                '{Authentication}'
+            ),{},
+            batch_rpc_request,
+            'Vault/prod/gnosis/quicknode/mainnet'
+        ) AS response
     FROM
         batch_reads
-        JOIN {{ source(
-            'streamline_crosschain',
-            'node_mapping'
-        ) }}
-        ON 1 = 1
-        AND chain = 'gnosis'
     WHERE
         EXISTS (
             SELECT
@@ -92,30 +98,30 @@ results AS (
             LIMIT
                 1
         )
-), FINAL AS (
+), flat_responses AS (
     SELECT
-        VALUE :id :: STRING AS read_id,
-        VALUE :result :: STRING AS read_result,
-        SPLIT(
-            read_id,
-            '-'
-        ) AS read_id_object,
-        read_id_object [0] :: STRING AS contract_address,
-        read_id_object [1] :: STRING AS block_number,
-        read_id_object [2] :: STRING AS function_sig,
-        read_id_object [3] :: STRING AS function_input
+        VALUE :id :: STRING AS call_id,
+        VALUE :result :: STRING AS read_result
     FROM
-        results,
-        LATERAL FLATTEN(
-            input => read_output [0] :data
+        node_call,
+        LATERAL FLATTEN (
+            input => response :data
         )
 )
 SELECT
-    contract_address,
-    block_number,
-    function_sig,
-    function_input,
+    SPLIT_PART(
+        call_id,
+        '-',
+        1
+    ) AS contract_address,
+    SPLIT_PART(
+        call_id,
+        '-',
+        3
+    ) AS block_number,
+    LEFT(SPLIT_PART(call_id, '-', 2), 10) AS function_sig,
+    NULL AS function_input,
     read_result,
     SYSDATE() :: TIMESTAMP AS _inserted_timestamp
 FROM
-    FINAL
+    flat_responses
diff --git a/models/silver/defi/bridge/hop/silver_bridge__hop_ammwrapper.sql b/models/silver/defi/bridge/hop/silver_bridge__hop_ammwrapper.sql
@@ -70,18 +70,18 @@ contract_reads AS (
             [{ 'to': contract_address, 'from': null, 'data': data }, utils.udf_int_to_hex(block_number) ]
         ) AS rpc_request,
         live.udf_api(
-            node_url,
-            rpc_request
+            'POST',
+            CONCAT(
+                '{service}',
+                '/',
+                '{Authentication}'
+            ),{},
+            rpc_request,
+            'Vault/prod/gnosis/quicknode/mainnet'
         ) AS read_output,
         SYSDATE() AS _inserted_timestamp
     FROM
         inputs
-        JOIN {{ source(
-            'streamline_crosschain',
-            'node_mapping'
-        ) }}
-        ON 1 = 1
-        AND chain = 'gnosis'
 ),
 reads_flat AS (
     SELECT

diff --git a/models/silver/defi/bridge/hop/silver_bridge__hop_l2canonicaltoken.sql b/models/silver/defi/bridge/hop/silver_bridge__hop_l2canonicaltoken.sql
@@ -62,18 +62,18 @@ contract_reads AS (
             [{ 'to': amm_wrapper_address, 'from': null, 'data': data }, utils.udf_int_to_hex(block_number) ]
         ) AS rpc_request,
         live.udf_api(
-            node_url,
-            rpc_request
+            'POST',
+            CONCAT(
+                '{service}',
+                '/',
+                '{Authentication}'
+            ),{},
+            rpc_request,
+            'Vault/prod/gnosis/quicknode/mainnet'
         ) AS read_output,
         SYSDATE() AS _inserted_timestamp
     FROM
         inputs
-        JOIN {{ source(
-            'streamline_crosschain',
-            'node_mapping'
-        ) }}
-        ON 1 = 1
-        AND chain = 'gnosis'
 ),
 reads_flat AS (
     SELECT

diff --git a/models/silver/defi/dex/balancer/silver_dex__balancer_pools.sql b/models/silver/defi/dex/balancer/silver_dex__balancer_pools.sql
@@ -90,36 +90,57 @@ inputs_pools AS (
     SELECT
         pool_address,
         block_number,
-        function_sig,
-        (ROW_NUMBER() over (PARTITION BY pool_address
-    ORDER BY
-        block_number)) - 1 AS function_input
+        function_sig
     FROM
         pools_registered
         JOIN function_sigs
         ON 1 = 1
 ),
-pool_token_reads AS ({% for item in range(50) %}
+build_rpc_requests AS (
+    SELECT
+        pool_address,
+        block_number,
+        function_sig,
+        RPAD(
+            function_sig,
+            64,
+            '0'
+        ) AS input,
+        utils.udf_json_rpc_call(
+            'eth_call',
+            [{'to': pool_address, 'from': null, 'data': input}, utils.udf_int_to_hex(block_number)],
+            concat_ws(
+                '-',
+                pool_address,
+                input,
+                block_number
+            )
+        ) AS rpc_request,
+        row_num,
+        CEIL(
+            row_num / 250
+        ) AS batch_no
+    FROM
+        inputs_pools
+        LEFT JOIN pools_registered USING(pool_address)
+),
+pool_token_reads AS ({% for item in range(10) %}
     (
 SELECT
-    ethereum.streamline.udf_json_rpc_read_calls(node_url, headers, PARSE_JSON(batch_read)) AS read_output, SYSDATE() AS _inserted_timestamp
+    live.udf_api('POST', CONCAT('{service}', '/', '{Authentication}'),{}, batch_rpc_request, 'Vault/prod/gnosis/quicknode/mainnet') AS read_output, SYSDATE() AS _inserted_timestamp
 FROM
     (
 SELECT
-    CONCAT('[', LISTAGG(read_input, ','), ']') AS batch_read
+    ARRAY_AGG(rpc_request) batch_rpc_request
 FROM
-    (
+    build_rpc_requests
+WHERE
+    batch_no = {{ item }} + 1
+    AND batch_no IN (
 SELECT
-    pool_address, block_number, function_sig, function_input, CONCAT('[\'', pool_address, '\',', block_number, ',\'', function_sig, '\',\'', function_input, '\']') AS read_input, row_num
+    DISTINCT batch_no
 FROM
-    inputs_pools
-    LEFT JOIN pools_registered USING(pool_address)) ready_reads_pools
-WHERE
-    row_num BETWEEN {{ item * 50 + 1 }}
-    AND {{(item + 1) * 50 }}) batch_reads_pools
-    JOIN {{ source('streamline_crosschain', 'node_mapping') }}
-    ON 1 = 1
-    AND chain = 'gnosis') {% if not loop.last %}
+    build_rpc_requests))) {% if not loop.last %}
     UNION ALL
     {% endif %}
 {% endfor %}),
@@ -132,14 +153,16 @@ reads_adjusted AS (
             '-'
         ) AS read_id_object,
         read_id_object [0] :: STRING AS pool_address,
-        read_id_object [1] :: STRING AS block_number,
-        read_id_object [2] :: STRING AS function_sig,
-        read_id_object [3] :: STRING AS function_input,
+        LEFT(
+            read_id_object [1] :: STRING,
+            10
+        ) AS function_sig,
+        read_id_object [2] :: STRING AS block_number,
         _inserted_timestamp
     FROM
         pool_token_reads,
         LATERAL FLATTEN(
-            input => read_output [0] :data
+            input => read_output :data
         )
 ),
 pool_details AS (