add transfers and balances for erc1155 tokens on ethereum (#1188)

* add transfers and balances for erc1155 tokens on ethereum * small fixes * reverse weird erc721 changes * delete view from configs * add manual test for erc1155 balances * Update balances_ethereum_erc1155_june17th.csv
duneanalytics · Jun 17, 2022 · 4e55585 · 4e55585
1 parent 9ae14c5
commit 4e55585
Show file tree

Hide file tree

Showing 18 changed files with 1,484 additions and 11 deletions.
diff --git a/spellbook/models/balances/ethereum/balances_ethereum_schema.yml b/spellbook/models/balances/ethereum/balances_ethereum_schema.yml
@@ -173,4 +173,75 @@ models:
       - *token_address
       - *tokenId
       - *collection
+      - *category
+
+  - name: balances_ethereum_erc1155_day
+    meta:
+      blockchain: ethereum
+      sector: balances
+      project: erc1155
+      contibutors: soispoke
+    config:
+      tags: ['balances', 'ethereum', 'erc1155', 'day', 'soispoke']
+    description: >
+        Daily token balances of ERC1155 Ethereum tokens per wallet and contract address pair.
+        Depends on erc1155_ethereum_transfersingle and erc1155_ethereum_transferbatch.
+    columns:
+      - *blockchain
+      - *day
+      - &wallet_address_erc1155
+        name: wallet_address
+        description: "Address of the wallet holding this ERC721 token"
+      - &token_address_erc1155
+        name: token_address
+        description: "Contract address for the ERC721 token"
+      - &tokenId_erc1155
+        name: tokenId
+        description: "ID of this ERC721 token"
+      - &amount_erc1155
+        name: amount
+        description: "Amount of ERC1155 tokens"
+      - *collection
+      - *category
+
+  - name: balances_ethereum_erc1155_hour
+    meta:
+      blockchain: ethereum
+      sector: balances
+      project: erc1155
+      contibutors: soispoke
+    config:
+      tags: ['balances', 'ethereum', 'erc1155', 'hour', 'soispoke']
+    description: >
+        Hourly token balances of ERC1155 Ethereum tokens per wallet and contract address pair.
+        Depends on erc1155_ethereum_transfersingle and erc1155_ethereum_transferbatch.
+    columns:
+      - *blockchain
+      - *hour
+      - *wallet_address_erc1155
+      - *token_address_erc1155
+      - *tokenId_erc1155
+      - *amount_erc1155
+      - *collection
+      - *category
+
+  - name: balances_ethereum_erc1155_latest
+    meta:
+      blockchain: ethereum
+      sector: balances
+      project: erc1155
+      contibutors: soispoke
+    config:
+      tags: ['balances', 'ethereum', 'erc1155', 'latest', 'soispoke']
+    description: >
+        Latest token balances of ERC1155 Ethereum tokens per wallet and contract address pair.
+        Depends on erc1155_ethereum_transfersingle and erc1155_ethereum_transferbatch.
+    columns:
+      - *blockchain
+      - *last_updated
+      - *wallet_address
+      - *token_address
+      - *tokenId_erc1155
+      - *amount_erc1155
+      - *collection
       - *category
diff --git a/spellbook/models/balances/ethereum/erc1155/balances_ethereum_erc1155_day.sql b/spellbook/models/balances/ethereum/erc1155/balances_ethereum_erc1155_day.sql
@@ -0,0 +1,37 @@
+{{ config(
+        alias='erc1155_day'
+        )
+}}
+
+with
+    days as (
+        select
+            explode(
+                sequence(
+                    to_date('2015-01-01'), date_trunc('day', now()), interval 1 day
+                )
+            ) as day
+    )
+
+, daily_balances as
+ (SELECT
+    wallet_address,
+    token_address,
+    tokenId,
+    day,
+    amount,
+    lead(day, 1, now()) OVER (PARTITION BY token_address, wallet_address ORDER BY day) AS next_day
+    FROM {{ ref('transfers_ethereum_erc1155_rolling_day') }})
+
+SELECT
+    'ethereum' as blockchain,
+    d.day,
+    b.wallet_address,
+    b.token_address,
+    b.tokenId,
+    b.amount,
+    nft_tokens.name as collection,
+    nft_tokens.category as category
+FROM daily_balances b
+INNER JOIN days d ON b.day <= d.day AND d.day < b.next_day
+LEFT JOIN {{ ref('tokens_ethereum_nft') }} nft_tokens ON nft_tokens.contract_address = b.token_address
diff --git a/spellbook/models/balances/ethereum/erc1155/balances_ethereum_erc1155_hour.sql b/spellbook/models/balances/ethereum/erc1155/balances_ethereum_erc1155_hour.sql
@@ -0,0 +1,37 @@
+{{ config(
+        alias='erc1155_hour'
+        )
+}}
+
+with
+    hours as (
+        select
+            explode(
+                sequence(
+                    to_date('2015-01-01'), date_trunc('hour', now()), interval 1 hour
+                )
+            ) as hour
+    )
+
+, hourly_balances as
+ (SELECT
+    wallet_address,
+    token_address,
+    tokenId,
+    hour,
+    amount,
+    lead(hour, 1, now()) OVER (PARTITION BY token_address, wallet_address ORDER BY hour) AS next_hour
+    FROM {{ ref('transfers_ethereum_erc1155_rolling_hour') }})
+
+SELECT
+    'ethereum' as blockchain,
+    d.hour,
+    b.wallet_address,
+    b.token_address,
+    b.tokenId,
+    b.amount,
+    nft_tokens.name as collection,
+    nft_tokens.category as category
+FROM hourly_balances b
+INNER JOIN hours d ON b.hour <= d.hour AND d.hour < b.next_hour
+LEFT JOIN {{ ref('tokens_ethereum_nft') }} nft_tokens ON nft_tokens.contract_address = b.token_address
diff --git a/spellbook/models/balances/ethereum/erc1155/balances_ethereum_erc1155_latest.sql b/spellbook/models/balances/ethereum/erc1155/balances_ethereum_erc1155_latest.sql
@@ -0,0 +1,14 @@
+{{ config(
+        alias='erc1155_latest'
+        )
+}}
+SELECT
+    wallet_address,
+    token_address,
+    tokenId,
+    amount,
+    nft_tokens.name as collection,
+    nft_tokens.category as category,
+    updated_at
+FROM {{ ref('transfers_ethereum_erc1155_rolling_hour') }}
+LEFT JOIN {{ ref('tokens_ethereum_nft') }} nft_tokens ON nft_tokens.contract_address = token_address
diff --git a/spellbook/models/transfers/ethereum/erc1155/transfers_ethereum_erc1155.sql b/spellbook/models/transfers/ethereum/erc1155/transfers_ethereum_erc1155.sql
@@ -0,0 +1,98 @@
+{{ config(materialized='view', alias='erc1155') }}
+
+with
+  erc1155_ids_batch AS (
+    SELECT
+      *,
+      explode(ids) as explode_id,
+      evt_tx_hash || '-' || cast(
+        row_number() OVER (
+          PARTITION BY evt_tx_hash,
+          ids
+          ORDER BY
+            ids
+        ) as string
+      ) as unique_transfer_id
+    FROM {{source('erc1155_ethereum', 'evt_transferbatch')}}
+  ),
+
+  erc1155_values_batch AS (
+    SELECT
+      *,
+      explode(
+        values
+      ) as explode_value,
+      evt_tx_hash || '-' || cast(
+        row_number() OVER (
+          PARTITION BY evt_tx_hash,
+          ids
+          ORDER BY
+            ids
+        ) as string
+      ) as unique_transfer_id
+    FROM {{source('erc1155_ethereum', 'evt_transferbatch')}}
+  ),
+
+  erc1155_transfers_batch AS (
+    SELECT
+      DISTINCT erc1155_ids_batch.explode_id,
+      erc1155_values_batch.explode_value,
+      erc1155_ids_batch.evt_tx_hash,
+      erc1155_ids_batch.to,
+      erc1155_ids_batch.from,
+      erc1155_ids_batch.contract_address,
+      erc1155_ids_batch.evt_index,
+      erc1155_ids_batch.evt_block_time
+    FROM erc1155_ids_batch
+      JOIN erc1155_values_batch ON erc1155_ids_batch.unique_transfer_id = erc1155_values_batch.unique_transfer_id
+  ),
+
+  sent_transfers as (
+    select
+      evt_tx_hash,
+      evt_tx_hash || '-' || evt_index || '-' || to as unique_tx_id,
+      to as wallet_address,
+      contract_address as token_address,
+      evt_block_time,
+      id as tokenId,
+      value as amount
+    FROM {{source('erc1155_ethereum', 'evt_transfersingle')}} single
+    UNION ALL
+    select
+      evt_tx_hash,
+      evt_tx_hash || '-' || evt_index || '-' || to as unique_tx_id,
+      to as wallet_address,
+      contract_address as token_address,
+      evt_block_time,
+      explode_id as tokenId,
+      explode_value as amount
+    FROM erc1155_transfers_batch
+  ),
+
+  received_transfers as (
+    select
+      evt_tx_hash,
+      evt_tx_hash || '-' || evt_index || '-' || to as unique_tx_id,
+      from as wallet_address,
+      contract_address as token_address,
+      evt_block_time,
+      id as tokenId,
+      - value as amount
+    FROM {{source('erc1155_ethereum', 'evt_transfersingle')}}
+    UNION ALL
+    select
+      evt_tx_hash,
+      evt_tx_hash || '-' || evt_index || '-' || to as unique_tx_id,
+      from as wallet_address,
+      contract_address as token_address,
+      evt_block_time,
+      explode_id as tokenId,
+      - explode_value as amount
+    FROM erc1155_transfers_batch
+  )
+
+select 'ethereum' as blockchain, wallet_address, token_address, evt_block_time, tokenId, amount, evt_tx_hash, unique_tx_id
+from sent_transfers
+union all
+select 'ethereum' as blockchain, wallet_address, token_address, evt_block_time, tokenId, amount, evt_tx_hash, unique_tx_id
+from received_transfers
diff --git a/spellbook/models/transfers/ethereum/erc1155/transfers_ethereum_erc1155_agg_day.sql b/spellbook/models/transfers/ethereum/erc1155/transfers_ethereum_erc1155_agg_day.sql
@@ -0,0 +1,24 @@
+{{ config(
+        alias ='erc1155_agg_day',
+        materialized ='incremental',
+        file_format ='delta',
+        incremental_strategy='merge',
+        unique_key='unique_transfer_id'
+        )
+}}
+
+select
+    'ethereum' as blockchain,
+    date_trunc('day', evt_block_time) as day,
+    wallet_address,
+    token_address,
+    tokenId,
+    sum(amount) as amount,
+    unique_tx_id || '-' || wallet_address || '-' || token_address || tokenId || '-' || sum(amount)::string as unique_transfer_id
+FROM {{ ref('transfers_ethereum_erc1155') }}
+{% if is_incremental() %}
+-- this filter will only be applied on an incremental run
+where evt_block_time > now() - interval 2 days
+{% endif %}
+group by
+    date_trunc('day', evt_block_time), wallet_address, token_address, tokenId, unique_tx_id
diff --git a/spellbook/models/transfers/ethereum/erc1155/transfers_ethereum_erc1155_agg_hour.sql b/spellbook/models/transfers/ethereum/erc1155/transfers_ethereum_erc1155_agg_hour.sql
@@ -0,0 +1,24 @@
+{{ config(
+        alias ='erc1155_agg_hour',
+        materialized ='incremental',
+        file_format ='delta',
+        incremental_strategy='merge',
+        unique_key='unique_transfer_id'
+        )
+}}
+
+select
+    'ethereum' as blockchain,
+    date_trunc('hour', evt_block_time) as hour,
+    wallet_address,
+    token_address,
+    tokenId,
+    sum(amount) as amount,
+    unique_tx_id || '-' || wallet_address || '-' || token_address || tokenId || '-' || sum(amount)::string as unique_transfer_id
+FROM {{ ref('transfers_ethereum_erc1155') }}
+{% if is_incremental() %}
+-- this filter will only be applied on an incremental run
+where evt_block_time > now() - interval 2 days
+{% endif %}
+group by
+    date_trunc('hour', evt_block_time), wallet_address, token_address, tokenId, unique_tx_id
diff --git a/spellbook/models/transfers/ethereum/erc1155/transfers_ethereum_erc1155_rolling_day.sql b/spellbook/models/transfers/ethereum/erc1155/transfers_ethereum_erc1155_rolling_day.sql
@@ -0,0 +1,17 @@
+{{ config(
+        alias ='erc1155_rolling_day'
+        )
+}}
+
+        select
+            'ethereum' as blockchain,
+            day,
+            wallet_address,
+            token_address,
+            tokenId,
+            current_timestamp() as updated_at,
+            row_number() over (partition by token_address, tokenId, wallet_address order by day desc) as recency_index,
+            sum(amount) over (
+                partition by token_address, wallet_address order by day
+            ) as amount
+        from {{ ref('transfers_ethereum_erc1155_agg_day') }}
diff --git a/spellbook/models/transfers/ethereum/erc1155/transfers_ethereum_erc1155_rolling_hour.sql b/spellbook/models/transfers/ethereum/erc1155/transfers_ethereum_erc1155_rolling_hour.sql
@@ -0,0 +1,17 @@
+{{ config(
+        alias ='erc1155_rolling_hour'
+        )
+}}
+
+        select
+            'ethereum' as blockchain,
+            hour,
+            wallet_address,
+            token_address,
+            tokenId,
+            current_timestamp() as updated_at,
+            row_number() over (partition by token_address, tokenId, wallet_address order by hour desc) as recency_index,
+            sum(amount) over (
+                partition by token_address, wallet_address order by hour
+            ) as amount
+        from {{ ref('transfers_ethereum_erc1155_agg_hour') }}
diff --git a/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20_rolling_day.sql b/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20_rolling_day.sql
@@ -1,7 +1,5 @@
 {{ config(
-        alias ='erc20_rolling_day',
-        materialized ='view'
-        )
+        alias ='erc20_rolling_day')
 }}
 
         select

diff --git a/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20_rolling_hour.sql b/spellbook/models/transfers/ethereum/erc20/transfers_ethereum_erc20_rolling_hour.sql
@@ -1,6 +1,5 @@
 {{ config(
-        alias ='erc20_rolling_hour',
-        materialized ='view'
+        alias ='erc20_rolling_hour'
         )
 }}
 

diff --git a/spellbook/models/transfers/ethereum/erc721/transfers_ethereum_erc721_rolling_day.sql b/spellbook/models/transfers/ethereum/erc721/transfers_ethereum_erc721_rolling_day.sql
@@ -1,7 +1,5 @@
 {{ config(
-        alias ='erc721_rolling_day',
-        materialized ='view'
-        )
+        alias ='erc721_rolling_day')
 }}
 
         select

diff --git a/spellbook/models/transfers/ethereum/erc721/transfers_ethereum_erc721_rolling_hour.sql b/spellbook/models/transfers/ethereum/erc721/transfers_ethereum_erc721_rolling_hour.sql
@@ -1,7 +1,5 @@
 {{ config(
-        alias ='erc721_rolling_hour',
-        materialized ='view'
-        )
+        alias ='erc721_rolling_hour')
 }}
 
         select