From 4603bf5eb3e5915d1eb0a27423d94d55f029db7a Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Fri, 7 Jun 2024 05:43:59 -0700 Subject: [PATCH] Token 2022 fungible (add metadata model first (#6113) * 1 * 2 * add fees model * incremental * add sources * Fix comma --------- Co-authored-by: Alan Ghobadi --- .../tokens/solana/tokens_solana.sources.yml | 23 +++ .../solana/tokens_solana_fees_history.sql | 61 +++++++ .../tokens/solana/tokens_solana_fungible.sql | 158 +++++++++++++++--- models/tokens/solana/tokens_solana_schema.yml | 25 +++ 4 files changed, 240 insertions(+), 27 deletions(-) create mode 100644 models/tokens/solana/tokens_solana_fees_history.sql diff --git a/models/tokens/solana/tokens_solana.sources.yml b/models/tokens/solana/tokens_solana.sources.yml index 0636d8254fe..5a17103eee3 100644 --- a/models/tokens/solana/tokens_solana.sources.yml +++ b/models/tokens/solana/tokens_solana.sources.yml @@ -24,6 +24,29 @@ sources: - name: spl_token_call_burnChecked loaded_at_field: call_block_time + - name: spl_token_2022_solana + description: "spl_token 2022 decoded tables" + freshness: # default freshness + warn_after: { count: 12, period: hour } + error_after: { count: 24, period: hour } + tables: + - name: spl_token_2022_call_initializeMint + loaded_at_field: call_block_time + - name: spl_token_2022_call_initializeMint2 + loaded_at_field: call_block_time + - name: spl_token_2022_call_transferFeeExtension + loaded_at_field: call_block_time + - name: spl_token_2022_call_transferChecked + loaded_at_field: call_block_time + - name: spl_token_2022_call_mintTo + loaded_at_field: call_block_time + - name: spl_token_2022_call_mintToChecked + loaded_at_field: call_block_time + - name: spl_token_2022_call_burn + loaded_at_field: call_block_time + - name: spl_token_2022_call_burnChecked + loaded_at_field: call_block_time + - name: mpl_token_metadata_solana description: "mpl metadata decoded tables" freshness: # default freshness diff --git a/models/tokens/solana/tokens_solana_fees_history.sql b/models/tokens/solana/tokens_solana_fees_history.sql new file mode 100644 index 00000000000..0414ee66262 --- /dev/null +++ b/models/tokens/solana/tokens_solana_fees_history.sql @@ -0,0 +1,61 @@ + {{ + config( + schema = 'tokens_solana', + alias = 'fees_history', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.fee_time')], + unique_key = ['account_mint','fee_time'], + post_hook='{{ expose_spells(\'["solana"]\', + "sector", + "tokens", + \'["ilemi"]\') }}') +}} + +--we need the fee basis points and maximum fee for token2022 transfers because the fee amount is not emitted in transferChecked +SELECT +call_account_arguments[1] as account_mint +, try(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data, + 1+1+1+1+1+case when bytearray_substring(call_data,1+1+1,1) = 0x01 and bytearray_substring(call_data,1+1+1+32+1,1) = 0x01 + then 64 + when bytearray_substring(call_data,1+1+1,1) = 0x01 and bytearray_substring(call_data,1+1+1+32+1,1) = 0x00 + then 32 + when bytearray_substring(call_data,1+1+1,1) = 0x00 and bytearray_substring(call_data,1+1+1+1,1) = 0x01 + then 32 + when bytearray_substring(call_data,1+1+1,1) = 0x00 and bytearray_substring(call_data,1+1+1+1,1) = 0x00 + then 0 + end --variations of COPTION enums for first two arguments + ,2)))) as fee_basis +, try(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data, + 1+1+1+1+1+case when bytearray_substring(call_data,1+1+1,1) = 0x01 and bytearray_substring(call_data,1+1+1+32+1,1) = 0x01 + then 64 + when bytearray_substring(call_data,1+1+1,1) = 0x01 and bytearray_substring(call_data,1+1+1+32+1,1) = 0x00 + then 32 + when bytearray_substring(call_data,1+1+1,1) = 0x00 and bytearray_substring(call_data,1+1+1+1,1) = 0x01 + then 32 + when bytearray_substring(call_data,1+1+1,1) = 0x00 and bytearray_substring(call_data,1+1+1+1,1) = 0x00 + then 0 + end + +2 + ,16)))) as fee_maximum +, call_block_time as fee_time +FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferFeeExtension') }} +WHERE bytearray_substring(call_data,1+1,1) = 0x00 --https://github.com/solana-labs/solana-program-library/blob/8f50c6fabc6ec87ada229e923030381f573e0aed/token/program-2022/src/extension/transfer_fee/instruction.rs#L38 +{% if is_incremental() %} +AND {{incremental_predicate('call_block_time')}} +{% endif %} + +UNION ALL +SELECT +call_account_arguments[1] as account_mint +, try(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data, + 1+1+1,2)))) as fee_basis +, try(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data, + 1+1+1+2,16)))) as fee_maximum +, call_block_time as fee_time +FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferFeeExtension') }} +WHERE bytearray_substring(call_data,1+1,1) = 0x05 --https://github.com/solana-labs/solana-program-library/blob/8f50c6fabc6ec87ada229e923030381f573e0aed/token/program-2022/src/extension/transfer_fee/instruction.rs#L147 +{% if is_incremental() %} +AND {{incremental_predicate('call_block_time')}} +{% endif %} \ No newline at end of file diff --git a/models/tokens/solana/tokens_solana_fungible.sql b/models/tokens/solana/tokens_solana_fungible.sql index 5f26d007d6c..6a66471188c 100644 --- a/models/tokens/solana/tokens_solana_fungible.sql +++ b/models/tokens/solana/tokens_solana_fungible.sql @@ -1,7 +1,7 @@ {{ config ( alias = 'fungible', - + post_hook='{{ expose_spells(\'["solana"]\', "sector", "tokens", @@ -10,26 +10,32 @@ }} -with +with tokens as ( SELECT bytearray_to_bigint(bytearray_reverse(bytearray_substring(call_data, 2, 1))) as decimals , call_data , account_mint + , token_version , call_tx_id , call_block_time + , row_number() over (partition by account_mint order by call_block_time desc) as latest FROM ( - SELECT call_data, account_mint, call_tx_id, call_block_time FROM {{ source('spl_token_solana', 'spl_token_call_initializeMint') }} - UNION ALL - SELECT call_data, account_mint, call_tx_id, call_block_time FROM {{ source('spl_token_solana', 'spl_token_call_initializeMint2') }} + SELECT call_data, account_mint, call_tx_id, call_block_time, 'spl_token' as token_version FROM {{ source('spl_token_solana', 'spl_token_call_initializeMint') }} + UNION ALL + SELECT call_data, account_mint, call_tx_id, call_block_time, 'spl_token' as token_version FROM {{ source('spl_token_solana', 'spl_token_call_initializeMint2') }} + UNION ALL + SELECT call_data, account_mint, call_tx_id, call_block_time, 'token2022' as token_version FROM {{ source('spl_token_2022_solana', 'spl_token_2022_call_initializeMint') }} + UNION ALL + SELECT call_data, account_mint, call_tx_id, call_block_time, 'token2022' as token_version FROM {{ source('spl_token_2022_solana', 'spl_token_2022_call_initializeMint2') }} ) {% if is_incremental() %} - where call_block_time >= date_trunc('day', now() - interval '7' day) + where {{ incremental_predicate('call_block_time') }} {% endif %} ) - + , metadata as ( - SELECT + SELECT meta.call_tx_id , meta.call_block_slot , meta.call_block_time @@ -38,8 +44,10 @@ with , meta.account_mint , meta.call_block_time , master.account_edition as master_edition + , metadata_program + , row_number() over (partition by meta.account_mint order by meta.call_block_time desc) as latest FROM ( - SELECT + SELECT call_tx_id , call_outer_instruction_index , call_inner_instruction_index @@ -48,9 +56,10 @@ with , json_query(createMetadataAccountArgs, 'lax $.CreateMetadataAccountArgs.data.Data') as args , account_metadata , account_mint + , call_executing_account as metadata_program FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMetadataAccount') }} - UNION ALL - SELECT + UNION ALL + SELECT call_tx_id , call_outer_instruction_index , call_inner_instruction_index @@ -59,51 +68,143 @@ with , json_query(createMetadataAccountArgsV2, 'lax $.CreateMetadataAccountArgsV2.data.DataV2') as args , account_metadata , account_mint + , call_executing_account as metadata_program FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMetadataAccountV2') }} - UNION ALL - SELECT + UNION ALL + SELECT call_tx_id , call_outer_instruction_index - , call_inner_instruction_index + , call_inner_instruction_index , call_block_slot , call_block_time , json_query(createMetadataAccountArgsV3, 'lax $.CreateMetadataAccountArgsV3.data.DataV2') as args , account_metadata , account_mint - FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMetadataAccountV3') }} - ) meta + , call_executing_account as metadata_program + FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMetadataAccountV3') }} + ) meta LEFT JOIN ( - SELECT account_mintAuthority, account_edition, account_metadata FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMasterEdition') }} + SELECT account_mintAuthority, account_edition, account_metadata FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMasterEdition') }} UNION ALL SELECT account_mintAuthority, account_edition, account_metadata FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMasterEditionV3') }} ) master ON master.account_metadata = meta.account_metadata {% if is_incremental() %} - WHERE meta.call_block_time >= date_trunc('day', now() - interval '7' day) + WHERE {{ incremental_predicate('meta.call_block_time') }} + {% endif %} + ) + + , token2022_metadata as ( + --token2022 direct metadata extension + SELECT + from_utf8(bytearray_substring(data,1+8+4,bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))))) as name + , from_utf8(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))) + 4 --start from end of name and end of length of symbol + , bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))),4))) --get length of symbol from end of name + )) as symbol + , from_utf8(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))) + 4 --end of name and end of length of symbol + + bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))),4))) + 4 --start from end of symbol and end of length of uri + , bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))) + 4 + + bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))),4))),4))) --get length of uri from end of symbol + )) as uri + , tx_id as metadata_tx + , account_arguments[3] as account_mint + , block_time + , executing_account as metadata_program + , row_number() over (partition by account_arguments[3] order by block_time desc) as latest + FROM {{ source('solana','instruction_calls') }} + WHERE executing_account = 'TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb' + AND bytearray_substring(data,1,1) = 0xd2 --deal with updateField later 0xdd + AND tx_success + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% endif %} + ) + + , token_metadata_other as ( + --some other metadata program (idk the owner) + SELECT + from_utf8(bytearray_substring(data,1+1+4,bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))))) as name + , from_utf8(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))) + 4 --start from end of name and end of length of symbol + , bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))),4))) --get length of symbol from end of name + )) as symbol + , from_utf8(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))) + 4 --end of name and end of length of symbol + + bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))),4))) + 4 --start from end of symbol and end of length of uri + , bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))) + 4 + + bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))),4))),4))) --get length of uri from end of symbol + )) as uri + , tx_id as metadata_tx + , account_arguments[2] as account_mint + , block_time + , executing_account as metadata_program + , row_number() over (partition by account_arguments[2] order by block_time desc) as latest + FROM {{ source('solana','instruction_calls') }} + WHERE executing_account = 'META4s4fSmpkTbZoUsgC1oBnWB31vQcmnN8giPw51Zu' + AND bytearray_substring(data,1,1) = 0x21 + AND tx_success + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} {% endif %} ) SELECT tk.account_mint as token_mint_address , tk.decimals - , trim(json_value(args, 'strict $.name'))as name - , trim(json_value(args, 'strict $.symbol')) as symbol - , trim(json_value(args, 'strict $.uri')) as token_uri + , coalesce(m22.name,mo.name,trim(json_value(args, 'strict $.name'))) as name + , coalesce(m22.symbol,mo.symbol,trim(json_value(args, 'strict $.symbol'))) as symbol + , coalesce(m22.uri,mo.uri,trim(json_value(args, 'strict $.uri'))) as token_uri , tk.call_block_time as created_at + , coalesce(m22.metadata_program,mo.metadata_program,m.metadata_program) as metadata_program + , tk.token_version + , tk.call_tx_id as init_tx FROM tokens tk -LEFT JOIN metadata m ON tk.account_mint = m.account_mint +LEFT JOIN token2022_metadata m22 ON tk.account_mint = m22.account_mint AND m22.latest = 1 +LEFT JOIN token_metadata_other mo ON tk.account_mint = mo.account_mint AND mo.latest = 1 +LEFT JOIN metadata m ON tk.account_mint = m.account_mint AND m.latest = 1 WHERE m.master_edition is null +AND tk.latest = 1 + +UNION ALL + +--token2022 wrapped sol https://solscan.io/tx/2L1o7sDMCMJ6PYqfNrnY6ozJC1DEx61pRYiLdfCCggxw81naQXsmHKDLn6EhJXmDmDSQ2eCKjUMjZAQuUsyNnYUv +SELECT + trim(token_mint_address) as token_mint_address + , decimals + , trim(name) as name + , trim(symbol) as symbol + , token_uri + , cast(created_at as timestamp) created_at + , metadata_program + , token_version + , init_tx +FROM +( + VALUES +( + '9pan9bMn5HatX4EJdBwg9VgCa7Uz5HL8N1m5D3NdXejP', + 9, + 'wrapped SOL', + 'SOL', + null, + '2023-08-02 00:00:00', + null, + 'token2022', + '2L1o7sDMCMJ6PYqfNrnY6ozJC1DEx61pRYiLdfCCggxw81naQXsmHKDLn6EhJXmDmDSQ2eCKjUMjZAQuUsyNnYUv' +) +) AS temp_table (token_mint_address, decimals, name, symbol, token_uri, created_at, metadata_program, token_version, init_tx) UNION ALL ---wrapped sol is special and doesn't have a init tx (that I can find) -SELECT +--old wrapped sol is special and doesn't have a init tx (that I can find) +SELECT trim(token_mint_address) as token_mint_address , decimals , trim(name) as name , trim(symbol) as symbol , token_uri , cast(created_at as timestamp) created_at -FROM + , metadata_program + , token_version + , init_tx +FROM ( VALUES ( @@ -112,6 +213,9 @@ FROM 'wrapped SOL', 'SOL', null, - '2021-01-31 00:00:00' + '2021-01-31 00:00:00', + null, + 'spl_token', + null ) -) AS temp_table (token_mint_address, decimals, name, symbol, token_uri, created_at) +) AS temp_table (token_mint_address, decimals, name, symbol, token_uri, created_at, metadata_program, token_version, init_tx) diff --git a/models/tokens/solana/tokens_solana_schema.yml b/models/tokens/solana/tokens_solana_schema.yml index ab3b17f52e5..4a6bd1a599e 100644 --- a/models/tokens/solana/tokens_solana_schema.yml +++ b/models/tokens/solana/tokens_solana_schema.yml @@ -19,8 +19,33 @@ models: description: "token symbol" - name: decimals description: "Number of decimals, refers to how divisible a token can be" + - name: metadata_program + description: program used for creating token metadata + - name: token_version + description: version of the token program used (spl_token, token2022) - name: created_at description: token mint created at + - name: init_tx + description: "transaction that initialized the mint" + + - name: tokens_solana_fees_history + meta: + blockchain: solana + sector: tokens + project: fees + contributors: ilemi + config: + tags: ['table', 'metadata', 'fess', 'solana'] + description: "fee updates on token2022 tokens" + columns: + - name: account_mint + description: "fungible token mint address on Solana" + - name: fee_basis + description: basis points fee on transfers + - name: fee_maximum + description: maximum fee amount on a given transfer + - name: fee_time + description: fee update time - name: tokens_solana_nft meta: