Add Llama Aave Treasury Tables - Polygon (#701)

* Create aave_tokens.sql

* Create aave_fees_by_day.sql

* Create insert_aave_fees_by_day.sql

No V1 fees

* Create aave_daily_treasury_events.sql

* Create insert_aave_daily_treasury_events.sql

* Update insert_aave_daily_treasury_events.sql

* Create aave_daily_interest_rates.sql

* Update aave_daily_interest_rates.sql

* Create llama_aave_daily_atoken_balances.sql

* Update llama_aave_daily_atoken_balances.sql

* Create insert_aave_daily_atoken_balances.sql

* Rename llama_aave_daily_atoken_balances.sql to aave_daily_atoken_balances.sql

* Update insert_aave_daily_atoken_balances.sql

* Create insert_aave_daily_interest_rates.sql

* Update insert_aave_daily_interest_rates.sql

* Update insert_aave_daily_atoken_balances.sql

* Update insert_aave_daily_interest_rates.sql

* Create readme.md

* Update insert_aave_fees_by_day.sql

* Rename insert_aave_fees_by_day.sql to insert_aave_treasury_fees_by_day.sql

* Update aave_daily_atoken_balances.sql

* Update aave_daily_interest_rates.sql

* Update aave_daily_treasury_events.sql

* Update aave_fees_by_day.sql

* Update and rename aave_fees_by_day.sql to aave_treasury_fees_by_day.sql

* Update insert_aave_daily_atoken_balances.sql

* Update insert_aave_daily_interest_rates.sql

* Update insert_aave_daily_treasury_events.sql

* Update and rename insert_aave_treasury_fees_by_day.sql to insert_aave_daily_treasury_feesy.sql

* Rename insert_aave_daily_treasury_feesy.sql to insert_aave_daily_treasury_fees.sql

* Update aave_tokens.sql

* Move to aave schema

* Move tokens and rates to top level

* rename folder

* Create aave_daily_liquidity_mining_rates.sql

* Create insert_aave_daily_liquidity_mining_rates.sql

* Update insert_aave_daily_liquidity_mining_rates.sql

* Update insert_aave_daily_treasury_events.sql

* Update aave_tokens.sql

* Update insert_aave_daily_liquidity_mining_rates.sql

* Update insert_aave_daily_liquidity_mining_rates.sql

* Update insert_aave_daily_treasury_events.sql

* Update insert_aave_daily_treasury_fees.sql

* Update insert_aave_daily_atoken_balances.sql

* Update insert_aave_daily_interest_rates.sql

* Update insert_aave_daily_treasury_events.sql

* Update insert_aave_daily_atoken_balances.sql

update date_trunc to day

* Update aave_tokens.sql

address to token_address

* Update insert_aave_daily_atoken_balances.sql

address to token_address

* Update aave_tokens.sql

llama to aave naming

* llama to aave naming

llama to aave naming

* fix column data type, fix copy command, fix typo in index column

* update backfill, update cron command

* update backfill, update cron command

* fix address column, add start_time variable, fix cron job

* remove extra backfill command

* add file to create llama schema and address table on polygon

* fix fees table name, fix cron command, update backfill command

* update fees name, update backfill logic, fix cron command

Co-authored-by: jeff-dude <[email protected]>
Co-authored-by: j__dude <[email protected]>

polygon/aave/aave_daily_atoken_balances.sql

@@ -0,0 +1,15 @@
+CREATE SCHEMA IF NOT EXISTS aave;
+
+CREATE TABLE IF NOT EXISTS aave.aave_daily_atoken_balances (   
+	day timestamptz,
+	token_address bytea,
+	daily_change numeric,
+	starting_balance numeric,
+	interest_rate_apr numeric,
+	int_earned numeric,
+	total_bal numeric,
+		PRIMARY KEY (day, token_address)
+);
+
+CREATE INDEX IF NOT EXISTS llama_aave_daily_atoken_balances_token_day_idx ON aave.aave_daily_atoken_balances (token_address, day);
+CREATE INDEX IF NOT EXISTS llama_aave_daily_atoken_balances_day_idx ON aave.aave_daily_atoken_balances (day);

polygon/aave/aave_daily_interest_rates.sql

@@ -0,0 +1,16 @@
+CREATE SCHEMA IF NOT EXISTS aave;
+
+CREATE TABLE IF NOT EXISTS aave.aave_daily_interest_rates (   
+	underlying_token bytea,
+	token bytea,
+	day timestamptz,
+	interest_rate_raw numeric,
+	interest_rate_ray numeric,
+	interest_rate_apr numeric,
+		PRIMARY KEY (underlying_token, token, day)
+);
+
+CREATE INDEX IF NOT EXISTS llama_aave_daily_interest_rates_underlying_token_day_idx ON aave.aave_daily_interest_rates (underlying_token, token, day);
+CREATE INDEX IF NOT EXISTS llama_aave_daily_interest_rates_underlying_day_idx ON aave.aave_daily_interest_rates (underlying_token, day);
+CREATE INDEX IF NOT EXISTS llama_aave_daily_interest_rates_token_day_idx ON aave.aave_daily_interest_rates (token, day);
+CREATE INDEX IF NOT EXISTS llama_aave_daily_interest_rates_day_idx ON aave.aave_daily_interest_rates (day);

polygon/aave/aave_daily_liquidity_mining_rates.sql

@@ -0,0 +1,16 @@
+CREATE SCHEMA IF NOT EXISTS aave;
+
+CREATE TABLE IF NOT EXISTS aave.aave_daily_liquidity_mining_rates ( 
+
+day timestamptz,
+token_address bytea,
+lm_reward_apr_yr numeric,
+lm_reward_apr_daily numeric,
+lm_token_yr_raw numeric,
+lm_token_daily_raw numeric,
+aave_decimals numeric,
+	PRIMARY KEY (day, token_address)
+);
+
+CREATE INDEX IF NOT EXISTS llama_aave_daily_liquidity_mining_rates_token_day_idx ON aave.aave_daily_liquidity_mining_rates (token_address, day);
+CREATE INDEX IF NOT EXISTS llama_aave_daily_liquidity_mining_rates_day_idx ON aave.aave_daily_liquidity_mining_rates (day);

polygon/aave/aave_tokens.sql

@@ -0,0 +1,30 @@
+CREATE TABLE IF NOT EXISTS aave.aave_tokens (
+	token_address bytea UNIQUE,
+	symbol text,
+	decimals int4,
+  	underlying_token_address bytea,
+  	underlying_token_symbol text,
+  	underlying_token_decimals int4
+);
+
+BEGIN;
+DELETE FROM aave.aave_tokens *;
+
+
+COPY aave.aave_tokens (token_address, symbol, decimals,underlying_token_address,underlying_token_symbol,underlying_token_decimals) FROM stdin;
+\\x1d2a0e5ec8e5bbdca5cb219e649b565d8e5c3360	amAAVE	18	\\xd6df932a45c0f255f85145f286ea0b292b21c90b	AAVE	18
+\\x27f8d03b3a2196956ed754badc28d73be8830a6e	amDAI	18	\\x8f3cf7ad23cd3cadbd9735aff958023239c6a063	DAI	18
+\\x1a13f4ca1d028320a707d99520abfefca3998b7f	amUSDC	6	\\x2791bca1f2de4661ed88a30c99a7a9449aa84174	USDC	6
+\\x60d55f02a771d515e077c9c2403a1ef324885cec	amUSDT	6	\\xc2132d05d31c914a87c6611c10748aeb04b58e8f	USDT	6
+\\x5c2ed810328349100a66b82b78a1791b101c9d61	amWBTC	8	\\x1bfd67037b42cf73acf2047067bd4f2c47d9bfd6	WBTC	8
+\\x28424507fefb6f7f8e9d3860f56504e4e5f5f390	amWETH	18	\\x7ceb23fd6bc0add59e62ac25578270cff1b9f619	WETH	18
+\\x8df3aad3a84da6b69a4da8aec3ea40d9091b2ac4	amWMATIC	18	\\x0d500b1d8e8ef31e21c99d1db9a6444d3adf1270	WMATIC	18
+\.
+
+
+COMMIT;
+
+CREATE INDEX IF NOT EXISTS llama_aave_tokens_address_decimals_idx ON aave.aave_tokens USING btree (token_address) INCLUDE (decimals);
+CREATE INDEX IF NOT EXISTS llama_aave_tokens_symbol_decimals_idx ON aave.aave_tokens USING btree (symbol) INCLUDE (decimals);
+CREATE INDEX IF NOT EXISTS llama_aave_tokens_erc20address_decimals_idx ON aave.aave_tokens USING btree (underlying_token_address) INCLUDE (underlying_token_decimals);
+CREATE INDEX IF NOT EXISTS llama_aave_tokens_erc20symbol_decimals_idx ON aave.aave_tokens USING btree (underlying_token_symbol) INCLUDE (underlying_token_decimals);

polygon/aave/insert_aave_daily_atoken_balances.sql

@@ -0,0 +1,142 @@
+CREATE OR REPLACE FUNCTION aave.insert_aave_daily_atoken_balances(start_time timestamptz, end_time timestamptz) RETURNS integer
+LANGUAGE plpgsql AS $function$
+DECLARE r integer;
+	start_time_day timestamptz := DATE_TRUNC('day',start_time);
+	end_time_day timestamptz := DATE_TRUNC('day',end_time) + interval '1 day'; --since we trunc to day
+BEGIN
+WITH rows AS (
+    INSERT INTO aave.aave_daily_atoken_balances (
+	day,
+	token_address,
+	daily_change,
+	starting_balance,
+	interest_rate_apr,
+	int_earned,
+	total_bal
+    )
+
+WITH RECURSIVE 
+gs AS --doing a series so that we don't skip days when there's no deposits/withraws
+(
+SELECT DISTINCT d.token_address AS token_address, gs.day FROM aave."aave_tokens" d
+INNER JOIN 
+(SELECT generate_series(start_time_day, end_time_day, '1 day') AS day) gs -- Generate all days since the first contract
+ON 1=1
+)
+
+, rawbals AS ( --raw balances without interest. Doing these in a cte helps with runtime
+SELECT
+gs.day,gs."token_address" AS "contract_address", COALESCE(a.value,0) AS value,
+COALESCE("interest_rate_apr",0) AS interest_rate_apr, drank
+FROM (
+SELECT pre.*,
+DENSE_RANK() OVER (PARTITION BY pre."contract_address" ORDER BY day ASC) AS drank
+FROM
+(
+    SELECT
+    DATE_TRUNC('day',e."evt_block_time") AS day, e."contract_address",
+    SUM(
+    CASE WHEN "from" = '\x0000000000000000000000000000000000000000' THEN "value"
+        WHEN "to" = '\x0000000000000000000000000000000000000000' THEN -"value"
+        ELSE 0 END) AS value
+    FROM erc20."ERC20_evt_Transfer" e
+    INNER JOIN aave."aave_tokens" at
+        ON e."contract_address" = at."token_address"
+
+    WHERE ("from" = '\x0000000000000000000000000000000000000000'
+    OR "to" = '\x0000000000000000000000000000000000000000')
+	AND e.evt_block_time >= start_time_day AND e.evt_block_time <= end_time_day
+    GROUP BY 1,2
+
+UNION ALL --start balances
+	SELECT day, token_address, total_bal
+	FROM (
+		SELECT
+		day, token_address, total_bal,
+			DENSE_RANK() OVER (PARTITION BY token_address ORDER BY day DESC) AS rnk
+		FROM aave.aave_daily_atoken_balances
+		WHERE day < start_time_day
+	) start_bal
+	WHERE rnk = 1
+) pre
+) a
+RIGHT JOIN gs
+ON gs.day = a.day
+AND a."contract_address" = gs."token_address"
+INNER JOIN aave."aave_daily_interest_rates" di
+ON di.day = gs.day
+AND di."token" = gs."token_address"
+)
+
+,abalances AS --recursive query so that we can compound atoken interest each day.
+(
+SELECT day, contract_address,value, interest_rate_apr,
+value::decimal AS value_diff,
+value::decimal AS starting_value, value::decimal AS pre_int_balance,
+value*interest_rate_apr::decimal AS int_earned,
+value::decimal + (value*interest_rate_apr::decimal) AS total_bal
+FROM rawbals
+WHERE drank = 1 --first daily entry for each token
+
+UNION
+
+SELECT
+gr.day,gr."contract_address", gr.value,gr."interest_rate_apr",
+gr.value,
+COALESCE(c.total_bal,0)::decimal AS starting_value,
+(COALESCE(c.total_bal,0) + gr.value)::decimal AS pre_int_balance,
+( (COALESCE(c.total_bal,0) + gr.value)::decimal * COALESCE(gr."interest_rate_apr",0)::decimal ) AS int_earned,
+( (COALESCE(c.total_bal,0) + gr.value)::decimal )
++
+( (COALESCE(c.total_bal,0) + gr.value)::decimal * COALESCE(gr."interest_rate_apr",0)::decimal )
+AS total_bal
+FROM rawbals gr
+
+INNER JOIN abalances c --yesterday
+ON gr.contract_address = c.contract_address
+AND gr.day = c.day + '1 day'::interval
+)
+
+
+
+SELECT day, contract_address AS token_address, value_diff AS daily_change,
+starting_value AS starting_balance, interest_rate_apr, int_earned, total_bal
+FROM abalances
+
+
+    ON CONFLICT (day, token_address) DO UPDATE SET
+
+	daily_change = EXCLUDED.daily_change,
+	starting_balance = EXCLUDED.starting_balance,
+	interest_rate_apr = EXCLUDED.interest_rate_apr,
+	int_earned = EXCLUDED.int_earned,
+	total_bal = EXCLUDED.total_bal
+
+    RETURNING 1
+)
+SELECT count(*) INTO r from rows;
+RETURN r;
+END
+$function$;
+
+-- Get the table started
+SELECT aave.insert_aave_daily_atoken_balances(
+	'2021-04-13'
+	,'2022-01-01'
+	)
+;
+
+-- Get the table started
+SELECT aave.insert_aave_daily_atoken_balances(
+	'2022-01-01'
+	,NOW()
+	)
+;
+
+INSERT INTO cron.job (schedule, command)
+VALUES ('15,45 * * * *', $$
+    SELECT aave.insert_aave_daily_atoken_balances(
+        (SELECT NOW() - interval '3 days'),
+        (SELECT NOW());	
+$$)
+ON CONFLICT (command) DO UPDATE SET schedule=EXCLUDED.schedule;

polygon/aave/insert_aave_daily_interest_rates.sql

@@ -0,0 +1,88 @@
+CREATE OR REPLACE FUNCTION aave.insert_aave_daily_interest_rates(start_time timestamptz, end_time timestamptz) RETURNS integer
+LANGUAGE plpgsql AS $function$
+DECLARE r integer;
+	start_time_day timestamptz := DATE_TRUNC('day',start_time);
+	end_time_day timestamptz := DATE_TRUNC('day',end_time) + interval '1 day'; --since we trunc to day
+BEGIN
+WITH rows AS (
+    INSERT INTO aave.aave_daily_interest_rates (
+	underlying_token,
+	    token,
+	    day,
+	    interest_rate_raw,
+	    interest_rate_ray,
+	    interest_rate_apr
+    )
+SELECT
+underlying_token, token, day, interest_rate_raw, interest_rate_ray, interest_rate_apr
+
+FROM
+( 
+SELECT 
+underlying_token, token, gs.day AS day, interest_rate_raw, interest_rate_ray --ray matches aave UI
+,((1+interest_rate_ray)^(1.0/365.0)-1) AS interest_rate_apr --convert apy to daily apr
+FROM (
+SELECT
+"reserve" AS underlying_token, a."output_aTokenAddress" AS token, day,
+lead(day, 1, DATE_TRUNC('day',now() + '1 day'::interval) ) OVER (PARTITION BY "reserve"
+                            ORDER BY day asc) AS next_day,
+interest_rate AS interest_rate_raw,
+interest_rate/(10^27) AS interest_rate_ray
+FROM
+(   SELECT
+    "reserve",
+    DATE_TRUNC('day',"evt_block_time") AS day,
+    AVG("liquidityRate") AS interest_rate
+    FROM
+    aave_v2."LendingPool_evt_ReserveDataUpdated"
+    GROUP BY 1,2
+) ra
+LEFT JOIN ( SELECT DISTINCT asset, "output_aTokenAddress"
+            FROM aave_v2."ProtocolDataProvider_call_getReserveTokensAddresses"
+            WHERE "output_aTokenAddress" IS NOT NULL
+            ) a --asset is raw, atoken is atoken
+ON ra."reserve" = a.asset
+
+) r
+INNER JOIN 
+(SELECT generate_series(start_time_day, end_time_day, '1 day') AS day) gs
+ON r.day <= gs.day
+AND gs.day < r.next_day
+) f	
+
+
+    ON CONFLICT (underlying_token, token, day) DO UPDATE SET
+
+	interest_rate_raw = EXCLUDED.interest_rate_raw,
+	interest_rate_ray = EXCLUDED.interest_rate_ray,
+	interest_rate_apr = EXCLUDED.interest_rate_apr
+
+
+    RETURNING 1
+)
+SELECT count(*) INTO r from rows;
+RETURN r;
+END
+$function$;
+
+-- Get the table started
+SELECT aave.insert_aave_daily_interest_rates(
+    '2021-04-13'
+    ,'2022-01-01'
+    )
+;
+
+-- Get the table started
+SELECT aave.insert_aave_daily_interest_rates(
+    '2022-01-01'
+    ,NOW() 
+    )
+;
+
+INSERT INTO cron.job (schedule, command)
+VALUES ('14,44 * * * *', $$
+    SELECT aave.insert_aave_daily_interest_rates(
+        (SELECT NOW() - interval '3 days'),
+        (SELECT NOW());
+$$)
+ON CONFLICT (command) DO UPDATE SET schedule=EXCLUDED.schedule;