Merge branch 'main' into feat/merkle-airdrop

* main: (40 commits) fix: delete swap file fix: decimal type conversion Fix none case in indexer Update alembic/versions/33b920ecf3d6_add_incentives_apy_column.py Update alembic/versions/33b920ecf3d6_add_incentives_apy_column.py fix strategies view Add incentives_apy to indexer state Restructure query Add migration Add incentives_apy field to and Remove truefi production, add maple address fix: log unix timestamp fix: migration feat: remove old migrations feat: log maple apy Make consistent Update meta with hook and usdc-incentives Remove Euler strategy reference Add comment re-add apy edge case handling ...
sherlock-protocol · Sep 2, 2022 · c255a57 · c255a57
2 parents ad41447 + fc77ba0
commit c255a57
Show file tree

Hide file tree

Showing 19 changed files with 479 additions and 16 deletions.
diff --git a/README.md b/README.md
@@ -54,4 +54,6 @@ SHERLOCK_V2_CORE_PATH=/home/evert/sherlock/sherlock-v2-core
 MERKLE_DISTRIBUTOR_PATH=/Users/acelasi/work/sherlock/merkle-distributor
 MERKLE_DISTRIBUTOR_ADDRESSES=0xa,0xb,0xc
 INDEXER_SLEEP_BETWEEN_CALL=0.1
+SENTRY_DSN=
+SENTRY_ENVIRONMENT=production
 ```
diff --git a/alembic/versions/33b920ecf3d6_add_incentives_apy_column.py b/alembic/versions/33b920ecf3d6_add_incentives_apy_column.py
@@ -0,0 +1,27 @@
+"""add incentives_apy column
+
+Revision ID: 33b920ecf3d6
+Revises: 9563fd232d2f
+Create Date: 2022-08-25 08:58:09.448640
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '33b920ecf3d6'
+down_revision = '9563fd232d2f'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # TODO decide if nullable=true and server_default=None or not.
+    op.add_column("stats_apy", sa.Column("incentives_apy", sa.Float(), nullable=False, server_default=sa.schema.DefaultClause("0")))
+    op.add_column("indexer_state", sa.Column("incentives_apy", sa.Float(), nullable=False, server_default=sa.schema.DefaultClause("0")))
+
+
+def downgrade():
+    op.drop_column("stats_apy", "incentives_apy")
+    op.drop_column("indexer_state", "incentives_apy")
diff --git a/alembic/versions/9563fd232d2f_add_additional_apy.py b/alembic/versions/9563fd232d2f_add_additional_apy.py
@@ -0,0 +1,28 @@
+"""Add additional_apy
+
+Revision ID: 9563fd232d2f
+Revises: fe1a6b4ecd8a
+Create Date: 2022-08-29 13:49:43.314919
+
+"""
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "9563fd232d2f"
+down_revision = "fe1a6b4ecd8a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("indexer_state", sa.Column("additional_apy", sa.Float(), server_default="0", nullable=False))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("indexer_state", "additional_apy")
+    # ### end Alembic commands ###
diff --git a/app.py b/app.py
@@ -3,6 +3,7 @@
 import sys
 from threading import Thread
 
+import sentry  # noqa
 import settings
 from flask_app import app
 from indexer import Indexer

diff --git a/indexer.py b/indexer.py
@@ -11,6 +11,7 @@
 from web3.constants import ADDRESS_ZERO
 from web3.exceptions import ContractLogicError
 
+import sentry
 import settings
 from models import (
     Airdrop,
@@ -30,6 +31,7 @@
     StrategyBalance,
 )
 from models.interval_function import IntervalFunction
+from strategies.custom_yields import CUSTOM_YIELDS, MapleYield
 from strategies.strategies import Strategies
 from utils import get_event_logs_in_range, get_premiums_apy, requests_retry_session, time_delta_apy
 
@@ -84,13 +86,18 @@ def __init__(self, blocks_per_call=None):
 
         # Order is important, because some functions might depends on the result of the previous ones.
         # - `index_apy` must have an up to date APY computed, so it must come after `calc_apy`
+        # -- NOTE: `calc_apy` can fail to store an up to date APY under specific conditions,
+        #          this would cause `index_apy` to use an old value.
+        # - `calc_additional_apy` must have TVL computed, so it must come after `calc_tvl`
         self.intervals = {
             self.calc_tvl: settings.INDEXER_STATS_BLOCKS_PER_CALL,
             self.calc_tvc: settings.INDEXER_STATS_BLOCKS_PER_CALL,
             self.calc_apy: settings.INDEXER_STATS_BLOCKS_PER_CALL,
+            self.calc_additional_apy: settings.INDEXER_STATS_BLOCKS_PER_CALL,
             self.index_apy: settings.INDEXER_STATS_BLOCKS_PER_CALL,
             self.reset_balance_factor: settings.INDEXER_STATS_BLOCKS_PER_CALL,
             self.index_strategy_balances: settings.INDEXER_STATS_BLOCKS_PER_CALL,
+            self.log_maple_apy: 240,  # 1 hour
         }
 
     def calc_balance_factor(self, session, indx, block):
@@ -144,10 +151,11 @@ def index_apy(self, session, indx, block):
             block: Current block
         """
         timestamp = datetime.fromtimestamp(settings.WEB3_WSS.eth.get_block(block)["timestamp"])
-        apy = indx.apy
+        apy = Decimal(indx.apy) + Decimal(indx.additional_apy)
         premiums_apy = indx.premiums_apy
+        incentives_apy = indx.incentives_apy
 
-        StatsAPY.insert(session, block, timestamp, apy, premiums_apy)
+        StatsAPY.insert(session, block, timestamp, apy, premiums_apy, incentives_apy)
 
     def calc_tvl(self, session, indx, block):
         timestamp = datetime.fromtimestamp(settings.WEB3_WSS.eth.get_block(block)["timestamp"])
@@ -255,20 +263,56 @@ def calc_apy(self, session, indx, block):
 
         # Update APY only if relevant, that means:
         # - skip negative APYs generated by payouts
+        # - skip short term, very high APYs, generated by strategies (e.g. a loan is paid back in Maple)
+        # -- skip very high APYs (15%)
+        # -- skip if apy is 2.5 times as high as previous apy
         # Position balances are still being correctly kept up to date
         # using the balance factor which accounts for payouts.
         if apy < 0:
             logger.warning("APY %s is being skipped because is negative." % apy)
-        else:
-            indx.apy = apy
+            sentry.report_message(
+                "APY is being skipped because it is negative!",
+                "warning",
+                {"current_apy": float(apy * 100)},
+            )
+            return
+
+        if apy > 0.15:
+            logger.warning("APY %s is being skipped because is higher than 15%%." % apy)
+            sentry.report_message(
+                "APY is being skipped because it higher than 15%!",
+                "warning",
+                {"current_apy": float(apy * 100)},
+            )
+            return
+
+        if indx.apy != 0 and apy > indx.apy * 2.5:
+            logger.warning(
+                "APY %s is being skipped because it is 2.5 times higher than the previous APY of %s" % (apy, indx.apy)
+            )
+            sentry.report_message(
+                "APY is 2.5 times higher than the previous APY!",
+                "warning",
+                {"current_apy": float(apy * 100), "previous_apy": float(indx.apy * 100)},
+            )
+            return
+
+        indx.apy = apy
 
         # Compute the APY coming from protocol premiums
         tvl = StatsTVL.get_current_tvl(session)
         if not tvl:
             return
 
         premiums_per_second = ProtocolPremium.get_sum_of_premiums(session)
+        incentives_per_second = ProtocolPremium.get_usdc_incentive_premiums(session)
+
+        # Incentives are included in the total premiums, exclude them here
+        if premiums_per_second is not None and incentives_per_second is not None:
+            premiums_per_second -= incentives_per_second
+
         premiums_apy = get_premiums_apy(tvl.value, premiums_per_second) if premiums_per_second else 0
+        incentives_apy = get_premiums_apy(tvl.value, incentives_per_second) if incentives_per_second else 0
 
         # When an increase in a protocol's premium takes place, and the TVL has not increased yet proportionally,
         # the premiums APY will be higher than the total APY.
@@ -278,6 +322,11 @@ def calc_apy(self, session, indx, block):
         else:
             indx.premiums_apy = premiums_apy
 
+        if incentives_apy > indx.apy:
+            logger.warning("Incentive APY %s is being skipped beacuse it is higher than the total APY.")
+        else:
+            indx.incentives_apy = incentives_apy
+
     def reset_balance_factor(self, session, indx, block):
         """Update staking positions' balances to become up to date
         and reflect the real-time data from the contract, without
@@ -329,6 +378,59 @@ def index_strategy_balances(self, session, indx, block):
                 # If strategy is deployed and active
                 StrategyBalance.insert(session, block, timestamp, strategy.address, balance)
 
+    def calc_additional_apy(self, session, indx, block):
+        """Compute the additionl APY coming from custom yield strtegies.
+        (e.g. Maple, TrueFi)
+
+        Args:
+            session: DB session
+            indx: Indexer state
+            block: Block number
+        """
+        timestamp = datetime.fromtimestamp(settings.WEB3_WSS.eth.get_block(block)["timestamp"])
+
+        additional_apy = 0.0
+        for custom_yield in CUSTOM_YIELDS:
+            apy = custom_yield.get_apy(block, timestamp)
+            balance = custom_yield.strategy.get_balance(block)
+
+            logger.info("Strategy %s has balance %s and APY %s" % (custom_yield.strategy, balance, apy))
+
+            # If strategy is deployed and active and the APY has been successfully fetched
+            if balance is not None and apy is not None:
+                TVL = session.query(StatsTVL).order_by(StatsTVL.timestamp.desc()).first()
+
+                logger.info("Balance is %s and TVL value is %s" % (balance, str(TVL.value)))
+
+                # Compute the additional APY generated by this strategy by multipliying the
+                # computed APY with the weights of this strategy in the entire TVL
+                strategy_weight = balance / (TVL.value)
+                logger.info("Strategy weight %s" % strategy_weight)
+                weighted_apy = float(strategy_weight) * apy
+                logger.info("Weghted APY %s" % weighted_apy)
+
+                additional_apy += weighted_apy
+
+        logger.info("Computed additional APY of %s" % additional_apy)
+        indx.additional_apy = additional_apy
+
+    def log_maple_apy(self, session, indx, block):
+        """Log Maple APY to a file in order to save historical data.
+
+        Args:
+            session: DB session
+            indx: Indexer state
+            block: Block number
+        """
+        logger.info("Saving historical Maple APY")
+        timestamp = settings.WEB3_WSS.eth.get_block(block)["timestamp"]
+
+        apy = MapleYield(Strategies.MAPLE).get_apy(0, 0)
+        logger.info("Maple APY: %s" % apy)
+
+        with open("maple.csv", "a") as f:
+            f.write(f"{block},{timestamp},{apy}\n")
+
     class Transfer:
         def new(self, session, indx, block, tx_hash, args, contract_address):
             if args["to"] == ADDRESS_ZERO:

diff --git a/meta/.protocols.csv.swp b/meta/.protocols.csv.swp
diff --git a/meta/protocols.csv b/meta/protocols.csv
@@ -2,5 +2,7 @@ id,tag,name,defi_llama_slug,networks,website,logo,description,agreement,agreemen
 0x3019e52a670390f24e4b9b58af62a7367658e457bbb07f86b19b213ec74b5be7,euler,Euler,euler,Ethereum,https://www.euler.finance/,,Euler is a non-custodial protocol on Ethereum that allows users to lend and borrow almost any crypto asset.,https://v1.sherlock.xyz/static/pdf/Euler%20Statement%20of%20Coverage%2012.14.21.pdf,0x8dea03f791159d92aed9eebd520a79490f891142f740e7136be323e61db4ac96,0xE130bA997B941f159ADc597F0d89a328554D4B3E,"$170,000.00",5391,1.50%
 0x99b8883ea932491b57118762f4b507ebcac598bee27b98f443c06d889237d9a4,opyn,Squeeth by Opyn,opyn,Ethereum,https://www.opyn.co/,,"Squeeth (squared ETH) is a Power Perpetual that tracks the price of ETH². This functions similar to a perpetual swap where you are targeting ETH² rather than ETH. Long Squeeth gives traders a leveraged position with unlimited ETH² upside, protected downside, and no liquidations. Squeeth buyers pay a funding rate for this position. In contrast, short Squeeth is a short ETH² position, collateralized with ETH. Traders earn a funding rate for taking on this position, paid by long Squeeth holders.",https://v1.sherlock.xyz/static/pdf/PUBLIC_Opyn_Statement_of_Coverage.pdf,0x818980ecff06fdc814bc138a2cade7b8895c7265641c4d2f61150eeefd7926e6,0x609FFF64429e2A275a879e5C50e415cec842c629,"$150,000.00",4756,1.70%
 0x615307f589ff909e3b7cfbf4d4b4371eb99fa64353970d40b76c1c37381e5cf0,tempus,Tempus Finance,tempus-finance,Ethereum,https://tempus.finance/,,"Tempus is a multi-chain fixed-income protocol that integrates with lending protocols, staking protocols, and yield aggregators, and lets users fix or speculate on the yield generated by them.",https://v1.sherlock.xyz/static/pdf/Tempus_Statement_of_Coverage.pdf,0x4222be442d853bca3663c7984f475d271344fb9a4aff2145406f1939a439b275,0xab40a7e3cef4afb323ce23b6565012ac7c76bfef,"$37,125.00",1177,2.00%
-0x69f4668c272ce31fadcd9c3baa18d332f7b51237a757c2a883b7c95c84d204e3,liquifi,LiquiFi,,Ethereum,https://www.liquifi.finance/,,"LiquiFi helps protocols automate their token vesting, manage compliance, and provide token holders greater visibility into their ownership",https://sherlock-files.ams3.digitaloceanspaces.com/reports/2022.05.31%20-%20Sherlock_LiquiFi%20Audit%20+%20Coverage%20Overview.pdf,0x14fd6e8528bf9a08486933cfa6f41abb3a1b3d9ed5deb95c901860df60ed206e,0x2d697A19192f0e4B9887f3679FA50B9BB89D886c,"$12,500.00",396,2.50%
-0x9c832ff12f1059a111aeb390ae646e686435ffa13c2bdc61d499758b85c1a716,lyra,Lyra,lyra,"Ethereum,Optimism,staking",https://www.lyra.finance/,,"Lyra is a decentralized options exchange on Optimistic Ethereum, giving traders 24/7 access to crypto markets with low fees and subsecond transaction speeds.",https://github.com/sherlock-protocol/sherlock-reports/raw/a0dbb56763a336e4b90575adcb0bc9014f4beac9/coverage-agreements/Lyra%20Coverage%20Agreement.pdf,0x5b68e6e7068f78bd177828f7adc5ce0fd5ce6faa8849a44a6ec57cea9b218263,0x246d38588b16dd877c558b245e6d5a711c649fcf,"$83,200.00",2638,2.00%
+0x69f4668c272ce31fadcd9c3baa18d332f7b51237a757c2a883b7c95c84d204e3,liquifi,LiquiFi,,Ethereum,https://www.liquifi.finance/,,"LiquiFi helps protocols automate their token vesting, manage compliance, and provide token holders greater visibility into their ownership",https://sherlock-files.ams3.digitaloceanspaces.com/reports/2022.05.31%20-%20Sherlock_LiquiFi%20Audit%20+%20Coverage%20Overview.pdf,0x14fd6e8528bf9a08486933cfa6f41abb3a1b3d9ed5deb95c901860df60ed206e,0x2d697A19192f0e4B9887f3679FA50B9BB89D886c,"$68,750.00",2180,2.50%
+0x9c832ff12f1059a111aeb390ae646e686435ffa13c2bdc61d499758b85c1a716,lyra,Lyra,lyra,"Ethereum,Optimism,staking",https://www.lyra.finance/,,"Lyra is a decentralized options exchange on Optimistic Ethereum, giving traders 24/7 access to crypto markets with low fees and subsecond transaction speeds.",https://github.com/sherlock-protocol/sherlock-reports/raw/a0dbb56763a336e4b90575adcb0bc9014f4beac9/coverage-agreements/Lyra%20Coverage%20Agreement.pdf,0x5b68e6e7068f78bd177828f7adc5ce0fd5ce6faa8849a44a6ec57cea9b218263,0x246d38588b16dd877c558b245e6d5a711c649fcf,"$200,000.00",6342,2.00%
+0x47a46b3628edc31155b950156914c27d25890563476422202887ed4298fc3c98,usdc-incentives,USDC Incentives,,Ethereum,https://sherlock.xyz,,USDC Incentives for stakers,https://i.imgur.com/gATHiEc.png,0x1cfc44755d472e291ca6de38551abba24b75680fb97572ef02167b02a472df1a,0x666B8EbFbF4D5f0CE56962a25635CfF563F13161,"$400,000.00",12684,
+0x7141e52f1187d2baa72e449b5470b3cd2b2cfe77ccade306ff9bcadf941a7a8d,hook,Hook,,Ethereum,https://www.hook.xyz/,,,https://github.com/sherlock-protocol/sherlock-reports/raw/2cf79724c5bece94d9e2eb788411ee330cc43d9b/coverage-agreements/Hook%20Protocol%20Coverage%20Agreement.pdf,0x5e0a213389218ccdbcc4d6cad14e680cdbc39eb4b8e7e46463080ff7c1a961ce,0xd8a0852fe7732d51e81d9bb398fb84543bad3240,"$5,000.00",159,2.00%
diff --git a/meta/tvl_history.csv b/meta/tvl_history.csv
@@ -3,4 +3,8 @@ id,tag,timestamp,tvl
 0x9c832ff12f1059a111aeb390ae646e686435ffa13c2bdc61d499758b85c1a716,lyra,0,"2,400,000.00"
 0x9c832ff12f1059a111aeb390ae646e686435ffa13c2bdc61d499758b85c1a716,lyra,1656584454,"2,500,000.00"
 0x9c832ff12f1059a111aeb390ae646e686435ffa13c2bdc61d499758b85c1a716,lyra,1656951957,"3,222,699.74"
-0x9c832ff12f1059a111aeb390ae646e686435ffa13c2bdc61d499758b85c1a716,lyra,1658330237,"4,162,131.55"
+0x9c832ff12f1059a111aeb390ae646e686435ffa13c2bdc61d499758b85c1a716,lyra,1658330237,"4,162,131.55"
+0x9c832ff12f1059a111aeb390ae646e686435ffa13c2bdc61d499758b85c1a716,lyra,1659974400,"17,050,745.91"
+0x69f4668c272ce31fadcd9c3baa18d332f7b51237a757c2a883b7c95c84d204e3,liquifi,1660241801,"2,750,000.00"
+0x47a46b3628edc31155b950156914c27d25890563476422202887ed4298fc3c98,usdc-incentives,1661367559,"0.00"
+0x7141e52f1187d2baa72e449b5470b3cd2b2cfe77ccade306ff9bcadf941a7a8d,hook,1661367559,"250,000.00"
diff --git a/models/indexer_state.py b/models/indexer_state.py
@@ -15,4 +15,6 @@ class IndexerState(Base):
     balance_factor = Column(NUMERIC(78, 70), nullable=False, default=1.0)
     apy = Column(Float, nullable=False, default=0.0)
     premiums_apy = Column(Float, nullable=False, default=0.0)
+    incentives_apy = Column(Float, nullable=False, default=0.0)
     apy_50ms_factor = Column(NUMERIC(78, 70), nullable=False, default=0.0)  # TODO: Remove unused column
+    additional_apy = Column(Float, nullable=False, default=0.0, server_default="0")
diff --git a/models/protocol_premium.py b/models/protocol_premium.py
@@ -4,7 +4,9 @@
 from sqlalchemy import Column, ForeignKey, Integer, func
 from sqlalchemy.dialects.postgresql import NUMERIC, TIMESTAMP
 
+import settings
 from models.base import Base
+from models.protocol import Protocol
 
 logger = logging.getLogger(__name__)
 
@@ -54,6 +56,19 @@ def get_sum_of_premiums(session):
             .scalar()
         )
 
+    @staticmethod
+    def get_usdc_incentive_premiums(session):
+        # Retrieve the latest premium paid by the USDC incentive protocol
+        # Could return None
+        return (
+            session.query(ProtocolPremium.premium)
+            .join(Protocol, ProtocolPremium.protocol_id == Protocol.id)
+            .filter(Protocol.bytes_identifier == settings.USDC_INCENTIVES_PROTOCOL)
+            .order_by(ProtocolPremium.premium_set_at.desc())
+            .limit(1)
+            .scalar()
+        )
+
     def to_dict(self):
         return {
             "premium": self.premium,

diff --git a/models/stats_apy.py b/models/stats_apy.py
@@ -14,14 +14,16 @@ class StatsAPY(Base):
     timestamp = Column(TIMESTAMP, nullable=False, default=datetime.min)
     value = Column(Float, nullable=False)
     premiums_apy = Column(Float, nullable=False)
+    incentives_apy = Column(Float, nullable=False)
     block = Column(Integer, default=0)
 
     @staticmethod
-    def insert(session, block, timestamp, total_apy, premiums_apy):
+    def insert(session, block, timestamp, total_apy, premiums_apy, incentives_apy):
         apy = StatsAPY()
         apy.timestamp = timestamp
         apy.value = total_apy
         apy.premiums_apy = premiums_apy
+        apy.incentives_apy = incentives_apy
         apy.block = block
 
         session.add(apy)

diff --git a/models/strategy_balance.py b/models/strategy_balance.py
@@ -21,7 +21,6 @@ class StrategyBalance(Base):
     def insert(session, block, timestamp, address, value):
         new_balance = StrategyBalance()
         new_balance.address = address
-        new_balance.value = value
         new_balance.block = block
         new_balance.value = value
         new_balance.timestamp = timestamp

diff --git a/requirements.txt b/requirements.txt
@@ -61,3 +61,4 @@ flake8==4.0.1  # https://github.com/PyCQA/flake8
 flake8-isort==4.1.1  # https://github.com/gforcada/flake8-isort
 pre-commit==2.15.0  # https://github.com/pre-commit/pre-commit
 alembic==1.7.7  # https://alembic.sqlalchemy.org/en/latest/
+sentry-sdk[flask]==1.5.12  # https://github.com/getsentry/sentry-python
diff --git a/sentry.py b/sentry.py
@@ -0,0 +1,45 @@
+import sentry_sdk
+from sentry_sdk.integrations.flask import FlaskIntegration
+
+import settings
+
+sentry_sdk.init(
+    dsn=settings.SENTRY_DSN,
+    environment=settings.SENTRY_ENVIRONMENT,
+    integrations=[
+        FlaskIntegration(),
+    ],
+    # Set traces_sample_rate to 1.0 to capture 100%
+    # of transactions for performance monitoring.
+    # We recommend adjusting this value in production.
+    traces_sample_rate=0.01,
+    # By default the SDK will try to use the SENTRY_RELEASE
+    # environment variable, or infer a git commit
+    # SHA as release, however you may want to set
+    # something more human-readable.
+    # release="[email protected]",
+)
+
+
+def report_message(message: str, level: str = None, extra={}):
+    """Capture a message and send it to Sentry
+
+    Available levels are:
+        - fatal
+        - critical
+        - error
+        - warning
+        - log
+        - info
+        - debug
+
+    Args:
+        message (str): Message text
+        extra (dict): Dict of extra items to send with the message
+    """
+
+    with sentry_sdk.push_scope() as scope:
+        for key, value in extra.items():
+            scope.set_extra(key, value)
+
+        sentry_sdk.capture_message(message, level)