diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 392d978..d5869c5 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,112 +1,24 @@ -## Title: - -- [ ] Pattern for PR title "protocol_name : protocol_category : other comments". Example - "uniswap: dex : tvl by user" +# Protocol Checklist -## Checklist before requesting a review -1. **index.ts file** +Welcome! Thank you for submitting a PR with your protocol data. The [README](../README.md) should serve as a general guide for you. But please follow this checklist to ensure you have everything. - - [ ] **Contains function** +- [ ] Read the [README](../README.md) and explore the other adapters (see `adapters/example`) +- [ ] See the schema that we are requesting based on your protocol type (currently supporting, lending and perps) +- [ ] Submit a PR with progress with a new folder containing your adapter (in `adapter/{protocol-name}`) +- [ ] Build your adapter + - [ ] Ensure your `package.json` has a `start` command that executes the proper code + - [ ] Follow the schema/output format, standard input file format, and standard function inputs + - [ ] Ensure the adapter runs and executes in the proper workflow (see: `How to execute this project?` in [README](../README.md)) +- [ ] QA your data +- [ ] Notify our team when you are ready for a review! - ```export const getUserTVLByBlock = async (blocks: BlockData) => { - const { blockNumber, blockTimestamp } = blocks - // Retrieve data using block number and timestamp - // YOUR LOGIC HERE - - return csvRows +## Protocol Notes - }; - ``` - - [ ] **getUserTVLByBlock function takes input with this schema** - - ``` - interface BlockData { - blockNumber: number; - blockTimestamp: number; - } - ``` - - [ ] **getUserTVLByBlock function returns output in this schema** +### Adapter Logic Flow + - ``` - const csvRows: OutputDataSchemaRow[] = []; +### QA Notes + - type OutputDataSchemaRow = { - block_number: number; //block_number which was given as input - timestamp: number; // block timestamp which was given an input, epoch format - user_address: string; // wallet address, all lowercase - token_address: string; // token address all lowercase - token_balance: bigint; // token balance, raw amount. Please dont divide by decimals - token_symbol: string; //token symbol should be empty string if it is not available - usd_price: number; //assign 0 if not available - }; - ``` - - [ ] **contains function** - - ``` - const readBlocksFromCSV = async (filePath: string): Promise => { - const blocks: BlockData[] = []; - - await new Promise((resolve, reject) => { - fs.createReadStream(filePath) - .pipe(csv()) // Specify the separator as '\t' for TSV files - .on('data', (row) => { - const blockNumber = parseInt(row.number, 10); - const blockTimestamp = parseInt(row.timestamp, 10); - if (!isNaN(blockNumber) && blockTimestamp) { - blocks.push({ blockNumber: blockNumber, blockTimestamp }); - } - }) - .on('end', () => { - resolve(); - }) - .on('error', (err) => { - reject(err); - }); - }); - - return blocks; - }; - - ``` - - [ ] **has this code** - - ``` - readBlocksFromCSV('hourly_blocks.csv').then(async (blocks: any[]) => { - console.log(blocks); - const allCsvRows: any[] = []; - - for (const block of blocks) { - try { - const result = await getUserTVLByBlock(block); - allCsvRows.push(...result); - } catch (error) { - console.error(`An error occurred for block ${block}:`, error); - } - } - await new Promise((resolve, reject) => { - const ws = fs.createWriteStream(`outputData.csv`, { flags: 'w' }); - write(allCsvRows, { headers: true }) - .pipe(ws) - .on("finish", () => { - console.log(`CSV file has been written.`); - resolve; - }); - }); - - }).catch((err) => { - console.error('Error reading CSV file:', err); - }); - ``` - - [ ] **Your code is handling Pagination to make sure all data for a given block is returned** - -2. **Output data** - - [ ] Created a folder test on the same level as src and put sample outputData.csv with 15-20 records generated by your code - - [ ] Data is returned for underlying tokens only. Not for special tokens (lp/veTokens etc) - - [ ] Follows the exact sequence mentioned in OutputDataSchemaRow . This is needed as we want same column ordering in output csv - - Value of each field is : - - [ ] block_number *is same as input block number. This signifies TVL is as of this block_number.* - - [ ] timestamp is same as input timestamp. This signifies TVL is as of this timestamp. It is in epoch format. - - [ ] user_address is in lowercase - - [ ] token_address is in lowercase - - [ ] token_balance is in raw amount. Please dont divide by decimals. - - [ ] token_symbol value if present, empty string if value is not available. - - [ ] usd_price if value is available, 0 if value is not available. +### General Notes + diff --git a/README.md b/README.md index f1ae125..af6f1f0 100644 --- a/README.md +++ b/README.md @@ -1,124 +1,198 @@ # scroll-adapters -## TVL by User - Adapters ### Onboarding Checklist + Please complete the following: -1. Set up a subquery indexer (e.g. Goldsky Subgraph) - 1. Follow the docs here: https://docs.goldsky.com/guides/create-a-no-code-subgraph - 2. General Steps - 1. create an account at app.goldsky.com - 2. deploy a subgraph or migrate an existing subgraph - https://docs.goldsky.com/subgraphs/introduction - 3. Use the slugs `scroll-testnet` and `scroll` when deploying the config -2. Prepare Subquery query code according to the Data Requirement section below. -3. Submit your response as a Pull Request to: https://github.com/delta-hq/scroll-adapters - 1. With path being `/` +- [ ] Read this document and explore the other adapters (see `adapters/example`) +- [ ] See the schema that we are requesting based on your protocol type (currently supporting, lending and perps) +- [ ] Submit a PR with progress with a new folder containing your adapter (in `adapter/{protocol-name}`) +- [ ] Build your adapter + - [ ] Ensure your `package.json` has a `start` command that executes the proper code + - [ ] Follow the schema/output format, standard input file format, and standard function inputs + - [ ] Ensure the adapter runs and executes in the proper workflow (see: `How to execute this project?`) +- [ ] QA your data +- [ ] Notify our team when you are ready for a review! +## How to execute this project? -### Code Changes Expected +### Standard Adapter Flow -1. Create a function like below: -``` - export const getUserTVLByBlock = async (blocks: BlockData) => { - const { blockNumber, blockTimestamp } = blocks - // Retrieve data using block number and timestamp - // YOUR LOGIC HERE - - return csvRows +1. Call standard function and input block number read from `hourly_blocks.csv` +2. Build a pandas dataframe with the user level data for at the given block +3. Query the subgraph/make `eth_call`s (ensure they are historical) to get all of the data + 1. Tip: use `viem` `multicall()` to save time and rpc compute +4. Write the data to `outputData.csv` - }; -``` -2. Interface for input Block Data is, in below blockTimestamp is in epoch format. - ``` - interface BlockData { - blockNumber: number; - blockTimestamp: number; -} -``` -3. Output "csvRow" is a list. +### Standard Execution Flow (detailed) + +Create your block file (`adapters/{protocol-name}/hourly_blocks.csv`) using **comma-delineated** csv. + +Example: + +```csv +number,block_timestamp +6916213,1719322991 ``` -const csvRows: OutputDataSchemaRow[] = []; - type OutputDataSchemaRow = { - block_number: number; - timestamp: number; - user_address: string; - token_address: string; - token_balance: bigint; - token_symbol: string; //token symbol should be empty string if it is not available - usd_price: number; //assign 0 if not available - }; +Run these commands: + +```bash +cd adapters/{protocol-name} +npm install +tsc +npm run start # should execute node dist/index.js ``` -4. Make sure you add relevant package.json and tsconfig.json +Your adapter should write the data to a csv file named `adapters/{protocol-name}/outputData.csv` following the proper schema below. + +## Schema Requirements + +Generally each row corresponds to a user / market pair at a given block. -### Data Requirement -Goal: **Hourly snapshot of TVL by User by Asset** +Please note the following: -For each protocol, we are looking for the following: -1. Query that fetches all relevant events required to calculate User TVL in the Protocol at hourly level. -2. Code that uses the above query, fetches all the data and converts it to csv file in below given format. -3. Token amount should be raw token amount. Please do not divide by decimals. +- We will be querying **hourly snapshots of user level data by asset/market** +- A standard function / entry point that inputs a block number and fetches all user level data at that block. (see below) +- Accepts the standard input defined in `hourly_blocks.csv` (see more details below) +- All values are in the underlying token amount (no lp or output tokens such as, cTokens, aTokens, uni pools) +- Token amounts are normalized +- All strings/addresses are lowercase with no spaces -Teams can refer to the example we have in there to write the code required. +> Note: **Expect multiple entries per user if the protocol has more than one token asset** + +### Lending Schema + +| Data Field | Notes | +|---------------------------|----------------------------------------------------------------------------------------| +| protocol | Name of the protocol (no spaces, should match the folder name) | +| block_number | | +| timestamp | Block timestamp | +| user_address | The address of the user who's data is being recorded | +| market | The smart contract address of the market | +| token_address (optional) | The smart contract address of the underlying token for this position | +| token_symbol (optional) | Symbol of the underlying token | +| supply_token | Balance of the supplied amount in this market from `user_address` | +| borrow_token | balance of the borrowed amount in this market from `user_address` | +| etl_timestamp | Run timestamp of this row | -### Output Data Schema +### Perps Schema | Data Field | Notes | |---------------------------|----------------------------------------------------------------------------------------| +| protocol | Name of the protocol (no spaces, should match the folder name) | | block_number | | | timestamp | Block timestamp | -| user_address | | -| token_address | | -| token_symbol (optional) | Symbol of token | -| token_balance | Balance of token (**If the token was borrowed, this balance should be negative**) | -| usd_price (from oracle) | Price of token (optional) | - - -Sample output row will look like this: - -| blocknumber | timestamp | user_address | token_address | token_balance | token_symbol (optional) | usd_price(optional)| -|---|---|---|---|---|---|---| -| 2940306 | 2024-03-16 19:19:57 | 0x4874459FE…d689 | 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 | 100 | WETH | 0| - -Note: **Expect multiple entries per user if the protocols has more than one token asset** - -### index.ts -On this scope, the code must read a CSV file with headers named `hourly_blocks.csv` that contains the following columns: -- `number` - block number -- `timestamp` - block timestamp - -And output a CSV file named `outputData.csv` with headers with the following columns: -- `block_number` - block number -- `timestamp` - block timestamp -- `user_address` - user address -- `token_address` - token address -- `token_symbol` - token symbol -- `token_balance` - token balance -- `usd_price` - USD price -e.g. `adapters/renzo/src/index.ts` - -For testing the adapter code for a single hourly block, use the following `hourly_blocks.csv` file: -``` -number,timestamp -4243360,1714773599 +| user_address | The address of the user who's data is being recorded | +| market | The smart contract address of the market | +| trade_pair_symbol | Symbol of the trade pair | +| daily_user_volume_usd | The cumulative volume of this user for this trade pair for the preceding 24h in USD | +| funding_rate | The funding rate for this trade pair (in percentage, ie: 63.4 = 63.4%) | +| supplied_amount_usd | The TVL or deposited amount of this user for this trade pair at this timestamp in USD (same as UI) | +| open_shorts_usd | Total notional value (in USD) of the shorts for this trade pair of the user | +| open_longs_usd | Total notional value (in USD) of the longs for this trade pair of the user | +| protocol_fees_usd | Revenue fees for the protocol in USD, generated from this user and trade pair (cumulative for the preceding 24h) | +| users_fees_usd | Revenue fees for the users (LP / takers) in USD, generated from this user and trade pair (cumulative for the preceding 24h) | +| etl_timestamp | Run timestamp of this row | + +## Code Expectations + +### Standard Input function + +This is a general guideline to help begin your project. See the following examples for how to call this function and write to `outputData.csv`: + +- [Example](./adapters/example/dex/src/index.ts) +- [Layerbank](./adapters/layerbank/src/index.ts) (compound v2 fork) +- [Rhomarkets](./adapters/rhomarkets/src/index.ts) (compound v2 fork) + +```typescript + export const getUserDataByBlock = async (blocks: BlockData) => { + const { blockNumber, blockTimestamp } = blocks + // Retrieve data using block number and timestamp + // YOUR LOGIC HERE + + return csvRows + }; ``` -### Adapter Example -In this repo, there is an adapter example. This adapter aims to get data positions from the subrgaph and calculate the TVL by users. -The main scripts is generating a output as CSV file. +### Standard Interfaces -[Adapter Example](adapters/example/dex/src/index.ts) +It is good practice to provide typing of newly defined data structures in typescript. Let's take a look at how we can standardize these based on your protocol type. -## Notes -1. Please make sure to have a "compile" script in package.json file. So, we are able to compile the typescript files into `dist/index.js` file. +**Input Data Type** -## How to execute this project? +```typescript +interface BlockData { + blockNumber: number; + blockTimestamp: number; +} +``` + +**Lending Schema Output** + +```typescript +const csvRows: OutputDataSchemaRow[] = []; +type OutputDataSchemaRow = { + // User / Market data + user_address: string; + market: string; + token_address: string | null; + token_symbol: string | null; + + // Financial data + supply_token: bigint; + borrow_token: bigint; + + // Metadata + block_number: number; + timestamp: number; + protocol: string; + etl_timestamp: number; +}; ``` -npm install // install all packages -npm run watch //other terminal tab -npm run start // other terminal tab + +**Perps Schema Output** + +```typescript +const csvRows: OutputDataSchemaRow[] = []; + +type OutputDataSchemaRow = { + // User / Market info + user_address: string; + market: string; + trade_pair_symbol: string; + funding_rate: number; + + // Financial data + daily_user_volume_usd: number; + supplied_amount_usd: number; + open_shorts_usd: number; + open_longs_usd: number; + protocol_fees_usd: number; + users_fees_usd: number; + + // Metadata + protocol: string; + block_number: number; + timestamp: number; + etl_timestamp: number; +}; ``` -By this, we'll be able to generate the output csv file. +## Deploying a Subgraph using Goldsky + +1. Set up a subquery indexer (e.g. Goldsky Subgraph) + 1. Follow the docs here: + 2. General Steps + 1. create an account at app.goldsky.com + 2. deploy a subgraph or migrate an existing subgraph - + 3. Use the slugs `scroll-testnet` and `scroll` when deploying the config +2. Prepare Subquery query code according to the Schema Requirements section + +## Adapter Example + +In this repo, there is an adapter example. This adapter aims to get data positions from the subrgaph and calculate the TVL by users. +The main scripts is generating a output as CSV file. + +[Adapter Example](adapters/example/dex/src/index.ts) diff --git a/adapters/layerbank/hourly_blocks.csv b/adapters/layerbank/hourly_blocks.csv index 8ffabfa..a47231b 100644 --- a/adapters/layerbank/hourly_blocks.csv +++ b/adapters/layerbank/hourly_blocks.csv @@ -1,2 +1,2 @@ -number,timestamp +number,timestamp 4243360,1714773599 \ No newline at end of file diff --git a/adapters/package.json b/adapters/package.json deleted file mode 100644 index ec94e15..0000000 --- a/adapters/package.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "adapters", - "version": "1.0.0", - "description": "", - "main": "index.js", - "type": "commonjs", - "scripts": { - "test": "node testScript.js", - "start": "node runScript.js" - }, - "keywords": [], - "author": "Openblocklabs", - "license": "UNLICENSED", - "dependencies": { - "csv-parser": "^3.0.0", - "fast-csv": "^5.0.1", - "viem": "^2.9.20" - } -} diff --git a/adapters/rhomarkets/hourly_blocks.csv b/adapters/rhomarkets/hourly_blocks.csv index 73da4b5..d27c542 100644 --- a/adapters/rhomarkets/hourly_blocks.csv +++ b/adapters/rhomarkets/hourly_blocks.csv @@ -1,2 +1,2 @@ -number,block_timestamp -6916213,1719322991 \ No newline at end of file +number,block_timestamp +7043812,1719856798 \ No newline at end of file diff --git a/adapters/runScript.js b/adapters/runScript.js deleted file mode 100644 index e3c154c..0000000 --- a/adapters/runScript.js +++ /dev/null @@ -1,94 +0,0 @@ -const fs = require('fs'); -const path = require('path'); - -const csv = require('csv-parser'); -const { write } =require('fast-csv'); - -// Get the folder name from command line arguments -const folderName = process.argv[2]; - -if (!folderName) { - console.error('Folder name not provided. Please provide the folder name as an argument.'); - process.exit(1); -} - -// Get the absolute path of the provided folder -const folderPath = path.resolve(folderName); - -// Check if the provided folder exists -if (!fs.existsSync(folderPath)) { - console.error(`Folder '${folderName}' does not exist.`); - process.exit(1); -} - -// Check if the provided folder contains index.ts file -const indexPath = path.join(folderPath, 'dist/index.js'); -if (!fs.existsSync(indexPath)) { - console.error(`Folder '${folderName}' does not contain index.ts file.`); - process.exit(1); -} - -// Import the funct function from the provided folder -const { getUserTVLByBlock } = require(indexPath); - -const readBlocksFromCSV = async (filePath) => { - const blocks = []; - - await new Promise((resolve, reject) => { - fs.createReadStream(filePath) - .pipe(csv({ separator: ',' })) - .on('data', (row) => { - const blockNumber = parseInt(row.number, 10); - const blockTimestamp = parseInt(row.block_timestamp, 10); - if (!isNaN(blockNumber) && blockTimestamp) { - blocks.push({ blockNumber: blockNumber, blockTimestamp }); - } - }) - .on('end', () => { - resolve(); - }) - .on('error', (err) => { - reject(err); - }); - }); - - return blocks; -}; - -readBlocksFromCSV(path.join(folderPath, 'hourly_blocks.csv')) -.then(async (blocks) => { - const allCsvRows = []; // Array to accumulate CSV rows for all blocks - const batchSize = 10; // Size of batch to trigger writing to the file - let i = 0; - - for (const block of blocks) { - try { - const result = await getUserTVLByBlock(block); - - // Accumulate CSV rows for all blocks - allCsvRows.push(...result); - - i++; - console.log(`Processed block ${i}`); - - // Write to file when batch size is reached or at the end of loop - if (i % batchSize === 0 || i === blocks.length) { - const ws = fs.createWriteStream(`${folderName}/outputData.csv`, { flags: i === batchSize ? 'w' : 'a' }); - write(allCsvRows, { headers: i === batchSize ? true : false }) - .pipe(ws) - .on("finish", () => { - console.log(`CSV file has been written.`); - }); - - // Clear the accumulated CSV rows - allCsvRows.length = 0; - } - } catch (error) { - console.error(`An error occurred for block ${block}:`, error); - } - - } -}) -.catch((err) => { - console.error('Error reading CSV file:', err); -}); diff --git a/adapters/testScript.js b/adapters/testScript.js deleted file mode 100644 index 86965c0..0000000 --- a/adapters/testScript.js +++ /dev/null @@ -1,42 +0,0 @@ -// runScript.js -const fs = require('fs'); -const path = require('path'); - -// Get the folder name from command line arguments -const folderName = process.argv[2]; - -if (!folderName) { - console.error('Folder name not provided. Please provide the folder name as an argument.'); - process.exit(1); -} - -// Get the absolute path of the provided folder -const folderPath = path.resolve(folderName); - -// Check if the provided folder exists -if (!fs.existsSync(folderPath)) { - console.error(`Folder '${folderName}' does not exist.`); - process.exit(1); -} - -// Check if the provided folder contains index.ts file -const indexPath = path.join(folderPath, 'dist/index.js'); -if (!fs.existsSync(indexPath)) { - console.error(`Folder '${folderName}' does not contain index.ts file.`); - process.exit(1); -} - -// Import the funct function from the provided folder -const { getUserTVLByBlock } = require(indexPath); - -// Call the getUserTVLByBlock function with desired arguments -const result = getUserTVLByBlock({ - blockTimestamp: 1711023841, - blockNumber: 3041467 -}).then((result) => { - if(!result.length){ - throw new Error("Empty result") - } -}); - -