diff --git a/conseilUtil.js b/conseilUtil.js index b022581..4c62df2 100644 --- a/conseilUtil.js +++ b/conseilUtil.js @@ -2,6 +2,8 @@ const conseiljs = require('conseiljs') const fetch = require('node-fetch') const log = require('loglevel') const BigNumber = require('bignumber.js') +const pThrottle = require('p-throttle') +// const { performance } = require('perf_hooks'); // NOTE: disabled for prod const logger = log.getLogger('conseiljs') logger.setLevel('error', false) @@ -11,6 +13,8 @@ const conseilServer = 'https://conseil-prod.cryptonomic-infra.tech' const conseilApiKey = 'aa73fa8a-8626-4f43-a605-ff63130f37b1' // signup at nautilus.cloud const tezosNode = '' +const throttleConseil = pThrottle({ limit: 15, interval: 1200 }) + const mainnet = require('./config').networkConfig /** @@ -275,12 +279,13 @@ const getArtisticUniverse = async (max_time) => { const objectQueries = queryChunks.map(c => makeObjectQuery(c)) + // const a = performance.now() let universe = [] await Promise.all( objectQueries.map(async (q) => { - const r = [] + let r = [] try { - r = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', q) + r = await throttleConseilQuery('big_map_contents', q) .then(result => result.map(row => { const objectId = row['value'].toString().replace(/^Pair ([0-9]{1,}) .*/, '$1') const objectUrl = row['value'].toString().replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1') @@ -288,10 +293,15 @@ const getArtisticUniverse = async (max_time) => { universe.push({ objectId, ipfsHash, minter: artistMap[objectId], swaps: swapMap[objectId] !== undefined ? swapMap[objectId] : []}) })) // NOTE: it's a work in progress, this will drop failed requests and return a smaller set than expected + } catch (error) { + console.log('failed at query', q, 'with error', error) } finally { return r }})) + // const b = performance.now() + // console.log(`time ${b - a}`) + return universe } @@ -341,6 +351,92 @@ const getObjectById = async (objectId) => { return { objectId, ipfsHash, swaps } } +const getObjectOwnersById = async (objectId) => { + let objectQuery = conseiljs.ConseilQueryBuilder.blankQuery(); + objectQuery = conseiljs.ConseilQueryBuilder.addFields(objectQuery, 'key', 'value'); + objectQuery = conseiljs.ConseilQueryBuilder.addPredicate(objectQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftLedger]) + objectQuery = conseiljs.ConseilQueryBuilder.addPredicate(objectQuery, 'key', conseiljs.ConseilOperator.ENDSWITH, [` ${objectId}`]) + objectQuery = conseiljs.ConseilQueryBuilder.setLimit(objectQuery, 20_000) + + const objectResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', objectQuery) + + const ownerMap = objectResult.map(r => { + const address = conseiljs.TezosMessageUtils.readAddress(r['key'].replace(/Pair 0x([0-9a-z]{1,}) [0-9]+$/, '$1')) + const amount = parseInt(r['value']) + + return { address, amount } + }) + + return ownerMap +} + +const listSwaps = async () => { + let swapsQuery = conseiljs.ConseilQueryBuilder.blankQuery() + swapsQuery = conseiljs.ConseilQueryBuilder.addFields(swapsQuery, 'key', 'value') + swapsQuery = conseiljs.ConseilQueryBuilder.addPredicate(swapsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftSwapMap]) + swapsQuery = conseiljs.ConseilQueryBuilder.addOrdering(swapsQuery, 'block_level', conseiljs.ConseilSortDirection.DESC) + swapsQuery = conseiljs.ConseilQueryBuilder.setLimit(swapsQuery, 50_000) + + const swapsResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', swapsQuery) + + const swapValuePattern = new RegExp('Pair [(]Pair 0x([0-9a-z]{44}) ([0-9]+)[)] [(]Pair ([0-9]+) ([0-9]+)[)]'); + const swapList = swapsResult + .filter(row => row['value'] && row['value'].length > 0) + .map(row => { + const swapDefinition = row['value'] + + if (swapValuePattern.test(swapDefinition)) { + match = swapDefinition.match(swapValuePattern); + + return { + swap_id: row['key'], + issuer: conseiljs.TezosMessageUtils.readAddress(match[1]), + objkt_id: match[3], + objkt_amount: match[2], + xtz_per_objkt: (new BigNumber(match[4])).dividedBy(1_000_000).toFixed(6) + } + } + + return undefined + }).filter(row => row) + + const distinctIds = [... new Set(swapList.map(swap => swap['swap_id']))] + const ipfsMap = await getObjktInfoUrl(distinctIds) + + swapList.forEach(row => { + row['ipfsHash'] = ipfsMap[row['objkt_id']] || '' + }) + + return swapList.filter(row => row['ipfsHash'].length) +} + +const getObjktInfoUrl = async (objktList) => { + const queryChunks = chunkArray(objktList, 100) + + const makeObjectQuery = (keys) => { + let mintedObjectsQuery = conseiljs.ConseilQueryBuilder.blankQuery(); + mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addFields(mintedObjectsQuery, 'key_hash', 'value'); + mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintedObjectsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftMetadataMap]) + mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintedObjectsQuery, 'key', (keys.length > 1 ? conseiljs.ConseilOperator.IN : conseiljs.ConseilOperator.EQ), keys) + mintedObjectsQuery = conseiljs.ConseilQueryBuilder.setLimit(mintedObjectsQuery, keys.length) + + return mintedObjectsQuery + } + + const objectQueries = queryChunks.map(c => makeObjectQuery(c)) + const objectIpfsMap = {} + await Promise.all(objectQueries.map(async (q) => await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', q) + .then(result => result.map(row => { + const objectId = row['value'].toString().replace(/^Pair ([0-9]{1,}) .*/, '$1') + const objectUrl = row['value'].toString().replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1') + const ipfsHash = Buffer.from(objectUrl, 'hex').toString().slice(7); + + objectIpfsMap[objectId] = ipfsHash + })))) + + return objectIpfsMap +} + const chunkArray = (arr, len) => { // TODO: move to util.js let chunks = [], i = 0, @@ -353,10 +449,18 @@ const chunkArray = (arr, len) => { // TODO: move to util.js return chunks; } +const throttleConseilQuery = throttleConseil(async (table, query) => { + const result = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', table, query) + + return Promise.resolve(result) +}) + module.exports = { getCollectionForAddress, gethDaoBalanceForAddress, getArtisticOutputForAddress, getObjectById, - getArtisticUniverse + getObjectOwnersById, + getArtisticUniverse, + listSwaps } diff --git a/package.json b/package.json index 04d9450..301e8ef 100644 --- a/package.json +++ b/package.json @@ -1,38 +1,40 @@ { - "name": "hicetnunc-apiv2", - "version": "1.0.0", - "description": "", - "main": "index.js", - "scripts": { - "start": "node index.js" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/hicetnunc2000/hicetnunc.git" - }, - "author": "@hicetnunc2000", - "license": "MIT", - "bugs": { - "url": "https://github.com/hicetnunc2000/hicetnunc/issues" - }, - "homepage": "https://github.com/hicetnunc2000/hicetnunc#readme", - "dependencies": { - "axios": "^0.21.1", - "bignumber.js": "9.0.1", - "cloud-local-storage": "0.0.11", - "conseiljs": "5.0.7-2", - "cors": "^2.8.5", - "dotenv": "^8.2.0", - "express": "^4.17.1", - "fetch": "^1.1.0", - "lodash": "^4.17.21", - "loglevel": "1.7.1", - "node-fetch": "2.6.1", - "serverless-dotenv-plugin": "^3.8.1", - "serverless-http": "^2.7.0" - }, - "engines": { - "node": "12.20.1", - "npm": "6.14.10" - } + "name": "hicetnunc-apiv2", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "start": "node index.js" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/hicetnunc2000/hicetnunc-api.git" + }, + "author": "@hicetnunc2000", + "license": "MIT", + "bugs": { + "url": "https://github.com/hicetnunc2000/hicetnunc-api/issues" + }, + "homepage": "https://github.com/hicetnunc2000/hicetnunc-api#readme", + "dependencies": { + "axios": "^0.21.1", + "bignumber.js": "9.0.1", + "cloud-local-storage": "0.0.11", + "conseiljs": "5.0.7-2", + "cors": "^2.8.5", + "dotenv": "^8.2.0", + "express": "^4.17.1", + "fetch": "^1.1.0", + "lodash": "^4.17.21", + "loglevel": "1.7.1", + "node-fetch": "2.6.1", + "perf_hooks": "0.0.1", + "p-throttle": "4.1.1", + "serverless-dotenv-plugin": "^3.8.1", + "serverless-http": "^2.7.0" + }, + "engines": { + "node": "12.20.1", + "npm": "6.14.10" + } }