diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 0000000..b07d135 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,10 @@ +{ + "extends": "eslint:recommended", + "env": { + "node": true, + "es6": true + }, + "parserOptions": { + "ecmaVersion": 8 + } +} \ No newline at end of file diff --git a/.node-version b/.node-version new file mode 100644 index 0000000..1cd71be --- /dev/null +++ b/.node-version @@ -0,0 +1 @@ +12.20.1 \ No newline at end of file diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..1cd71be --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +12.20.1 \ No newline at end of file diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..c3481a7 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,4 @@ +{ + "singleQuote": true, + "semi": false +} \ No newline at end of file diff --git a/conseilUtil.js b/conseilUtil.js deleted file mode 100644 index f5c95dd..0000000 --- a/conseilUtil.js +++ /dev/null @@ -1,570 +0,0 @@ -const conseiljs = require('conseiljs') -const fetch = require('node-fetch') -const log = require('loglevel') -const BigNumber = require('bignumber.js') - -const logger = log.getLogger('conseiljs') -logger.setLevel('error', false) -conseiljs.registerLogger(logger) -conseiljs.registerFetch(fetch) -const conseilServer = 'https://conseil-prod.cryptonomic-infra.tech' -const conseilApiKey = 'aa73fa8a-8626-4f43-a605-ff63130f37b1' // signup at nautilus.cloud -const mainnet = require('./config').networkConfig - - -const hDAOFeed = async () => { - - let hDAOQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - hDAOQuery = conseiljs.ConseilQueryBuilder.addFields(hDAOQuery, 'key', 'value'); - hDAOQuery = conseiljs.ConseilQueryBuilder.addPredicate(hDAOQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.curationsPtr]) - hDAOQuery = conseiljs.ConseilQueryBuilder.setLimit(hDAOQuery, 30_000) - - let hDAOResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', hDAOQuery); - return hDAOResult.map(e => { - return { - token_id : parseInt(e.key), - hDAO_balance : parseInt((e.value).split(' ')[1]) - } - }) -} -/** - * Returns a list of nft token ids and amounts that a given address owns. - * - * @param {string} address - * @returns - */ -const getCollectionForAddress = async (address) => { - let collectionQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - collectionQuery = conseiljs.ConseilQueryBuilder.addFields(collectionQuery, 'key', 'value', 'operation_group_id'); - collectionQuery = conseiljs.ConseilQueryBuilder.addPredicate(collectionQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftLedger]) - collectionQuery = conseiljs.ConseilQueryBuilder.addPredicate(collectionQuery, 'key', conseiljs.ConseilOperator.STARTSWITH, [ - `Pair 0x${conseiljs.TezosMessageUtils.writeAddress(address)}`, - ]) - collectionQuery = conseiljs.ConseilQueryBuilder.addPredicate(collectionQuery, 'value', conseiljs.ConseilOperator.EQ, [0], true) - collectionQuery = conseiljs.ConseilQueryBuilder.setLimit(collectionQuery, 30_000) - - const collectionResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', collectionQuery); - let collection = collectionResult.map((i) => { - return { - piece: i['key'].toString().replace(/.* ([0-9]{1,}$)/, '$1'), - amount: Number(i['value']), - opId: i['operation_group_id'] - } - }) - - const queryChunks = chunkArray(collection.map(i => i.piece), 50) // NOTE: consider increasing this number somewhat - const makeObjectQuery = (keys) => { - let mintedObjectsQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addFields(mintedObjectsQuery, 'key_hash', 'value'); - mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintedObjectsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftMetadataMap]) - mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintedObjectsQuery, 'key', (keys.length > 1 ? conseiljs.ConseilOperator.IN : conseiljs.ConseilOperator.EQ), keys) - mintedObjectsQuery = conseiljs.ConseilQueryBuilder.setLimit(mintedObjectsQuery, keys.length) - - return mintedObjectsQuery - } - - const objectQueries = queryChunks.map(c => makeObjectQuery(c)) - const objectIpfsMap = {} - await Promise.all(objectQueries.map(async (q) => await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', q) - .then(result => result.map(row => { - const objectId = row['value'].toString().replace(/^Pair ([0-9]{1,}) .*/, '$1') - const objectUrl = row['value'].toString().replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1') - const ipfsHash = Buffer.from(objectUrl, 'hex').toString().slice(7); - - objectIpfsMap[objectId] = ipfsHash - })))) - - const operationGroupIds = collectionResult.map((r) => r.operation_group_id) - const priceQueryChunks = chunkArray(operationGroupIds, 30) - const makeLastPriceQuery = (opIds) => { - let lastPriceQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - lastPriceQuery = conseiljs.ConseilQueryBuilder.addFields(lastPriceQuery, 'timestamp', 'amount', 'operation_group_hash', 'parameters_entrypoints', 'parameters'); - lastPriceQuery = conseiljs.ConseilQueryBuilder.addPredicate(lastPriceQuery, 'kind', conseiljs.ConseilOperator.EQ, ['transaction']); - lastPriceQuery = conseiljs.ConseilQueryBuilder.addPredicate(lastPriceQuery, 'status', conseiljs.ConseilOperator.EQ, ['applied']); - lastPriceQuery = conseiljs.ConseilQueryBuilder.addPredicate(lastPriceQuery, 'internal', conseiljs.ConseilOperator.EQ, ['false']); - lastPriceQuery = conseiljs.ConseilQueryBuilder.addPredicate(lastPriceQuery, 'operation_group_hash', opIds.length > 1 ? conseiljs.ConseilOperator.IN : conseiljs.ConseilOperator.EQ, opIds); - lastPriceQuery = conseiljs.ConseilQueryBuilder.setLimit(lastPriceQuery, opIds.length); - - return lastPriceQuery; - } - - const priceQueries = priceQueryChunks.map((c) => makeLastPriceQuery(c)) - const priceMap = {}; - await Promise.all( - priceQueries.map( - async (q) => - await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'operations', q).then((result) => - result.map((row) => { - let amount = 0; - const action = row.parameters_entrypoints; - - if (action === 'collect') { - amount = Number(row.parameters.toString().replace(/^Pair ([0-9]+) [0-9]+/, '$1')); - } else if (action === 'transfer') { - amount = Number( - row.parameters - .toString() - .replace( - /[{] Pair \"[1-9A-HJ-NP-Za-km-z]{36}\" [{] Pair \"[1-9A-HJ-NP-Za-km-z]{36}\" [(]Pair [0-9]+ [0-9]+[)] [}] [}]/, - '$1' - ) - ); - } - - priceMap[row.operation_group_hash] = { - price: new BigNumber(row.amount), - amount, - timestamp: row.timestamp, - action, - }; - }) - ) - ) - ) - - collection = collection.map(i => { - let price = 0 - let receivedOn = new Date() - let action = '' - - try { - const priceRecord = priceMap[i.opId] - price = priceRecord.price.dividedToIntegerBy(priceRecord.amount).toNumber() - receivedOn = new Date(priceRecord.timestamp) - action = priceRecord.action === 'collect' ? 'Purchased' : 'Received' - } catch { - // - } - - delete i.opId - - return { - price: isNaN(price) ? 0 : price, - receivedOn, - action, - ipfsHash: objectIpfsMap[i.piece.toString()], - ...i - }}) - - return collection.sort((a, b) => b.receivedOn.getTime() - a.receivedOn.getTime()) // sort descending by date – most-recently acquired art first -} - -const gethDaoBalanceForAddress = async (address) => { - let hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.blankQuery() - hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addFields(hDaoBalanceQuery, 'value') - hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(hDaoBalanceQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.daoLedger]) - hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(hDaoBalanceQuery, 'key', conseiljs.ConseilOperator.EQ, [ - `Pair 0x${conseiljs.TezosMessageUtils.writeAddress(address)} 0` - ]) - hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(hDaoBalanceQuery, 'value', conseiljs.ConseilOperator.EQ, [0], true) - hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.setLimit(hDaoBalanceQuery, 1) - - let balance = 0 - - try { - const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', hDaoBalanceQuery) - balance = balanceResult[0]['value'] // TODO: consider bigNumber here, for the moment there is no reason for it - } catch (error) { - console.log(`gethDaoBalanceForAddress failed for ${JSON.stringify(hDaoBalanceQuery)} with ${error}`) - } - - return balance -} - -const getTokenBalance = async (big_map_id, address, fa2=false, token_id=0) => { - let tokenBalanceQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - tokenBalanceQuery = conseiljs.ConseilQueryBuilder.addFields(tokenBalanceQuery, 'value'); - tokenBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(tokenBalanceQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [big_map_id]) - if (fa2) { - tokenBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(tokenBalanceQuery, 'key', conseiljs.ConseilOperator.EQ, [ - `Pair 0x${conseiljs.TezosMessageUtils.writeAddress(address)} ${token_id}` - ]) - } - else { - tokenBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(tokenBalanceQuery, 'key', conseiljs.ConseilOperator.EQ, [ - `0x${conseiljs.TezosMessageUtils.writeAddress(address)}` - ]) - } - tokenBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(tokenBalanceQuery, 'value', conseiljs.ConseilOperator.EQ, [0], true) - tokenBalanceQuery = conseiljs.ConseilQueryBuilder.setLimit(tokenBalanceQuery, 1) - - let balance = 0 - - try { - const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', tokenBalanceQuery); - balance = balanceResult[0]['value'] // TODO: consider bigNumber here, for the moment there is no reason for it - } catch (error) { - console.log(`getTokenBalance failed for ${JSON.stringify(tokenBalanceQuery)} with ${error}`) - } - - return balance -} - - -const getTezBalanceForAddress = async (address) => { - let accountQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - accountQuery = conseiljs.ConseilQueryBuilder.addFields(accountQuery, 'balance'); - accountQuery = conseiljs.ConseilQueryBuilder.addPredicate(accountQuery, 'account_id', conseiljs.ConseilOperator.EQ, [address], false); - accountQuery = conseiljs.ConseilQueryBuilder.setLimit(accountQuery, 1); - - try { - const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'accounts', accountQuery); - balance = balanceResult[0]['balance'] // TODO: consider bigNumber here, for the moment there is no reason for it - } catch (error) { - console.log(`getTezBalanceForAddress failed for ${JSON.stringify(accountQuery)} with ${error}`) - } - - return balance -} - - -const gethDAOPerTez = async() => { - const tezBalance = await(getTezBalanceForAddress(mainnet.hDaoSwap)) - const hdaoBalance = await(gethDaoBalanceForAddress(mainnet.hDaoSwap)) - return hdaoBalance / tezBalance -} - -const getKolibriPerTez = async() => { - const tezBalance = await(getTezBalanceForAddress(mainnet.kolibriSwap)) - var kolibriBalance = await(getTokenBalance(mainnet.kolibriLedger, mainnet.kolibriSwap)) - - // TODO: Find a better way to get the balance, this is FA1.2, mike? - kolibriBalance = parseInt(kolibriBalance.replace("Pair {} ", "")) / (10**((18 - 6))) - return kolibriBalance / tezBalance -} - - -const gethDaoBalances = async () => { - let hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addFields(hDaoBalanceQuery, 'key', 'value'); - hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(hDaoBalanceQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.daoLedger]) - hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(hDaoBalanceQuery, 'value', conseiljs.ConseilOperator.EQ, [0], true) - hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.setLimit(hDaoBalanceQuery, 500_000) - - let balance = 0 - let hdaoMap = {} - - try { - const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', hDaoBalanceQuery); - - - balanceResult.forEach(row => { - hdaoMap[conseiljs.TezosMessageUtils.readAddress(row['key'].toString().replace(/^Pair 0x([0-9a-z]{1,}) [0-9]+/, '$1'))] = row['value'] - }) - //#balance = balanceResult[0]['value'] // TODO: consider bigNumber here, for the moment there is no reason for it - } catch (error) { - console.log(`gethDaoBalanceForAddress failed for ${JSON.stringify(hDaoBalanceQuery)} with ${error}`) - } - - - return hdaoMap - -} - -const getObjektOwners = async (objekt_id) => { - let objektBalanceQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - objektBalanceQuery = conseiljs.ConseilQueryBuilder.addFields(objektBalanceQuery, 'key', 'value'); - objektBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(objektBalanceQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftLedger]) - objektBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(objektBalanceQuery, 'key', conseiljs.ConseilOperator.ENDSWITH, [` ${objekt_id}`], false) - objektBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate(objektBalanceQuery, 'value', conseiljs.ConseilOperator.EQ, [0], true) - objektBalanceQuery = conseiljs.ConseilQueryBuilder.setLimit(objektBalanceQuery, 500_000) - - let objektMap = {} - - try { - const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', objektBalanceQuery); - - - balanceResult.forEach(row => { - objektMap[conseiljs.TezosMessageUtils.readAddress(row['key'].toString().replace(/^Pair 0x([0-9a-z]{1,}) [0-9]+/, '$1'))] = row['value'] - }) - //#balance = balanceResult[0]['value'] // TODO: consider bigNumber here, for the moment there is no reason for it - } catch (error) { - console.log(`getObjektOwners failed for ${JSON.stringify(objektBalanceQuery)} with ${error}`) - } - - - return objektMap - -} - -const getObjektMintingsLastWeek = async () => { - var d = new Date(); - d.setDate(d.getDate()-5); - let mintOperationQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - mintOperationQuery = conseiljs.ConseilQueryBuilder.addFields(mintOperationQuery, 'source'); - mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'kind', conseiljs.ConseilOperator.EQ, ['transaction']) - mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'timestamp', conseiljs.ConseilOperator.AFTER, [d.getTime()]) // 2021 Feb 1 - mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'status', conseiljs.ConseilOperator.EQ, ['applied']) - mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'destination', conseiljs.ConseilOperator.EQ, [mainnet.protocol]) - mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'parameters_entrypoints', conseiljs.ConseilOperator.EQ, ['mint_OBJKT']) - mintOperationQuery = conseiljs.ConseilQueryBuilder.addOrdering(mintOperationQuery, 'block_level', conseiljs.ConseilSortDirection.DESC) - mintOperationQuery = conseiljs.ConseilQueryBuilder.setLimit(mintOperationQuery, 900_000) // TODO: this is hardwired and will not work for highly productive artists - - const mintOperationResult = await conseiljs.TezosConseilClient.getTezosEntityData( - { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, - 'mainnet', - 'operations', - mintOperationQuery); - - const mints = mintOperationResult.map(r => r['source']) - - var initialValue = {} - var reducer = function(minters, mintOp) { - if (!minters[mintOp]) { - minters[mintOp] = 1; - } else { - minters[mintOp] = minters[mintOp] + 1; - } - return minters; - } - return mints.reduce(reducer, initialValue) -} - - -/** - * Queries Conseil in two steps to get all the objects minted by a specific address. Step 1 is to query for all 'mint_OBJKT' operations performed by the account to get the list of operation group hashes. Then that list is partitioned into chunks and another query (or set of queries) is run to get big_map values. These values are then parsed into an array of 3-tuples containing the hashed big_map key that can be used to query a Tezos node directly, the nft token id and the ipfs item hash. - * - * @param {string} address - * @returns - */ -const getArtisticOutputForAddress = async (address) => { - let mintOperationQuery = conseiljs.ConseilQueryBuilder.blankQuery() - mintOperationQuery = conseiljs.ConseilQueryBuilder.addFields(mintOperationQuery, 'operation_group_hash') - mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'kind', conseiljs.ConseilOperator.EQ, ['transaction']) - mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'timestamp', conseiljs.ConseilOperator.AFTER, [1612240919000]) // 2021 Feb 1 - mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'status', conseiljs.ConseilOperator.EQ, ['applied']) - mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'destination', conseiljs.ConseilOperator.EQ, [mainnet.protocol]) - mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'parameters_entrypoints', conseiljs.ConseilOperator.EQ, ['mint_OBJKT']) - mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'source', conseiljs.ConseilOperator.EQ, [address]) - mintOperationQuery = conseiljs.ConseilQueryBuilder.addOrdering(mintOperationQuery, 'block_level', conseiljs.ConseilSortDirection.DESC) - mintOperationQuery = conseiljs.ConseilQueryBuilder.setLimit(mintOperationQuery, 256) // TODO: this is hardwired and will not work for highly productive artists - - const mintOperationResult = await conseiljs.TezosConseilClient.getTezosEntityData( - { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, - 'mainnet', - 'operations', - mintOperationQuery) - - const operationGroupIds = mintOperationResult.map(r => r['operation_group_hash']) - const queryChunks = chunkArray(operationGroupIds, 30) - - const makeObjectQuery = (opIds) => { - let mintedObjectsQuery = conseiljs.ConseilQueryBuilder.blankQuery() - mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addFields(mintedObjectsQuery, 'key_hash', 'value') - mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintedObjectsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftMetadataMap]) - mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintedObjectsQuery, 'operation_group_id', (opIds.length > 1 ? conseiljs.ConseilOperator.IN : conseiljs.ConseilOperator.EQ), opIds) - mintedObjectsQuery = conseiljs.ConseilQueryBuilder.setLimit(mintedObjectsQuery, opIds.length) - - return mintedObjectsQuery - } - - const objectQueries = queryChunks.map(c => makeObjectQuery(c)) - - const objectInfo = await Promise.all(objectQueries.map(async (q) => await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', q) - .then(result => result.map(row => { - const objectId = row['value'].toString().replace(/^Pair ([0-9]{1,}) .*/, '$1') - const objectUrl = row['value'].toString().replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1') - const ipfsHash = Buffer.from(objectUrl, 'hex').toString().slice(7); - - return { key: row['key_hash'], objectId, ipfsHash } - })))) - - return objectInfo.flat(1).sort((a, b) => parseInt(b.objectId) - parseInt(a.objectId)) -} - -const getArtisticUniverse = async (max_time) => { - var d = new Date(); - d.setDate(d.getDate()-14); - let mintOperationQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - mintOperationQuery = conseiljs.ConseilQueryBuilder.addFields(mintOperationQuery, 'operation_group_hash'); - mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'kind', conseiljs.ConseilOperator.EQ, ['transaction']) - mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'timestamp', conseiljs.ConseilOperator.BETWEEN, [max_time, d.getTime()]) //Two weeks ago - mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'status', conseiljs.ConseilOperator.EQ, ['applied']) - mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'destination', conseiljs.ConseilOperator.EQ, [mainnet.protocol]) - mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate(mintOperationQuery, 'parameters_entrypoints', conseiljs.ConseilOperator.EQ, ['mint_OBJKT']) - mintOperationQuery = conseiljs.ConseilQueryBuilder.addOrdering(mintOperationQuery, 'block_level', conseiljs.ConseilSortDirection.DESC) - mintOperationQuery = conseiljs.ConseilQueryBuilder.setLimit(mintOperationQuery, 2500) - - const mintOperationResult = await conseiljs.TezosConseilClient.getTezosEntityData( - { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, - 'mainnet', - 'operations', - mintOperationQuery); - - const operationGroupIds = mintOperationResult.map(r => r['operation_group_hash']) - - let royaltiesQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - royaltiesQuery = conseiljs.ConseilQueryBuilder.addFields(royaltiesQuery, 'key', 'value'); - royaltiesQuery = conseiljs.ConseilQueryBuilder.addPredicate(royaltiesQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftRoyaltiesMap]) - royaltiesQuery = conseiljs.ConseilQueryBuilder.setLimit(royaltiesQuery, 30_000) - const royaltiesResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', royaltiesQuery) - let artistMap = {} - royaltiesResult.forEach(row => { - artistMap[row['key']] = conseiljs.TezosMessageUtils.readAddress(row['value'].toString().replace(/^Pair 0x([0-9a-z]{1,}) [0-9]+/, '$1')) - }) - - let swapsQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - swapsQuery = conseiljs.ConseilQueryBuilder.addFields(swapsQuery, 'key', 'value') - swapsQuery = conseiljs.ConseilQueryBuilder.addPredicate(swapsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftSwapMap]) - swapsQuery = conseiljs.ConseilQueryBuilder.setLimit(swapsQuery, 30_000) // NOTE, limited to 30_000 - - const swapsResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', swapsQuery) - - const swapStoragePattern = new RegExp(`Pair [(]Pair 0x([0-9a-z]{44}) ([0-9]+)[)] [(]Pair ([0-9]+) ([0-9]+)[)]`); - let swapMap = {} - - swapsResult.forEach(row => { - swap_id = row['key'] - const match = swapStoragePattern.exec(row['value']) - if (!match) { return; } - const amount = match[2] - const objkt_id = match[3] - const xtz_per_objkt = match[4] - - if (swapMap[row['key']]) { - swapMap[row['key']].push({ }) - } else { - swapMap[row['key']] = [{ swap_id, objkt_id, amount, xtz_per_objkt }] - } - }) - - const queryChunks = chunkArray(operationGroupIds, 50) - - const makeObjectQuery = (opIds) => { - let objectsQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - objectsQuery = conseiljs.ConseilQueryBuilder.addFields(objectsQuery, 'key', 'value', 'operation_group_id'); - objectsQuery = conseiljs.ConseilQueryBuilder.addPredicate(objectsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftMetadataMap]) - objectsQuery = conseiljs.ConseilQueryBuilder.addPredicate(objectsQuery, 'operation_group_id', (opIds.length > 1 ? conseiljs.ConseilOperator.IN : conseiljs.ConseilOperator.EQ), opIds) - objectsQuery = conseiljs.ConseilQueryBuilder.setLimit(objectsQuery, opIds.length) - - return objectsQuery - } - - const objectQueries = queryChunks.map(c => makeObjectQuery(c)) - - let universe = [] - await Promise.all( - objectQueries.map(async (q) => { - const r = [] - try { - r = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', q) - .then(result => result.map(row => { - const objectId = row['value'].toString().replace(/^Pair ([0-9]{1,}) .*/, '$1') - const objectUrl = row['value'].toString().replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1') - const ipfsHash = Buffer.from(objectUrl, 'hex').toString().slice(7); - - universe.push({ objectId, ipfsHash, minter: artistMap[objectId], swaps: swapMap[objectId] !== undefined ? swapMap[objectId] : []}) - })) // NOTE: it's a work in progress, this will drop failed requests and return a smaller set than expected - } finally { - return r - }})) - - return universe -} - - -const getFeaturedArtisticUniverse = async(max_time) => { - - hdaoMap = await gethDaoBalances() - - mintsPerCreator = await getObjektMintingsLastWeek() - - artisticUniverse = await getArtisticUniverse(max_time) - - hdaoPerTez = await gethDAOPerTez() - - // Cost to be on feed per objekt last 7 days shouldn't be higher than: - // 0.1tez - // 1 hDAO - // But not lower than: - // 0.01 hDAO - // - // We should probably add more thresholds like $, € and yen - // It should be cheap but not too cheap and it shouldn't be - // affected by tez or hDAO volatility - - thresholdHdao = Math.min(1_000_000, Math.max(100_000 * hdaoPerTez, 10_000)) - - return artisticUniverse.filter(function (o) { - return (((hdaoMap[o.minter] || 0) / Math.max(mintsPerCreator[o.minter] || 1, 1)) > thresholdHdao) - }) -} - -const getRecommendedCurateDefault = async() => { - hdaoPerTez = await gethDAOPerTez() - kolPerTez = await getKolibriPerTez() - hdaoPerKol = hdaoPerTez / kolPerTez - //Minimum of $0.1, 0.1 hDAO, and 0.1tez, in hDAO - return Math.floor(Math.min(hdaoPerKol * 0.1, 0.1, 0.1 * hdaoPerTez) * 1_000_000) -} - -/** - * Returns object ipfs hash and swaps if any - * - * @param {number} objectId - * @returns - */ -const getObjectById = async (objectId) => { - let objectQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - objectQuery = conseiljs.ConseilQueryBuilder.addFields(objectQuery, 'value'); - objectQuery = conseiljs.ConseilQueryBuilder.addPredicate(objectQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftMetadataMap]) - objectQuery = conseiljs.ConseilQueryBuilder.addPredicate(objectQuery, 'key', conseiljs.ConseilOperator.EQ, [objectId]) - objectQuery = conseiljs.ConseilQueryBuilder.setLimit(objectQuery, 1) - - const objectResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', objectQuery) - - const objectUrl = objectResult[0]['value'].toString().replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1') - const ipfsHash = Buffer.from(objectUrl, 'hex').toString().slice(7); - - let swapsQuery = conseiljs.ConseilQueryBuilder.blankQuery(); - swapsQuery = conseiljs.ConseilQueryBuilder.addFields(swapsQuery, 'key', 'value'); - swapsQuery = conseiljs.ConseilQueryBuilder.addPredicate(swapsQuery, 'big_map_id', conseiljs.ConseilOperator.EQ, [mainnet.nftSwapMap]) - swapsQuery = conseiljs.ConseilQueryBuilder.addPredicate(swapsQuery, 'value', conseiljs.ConseilOperator.LIKE, [`) (Pair ${objectId} `]) - swapsQuery = conseiljs.ConseilQueryBuilder.setLimit(swapsQuery, 1000) // NOTE, limited to 1000 swaps for a given object - - const swapsResult = await conseiljs.TezosConseilClient.getTezosEntityData({ url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, 'mainnet', 'big_map_contents', swapsQuery) - const swapStoragePattern = new RegExp(`Pair [(]Pair 0x([0-9a-z]{44}) ([0-9]+)[)] [(]Pair ${objectId} ([0-9]+)[)]`); - - let swaps = [] - try { - swapsResult.map(row => { - const match = swapStoragePattern.exec(row['value']) - - swaps.push({ - swap_id: row['key'], - issuer: conseiljs.TezosMessageUtils.readAddress(match[1]), - objkt_amount: match[2], - xtz_per_objkt: match[3] - }) - }) - } catch (error) { - console.log(`failed to process swaps for ${objectId} with ${error}`) - } - - return { objectId, ipfsHash, swaps } -} - -const chunkArray = (arr, len) => { // TODO: move to util.js - let chunks = [], - i = 0, - n = arr.length; - - while (i < n) { - chunks.push(arr.slice(i, i += len)); - } - - return chunks; -} - -module.exports = { - getCollectionForAddress, - gethDaoBalanceForAddress, - getArtisticOutputForAddress, - getObjectById, - getArtisticUniverse, - getFeaturedArtisticUniverse, - hDAOFeed, - getRecommendedCurateDefault, - getObjektOwners -} diff --git a/index.js b/index.js deleted file mode 100644 index 7db1f96..0000000 --- a/index.js +++ /dev/null @@ -1,294 +0,0 @@ -const serverless = require('serverless-http') -const axios = require('axios') -const express = require('express') -const cors = require('cors') -const _ = require('lodash') -const conseilUtil = require('./conseilUtil') -const { random } = require('lodash') - -const BURN_ADDRESS = 'tz1burnburnburnburnburnburnburjAYjjX' - -require('dotenv').config() - -const reducer = (accumulator, currentValue) => parseInt(accumulator) + parseInt(currentValue) - -const getIpfsHash = async (ipfsHash) => { - return await axios.get('https://cloudflare-ipfs.com/ipfs/' + ipfsHash).then(res => res.data) -} - - -const owners = async (obj) => { - var owners = await conseilUtil.getObjektOwners(obj.token_id) - var values_arr = (_.values(owners)) - obj.total_amount = (values_arr.map(e => parseInt(e))).length > 0 ? values_arr.filter(e => parseInt(e) > 0).reduce(reducer) : 0 - obj.owners = owners - return obj -} - -const desc = arr => _.sortBy(arr, e => parseInt(e.objectId)).reverse() -const offset = (arr, set) => arr.slice(set * 30, set * 30 + 30) - - -const customFloor = function (value, roundTo) { - return Math.floor(value / roundTo) * roundTo; -} - -const ONE_MINUTE_MILLIS = 60 * 1000 - -const randomFeed = async (counter, res) => { - var feed = await conseilUtil.getArtisticUniverse(0) - feed = offset(_.shuffle(feed), counter) - feed = await feed.map(async e => { - e.token_info = await getIpfsHash(e.ipfsHash) - e.token_id = parseInt(e.objectId) - return e - }) - var promise = Promise.all(feed.map(e => e)) - promise.then(async (results) => { - var aux_arr = results.map(e => e) - res.json({ result: aux_arr }) - }) -} - -const getFeed = async (res, counter, featured, max_time) => { - const now_time = Date.now() - const immutable = (typeof max_time !== 'undefined') && (max_time < now_time) - max_time = (typeof max_time !== 'undefined') ? max_time : customFloor(now_time, ONE_MINUTE_MILLIS) - - console.log(`feed, featured: ${featured}`) - var arr - if (featured) { - arr = await conseilUtil.getFeaturedArtisticUniverse(0, max_time) - } else { - arr = await conseilUtil.getArtisticUniverse(0, max_time) - } - - var feed = offset(desc(arr), counter) - feed = await feed.map(async e => { - e.token_info = await getIpfsHash(e.ipfsHash) - e.token_id = parseInt(e.objectId) - console.log(e) - return e - }) - var cache_time - if (immutable) { - cache_time = 60 * 10 - } - else { - cache_time = Math.floor(((max_time + ONE_MINUTE_MILLIS) - now_time) / 1000) - } - var promise = Promise.all(feed.map(e => e)) - return promise.then(async (results) => { - var aux_arr = results.map(e => e) - res.set('Cache-Control', `public, max-age=${cache_time}`) - res.json({ result: aux_arr }) - }) -} - -const getTzLedger = async (tz, res) => { - /* var ledger = desc(await getObjktLedger()) - var objkts = await getObjkts() - var tzLedger = _.map(filterTz(ledger, tz), (obj) => _.assign(obj, _.find(objkts, { token_id : obj.tk_id }))) - */ - var collection = await conseilUtil.getCollectionForAddress(tz) - var creations = await conseilUtil.getArtisticOutputForAddress(tz) - var hdao = await conseilUtil.gethDaoBalanceForAddress(tz) - - console.log(hdao) - - var validCreations = [] - - await Promise.all(creations.map(async (c) => { - c.token_id = c.objectId - - await owners(c) - - var burnAddrCount = c.owners[BURN_ADDRESS] - var allIssuesBurned = burnAddrCount && burnAddrCount === c.total_amount - - if (!allIssuesBurned) { - delete c.owners - - validCreations.push(c) - } - - return arr - })) - - validCreations = validCreations.sort((a, b) => parseInt(b.objectId) - parseInt(a.objectId)) - - var arr = [] - console.log([...collection, ...validCreations]) - var arr = [...collection, ...validCreations] - - var result = arr.map(async e => { - e.token_info = await getIpfsHash(e.ipfsHash) - - if (e.piece != undefined) { - e.token_id = parseInt(e.piece) - } else { - e.token_id = parseInt(e.objectId) - - } - console.log(e) - return e - }) - - var promise = Promise.all(result.map(e => e)) - promise.then(async results => { - var result = results.map(e => e) - console.log(result) - - res.json({ - result: _.uniqBy(result, (e) => { - return e.token_id - }), - hdao: hdao - }) - }) -} - -const getObjktById = async (id, res) => { - var objkt = await conseilUtil.getObjectById(id) - objkt.token_id = objkt.objectId - objkt = await owners(objkt) - objkt.token_info = await getIpfsHash(objkt.ipfsHash) - console.log(objkt) - - return objkt -} - -const mergehDAO = async (obj) => { - var obj_aux = await getObjktById(obj.token_id) - obj_aux.hDAO_balance = obj.hDAO_balance - return obj_aux -} - -const hDAOFeed = async (counter, res) => { - - var hDAO = await conseilUtil.hDAOFeed() - var set = _.orderBy(hDAO, ['hDAO_balance'], ['desc']) - var objkts = await (offset(set, counter)).map(async e => await mergehDAO(e)) - - var promise = Promise.all(objkts.map(e => e)) - promise.then(results => { - var result = results.map(e => e) - console.log(result) - res.set('Cache-Control', `public, max-age=300`) - res.json({ result: result }) - }).catch(e => { - res.status(500).json({ error: 'downstream API failure' }) - }) -} - -// list of restricted addresses -const getRestrictedAddresses = async () => { - const list = await axios.get('https://raw.githubusercontent.com/hicetnunc2000/hicetnunc/main/filters/w.json').then(res => res.data) - return list -} - -// list of restricted objkts -const getRestrictedObjkts = async () => { - const list = await axios.get('https://raw.githubusercontent.com/hicetnunc2000/hicetnunc/main/filters/o.json').then(res => res.data) - return list -} - -const app = express() - -app.use(express.json()) -app.use(cors({ origin: '*' })) - -app.post('/feed|/featured', async (req, res) => { - const feedOffset = req.body.counter || 0 - const isFeatured = req.path === '/featured' - const max_time = req.body.max_time - await getFeed(res, feedOffset, isFeatured, max_time) -}) - -app.get('/feed|/featured', async (req, res) => { - const feedOffset = req.query.counter || 0 - const isFeatured = req.path === '/featured' - const max_time = req.query.max_time - await getFeed(res, feedOffset, isFeatured, max_time) -}) - - -// Random - -app.post('/random', async (req, res) => { - res.set('Cache-Control', `public, max-age=300`) - await randomFeed(parseInt(req.body.counter), res) -}) - -app.get('/random', async (req, res) => { - res.set('Cache-Control', `public, max-age=300`) - await randomFeed(parseInt(req.query.counter), res) -}) - - -// TZ - -const get_tz = async(tz, res) => { - // list of restricted addresses - var list = await getRestrictedAddresses() - res.set('Cache-Control', `public, max-age=120`) - list.includes(tz) - ? - res.json({ result: [] }) - : - await getTzLedger(tz, res) -} - -app.post('/tz', async (req, res) => { - await get_tz(req.body.tz, res) -}) -app.get('/tz', async (req, res) => { - await get_tz(req.query.tz, res) -}) - -// OBJEKT - -const objkt = async(id, res) => { - - // list of restricted objkts - var list = await getRestrictedObjkts() - - res.set('Cache-Control', `public, max-age=120`) - list.includes(parseInt(id)) - ? - res.json({ result: [] }) - : - res.json({ result: await getObjktById(id) }) -} - -app.post('/objkt', async (req, res) => { - await objkt(req.body.objkt_id, res) - -}) -app.get('/objkt', async (req, res) => { - await objkt(req.query.id, res) -}) - - - - -app.get('/recommend_curate', async (req, res) => { - const amt = await conseilUtil.getRecommendedCurateDefault() - res.set('Cache-Control', `public, max-age=300`) - res.json({ amount: amt }) -}) - - -// HDAO -app.post('/hdao', async (req, res) => { - await hDAOFeed(parseInt(req.body.counter), res) -}) - -app.get('/hdao', async (req, res) => { - await hDAOFeed(parseInt(req.query.counter), res) -}) - -//app.listen(3001) -console.log('SERVER RUNNING ON localhost:3001') -module.exports.handler = serverless(app) - diff --git a/package.json b/package.json index 7a43524..c3a7bd5 100644 --- a/package.json +++ b/package.json @@ -1,39 +1,44 @@ { - "name": "hicetnunc-apiv2", - "version": "1.0.0", - "description": "", - "main": "index.js", - "scripts": { - "start": "node index.js" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/hicetnunc2000/hicetnunc-api.git" - }, - "author": "@hicetnunc2000", - "license": "MIT", - "bugs": { - "url": "https://github.com/hicetnunc2000/hicetnunc-api/issues" - }, - "homepage": "https://github.com/hicetnunc2000/hicetnunc-api#readme", - "dependencies": { - "axios": "^0.21.1", - "bignumber.js": "9.0.1", - "cloud-local-storage": "0.0.11", - "conseiljs": "5.0.8-1", - "cors": "^2.8.5", - "dotenv": "^8.2.0", - "express": "^4.17.1", - "fetch": "^1.1.0", - "lodash": "^4.17.21", - "loglevel": "1.7.1", - "node-fetch": "2.6.1", - "prex": "^0.4.7", - "serverless-dotenv-plugin": "^3.8.1", - "serverless-http": "^2.7.0" - }, - "engines": { - "node": "12.20.1", - "npm": "6.14.10" - } + "name": "hicetnunc-apiv2", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "start": "node src/index.js", + "start-dev": "NODE_ENV=development ./node_modules/.bin/nodemon src/index.js", + "pretty": "./node_modules/.bin/prettier --loglevel warn --write src" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/hicetnunc2000/hicetnunc-api.git" + }, + "author": "@hicetnunc2000", + "license": "MIT", + "bugs": { + "url": "https://github.com/hicetnunc2000/hicetnunc-api/issues" + }, + "homepage": "https://github.com/hicetnunc2000/hicetnunc-api#readme", + "dependencies": { + "axios": "^0.21.1", + "bignumber.js": "9.0.1", + "cloud-local-storage": "0.0.11", + "conseiljs": "5.0.8-1", + "cors": "^2.8.5", + "dotenv": "^8.2.0", + "express": "^4.17.1", + "fetch": "^1.1.0", + "lodash": "^4.17.21", + "loglevel": "1.7.1", + "node-fetch": "2.6.1", + "serverless-dotenv-plugin": "^3.8.1", + "serverless-http": "^2.7.0" + }, + "engines": { + "node": "12.20.1", + "npm": "6.14.10" + }, + "devDependencies": { + "nodemon": "^2.0.7", + "prettier": "^2.2.1" + } } diff --git a/serverless.yml b/serverless.yml index ff6d4b1..315359e 100644 --- a/serverless.yml +++ b/serverless.yml @@ -16,7 +16,7 @@ provider: functions: handler: - handler: index.handler + handler: src/index.handler timeout: 120 events: - http: diff --git a/src/index.js b/src/index.js new file mode 100644 index 0000000..13aee7d --- /dev/null +++ b/src/index.js @@ -0,0 +1,28 @@ +'use strict' + +// set node path to import from this directory as if they are node modules +process.env.NODE_PATH = 'src/lib' + +require('module').Module._initPaths() +require('dotenv').config() + +const cors = require('cors') +const express = require('express') +const router = require('router') +const serverless = require('serverless-http') + +const { serverPort: PORT } = require('config') + +const app = express() + +app.use(express.json()) +app.use(cors({ origin: '*' })) +app.use(router) + +if (process.env.NODE_ENV === 'development') { + app.listen(PORT, () => { + console.log(`SERVER RUNNING ON localhost:${PORT}`) + }) +} + +module.exports.handler = serverless(app) diff --git a/config.js b/src/lib/config.js similarity index 72% rename from config.js rename to src/lib/config.js index ccd3cfd..f3ccadd 100644 --- a/config.js +++ b/src/lib/config.js @@ -1,4 +1,10 @@ -const mainnet = { +'use strict' + +module.exports = { + burnAddress: + process.env.BURN_ADDRESS || 'tz1burnburnburnburnburnburnburjAYjjX', + feedItemsPerPage: process.env.FEED_ITEMS_PER_PAGE || 30, + networkConfig: { network: 'mainnet', nftContract: 'KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton', hDAOToken: 'KT1AFA2mwNUMNd4SsujE1YYp29vd8BZejyKW', @@ -13,8 +19,6 @@ const mainnet = { kolibriLedger: 380, hDaoSwap: "KT1V41fGzkdTJki4d11T1Rp9yPkCmDhB7jph", kolibriSwap: "KT1CiSKXR68qYSxnbzjwvfeMCRburaSDonT2", -} - -module.exports = { - networkConfig: mainnet + }, + serverPort: 3001, } diff --git a/src/lib/conseil.js b/src/lib/conseil.js new file mode 100644 index 0000000..a0e3e23 --- /dev/null +++ b/src/lib/conseil.js @@ -0,0 +1,1111 @@ +const conseiljs = require('conseiljs') +const fetch = require('node-fetch') +const log = require('loglevel') +const BigNumber = require('bignumber.js') + +const logger = log.getLogger('conseiljs') +logger.setLevel('error', false) +conseiljs.registerLogger(logger) +conseiljs.registerFetch(fetch) +const conseilServer = 'https://conseil-prod.cryptonomic-infra.tech' +const conseilApiKey = 'aa73fa8a-8626-4f43-a605-ff63130f37b1' // signup at nautilus.cloud +const mainnet = require('./config').networkConfig + +const hDAOFeed = async () => { + let hDAOQuery = conseiljs.ConseilQueryBuilder.blankQuery() + hDAOQuery = conseiljs.ConseilQueryBuilder.addFields(hDAOQuery, 'key', 'value') + hDAOQuery = conseiljs.ConseilQueryBuilder.addPredicate( + hDAOQuery, + 'big_map_id', + conseiljs.ConseilOperator.EQ, + [mainnet.curationsPtr] + ) + hDAOQuery = conseiljs.ConseilQueryBuilder.setLimit(hDAOQuery, 300_000) + + let hDAOResult = await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'big_map_contents', + hDAOQuery + ) + return hDAOResult.map((e) => { + return { + token_id: parseInt(e.key), + hDAO_balance: parseInt(e.value.split(' ')[1]), + } + }) +} +/** + * Returns a list of nft token ids and amounts that a given address owns. + * + * @param {string} address + * @returns + */ +const getCollectionForAddress = async (address) => { + let collectionQuery = conseiljs.ConseilQueryBuilder.blankQuery() + collectionQuery = conseiljs.ConseilQueryBuilder.addFields( + collectionQuery, + 'key', + 'value', + 'operation_group_id' + ) + collectionQuery = conseiljs.ConseilQueryBuilder.addPredicate( + collectionQuery, + 'big_map_id', + conseiljs.ConseilOperator.EQ, + [mainnet.nftLedger] + ) + collectionQuery = conseiljs.ConseilQueryBuilder.addPredicate( + collectionQuery, + 'key', + conseiljs.ConseilOperator.STARTSWITH, + [`Pair 0x${conseiljs.TezosMessageUtils.writeAddress(address)}`] + ) + collectionQuery = conseiljs.ConseilQueryBuilder.addPredicate( + collectionQuery, + 'value', + conseiljs.ConseilOperator.EQ, + [0], + true + ) + collectionQuery = conseiljs.ConseilQueryBuilder.setLimit( + collectionQuery, + 300_000 + ) + + const collectionResult = await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'big_map_contents', + collectionQuery + ) + let collection = collectionResult.map((i) => { + return { + piece: i['key'].toString().replace(/.* ([0-9]{1,}$)/, '$1'), + amount: Number(i['value']), + opId: i['operation_group_id'], + } + }) + + const queryChunks = chunkArray( + collection.map((i) => i.piece), + 50 + ) // NOTE: consider increasing this number somewhat + const makeObjectQuery = (keys) => { + let mintedObjectsQuery = conseiljs.ConseilQueryBuilder.blankQuery() + mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addFields( + mintedObjectsQuery, + 'key_hash', + 'value' + ) + mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintedObjectsQuery, + 'big_map_id', + conseiljs.ConseilOperator.EQ, + [mainnet.nftMetadataMap] + ) + mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintedObjectsQuery, + 'key', + keys.length > 1 + ? conseiljs.ConseilOperator.IN + : conseiljs.ConseilOperator.EQ, + keys + ) + mintedObjectsQuery = conseiljs.ConseilQueryBuilder.setLimit( + mintedObjectsQuery, + keys.length + ) + + return mintedObjectsQuery + } + + const objectQueries = queryChunks.map((c) => makeObjectQuery(c)) + const objectIpfsMap = {} + await Promise.all( + objectQueries.map( + async (q) => + await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'big_map_contents', + q + ).then((result) => + result.map((row) => { + const objectId = row['value'] + .toString() + .replace(/^Pair ([0-9]{1,}) .*/, '$1') + const objectUrl = row['value'] + .toString() + .replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1') + const ipfsHash = Buffer.from(objectUrl, 'hex').toString().slice(7) + + objectIpfsMap[objectId] = ipfsHash + }) + ) + ) + ) + + const operationGroupIds = collectionResult.map((r) => r.operation_group_id) + const priceQueryChunks = chunkArray(operationGroupIds, 30) + const makeLastPriceQuery = (opIds) => { + let lastPriceQuery = conseiljs.ConseilQueryBuilder.blankQuery() + lastPriceQuery = conseiljs.ConseilQueryBuilder.addFields( + lastPriceQuery, + 'timestamp', + 'amount', + 'operation_group_hash', + 'parameters_entrypoints', + 'parameters' + ) + lastPriceQuery = conseiljs.ConseilQueryBuilder.addPredicate( + lastPriceQuery, + 'kind', + conseiljs.ConseilOperator.EQ, + ['transaction'] + ) + lastPriceQuery = conseiljs.ConseilQueryBuilder.addPredicate( + lastPriceQuery, + 'status', + conseiljs.ConseilOperator.EQ, + ['applied'] + ) + lastPriceQuery = conseiljs.ConseilQueryBuilder.addPredicate( + lastPriceQuery, + 'internal', + conseiljs.ConseilOperator.EQ, + ['false'] + ) + lastPriceQuery = conseiljs.ConseilQueryBuilder.addPredicate( + lastPriceQuery, + 'operation_group_hash', + opIds.length > 1 + ? conseiljs.ConseilOperator.IN + : conseiljs.ConseilOperator.EQ, + opIds + ) + lastPriceQuery = conseiljs.ConseilQueryBuilder.setLimit( + lastPriceQuery, + opIds.length + ) + + return lastPriceQuery + } + + const priceQueries = priceQueryChunks.map((c) => makeLastPriceQuery(c)) + const priceMap = {} + await Promise.all( + priceQueries.map( + async (q) => + await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'operations', + q + ).then((result) => + result.map((row) => { + let amount = 0 + const action = row.parameters_entrypoints + + if (action === 'collect') { + amount = Number( + row.parameters.toString().replace(/^Pair ([0-9]+) [0-9]+/, '$1') + ) + } else if (action === 'transfer') { + amount = Number( + row.parameters + .toString() + .replace( + /[{] Pair \"[1-9A-HJ-NP-Za-km-z]{36}\" [{] Pair \"[1-9A-HJ-NP-Za-km-z]{36}\" [(]Pair [0-9]+ [0-9]+[)] [}] [}]/, + '$1' + ) + ) + } + + priceMap[row.operation_group_hash] = { + price: new BigNumber(row.amount), + amount, + timestamp: row.timestamp, + action, + } + }) + ) + ) + ) + + collection = collection.map((i) => { + let price = 0 + let receivedOn = new Date() + let action = '' + + try { + const priceRecord = priceMap[i.opId] + price = priceRecord.price + .dividedToIntegerBy(priceRecord.amount) + .toNumber() + receivedOn = new Date(priceRecord.timestamp) + action = priceRecord.action === 'collect' ? 'Purchased' : 'Received' + } catch { + // + } + + delete i.opId + + return { + price: isNaN(price) ? 0 : price, + receivedOn, + action, + ipfsHash: objectIpfsMap[i.piece.toString()], + ...i, + } + }) + + return collection.sort( + (a, b) => b.receivedOn.getTime() - a.receivedOn.getTime() + ) // sort descending by date – most-recently acquired art first +} + +const gethDaoBalanceForAddress = async (address) => { + let hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.blankQuery() + hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addFields( + hDaoBalanceQuery, + 'value' + ) + hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate( + hDaoBalanceQuery, + 'big_map_id', + conseiljs.ConseilOperator.EQ, + [mainnet.daoLedger] + ) + hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate( + hDaoBalanceQuery, + 'key', + conseiljs.ConseilOperator.EQ, + [`Pair 0x${conseiljs.TezosMessageUtils.writeAddress(address)} 0`] + ) + hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate( + hDaoBalanceQuery, + 'value', + conseiljs.ConseilOperator.EQ, + [0], + true + ) + hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.setLimit(hDaoBalanceQuery, 1) + + let balance = 0 + + try { + const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'big_map_contents', + hDaoBalanceQuery + ) + balance = balanceResult[0]['value'] // TODO: consider bigNumber here, for the moment there is no reason for it + } catch (error) { + console.log( + `gethDaoBalanceForAddress failed for ${JSON.stringify( + hDaoBalanceQuery + )} with ${error}` + ) + } + + return balance +} + +const getTokenBalance = async ( + big_map_id, + address, + fa2 = false, + token_id = 0 +) => { + let tokenBalanceQuery = conseiljs.ConseilQueryBuilder.blankQuery() + tokenBalanceQuery = conseiljs.ConseilQueryBuilder.addFields( + tokenBalanceQuery, + 'value' + ) + tokenBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate( + tokenBalanceQuery, + 'big_map_id', + conseiljs.ConseilOperator.EQ, + [big_map_id] + ) + if (fa2) { + tokenBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate( + tokenBalanceQuery, + 'key', + conseiljs.ConseilOperator.EQ, + [ + `Pair 0x${conseiljs.TezosMessageUtils.writeAddress( + address + )} ${token_id}`, + ] + ) + } else { + tokenBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate( + tokenBalanceQuery, + 'key', + conseiljs.ConseilOperator.EQ, + [`0x${conseiljs.TezosMessageUtils.writeAddress(address)}`] + ) + } + tokenBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate( + tokenBalanceQuery, + 'value', + conseiljs.ConseilOperator.EQ, + [0], + true + ) + tokenBalanceQuery = conseiljs.ConseilQueryBuilder.setLimit( + tokenBalanceQuery, + 1 + ) + + let balance = 0 + + try { + const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'big_map_contents', + tokenBalanceQuery + ) + balance = balanceResult[0]['value'] // TODO: consider bigNumber here, for the moment there is no reason for it + } catch (error) { + console.log( + `getTokenBalance failed for ${JSON.stringify( + tokenBalanceQuery + )} with ${error}` + ) + } + + return balance +} + +const getTezBalanceForAddress = async (address) => { + let accountQuery = conseiljs.ConseilQueryBuilder.blankQuery() + accountQuery = conseiljs.ConseilQueryBuilder.addFields( + accountQuery, + 'balance' + ) + accountQuery = conseiljs.ConseilQueryBuilder.addPredicate( + accountQuery, + 'account_id', + conseiljs.ConseilOperator.EQ, + [address], + false + ) + accountQuery = conseiljs.ConseilQueryBuilder.setLimit(accountQuery, 1) + + try { + const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'accounts', + accountQuery + ) + balance = balanceResult[0]['balance'] // TODO: consider bigNumber here, for the moment there is no reason for it + } catch (error) { + console.log( + `getTezBalanceForAddress failed for ${JSON.stringify( + accountQuery + )} with ${error}` + ) + } + + return balance +} + +const gethDAOPerTez = async () => { + const tezBalance = await getTezBalanceForAddress(mainnet.hDaoSwap) + const hdaoBalance = await gethDaoBalanceForAddress(mainnet.hDaoSwap) + return hdaoBalance / tezBalance +} + +const getKolibriPerTez = async () => { + const tezBalance = await getTezBalanceForAddress(mainnet.kolibriSwap) + var kolibriBalance = await getTokenBalance( + mainnet.kolibriLedger, + mainnet.kolibriSwap + ) + + // TODO: Find a better way to get the balance, this is FA1.2, mike? + kolibriBalance = + parseInt(kolibriBalance.replace('Pair {} ', '')) / 10 ** (18 - 6) + return kolibriBalance / tezBalance +} + +const gethDaoBalances = async () => { + let hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.blankQuery() + hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addFields( + hDaoBalanceQuery, + 'key', + 'value' + ) + hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate( + hDaoBalanceQuery, + 'big_map_id', + conseiljs.ConseilOperator.EQ, + [mainnet.daoLedger] + ) + hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate( + hDaoBalanceQuery, + 'value', + conseiljs.ConseilOperator.EQ, + [0], + true + ) + hDaoBalanceQuery = conseiljs.ConseilQueryBuilder.setLimit( + hDaoBalanceQuery, + 500_000 + ) + + let balance = 0 + let hdaoMap = {} + + try { + const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'big_map_contents', + hDaoBalanceQuery + ) + + balanceResult.forEach((row) => { + hdaoMap[ + conseiljs.TezosMessageUtils.readAddress( + row['key'].toString().replace(/^Pair 0x([0-9a-z]{1,}) [0-9]+/, '$1') + ) + ] = row['value'] + }) + //#balance = balanceResult[0]['value'] // TODO: consider bigNumber here, for the moment there is no reason for it + } catch (error) { + console.log( + `gethDaoBalanceForAddress failed for ${JSON.stringify( + hDaoBalanceQuery + )} with ${error}` + ) + } + + return hdaoMap +} + +const getObjektOwners = async (objekt_id) => { + let objektBalanceQuery = conseiljs.ConseilQueryBuilder.blankQuery() + objektBalanceQuery = conseiljs.ConseilQueryBuilder.addFields( + objektBalanceQuery, + 'key', + 'value' + ) + objektBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate( + objektBalanceQuery, + 'big_map_id', + conseiljs.ConseilOperator.EQ, + [mainnet.nftLedger] + ) + objektBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate( + objektBalanceQuery, + 'key', + conseiljs.ConseilOperator.ENDSWITH, + [` ${objekt_id}`], + false + ) + objektBalanceQuery = conseiljs.ConseilQueryBuilder.addPredicate( + objektBalanceQuery, + 'value', + conseiljs.ConseilOperator.EQ, + [0], + true + ) + objektBalanceQuery = conseiljs.ConseilQueryBuilder.setLimit( + objektBalanceQuery, + 500_000 + ) + + let objektMap = {} + + try { + const balanceResult = await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'big_map_contents', + objektBalanceQuery + ) + + balanceResult.forEach((row) => { + objektMap[ + conseiljs.TezosMessageUtils.readAddress( + row['key'].toString().replace(/^Pair 0x([0-9a-z]{1,}) [0-9]+/, '$1') + ) + ] = row['value'] + }) + //#balance = balanceResult[0]['value'] // TODO: consider bigNumber here, for the moment there is no reason for it + } catch (error) { + console.log( + `getObjektOwners failed for ${JSON.stringify( + objektBalanceQuery + )} with ${error}` + ) + } + + return objektMap +} + +const getObjektMintingsLastWeek = async () => { + var d = new Date() + d.setDate(d.getDate() - 7) + let mintOperationQuery = conseiljs.ConseilQueryBuilder.blankQuery() + mintOperationQuery = conseiljs.ConseilQueryBuilder.addFields( + mintOperationQuery, + 'source' + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintOperationQuery, + 'kind', + conseiljs.ConseilOperator.EQ, + ['transaction'] + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintOperationQuery, + 'timestamp', + conseiljs.ConseilOperator.AFTER, + [d.getTime()] + ) // 2021 Feb 1 + mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintOperationQuery, + 'status', + conseiljs.ConseilOperator.EQ, + ['applied'] + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintOperationQuery, + 'destination', + conseiljs.ConseilOperator.EQ, + [mainnet.protocol] + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintOperationQuery, + 'parameters_entrypoints', + conseiljs.ConseilOperator.EQ, + ['mint_OBJKT'] + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.addOrdering( + mintOperationQuery, + 'block_level', + conseiljs.ConseilSortDirection.DESC + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.setLimit( + mintOperationQuery, + 500_000 + ) // TODO: this is hardwired and will not work for highly productive artists + + const mintOperationResult = await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'operations', + mintOperationQuery + ) + + const mints = mintOperationResult.map((r) => r['source']) + + var initialValue = {} + var reducer = function (minters, mintOp) { + if (!minters[mintOp]) { + minters[mintOp] = 1 + } else { + minters[mintOp] = minters[mintOp] + 1 + } + return minters + } + return mints.reduce(reducer, initialValue) +} + +/** + * Queries Conseil in two steps to get all the objects minted by a specific address. Step 1 is to query for all 'mint_OBJKT' operations performed by the account to get the list of operation group hashes. Then that list is partitioned into chunks and another query (or set of queries) is run to get big_map values. These values are then parsed into an array of 3-tuples containing the hashed big_map key that can be used to query a Tezos node directly, the nft token id and the ipfs item hash. + * + * @param {string} address + * @returns + */ +const getArtisticOutputForAddress = async (address) => { + let mintOperationQuery = conseiljs.ConseilQueryBuilder.blankQuery() + mintOperationQuery = conseiljs.ConseilQueryBuilder.addFields( + mintOperationQuery, + 'operation_group_hash' + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintOperationQuery, + 'kind', + conseiljs.ConseilOperator.EQ, + ['transaction'] + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintOperationQuery, + 'timestamp', + conseiljs.ConseilOperator.AFTER, + [1612240919000] + ) // 2021 Feb 1 + mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintOperationQuery, + 'status', + conseiljs.ConseilOperator.EQ, + ['applied'] + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintOperationQuery, + 'destination', + conseiljs.ConseilOperator.EQ, + [mainnet.protocol] + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintOperationQuery, + 'parameters_entrypoints', + conseiljs.ConseilOperator.EQ, + ['mint_OBJKT'] + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintOperationQuery, + 'source', + conseiljs.ConseilOperator.EQ, + [address] + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.addOrdering( + mintOperationQuery, + 'block_level', + conseiljs.ConseilSortDirection.DESC + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.setLimit( + mintOperationQuery, + 256 + ) // TODO: this is hardwired and will not work for highly productive artists + + const mintOperationResult = await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'operations', + mintOperationQuery + ) + + const operationGroupIds = mintOperationResult.map( + (r) => r['operation_group_hash'] + ) + const queryChunks = chunkArray(operationGroupIds, 30) + + const makeObjectQuery = (opIds) => { + let mintedObjectsQuery = conseiljs.ConseilQueryBuilder.blankQuery() + mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addFields( + mintedObjectsQuery, + 'key_hash', + 'value' + ) + mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintedObjectsQuery, + 'big_map_id', + conseiljs.ConseilOperator.EQ, + [mainnet.nftMetadataMap] + ) + mintedObjectsQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintedObjectsQuery, + 'operation_group_id', + opIds.length > 1 + ? conseiljs.ConseilOperator.IN + : conseiljs.ConseilOperator.EQ, + opIds + ) + mintedObjectsQuery = conseiljs.ConseilQueryBuilder.setLimit( + mintedObjectsQuery, + opIds.length + ) + + return mintedObjectsQuery + } + + const objectQueries = queryChunks.map((c) => makeObjectQuery(c)) + + const objectInfo = await Promise.all( + objectQueries.map( + async (q) => + await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'big_map_contents', + q + ).then((result) => + result.map((row) => { + const objectId = row['value'] + .toString() + .replace(/^Pair ([0-9]{1,}) .*/, '$1') + const objectUrl = row['value'] + .toString() + .replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1') + const ipfsHash = Buffer.from(objectUrl, 'hex').toString().slice(7) + + return { key: row['key_hash'], objectId, ipfsHash } + }) + ) + ) + ) + + return objectInfo + .flat(1) + .sort((a, b) => parseInt(b.objectId) - parseInt(a.objectId)) +} + +const getArtisticUniverse = async (max_time) => { + max_time = ((typeof max_time !== 'undefined') && max_time != 0) ? max_time : (new Date()).getTime() + var d = new Date() + d.setDate(d.getDate() - 14) + let mintOperationQuery = conseiljs.ConseilQueryBuilder.blankQuery() + mintOperationQuery = conseiljs.ConseilQueryBuilder.addFields( + mintOperationQuery, + 'operation_group_hash' + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintOperationQuery, + 'kind', + conseiljs.ConseilOperator.EQ, + ['transaction'] + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintOperationQuery, + 'timestamp', + conseiljs.ConseilOperator.BETWEEN, + [d.getTime(), max_time] + ) //Two weeks ago + mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintOperationQuery, + 'status', + conseiljs.ConseilOperator.EQ, + ['applied'] + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintOperationQuery, + 'destination', + conseiljs.ConseilOperator.EQ, + [mainnet.protocol] + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.addPredicate( + mintOperationQuery, + 'parameters_entrypoints', + conseiljs.ConseilOperator.EQ, + ['mint_OBJKT'] + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.addOrdering( + mintOperationQuery, + 'block_level', + conseiljs.ConseilSortDirection.DESC + ) + mintOperationQuery = conseiljs.ConseilQueryBuilder.setLimit( + mintOperationQuery, + 2500 + ) + + const mintOperationResult = await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'operations', + mintOperationQuery + ) + + const operationGroupIds = mintOperationResult.map( + (r) => r['operation_group_hash'] + ) + + let royaltiesQuery = conseiljs.ConseilQueryBuilder.blankQuery() + royaltiesQuery = conseiljs.ConseilQueryBuilder.addFields( + royaltiesQuery, + 'key', + 'value' + ) + royaltiesQuery = conseiljs.ConseilQueryBuilder.addPredicate( + royaltiesQuery, + 'big_map_id', + conseiljs.ConseilOperator.EQ, + [mainnet.nftRoyaltiesMap] + ) + royaltiesQuery = conseiljs.ConseilQueryBuilder.setLimit( + royaltiesQuery, + 5_000_000 + ) + const royaltiesResult = await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'big_map_contents', + royaltiesQuery + ) + let artistMap = {} + royaltiesResult.forEach((row) => { + artistMap[row['key']] = conseiljs.TezosMessageUtils.readAddress( + row['value'].toString().replace(/^Pair 0x([0-9a-z]{1,}) [0-9]+/, '$1') + ) + }) + + let swapsQuery = conseiljs.ConseilQueryBuilder.blankQuery() + swapsQuery = conseiljs.ConseilQueryBuilder.addFields( + swapsQuery, + 'key', + 'value' + ) + swapsQuery = conseiljs.ConseilQueryBuilder.addPredicate( + swapsQuery, + 'big_map_id', + conseiljs.ConseilOperator.EQ, + [mainnet.nftSwapMap] + ) + swapsQuery = conseiljs.ConseilQueryBuilder.setLimit(swapsQuery, 5_000_000) // NOTE, limited to 30_000 + + const swapsResult = await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'big_map_contents', + swapsQuery + ) + + const swapStoragePattern = new RegExp( + `Pair [(]Pair 0x([0-9a-z]{44}) ([0-9]+)[)] [(]Pair ([0-9]+) ([0-9]+)[)]` + ) + let swapMap = {} + + swapsResult.forEach((row) => { + swap_id = row['key'] + const match = swapStoragePattern.exec(row['value']) + if (!match) { + return + } + const amount = match[2] + const objkt_id = match[3] + const xtz_per_objkt = match[4] + + if (swapMap[row['key']]) { + swapMap[row['key']].push({}) + } else { + swapMap[row['key']] = [{ swap_id, objkt_id, amount, xtz_per_objkt }] + } + }) + + const queryChunks = chunkArray(operationGroupIds, 50) + + const makeObjectQuery = (opIds) => { + let objectsQuery = conseiljs.ConseilQueryBuilder.blankQuery() + objectsQuery = conseiljs.ConseilQueryBuilder.addFields( + objectsQuery, + 'key', + 'value', + 'operation_group_id' + ) + objectsQuery = conseiljs.ConseilQueryBuilder.addPredicate( + objectsQuery, + 'big_map_id', + conseiljs.ConseilOperator.EQ, + [mainnet.nftMetadataMap] + ) + objectsQuery = conseiljs.ConseilQueryBuilder.addPredicate( + objectsQuery, + 'operation_group_id', + opIds.length > 1 + ? conseiljs.ConseilOperator.IN + : conseiljs.ConseilOperator.EQ, + opIds + ) + objectsQuery = conseiljs.ConseilQueryBuilder.setLimit( + objectsQuery, + opIds.length + ) + + return objectsQuery + } + + const objectQueries = queryChunks.map((c) => makeObjectQuery(c)) + + let universe = [] + await Promise.all( + objectQueries.map(async (q) => { + const r = [] + try { + r = await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'big_map_contents', + q + ).then((result) => + result.map((row) => { + const objectId = row['value'] + .toString() + .replace(/^Pair ([0-9]{1,}) .*/, '$1') + const objectUrl = row['value'] + .toString() + .replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1') + const ipfsHash = Buffer.from(objectUrl, 'hex').toString().slice(7) + + universe.push({ + objectId, + ipfsHash, + minter: artistMap[objectId], + swaps: swapMap[objectId] !== undefined ? swapMap[objectId] : [], + }) + }) + ) // NOTE: it's a work in progress, this will drop failed requests and return a smaller set than expected + } finally { + return r + } + }) + ) + + return universe +} + +const getFeaturedArtisticUniverse = async (max_time) => { + hdaoMap = await gethDaoBalances() + + mintsPerCreator = await getObjektMintingsLastWeek() + + artisticUniverse = await getArtisticUniverse(max_time) + + hdaoPerTez = await gethDAOPerTez() + kolPerTez = await getKolibriPerTez() + hdaoPerKol = hdaoPerTez / kolPerTez + + + // Cost to be on feed per objekt last 7 days shouldn't be higher than: + // 0.1tez + // 1 hDAO + // $1 + // But not lower than: + // 0.01 hDAO + // + // We should probably add more thresholds like $, € and yen + // It should be cheap but not too cheap and it shouldn't be + // affected by tez or hDAO volatility + + let thresholdHdao = Math.floor( + Math.max(10_000, + Math.min(1_000_000, + Math.max( + 100_000 * hdaoPerTez, + 100_000 * hdaoPerKol) + ) + )) + //thresholdHdao = 0 + + return artisticUniverse.filter(o => + (hdaoMap[o.minter] || 100_000) / Math.max(mintsPerCreator[o.minter] || 1, 1) > + thresholdHdao) +} + +const getRecommendedCurateDefault = async () => { + hdaoPerTez = await gethDAOPerTez() + kolPerTez = await getKolibriPerTez() + hdaoPerKol = hdaoPerTez / kolPerTez + //Minimum of $0.1, 0.1 hDAO, and 0.1tez, in hDAO + return Math.floor( + Math.min(hdaoPerKol * 0.1, 0.1, 0.1 * hdaoPerTez) * 1_000_000 + ) +} + +/** + * Returns object ipfs hash and swaps if any + * + * @param {number} objectId + * @returns + */ +const getObjectById = async (objectId) => { + let objectQuery = conseiljs.ConseilQueryBuilder.blankQuery() + objectQuery = conseiljs.ConseilQueryBuilder.addFields(objectQuery, 'value') + objectQuery = conseiljs.ConseilQueryBuilder.addPredicate( + objectQuery, + 'big_map_id', + conseiljs.ConseilOperator.EQ, + [mainnet.nftMetadataMap] + ) + objectQuery = conseiljs.ConseilQueryBuilder.addPredicate( + objectQuery, + 'key', + conseiljs.ConseilOperator.EQ, + [objectId] + ) + objectQuery = conseiljs.ConseilQueryBuilder.setLimit(objectQuery, 1) + + const objectResult = await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'big_map_contents', + objectQuery + ) + + const objectUrl = objectResult[0]['value'] + .toString() + .replace(/.* 0x([0-9a-z]{1,}) \}$/, '$1') + const ipfsHash = Buffer.from(objectUrl, 'hex').toString().slice(7) + + let swapsQuery = conseiljs.ConseilQueryBuilder.blankQuery() + swapsQuery = conseiljs.ConseilQueryBuilder.addFields( + swapsQuery, + 'key', + 'value' + ) + swapsQuery = conseiljs.ConseilQueryBuilder.addPredicate( + swapsQuery, + 'big_map_id', + conseiljs.ConseilOperator.EQ, + [mainnet.nftSwapMap] + ) + swapsQuery = conseiljs.ConseilQueryBuilder.addPredicate( + swapsQuery, + 'value', + conseiljs.ConseilOperator.LIKE, + [`) (Pair ${objectId} `] + ) + swapsQuery = conseiljs.ConseilQueryBuilder.setLimit(swapsQuery, 1000) // NOTE, limited to 1000 swaps for a given object + + const swapsResult = await conseiljs.TezosConseilClient.getTezosEntityData( + { url: conseilServer, apiKey: conseilApiKey, network: 'mainnet' }, + 'mainnet', + 'big_map_contents', + swapsQuery + ) + const swapStoragePattern = new RegExp( + `Pair [(]Pair 0x([0-9a-z]{44}) ([0-9]+)[)] [(]Pair ${objectId} ([0-9]+)[)]` + ) + + let swaps = [] + try { + swapsResult.map((row) => { + const match = swapStoragePattern.exec(row['value']) + + swaps.push({ + swap_id: row['key'], + issuer: conseiljs.TezosMessageUtils.readAddress(match[1]), + objkt_amount: match[2], + xtz_per_objkt: match[3], + }) + }) + } catch (error) { + console.log(`failed to process swaps for ${objectId} with ${error}`) + } + + return { objectId, ipfsHash, swaps } +} + +const chunkArray = (arr, len) => { + // TODO: move to util.js + let chunks = [], + i = 0, + n = arr.length + + while (i < n) { + chunks.push(arr.slice(i, (i += len))) + } + + return chunks +} + +module.exports = { + getCollectionForAddress, + gethDaoBalanceForAddress, + getArtisticOutputForAddress, + getObjectById, + getArtisticUniverse, + getFeaturedArtisticUniverse, + hDAOFeed, + getRecommendedCurateDefault, + getObjektOwners, +} diff --git a/src/lib/router/index.js b/src/lib/router/index.js new file mode 100644 index 0000000..3098fa8 --- /dev/null +++ b/src/lib/router/index.js @@ -0,0 +1,163 @@ +'use strict' + +const router = require('express').Router() +const readFeed = require('./readFeed') +const readRandomFeed = require('./readRandomFeed') +const readIssuer = require('./readIssuer') +const readObjkt = require('./readObjkt') +const readHdaoFeed = require('./readHdaoFeed') +const readRecommendCurate = require('./readRecommendCurate') + +const MILLISECOND_MODIFIER = 1000 +const ONE_MINUTE_MILLIS = 60 * MILLISECOND_MODIFIER +const DEFAULT_CACHE_TTL = 60 * 10 +const STATUS_CODE_SUCCESS = 200 +const CACHE_MAX_AGE_MAP = { + hdao: 300, + issuer: 120, + objkt: 120, + random: 300, + recommendCurate: 300, +} + +router + .route('/feed|/featured') + .all(_processClientCache) + .get((req, res, next) => { + req.body.max_time = req.query.max_time + req.body.counter = req.query.counter + + return next() + }, _asyncHandler(readFeed)) + .post(_asyncHandler(readFeed)) + +router + .route('/random') + .all((req, res, next) => { + _setCacheHeaderOnSuccess(res, CACHE_MAX_AGE_MAP.random) + + return next() + }) + .get((req, res, next) => { + req.body.counter = req.query.counter + req.body.max_time = req.query.max_time + return next() + }, _asyncHandler(readRandomFeed)) + .post(_asyncHandler(readRandomFeed)) + +router + .route('/tz') + .all((req, res, next) => { + _setCacheHeaderOnSuccess(res, CACHE_MAX_AGE_MAP.issuer) + + return next() + }) + .get((req, res, next) => { + req.body.tz = req.query.tz + + return next() + }, _asyncHandler(readIssuer)) + .post(_asyncHandler(readIssuer)) + +router + .route('/objkt') + .all((req, res, next) => { + _setCacheHeaderOnSuccess(res, CACHE_MAX_AGE_MAP.objkt) + + return next() + }) + .get((req, res, next) => { + req.body.objkt_id = req.query.id + + return next() + }, _asyncHandler(readObjkt)) + .post(_asyncHandler(readObjkt)) + +router + .route('/hdao') + .all((req, res, next) => { + _setCacheHeaderOnSuccess(res, CACHE_MAX_AGE_MAP.hdao) + + return next() + }) + .get((req, res, next) => { + req.body.counter = req.query.counter + + return next() + }, _asyncHandler(readHdaoFeed)) + .post(_asyncHandler(readHdaoFeed)) + +router.get( + '/recommend_curate', + (req, res, next) => { + _setCacheHeaderOnSuccess(res, CACHE_MAX_AGE_MAP.recommendCurate) + + return next() + }, + _asyncHandler(readRecommendCurate) +) + +module.exports = router + +/** + * Express cannot process promise rejections correctly. We need to encapsulate async + * route handlers with logic to pass promise rejections to express's middleware chain + * as an error. + */ +function _asyncHandler(cb) { + return function (req, res, next) { + Promise.resolve(cb(req, res, next)).catch(next) + } +} + +/** + * Set the fetch time for conseil requests and set the Cache-Control header + * on successful responses. + */ +function _processClientCache(req, res, next) { + const clientCacheMaxAge = req.body.max_time || req.query.max_time + const isValidClientCacheTime = Number.isInteger(clientCacheMaxAge) + const now = Date.now() + const fetchTime = isValidClientCacheTime + ? clientCacheMaxAge + : _floorTimeToMinute(now) + + // Set fetch time for feed conseil requests + req.feedFetchAt = fetchTime + + const isImmutable = isValidClientCacheTime && clientCacheMaxAge < now + const cacheMaxAge = isImmutable + ? DEFAULT_CACHE_TTL + : _calcCacheTtl(fetchTime, now) + + // Set cache on successful request + _setCacheHeaderOnSuccess(res, cacheMaxAge) + + return next() + + function _floorTimeToMinute(currentTime) { + return Math.floor(currentTime / ONE_MINUTE_MILLIS) * ONE_MINUTE_MILLIS + } + + function _calcCacheTtl(dataGatheredAt, currentTime) { + return Math.floor( + (dataGatheredAt + ONE_MINUTE_MILLIS - currentTime) / MILLISECOND_MODIFIER + ) + } +} + +/** + * Set cache time for cloudfront if request is successful. + * res.end is called by res.send which is called by res.json. + */ +function _setCacheHeaderOnSuccess(res, maxAge) { + const responseEnd = res.end.bind(res) + + res.end = function (...args) { + if (res.statusCode === STATUS_CODE_SUCCESS) { + res.set('Cache-Control', `public, max-age=${maxAge}`) + } + + return responseEnd(...args) + } +} diff --git a/src/lib/router/readFeed.js b/src/lib/router/readFeed.js new file mode 100644 index 0000000..f86df1b --- /dev/null +++ b/src/lib/router/readFeed.js @@ -0,0 +1,26 @@ +'use strict' + +const conseil = require('conseil') + +const { getIpfsHash, paginateFeed, sortFeed } = require('utils') + +module.exports = async function readFeed(req, res) { + const isFeatured = req.path === '/featured' + const pageCursor = req.body.counter + const fetchTime = req.feedFetchAt + const rawFeed = await (isFeatured + ? conseil.getFeaturedArtisticUniverse(fetchTime) + : conseil.getArtisticUniverse(fetchTime)) + + const paginatedFeed = paginateFeed(sortFeed(rawFeed), pageCursor) + const feed = await Promise.all( + paginatedFeed.map(async (objkt) => { + objkt.token_info = await getIpfsHash(objkt.ipfsHash) + objkt.token_id = parseInt(objkt.objectId) + + return objkt + }) + ) + + return res.json({ result: feed }) +} diff --git a/src/lib/router/readHdaoFeed.js b/src/lib/router/readHdaoFeed.js new file mode 100644 index 0000000..9caa4c0 --- /dev/null +++ b/src/lib/router/readHdaoFeed.js @@ -0,0 +1,26 @@ +'use strict' + +const _ = require('lodash') +const conseil = require('conseil') + +const { getObjktById, paginateFeed } = require('utils') + +module.exports = async function readHdaoFeed(req, res) { + const rawFeed = await conseil.hDAOFeed() + const sortedFeed = _.orderBy(rawFeed, ['hDAO_balance'], ['desc']) + const paginatedFeed = paginateFeed(sortedFeed, 0) + + res.json({ + result: await Promise.all( + paginatedFeed.map(async (objkt) => await _mergeHdao(objkt)) + ), + }) +} + +async function _mergeHdao(objkt) { + const mergedObjkt = await getObjktById(objkt.token_id) + + mergedObjkt.hDAO_balance = objkt.hDAO_balance + + return mergedObjkt +} diff --git a/src/lib/router/readIssuer.js b/src/lib/router/readIssuer.js new file mode 100644 index 0000000..2b6c284 --- /dev/null +++ b/src/lib/router/readIssuer.js @@ -0,0 +1,86 @@ +'use strict' + +const conseil = require('conseil') +const _ = require('lodash') + +const { getIpfsHash, getObjktOwners, getRestrictedAddresses } = require('utils') +const { burnAddress } = require('config') + +module.exports = async function readIssuer(req, res) { + const issuerAddress = req.body.tz + const restrictedAddresses = await getRestrictedAddresses() + + if (restrictedAddresses.includes(issuerAddress)) { + return res.json({ result: [] }) + } + + const [collection, creations, hdao] = await Promise.all([ + conseil.getCollectionForAddress(issuerAddress), + conseil.getArtisticOutputForAddress(issuerAddress), + conseil.gethDaoBalanceForAddress(issuerAddress), + ]) + + const filteredCreations = await _filteredBurnedCreations(creations) + + const unsortedResults = await Promise.all( + [...collection, ...filteredCreations].map(async (objkt) => { + objkt.token_info = await getIpfsHash(objkt.ipfsHash) + objkt.token_id = parseInt(objkt.piece || objkt.objectId) + + return objkt + }) + ) + + const sortedResults = _sortResults(unsortedResults) + + return res.json({ + result: _.uniqBy(sortedResults, (objkt) => { + return objkt.token_id + }), + hdao: hdao, + }) +} + +async function _filteredBurnedCreations(creations) { + const validCreations = [] + + await Promise.all( + creations.map(async (c) => { + c.token_id = c.objectId + + const ownerData = await getObjktOwners(c) + + Object.assign(c, ownerData) + + const burnAddrCount = + c.owners[burnAddress] && parseInt(c.owners[burnAddress]) + const allIssuesBurned = burnAddrCount && burnAddrCount === c.total_amount + + if (!allIssuesBurned) { + delete c.owners + + validCreations.push(c) + } + }) + ) + + return validCreations +} + +function _sortResults(results) { + const unsortedCollection = [] + const unsortedCreations = [] + + results.map((r) => + r.piece ? unsortedCollection.push(r) : unsortedCreations.push(r) + ) + + return [ + ..._sort(unsortedCollection, 'piece'), + ..._sort(unsortedCreations, 'objectId'), + ] + + function _sort(arr, sortKey) { + return arr.sort((a, b) => parseInt(b[sortKey]) - parseInt(a[sortKey])) + } +} diff --git a/src/lib/router/readObjkt.js b/src/lib/router/readObjkt.js new file mode 100644 index 0000000..bfdb02e --- /dev/null +++ b/src/lib/router/readObjkt.js @@ -0,0 +1,14 @@ +'use strict' + +const { getObjktById, getRestrictedObjkts } = require('utils') + +module.exports = async function readObjkt(req, res) { + const objktId = req.body.objkt_id + const restrictedObjkts = await getRestrictedObjkts() + + if (restrictedObjkts.includes(objktId)) { + return res.json({ result: [] }) + } + + return res.json({ result: await getObjktById(objktId) }) +} diff --git a/src/lib/router/readRandomFeed.js b/src/lib/router/readRandomFeed.js new file mode 100644 index 0000000..7e02058 --- /dev/null +++ b/src/lib/router/readRandomFeed.js @@ -0,0 +1,23 @@ +'use strict' + +const conseil = require('conseil') +const _ = require('lodash') + +const { getIpfsHash, paginateFeed } = require('utils') + +module.exports = async function readRandomFeed(req, res) { + const rawFeed = await conseil.getArtisticUniverse(0) + const pageCursor = req.body.cursor + const feed = paginateFeed(_.shuffle(rawFeed), pageCursor) + + res.json({ + result: await Promise.all( + feed.map(async (objkt) => { + objkt.token_info = await getIpfsHash(objkt.ipfsHash) + objkt.token_id = parseInt(objkt.objectId) + + return objkt + }) + ), + }) +} diff --git a/src/lib/router/readRecommendCurate.js b/src/lib/router/readRecommendCurate.js new file mode 100644 index 0000000..4b4b692 --- /dev/null +++ b/src/lib/router/readRecommendCurate.js @@ -0,0 +1,7 @@ +'use strict' + +const conseil = require('conseil') + +module.exports = async function readRecommendCurate(req, res) { + return res.json({ amount: await conseil.getRecommendedCurateDefault() }) +} diff --git a/src/lib/utils.js b/src/lib/utils.js new file mode 100644 index 0000000..b23941d --- /dev/null +++ b/src/lib/utils.js @@ -0,0 +1,93 @@ +'use strict' + +const axios = require('axios') +const conseil = require('conseil') +const _ = require('lodash') + +const { feedItemsPerPage } = require('config') + +module.exports = { + getIpfsHash, + getObjktById, + getObjktOwners, + getRestrictedAddresses, + getRestrictedObjkts, + paginateFeed, + sortFeed, +} + +async function getIpfsHash(ipfsHash) { + return (await axios.get('https://cloudflare-ipfs.com/ipfs/' + ipfsHash)).data +} + +async function getObjktById(id) { + const objkt = await conseil.getObjectById(id) + + objkt.token_id = objkt.objectId + + const [objktOwners, ipfsHash] = await Promise.all([ + getObjktOwners(objkt), + getIpfsHash(objkt.ipfsHash), + ]) + + Object.assign(objkt, objktOwners, { + token_info: ipfsHash, + }) + + return objkt +} + +async function getObjktOwners(objkt) { + const owners = ( + await axios.get( + 'https://api.better-call.dev/v1/contract/mainnet/KT1RJ6PbjHpwc3M5rw5s2Nbmefwbuwbdxton/tokens/holders?token_id=' + + objkt.token_id + ) + ).data + + const ownerCountList = _.values(owners) + + let total = 0 + + if (ownerCountList.length) { + total = ownerCountList.reduce((acc, i) => { + const owned = parseInt(i) + + return owned > 0 ? acc + owned : acc + }, 0) + } + + return { + total_amount: total, + owners, + } +} + +async function getRestrictedAddresses() { + return ( + await axios.get( + 'https://raw.githubusercontent.com/hicetnunc2000/hicetnunc/main/filters/w.json' + ) + ).data +} + +async function getRestrictedObjkts() { + return ( + await axios.get( + 'https://raw.githubusercontent.com/hicetnunc2000/hicetnunc/main/filters/o.json' + ) + ).data +} + +function paginateFeed(feed, cursor) { + const pageCursor = cursor ? parseInt(cursor) : 0 + + return feed.slice( + pageCursor * feedItemsPerPage, + pageCursor * feedItemsPerPage + feedItemsPerPage + ) +} + +function sortFeed(feed) { + return _.sortBy(feed, (i) => parseInt(i.objectId)).reverse() +}