From ac88f2a4a12aa5500e97805940339309dd354b4a Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Tue, 23 Jul 2024 14:31:55 -0700 Subject: [PATCH 1/6] Rename "Processor" to "DataLayer" and relevant types The processor is a DAL, so it's better to be named appropriately. All the type names will be suffixed with "Data" to distinguish them from other models of the same kind of object (e.g. TransactionData vs EdgeTransaction). --- src/common/plugin/types.ts | 12 +- .../db/{Processor.ts => DataLayer.ts} | 74 ++-- ...essorTransaction.ts => TransactionData.ts} | 18 +- src/common/utxobased/db/Models/baselet.ts | 8 +- src/common/utxobased/db/types.ts | 22 +- src/common/utxobased/db/util/utxo.ts | 20 +- src/common/utxobased/engine/UtxoEngine.ts | 88 ++--- .../utxobased/engine/UtxoEngineState.ts | 227 +++++++------ .../engine/util/getOwnUtxosFromTx.ts | 20 +- src/common/utxobased/engine/utils.ts | 4 +- src/common/utxobased/info/dash.ts | 6 +- src/common/utxobased/keymanager/keymanager.ts | 8 +- .../{Processor.spec.ts => DataLayer.spec.ts} | 318 +++++++++--------- test/common/utxobased/engine/engine.spec.ts | 4 +- .../coins/bitcointransactiontest.spec.ts | 14 +- 15 files changed, 421 insertions(+), 422 deletions(-) rename src/common/utxobased/db/{Processor.ts => DataLayer.ts} (91%) rename src/common/utxobased/db/Models/{ProcessorTransaction.ts => TransactionData.ts} (87%) rename test/common/utxobased/db/{Processor.spec.ts => DataLayer.spec.ts} (67%) diff --git a/src/common/plugin/types.ts b/src/common/plugin/types.ts index a129bf06..ba10dc3e 100644 --- a/src/common/plugin/types.ts +++ b/src/common/plugin/types.ts @@ -22,7 +22,7 @@ import { } from 'edge-core-js/types' import * as wif from 'wif' -import { asIUTXO, IProcessorTransaction, IUTXO } from '../utxobased/db/types' +import { asUtxoData, TransactionData, UtxoData } from '../utxobased/db/types' import { UtxoInitOptions } from '../utxobased/engine/types' import { ScriptTemplates } from '../utxobased/info/scriptTemplates/types' import { UtxoPicker } from '../utxobased/keymanager/utxopicker' @@ -45,12 +45,12 @@ export interface AddressPath { } export interface TxOptions { - utxos?: IUTXO[] + utxos?: UtxoData[] subtractFee?: boolean CPFP?: string } export const asTxOptions = asObject({ - utxos: asOptional(asArray(asIUTXO)), + utxos: asOptional(asArray(asUtxoData)), subtractFee: asOptional(asBoolean), CPFP: asOptional(asString) }) @@ -148,13 +148,13 @@ export interface EngineInfo { * Some currencies require an additional blockbook payload * 'getTransactionSpecific' in order to provide all relevant transaction * data. If this function is defined, it should merge the unknown `specialTx` - * data from the Blockbook endpoint with the `IProcessorTransaction` object + * data from the Blockbook endpoint with the `TransactionData` object * which is stored to disk. **/ txSpecificHandling?: ( - tx: IProcessorTransaction, + tx: TransactionData, specialTx: unknown - ) => IProcessorTransaction + ) => TransactionData } export interface ServerConfig { diff --git a/src/common/utxobased/db/Processor.ts b/src/common/utxobased/db/DataLayer.ts similarity index 91% rename from src/common/utxobased/db/Processor.ts rename to src/common/utxobased/db/DataLayer.ts index f35d7719..b18dca10 100644 --- a/src/common/utxobased/db/Processor.ts +++ b/src/common/utxobased/db/DataLayer.ts @@ -9,23 +9,23 @@ import { AddressPath, ChangePath } from '../../plugin/types' import { makeBaselets } from './Baselets' import { addressPathToPrefix, TxIdByDate } from './Models/baselet' import { - IAddress, - IProcessorTransaction, - ITransactionInput, - ITransactionOutput, - IUTXO + AddressData, + TransactionData, + TransactionDataInput, + TransactionDataOutput, + UtxoData } from './types' const BASELET_DIR = 'tables' -interface ProcessorConfig { +interface DataLayerConfig { disklet: Disklet } /* Transaction table interfaces */ interface SaveTransactionArgs { - tx: IProcessorTransaction + tx: TransactionData scriptPubkeys?: string[] } @@ -51,7 +51,13 @@ interface DumpDataReturn { data: unknown } -export interface Processor { +/** + * The Data Access Layer for the UTXO-based wallet engine. + * + * It provides all the methods necessary to interact with underlying + * database/storage mechanism. + */ +export interface DataLayer { clearAll: () => Promise dumpData: () => Promise @@ -64,13 +70,13 @@ export interface Processor { Used to store UTXOs. Needs to be able to track UTXOs that are already used in a transaction, but not confirmed yet */ - saveUtxo: (utxo: IUTXO) => Promise + saveUtxo: (utxo: UtxoData) => Promise // remove either all UTXOs if the array is empty, or as selected from an array // of UTXO ids removeUtxos: (utxoIds: string[]) => Promise // fetch either all UTXOs if the array is empty or as selected from an array // of UTXO ids - fetchUtxos: (args: FetchUtxosArgs) => Promise> + fetchUtxos: (args: FetchUtxosArgs) => Promise> /* Transaction processing ********************** @@ -87,12 +93,12 @@ export interface Processor { Used to store transactions. Needs to be updated for confirmation heights, and detecting used script pubkeys */ - saveTransaction: (args: SaveTransactionArgs) => Promise + saveTransaction: (args: SaveTransactionArgs) => Promise numTransactions: () => number removeTransaction: (txId: string) => Promise fetchTransactions: ( args: FetchTransactionArgs - ) => Promise> + ) => Promise> /* Address processing ********************* @@ -105,17 +111,17 @@ export interface Processor { Used to store script pubkeys / addresses. Needs to be updated for 'used' flag and path */ - saveAddress: (args: IAddress) => Promise + saveAddress: (args: AddressData) => Promise // used to calculate total number of addresses numAddressesByFormatPath: (path: ChangePath) => number // get the last used address index for a specific format lastUsedIndexByFormatPath: (path: ChangePath) => Promise - fetchAddress: (args: AddressPath | string) => Promise + fetchAddress: (args: AddressPath | string) => Promise } -export async function makeProcessor( - config: ProcessorConfig -): Promise { +export async function makeDataLayer( + config: DataLayerConfig +): Promise { const disklet = navigateDisklet(config.disklet, BASELET_DIR) let memlet = makeMemlet(disklet) let baselets = await makeBaselets({ storage: memlet }).catch(async error => { @@ -126,11 +132,11 @@ export async function makeProcessor( /** * Calculates the transaction value supplied (negative) or received (positive). In order to calculate * a value, the `ourIns` and `ourOuts` of the object must be populated with indices. - * @param tx {IProcessorTransaction} A transaction object with `ourIns` and `ourOuts` populated + * @param tx {TransactionData} A transaction object with `ourIns` and `ourOuts` populated */ - const calculateTxAmount = (tx: IProcessorTransaction): string => { + const calculateTxAmount = (tx: TransactionData): string => { interface TxIndexMap { - [index: string]: ITransactionInput | ITransactionOutput + [index: string]: TransactionDataInput | TransactionDataOutput } let ourAmount = '0' let txIndexMap: TxIndexMap = {} @@ -152,7 +158,7 @@ export async function makeProcessor( return ourAmount } - const processor: Processor = { + const dataLayer: DataLayer = { async clearAll(): Promise { await memlet.onFlush.next().value await clearMemletCache() @@ -176,7 +182,7 @@ export async function makeProcessor( ) }, - async saveUtxo(utxo: IUTXO): Promise { + async saveUtxo(utxo: UtxoData): Promise { return await baselets.utxo(async tables => { const [utxoIds = []] = await tables.utxoIdsByScriptPubkey.query('', [ utxo.scriptPubkey @@ -203,7 +209,7 @@ export async function makeProcessor( // 2. Use the map to remove utxoIds from the utxoIdsByScriptPubkey table const utxos = (await tables.utxoById.query('', utxoIds)).filter( utxo => utxo != null - ) as IUTXO[] + ) as UtxoData[] const utxoIdsMap: { [scriptPubkey: string]: string[] } = {} for (const utxo of utxos) { utxoIdsMap[utxo.scriptPubkey] = [ @@ -234,7 +240,7 @@ export async function makeProcessor( }) }, - async fetchUtxos(args): Promise> { + async fetchUtxos(args): Promise> { const { scriptPubkey, utxoIds = [] } = args return await baselets.utxo(async tables => { if (scriptPubkey != null) { @@ -261,19 +267,17 @@ export async function makeProcessor( }) }, - async saveTransaction( - args: SaveTransactionArgs - ): Promise { + async saveTransaction(args: SaveTransactionArgs): Promise { const { scriptPubkeys = [], tx } = args return await baselets.tx(async tables => { // Check if the transaction already exists - const processorTx = await tables.txById + const transactionData = await tables.txById .query('', [tx.txid]) .then(transactions => transactions[0]) .catch(_ => undefined) // Use the existing transaction if it does exist. - const transaction = processorTx ?? tx + const transaction = transactionData ?? tx // Mark the used inputs with the provided script pubkey for (const scriptPubkey of scriptPubkeys) { @@ -325,7 +329,7 @@ export async function makeProcessor( await tables.txById.insert('', transaction.txid, transaction) // Save index entry only for first transaction insert - if (processorTx == null) { + if (transactionData == null) { await tables.txIdsByDate.insert('', { txid: tx.txid, date: tx.date @@ -346,10 +350,10 @@ export async function makeProcessor( async fetchTransactions( args: FetchTransactionArgs - ): Promise> { + ): Promise> { const { blockHeightMax, txId, options } = args let { blockHeight } = args - const txs: Array = [] + const txs: Array = [] await baselets.tx(async tables => { // Fetch transactions by id if (txId != null) { @@ -407,7 +411,7 @@ export async function makeProcessor( return txs }, - async saveAddress(address: IAddress): Promise { + async saveAddress(address: AddressData): Promise { await baselets.address(async tables => { // This variable is used to update the scriptPubkeyByPath table. // The path table must be written after the address table because the @@ -564,7 +568,7 @@ export async function makeProcessor( async fetchAddress( fetchAddressArg: AddressPath | string - ): Promise { + ): Promise { return await baselets.address(async tables => { if (typeof fetchAddressArg === 'string') { // if it is a string, it is a scriptPubkey @@ -594,5 +598,5 @@ export async function makeProcessor( }) } } - return processor + return dataLayer } diff --git a/src/common/utxobased/db/Models/ProcessorTransaction.ts b/src/common/utxobased/db/Models/TransactionData.ts similarity index 87% rename from src/common/utxobased/db/Models/ProcessorTransaction.ts rename to src/common/utxobased/db/Models/TransactionData.ts index 620545cc..cd2e7716 100644 --- a/src/common/utxobased/db/Models/ProcessorTransaction.ts +++ b/src/common/utxobased/db/Models/TransactionData.ts @@ -6,12 +6,10 @@ import { EdgeTransaction, JsonObject } from 'edge-core-js/types' import { PluginInfo } from '../../../plugin/types' import { UtxoTxOtherParams } from '../../engine/types' import { UtxoWalletTools } from '../../engine/UtxoWalletTools' -import { Processor } from '../Processor' -import { IProcessorTransaction } from '../types' +import { DataLayer } from '../DataLayer' +import { TransactionData } from '../types' -export const fromEdgeTransaction = ( - tx: EdgeTransaction -): IProcessorTransaction => { +export const fromEdgeTransaction = (tx: EdgeTransaction): TransactionData => { const otherParams = tx.otherParams as UtxoTxOtherParams if (otherParams == null) throw new Error('Invalid transaction data') if (otherParams.psbt == null) @@ -44,7 +42,7 @@ export const fromEdgeTransaction = ( inputs: inputs, outputs: outputs, // We can leave ourIns/ourOuts blank because they'll be updated by the - // processor when receiving the transaction from blockbook. + // DataLayer when receiving the transaction from blockbook. // We may want to calculate these preemptively, but for now this will work. ourIns: [], ourOuts: [], @@ -54,21 +52,21 @@ export const fromEdgeTransaction = ( interface ToEdgeTransactionArgs { walletId: string - tx: IProcessorTransaction + tx: TransactionData walletTools: UtxoWalletTools - processor: Processor + dataLayer: DataLayer pluginInfo: PluginInfo } export const toEdgeTransaction = async ( args: ToEdgeTransactionArgs ): Promise => { - const { tx, processor, walletTools, pluginInfo, walletId } = args + const { tx, dataLayer, walletTools, pluginInfo, walletId } = args const { currencyInfo } = pluginInfo const ourReceiveAddresses: string[] = [] for (const out of tx.ourOuts) { const { scriptPubkey } = tx.outputs[parseInt(out)] - const address = await processor.fetchAddress(scriptPubkey) + const address = await dataLayer.fetchAddress(scriptPubkey) if (address?.path != null) { const { address: addrStr } = walletTools.scriptPubkeyToAddress({ diff --git a/src/common/utxobased/db/Models/baselet.ts b/src/common/utxobased/db/Models/baselet.ts index 6b317575..23965806 100644 --- a/src/common/utxobased/db/Models/baselet.ts +++ b/src/common/utxobased/db/Models/baselet.ts @@ -8,7 +8,7 @@ import { } from 'baselet' import { ChangePath } from '../../../plugin/types' -import { IAddress, IProcessorTransaction, IUTXO } from '../types' +import { AddressData, TransactionData, UtxoData } from '../types' export const addressPathToPrefix = (path: ChangePath): string => `${path.format}_${path.changeIndex}` @@ -19,7 +19,7 @@ export const scriptPubkeyByPathOptions: CountBaseOptions = { bucketSize: 50 } -export type AddressByScriptPubkeyBaselet = HashBase +export type AddressByScriptPubkeyBaselet = HashBase export const addressByScriptPubkeyOptions: HashBaseOptions = { name: 'addressByScriptPubkey', prefixSize: 4 @@ -50,7 +50,7 @@ export const txIdsByBlockHeightOptions: RangeBaseOptions< idKey: 'txid' } -export type TxByIdBaselet = HashBase +export type TxByIdBaselet = HashBase export const txByIdOptions: HashBaseOptions = { name: 'txById', prefixSize: 2 @@ -68,7 +68,7 @@ export const txIdsByDateOptions: RangeBaseOptions<'date', 'txid'> = { idKey: 'txid' } -export type UtxoByIdBaselet = HashBase +export type UtxoByIdBaselet = HashBase export const utxoByIdOptions: HashBaseOptions = { name: 'utxoById', prefixSize: 2 diff --git a/src/common/utxobased/db/types.ts b/src/common/utxobased/db/types.ts index 7d385475..45af26e2 100644 --- a/src/common/utxobased/db/types.ts +++ b/src/common/utxobased/db/types.ts @@ -4,7 +4,7 @@ import { SoftPick } from '../../../util/typeUtil' import { AddressPath } from '../../plugin/types' import { asScriptTypeEnum, ScriptTypeEnum } from '../keymanager/keymanager' -export interface IAddress { +export interface AddressData { scriptPubkey: string redeemScript?: string lastQueriedBlockHeight: number @@ -15,9 +15,9 @@ export interface IAddress { balance?: string } -export const makeIAddress = ( - addressFields: SoftPick -): IAddress => { +export const makeAddressData = ( + addressFields: SoftPick +): AddressData => { const { scriptPubkey, used = false, ...rest } = addressFields return { @@ -30,7 +30,7 @@ export const makeIAddress = ( } } -export interface IUTXO { +export interface UtxoData { id: string txid: string vout: number @@ -42,7 +42,7 @@ export interface IUTXO { blockHeight: number spent: boolean } -export const asIUTXO = asObject({ +export const asUtxoData = asObject({ id: asString, txid: asString, vout: asNumber, @@ -55,27 +55,27 @@ export const asIUTXO = asObject({ spent: asBoolean }) -export interface IProcessorTransaction { +export interface TransactionData { txid: string hex: string blockHeight: number confirmations?: 'confirmed' | 'unconfirmed' | 'dropped' | number date: number fees: string - inputs: ITransactionInput[] - outputs: ITransactionOutput[] + inputs: TransactionDataInput[] + outputs: TransactionDataOutput[] ourIns: string[] ourOuts: string[] ourAmount: string } -export interface ITransactionOutput { +export interface TransactionDataOutput { amount: string n: number scriptPubkey: string } -export interface ITransactionInput { +export interface TransactionDataInput { amount: string n: number outputIndex: number diff --git a/src/common/utxobased/db/util/utxo.ts b/src/common/utxobased/db/util/utxo.ts index 07dc07b6..611deef7 100644 --- a/src/common/utxobased/db/util/utxo.ts +++ b/src/common/utxobased/db/util/utxo.ts @@ -3,17 +3,17 @@ import { BIP43PurposeTypeEnum, ScriptTypeEnum } from '../../keymanager/keymanager' -import { Processor } from '../Processor' -import { IProcessorTransaction, IUTXO } from '../types' +import { DataLayer } from '../DataLayer' +import { TransactionData, UtxoData } from '../types' -export const utxoFromProcessorTransactionInput = async ( - processor: Processor, - processorTransaction: IProcessorTransaction, +export const utxoFromTransactionDataInput = async ( + dataLayer: DataLayer, + transactionData: TransactionData, inputIndex: number -): Promise => { - const input = processorTransaction.inputs[inputIndex] +): Promise => { + const input = transactionData.inputs[inputIndex] const { scriptPubkey } = input - const address = await processor.fetchAddress(scriptPubkey) + const address = await dataLayer.fetchAddress(scriptPubkey) if (address == null) throw new Error(`Cannot find address for ${scriptPubkey}`) @@ -34,7 +34,7 @@ export const utxoFromProcessorTransactionInput = async ( case BIP43PurposeTypeEnum.Airbitz: case BIP43PurposeTypeEnum.Legacy: { return { - script: processorTransaction.hex, + script: transactionData.hex, scriptType: redeemScript != null ? ScriptTypeEnum.p2sh : ScriptTypeEnum.p2pkh } @@ -64,7 +64,7 @@ export const utxoFromProcessorTransactionInput = async ( script, redeemScript, scriptType, - blockHeight: processorTransaction.blockHeight, + blockHeight: transactionData.blockHeight, spent: true } } diff --git a/src/common/utxobased/engine/UtxoEngine.ts b/src/common/utxobased/engine/UtxoEngine.ts index 94475215..5b00ac48 100644 --- a/src/common/utxobased/engine/UtxoEngine.ts +++ b/src/common/utxobased/engine/UtxoEngine.ts @@ -25,13 +25,13 @@ import { EngineEmitter, EngineEvent } from '../../plugin/EngineEmitter' import { makeMetadata } from '../../plugin/Metadata' import { EngineConfig, TxOptions } from '../../plugin/types' import { upgradeMemos } from '../../upgradeMemos' +import { makeDataLayer } from '../db/DataLayer' import { fromEdgeTransaction, toEdgeTransaction -} from '../db/Models/ProcessorTransaction' -import { makeProcessor } from '../db/Processor' -import { IProcessorTransaction, IUTXO } from '../db/types' -import { utxoFromProcessorTransactionInput } from '../db/util/utxo' +} from '../db/Models/TransactionData' +import { TransactionData, UtxoData } from '../db/types' +import { utxoFromTransactionDataInput } from '../db/util/utxo' import { asPrivateKey, asSafeWalletInfo, @@ -125,14 +125,14 @@ export async function makeUtxoEngine( emitter, log }) - const processor = await makeProcessor({ + const dataLayer = await makeDataLayer({ disklet: walletLocalDisklet }) const engineState = makeUtxoEngineState({ ...config, walletTools, walletInfo, - processor, + dataLayer, pluginState }) @@ -145,9 +145,9 @@ export async function makeUtxoEngine( if (privateKeyFormat === 'bip32' || privateKeyFormat === 'bip44') return null - // Get the replaced transaction from the processor: + // Get the replaced transaction from the DataLayer: const replacedTxid = edgeTx.txid - const [replacedTx] = await processor.fetchTransactions({ + const [replacedTx] = await dataLayer.fetchTransactions({ txId: replacedTxid }) @@ -165,11 +165,11 @@ export async function makeUtxoEngine( const newFeeRate = Math.round((parseInt(replacedTx.fees) / vBytes) * 2) const replacedTxInputs = replacedTx.inputs - // Recreate UTXOs from processor transaction and mark them as unspent: + // Recreate UTXOs from DataLayer transaction and mark them as unspent: const replacedTxUtxos = await Promise.all( replacedTxInputs.map(async (_, index) => { - const utxo = await utxoFromProcessorTransactionInput( - processor, + const utxo = await utxoFromTransactionDataInput( + dataLayer, replacedTx, index ) @@ -189,7 +189,7 @@ export async function makeUtxoEngine( let foundChangeAddress: string | undefined for (const output of replacedTx.outputs) { // Fetch address by output's scriptPubkey to determine output ownership - const ourAddress = await processor.fetchAddress(output.scriptPubkey) + const ourAddress = await dataLayer.fetchAddress(output.scriptPubkey) // This isn't our output, so include it as a target if (ourAddress == null) { @@ -242,7 +242,7 @@ export async function makeUtxoEngine( // Get all current UTXOs: const currentUtxos = filterUndefined( - await processor.fetchUtxos({ + await dataLayer.fetchUtxos({ utxoIds: [] }) ) @@ -305,7 +305,7 @@ export async function makeUtxoEngine( let nativeAmount = '0' for (const output of newTx.outputs) { const scriptPubkey = output.scriptPubkey.toString('hex') - const own = await processor.fetchAddress(scriptPubkey) + const own = await dataLayer.fetchAddress(scriptPubkey) if (own == null) { // Not our output nativeAmount = bs.sub(nativeAmount, output.value.toString()) @@ -432,7 +432,7 @@ export async function makeUtxoEngine( walletFormats }, metadataState: metadata.state, - processorState: await processor.dumpData(), + storageState: await dataLayer.dumpData(), pluginState: pluginState.dumpData() } } @@ -465,7 +465,7 @@ export async function makeUtxoEngine( }, getNumTransactions(_opts: EdgeTokenIdOptions): number { - return processor.numTransactions() + return dataLayer.numTransactions() }, async getPaymentProtocolInfo( @@ -482,16 +482,16 @@ export async function makeUtxoEngine( options: EdgeGetTransactionsOptions ): Promise { const txs = filterUndefined( - await processor.fetchTransactions({ options }) + await dataLayer.fetchTransactions({ options }) ) return await Promise.all( txs.map( - async (tx: IProcessorTransaction) => + async (tx: TransactionData) => await toEdgeTransaction({ walletId: walletInfo.id, tx, walletTools, - processor, + dataLayer, pluginInfo }) ) @@ -500,7 +500,7 @@ export async function makeUtxoEngine( async isAddressUsed(address: string): Promise { const scriptPubkey = walletTools.addressToScriptPubkey(address) - const addressData = await processor.fetchAddress(scriptPubkey) + const addressData = await dataLayer.fetchAddress(scriptPubkey) if (addressData == null) throw new Error('Address not found in wallet') return addressData.used }, @@ -534,10 +534,10 @@ export async function makeUtxoEngine( const utxos = txOptions.utxos ?? filterUndefined( - (await processor.fetchUtxos({ + (await dataLayer.fetchUtxos({ scriptPubkey: utxoScriptPubkey, utxoIds: [] - })) as IUTXO[] + })) as UtxoData[] ) if ( @@ -575,7 +575,7 @@ export async function makeUtxoEngine( const scriptPubkey = walletTools.addressToScriptPubkey( target.publicAddress ) - if ((await processor.fetchAddress(scriptPubkey)) != null) { + if ((await dataLayer.fetchAddress(scriptPubkey)) != null) { ourReceiveAddresses.push(target.publicAddress) } } @@ -601,15 +601,15 @@ export async function makeUtxoEngine( const feeRate = parseInt(await fees.getRate(edgeSpendInfo)) - let maxUtxo: undefined | IUTXO + let maxUtxo: undefined | UtxoData if (txOptions.CPFP != null) { - const [childTx] = await processor.fetchTransactions({ + const [childTx] = await dataLayer.fetchTransactions({ txId: txOptions.CPFP }) if (childTx == null) throw new Error('transaction not found') - const utxos: IUTXO[] = [] + const utxos: UtxoData[] = [] for (const txid of childTx.ourOuts) { - const [output] = await processor.fetchUtxos({ utxoIds: [txid] }) + const [output] = await dataLayer.fetchUtxos({ utxoIds: [txid] }) if (output != null) utxos.push(output) } maxUtxo = utxos.reduce((a, b) => (bs.gt(a.value, b.value) ? a : b)) @@ -651,7 +651,7 @@ export async function makeUtxoEngine( ) for (const output of tx.outputs) { const scriptPubkey = output.scriptPubkey.toString('hex') - const own = await processor.fetchAddress(scriptPubkey) + const own = await dataLayer.fetchAddress(scriptPubkey) if (own == null) { // Not our output nativeAmount = bs.sub(nativeAmount, output.value.toString()) @@ -702,7 +702,7 @@ export async function makeUtxoEngine( await engine.killEngine() // Clear cache and state - await processor.clearAll() + await dataLayer.clearAll() await pluginState.clearCache() await metadata.clear() await fees.clearCache() @@ -719,7 +719,7 @@ export async function makeUtxoEngine( const replacedTxid: string | undefined = edgeTx.otherParams?.replacedTxid if (replacedTxid != null) { // Get the replaced transaction using the replacedTxid - const [rbfTx] = await processor.fetchTransactions({ + const [rbfTx] = await dataLayer.fetchTransactions({ txId: replacedTxid }) if (rbfTx != null) { @@ -730,7 +730,7 @@ export async function makeUtxoEngine( pluginInfo, emitter, walletTools, - processor + dataLayer }) } } @@ -742,9 +742,9 @@ export async function makeUtxoEngine( pluginInfo, emitter, walletTools, - processor + dataLayer }) - await processor.saveTransaction({ + await dataLayer.saveTransaction({ tx, scriptPubkeys: edgeTx.otherParams?.ourScriptPubkeys }) @@ -752,7 +752,7 @@ export async function makeUtxoEngine( /* Get the wallet's UTXOs from the new transaction and save them to the processsor. */ - const ownUtxos = await getOwnUtxosFromTx(engineInfo, processor, tx) + const ownUtxos = await getOwnUtxosFromTx(engineInfo, dataLayer, tx) await engineState.processUtxos(ownUtxos) }, @@ -764,9 +764,9 @@ export async function makeUtxoEngine( const otherParams = asUtxoSignMessageOtherParams(opts.otherParams) const { publicAddress } = otherParams const scriptPubkey = walletTools.addressToScriptPubkey(publicAddress) - const processorAddress = await processor.fetchAddress(scriptPubkey) - if (processorAddress?.path == null) { - throw new Error('Missing address to sign with') + const addressData = await dataLayer.fetchAddress(scriptPubkey) + if (addressData?.path == null) { + throw new Error('Missing data-layer address to sign with') } const privateKey = asMaybeCurrencyPrivateKey(privateKeys) @@ -781,7 +781,7 @@ export async function makeUtxoEngine( }) const signature = await walletTools.signMessageBase64({ - path: processorAddress?.path, + path: addressData?.path, message, xprivKeys }) @@ -824,7 +824,7 @@ export async function makeUtxoEngine( psbt.inputs.map(async ({ hash, index: vout }) => { const txId = Buffer.from(hash).reverse().toString('hex') - const [transaction] = await processor.fetchTransactions({ txId }) + const [transaction] = await dataLayer.fetchTransactions({ txId }) if (transaction == null) throw new Error( 'Unable to find previous transaction data for input' @@ -838,7 +838,7 @@ export async function makeUtxoEngine( // Use the scriptPubkey to find the private key from the address // derivation path - const address = await processor.fetchAddress(scriptPubkey) + const address = await dataLayer.fetchAddress(scriptPubkey) if (address == null) { throw new Error( `Address for scriptPubkey '${scriptPubkey}' not found` @@ -931,7 +931,7 @@ export async function makeUtxoEngine( log } const tmpMetadata = await makeMetadata(tmpConfig) - const tmpProcessor = await makeProcessor(tmpConfig) + const tmpDataLayer = await makeDataLayer(tmpConfig) const tmpWalletTools = makeUtxoWalletTools({ pluginInfo, publicKey: tmpWalletInfo.keys.publicKey @@ -943,9 +943,9 @@ export async function makeUtxoEngine( try { await tmpState.stop() - const tmpUtxos = (await tmpProcessor.fetchUtxos({ + const tmpUtxos = (await tmpDataLayer.fetchUtxos({ utxoIds: [] - })) as IUTXO[] + })) as UtxoData[] if (tmpUtxos === null || tmpUtxos.length < 1) { throw new Error('Private key has no funds') } @@ -983,7 +983,7 @@ export async function makeUtxoEngine( gapLimit: 0 } }, - processor: tmpProcessor, + dataLayer: tmpDataLayer, walletTools: tmpWalletTools, walletInfo: tmpWalletInfo }) diff --git a/src/common/utxobased/engine/UtxoEngineState.ts b/src/common/utxobased/engine/UtxoEngineState.ts index 93443e7c..6cfc6f66 100644 --- a/src/common/utxobased/engine/UtxoEngineState.ts +++ b/src/common/utxobased/engine/UtxoEngineState.ts @@ -18,13 +18,13 @@ import { PluginInfo } from '../../plugin/types' import { removeItem } from '../../plugin/utils' -import { toEdgeTransaction } from '../db/Models/ProcessorTransaction' -import { Processor } from '../db/Processor' +import { DataLayer } from '../db/DataLayer' +import { toEdgeTransaction } from '../db/Models/TransactionData' import { - IAddress, - IProcessorTransaction, - IUTXO, - makeIAddress + AddressData, + makeAddressData, + TransactionData, + UtxoData } from '../db/types' import { getSupportedFormats, SafeWalletInfo } from '../keymanager/cleaners' import { @@ -78,13 +78,13 @@ export interface UtxoEngineState { loadWifs: (wifs: string[]) => Promise - processUtxos: (utxos: IUTXO[]) => Promise + processUtxos: (utxos: UtxoData[]) => Promise } export interface UtxoEngineStateConfig extends EngineConfig { walletTools: UtxoWalletTools walletInfo: SafeWalletInfo - processor: Processor + dataLayer: DataLayer } export function makeUtxoEngineState( @@ -96,7 +96,7 @@ export function makeUtxoEngineState( options, pluginState, pluginInfo, - processor, + dataLayer, walletInfo, walletTools } = config @@ -110,7 +110,7 @@ export function makeUtxoEngineState( addressTransactionCache: {}, addressUtxoCache: {}, rawUtxoCache: {}, - processorUtxoCache: {}, + dataLayerUtxoCache: {}, updateTransactionCache: {}, updateTransactionSpecificCache: {} } @@ -129,8 +129,8 @@ export function makeUtxoEngineState( for (const key of Object.keys(taskCache.rawUtxoCache)) { removeItem(taskCache.rawUtxoCache, key) } - for (const key of Object.keys(taskCache.processorUtxoCache)) { - removeItem(taskCache.processorUtxoCache, key) + for (const key of Object.keys(taskCache.dataLayerUtxoCache)) { + removeItem(taskCache.dataLayerUtxoCache, key) } for (const key of Object.keys(taskCache.updateTransactionCache)) { removeItem(taskCache.updateTransactionCache, key) @@ -188,7 +188,7 @@ export function makeUtxoEngineState( pluginInfo, walletInfo, walletTools, - processor, + dataLayer, emitter, taskCache, updateProgressRatio, @@ -216,7 +216,7 @@ export function makeUtxoEngineState( emitter.on( EngineEvent.BLOCK_HEIGHT_CHANGED, async (_uri: string, _blockHeight: number): Promise => { - const txs = await processor.fetchTransactions({ + const txs = await dataLayer.fetchTransactions({ blockHeight: 0 }) for (const tx of txs) { @@ -253,7 +253,7 @@ export function makeUtxoEngineState( ) // Initialize the addressSubscribeCache with the existing addresses already - // processed by the processor. This happens only once before any call to + // processed by the DataLayer. This happens only once before any call to // setLookAhead. const initializeAddressSubscriptions = async (): Promise => { const addressBalanceChanges: Array<{ @@ -269,10 +269,10 @@ export function makeUtxoEngineState( format, changeIndex: branch } - const branchAddressCount = processor.numAddressesByFormatPath( + const branchAddressCount = dataLayer.numAddressesByFormatPath( changePath ) - // If the processor has not processed any addresses then the loop + // If the DataLayer has not processed any addresses then the loop // condition will only iterate once when branchAddressCount is 0 for the // first address in the derivation path. for ( @@ -280,25 +280,25 @@ export function makeUtxoEngineState( addressIndex < branchAddressCount; addressIndex++ ) { - const processorAddress = await processor.fetchAddress({ + const addressData = await dataLayer.fetchAddress({ format, changeIndex: branch, addressIndex }) - if (processorAddress == null) { + if (addressData == null) { throw new Error( - `Missing processor address ${format} ${branch} ${addressIndex} during initialization` + `Missing data-layer address with '${format}/${branch}/${addressIndex}' path during initialization` ) } const { address } = walletTools.scriptPubkeyToAddress({ changePath, - scriptPubkey: processorAddress.scriptPubkey + scriptPubkey: addressData.scriptPubkey }) addressesToSubscribe.add(address) - if (processorAddress.balance != null) { + if (addressData.balance != null) { addressBalanceChanges.push({ - scriptPubkey: processorAddress.scriptPubkey, - balance: processorAddress.balance + scriptPubkey: addressData.scriptPubkey, + balance: addressData.balance }) } } @@ -397,7 +397,7 @@ export function makeUtxoEngineState( const { address } = await internalDeriveScriptAddress({ walletTools: commonArgs.walletTools, engineInfo: commonArgs.pluginInfo.engineInfo, - processor: commonArgs.processor, + dataLayer: commonArgs.dataLayer, taskCache: commonArgs.taskCache, format: walletInfo.keys.privateKeyFormat, script @@ -410,8 +410,8 @@ export function makeUtxoEngineState( async addGapLimitAddresses(addresses: string[]): Promise { const promises = addresses.map(async address => { const scriptPubkey = walletTools.addressToScriptPubkey(address) - await processor.saveAddress( - makeIAddress({ + await dataLayer.saveAddress( + makeAddressData({ scriptPubkey, used: true }) @@ -457,8 +457,8 @@ export function makeUtxoEngineState( }) // Make a new IAddress and save it - await processor.saveAddress( - makeIAddress({ scriptPubkey, redeemScript, path }) + await dataLayer.saveAddress( + makeAddressData({ scriptPubkey, redeemScript, path }) ) taskCache.addressSubscribeCache[address] = { @@ -468,9 +468,8 @@ export function makeUtxoEngineState( } } }, - async processUtxos(utxos: IUTXO[]) { - // Map of scriptPubkey to IUTXOs - const utxoMap: Map = new Map() + async processUtxos(utxos: UtxoData[]) { + const utxoMap: Map = new Map() const utxoIds: Set = new Set() // Map updated utxos @@ -487,16 +486,16 @@ export function makeUtxoEngineState( // Process UTXO sets for each scriptPubkey in map for (const [scriptPubkey, utxos] of utxoMap.entries()) { // Get saved utxo set - const savedUtxos = await processor.fetchUtxos({ scriptPubkey }) + const savedUtxos = await dataLayer.fetchUtxos({ scriptPubkey }) // Filter UTXOs to de-duplicate (and undefined utxo as a type assert) const filteredUtxos = savedUtxos.filter( utxo => utxo != null && !utxoIds.has(utxo?.id) - ) as IUTXO[] + ) as UtxoData[] // Add updated utxos to utxo set const combinedUtxos = [...filteredUtxos, ...utxos] - await processProcessorUtxos({ + await processDataLayerUtxos({ ...commonArgs, scriptPubkey, utxos: combinedUtxos @@ -510,7 +509,7 @@ interface CommonArgs { pluginInfo: PluginInfo walletInfo: SafeWalletInfo walletTools: UtxoWalletTools - processor: Processor + dataLayer: DataLayer emitter: EngineEmitter taskCache: TaskCache updateProgressRatio: () => void @@ -527,7 +526,7 @@ interface TaskCache { readonly addressSubscribeCache: AddressSubscribeCache readonly addressUtxoCache: AddressUtxoCache readonly rawUtxoCache: RawUtxoCache - readonly processorUtxoCache: ProcessorUtxoCache + readonly dataLayerUtxoCache: DataLayerUtxoCache readonly addressTransactionCache: AddressTransactionCache readonly updateTransactionCache: UpdateTransactionCache readonly updateTransactionSpecificCache: UpdateTransactionSpecificCache @@ -545,11 +544,11 @@ interface AddressSubscribeCache { interface AddressUtxoCache { [key: string]: { processing: boolean; path: ChangePath } } -interface ProcessorUtxoCache { +interface DataLayerUtxoCache { [key: string]: { processing: boolean full: boolean - utxos: IUTXO[] + utxos: UtxoData[] path: ChangePath } } @@ -558,7 +557,7 @@ interface RawUtxoCache { blockbookUtxo: BlockbookAccountUtxo processing: boolean path: ChangePath - address: IAddress + address: AddressData requiredCount: number } } @@ -577,7 +576,7 @@ const setLookAhead = async (common: CommonArgs): Promise => { const { pluginInfo: { engineInfo }, lock, - processor, + dataLayer, walletFormats, walletTools } = common @@ -599,8 +598,8 @@ const setLookAhead = async (common: CommonArgs): Promise => { async function deriveKeys(changePath: ChangePath): Promise { const addressesToSubscribe = new Set() - const totalAddressCount = processor.numAddressesByFormatPath(changePath) - let lastUsedIndex = await processor.lastUsedIndexByFormatPath(changePath) + const totalAddressCount = dataLayer.numAddressesByFormatPath(changePath) + let lastUsedIndex = await dataLayer.lastUsedIndexByFormatPath(changePath) // Loop until the total address count equals the lookahead count let lookAheadIndex = lastUsedIndex + engineInfo.gapLimit @@ -618,17 +617,17 @@ const setLookAhead = async (common: CommonArgs): Promise => { }) // Make a new IAddress and save it - await processor.saveAddress( - makeIAddress({ scriptPubkey, redeemScript, path }) + await dataLayer.saveAddress( + makeAddressData({ scriptPubkey, redeemScript, path }) ) // Add the displayAddress to the set of addresses to subscribe to after loop addressesToSubscribe.add(address) // Update the state for the loop - lastUsedIndex = await processor.lastUsedIndexByFormatPath(changePath) + lastUsedIndex = await dataLayer.lastUsedIndexByFormatPath(changePath) lookAheadIndex = lastUsedIndex + engineInfo.gapLimit - nextAddressIndex = processor.numAddressesByFormatPath(changePath) + nextAddressIndex = dataLayer.numAddressesByFormatPath(changePath) } // Add all the addresses to the subscribe cache for registering subscriptions later @@ -660,13 +659,13 @@ const addToAddressTransactionCache = async ( blockHeight: number, addressTransactionCache: AddressTransactionCache ): Promise => { - const { walletTools, processor } = common + const { walletTools, dataLayer } = common // Fetch the blockHeight for the address from the database const scriptPubkey = walletTools.addressToScriptPubkey(address) if (blockHeight === 0) { const { lastQueriedBlockHeight = 0 } = - (await processor.fetchAddress(scriptPubkey)) ?? {} + (await dataLayer.fetchAddress(scriptPubkey)) ?? {} blockHeight = lastQueriedBlockHeight } @@ -680,23 +679,23 @@ const addToAddressTransactionCache = async ( interface TransactionChangedArgs { walletId: string - tx: IProcessorTransaction + tx: TransactionData emitter: EngineEmitter walletTools: UtxoWalletTools pluginInfo: PluginInfo - processor: Processor + dataLayer: DataLayer } export const transactionChanged = async ( args: TransactionChangedArgs ): Promise => { - const { emitter, walletTools, processor, pluginInfo, tx, walletId } = args + const { emitter, walletTools, dataLayer, pluginInfo, tx, walletId } = args emitter.emit(EngineEvent.TRANSACTIONS_CHANGED, [ await toEdgeTransaction({ walletId, tx, walletTools, - processor, + dataLayer, pluginInfo }) ]) @@ -712,7 +711,7 @@ export const pickNextTask = async ( addressSubscribeCache, addressUtxoCache, rawUtxoCache, - processorUtxoCache, + dataLayerUtxoCache, addressTransactionCache, updateTransactionCache, updateTransactionSpecificCache @@ -735,17 +734,17 @@ export const pickNextTask = async ( } // Loop processed utxos, these are just database ops, triggers setLookAhead - if (Object.keys(processorUtxoCache).length > 0) { - for (const [scriptPubkey, state] of Object.entries(processorUtxoCache)) { + if (Object.keys(dataLayerUtxoCache).length > 0) { + for (const [scriptPubkey, state] of Object.entries(dataLayerUtxoCache)) { // Only process when all utxos for a specific address have been gathered if (!state.processing && state.full) { state.processing = true - await processProcessorUtxos({ + await processDataLayerUtxos({ ...args, scriptPubkey, utxos: state.utxos }) - removeItem(processorUtxoCache, scriptPubkey) + removeItem(dataLayerUtxoCache, scriptPubkey) return true } } @@ -969,7 +968,7 @@ const updateTransactionsSpecific = ( emitter, walletTools, pluginInfo, - processor, + dataLayer, serverUri, txId, taskCache @@ -978,7 +977,7 @@ const updateTransactionsSpecific = ( deferred.promise .then(async (txResponse: unknown) => { // Grab tx to update it - const txs = await processor.fetchTransactions({ txId }) + const txs = await dataLayer.fetchTransactions({ txId }) let tx = txs[0] if (tx == null) return @@ -988,7 +987,7 @@ const updateTransactionsSpecific = ( } // Process and save new tx - const processedTx = await processor.saveTransaction({ + const processedTx = await dataLayer.saveTransaction({ tx }) @@ -996,7 +995,7 @@ const updateTransactionsSpecific = ( walletId: walletInfo.id, emitter, walletTools, - processor, + dataLayer, pluginInfo, tx: processedTx }) @@ -1030,7 +1029,7 @@ const updateTransactions = ( txId, needsTxSpecific, pluginInfo, - processor, + dataLayer, serverUri, taskCache } = args @@ -1041,22 +1040,22 @@ const updateTransactions = ( if (txResponse.blockHeight < 1) return // Create new tx from raw tx const tx = processTransactionResponse({ ...args, txResponse }) - // Remove any existing input utxos from the processor + // Remove any existing input utxos from the dataLayer for (const input of tx.inputs) { - await processor.removeUtxos([`${input.txId}_${input.outputIndex}`]) + await dataLayer.removeUtxos([`${input.txId}_${input.outputIndex}`]) } - // Update output utxos's blockHeight any existing input utxos from the processor + // Update output utxos's blockHeight any existing input utxos from the dataLayer const utxoIds = tx.outputs.map(output => `${tx.txid}_${output.n}`) - const utxos = await processor.fetchUtxos({ + const utxos = await dataLayer.fetchUtxos({ utxoIds }) for (const utxo of utxos) { if (utxo == null) continue utxo.blockHeight = tx.blockHeight - await processor.saveUtxo(utxo) + await dataLayer.saveUtxo(utxo) } // Process and save new tx - const processedTx = await processor.saveTransaction({ + const processedTx = await dataLayer.saveTransaction({ tx }) @@ -1064,7 +1063,7 @@ const updateTransactions = ( walletId: walletInfo.id, emitter, walletTools, - processor, + dataLayer, pluginInfo, tx: processedTx }) @@ -1088,7 +1087,7 @@ const updateTransactions = ( interface DeriveScriptAddressArgs { walletTools: UtxoWalletTools engineInfo: EngineInfo - processor: Processor + dataLayer: DataLayer format: CurrencyFormat taskCache: TaskCache script: string @@ -1103,7 +1102,7 @@ interface DeriveScriptAddressReturn { const internalDeriveScriptAddress = async ({ walletTools, engineInfo, - processor, + dataLayer, format, taskCache, script @@ -1122,13 +1121,13 @@ const internalDeriveScriptAddress = async ({ addressIndex: 0 } - // save the address to the processor and add it to the cache + // save the address to the dataLayer and add it to the cache const { address, scriptPubkey, redeemScript } = walletTools.getScriptAddress({ path, scriptTemplate }) - await processor.saveAddress( - makeIAddress({ scriptPubkey, redeemScript, path }) + await dataLayer.saveAddress( + makeAddressData({ scriptPubkey, redeemScript, path }) ) const addresses = new Set() addresses.add(address) @@ -1156,11 +1155,11 @@ const internalGetFreshAddress = async ( format, changeIndex: branch, walletTools, - processor, + dataLayer, forceIndex } = args - const numAddresses = processor.numAddressesByFormatPath({ + const numAddresses = dataLayer.numAddressesByFormatPath({ format, changeIndex: branch }) @@ -1178,7 +1177,7 @@ const internalGetFreshAddress = async ( path.addressIndex = forceIndex } - const iAddress = await processor.fetchAddress(path) + const iAddress = await dataLayer.fetchAddress(path) const nativeBalance = iAddress?.balance ?? '0' const { scriptPubkey } = iAddress ?? (await walletTools.getScriptPubkey(path)) @@ -1213,7 +1212,7 @@ const processAddressTransactions = async ( emitter, needsTxSpecific, pluginInfo, - processor, + dataLayer, walletTools, taskCache, pluginState, @@ -1223,7 +1222,7 @@ const processAddressTransactions = async ( const addressTransactionCache = taskCache.addressTransactionCache const scriptPubkey = walletTools.addressToScriptPubkey(address) - const addressData = await processor.fetchAddress(scriptPubkey) + const addressData = await dataLayer.fetchAddress(scriptPubkey) if (addressData == null) { throw new Error(`could not find address with script pubkey ${scriptPubkey}`) } @@ -1244,7 +1243,7 @@ const processAddressTransactions = async ( // Process and save the address's transactions for (const txResponse of transactions) { const tx = processTransactionResponse({ ...args, txResponse }) - const processedTx = await processor.saveTransaction({ + const processedTx = await dataLayer.saveTransaction({ tx, scriptPubkeys: [scriptPubkey] }) @@ -1252,7 +1251,7 @@ const processAddressTransactions = async ( walletId: walletInfo.id, emitter, walletTools, - processor, + dataLayer, pluginInfo, tx: processedTx }) @@ -1281,7 +1280,7 @@ const processAddressTransactions = async ( addressData.lastQueriedBlockHeight = blockHeight // Save/update the fully-processed address - await processor.saveAddress(addressData) + await dataLayer.saveAddress(addressData) // Update the progress now that the transactions for an address have processed await args.updateProgressRatio() @@ -1313,7 +1312,7 @@ interface processTransactionResponseArgs extends CommonArgs { const processTransactionResponse = ( args: processTransactionResponseArgs -): IProcessorTransaction => { +): TransactionData => { const { txResponse, pluginInfo: { coinInfo } @@ -1383,26 +1382,26 @@ const processAddressUtxos = async ( const { address, walletTools, - processor, + dataLayer, taskCache, path, pluginState, serverUri } = args - const { addressUtxoCache, rawUtxoCache, processorUtxoCache } = taskCache + const { addressUtxoCache, rawUtxoCache, dataLayerUtxoCache } = taskCache const queryTime = Date.now() const deferred = new Deferred() deferred.promise .then(async (utxos: AddressUtxosResponse) => { pluginState.serverScoreUp(serverUri, Date.now() - queryTime) const scriptPubkey = walletTools.addressToScriptPubkey(address) - const addressData = await processor.fetchAddress(scriptPubkey) + const addressData = await dataLayer.fetchAddress(scriptPubkey) if (addressData == null || addressData.path == null) { return } if (utxos.length === 0) { - addToProcessorUtxoCache(processorUtxoCache, path, scriptPubkey, 0) + addToDataLayerUtxoCache(dataLayerUtxoCache, path, scriptPubkey, 0) return } @@ -1429,28 +1428,28 @@ const processAddressUtxos = async ( return args.serverStates.utxoListQueryTask(serverUri, address, deferred) } -interface ProcessUtxoTransactionArgs extends CommonArgs { +interface ProcessDataLayerUtxosArgs extends CommonArgs { scriptPubkey: string - utxos: IUTXO[] + utxos: UtxoData[] } -const processProcessorUtxos = async ( - args: ProcessUtxoTransactionArgs +const processDataLayerUtxos = async ( + args: ProcessDataLayerUtxosArgs ): Promise => { const { utxos, - processor, + dataLayer, scriptPubkey, log, emitter, pluginInfo: { currencyInfo } } = args - const updatedUtxos: { [utxoId: string]: IUTXO } = Object.fromEntries( + const updatedUtxos: { [utxoId: string]: UtxoData } = Object.fromEntries( [...utxos].map(utxo => [utxo.id, utxo]) ) const utxoIdsToRemove: string[] = [] - const currentUtxos = await processor.fetchUtxos({ scriptPubkey }) + const currentUtxos = await dataLayer.fetchUtxos({ scriptPubkey }) // // Modify existing UTXO set @@ -1475,7 +1474,7 @@ const processProcessorUtxos = async ( } // Remove any spent UTXOs that have confirmations - await processor.removeUtxos(utxoIdsToRemove) + await dataLayer.removeUtxos(utxoIdsToRemove) // // Save updated UTXO set @@ -1488,7 +1487,7 @@ const processProcessorUtxos = async ( newBalance = add(utxo.value, newBalance) } // Save new UTXOs - await processor.saveUtxo(utxo) + await dataLayer.saveUtxo(utxo) } // @@ -1510,13 +1509,13 @@ const processProcessorUtxos = async ( ) // Update balances for address that have this scriptPubkey - const address = await processor.fetchAddress(scriptPubkey) + const address = await dataLayer.fetchAddress(scriptPubkey) if (address == null) { throw new Error('address not found when processing UTXO transactions') } - await processor.saveAddress({ + await dataLayer.saveAddress({ ...address, balance: newBalance, used: true @@ -1533,7 +1532,7 @@ const processProcessorUtxos = async ( } interface ProcessRawUtxoArgs extends CommonArgs { - address: IAddress + address: AddressData path: ChangePath id: string requiredCount: number @@ -1549,7 +1548,7 @@ const processRawUtxo = async ( id, address, pluginInfo, - processor, + dataLayer, path, taskCache, requiredCount, @@ -1557,7 +1556,7 @@ const processRawUtxo = async ( serverUri, log } = args - const { rawUtxoCache, processorUtxoCache } = taskCache + const { rawUtxoCache, dataLayerUtxoCache } = taskCache const purposeType = pathToPurposeType( path, pluginInfo.engineInfo.scriptTemplates @@ -1568,7 +1567,7 @@ const processRawUtxo = async ( // Function to call once we are finished const done = (): void => { - const processorUtxo: IUTXO = { + const utxoData: UtxoData = { id, txid: utxo.txid, vout: utxo.vout, @@ -1580,12 +1579,12 @@ const processRawUtxo = async ( blockHeight: utxo.height ?? -1, spent: false } - addToProcessorUtxoCache( - processorUtxoCache, + addToDataLayerUtxoCache( + dataLayerUtxoCache, path, address.scriptPubkey, requiredCount, - processorUtxo + utxoData ) } @@ -1598,7 +1597,7 @@ const processRawUtxo = async ( // Legacy UTXOs need the previous transaction hex as the script // If we do not currently have it, add it to the queue to fetch it { - const [tx] = await processor.fetchTransactions({ + const [tx] = await dataLayer.fetchTransactions({ txId: utxo.txid }) if (tx == null) { @@ -1659,20 +1658,20 @@ const processRawUtxo = async ( done() } -const addToProcessorUtxoCache = ( - processorUtxoCache: ProcessorUtxoCache, +const addToDataLayerUtxoCache = ( + dataLayerUtxoCache: DataLayerUtxoCache, path: ChangePath, scriptPubkey: string, requiredCount: number, - utxo?: IUTXO + utxo?: UtxoData ): void => { - const processorUtxos = processorUtxoCache[scriptPubkey] ?? { + const dataLayerUtxos = dataLayerUtxoCache[scriptPubkey] ?? { utxos: [], processing: false, path, full: false } - if (utxo != null) processorUtxos.utxos.push(utxo) - processorUtxoCache[scriptPubkey] = processorUtxos - processorUtxos.full = processorUtxos.utxos.length >= requiredCount + if (utxo != null) dataLayerUtxos.utxos.push(utxo) + dataLayerUtxoCache[scriptPubkey] = dataLayerUtxos + dataLayerUtxos.full = dataLayerUtxos.utxos.length >= requiredCount } diff --git a/src/common/utxobased/engine/util/getOwnUtxosFromTx.ts b/src/common/utxobased/engine/util/getOwnUtxosFromTx.ts index b94c5c16..b277651c 100644 --- a/src/common/utxobased/engine/util/getOwnUtxosFromTx.ts +++ b/src/common/utxobased/engine/util/getOwnUtxosFromTx.ts @@ -1,15 +1,15 @@ import { EngineInfo } from '../../../plugin/types' -import { Processor } from '../../db/Processor' -import { IProcessorTransaction, IUTXO } from '../../db/types' +import { DataLayer } from '../../db/DataLayer' +import { TransactionData, UtxoData } from '../../db/types' import { BIP43PurposeTypeEnum } from '../../keymanager/keymanager' import { getScriptTypeFromPurposeType, pathToPurposeType } from '../utils' export const getOwnUtxosFromTx = async ( engineInfo: EngineInfo, - processor: Processor, - tx: IProcessorTransaction -): Promise => { - const ownUtxos: IUTXO[] = [] + dataLayer: DataLayer, + tx: TransactionData +): Promise => { + const ownUtxos: UtxoData[] = [] // // Spent UTXOs (Inputs) @@ -18,12 +18,12 @@ export const getOwnUtxosFromTx = async ( const inputUtxoIds = tx.inputs.map( input => `${input.txId}_${input.outputIndex}` ) - const inputUtxos = await processor.fetchUtxos({ + const inputUtxos = await dataLayer.fetchUtxos({ utxoIds: inputUtxoIds }) for (const utxo of inputUtxos) { if (utxo == null) continue - // Must create a new IUTXO object when mutating processor objects because + // Must create a new UtxoData object when mutating DataLayer objects because // memlet may keep a reference in memory. ownUtxos.push({ ...utxo, spent: true }) } @@ -34,7 +34,7 @@ export const getOwnUtxosFromTx = async ( for (const output of tx.outputs) { const scriptPubkey = output.scriptPubkey - const address = await processor.fetchAddress(scriptPubkey) + const address = await dataLayer.fetchAddress(scriptPubkey) if (address != null) { const id = `${tx.txid}_${output.n}` const path = address.path @@ -50,7 +50,7 @@ export const getOwnUtxosFromTx = async ( const scriptType = getScriptTypeFromPurposeType(purposeType) const script = isAirbitzOrLegacy ? tx.hex : address.scriptPubkey - const utxo: IUTXO = { + const utxo: UtxoData = { id, txid: tx.txid, vout: output.n, diff --git a/src/common/utxobased/engine/utils.ts b/src/common/utxobased/engine/utils.ts index 2fc7f235..61363ef2 100644 --- a/src/common/utxobased/engine/utils.ts +++ b/src/common/utxobased/engine/utils.ts @@ -3,7 +3,7 @@ import { Disklet } from 'disklet' import { EdgeParsedUri } from 'edge-core-js/types' import { ChangePath, CurrencyFormat, EngineInfo } from '../../plugin/types' -import { IUTXO } from '../db/types' +import { UtxoData } from '../db/types' import { ScriptTemplates } from '../info/scriptTemplates/types' import { getSupportedFormats, PrivateKey } from '../keymanager/cleaners' import { @@ -269,7 +269,7 @@ export const parsePathname = (args: { return edgeParsedUri } -export const sumUtxos = (utxos: IUTXO[]): string => +export const sumUtxos = (utxos: UtxoData[]): string => utxos.reduce( (sum, { spent, value }) => (spent ? sum : bs.add(sum, value)), '0' diff --git a/src/common/utxobased/info/dash.ts b/src/common/utxobased/info/dash.ts index ef042997..0e390c7b 100644 --- a/src/common/utxobased/info/dash.ts +++ b/src/common/utxobased/info/dash.ts @@ -3,7 +3,7 @@ import { EdgeCurrencyInfo } from 'edge-core-js/types' import { CoinInfo, EngineInfo, PluginInfo } from '../../plugin/types' import { maximumFeeRateCalculator } from '../../plugin/util/maximumFeeRateCalculator' -import { IProcessorTransaction } from '../db/types' +import { TransactionData } from '../db/types' import { legacyMemoInfo, utxoCustomFeeTemplate, @@ -70,9 +70,9 @@ const engineInfo: EngineInfo = { maximumFeeRate: maximumFeeRateCalculator(currencyInfo, 25.8) }, txSpecificHandling: ( - tx: IProcessorTransaction, + tx: TransactionData, txSpecific: unknown - ): IProcessorTransaction => { + ): TransactionData => { const asDashTransactionSpecific = asObject({ instantlock: asOptional(asBoolean) }) diff --git a/src/common/utxobased/keymanager/keymanager.ts b/src/common/utxobased/keymanager/keymanager.ts index 605c37ef..eeec38c9 100644 --- a/src/common/utxobased/keymanager/keymanager.ts +++ b/src/common/utxobased/keymanager/keymanager.ts @@ -19,7 +19,7 @@ import { EdgeLog, EdgeMemo } from 'edge-core-js/types' import { indexAtProtected } from '../../../util/indexAtProtected' import { undefinedIfEmptyString } from '../../../util/undefinedIfEmptyString' import { ChangePath, CoinInfo, CoinPrefixes, FeeInfo } from '../../plugin/types' -import { IUTXO } from '../db/types' +import { UtxoData } from '../db/types' import { ScriptTemplate, ScriptTemplates } from '../info/scriptTemplates/types' import { sortInputs, sortOutputs } from './bip69' import { @@ -223,8 +223,8 @@ export interface TxOutput { } export interface MakeTxArgs { - forceUseUtxo: IUTXO[] - utxos: IUTXO[] + forceUseUtxo: UtxoData[] + utxos: UtxoData[] targets: MakeTxTarget[] memos: EdgeMemo[] feeRate: number @@ -961,7 +961,7 @@ export function makeTx(args: MakeTxArgs): MakeTxReturn { const uniqueUtxos = [ ...mergedArray .filter(utxo => !utxo.spent) - .reduce((map, obj) => map.set(obj.id, obj), new Map()) + .reduce((map, obj) => map.set(obj.id, obj), new Map()) .values() ].sort((a, b) => { if (a.blockHeight <= 0 && b.blockHeight > 0) return 1 diff --git a/test/common/utxobased/db/Processor.spec.ts b/test/common/utxobased/db/DataLayer.spec.ts similarity index 67% rename from test/common/utxobased/db/Processor.spec.ts rename to test/common/utxobased/db/DataLayer.spec.ts index 904b5c5c..f546849f 100644 --- a/test/common/utxobased/db/Processor.spec.ts +++ b/test/common/utxobased/db/DataLayer.spec.ts @@ -4,15 +4,15 @@ import { expect } from 'chai' import { makeMemoryDisklet } from 'disklet' import { - makeProcessor, - Processor -} from '../../../../src/common/utxobased/db/Processor' + DataLayer, + makeDataLayer +} from '../../../../src/common/utxobased/db/DataLayer' import { - IAddress, - IProcessorTransaction, - ITransactionInput, - ITransactionOutput, - IUTXO + AddressData, + TransactionData, + TransactionDataInput, + TransactionDataOutput, + UtxoData } from '../../../../src/common/utxobased/db/types' import { ScriptTypeEnum } from '../../../../src/common/utxobased/keymanager/keymanager' import { unixTime } from '../../../../src/util/unixTime' @@ -22,19 +22,19 @@ chai.should() interface Fixtures { assertNumAddressesWithPaths: (expectedNum: number) => void assertLastUsedByFormatPath: (toBe: number) => Promise - assertNumTransactions: (expectedNum: number, processor: Processor) => void - assertProcessorObjectNotUndefined: (object: T | undefined) => T - processor: Processor + assertNumTransactions: (expectedNum: number, dataLayer: DataLayer) => void + assertObjectNotUndefined: (object: T | undefined) => T + dataLayer: DataLayer } const makeFixtures = async (): Promise => { const storage = {} const disklet = makeMemoryDisklet(storage) - const processor = await makeProcessor({ disklet }) + const dataLayer = await makeDataLayer({ disklet }) return { assertNumAddressesWithPaths: expectedNum => { - const num = processor.numAddressesByFormatPath({ + const num = dataLayer.numAddressesByFormatPath({ format: 'bip44', changeIndex: 0 }) @@ -42,7 +42,7 @@ const makeFixtures = async (): Promise => { }, assertLastUsedByFormatPath: async toBe => { - const result = await processor.lastUsedIndexByFormatPath({ + const result = await dataLayer.lastUsedIndexByFormatPath({ format: 'bip44', changeIndex: 0 }) @@ -51,36 +51,36 @@ const makeFixtures = async (): Promise => { assertNumTransactions: ( expectedNum: number, - processor: Processor + dataLayer: DataLayer ): void => { - const num = processor.numTransactions() + const num = dataLayer.numTransactions() expect(num).eql(expectedNum) }, - assertProcessorObjectNotUndefined: (object: T | undefined): T => { + assertObjectNotUndefined: (object: T | undefined): T => { if (object == null) - throw new Error(`unable to retrieve from the processor`) + throw new Error(`unable to retrieve from the dataLayer`) return object }, - processor + dataLayer } } -describe('Processor address tests', () => { +describe('DataLayer address tests', () => { it('empty address baselets', async () => { const { assertNumAddressesWithPaths, assertLastUsedByFormatPath, - processor + dataLayer } = await makeFixtures() assertNumAddressesWithPaths(0) await assertLastUsedByFormatPath(-1) - expect(await processor.fetchAddress('doesnotexist')).to.be.undefined + expect(await dataLayer.fetchAddress('doesnotexist')).to.be.undefined expect( - await processor.fetchAddress({ + await dataLayer.fetchAddress({ format: 'bip44', changeIndex: 0, addressIndex: 0 @@ -92,12 +92,12 @@ describe('Processor address tests', () => { const { assertNumAddressesWithPaths, assertLastUsedByFormatPath, - assertProcessorObjectNotUndefined, - processor + assertObjectNotUndefined, + dataLayer } = await makeFixtures() // Insert an unused address without a path - const address1: IAddress = { + const address1: AddressData = { scriptPubkey: 'testscriptpubkey1', lastQueriedBlockHeight: 0, lastQuery: 0, @@ -105,17 +105,17 @@ describe('Processor address tests', () => { used: false, balance: '' } - await processor.saveAddress(address1) + await dataLayer.saveAddress(address1) // Assertions assertNumAddressesWithPaths(0) await assertLastUsedByFormatPath(-1) - const processorAddress1 = assertProcessorObjectNotUndefined( - await processor.fetchAddress(address1.scriptPubkey) + const addressData1 = assertObjectNotUndefined( + await dataLayer.fetchAddress(address1.scriptPubkey) ) - expect(processorAddress1?.scriptPubkey).to.eqls(address1.scriptPubkey) + expect(addressData1?.scriptPubkey).to.eqls(address1.scriptPubkey) // Insert an unused address with a path - const address2: IAddress = { + const address2: AddressData = { scriptPubkey: 'testscriptpubkey2', lastQueriedBlockHeight: 0, lastQuery: 0, @@ -124,17 +124,17 @@ describe('Processor address tests', () => { used: false, balance: '' } - await processor.saveAddress(address2) + await dataLayer.saveAddress(address2) // Assertions assertNumAddressesWithPaths(1) await assertLastUsedByFormatPath(-1) - const processorAddress2 = assertProcessorObjectNotUndefined( - await processor.fetchAddress(address2.scriptPubkey) + const addressData2 = assertObjectNotUndefined( + await dataLayer.fetchAddress(address2.scriptPubkey) ) - expect(processorAddress2?.scriptPubkey).to.eqls(address2.scriptPubkey) + expect(addressData2?.scriptPubkey).to.eqls(address2.scriptPubkey) // Insert a used address with a conflicting path - const address3: IAddress = { + const address3: AddressData = { scriptPubkey: 'testscriptpubkey3', lastQueriedBlockHeight: 0, lastQuery: 0, @@ -143,20 +143,18 @@ describe('Processor address tests', () => { used: true, balance: '' } - await processor + await dataLayer .saveAddress(address3) .should.be.rejectedWith( 'Attempted to save address with an existing path, but different script pubkey' ) // Assertions assertNumAddressesWithPaths(1) - const processorAddress3 = await processor.fetchAddress( - address3.scriptPubkey - ) - expect(processorAddress3).to.be.undefined + const addressData3 = await dataLayer.fetchAddress(address3.scriptPubkey) + expect(addressData3).to.be.undefined // Insert a used address with a path - const address4: IAddress = { + const address4: AddressData = { scriptPubkey: 'testscriptpubkey3', lastQueriedBlockHeight: 0, lastQuery: 0, @@ -165,19 +163,19 @@ describe('Processor address tests', () => { used: true, balance: '' } - await processor.saveAddress(address4) + await dataLayer.saveAddress(address4) // Assertions assertNumAddressesWithPaths(2) - const processorAddress4 = assertProcessorObjectNotUndefined( - await processor.fetchAddress(address4.scriptPubkey) + const addressData4 = assertObjectNotUndefined( + await dataLayer.fetchAddress(address4.scriptPubkey) ) - expect(processorAddress4?.scriptPubkey).to.eqls(address4.scriptPubkey) + expect(addressData4?.scriptPubkey).to.eqls(address4.scriptPubkey) await assertLastUsedByFormatPath(1) // check behavior of not found addresses in populated baselets: - expect(await processor.fetchAddress('doesnotexist')).to.be.undefined + expect(await dataLayer.fetchAddress('doesnotexist')).to.be.undefined expect( - await processor.fetchAddress({ + await dataLayer.fetchAddress({ format: 'bip32', changeIndex: 0, addressIndex: 0 @@ -189,12 +187,12 @@ describe('Processor address tests', () => { const { assertNumAddressesWithPaths, assertLastUsedByFormatPath, - assertProcessorObjectNotUndefined, - processor + assertObjectNotUndefined, + dataLayer } = await makeFixtures() // Insert an unused address without a path - let address: IAddress = { + let address: AddressData = { scriptPubkey: 'testscriptpubkey1', lastQueriedBlockHeight: 0, lastQuery: 0, @@ -202,14 +200,14 @@ describe('Processor address tests', () => { used: false, balance: '' } - await processor.saveAddress(address) + await dataLayer.saveAddress(address) // Assertions assertNumAddressesWithPaths(0) await assertLastUsedByFormatPath(-1) - const processorAddress1 = assertProcessorObjectNotUndefined( - await processor.fetchAddress(address.scriptPubkey) + const addressData1 = assertObjectNotUndefined( + await dataLayer.fetchAddress(address.scriptPubkey) ) - expect(processorAddress1?.scriptPubkey).to.eqls(address.scriptPubkey) + expect(addressData1?.scriptPubkey).to.eqls(address.scriptPubkey) // Update the address with a path address = { @@ -220,24 +218,24 @@ describe('Processor address tests', () => { addressIndex: 0 } } - await processor.saveAddress(address) + await dataLayer.saveAddress(address) // Assertions assertNumAddressesWithPaths(1) await assertLastUsedByFormatPath(-1) - const processorAddress2 = assertProcessorObjectNotUndefined( - await processor.fetchAddress(address.scriptPubkey) + const addressData2 = assertObjectNotUndefined( + await dataLayer.fetchAddress(address.scriptPubkey) ) - expect(processorAddress2.scriptPubkey).to.eqls(address.scriptPubkey) + expect(addressData2.scriptPubkey).to.eqls(address.scriptPubkey) // Update the used flag of an existing address with a path address = { ...address, used: true } - await processor.saveAddress(address) + await dataLayer.saveAddress(address) // Assertions assertNumAddressesWithPaths(1) - const processorAddress3 = assertProcessorObjectNotUndefined( - await processor.fetchAddress(address.scriptPubkey) + const addressData3 = assertObjectNotUndefined( + await dataLayer.fetchAddress(address.scriptPubkey) ) - expect(processorAddress3.scriptPubkey).to.eqls(address.scriptPubkey) + expect(addressData3.scriptPubkey).to.eqls(address.scriptPubkey) await assertLastUsedByFormatPath(0) // Update various address fields of an existing address with a path @@ -248,26 +246,26 @@ describe('Processor address tests', () => { lastTouched: 1, balance: '0' } - await processor.saveAddress(address) + await dataLayer.saveAddress(address) // Assertions assertNumAddressesWithPaths(1) - const processorAddress4 = assertProcessorObjectNotUndefined( - await processor.fetchAddress(address.scriptPubkey) + const addressData4 = assertObjectNotUndefined( + await dataLayer.fetchAddress(address.scriptPubkey) ) - expect(processorAddress4.scriptPubkey).to.eqls(address.scriptPubkey) - expect(processorAddress4.lastQueriedBlockHeight).to.eqls(1) - expect(processorAddress4.lastQuery).to.eqls(1) - expect(processorAddress4.lastTouched).to.eqls(1) - expect(processorAddress4.balance).to.eqls('0') + expect(addressData4.scriptPubkey).to.eqls(address.scriptPubkey) + expect(addressData4.lastQueriedBlockHeight).to.eqls(1) + expect(addressData4.lastQuery).to.eqls(1) + expect(addressData4.lastTouched).to.eqls(1) + expect(addressData4.balance).to.eqls('0') }) }) -describe('Processor utxo tests', () => { +describe('DataLayer utxo tests', () => { it('insert utxo to baselets', async () => { - const { processor } = await makeFixtures() + const { dataLayer } = await makeFixtures() // Insert a utxo - const utxo1: IUTXO = { + const utxo1: UtxoData = { id: 'utxo000001', txid: 'transaction1', vout: 0, @@ -278,18 +276,18 @@ describe('Processor utxo tests', () => { blockHeight: 0, spent: false } - await processor.saveUtxo(utxo1) + await dataLayer.saveUtxo(utxo1) // Fetch all - await processor.fetchUtxos({ utxoIds: [] }).then(utxos => { + await dataLayer.fetchUtxos({ utxoIds: [] }).then(utxos => { expect(utxos).to.eqls([utxo1]) }) // Fetch one - await processor.fetchUtxos({ utxoIds: ['utxo000001'] }).then(utxos => { + await dataLayer.fetchUtxos({ utxoIds: ['utxo000001'] }).then(utxos => { expect(utxos).to.eqls([utxo1]) }) // Insert a second utxo - const utxo2: IUTXO = { + const utxo2: UtxoData = { id: 'utxo000002', txid: 'transaction2', vout: 0, @@ -300,25 +298,25 @@ describe('Processor utxo tests', () => { blockHeight: 0, spent: false } - await processor.saveUtxo(utxo2) + await dataLayer.saveUtxo(utxo2) // Fetch all - await processor.fetchUtxos({ utxoIds: [] }).then(utxos => { + await dataLayer.fetchUtxos({ utxoIds: [] }).then(utxos => { expect(utxos).to.eqls([utxo1, utxo2]) }) // Fetch two - await processor + await dataLayer .fetchUtxos({ utxoIds: ['utxo000001', 'utxo000002'] }) .then(utxos => { expect(utxos).to.eqls([utxo1, utxo2]) }) // Fetch by scriptPubkey - await processor + await dataLayer .fetchUtxos({ scriptPubkey: utxo1.scriptPubkey }) .then(utxos => { expect(utxos).to.eqls([utxo1]) }) - await processor + await dataLayer .fetchUtxos({ scriptPubkey: utxo2.scriptPubkey }) .then(utxos => { expect(utxos).to.eqls([utxo2]) @@ -326,10 +324,10 @@ describe('Processor utxo tests', () => { }) it('update utxo in baselets', async () => { - const { processor } = await makeFixtures() + const { dataLayer } = await makeFixtures() // Insert a utxo - const utxoOriginal: IUTXO = { + const utxoOriginal: UtxoData = { id: 'utxo000001', txid: 'transaction1', vout: 0, @@ -340,18 +338,18 @@ describe('Processor utxo tests', () => { blockHeight: 0, spent: false } - await processor.saveUtxo(utxoOriginal) + await dataLayer.saveUtxo(utxoOriginal) // Fetch all - await processor.fetchUtxos({ utxoIds: [] }).then(utxos => { + await dataLayer.fetchUtxos({ utxoIds: [] }).then(utxos => { expect(utxos).to.eqls([utxoOriginal]) }) // Fetch one - await processor.fetchUtxos({ utxoIds: ['utxo000001'] }).then(utxos => { + await dataLayer.fetchUtxos({ utxoIds: ['utxo000001'] }).then(utxos => { expect(utxos).to.eqls([utxoOriginal]) }) // Insert a second utxo - const utxoUpdated: IUTXO = { + const utxoUpdated: UtxoData = { id: 'utxo000001', txid: 'transaction1', vout: 0, @@ -362,22 +360,22 @@ describe('Processor utxo tests', () => { blockHeight: 0, spent: false } - await processor.saveUtxo(utxoUpdated) + await dataLayer.saveUtxo(utxoUpdated) // Fetch all - await processor.fetchUtxos({ utxoIds: [] }).then(utxos => { + await dataLayer.fetchUtxos({ utxoIds: [] }).then(utxos => { expect(utxos).to.eqls([utxoUpdated]) }) // Fetch one - await processor.fetchUtxos({ utxoIds: ['utxo000001'] }).then(utxos => { + await dataLayer.fetchUtxos({ utxoIds: ['utxo000001'] }).then(utxos => { expect(utxos).to.eqls([utxoUpdated]) }) }) it('remove utxo in baselets', async () => { - const { processor } = await makeFixtures() + const { dataLayer } = await makeFixtures() // Insert a utxo - const utxo: IUTXO = { + const utxo: UtxoData = { id: 'utxo000001', txid: 'transaction1', vout: 0, @@ -388,34 +386,34 @@ describe('Processor utxo tests', () => { blockHeight: 0, spent: false } - await processor.saveUtxo(utxo) + await dataLayer.saveUtxo(utxo) // Fetch all - await processor.fetchUtxos({ utxoIds: [] }).then(utxos => { + await dataLayer.fetchUtxos({ utxoIds: [] }).then(utxos => { expect(utxos).to.eqls([utxo]) }) // Fetch all - await processor.fetchUtxos({ utxoIds: ['utxo000001'] }).then(utxos => { + await dataLayer.fetchUtxos({ utxoIds: ['utxo000001'] }).then(utxos => { expect(utxos).to.eqls([utxo]) }) // Fetch by scriptPubkey - await processor + await dataLayer .fetchUtxos({ scriptPubkey: utxo.scriptPubkey }) .then(utxos => { expect(utxos).to.eqls([utxo]) }) // Remove utxo - await processor.removeUtxos(['utxo000001']) + await dataLayer.removeUtxos(['utxo000001']) // Fetch all - await processor.fetchUtxos({ utxoIds: [] }).then(utxos => { + await dataLayer.fetchUtxos({ utxoIds: [] }).then(utxos => { expect(utxos).to.eqls([]) }) // Fetch one - await processor.fetchUtxos({ utxoIds: ['utxo000001'] }).then(utxos => { + await dataLayer.fetchUtxos({ utxoIds: ['utxo000001'] }).then(utxos => { expect(utxos).to.eqls([undefined]) }) // Fetch by scriptPubkey - await processor + await dataLayer .fetchUtxos({ scriptPubkey: utxo.scriptPubkey }) .then(utxos => { expect(utxos).to.eqls([]) @@ -423,10 +421,10 @@ describe('Processor utxo tests', () => { }) it('query all utxos in baselets', async () => { - const { processor } = await makeFixtures() + const { dataLayer } = await makeFixtures() // Insert a utxo - const utxos: IUTXO[] = [ + const utxos: UtxoData[] = [ { id: 'utxo000001', txid: 'transaction1', @@ -484,16 +482,16 @@ describe('Processor utxo tests', () => { } ] for (const utxo of utxos) { - await processor.saveUtxo(utxo) + await dataLayer.saveUtxo(utxo) } // Fetch all - await processor.fetchUtxos({ utxoIds: [] }).then(utxos => { + await dataLayer.fetchUtxos({ utxoIds: [] }).then(utxos => { expect(utxos).to.eqls(utxos) }) for (const utxo of utxos) { // Fetch by scriptPubkey - await processor + await dataLayer .fetchUtxos({ scriptPubkey: utxo.scriptPubkey }) .then(utxos => { expect(utxos).to.eqls([utxo]) @@ -501,12 +499,12 @@ describe('Processor utxo tests', () => { } }) }) -describe('Processor transactions tests', () => { +describe('DataLayer transactions tests', () => { function assertNumTransactions( expectedNum: number, - processor: Processor + dataLayer: DataLayer ): void { - const num = processor.numTransactions() + const num = dataLayer.numTransactions() expect(num).eql(expectedNum) } @@ -514,16 +512,16 @@ describe('Processor transactions tests', () => { const storage = {} const disklet = makeMemoryDisklet(storage) - const processor = await makeProcessor({ disklet }) - assertNumTransactions(0, processor) + const dataLayer = await makeDataLayer({ disklet }) + assertNumTransactions(0, dataLayer) }) it('insert a transaction to transaction baselets', async () => { const storage = {} const disklet = makeMemoryDisklet(storage) - const processor = await makeProcessor({ disklet }) + const dataLayer = await makeDataLayer({ disklet }) - const input1: ITransactionInput = { + const input1: TransactionDataInput = { txId: 'random', outputIndex: 0, scriptPubkey: 'pubkeyin1', @@ -531,17 +529,17 @@ describe('Processor transactions tests', () => { n: 0, amount: '1' } - const output1: ITransactionOutput = { + const output1: TransactionDataOutput = { amount: '1', n: 0, scriptPubkey: 'pubkeyout1' } - const output2: ITransactionOutput = { + const output2: TransactionDataOutput = { amount: '1', n: 1, scriptPubkey: 'pubkeyout2' } - const transaction1: IProcessorTransaction = { + const transaction1: TransactionData = { txid: 'transaction1', hex: '', blockHeight: 1, @@ -554,50 +552,50 @@ describe('Processor transactions tests', () => { ourAmount: '0' } - await processor.saveTransaction({ + await dataLayer.saveTransaction({ tx: transaction1, scriptPubkeys: [output1.scriptPubkey] }) - assertNumTransactions(1, processor) - const [tx1] = await processor.fetchTransactions({ txId: transaction1.txid }) + assertNumTransactions(1, dataLayer) + const [tx1] = await dataLayer.fetchTransactions({ txId: transaction1.txid }) expect(tx1?.ourOuts[0]).to.eqls('0') expect(tx1?.ourAmount).to.eqls('1') - const [txByBlockHeight1] = await processor.fetchTransactions({ + const [txByBlockHeight1] = await dataLayer.fetchTransactions({ blockHeight: 1 }) expect(txByBlockHeight1?.blockHeight).to.eqls(1) // insert the same transaction, but with a script pubkey referencing an input - await processor.saveTransaction({ + await dataLayer.saveTransaction({ tx: transaction1, scriptPubkeys: [input1.scriptPubkey] }) - const [tx2] = await processor.fetchTransactions({ txId: transaction1.txid }) + const [tx2] = await dataLayer.fetchTransactions({ txId: transaction1.txid }) expect(tx2?.ourOuts[0]).to.eqls('0') expect(tx2?.ourIns[0]).to.eqls('0') expect(tx2?.ourAmount).to.eqls('0') // insert the same transaction, but with a script pubkey referencing another output - await processor.saveTransaction({ + await dataLayer.saveTransaction({ tx: transaction1, scriptPubkeys: [output2.scriptPubkey] }) - const [tx3] = await processor.fetchTransactions({ txId: transaction1.txid }) + const [tx3] = await dataLayer.fetchTransactions({ txId: transaction1.txid }) expect(tx3?.ourOuts[1]).to.eqls('1') expect(tx3?.ourIns[0]).to.eqls('0') expect(tx3?.ourAmount).to.eqls('1') - const [tx4] = await processor.fetchTransactions({ + const [tx4] = await dataLayer.fetchTransactions({ options: { tokenId: null } }) expect(tx4).not.to.be.undefined - const results = await processor.fetchTransactions({ + const results = await dataLayer.fetchTransactions({ options: { tokenId: null, startDate: new Date(11_000), @@ -606,7 +604,7 @@ describe('Processor transactions tests', () => { }) expect(results).to.deep.equal([]) - const [tx6] = await processor.fetchTransactions({ + const [tx6] = await dataLayer.fetchTransactions({ options: { tokenId: null, startDate: new Date(9_000), @@ -619,9 +617,9 @@ describe('Processor transactions tests', () => { it('insert multiple transactions to baselets', async () => { const storage = {} const disklet = makeMemoryDisklet(storage) - const processor = await makeProcessor({ disklet }) + const dataLayer = await makeDataLayer({ disklet }) - const input1: ITransactionInput = { + const input1: TransactionDataInput = { txId: 'random', outputIndex: 0, scriptPubkey: 'pubkeyin1', @@ -629,17 +627,17 @@ describe('Processor transactions tests', () => { n: 0, amount: '1' } - const output1: ITransactionOutput = { + const output1: TransactionDataOutput = { amount: '1', n: 0, scriptPubkey: 'pubkeyout1' } - const output2: ITransactionOutput = { + const output2: TransactionDataOutput = { amount: '1', n: 1, scriptPubkey: 'pubkeyout2' } - const transaction1: IProcessorTransaction = { + const transaction1: TransactionData = { txid: 'transaction1', hex: '', blockHeight: 1, @@ -652,7 +650,7 @@ describe('Processor transactions tests', () => { ourAmount: '0' } - const transaction2: IProcessorTransaction = { + const transaction2: TransactionData = { txid: 'transaction2', hex: '', blockHeight: 1, @@ -665,53 +663,53 @@ describe('Processor transactions tests', () => { ourAmount: '0' } - await processor.saveTransaction({ + await dataLayer.saveTransaction({ tx: transaction1, scriptPubkeys: [output1.scriptPubkey] }) - await processor.saveTransaction({ + await dataLayer.saveTransaction({ tx: transaction2, scriptPubkeys: [output2.scriptPubkey] }) - assertNumTransactions(2, processor) - const [tx1] = await processor.fetchTransactions({ txId: transaction1.txid }) + assertNumTransactions(2, dataLayer) + const [tx1] = await dataLayer.fetchTransactions({ txId: transaction1.txid }) expect(tx1?.ourOuts[0]).to.eqls('0') expect(tx1?.ourAmount).to.eqls('1') - const txsByBlockHeight = await processor.fetchTransactions({ + const txsByBlockHeight = await dataLayer.fetchTransactions({ blockHeight: 1 }) expect(txsByBlockHeight[0]?.blockHeight).to.eqls(1) expect(txsByBlockHeight[1]?.blockHeight).to.eqls(1) - await processor.saveTransaction({ + await dataLayer.saveTransaction({ tx: transaction1, scriptPubkeys: [input1.scriptPubkey] }) - const [tx2] = await processor.fetchTransactions({ txId: transaction1.txid }) + const [tx2] = await dataLayer.fetchTransactions({ txId: transaction1.txid }) expect(tx2?.ourOuts[0]).to.eqls('0') expect(tx2?.ourIns[0]).to.eqls('0') expect(tx2?.ourAmount).to.eqls('0') - await processor.saveTransaction({ + await dataLayer.saveTransaction({ tx: transaction1, scriptPubkeys: [output2.scriptPubkey] }) - const [tx3] = await processor.fetchTransactions({ txId: transaction1.txid }) + const [tx3] = await dataLayer.fetchTransactions({ txId: transaction1.txid }) expect(tx3?.ourOuts[1]).to.eqls('1') expect(tx3?.ourIns[0]).to.eqls('0') expect(tx3?.ourAmount).to.eqls('1') - const [tx4] = await processor.fetchTransactions({ + const [tx4] = await dataLayer.fetchTransactions({ options: { tokenId: null } }) expect(tx4).not.to.be.undefined - const [tx5] = await processor.fetchTransactions({ + const [tx5] = await dataLayer.fetchTransactions({ options: { tokenId: null, startDate: new Date(11_000), @@ -720,7 +718,7 @@ describe('Processor transactions tests', () => { }) expect(tx5).not.to.be.undefined - const tx6 = await processor.fetchTransactions({ + const tx6 = await dataLayer.fetchTransactions({ options: { tokenId: null, startDate: new Date(9_000), @@ -733,9 +731,9 @@ describe('Processor transactions tests', () => { it('update transaction blockheight in transaction baselets', async () => { const storage = {} const disklet = makeMemoryDisklet(storage) - const processor = await makeProcessor({ disklet }) + const dataLayer = await makeDataLayer({ disklet }) - const input1: ITransactionInput = { + const input1: TransactionDataInput = { txId: 'random', outputIndex: 0, scriptPubkey: 'pubkeyin1', @@ -743,17 +741,17 @@ describe('Processor transactions tests', () => { n: 0, amount: '1' } - const output1: ITransactionOutput = { + const output1: TransactionDataOutput = { amount: '1', n: 0, scriptPubkey: 'pubkeyout1' } - const output2: ITransactionOutput = { + const output2: TransactionDataOutput = { amount: '1', n: 1, scriptPubkey: 'pubkeyout2' } - const transaction1: IProcessorTransaction = { + const transaction1: TransactionData = { txid: 'transaction1', hex: '', blockHeight: 1, @@ -766,7 +764,7 @@ describe('Processor transactions tests', () => { ourAmount: '0' } - const transaction2: IProcessorTransaction = { + const transaction2: TransactionData = { txid: 'transaction2', hex: '', blockHeight: 1, @@ -779,7 +777,7 @@ describe('Processor transactions tests', () => { ourAmount: '0' } - const transaction2updated: IProcessorTransaction = { + const transaction2updated: TransactionData = { txid: 'transaction2', hex: '', blockHeight: 10, @@ -792,40 +790,40 @@ describe('Processor transactions tests', () => { ourAmount: '0' } - await processor.saveTransaction({ + await dataLayer.saveTransaction({ tx: transaction1, scriptPubkeys: [output1.scriptPubkey] }) - await processor.saveTransaction({ + await dataLayer.saveTransaction({ tx: transaction2, scriptPubkeys: [output2.scriptPubkey] }) - const txsByBlockHeight = await processor.fetchTransactions({ + const txsByBlockHeight = await dataLayer.fetchTransactions({ blockHeight: 1 }) expect(txsByBlockHeight.length).to.be.eqls(2) expect(txsByBlockHeight[0]?.blockHeight).to.eqls(1) expect(txsByBlockHeight[1]?.blockHeight).to.eqls(1) - await processor.saveTransaction({ tx: transaction2updated }) + await dataLayer.saveTransaction({ tx: transaction2updated }) // should return for a single block height - const txsByBlockHeight1 = await processor.fetchTransactions({ + const txsByBlockHeight1 = await dataLayer.fetchTransactions({ blockHeight: 1 }) expect(txsByBlockHeight1.length).to.be.eqls(1) expect(txsByBlockHeight1[0]?.blockHeight).to.eqls(1) // should return between (including) a range of block heights - const txsByBlockHeight2 = await processor.fetchTransactions({ + const txsByBlockHeight2 = await dataLayer.fetchTransactions({ blockHeight: 1, blockHeightMax: 10 }) expect(txsByBlockHeight2.length).to.be.equals(2) // should return by a range of block heights from 0 to 10 - const txsByBlockHeight3 = await processor.fetchTransactions({ + const txsByBlockHeight3 = await dataLayer.fetchTransactions({ blockHeightMax: 10 }) expect(txsByBlockHeight3.length).to.be.equals(2) diff --git a/test/common/utxobased/engine/engine.spec.ts b/test/common/utxobased/engine/engine.spec.ts index 688bc671..8f5911a6 100644 --- a/test/common/utxobased/engine/engine.spec.ts +++ b/test/common/utxobased/engine/engine.spec.ts @@ -391,11 +391,11 @@ describe('engine.spec', function () { it('Should provide a non used BTC address when no options are provided', async function () { this.timeout(3000) const address = await engine.getFreshAddress({}) // TODO - const processor = await makeProcessor({ + const dataLayer = await makeDataLayer({ disklet: engineOpts.walletLocalDisklet }) - const txs = await processor.fetchTransactions({ blockHeight: 0 }) + const txs = await dataLayer.fetchTransactions({ blockHeight: 0 }) // $FlowFixMe const engineState: any = engine.engineState diff --git a/test/common/utxobased/keymanager/coins/bitcointransactiontest.spec.ts b/test/common/utxobased/keymanager/coins/bitcointransactiontest.spec.ts index b18b83f1..698d3d82 100644 --- a/test/common/utxobased/keymanager/coins/bitcointransactiontest.spec.ts +++ b/test/common/utxobased/keymanager/coins/bitcointransactiontest.spec.ts @@ -1,7 +1,7 @@ import { expect } from 'chai' import { describe, it } from 'mocha' -import { IUTXO } from '../../../../../src/common/utxobased/db/types' +import { UtxoData } from '../../../../../src/common/utxobased/db/types' import { info as bitcoin } from '../../../../../src/common/utxobased/info/bitcoin' import { addressToScriptPubkey, @@ -391,7 +391,7 @@ describe('bitcoin transaction creation and signing test', function () { }) it.skip('create a segwit tx with segwit outputs, then create another tx consuming these outputs', async () => { - const getUtxo = (i: number, value: string = '100'): IUTXO => ({ + const getUtxo = (i: number, value: string = '100'): UtxoData => ({ id: `${i}`, scriptType: ScriptTypeEnum.p2wpkh, txid: '8b26fa4d0238788ffc3a7d96e4169acf6fe993a28791e9e748819ac216ee85b3', @@ -433,7 +433,7 @@ describe('bitcoin transaction creation and signing test', function () { }) it.skip('create a mixed input and mixed output transaction', async () => { - const utxoLegacy: IUTXO = { + const utxoLegacy: UtxoData = { id: '0', scriptType: ScriptTypeEnum.p2pkh, txid: '7d067b4a697a09d2c3cff7d4d9506c9955e93bff41bf82d439da7d030382bc3e', @@ -451,7 +451,7 @@ describe('bitcoin transaction creation and signing test', function () { vout: 0 } - const utxoSegwit: IUTXO = { + const utxoSegwit: UtxoData = { id: '1', scriptType: ScriptTypeEnum.p2wpkh, txid: '8b26fa4d0238788ffc3a7d96e4169acf6fe993a28791e9e748819ac216ee85b3', @@ -463,7 +463,7 @@ describe('bitcoin transaction creation and signing test', function () { vout: 1 } - const utxoWrappedSegwit: IUTXO = { + const utxoWrappedSegwit: UtxoData = { id: '2', scriptType: ScriptTypeEnum.p2wpkhp2sh, txid: 'e9f28846381667b6beb57698ab824b597312428cd026d45e9e3a13c95e335d9e', @@ -514,7 +514,7 @@ describe('bitcoin transaction creation and signing test', function () { it.skip('create a legacy tx with one input and 100 outputs, then create another legacy tx with 100 inputs and two outputs', async () => { const nOutputs = 100 - const utxo: IUTXO = { + const utxo: UtxoData = { id: '0', scriptType: ScriptTypeEnum.p2pkh, txid: '7d067b4a697a09d2c3cff7d4d9506c9955e93bff41bf82d439da7d030382bc3e', @@ -562,7 +562,7 @@ describe('bitcoin transaction creation and signing test', function () { psbtBase64 }) - const utxos: IUTXO[] = Array(100) + const utxos: UtxoData[] = Array(100) .fill(1) .map((_, i) => ({ id: `${i}`, From ed4d0dd92ad651be30fc1c0fbbcac4d7b05a0d04 Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Tue, 23 Jul 2024 14:38:47 -0700 Subject: [PATCH 2/6] Rename UtxoEngineState to UtxoEngineProcessor This component of the plugin makes more sense to be name "processor" because it's the component that processes data from the network and storage for the engine. --- src/common/plugin/PluginState.ts | 16 +++++----- src/common/utxobased/engine/UtxoEngine.ts | 31 ++++++++++--------- ...oEngineState.ts => UtxoEngineProcessor.ts} | 10 +++--- test/common/utxobased/engine/engine.spec.ts | 6 ++-- 4 files changed, 33 insertions(+), 30 deletions(-) rename src/common/utxobased/engine/{UtxoEngineState.ts => UtxoEngineProcessor.ts} (99%) diff --git a/src/common/plugin/PluginState.ts b/src/common/plugin/PluginState.ts index 66e65cdb..54b06743 100644 --- a/src/common/plugin/PluginState.ts +++ b/src/common/plugin/PluginState.ts @@ -6,7 +6,7 @@ import { EdgeIo, EdgeLog } from 'edge-core-js/types' import { makeMemlet } from 'memlet' import { UtxoUserSettings } from '../utxobased/engine/types' -import { UtxoEngineState } from '../utxobased/engine/UtxoEngineState' +import { UtxoEngineProcessor } from '../utxobased/engine/UtxoEngineProcessor' import { asServerCache, ServerCache, @@ -42,8 +42,8 @@ export interface PluginStateSettings { } export interface PluginState { - addEngine: (engineState: UtxoEngineState) => void - removeEngine: (engineState: UtxoEngineState) => void + addEngine: (engineProcessor: UtxoEngineProcessor) => void + removeEngine: (engineProcessor: UtxoEngineProcessor) => void dumpData: () => JsonObject load: () => Promise serverScoreDown: (uri: string) => void @@ -67,7 +67,7 @@ export function makePluginState(settings: PluginStateSettings): PluginState { log } = settings - let engines: UtxoEngineState[] = [] + let engines: UtxoEngineProcessor[] = [] const memlet = makeMemlet(pluginDisklet) let serverCache: ServerCache = { @@ -142,15 +142,15 @@ export function makePluginState(settings: PluginStateSettings): PluginState { /** * Begins notifying the engine of state changes. Used at connection time. */ - addEngine(engineState: UtxoEngineState): void { - engines.push(engineState) + addEngine(engineProcessor: UtxoEngineProcessor): void { + engines.push(engineProcessor) }, /** * Stops notifying the engine of state changes. Used at disconnection time. */ - removeEngine(engineState: UtxoEngineState): void { - engines = engines.filter(engine => engine !== engineState) + removeEngine(engineProcessor: UtxoEngineProcessor): void { + engines = engines.filter(engine => engine !== engineProcessor) }, dumpData(): JsonObject { diff --git a/src/common/utxobased/engine/UtxoEngine.ts b/src/common/utxobased/engine/UtxoEngine.ts index 5b00ac48..7ab5c9f5 100644 --- a/src/common/utxobased/engine/UtxoEngine.ts +++ b/src/common/utxobased/engine/UtxoEngine.ts @@ -61,7 +61,10 @@ import { pathToPurposeType, sumUtxos } from './utils' -import { makeUtxoEngineState, transactionChanged } from './UtxoEngineState' +import { + makeUtxoEngineProcessor, + transactionChanged +} from './UtxoEngineProcessor' import { makeUtxoWalletTools } from './UtxoWalletTools' export async function makeUtxoEngine( @@ -128,7 +131,7 @@ export async function makeUtxoEngine( const dataLayer = await makeDataLayer({ disklet: walletLocalDisklet }) - const engineState = makeUtxoEngineState({ + const engineProcessor = makeUtxoEngineProcessor({ ...config, walletTools, walletInfo, @@ -235,7 +238,7 @@ export async function makeUtxoEngine( // Use the found change address or generate a new one: const freshAddress = foundChangeAddress == null - ? await engineState.getFreshAddress({ branch: 1 }) + ? await engineProcessor.getFreshAddress({ branch: 1 }) : { publicAddress: foundChangeAddress } const freshChangeAddress = freshAddress.segwitAddress ?? freshAddress.publicAddress @@ -357,15 +360,15 @@ export async function makeUtxoEngine( metadata.state.balance ) - pluginState.addEngine(engineState) + pluginState.addEngine(engineProcessor) await fees.start() - await engineState.start() + await engineProcessor.start() }, async killEngine(): Promise { - await engineState.stop() + await engineProcessor.stop() fees.stop() - pluginState.removeEngine(engineState) + pluginState.removeEngine(engineProcessor) }, getBalance(_opts: EdgeTokenIdOptions): string { @@ -381,7 +384,7 @@ export async function makeUtxoEngine( }, async addGapLimitAddresses(addresses: string[]): Promise { - return await engineState.addGapLimitAddresses(addresses) + return await engineProcessor.addGapLimitAddresses(addresses) }, async broadcastTx(transaction: EdgeTransaction): Promise { @@ -401,7 +404,7 @@ export async function makeUtxoEngine( ) } } - const id = await engineState.broadcastTx(transaction).catch(err => { + const id = await engineProcessor.broadcastTx(transaction).catch(err => { if (String(err).includes('Error: Blockbook Error: -26: dust')) { throw new DustSpendError() } @@ -461,7 +464,7 @@ export async function makeUtxoEngine( opts: EdgeGetReceiveAddressOptions ): Promise { const { forceIndex } = opts - return await engineState.getFreshAddress({ forceIndex }) + return await engineProcessor.getFreshAddress({ forceIndex }) }, getNumTransactions(_opts: EdgeTokenIdOptions): number { @@ -559,7 +562,7 @@ export async function makeUtxoEngine( const { script } = target.otherParams if (script.type === 'replayProtection') { // construct a replay protection p2sh address - const { publicAddress } = await engineState.deriveScriptAddress( + const { publicAddress } = await engineProcessor.deriveScriptAddress( script.type ) targets.push({ @@ -593,7 +596,7 @@ export async function makeUtxoEngine( throw new Error('Need to provide Spend Targets') } - const freshAddress = await engineState.getFreshAddress({ branch: 1 }) + const freshAddress = await engineProcessor.getFreshAddress({ branch: 1 }) const freshChangeAddress = forceChangeAddress ?? freshAddress.segwitAddress ?? @@ -753,7 +756,7 @@ export async function makeUtxoEngine( Get the wallet's UTXOs from the new transaction and save them to the processsor. */ const ownUtxos = await getOwnUtxosFromTx(engineInfo, dataLayer, tx) - await engineState.processUtxos(ownUtxos) + await engineProcessor.processUtxos(ownUtxos) }, async signMessage( @@ -967,7 +970,7 @@ export async function makeUtxoEngine( } }) - const tmpState = makeUtxoEngineState({ + const tmpState = makeUtxoEngineProcessor({ ...config, options: { ...config.options, diff --git a/src/common/utxobased/engine/UtxoEngineState.ts b/src/common/utxobased/engine/UtxoEngineProcessor.ts similarity index 99% rename from src/common/utxobased/engine/UtxoEngineState.ts rename to src/common/utxobased/engine/UtxoEngineProcessor.ts index 6cfc6f66..b00a4ad8 100644 --- a/src/common/utxobased/engine/UtxoEngineState.ts +++ b/src/common/utxobased/engine/UtxoEngineProcessor.ts @@ -52,7 +52,7 @@ import { } from './utils' import { UtxoWalletTools } from './UtxoWalletTools' -export interface UtxoEngineState { +export interface UtxoEngineProcessor { processedPercent: number start: () => Promise @@ -81,15 +81,15 @@ export interface UtxoEngineState { processUtxos: (utxos: UtxoData[]) => Promise } -export interface UtxoEngineStateConfig extends EngineConfig { +export interface UtxoEngineProcessorConfig extends EngineConfig { walletTools: UtxoWalletTools walletInfo: SafeWalletInfo dataLayer: DataLayer } -export function makeUtxoEngineState( - config: UtxoEngineStateConfig -): UtxoEngineState { +export function makeUtxoEngineProcessor( + config: UtxoEngineProcessorConfig +): UtxoEngineProcessor { const { initOptions, io, diff --git a/test/common/utxobased/engine/engine.spec.ts b/test/common/utxobased/engine/engine.spec.ts index 8f5911a6..89a76655 100644 --- a/test/common/utxobased/engine/engine.spec.ts +++ b/test/common/utxobased/engine/engine.spec.ts @@ -398,9 +398,9 @@ describe('engine.spec', function () { const txs = await dataLayer.fetchTransactions({ blockHeight: 0 }) // $FlowFixMe - const engineState: any = engine.engineState - const scriptHash = engineState.scriptHashes[address.publicAddress] - const transactions = engineState.addressInfos[scriptHash].txids + const engineProcessor: any = engine.engineProcessor + const scriptHash = engineProcessor.scriptHashes[address.publicAddress] + const transactions = engineProcessor.addressInfos[scriptHash].txids assert(transactions.length === 0, 'Should have never received coins') }) }) From 2191507a83e471505c6cf7ea00811101bd2017f7 Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Tue, 23 Jul 2024 15:56:21 -0700 Subject: [PATCH 3/6] Remove `FormatArgs` type from UtxoEngineProcessor --- .../utxobased/engine/UtxoEngineProcessor.ts | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/src/common/utxobased/engine/UtxoEngineProcessor.ts b/src/common/utxobased/engine/UtxoEngineProcessor.ts index b00a4ad8..04dde809 100644 --- a/src/common/utxobased/engine/UtxoEngineProcessor.ts +++ b/src/common/utxobased/engine/UtxoEngineProcessor.ts @@ -350,9 +350,11 @@ export function makeUtxoEngineProcessor( legacyAddress } = await internalGetFreshAddress({ ...commonArgs, - format: privateKeyFormat, forceIndex, - changeIndex: branch + changePath: { + format: privateKeyFormat, + changeIndex: branch + } }) const freshAddress: EdgeFreshAddress = { @@ -380,9 +382,11 @@ export function makeUtxoEngineProcessor( nativeBalance = '0' } = await internalGetFreshAddress({ ...commonArgs, - format, forceIndex, - changeIndex: branch + changePath: { + format, + changeIndex: branch + } }) freshAddress.segwitAddress = address @@ -570,8 +574,6 @@ interface AddressTransactionCache { } } -interface FormatArgs extends CommonArgs, ChangePath {} - const setLookAhead = async (common: CommonArgs): Promise => { const { pluginInfo: { engineInfo }, @@ -1138,8 +1140,9 @@ const internalDeriveScriptAddress = async ({ return { address, scriptPubkey, redeemScript } } -interface GetFreshAddressArgs extends FormatArgs { +interface GetFreshAddressArgs extends CommonArgs { forceIndex?: number + changePath: ChangePath } interface GetFreshAddressReturn { @@ -1151,22 +1154,13 @@ interface GetFreshAddressReturn { const internalGetFreshAddress = async ( args: GetFreshAddressArgs ): Promise => { - const { - format, - changeIndex: branch, - walletTools, - dataLayer, - forceIndex - } = args + const { changePath, walletTools, dataLayer, forceIndex } = args - const numAddresses = dataLayer.numAddressesByFormatPath({ - format, - changeIndex: branch - }) + const numAddresses = dataLayer.numAddressesByFormatPath(changePath) const path: AddressPath = { - format, - changeIndex: branch, + format: changePath.format, + changeIndex: changePath.changeIndex, // while syncing, we may hit negative numbers when only subtracting. Use the address at /0 in that case. addressIndex: Math.max( numAddresses - args.pluginInfo.engineInfo.gapLimit, From 2b39a6969a2ce1744ee9ae5fdfd69a0805c9ef11 Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Tue, 23 Jul 2024 16:49:44 -0700 Subject: [PATCH 4/6] Restructure function parameters in UtxoEngineProcessor --- .../utxobased/engine/UtxoEngineProcessor.ts | 567 ++++++++---------- 1 file changed, 246 insertions(+), 321 deletions(-) diff --git a/src/common/utxobased/engine/UtxoEngineProcessor.ts b/src/common/utxobased/engine/UtxoEngineProcessor.ts index 04dde809..ebb78447 100644 --- a/src/common/utxobased/engine/UtxoEngineProcessor.ts +++ b/src/common/utxobased/engine/UtxoEngineProcessor.ts @@ -184,7 +184,7 @@ export function makeUtxoEngineProcessor( pluginState, walletInfo }) - const commonArgs: CommonArgs = { + const common: CommonParams = { pluginInfo, walletInfo, walletTools, @@ -201,7 +201,7 @@ export function makeUtxoEngineProcessor( } serverStates.setPickNextTaskCB(async serverUri => { - return await pickNextTask(commonArgs, serverUri) + return await pickNextTask(common, serverUri) }) let running = false @@ -210,7 +210,7 @@ export function makeUtxoEngineProcessor( running = true await initializeAddressSubscriptions() - await setLookAhead(commonArgs) + await setLookAhead(common) } emitter.on( @@ -236,16 +236,15 @@ export function makeUtxoEngineProcessor( processing: false, path } - addToAddressTransactionCache( - commonArgs, - response.address, - path, - 0, - taskCache.addressTransactionCache - ).catch(() => { + addToAddressTransactionCache(common, { + address: response.address, + changePath: path, + blockHeight: 0, + addressTransactionCache: taskCache.addressTransactionCache + }).catch(() => { throw new Error('failed to add to transaction cache') }) - setLookAhead(commonArgs).catch(e => { + setLookAhead(common).catch(e => { log(e) }) } @@ -302,7 +301,7 @@ export function makeUtxoEngineProcessor( }) } } - addToAddressSubscribeCache(commonArgs.taskCache, addressesToSubscribe, { + addToAddressSubscribeCache(common.taskCache, addressesToSubscribe, { format, changeIndex: branch }) @@ -348,8 +347,7 @@ export function makeUtxoEngineProcessor( address: publicAddress, nativeBalance = '0', legacyAddress - } = await internalGetFreshAddress({ - ...commonArgs, + } = await internalGetFreshAddress(common, { forceIndex, changePath: { format: privateKeyFormat, @@ -380,8 +378,7 @@ export function makeUtxoEngineProcessor( const { address, nativeBalance = '0' - } = await internalGetFreshAddress({ - ...commonArgs, + } = await internalGetFreshAddress(common, { forceIndex, changePath: { format, @@ -399,10 +396,10 @@ export function makeUtxoEngineProcessor( async deriveScriptAddress(script): Promise { const { address } = await internalDeriveScriptAddress({ - walletTools: commonArgs.walletTools, - engineInfo: commonArgs.pluginInfo.engineInfo, - dataLayer: commonArgs.dataLayer, - taskCache: commonArgs.taskCache, + walletTools: common.walletTools, + engineInfo: common.pluginInfo.engineInfo, + dataLayer: common.dataLayer, + taskCache: common.taskCache, format: walletInfo.keys.privateKeyFormat, script }) @@ -499,8 +496,7 @@ export function makeUtxoEngineProcessor( // Add updated utxos to utxo set const combinedUtxos = [...filteredUtxos, ...utxos] - await processDataLayerUtxos({ - ...commonArgs, + await processDataLayerUtxos(common, { scriptPubkey, utxos: combinedUtxos }) @@ -509,7 +505,7 @@ export function makeUtxoEngineProcessor( } } -interface CommonArgs { +interface CommonParams { pluginInfo: PluginInfo walletInfo: SafeWalletInfo walletTools: UtxoWalletTools @@ -574,37 +570,36 @@ interface AddressTransactionCache { } } -const setLookAhead = async (common: CommonArgs): Promise => { - const { - pluginInfo: { engineInfo }, - lock, - dataLayer, - walletFormats, - walletTools - } = common - +const setLookAhead = async (common: CommonParams): Promise => { // Wait for the lock to be released before continuing invocation. // This is to ensure that setLockAhead is not called while the lock is held. - await lock.acquireAsync() + await common.lock.acquireAsync() try { - for (const format of walletFormats) { - const branches = getFormatSupportedBranches(engineInfo, format) + for (const format of common.walletFormats) { + const branches = getFormatSupportedBranches( + common.pluginInfo.engineInfo, + format + ) for (const branch of branches) { await deriveKeys({ format, changeIndex: branch }) } } } finally { - lock.release() + common.lock.release() } async function deriveKeys(changePath: ChangePath): Promise { const addressesToSubscribe = new Set() - const totalAddressCount = dataLayer.numAddressesByFormatPath(changePath) - let lastUsedIndex = await dataLayer.lastUsedIndexByFormatPath(changePath) + const totalAddressCount = common.dataLayer.numAddressesByFormatPath( + changePath + ) + let lastUsedIndex = await common.dataLayer.lastUsedIndexByFormatPath( + changePath + ) // Loop until the total address count equals the lookahead count - let lookAheadIndex = lastUsedIndex + engineInfo.gapLimit + let lookAheadIndex = lastUsedIndex + common.pluginInfo.engineInfo.gapLimit let nextAddressIndex = totalAddressCount while (nextAddressIndex <= lookAheadIndex) { const path: AddressPath = { @@ -612,14 +607,16 @@ const setLookAhead = async (common: CommonArgs): Promise => { addressIndex: nextAddressIndex } - const { scriptPubkey, redeemScript } = walletTools.getScriptPubkey(path) - const { address } = walletTools.scriptPubkeyToAddress({ + const { scriptPubkey, redeemScript } = common.walletTools.getScriptPubkey( + path + ) + const { address } = common.walletTools.scriptPubkeyToAddress({ changePath: path, scriptPubkey }) // Make a new IAddress and save it - await dataLayer.saveAddress( + await common.dataLayer.saveAddress( makeAddressData({ scriptPubkey, redeemScript, path }) ) @@ -627,9 +624,11 @@ const setLookAhead = async (common: CommonArgs): Promise => { addressesToSubscribe.add(address) // Update the state for the loop - lastUsedIndex = await dataLayer.lastUsedIndexByFormatPath(changePath) - lookAheadIndex = lastUsedIndex + engineInfo.gapLimit - nextAddressIndex = dataLayer.numAddressesByFormatPath(changePath) + lastUsedIndex = await common.dataLayer.lastUsedIndexByFormatPath( + changePath + ) + lookAheadIndex = lastUsedIndex + common.pluginInfo.engineInfo.gapLimit + nextAddressIndex = common.dataLayer.numAddressesByFormatPath(changePath) } // Add all the addresses to the subscribe cache for registering subscriptions later @@ -655,19 +654,22 @@ const addToAddressSubscribeCache = ( } const addToAddressTransactionCache = async ( - common: CommonArgs, - address: string, - changePath: ChangePath, - blockHeight: number, - addressTransactionCache: AddressTransactionCache + common: CommonParams, + args: { + address: string + changePath: ChangePath + blockHeight: number + addressTransactionCache: AddressTransactionCache + } ): Promise => { - const { walletTools, dataLayer } = common + const { address, changePath, addressTransactionCache } = args + let { blockHeight } = args // Fetch the blockHeight for the address from the database - const scriptPubkey = walletTools.addressToScriptPubkey(address) + const scriptPubkey = common.walletTools.addressToScriptPubkey(address) if (blockHeight === 0) { const { lastQueriedBlockHeight = 0 } = - (await dataLayer.fetchAddress(scriptPubkey)) ?? {} + (await common.dataLayer.fetchAddress(scriptPubkey)) ?? {} blockHeight = lastQueriedBlockHeight } @@ -679,19 +681,15 @@ const addToAddressTransactionCache = async ( } } -interface TransactionChangedArgs { - walletId: string - tx: TransactionData +export const transactionChanged = async (args: { + dataLayer: DataLayer emitter: EngineEmitter - walletTools: UtxoWalletTools pluginInfo: PluginInfo - dataLayer: DataLayer -} - -export const transactionChanged = async ( - args: TransactionChangedArgs -): Promise => { - const { emitter, walletTools, dataLayer, pluginInfo, tx, walletId } = args + tx: TransactionData + walletTools: UtxoWalletTools + walletId: string +}): Promise => { + const { dataLayer, emitter, pluginInfo, tx, walletTools, walletId } = args emitter.emit(EngineEvent.TRANSACTIONS_CHANGED, [ await toEdgeTransaction({ walletId, @@ -704,11 +702,9 @@ export const transactionChanged = async ( } export const pickNextTask = async ( - args: CommonArgs, + common: CommonParams, serverUri: string ): Promise | undefined | boolean> => { - const { pluginInfo, taskCache, serverStates } = args - const { addressSubscribeCache, addressUtxoCache, @@ -717,21 +713,22 @@ export const pickNextTask = async ( addressTransactionCache, updateTransactionCache, updateTransactionSpecificCache - } = taskCache + } = common.taskCache /** * Some currencies require an additional blockbook payload 'getTransactionSpecific' in order * to provide all relevant transaction data. Since this is currency-specific, we can limit - * the useage to currencies that require it. + * the usage to currencies that require it. **/ - const needsTxSpecific = pluginInfo.engineInfo.txSpecificHandling != null + const needsTxSpecific = + common.pluginInfo.engineInfo.txSpecificHandling != null - const serverState = serverStates.getServerState(serverUri) + const serverState = common.serverStates.getServerState(serverUri) if (serverState == null) return // subscribe all servers to new blocks if (serverState.blockSubscriptionStatus === 'unsubscribed') { - serverStates.watchBlocks(serverUri) + common.serverStates.watchBlocks(serverUri) return true } @@ -741,8 +738,7 @@ export const pickNextTask = async ( // Only process when all utxos for a specific address have been gathered if (!state.processing && state.full) { state.processing = true - await processDataLayerUtxos({ - ...args, + await processDataLayerUtxos(common, { scriptPubkey, utxos: state.utxos }) @@ -760,7 +756,7 @@ export const pickNextTask = async ( // check if we need to fetch additional network content for legacy purpose type const purposeType = pathToPurposeType( state.path, - pluginInfo.engineInfo.scriptTemplates + common.pluginInfo.engineInfo.scriptTemplates ) if ( purposeType === BIP43PurposeTypeEnum.Airbitz || @@ -768,12 +764,11 @@ export const pickNextTask = async ( purposeType === BIP43PurposeTypeEnum.ReplayProtection ) { // if we do need to make a network call, check with the serverState - if (!serverStates.serverCanGetTx(serverUri, utxo.txid)) return + if (!common.serverStates.serverCanGetTx(serverUri, utxo.txid)) return } state.processing = true removeItem(rawUtxoCache, utxoId) - const wsTask = await processRawUtxo({ - ...args, + const wsTask = await processRawUtxo(common, { ...state, address: state.address, serverUri, @@ -789,15 +784,14 @@ export const pickNextTask = async ( // Check if we need to fetch address UTXOs if ( !state.processing && - serverStates.serverCanGetAddress(serverUri, address) + common.serverStates.serverCanGetAddress(serverUri, address) ) { state.processing = true removeItem(addressUtxoCache, address) // Fetch and process address UTXOs - const wsTask = await processAddressUtxos({ - ...args, + const wsTask = await processAddressUtxos(common, { ...state, address, serverUri @@ -809,7 +803,7 @@ export const pickNextTask = async ( .catch(err => { addressUtxoCache[address] = state console.error(err) - args.log('error in addressUtxoCache:', err) + common.log('error in addressUtxoCache:', err) }) return wsTask } @@ -819,11 +813,11 @@ export const pickNextTask = async ( if (Object.keys(addressSubscribeCache).length > 0) { // These are addresses to which the server has not subscribed const newAddresses: string[] = [] - const blockHeight = serverStates.getBlockHeight(serverUri) + const blockHeight = common.serverStates.getBlockHeight(serverUri) // Loop each address in the cache for (const [address, state] of Object.entries(addressSubscribeCache)) { - const isAddressNewlySubscribed = !serverStates.serverIsAwareOfAddress( + const isAddressNewlySubscribed = !common.serverStates.serverIsAwareOfAddress( serverUri, address ) @@ -845,19 +839,18 @@ export const pickNextTask = async ( } } - await addToAddressTransactionCache( - args, + await addToAddressTransactionCache(common, { address, - state.path, + changePath: state.path, blockHeight, addressTransactionCache - ) + }) state.processing = true } // Subscribe to any new addresses if (newAddresses.length > 0) { - serverStates.watchAddresses(serverUri, newAddresses) + common.serverStates.watchAddresses(serverUri, newAddresses) return true } } @@ -868,15 +861,17 @@ export const pickNextTask = async ( for (const [txId, state] of Object.entries( updateTransactionSpecificCache )) { - if (!state.processing && serverStates.serverCanGetTx(serverUri, txId)) { + if ( + !state.processing && + common.serverStates.serverCanGetTx(serverUri, txId) + ) { hasProcessedAtLeastOnce = true state.processing = true removeItem(updateTransactionSpecificCache, txId) - const updateTransactionSpecificTask = updateTransactionsSpecific({ - ...args, - serverUri, - txId - }) + const updateTransactionSpecificTask = updateTransactionsSpecific( + common, + { serverUri, txId } + ) // once resolved, add the txid to the server cache updateTransactionSpecificTask.deferred.promise .then(() => { @@ -885,7 +880,7 @@ export const pickNextTask = async ( .catch(err => { updateTransactionSpecificCache[txId] = state console.error(err) - args.log('error in updateTransactionSpecificCache:', err) + common.log('error in updateTransactionSpecificCache:', err) }) return updateTransactionSpecificTask } @@ -898,14 +893,16 @@ export const pickNextTask = async ( if (Object.keys(updateTransactionCache).length > 0) { let hasProcessedAtLeastOnce = false for (const [txId, state] of Object.entries(updateTransactionCache)) { - if (!state.processing && serverStates.serverCanGetTx(serverUri, txId)) { + if ( + !state.processing && + common.serverStates.serverCanGetTx(serverUri, txId) + ) { hasProcessedAtLeastOnce = true state.processing = true removeItem(updateTransactionCache, txId) - const updateTransactionTask = updateTransactions({ - ...args, - serverUri, + const updateTransactionTask = updateTransactions(common, { needsTxSpecific, + serverUri, txId }) // once resolved, add the txid to the server cache @@ -916,7 +913,7 @@ export const pickNextTask = async ( .catch(err => { updateTransactionCache[txId] = state console.error(err) - args.log('error in updateTransactionCache:', err) + common.log('error in updateTransactionCache:', err) }) return updateTransactionTask } @@ -929,15 +926,14 @@ export const pickNextTask = async ( for (const [address, state] of Object.entries(addressTransactionCache)) { if ( !state.processing && - serverStates.serverCanGetAddress(serverUri, address) + common.serverStates.serverCanGetAddress(serverUri, address) ) { state.processing = true removeItem(addressTransactionCache, address) // Fetch and process address UTXOs - const wsTask = await processAddressTransactions({ - ...args, + const wsTask = await processAddressTransactions(common, { addressTransactionState: state, address, needsTxSpecific, @@ -950,165 +946,135 @@ export const pickNextTask = async ( .catch(err => { addressTransactionCache[address] = state console.error(err) - args.log('error in updateTransactionCache:', err) + common.log('error in updateTransactionCache:', err) }) return wsTask } } } -interface UpdateTransactionsSpecificArgs extends CommonArgs { - serverUri: string - txId: string -} - const updateTransactionsSpecific = ( - args: UpdateTransactionsSpecificArgs + common: CommonParams, + args: { serverUri: string; txId: string } ): WsTask => { - const { - walletInfo, - emitter, - walletTools, - pluginInfo, - dataLayer, - serverUri, - txId, - taskCache - } = args + const { serverUri, txId } = args const deferred = new Deferred() deferred.promise .then(async (txResponse: unknown) => { // Grab tx to update it - const txs = await dataLayer.fetchTransactions({ txId }) + const txs = await common.dataLayer.fetchTransactions({ txId }) let tx = txs[0] if (tx == null) return - if (pluginInfo.engineInfo.txSpecificHandling != null) { + if (common.pluginInfo.engineInfo.txSpecificHandling != null) { // Do coin-specific things to it - tx = pluginInfo.engineInfo.txSpecificHandling(tx, txResponse) + tx = common.pluginInfo.engineInfo.txSpecificHandling(tx, txResponse) } // Process and save new tx - const processedTx = await dataLayer.saveTransaction({ + const processedTx = await common.dataLayer.saveTransaction({ tx }) await transactionChanged({ - walletId: walletInfo.id, - emitter, - walletTools, - dataLayer, - pluginInfo, + walletId: common.walletInfo.id, + emitter: common.emitter, + walletTools: common.walletTools, + dataLayer: common.dataLayer, + pluginInfo: common.pluginInfo, tx: processedTx }) }) .catch(err => { console.error(err) - args.log('error in updateTransactionsSpecific:', err) - taskCache.updateTransactionSpecificCache[txId] = { processing: false } + common.log('error in updateTransactionsSpecific:', err) + common.taskCache.updateTransactionSpecificCache[txId] = { + processing: false + } }) - return args.serverStates.transactionSpecialQueryTask( + return common.serverStates.transactionSpecialQueryTask( serverUri, txId, deferred ) } -interface UpdateTransactionsArgs extends CommonArgs { - needsTxSpecific: boolean - serverUri: string - txId: string -} - const updateTransactions = ( - args: UpdateTransactionsArgs + common: CommonParams, + args: { + needsTxSpecific: boolean + serverUri: string + txId: string + } ): WsTask => { - const { - walletInfo, - emitter, - walletTools, - txId, - needsTxSpecific, - pluginInfo, - dataLayer, - serverUri, - taskCache - } = args + const { needsTxSpecific, serverUri, txId } = args const deferred = new Deferred() deferred.promise .then(async (txResponse: TransactionResponse) => { // check if raw tx is still not confirmed, if so, don't change anything if (txResponse.blockHeight < 1) return // Create new tx from raw tx - const tx = processTransactionResponse({ ...args, txResponse }) + const tx = processTransactionResponse(common, { txResponse }) // Remove any existing input utxos from the dataLayer for (const input of tx.inputs) { - await dataLayer.removeUtxos([`${input.txId}_${input.outputIndex}`]) + await common.dataLayer.removeUtxos([ + `${input.txId}_${input.outputIndex}` + ]) } // Update output utxos's blockHeight any existing input utxos from the dataLayer const utxoIds = tx.outputs.map(output => `${tx.txid}_${output.n}`) - const utxos = await dataLayer.fetchUtxos({ + const utxos = await common.dataLayer.fetchUtxos({ utxoIds }) for (const utxo of utxos) { if (utxo == null) continue utxo.blockHeight = tx.blockHeight - await dataLayer.saveUtxo(utxo) + await common.dataLayer.saveUtxo(utxo) } // Process and save new tx - const processedTx = await dataLayer.saveTransaction({ + const processedTx = await common.dataLayer.saveTransaction({ tx }) await transactionChanged({ - walletId: walletInfo.id, - emitter, - walletTools, - dataLayer, - pluginInfo, + walletId: common.walletInfo.id, + emitter: common.emitter, + walletTools: common.walletTools, + dataLayer: common.dataLayer, + pluginInfo: common.pluginInfo, tx: processedTx }) if (needsTxSpecific) { // Add task to grab transactionSpecific payload - taskCache.updateTransactionSpecificCache[txId] = { + common.taskCache.updateTransactionSpecificCache[txId] = { processing: false } } }) .catch(err => { console.error(err) - args.log('error in updateTransactions:', err) - taskCache.updateTransactionCache[txId] = { processing: false } + common.log('error in updateTransactions:', err) + common.taskCache.updateTransactionCache[txId] = { processing: false } }) - return args.serverStates.transactionQueryTask(serverUri, txId, deferred) + return common.serverStates.transactionQueryTask(serverUri, txId, deferred) } -interface DeriveScriptAddressArgs { +const internalDeriveScriptAddress = async (args: { walletTools: UtxoWalletTools engineInfo: EngineInfo dataLayer: DataLayer format: CurrencyFormat taskCache: TaskCache script: string -} - -interface DeriveScriptAddressReturn { +}): Promise<{ address: string scriptPubkey: string redeemScript: string -} - -const internalDeriveScriptAddress = async ({ - walletTools, - engineInfo, - dataLayer, - format, - taskCache, - script -}: DeriveScriptAddressArgs): Promise => { +}> => { + const { walletTools, engineInfo, dataLayer, format, taskCache, script } = args if (engineInfo.scriptTemplates == null) { throw new Error( `cannot derive script address ${script} without defined script template` @@ -1140,30 +1106,27 @@ const internalDeriveScriptAddress = async ({ return { address, scriptPubkey, redeemScript } } -interface GetFreshAddressArgs extends CommonArgs { - forceIndex?: number - changePath: ChangePath -} - -interface GetFreshAddressReturn { +const internalGetFreshAddress = async ( + common: CommonParams, + args: { + forceIndex?: number + changePath: ChangePath + } +): Promise<{ address: string legacyAddress: string nativeBalance?: string -} - -const internalGetFreshAddress = async ( - args: GetFreshAddressArgs -): Promise => { - const { changePath, walletTools, dataLayer, forceIndex } = args +}> => { + const { changePath, forceIndex } = args - const numAddresses = dataLayer.numAddressesByFormatPath(changePath) + const numAddresses = common.dataLayer.numAddressesByFormatPath(changePath) const path: AddressPath = { format: changePath.format, changeIndex: changePath.changeIndex, // while syncing, we may hit negative numbers when only subtracting. Use the address at /0 in that case. addressIndex: Math.max( - numAddresses - args.pluginInfo.engineInfo.gapLimit, + numAddresses - common.pluginInfo.engineInfo.gapLimit, 0 ) } @@ -1171,14 +1134,15 @@ const internalGetFreshAddress = async ( path.addressIndex = forceIndex } - const iAddress = await dataLayer.fetchAddress(path) + const iAddress = await common.dataLayer.fetchAddress(path) const nativeBalance = iAddress?.balance ?? '0' - const { scriptPubkey } = iAddress ?? (await walletTools.getScriptPubkey(path)) + const { scriptPubkey } = + iAddress ?? (await common.walletTools.getScriptPubkey(path)) if (scriptPubkey == null) { throw new Error('Unknown address path') } - const address = walletTools.scriptPubkeyToAddress({ + const address = common.walletTools.scriptPubkeyToAddress({ changePath: path, scriptPubkey }) @@ -1189,34 +1153,21 @@ const internalGetFreshAddress = async ( } } -interface ProcessAddressTxsArgs extends CommonArgs { - address: string - addressTransactionState: AddressTransactionCache[string] - serverUri: string - needsTxSpecific: boolean -} - const processAddressTransactions = async ( - args: ProcessAddressTxsArgs + common: CommonParams, + args: { + address: string + addressTransactionState: AddressTransactionCache[string] + needsTxSpecific: boolean + serverUri: string + } ): Promise> => { - const { - walletInfo, - address, - addressTransactionState, - emitter, - needsTxSpecific, - pluginInfo, - dataLayer, - walletTools, - taskCache, - pluginState, - serverUri - } = args + const { address, addressTransactionState, needsTxSpecific, serverUri } = args const { page = 1, blockHeight } = addressTransactionState - const addressTransactionCache = taskCache.addressTransactionCache + const addressTransactionCache = common.taskCache.addressTransactionCache - const scriptPubkey = walletTools.addressToScriptPubkey(address) - const addressData = await dataLayer.fetchAddress(scriptPubkey) + const scriptPubkey = common.walletTools.addressToScriptPubkey(address) + const addressData = await common.dataLayer.fetchAddress(scriptPubkey) if (addressData == null) { throw new Error(`could not find address with script pubkey ${scriptPubkey}`) } @@ -1225,7 +1176,7 @@ const processAddressTransactions = async ( const deferred = new Deferred() deferred.promise .then(async (value: AddressResponse) => { - pluginState.serverScoreUp(serverUri, Date.now() - queryTime) + common.pluginState.serverScoreUp(serverUri, Date.now() - queryTime) const { transactions = [], txs, unconfirmedTxs, totalPages } = value // If address is used and previously not marked as used, mark as used. @@ -1236,23 +1187,23 @@ const processAddressTransactions = async ( // Process and save the address's transactions for (const txResponse of transactions) { - const tx = processTransactionResponse({ ...args, txResponse }) - const processedTx = await dataLayer.saveTransaction({ + const tx = processTransactionResponse(common, { txResponse }) + const processedTx = await common.dataLayer.saveTransaction({ tx, scriptPubkeys: [scriptPubkey] }) await transactionChanged({ - walletId: walletInfo.id, - emitter, - walletTools, - dataLayer, - pluginInfo, + walletId: common.walletInfo.id, + emitter: common.emitter, + walletTools: common.walletTools, + dataLayer: common.dataLayer, + pluginInfo: common.pluginInfo, tx: processedTx }) if (needsTxSpecific) { // Add task to grab transactionSpecific payload - taskCache.updateTransactionSpecificCache[tx.txid] = { + common.taskCache.updateTransactionSpecificCache[tx.txid] = { processing: false } } @@ -1274,13 +1225,13 @@ const processAddressTransactions = async ( addressData.lastQueriedBlockHeight = blockHeight // Save/update the fully-processed address - await dataLayer.saveAddress(addressData) + await common.dataLayer.saveAddress(addressData) // Update the progress now that the transactions for an address have processed - await args.updateProgressRatio() + await common.updateProgressRatio() // Call setLookAhead to update the lookahead - await setLookAhead(args) + await setLookAhead(common) }) .catch(err => { // Log the error for debugging purposes without crashing the engine @@ -1289,7 +1240,7 @@ const processAddressTransactions = async ( addressTransactionCache[address] = addressTransactionState }) - return args.serverStates.addressQueryTask( + return common.serverStates.addressQueryTask( serverUri, address, { @@ -1300,17 +1251,11 @@ const processAddressTransactions = async ( ) } -interface processTransactionResponseArgs extends CommonArgs { - txResponse: TransactionResponse -} - const processTransactionResponse = ( - args: processTransactionResponseArgs + common: CommonParams, + args: { txResponse: TransactionResponse } ): TransactionData => { - const { - txResponse, - pluginInfo: { coinInfo } - } = args + const { txResponse } = args const inputs = txResponse.vin.map(vin => { const scriptPubkey = // Note: Blockbook has empirically not sent a hex value as the @@ -1321,7 +1266,7 @@ const processTransactionResponse = ( 'unknown' )({ address: vin.addresses[0], - coin: coinInfo.name + coin: common.pluginInfo.coinInfo.name }) return { txId: vin.txid, @@ -1340,7 +1285,7 @@ const processTransactionResponse = ( 'unknown' )({ address: vout.addresses[0], - coin: coinInfo.name + coin: common.pluginInfo.coinInfo.name }) return { n: vout.n, @@ -1363,33 +1308,28 @@ const processTransactionResponse = ( } } -interface ProcessAddressUtxosArgs extends CommonArgs { - address: string - path: ChangePath - processing: boolean - serverUri: string -} - const processAddressUtxos = async ( - args: ProcessAddressUtxosArgs + common: CommonParams, + args: { + address: string + path: ChangePath + processing: boolean + serverUri: string + } ): Promise> => { + const { address, path, serverUri } = args const { - address, - walletTools, - dataLayer, - taskCache, - path, - pluginState, - serverUri - } = args - const { addressUtxoCache, rawUtxoCache, dataLayerUtxoCache } = taskCache + addressUtxoCache, + rawUtxoCache, + dataLayerUtxoCache + } = common.taskCache const queryTime = Date.now() const deferred = new Deferred() deferred.promise .then(async (utxos: AddressUtxosResponse) => { - pluginState.serverScoreUp(serverUri, Date.now() - queryTime) - const scriptPubkey = walletTools.addressToScriptPubkey(address) - const addressData = await dataLayer.fetchAddress(scriptPubkey) + common.pluginState.serverScoreUp(serverUri, Date.now() - queryTime) + const scriptPubkey = common.walletTools.addressToScriptPubkey(address) + const addressData = await common.dataLayer.fetchAddress(scriptPubkey) if (addressData == null || addressData.path == null) { return } @@ -1419,31 +1359,23 @@ const processAddressUtxos = async ( } }) - return args.serverStates.utxoListQueryTask(serverUri, address, deferred) -} - -interface ProcessDataLayerUtxosArgs extends CommonArgs { - scriptPubkey: string - utxos: UtxoData[] + return common.serverStates.utxoListQueryTask(serverUri, address, deferred) } const processDataLayerUtxos = async ( - args: ProcessDataLayerUtxosArgs + common: CommonParams, + args: { + scriptPubkey: string + utxos: UtxoData[] + } ): Promise => { - const { - utxos, - dataLayer, - scriptPubkey, - log, - emitter, - pluginInfo: { currencyInfo } - } = args + const { utxos, scriptPubkey } = args const updatedUtxos: { [utxoId: string]: UtxoData } = Object.fromEntries( [...utxos].map(utxo => [utxo.id, utxo]) ) const utxoIdsToRemove: string[] = [] - const currentUtxos = await dataLayer.fetchUtxos({ scriptPubkey }) + const currentUtxos = await common.dataLayer.fetchUtxos({ scriptPubkey }) // // Modify existing UTXO set @@ -1468,7 +1400,7 @@ const processDataLayerUtxos = async ( } // Remove any spent UTXOs that have confirmations - await dataLayer.removeUtxos(utxoIdsToRemove) + await common.dataLayer.removeUtxos(utxoIdsToRemove) // // Save updated UTXO set @@ -1481,7 +1413,7 @@ const processDataLayerUtxos = async ( newBalance = add(utxo.value, newBalance) } // Save new UTXOs - await dataLayer.saveUtxo(utxo) + await common.dataLayer.saveUtxo(utxo) } // @@ -1490,10 +1422,14 @@ const processDataLayerUtxos = async ( // Address balance and emit address balance change event if (newBalance !== oldBalance) { - log('address balance changed:', { scriptPubkey, oldBalance, newBalance }) - emitter.emit( + common.log('address balance changed:', { + scriptPubkey, + oldBalance, + newBalance + }) + common.emitter.emit( EngineEvent.ADDRESS_BALANCE_CHANGED, - currencyInfo.currencyCode, + common.pluginInfo.currencyInfo.currencyCode, [ { scriptPubkey, @@ -1503,13 +1439,13 @@ const processDataLayerUtxos = async ( ) // Update balances for address that have this scriptPubkey - const address = await dataLayer.fetchAddress(scriptPubkey) + const address = await common.dataLayer.fetchAddress(scriptPubkey) if (address == null) { throw new Error('address not found when processing UTXO transactions') } - await dataLayer.saveAddress({ + await common.dataLayer.saveAddress({ ...address, balance: newBalance, used: true @@ -1517,43 +1453,30 @@ const processDataLayerUtxos = async ( } // Update the progress now that the UTXOs for an address have been processed - await args.updateProgressRatio() + await common.updateProgressRatio() - await setLookAhead(args).catch(err => { - log.error(err) + await setLookAhead(common).catch(err => { + common.log.error(err) throw err }) } -interface ProcessRawUtxoArgs extends CommonArgs { - address: AddressData - path: ChangePath - id: string - requiredCount: number - serverUri: string - utxo: BlockbookAccountUtxo -} - const processRawUtxo = async ( - args: ProcessRawUtxoArgs + common: CommonParams, + args: { + address: AddressData + path: ChangePath + id: string + requiredCount: number + serverUri: string + utxo: BlockbookAccountUtxo + } ): Promise | undefined> => { - const { - utxo, - id, - address, - pluginInfo, - dataLayer, - path, - taskCache, - requiredCount, - pluginState, - serverUri, - log - } = args - const { rawUtxoCache, dataLayerUtxoCache } = taskCache + const { address, path, id, requiredCount, serverUri, utxo } = args + const { rawUtxoCache, dataLayerUtxoCache } = common.taskCache const purposeType = pathToPurposeType( path, - pluginInfo.engineInfo.scriptTemplates + common.pluginInfo.engineInfo.scriptTemplates ) const scriptType: ScriptTypeEnum = getScriptTypeFromPurposeType(purposeType) let script: string @@ -1591,7 +1514,7 @@ const processRawUtxo = async ( // Legacy UTXOs need the previous transaction hex as the script // If we do not currently have it, add it to the queue to fetch it { - const [tx] = await dataLayer.fetchTransactions({ + const [tx] = await common.dataLayer.fetchTransactions({ txId: utxo.txid }) if (tx == null) { @@ -1599,9 +1522,11 @@ const processRawUtxo = async ( const deferred = new Deferred() deferred.promise .then((txResponse: TransactionResponse) => { - pluginState.serverScoreUp(serverUri, Date.now() - queryTime) - const processedTx = processTransactionResponse({ - ...args, + common.pluginState.serverScoreUp( + serverUri, + Date.now() - queryTime + ) + const processedTx = processTransactionResponse(common, { txResponse }) script = processedTx.hex @@ -1610,7 +1535,7 @@ const processRawUtxo = async ( }) .catch(err => { // If something went wrong, add the UTXO back to the queue - log('error in processRawUtxo:', err) + common.log('error in processRawUtxo:', err) const utxoId = `${utxo.txid}_${utxo.vout}` rawUtxoCache[utxoId] = { blockbookUtxo: utxo, @@ -1621,7 +1546,7 @@ const processRawUtxo = async ( } }) - return args.serverStates.transactionQueryTask( + return common.serverStates.transactionQueryTask( serverUri, utxo.txid, deferred From 7aadfc5ead24caf5a0500bf4d6aca2533ec94939 Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Wed, 24 Jul 2024 14:45:05 -0700 Subject: [PATCH 5/6] Rename processor functions and cache items for readability This adds documentation comments to each processor function to briefly describe the function's intent. --- .../utxobased/engine/UtxoEngineProcessor.ts | 239 ++++++++++-------- 1 file changed, 137 insertions(+), 102 deletions(-) diff --git a/src/common/utxobased/engine/UtxoEngineProcessor.ts b/src/common/utxobased/engine/UtxoEngineProcessor.ts index ebb78447..4fba5117 100644 --- a/src/common/utxobased/engine/UtxoEngineProcessor.ts +++ b/src/common/utxobased/engine/UtxoEngineProcessor.ts @@ -106,37 +106,37 @@ export function makeUtxoEngineProcessor( const taskCache: TaskCache = { blockWatching: false, + addressForTransactionsCache: {}, + addressForUtxosCache: {}, addressSubscribeCache: {}, - addressTransactionCache: {}, - addressUtxoCache: {}, - rawUtxoCache: {}, + blockbookUtxoCache: {}, dataLayerUtxoCache: {}, - updateTransactionCache: {}, - updateTransactionSpecificCache: {} + transactionSpecificUpdateCache: {}, + transactionUpdateCache: {} } const clearTaskCache = (): void => { taskCache.blockWatching = false - for (const key of Object.keys(taskCache.addressSubscribeCache)) { - removeItem(taskCache.addressSubscribeCache, key) + for (const key of Object.keys(taskCache.addressForTransactionsCache)) { + removeItem(taskCache.addressForTransactionsCache, key) } - for (const key of Object.keys(taskCache.addressTransactionCache)) { - removeItem(taskCache.addressTransactionCache, key) + for (const key of Object.keys(taskCache.addressForUtxosCache)) { + removeItem(taskCache.addressForUtxosCache, key) } - for (const key of Object.keys(taskCache.addressUtxoCache)) { - removeItem(taskCache.addressUtxoCache, key) + for (const key of Object.keys(taskCache.addressSubscribeCache)) { + removeItem(taskCache.addressSubscribeCache, key) } - for (const key of Object.keys(taskCache.rawUtxoCache)) { - removeItem(taskCache.rawUtxoCache, key) + for (const key of Object.keys(taskCache.blockbookUtxoCache)) { + removeItem(taskCache.blockbookUtxoCache, key) } for (const key of Object.keys(taskCache.dataLayerUtxoCache)) { removeItem(taskCache.dataLayerUtxoCache, key) } - for (const key of Object.keys(taskCache.updateTransactionCache)) { - removeItem(taskCache.updateTransactionCache, key) + for (const key of Object.keys(taskCache.transactionSpecificUpdateCache)) { + removeItem(taskCache.transactionSpecificUpdateCache, key) } - for (const key of Object.keys(taskCache.updateTransactionSpecificCache)) { - removeItem(taskCache.updateTransactionSpecificCache, key) + for (const key of Object.keys(taskCache.transactionUpdateCache)) { + removeItem(taskCache.transactionUpdateCache, key) } } @@ -221,7 +221,7 @@ export function makeUtxoEngineProcessor( }) for (const tx of txs) { if (tx == null) continue - taskCache.updateTransactionCache[tx.txid] = { processing: false } + taskCache.transactionUpdateCache[tx.txid] = { processing: false } } } ) @@ -232,15 +232,15 @@ export function makeUtxoEngineProcessor( const state = taskCache.addressSubscribeCache[response.address] if (state != null) { const { path } = state - taskCache.addressUtxoCache[response.address] = { + taskCache.addressForUtxosCache[response.address] = { processing: false, path } - addToAddressTransactionCache(common, { + addToAddressForTransactionsCache(common, { address: response.address, changePath: path, blockHeight: 0, - addressTransactionCache: taskCache.addressTransactionCache + addressForTransactionsCache: taskCache.addressForTransactionsCache }).catch(() => { throw new Error('failed to add to transaction cache') }) @@ -523,25 +523,27 @@ interface CommonParams { interface TaskCache { blockWatching: boolean + readonly addressForTransactionsCache: AddressForTransactionsCache + readonly addressForUtxosCache: AddressForUtxosCache readonly addressSubscribeCache: AddressSubscribeCache - readonly addressUtxoCache: AddressUtxoCache - readonly rawUtxoCache: RawUtxoCache + readonly blockbookUtxoCache: BlockbookUtxoCache readonly dataLayerUtxoCache: DataLayerUtxoCache - readonly addressTransactionCache: AddressTransactionCache - readonly updateTransactionCache: UpdateTransactionCache - readonly updateTransactionSpecificCache: UpdateTransactionSpecificCache + readonly transactionSpecificUpdateCache: TransactionSpecificUpdateCache + readonly transactionUpdateCache: TransactionUpdateCache } -interface UpdateTransactionCache { - [key: string]: { processing: boolean } -} -interface UpdateTransactionSpecificCache { - [key: string]: { processing: boolean } +interface AddressForTransactionsCache { + [key: string]: { + processing: boolean + path: ChangePath + page: number + blockHeight: number + } } -interface AddressSubscribeCache { +interface AddressForUtxosCache { [key: string]: { processing: boolean; path: ChangePath } } -interface AddressUtxoCache { +interface AddressSubscribeCache { [key: string]: { processing: boolean; path: ChangePath } } interface DataLayerUtxoCache { @@ -552,7 +554,7 @@ interface DataLayerUtxoCache { path: ChangePath } } -interface RawUtxoCache { +interface BlockbookUtxoCache { [key: string]: { blockbookUtxo: BlockbookAccountUtxo processing: boolean @@ -561,13 +563,11 @@ interface RawUtxoCache { requiredCount: number } } -interface AddressTransactionCache { - [key: string]: { - processing: boolean - path: ChangePath - page: number - blockHeight: number - } +interface TransactionSpecificUpdateCache { + [key: string]: { processing: boolean } +} +interface TransactionUpdateCache { + [key: string]: { processing: boolean } } const setLookAhead = async (common: CommonParams): Promise => { @@ -653,16 +653,16 @@ const addToAddressSubscribeCache = ( }) } -const addToAddressTransactionCache = async ( +const addToAddressForTransactionsCache = async ( common: CommonParams, args: { address: string changePath: ChangePath blockHeight: number - addressTransactionCache: AddressTransactionCache + addressForTransactionsCache: AddressForTransactionsCache } ): Promise => { - const { address, changePath, addressTransactionCache } = args + const { address, changePath, addressForTransactionsCache } = args let { blockHeight } = args // Fetch the blockHeight for the address from the database const scriptPubkey = common.walletTools.addressToScriptPubkey(address) @@ -673,7 +673,7 @@ const addToAddressTransactionCache = async ( blockHeight = lastQueriedBlockHeight } - addressTransactionCache[address] = { + addressForTransactionsCache[address] = { processing: false, path: changePath, page: 1, // Page starts on 1 @@ -706,13 +706,13 @@ export const pickNextTask = async ( serverUri: string ): Promise | undefined | boolean> => { const { + addressForTransactionsCache, + addressForUtxosCache, addressSubscribeCache, - addressUtxoCache, - rawUtxoCache, + blockbookUtxoCache, dataLayerUtxoCache, - addressTransactionCache, - updateTransactionCache, - updateTransactionSpecificCache + transactionSpecificUpdateCache, + transactionUpdateCache } = common.taskCache /** @@ -749,7 +749,7 @@ export const pickNextTask = async ( } // Loop unparsed utxos, some require a network call to get the full tx data - for (const [utxoId, state] of Object.entries(rawUtxoCache)) { + for (const [utxoId, state] of Object.entries(blockbookUtxoCache)) { const utxo = state.blockbookUtxo if (utxo == null) continue if (!state.processing) { @@ -767,8 +767,8 @@ export const pickNextTask = async ( if (!common.serverStates.serverCanGetTx(serverUri, utxo.txid)) return } state.processing = true - removeItem(rawUtxoCache, utxoId) - const wsTask = await processRawUtxo(common, { + removeItem(blockbookUtxoCache, utxoId) + const wsTask = await processBlockbookUtxo(common, { ...state, address: state.address, serverUri, @@ -780,7 +780,7 @@ export const pickNextTask = async ( } // Loop to process addresses to utxos - for (const [address, state] of Object.entries(addressUtxoCache)) { + for (const [address, state] of Object.entries(addressForUtxosCache)) { // Check if we need to fetch address UTXOs if ( !state.processing && @@ -788,10 +788,10 @@ export const pickNextTask = async ( ) { state.processing = true - removeItem(addressUtxoCache, address) + removeItem(addressForUtxosCache, address) - // Fetch and process address UTXOs - const wsTask = await processAddressUtxos(common, { + // Fetch and process address for UTXOs + const wsTask = await processAddressForUtxos(common, { ...state, address, serverUri @@ -801,9 +801,9 @@ export const pickNextTask = async ( serverState.addresses.add(address) }) .catch(err => { - addressUtxoCache[address] = state + addressForUtxosCache[address] = state console.error(err) - common.log('error in addressUtxoCache:', err) + common.log('error while processing address for UTXOs:', err) }) return wsTask } @@ -827,23 +827,23 @@ export const pickNextTask = async ( newAddresses.push(address) } - // Add to the addressTransactionCache if they're not yet added: + // Add to the addressForTransactionsCache if they're not yet added: // Only process newly watched addresses if (state.processing) continue // Add the newly watched addresses to the UTXO cache if (isAddressNewlySubscribed) { - addressUtxoCache[address] = { + addressForUtxosCache[address] = { processing: false, path: state.path } } - await addToAddressTransactionCache(common, { + await addToAddressForTransactionsCache(common, { address, changePath: state.path, blockHeight, - addressTransactionCache + addressForTransactionsCache }) state.processing = true } @@ -856,10 +856,10 @@ export const pickNextTask = async ( } // filled when transactions potentially changed (e.g. through new block notification) - if (Object.keys(updateTransactionSpecificCache).length > 0) { + if (Object.keys(transactionSpecificUpdateCache).length > 0) { let hasProcessedAtLeastOnce = false for (const [txId, state] of Object.entries( - updateTransactionSpecificCache + transactionSpecificUpdateCache )) { if ( !state.processing && @@ -867,8 +867,8 @@ export const pickNextTask = async ( ) { hasProcessedAtLeastOnce = true state.processing = true - removeItem(updateTransactionSpecificCache, txId) - const updateTransactionSpecificTask = updateTransactionsSpecific( + removeItem(transactionSpecificUpdateCache, txId) + const updateTransactionSpecificTask = processTransactionsSpecificUpdate( common, { serverUri, txId } ) @@ -878,9 +878,12 @@ export const pickNextTask = async ( serverState.txids.add(txId) }) .catch(err => { - updateTransactionSpecificCache[txId] = state + transactionSpecificUpdateCache[txId] = state console.error(err) - common.log('error in updateTransactionSpecificCache:', err) + common.log( + 'error while processing transaction specific update:', + err + ) }) return updateTransactionSpecificTask } @@ -890,17 +893,17 @@ export const pickNextTask = async ( } // filled when transactions potentially changed (e.g. through new block notification) - if (Object.keys(updateTransactionCache).length > 0) { + if (Object.keys(transactionUpdateCache).length > 0) { let hasProcessedAtLeastOnce = false - for (const [txId, state] of Object.entries(updateTransactionCache)) { + for (const [txId, state] of Object.entries(transactionUpdateCache)) { if ( !state.processing && common.serverStates.serverCanGetTx(serverUri, txId) ) { hasProcessedAtLeastOnce = true state.processing = true - removeItem(updateTransactionCache, txId) - const updateTransactionTask = updateTransactions(common, { + removeItem(transactionUpdateCache, txId) + const updateTransactionTask = processTransactionUpdate(common, { needsTxSpecific, serverUri, txId @@ -911,9 +914,9 @@ export const pickNextTask = async ( serverState.txids.add(txId) }) .catch(err => { - updateTransactionCache[txId] = state + transactionUpdateCache[txId] = state console.error(err) - common.log('error in updateTransactionCache:', err) + common.log('error while processing transaction update:', err) }) return updateTransactionTask } @@ -923,17 +926,17 @@ export const pickNextTask = async ( } // loop to get and process transaction history of single addresses, triggers setLookAhead - for (const [address, state] of Object.entries(addressTransactionCache)) { + for (const [address, state] of Object.entries(addressForTransactionsCache)) { if ( !state.processing && common.serverStates.serverCanGetAddress(serverUri, address) ) { state.processing = true - removeItem(addressTransactionCache, address) + removeItem(addressForTransactionsCache, address) // Fetch and process address UTXOs - const wsTask = await processAddressTransactions(common, { + const wsTask = await processAddressForTransactions(common, { addressTransactionState: state, address, needsTxSpecific, @@ -944,16 +947,21 @@ export const pickNextTask = async ( serverState.addresses.add(address) }) .catch(err => { - addressTransactionCache[address] = state + addressForTransactionsCache[address] = state console.error(err) - common.log('error in updateTransactionCache:', err) + common.log('error while processing address for transactions:', err) }) return wsTask } } } -const updateTransactionsSpecific = ( +/** + * Process a transaction specific update from the TransactionSpecificUpdateCache + * by querying the network for the transaction data using the specific handling + * query and processing the data into the DataLayer. + */ +const processTransactionsSpecificUpdate = ( common: CommonParams, args: { serverUri: string; txId: string } ): WsTask => { @@ -987,8 +995,8 @@ const updateTransactionsSpecific = ( }) .catch(err => { console.error(err) - common.log('error in updateTransactionsSpecific:', err) - common.taskCache.updateTransactionSpecificCache[txId] = { + common.log('error while processing transaction specific update:', err) + common.taskCache.transactionSpecificUpdateCache[txId] = { processing: false } }) @@ -1000,7 +1008,11 @@ const updateTransactionsSpecific = ( ) } -const updateTransactions = ( +/** + * Processes a transaction update from the TransactionUpdateCache by querying + * the network for the transaction data and processing it into the DataLayer. + */ +const processTransactionUpdate = ( common: CommonParams, args: { needsTxSpecific: boolean @@ -1048,15 +1060,15 @@ const updateTransactions = ( if (needsTxSpecific) { // Add task to grab transactionSpecific payload - common.taskCache.updateTransactionSpecificCache[txId] = { + common.taskCache.transactionSpecificUpdateCache[txId] = { processing: false } } }) .catch(err => { console.error(err) - common.log('error in updateTransactions:', err) - common.taskCache.updateTransactionCache[txId] = { processing: false } + common.log('error while processing transaction update:', err) + common.taskCache.transactionUpdateCache[txId] = { processing: false } }) return common.serverStates.transactionQueryTask(serverUri, txId, deferred) @@ -1153,18 +1165,23 @@ const internalGetFreshAddress = async ( } } -const processAddressTransactions = async ( +/** + * Processes an address for transactions by querying the network for the + * transaction data. + */ +const processAddressForTransactions = async ( common: CommonParams, args: { address: string - addressTransactionState: AddressTransactionCache[string] + addressTransactionState: AddressForTransactionsCache[string] needsTxSpecific: boolean serverUri: string } ): Promise> => { const { address, addressTransactionState, needsTxSpecific, serverUri } = args const { page = 1, blockHeight } = addressTransactionState - const addressTransactionCache = common.taskCache.addressTransactionCache + const addressForTransactionsCache = + common.taskCache.addressForTransactionsCache const scriptPubkey = common.walletTools.addressToScriptPubkey(address) const addressData = await common.dataLayer.fetchAddress(scriptPubkey) @@ -1203,7 +1220,7 @@ const processAddressTransactions = async ( if (needsTxSpecific) { // Add task to grab transactionSpecific payload - common.taskCache.updateTransactionSpecificCache[tx.txid] = { + common.taskCache.transactionSpecificUpdateCache[tx.txid] = { processing: false } } @@ -1213,7 +1230,7 @@ const processAddressTransactions = async ( // we have progressed through all of the blockbook pages if (page < totalPages) { // Add the address back to the cache, incrementing the page - addressTransactionCache[address] = { + addressForTransactionsCache[address] = { ...addressTransactionState, processing: false, page: page + 1 @@ -1237,7 +1254,7 @@ const processAddressTransactions = async ( // Log the error for debugging purposes without crashing the engine // This will cause frozen wallet syncs console.error(err) - addressTransactionCache[address] = addressTransactionState + addressForTransactionsCache[address] = addressTransactionState }) return common.serverStates.addressQueryTask( @@ -1251,6 +1268,11 @@ const processAddressTransactions = async ( ) } +/** + * Processes a blockbook transaction response object (a blockbook transaction). + * It will simply convert the blockbook transaction into a TransactionData + * object to be saved in the DataLayer + */ const processTransactionResponse = ( common: CommonParams, args: { txResponse: TransactionResponse } @@ -1308,7 +1330,12 @@ const processTransactionResponse = ( } } -const processAddressUtxos = async ( +/** + * Process a given address for UTXO data by querying the network for UTXO data + * and processing it into the `blockbookUtxoCache` to later be processed by + * `processBlockbookUtxo`. + */ +const processAddressForUtxos = async ( common: CommonParams, args: { address: string @@ -1319,8 +1346,8 @@ const processAddressUtxos = async ( ): Promise> => { const { address, path, serverUri } = args const { - addressUtxoCache, - rawUtxoCache, + addressForUtxosCache, + blockbookUtxoCache, dataLayerUtxoCache } = common.taskCache const queryTime = Date.now() @@ -1341,7 +1368,7 @@ const processAddressUtxos = async ( for (const utxo of utxos) { const utxoId = `${utxo.txid}_${utxo.vout}` - rawUtxoCache[utxoId] = { + blockbookUtxoCache[utxoId] = { blockbookUtxo: utxo, processing: false, requiredCount: utxos.length, @@ -1353,7 +1380,7 @@ const processAddressUtxos = async ( }) .catch(() => { args.processing = false - addressUtxoCache[address] = { + addressForUtxosCache[address] = { processing: args.processing, path } @@ -1461,7 +1488,10 @@ const processDataLayerUtxos = async ( }) } -const processRawUtxo = async ( +/** + * Process a blockbook UTXO from the cache into the DataLayer + */ +const processBlockbookUtxo = async ( common: CommonParams, args: { address: AddressData @@ -1473,7 +1503,7 @@ const processRawUtxo = async ( } ): Promise | undefined> => { const { address, path, id, requiredCount, serverUri, utxo } = args - const { rawUtxoCache, dataLayerUtxoCache } = common.taskCache + const { blockbookUtxoCache, dataLayerUtxoCache } = common.taskCache const purposeType = pathToPurposeType( path, common.pluginInfo.engineInfo.scriptTemplates @@ -1535,9 +1565,9 @@ const processRawUtxo = async ( }) .catch(err => { // If something went wrong, add the UTXO back to the queue - common.log('error in processRawUtxo:', err) + common.log('error while processing Blockbook UTXO:', err) const utxoId = `${utxo.txid}_${utxo.vout}` - rawUtxoCache[utxoId] = { + blockbookUtxoCache[utxoId] = { blockbookUtxo: utxo, processing: false, path, @@ -1577,6 +1607,11 @@ const processRawUtxo = async ( done() } +/** + * This adds a UtxoData object to the DataLayerUtxoCache. It will mark the cache + * with processing and full flags which tell the pickNextTask routine when to + * process the UtxoData into the data-later. + */ const addToDataLayerUtxoCache = ( dataLayerUtxoCache: DataLayerUtxoCache, path: ChangePath, From ade3359650f25bb5be8f25d6c21f22f4847136a8 Mon Sep 17 00:00:00 2001 From: Sam Holmes Date: Thu, 25 Jul 2024 14:11:45 -0700 Subject: [PATCH 6/6] Change test blockbook server connection URI Trezor is known to rate limit. In the future, a mock Blockbook interface is the correct fix. --- test/common/utxobased/network/Blockbook.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common/utxobased/network/Blockbook.spec.ts b/test/common/utxobased/network/Blockbook.spec.ts index af6531da..03e6c9ac 100644 --- a/test/common/utxobased/network/Blockbook.spec.ts +++ b/test/common/utxobased/network/Blockbook.spec.ts @@ -164,7 +164,7 @@ describe('Blockbook', function () { beforeEach(async () => { blockbook = makeBlockbook({ - connectionUri: 'wss://btc1.trezor.io/websocket', + connectionUri: 'wss://bitcoin.atomicwallet.io/websocket', engineEmitter, log, onQueueSpaceCB,