From 4559b99f68e407454179e938b5a86144e2af9853 Mon Sep 17 00:00:00 2001 From: William Wolff Date: Wed, 21 Aug 2024 14:38:58 +0200 Subject: [PATCH 01/29] refactor(ogmios-block-type-mapping): add block type; transaction cbor --- .../src/ChainHistory/DbSyncChainHistory/mappers.ts | 1 + .../src/ChainHistory/DbSyncChainHistory/types.ts | 1 + packages/core/src/Cardano/types/Block.ts | 3 +++ packages/core/src/Cardano/types/Transaction.ts | 2 ++ packages/ogmios/src/ogmiosToCore/block.ts | 2 ++ .../__snapshots__/ObservableOgmiosCardanoNode.test.ts.snap | 1 + .../test/ogmiosToCore/__snapshots__/block.test.ts.snap | 6 ++++++ 7 files changed, 16 insertions(+) diff --git a/packages/cardano-services/src/ChainHistory/DbSyncChainHistory/mappers.ts b/packages/cardano-services/src/ChainHistory/DbSyncChainHistory/mappers.ts index 2a30d72b83a..8a4d84a076a 100644 --- a/packages/cardano-services/src/ChainHistory/DbSyncChainHistory/mappers.ts +++ b/packages/cardano-services/src/ChainHistory/DbSyncChainHistory/mappers.ts @@ -492,6 +492,7 @@ export const mapBlock = ( : Cardano.SlotLeader(blockModel.slot_leader_hash.toString('hex')), totalOutput: BigInt(blockOutputModel?.output ?? 0), txCount: Number(blockModel.tx_count), + type: blockModel.type, vrf: blockModel.vrf as unknown as Cardano.VrfVkBech32 }); diff --git a/packages/cardano-services/src/ChainHistory/DbSyncChainHistory/types.ts b/packages/cardano-services/src/ChainHistory/DbSyncChainHistory/types.ts index 1790a03f792..8486cf0dc24 100644 --- a/packages/cardano-services/src/ChainHistory/DbSyncChainHistory/types.ts +++ b/packages/cardano-services/src/ChainHistory/DbSyncChainHistory/types.ts @@ -19,6 +19,7 @@ export interface BlockModel { slot_no: string; time: string; tx_count: string; + type: Cardano.BlockType; vrf: string; } diff --git a/packages/core/src/Cardano/types/Block.ts b/packages/core/src/Cardano/types/Block.ts index e04fa3d3d8d..0f5c56353e2 100644 --- a/packages/core/src/Cardano/types/Block.ts +++ b/packages/core/src/Cardano/types/Block.ts @@ -77,6 +77,8 @@ VrfVkBech32.fromHex = (value: string) => { return VrfVkBech32(BaseEncoding.bech32.encode('vrf_vk', words, 1023)); }; +export type BlockType = 'bft' | 'praos'; + /** Minimal Block type meant as a base for the more complete version `Block` */ // TODO: optionals (except previousBlock) are there because they are not calculated for Byron yet. // Remove them once calculation is done and remove the Required from interface Block @@ -89,6 +91,7 @@ export interface BlockInfo { /** Byron blocks size not calculated yet */ size?: BlockSize; previousBlock?: BlockId; + type: BlockType; vrf?: VrfVkBech32; /** * This is the operational cold verification key of the stake pool diff --git a/packages/core/src/Cardano/types/Transaction.ts b/packages/core/src/Cardano/types/Transaction.ts index 3d3fd4e1ceb..1f253693dd1 100644 --- a/packages/core/src/Cardano/types/Transaction.ts +++ b/packages/core/src/Cardano/types/Transaction.ts @@ -11,6 +11,7 @@ import { PlutusData } from './PlutusData'; import { ProposalProcedure, VotingProcedures } from './Governance'; import { RewardAccount } from '../Address'; import { Script } from './Script'; +import { Serialization } from '../..'; /** transaction hash as hex string */ export type TransactionId = OpaqueString<'TransactionId'>; @@ -151,6 +152,7 @@ export interface OnChainTx extends Omit, 'witness' | 'auxiliaryData'> { witness: Omit; auxiliaryData?: Omit; + cbor?: Serialization.TxCBOR; } export interface HydratedTx extends TxWithInputSource { diff --git a/packages/ogmios/src/ogmiosToCore/block.ts b/packages/ogmios/src/ogmiosToCore/block.ts index 9d9fc62b526..7fb9cf98cb7 100644 --- a/packages/ogmios/src/ogmiosToCore/block.ts +++ b/packages/ogmios/src/ogmiosToCore/block.ts @@ -74,6 +74,7 @@ const mapByronBlock = (block: Schema.BlockBFT): Cardano.Block => ({ size: mapBlockSize(block), totalOutput: mapTotalOutputs(block), txCount: mapTxCount(block), + type: block.type, vrf: undefined // no vrf key for byron. DbSync doesn't have one either }); @@ -86,6 +87,7 @@ const mapCommonBlock = (block: CommonBlock): Cardano.Block => ({ size: mapBlockSize(block), totalOutput: mapTotalOutputs(block), txCount: mapTxCount(block), + type: block.type, vrf: mapCommonVrf(block) }); diff --git a/packages/ogmios/test/CardanoNode/__snapshots__/ObservableOgmiosCardanoNode.test.ts.snap b/packages/ogmios/test/CardanoNode/__snapshots__/ObservableOgmiosCardanoNode.test.ts.snap index 27aee5644cf..fe0295f0efe 100644 --- a/packages/ogmios/test/CardanoNode/__snapshots__/ObservableOgmiosCardanoNode.test.ts.snap +++ b/packages/ogmios/test/CardanoNode/__snapshots__/ObservableOgmiosCardanoNode.test.ts.snap @@ -258,6 +258,7 @@ Array [ "size": 1880, "totalOutput": 29999998493561943n, "txCount": 1, + "type": "praos", "vrf": "vrf_vk1ny8dyz3pa9u7v7h8l5evsmxxjq07dk6j5uda60lxe3zsya5ea20s2c6jph", }, "eventType": 0, diff --git a/packages/ogmios/test/ogmiosToCore/__snapshots__/block.test.ts.snap b/packages/ogmios/test/ogmiosToCore/__snapshots__/block.test.ts.snap index ceb03db80fa..a1e29e3f8fe 100644 --- a/packages/ogmios/test/ogmiosToCore/__snapshots__/block.test.ts.snap +++ b/packages/ogmios/test/ogmiosToCore/__snapshots__/block.test.ts.snap @@ -632,6 +632,7 @@ Object { "size": 8511, "totalOutput": 29698651346n, "txCount": 6, + "type": "praos", "vrf": "vrf_vk1afy7gefvgc9eaek64lhunxw2velmuh4467n6a2aalah7r8q68j0s4f2sa9", } `; @@ -711,6 +712,7 @@ Object { "size": 1193, "totalOutput": 29699998493147869n, "txCount": 1, + "type": "praos", "vrf": "vrf_vk1wpa9axwwassnadt8drdrzptxm285latvhhvsgv0t6zhp0akgej9sfgecvl", } `; @@ -790,6 +792,7 @@ Object { "size": 1152, "totalOutput": 29699998492735907n, "txCount": 1, + "type": "praos", "vrf": "vrf_vk1aw6s04lqkquell8drg6jjusrzd8c5259kctvxkgug5kta7nphz9qfqtr4w", } `; @@ -846,6 +849,7 @@ Object { "size": 908, "totalOutput": 1000000n, "txCount": 1, + "type": "bft", "vrf": undefined, } `; @@ -925,6 +929,7 @@ Object { "size": 1151, "totalOutput": 29699998492941888n, "txCount": 1, + "type": "praos", "vrf": "vrf_vk1aw6s04lqkquell8drg6jjusrzd8c5259kctvxkgug5kta7nphz9qfqtr4w", } `; @@ -1157,6 +1162,7 @@ Object { "size": 1880, "totalOutput": 29999998493561943n, "txCount": 1, + "type": "praos", "vrf": "vrf_vk1ny8dyz3pa9u7v7h8l5evsmxxjq07dk6j5uda60lxe3zsya5ea20s2c6jph", } `; \ No newline at end of file From 32e5ffc4eeb6efb264be4810ce56c1bde18146a4 Mon Sep 17 00:00:00 2001 From: William Wolff Date: Wed, 21 Aug 2024 18:29:18 +0200 Subject: [PATCH 02/29] feat: add transaction projection indexed by credentials --- .../Projection/prepareTypeormProjection.ts | 30 +++- .../src/entity/Credential.entity.ts | 26 +++ .../src/entity/Transaction.entity.ts | 27 +++ .../projection-typeorm/src/entity/index.ts | 2 + .../projection-typeorm/src/operators/index.ts | 2 + .../src/operators/storeCredentials.ts | 164 ++++++++++++++++++ .../src/operators/storeTransactions.ts | 60 +++++++ .../Mappers/certificates/withCertificates.ts | 64 +++++-- .../projection/src/operators/Mappers/index.ts | 10 +- .../projection/src/operators/Mappers/util.ts | 40 +++++ .../src/operators/Mappers/withAddresses.ts | 59 ++----- .../src/operators/Mappers/withUtxo.ts | 128 +++++++++++--- .../Mappers/withValidByronAddresses.ts | 61 +++++++ .../certificates/withCertificates.test.ts | 9 +- .../withStakeKeyRegistrations.test.ts | 3 +- .../certificates/withStakeKeys.test.ts | 12 +- .../certificates/withStakePools.test.ts | 6 +- .../operators/Mappers/withAddresses.test.ts | 20 ++- .../test/operators/Mappers/withUtxo.test.ts | 48 ++++- 19 files changed, 663 insertions(+), 108 deletions(-) create mode 100644 packages/projection-typeorm/src/entity/Credential.entity.ts create mode 100644 packages/projection-typeorm/src/entity/Transaction.entity.ts create mode 100644 packages/projection-typeorm/src/operators/storeCredentials.ts create mode 100644 packages/projection-typeorm/src/operators/storeTransactions.ts create mode 100644 packages/projection/src/operators/Mappers/withValidByronAddresses.ts diff --git a/packages/cardano-services/src/Projection/prepareTypeormProjection.ts b/packages/cardano-services/src/Projection/prepareTypeormProjection.ts index 09e79dc1704..3a4751ebd25 100644 --- a/packages/cardano-services/src/Projection/prepareTypeormProjection.ts +++ b/packages/cardano-services/src/Projection/prepareTypeormProjection.ts @@ -3,6 +3,7 @@ import { AssetEntity, BlockDataEntity, BlockEntity, + CredentialEntity, CurrentPoolMetricsEntity, DataSourceExtensions, GovernanceActionEntity, @@ -18,11 +19,13 @@ import { StakeKeyRegistrationEntity, StakePoolEntity, TokensEntity, + TransactionEntity, createStorePoolMetricsUpdateJob, createStoreStakePoolMetadataJob, storeAddresses, storeAssets, storeBlock, + storeCredentials, storeGovernanceAction, storeHandleMetadata, storeHandles, @@ -30,10 +33,12 @@ import { storeStakeKeyRegistrations, storeStakePoolRewardsJob, storeStakePools, + storeTransactions, storeUtxo, willStoreAddresses, willStoreAssets, willStoreBlockData, + willStoreCredentials, willStoreGovernanceAction, willStoreHandleMetadata, willStoreHandles, @@ -42,6 +47,7 @@ import { willStoreStakePoolMetadataJob, willStoreStakePoolRewardsJob, willStoreStakePools, + willStoreTransactions, willStoreUtxo } from '@cardano-sdk/projection-typeorm'; import { Cardano, ChainSyncEventType } from '@cardano-sdk/core'; @@ -61,6 +67,7 @@ export enum ProjectionName { StakePoolMetadataJob = 'stake-pool-metadata-job', StakePoolMetricsJob = 'stake-pool-metrics-job', StakePoolRewardsJob = 'stake-pool-rewards-job', + Transaction = 'transaction', UTXO = 'utxo' } @@ -106,7 +113,8 @@ const createMapperOperators = ( withNftMetadata: Mapper.withNftMetadata({ logger }), withStakeKeyRegistrations: Mapper.withStakeKeyRegistrations(), withStakePools: Mapper.withStakePools(), - withUtxo: Mapper.withUtxo() + withUtxo: Mapper.withUtxo(), + withValidByronAddresses: Mapper.withValidByronAddresses() }; }; type MapperOperators = ReturnType; @@ -117,6 +125,7 @@ export const storeOperators = { storeAddresses: storeAddresses(), storeAssets: storeAssets(), storeBlock: storeBlock(), + storeCredentials: storeCredentials(), storeGovernanceAction: storeGovernanceAction(), storeHandleMetadata: storeHandleMetadata(), storeHandles: storeHandles(), @@ -129,6 +138,7 @@ export const storeOperators = { storeStakePoolMetadataJob: createStoreStakePoolMetadataJob()(), storeStakePoolRewardsJob: storeStakePoolRewardsJob(), storeStakePools: storeStakePools(), + storeTransactions: storeTransactions(), storeUtxo: storeUtxo() }; type StoreOperators = typeof storeOperators; @@ -144,6 +154,7 @@ type WillStore = { const willStore: Partial = { storeAddresses: willStoreAddresses, storeAssets: willStoreAssets, + storeCredentials: willStoreCredentials, storeGovernanceAction: willStoreGovernanceAction, storeHandleMetadata: willStoreHandleMetadata, storeHandles: willStoreHandles, @@ -152,6 +163,7 @@ const willStore: Partial = { storeStakePoolMetadataJob: willStoreStakePoolMetadataJob, storeStakePoolRewardsJob: willStoreStakePoolRewardsJob, storeStakePools: willStoreStakePools, + storeTransactions: willStoreTransactions, storeUtxo: willStoreUtxo }; @@ -160,6 +172,7 @@ const entities = { asset: AssetEntity, block: BlockEntity, blockData: BlockDataEntity, + credential: CredentialEntity, currentPoolMetrics: CurrentPoolMetricsEntity, governanceAction: GovernanceActionEntity, handle: HandleEntity, @@ -173,7 +186,8 @@ const entities = { poolRewards: PoolRewardsEntity, stakeKeyRegistration: StakeKeyRegistrationEntity, stakePool: StakePoolEntity, - tokens: TokensEntity + tokens: TokensEntity, + transaction: TransactionEntity }; export const allEntities = Object.values(entities); type Entities = typeof entities; @@ -184,6 +198,7 @@ const storeEntities: Partial> = { storeAddresses: ['address'], storeAssets: ['asset'], storeBlock: ['block', 'blockData'], + storeCredentials: ['credential', 'transaction', 'output'], storeGovernanceAction: ['governanceAction'], storeHandleMetadata: ['handleMetadata', 'output'], storeHandles: ['handle', 'asset', 'tokens', 'output'], @@ -195,6 +210,7 @@ const storeEntities: Partial> = { storeStakePoolMetadataJob: ['stakePool', 'currentPoolMetrics', 'poolMetadata'], storeStakePoolRewardsJob: ['poolRewards', 'stakePool'], storeStakePools: ['stakePool', 'currentPoolMetrics', 'poolMetadata', 'poolDelisted'], + storeTransactions: ['block', 'transaction'], storeUtxo: ['tokens', 'output'] }; @@ -202,6 +218,7 @@ const entityInterDependencies: Partial> = { address: ['stakeKeyRegistration'], asset: ['block', 'nftMetadata'], blockData: ['block'], + credential: [], currentPoolMetrics: ['stakePool'], governanceAction: ['block'], handle: ['asset'], @@ -213,7 +230,8 @@ const entityInterDependencies: Partial> = { poolRetirement: ['block'], stakeKeyRegistration: ['block'], stakePool: ['block', 'poolRegistration', 'poolRetirement'], - tokens: ['asset'] + tokens: ['asset'], + transaction: ['block', 'credential'] }; export const getEntities = (entityNames: EntityName[]): Entity[] => { @@ -245,12 +263,14 @@ const mapperInterDependencies: Partial> = { withHandles: ['withMint', 'filterMint', 'withUtxo', 'filterUtxo', 'withCIP67'], withNftMetadata: ['withCIP67', 'withMint', 'filterMint'], withStakeKeyRegistrations: ['withCertificates'], - withStakePools: ['withCertificates'] + withStakePools: ['withCertificates'], + withValidByronAddresses: ['withUtxo'] }; const storeMapperDependencies: Partial> = { storeAddresses: ['withAddresses'], storeAssets: ['withMint'], + storeCredentials: ['withAddresses', 'withCertificates', 'withUtxo', 'withValidByronAddresses'], storeGovernanceAction: ['withGovernanceActions'], storeHandleMetadata: ['withHandleMetadata'], storeHandles: ['withHandles'], @@ -272,6 +292,7 @@ const storeInterDependencies: Partial> = { storeStakePoolMetadataJob: ['storeBlock'], storeStakePoolRewardsJob: ['storeBlock'], storeStakePools: ['storeBlock'], + storeTransactions: ['storeCredentials', 'storeBlock', 'storeUtxo'], storeUtxo: ['storeBlock', 'storeAssets'] }; @@ -286,6 +307,7 @@ const projectionStoreDependencies: Record = { 'stake-pool-metadata-job': ['storeStakePoolMetadataJob'], 'stake-pool-metrics-job': ['storePoolMetricsUpdateJob'], 'stake-pool-rewards-job': ['storeStakePoolRewardsJob'], + transaction: ['storeCredentials', 'storeTransactions'], utxo: ['storeUtxo'] }; diff --git a/packages/projection-typeorm/src/entity/Credential.entity.ts b/packages/projection-typeorm/src/entity/Credential.entity.ts new file mode 100644 index 00000000000..024eb1ceae5 --- /dev/null +++ b/packages/projection-typeorm/src/entity/Credential.entity.ts @@ -0,0 +1,26 @@ +import { Column, Entity, Index, ManyToMany, PrimaryColumn } from 'typeorm'; +import { Hash28ByteBase16 } from '@cardano-sdk/crypto'; +import { TransactionEntity } from './Transaction.entity'; + +export enum CredentialType { + PaymentKey = 'payment_key', + PaymentScript = 'payment_script', + StakeKey = 'stake_key', + StakeScript = 'stake_script' +} + +@Entity() +export class CredentialEntity { + @Index() + @PrimaryColumn('varchar') + credentialHash?: Hash28ByteBase16; + + @Column('enum', { enum: CredentialType, nullable: false }) + credentialType?: CredentialType; + + @ManyToMany(() => TransactionEntity, (transaction) => transaction.credentials, { onDelete: 'CASCADE' }) + transactions?: TransactionEntity[]; +} + +export const credentialEntityComparator = (c1: CredentialEntity, c2: CredentialEntity) => + c1.credentialHash === c2.credentialHash && c1.credentialType === c2.credentialType; diff --git a/packages/projection-typeorm/src/entity/Transaction.entity.ts b/packages/projection-typeorm/src/entity/Transaction.entity.ts new file mode 100644 index 00000000000..19bc0664fc2 --- /dev/null +++ b/packages/projection-typeorm/src/entity/Transaction.entity.ts @@ -0,0 +1,27 @@ +import { BlockEntity } from './Block.entity'; +import { Cardano, Serialization } from '@cardano-sdk/core'; +import { Column, Entity, Index, JoinColumn, JoinTable, ManyToMany, ManyToOne, PrimaryColumn } from 'typeorm'; +import { CredentialEntity } from './Credential.entity'; +import { OnDeleteCascadeRelationOptions } from './util'; + +@Entity() +export class TransactionEntity { + @Index() + @PrimaryColumn('varchar') + txId?: Cardano.TransactionId; + + @Column('varchar', { nullable: false }) + cbor?: Serialization.TxCBOR; + + @ManyToOne(() => BlockEntity, OnDeleteCascadeRelationOptions) + @JoinColumn({ name: 'block_id' }) + block?: BlockEntity; + + @ManyToMany(() => CredentialEntity, (credential) => credential.transactions, { onDelete: 'CASCADE' }) + @JoinTable({ + inverseJoinColumn: { name: 'credential_id', referencedColumnName: 'credentialHash' }, + joinColumn: { name: 'transaction_id', referencedColumnName: 'txId' }, + name: 'transaction_credentials' + }) + credentials?: CredentialEntity[]; +} diff --git a/packages/projection-typeorm/src/entity/index.ts b/packages/projection-typeorm/src/entity/index.ts index a5abf1a2940..61033e85d94 100644 --- a/packages/projection-typeorm/src/entity/index.ts +++ b/packages/projection-typeorm/src/entity/index.ts @@ -2,6 +2,7 @@ export * from './Address.entity'; export * from './Asset.entity'; export * from './Block.entity'; export * from './BlockData.entity'; +export * from './Credential.entity'; export * from './CurrentPoolMetrics.entity'; export * from './Handle.entity'; export * from './HandleMetadata.entity'; @@ -16,4 +17,5 @@ export * from './PoolRewards.entity'; export * from './StakeKey.entity'; export * from './StakeKeyRegistration.entity'; export * from './StakePool.entity'; +export * from './Transaction.entity'; export * from './Tokens.entity'; diff --git a/packages/projection-typeorm/src/operators/index.ts b/packages/projection-typeorm/src/operators/index.ts index f0e7c6394cd..6e6c6675184 100644 --- a/packages/projection-typeorm/src/operators/index.ts +++ b/packages/projection-typeorm/src/operators/index.ts @@ -1,6 +1,7 @@ export * from './storeAddresses'; export * from './storeAssets'; export * from './storeBlock'; +export * from './storeCredentials'; export * from './storeGovernanceAction'; export * from './storeHandles'; export * from './storeHandleMetadata'; @@ -11,6 +12,7 @@ export * from './storeStakeKeyRegistrations'; export * from './storeStakePools'; export * from './storeStakePoolMetadataJob'; export * from './storeStakePoolRewardsJob'; +export * from './storeTransactions'; export * from './storeUtxo'; export * from './util'; export * from './withTypeormTransaction'; diff --git a/packages/projection-typeorm/src/operators/storeCredentials.ts b/packages/projection-typeorm/src/operators/storeCredentials.ts new file mode 100644 index 00000000000..be442a9f1e8 --- /dev/null +++ b/packages/projection-typeorm/src/operators/storeCredentials.ts @@ -0,0 +1,164 @@ +import { Cardano, ChainSyncEventType } from '@cardano-sdk/core'; +import { CredentialEntity, CredentialType, OutputEntity, credentialEntityComparator } from '../entity'; +import { Hash28ByteBase16 } from '@cardano-sdk/crypto'; +import { Mappers } from '@cardano-sdk/projection'; +import { Repository } from 'typeorm'; +import { typeormOperator } from './util'; +import uniqWith from 'lodash/uniqWith.js'; + +export interface WithTxCredentials { + credentialsByTx: Record; +} + +export const willStoreCredentials = ({ utxoByTx }: Mappers.WithUtxo) => Object.keys(utxoByTx).length > 0; + +const addInputCredentials = async ( + utxoByTx: Record, + utxoRepository: Repository, + addCredentialFromAddress: (txId: Cardano.TransactionId, address: Mappers.Address) => void +) => { + for (const txHash of Object.keys(utxoByTx) as Cardano.TransactionId[]) { + const txInLookups = utxoByTx[txHash].consumed.map(({ txId, index: outputIndex }) => ({ + outputIndex, + txId + })); + + const outputEntities = await utxoRepository.find({ + select: { address: true, outputIndex: true, txId: true }, + where: txInLookups.map(({ txId, outputIndex }) => ({ + outputIndex, + txId + })) + }); + + for (const hydratedTxIn of outputEntities) { + if (hydratedTxIn.address) { + addCredentialFromAddress(txHash, Mappers.credentialsFromAddress(hydratedTxIn.address)); + } + } + } +}; + +const addOutputCredentials = ( + addressesByTx: Record, + addCredentialFromAddress: (txId: Cardano.TransactionId, address: Mappers.Address) => void +) => { + for (const txId of Object.keys(addressesByTx) as Cardano.TransactionId[]) { + for (const address of addressesByTx[txId]) { + addCredentialFromAddress(txId, address); + } + } +}; + +const addCertificateCredentials = ( + credentialsByTx: Record, + addCredential: ( + txId: Cardano.TransactionId, + credentialHash: Hash28ByteBase16, + credentialType: CredentialType + ) => Map +) => { + for (const txId of Object.keys(credentialsByTx) as Cardano.TransactionId[]) { + for (const credential of credentialsByTx[txId]) { + addCredential( + txId, + credential.hash, + credential.type === 0 ? CredentialType.StakeKey : CredentialType.StakeScript + ); + } + } +}; + +type AddressPart = 'payment' | 'stake'; +const credentialTypeMap: { [key: number]: { payment: CredentialType | null; stake: CredentialType } } = { + [Cardano.AddressType.BasePaymentKeyStakeKey]: { payment: CredentialType.PaymentKey, stake: CredentialType.StakeKey }, + [Cardano.AddressType.EnterpriseKey]: { payment: CredentialType.PaymentKey, stake: CredentialType.StakeKey }, + [Cardano.AddressType.BasePaymentKeyStakeScript]: { + payment: CredentialType.PaymentKey, + stake: CredentialType.StakeScript + }, + [Cardano.AddressType.BasePaymentScriptStakeKey]: { + payment: CredentialType.PaymentScript, + stake: CredentialType.StakeKey + }, + [Cardano.AddressType.BasePaymentScriptStakeScript]: { + payment: CredentialType.PaymentScript, + stake: CredentialType.StakeScript + }, + [Cardano.AddressType.EnterpriseScript]: { payment: CredentialType.PaymentScript, stake: CredentialType.StakeScript }, + [Cardano.AddressType.RewardKey]: { payment: null, stake: CredentialType.StakeKey }, + [Cardano.AddressType.RewardScript]: { payment: null, stake: CredentialType.StakeScript } +}; + +export const storeCredentials = typeormOperator< + Mappers.WithUtxo & Mappers.WithAddresses & Mappers.WithCertificates, + WithTxCredentials +>(async (evt) => { + const { + addressesByTx, + block: { body: txs }, + eventType, + queryRunner, + stakeCredentialsByTx, + utxoByTx + } = evt; + + const txToCredentials = new Map(); + + // produced credentials will be automatically deleted via block cascade + if (txs.length === 0 || eventType !== ChainSyncEventType.RollForward) { + return { credentialsByTx: Object.fromEntries(txToCredentials) }; + } + const utxoRepository = queryRunner.manager.getRepository(OutputEntity); + const addCredential = ( + txId: Cardano.TransactionId, + credentialHash: Hash28ByteBase16, + credentialType: CredentialType + ) => + txToCredentials.set( + txId, + uniqWith([...(txToCredentials.get(txId) || []), { credentialHash, credentialType }], credentialEntityComparator) + ); + + const credentialTypeFromAddressType = (type: Cardano.AddressType, part: AddressPart) => { + const credential = credentialTypeMap[type]; + if (!credential) { + // FIXME: map byron address, pointer script, pointer key type + return null; + } + return credential[part]; + }; + + const addCredentialFromAddress = ( + txId: Cardano.TransactionId, + { paymentCredentialHash, stakeCredential, type }: Mappers.Address + ) => { + const paymentCredentialType = credentialTypeFromAddressType(type, 'payment'); + if (paymentCredentialHash && paymentCredentialType) { + addCredential(txId, paymentCredentialHash, paymentCredentialType); + } + + if (stakeCredential) { + const stakeCredentialType = credentialTypeFromAddressType(type, 'stake'); + // FIXME: support pointers + if (stakeCredentialType && typeof stakeCredential === 'string') { + addCredential(txId, stakeCredential, stakeCredentialType); + } + } + }; + + await addInputCredentials(utxoByTx, utxoRepository, addCredentialFromAddress); + addOutputCredentials(addressesByTx, addCredentialFromAddress); + addCertificateCredentials(stakeCredentialsByTx, addCredential); + + // insert new credentials & ignore conflicts of existing ones + await queryRunner.manager + .createQueryBuilder() + .insert() + .into(CredentialEntity) + .values([...txToCredentials.values()].flat()) + .orIgnore() + .execute(); + + return { credentialsByTx: Object.fromEntries(txToCredentials) }; +}); diff --git a/packages/projection-typeorm/src/operators/storeTransactions.ts b/packages/projection-typeorm/src/operators/storeTransactions.ts new file mode 100644 index 00000000000..7aeb0cd657c --- /dev/null +++ b/packages/projection-typeorm/src/operators/storeTransactions.ts @@ -0,0 +1,60 @@ +import { Cardano, ChainSyncEventType } from '@cardano-sdk/core'; +import { Hash28ByteBase16 } from '@cardano-sdk/crypto'; +import { TransactionEntity } from '../entity'; +import { WithBlock } from '@cardano-sdk/projection'; +import { WithTxCredentials } from './storeCredentials'; +import { typeormOperator } from './util'; + +export const willStoreTransactions = ({ block: { body } }: WithBlock) => body.length > 0; + +export const storeTransactions = typeormOperator(async (evt) => { + const { + block: { body: txs, header }, + credentialsByTx, + eventType, + queryRunner + } = evt; + + // produced txs will be automatically deleted via block cascade + if (txs.length === 0 || eventType !== ChainSyncEventType.RollForward) return; + + const transactionEntities = new Array(); + for (const tx of txs) { + const credentials = credentialsByTx[tx.id] || []; + const txEntity: TransactionEntity = { + block: header, + cbor: tx.cbor, + credentials, + txId: tx.id + }; + transactionEntities.push(txEntity); + } + + await queryRunner.manager + .createQueryBuilder() + .insert() + .into(TransactionEntity) + .values(transactionEntities) + .orIgnore() + .execute(); + + // Bulk insert relationships + await queryRunner.manager + .createQueryBuilder() + .insert() + .into('transaction_credentials') + .values( + Object.entries(credentialsByTx).reduce( + (arr, [txId, credentials]) => [ + ...arr, + ...credentials.map((credential) => ({ + credential_id: credential.credentialHash!, + transaction_id: txId as Cardano.TransactionId + })) + ], + new Array<{ transaction_id: Cardano.TransactionId; credential_id: Hash28ByteBase16 }>() + ) + ) + .orIgnore() + .execute(); +}); diff --git a/packages/projection/src/operators/Mappers/certificates/withCertificates.ts b/packages/projection/src/operators/Mappers/certificates/withCertificates.ts index 64f44f44898..963a55aa2fb 100644 --- a/packages/projection/src/operators/Mappers/certificates/withCertificates.ts +++ b/packages/projection/src/operators/Mappers/certificates/withCertificates.ts @@ -1,6 +1,6 @@ import { Cardano } from '@cardano-sdk/core'; -import { WithBlock } from '../../../types'; import { unifiedProjectorOperator } from '../../utils'; +import uniqWith from 'lodash/uniqWith.js'; export interface OnChainCertificate { pointer: Cardano.Pointer; @@ -9,29 +9,61 @@ export interface OnChainCertificate { export interface WithCertificates { certificates: OnChainCertificate[]; + stakeCredentialsByTx: Record; } -const blockCertificates = ({ - block: { +const isNotPhase2ValidationErrorTx = (tx: Cardano.OnChainTx) => + !Cardano.util.isPhase2ValidationErrTx(tx); + +const credentialComparator = (c1: Cardano.Credential, c2: Cardano.Credential) => + c1.hash === c2.hash && c1.type === c2.type; + +/** Adds flat array of certificates to event as well as a record of stake credentials grouped by transaction id. */ +export const withCertificates = unifiedProjectorOperator<{}, WithCertificates>((evt) => { + let blockCertificates: OnChainCertificate[] = []; + const txToStakeCredentials = new Map(); + + const { header: { slot }, body - } -}: WithBlock) => - body - .filter((tx) => !Cardano.util.isPhase2ValidationErrTx(tx)) - .flatMap(({ body: { certificates = [] } }, txIndex) => - certificates.map((certificate, certIndex) => ({ + } = evt.block; + const txs = body.filter(isNotPhase2ValidationErrorTx); + + const addCredential = (txId: Cardano.TransactionId, credential: Cardano.Credential) => + txToStakeCredentials.set( + txId, + uniqWith([...(txToStakeCredentials.get(txId) || []), credential], credentialComparator) + ); + + for (const [ + txIndex, + { + id: txId, + body: { certificates = [] } + } + ] of txs.filter(isNotPhase2ValidationErrorTx).entries()) { + const certs = new Array(); + + for (const [certIndex, certificate] of certificates.entries()) { + certs.push({ certificate, pointer: { certIndex: Cardano.CertIndex(certIndex), slot: BigInt(slot), txIndex: Cardano.TxIndex(txIndex) } - })) - ); + }); + + if ('stakeCredential' in certificate && certificate.stakeCredential) { + addCredential(txId, certificate.stakeCredential); + } + } + blockCertificates = [...blockCertificates, ...certs]; + } -/** Map ChainSyncEvents to a flat array of certificates. */ -export const withCertificates = unifiedProjectorOperator<{}, WithCertificates>((evt) => ({ - ...evt, - certificates: blockCertificates(evt) -})); + return { + ...evt, + certificates: blockCertificates, + stakeCredentialsByTx: Object.fromEntries(txToStakeCredentials) + }; +}); diff --git a/packages/projection/src/operators/Mappers/index.ts b/packages/projection/src/operators/Mappers/index.ts index 854ef8626fd..fcc3281c6da 100644 --- a/packages/projection/src/operators/Mappers/index.ts +++ b/packages/projection/src/operators/Mappers/index.ts @@ -1,9 +1,11 @@ export * from './certificates'; -export * from './withUtxo'; -export * from './withMint'; +export * from './withAddresses'; +export * from './withCIP67'; export * from './withGovernanceActions'; export * from './withHandles'; export * from './withHandleMetadata'; +export * from './withMint'; export * from './withNftMetadata'; -export * from './withCIP67'; -export * from './withAddresses'; +export * from './withUtxo'; +export * from './withValidByronAddresses'; +export { credentialsFromAddress } from './util'; diff --git a/packages/projection/src/operators/Mappers/util.ts b/packages/projection/src/operators/Mappers/util.ts index 4aeaeac24d4..520fb7a0e8f 100644 --- a/packages/projection/src/operators/Mappers/util.ts +++ b/packages/projection/src/operators/Mappers/util.ts @@ -1,4 +1,6 @@ +import { Address } from './withAddresses'; import { Asset, Cardano, Handle } from '@cardano-sdk/core'; +import { Hash28ByteBase16 } from '@cardano-sdk/crypto'; import { Logger } from 'ts-log'; /** Up to 100k transactions per block. Fits in 64-bit signed integer. */ @@ -12,3 +14,41 @@ export const assetNameToUTF8Handle = (assetName: Cardano.AssetName, logger: Logg } return handle; }; + +export const credentialsFromAddress = (address: Cardano.PaymentAddress): Address => { + const parsed = Cardano.Address.fromString(address)!; + let paymentCredentialHash: Hash28ByteBase16 | undefined; + let stakeCredentialHash: Hash28ByteBase16 | undefined; + let pointer: Cardano.Pointer | undefined; + const type = parsed.getType(); + switch (type) { + case Cardano.AddressType.BasePaymentKeyStakeKey: + case Cardano.AddressType.BasePaymentKeyStakeScript: + case Cardano.AddressType.BasePaymentScriptStakeKey: + case Cardano.AddressType.BasePaymentScriptStakeScript: { + const baseAddress = parsed.asBase()!; + paymentCredentialHash = baseAddress.getPaymentCredential().hash; + stakeCredentialHash = baseAddress.getStakeCredential().hash; + break; + } + case Cardano.AddressType.EnterpriseKey: + case Cardano.AddressType.EnterpriseScript: { + const enterpriseAddress = parsed.asEnterprise()!; + paymentCredentialHash = enterpriseAddress.getPaymentCredential().hash; + break; + } + case Cardano.AddressType.PointerKey: + case Cardano.AddressType.PointerScript: { + const pointerAddress = parsed.asPointer()!; + paymentCredentialHash = pointerAddress.getPaymentCredential().hash; + pointer = pointerAddress.getStakePointer(); + break; + } + } + return { + address, + paymentCredentialHash, + stakeCredential: stakeCredentialHash || pointer, + type + }; +}; diff --git a/packages/projection/src/operators/Mappers/withAddresses.ts b/packages/projection/src/operators/Mappers/withAddresses.ts index 02ba2a4ec2c..ed87db96e6f 100644 --- a/packages/projection/src/operators/Mappers/withAddresses.ts +++ b/packages/projection/src/operators/Mappers/withAddresses.ts @@ -1,6 +1,7 @@ import { Cardano } from '@cardano-sdk/core'; import { Hash28ByteBase16 } from '@cardano-sdk/crypto'; import { WithUtxo } from './withUtxo'; +import { credentialsFromAddress } from './util'; import { unifiedProjectorOperator } from '../utils'; import uniq from 'lodash/uniq.js'; @@ -15,46 +16,24 @@ export interface Address { export interface WithAddresses { addresses: Address[]; + addressesByTx: Record; } /** Collect all unique addresses from produced utxo */ -export const withAddresses = unifiedProjectorOperator((evt) => ({ - ...evt, - addresses: uniq(evt.utxo.produced.map(([_, txOut]) => txOut.address)).map((address): Address => { - const parsed = Cardano.Address.fromString(address)!; - let paymentCredentialHash: Hash28ByteBase16 | undefined; - let stakeCredentialHash: Hash28ByteBase16 | undefined; - let pointer: Cardano.Pointer | undefined; - const type = parsed.getType(); - switch (type) { - case Cardano.AddressType.BasePaymentKeyStakeKey: - case Cardano.AddressType.BasePaymentKeyStakeScript: - case Cardano.AddressType.BasePaymentScriptStakeKey: - case Cardano.AddressType.BasePaymentScriptStakeScript: { - const baseAddress = parsed.asBase()!; - paymentCredentialHash = baseAddress.getPaymentCredential().hash; - stakeCredentialHash = baseAddress.getStakeCredential().hash; - break; - } - case Cardano.AddressType.EnterpriseKey: - case Cardano.AddressType.EnterpriseScript: { - const enterpriseAddress = parsed.asEnterprise()!; - paymentCredentialHash = enterpriseAddress.getPaymentCredential().hash; - break; - } - case Cardano.AddressType.PointerKey: - case Cardano.AddressType.PointerScript: { - const pointerAddress = parsed.asPointer()!; - paymentCredentialHash = pointerAddress.getPaymentCredential().hash; - pointer = pointerAddress.getStakePointer(); - break; - } - } - return { - address, - paymentCredentialHash, - stakeCredential: stakeCredentialHash || pointer, - type - }; - }) -})); +export const withAddresses = unifiedProjectorOperator((evt) => { + const addressesByTx = { + ...Object.entries(evt.utxoByTx).reduce( + (map, [txId, utxo]) => ({ + ...map, + [txId]: uniq(utxo.produced.map(([_, txOut]) => txOut.address)).map(credentialsFromAddress) + }), + new Map() + ) + } as Record; + + return { + ...evt, + addresses: uniq(Object.values(addressesByTx).flat()), + addressesByTx + }; +}); diff --git a/packages/projection/src/operators/Mappers/withUtxo.ts b/packages/projection/src/operators/Mappers/withUtxo.ts index 5eac409e28d..7c5ef0fec0d 100644 --- a/packages/projection/src/operators/Mappers/withUtxo.ts +++ b/packages/projection/src/operators/Mappers/withUtxo.ts @@ -6,30 +6,51 @@ import { unifiedProjectorOperator } from '../utils'; export type ProducedUtxo = [Cardano.TxIn, Cardano.TxOut]; +export interface WithProducedUTxO { + produced: Array; +} +export interface WithConsumedTxIn { + /** Refers to `compactUtxoId` of a previously produced utxo */ + consumed: Cardano.TxIn[]; +} export interface WithUtxo { - utxo: { - produced: Array; - /** Refers to `compactUtxoId` of a previously produced utxo */ - consumed: Cardano.TxIn[]; - }; + /** Complete utxo set from block including all transactions */ + utxo: WithConsumedTxIn & WithProducedUTxO; + /** Utxo set grouped by transaction id */ + utxoByTx: Record; } export const withUtxo = unifiedProjectorOperator<{}, WithUtxo>((evt) => { - const produced = evt.block.body.flatMap(({ body: { outputs, collateralReturn }, inputSource, id }) => - (inputSource === Cardano.InputSource.inputs ? outputs : collateralReturn ? [collateralReturn] : []).map( - (txOut, outputIndex): [Cardano.TxIn, Cardano.TxOut] => [ - { - index: outputIndex, - txId: id - }, - txOut - ] - ) - ); - const consumed = evt.block.body.flatMap(({ body: { inputs, collaterals }, inputSource }) => - inputSource === Cardano.InputSource.inputs ? inputs : collaterals || [] - ); - return { ...evt, utxo: { consumed, produced } }; + const txToUtxos = new Map(); + + for (const { + body: { collaterals, inputs, outputs, collateralReturn }, + inputSource, + id + } of evt.block.body) { + txToUtxos.set(id, { + consumed: inputSource === Cardano.InputSource.inputs ? inputs : collaterals || [], + produced: (inputSource === Cardano.InputSource.inputs ? outputs : collateralReturn ? [collateralReturn] : []).map( + (txOut, outputIndex): [Cardano.TxIn, Cardano.TxOut] => [ + { + index: outputIndex, + txId: id + }, + txOut + ] + ) + }); + } + + const utxoByTx = Object.fromEntries(txToUtxos); + return { + ...evt, + utxo: { + consumed: Object.values(utxoByTx).flatMap((tx) => tx.consumed), + produced: Object.values(utxoByTx).flatMap((tx) => tx.produced) + }, + utxoByTx + }; }); export interface FilterByPaymentAddresses { @@ -40,10 +61,23 @@ export const filterProducedUtxoByAddresses = ({ addresses }: FilterByPaymentAddresses): ProjectionOperator => (evt$) => evt$.pipe( - map((evt) => ({ - ...evt, - utxo: { ...evt.utxo, produced: evt.utxo.produced.filter(([_, { address }]) => addresses.includes(address)) } - })) + map((evt) => { + const filteredTxs: Record = Object.fromEntries( + Object.entries(evt.utxoByTx).reduce((txToUtxo, [txId, { consumed, produced }]) => { + txToUtxo.set(Cardano.TransactionId(txId), { + consumed, + produced: produced.filter(([_, { address }]) => addresses.includes(address)) + }); + return txToUtxo; + }, new Map()) + ); + + return { + ...evt, + utxo: { ...evt.utxo, produced: Object.values(filteredTxs).flatMap((utxos) => utxos.produced) }, + utxoByTx: filteredTxs + }; + }) ); export const filterProducedUtxoByAssetsPresence = @@ -56,7 +90,20 @@ export const filterProducedUtxoByAssetsPresence = utxo: { ...evt.utxo, produced: evt.utxo.produced.filter(([_, { value }]) => value.assets && value.assets.size > 0) - } + }, + utxoByTx: { + ...evt.utxoByTx, + ...Object.entries(evt.utxoByTx).reduce( + (txToUtxo, [txId, utxos]) => ({ + ...txToUtxo, + [txId]: { + ...utxos, + produced: utxos.produced.filter(([_, { value }]) => value.assets && value.assets.size > 0) + } + }), + new Map() + ) + } as Record })) ); @@ -96,6 +143,37 @@ export const filterProducedUtxoByAssetPolicyId = } ]) => assets && assets.size > 0 ) + }, + utxoByTx: { + ...evt.utxoByTx, + ...Object.entries(evt.utxoByTx).reduce( + (txToUtxo, [txId, utxos]) => ({ + ...txToUtxo, + [txId]: { + ...utxos, + produced: { + ...utxos.produced, + ...utxos.produced.map(([txIn, txOut]) => [ + txIn, + { + ...txOut, + value: { + ...txOut.value, + assets: txOut.value.assets + ? new Map( + [...txOut.value.assets.entries()].filter(([assetId]) => + policyIds.includes(Cardano.AssetId.getPolicyId(assetId)) + ) + ) + : undefined + } + } + ]) + } + } + }), + new Map() + ) } })) ); diff --git a/packages/projection/src/operators/Mappers/withValidByronAddresses.ts b/packages/projection/src/operators/Mappers/withValidByronAddresses.ts new file mode 100644 index 00000000000..7d8434158a9 --- /dev/null +++ b/packages/projection/src/operators/Mappers/withValidByronAddresses.ts @@ -0,0 +1,61 @@ +import * as Crypto from '@cardano-sdk/crypto'; +import { Cardano } from '@cardano-sdk/core'; +import { WithConsumedTxIn, WithProducedUTxO, WithUtxo } from './withUtxo'; +import { unifiedProjectorOperator } from '../utils'; + +/** + * Byron addresses in general do NOT define a maximum length. + * This upper limit originates from the maximum length of an index row defined + * in postgres. + */ +const MAX_BYRON_OUTPUT_ADDRESS_BYTES_LENGTH = 8191; +const ICARUS_ADDR_BECH32_PREFIX = 'Ae2'; +const DAEDALUS_ADDR_BECH32_PREFIX = 'DdzFF'; + +const isBFT = ({ type }: Cardano.Block) => type === 'bft'; +const hasByronAddressPrefix = (address: string): boolean => + address.startsWith(ICARUS_ADDR_BECH32_PREFIX) || address.startsWith(DAEDALUS_ADDR_BECH32_PREFIX); + +const transformByronAddress = (address: Cardano.PaymentAddress) => { + if (!hasByronAddressPrefix(address) || address.length > MAX_BYRON_OUTPUT_ADDRESS_BYTES_LENGTH) { + const byronAddress = Cardano.Address.fromBase58(address); + const keyHashBytes = Buffer.from(byronAddress.toBytes(), 'hex'); + const byronBase16CredentialHash = Crypto.Hash28ByteBase16(Crypto.blake2b(28).update(keyHashBytes).digest('hex')); + return Cardano.ByronAddress.fromCredentials(byronBase16CredentialHash, {}, 0).toAddress().toBase58(); + } + return address; +}; + +/** + * This mapper transforms invalid (very long) Byron output addresses by re-hashing them + * such so their length does not exceed the maximum defined row index of postgres. + * + * Example Tx (Mainnet): + * {@link https://cardanoscan.io/transaction/bc61865d72bd8a0956f1b12595e314a60cc8e3f4350c044b2a86f3230ace923a?tab=summary bc61865d72bd8a0956f1b12595e314a60cc8e3f4350c044b2a86f3230ace923a} + */ +export const withValidByronAddresses = unifiedProjectorOperator((evt) => { + if (isBFT(evt.block)) { + const txToUtxos = new Map(); + + for (const txId of Object.keys(evt.utxoByTx) as Cardano.TransactionId[]) { + txToUtxos.set(txId, { + consumed: evt.utxoByTx[txId]!.consumed, + produced: evt.utxoByTx[txId]!.produced.map(([txIn, txOut]): [Cardano.TxIn, Cardano.TxOut] => [ + txIn, + { ...txOut, address: transformByronAddress(txOut.address) } + ]) + }); + } + + const utxoByTx = Object.fromEntries(txToUtxos); + return { + ...evt, + utxo: { + consumed: evt.utxo.consumed, + produced: Object.values(utxoByTx).flatMap((tx) => tx.produced) + }, + utxoByTx + }; + } + return evt; +}); diff --git a/packages/projection/test/operators/Mappers/certificates/withCertificates.test.ts b/packages/projection/test/operators/Mappers/certificates/withCertificates.test.ts index dea75208440..bfa73dd870d 100644 --- a/packages/projection/test/operators/Mappers/certificates/withCertificates.test.ts +++ b/packages/projection/test/operators/Mappers/certificates/withCertificates.test.ts @@ -67,11 +67,13 @@ describe('withCertificates', () => { txIndex: 1 } } - ] + ], + stakeCredentialsByTx: {} }, b: { ...createEvent(ChainSyncEventType.RollForward, Cardano.Slot(2), []), - certificates: [] + certificates: [], + stakeCredentialsByTx: {} }, c: { ...createEvent( @@ -80,7 +82,8 @@ describe('withCertificates', () => { certificates, Cardano.InputSource.collaterals ), - certificates: [] + certificates: [], + stakeCredentialsByTx: {} } }); expectSubscriptions(source$.subscriptions).toBe('^'); diff --git a/packages/projection/test/operators/Mappers/certificates/withStakeKeyRegistrations.test.ts b/packages/projection/test/operators/Mappers/certificates/withStakeKeyRegistrations.test.ts index 9d3cd62641c..66f14612b1d 100644 --- a/packages/projection/test/operators/Mappers/certificates/withStakeKeyRegistrations.test.ts +++ b/packages/projection/test/operators/Mappers/certificates/withStakeKeyRegistrations.test.ts @@ -35,7 +35,8 @@ describe('withStakeKeyRegistrations', () => { pointer: {} as Cardano.Pointer } ], - eventType: ChainSyncEventType.RollForward + eventType: ChainSyncEventType.RollForward, + stakeCredentialsByTx: {} }; const result = await firstValueFrom( diff --git a/packages/projection/test/operators/Mappers/certificates/withStakeKeys.test.ts b/packages/projection/test/operators/Mappers/certificates/withStakeKeys.test.ts index 8e745bff866..8d154d9528b 100644 --- a/packages/projection/test/operators/Mappers/certificates/withStakeKeys.test.ts +++ b/packages/projection/test/operators/Mappers/certificates/withStakeKeys.test.ts @@ -33,7 +33,8 @@ describe('withStakeKeys', () => { pointer: {} as Cardano.Pointer } ], - eventType: ChainSyncEventType.RollForward + eventType: ChainSyncEventType.RollForward, + stakeCredentialsByTx: {} }; const result = await firstValueFrom( @@ -68,7 +69,8 @@ describe('withStakeKeys', () => { pointer: {} as Cardano.Pointer } ], - eventType: ChainSyncEventType.RollBackward + eventType: ChainSyncEventType.RollBackward, + stakeCredentialsByTx: {} }; const result = await firstValueFrom( @@ -102,7 +104,8 @@ describe('withStakeKeys', () => { pointer: {} as Cardano.Pointer } ], - eventType: ChainSyncEventType.RollForward + eventType: ChainSyncEventType.RollForward, + stakeCredentialsByTx: {} }; const result = await firstValueFrom( Mappers.withStakeKeys()(of(data as UnifiedExtChainSyncEvent)) @@ -135,7 +138,8 @@ describe('withStakeKeys', () => { pointer: {} as Cardano.Pointer } ], - eventType: ChainSyncEventType.RollForward + eventType: ChainSyncEventType.RollForward, + stakeCredentialsByTx: {} }; const result = await firstValueFrom( Mappers.withStakeKeys()(of(data as UnifiedExtChainSyncEvent)) diff --git a/packages/projection/test/operators/Mappers/certificates/withStakePools.test.ts b/packages/projection/test/operators/Mappers/certificates/withStakePools.test.ts index 58e72efa3fb..16a2c267932 100644 --- a/packages/projection/test/operators/Mappers/certificates/withStakePools.test.ts +++ b/packages/projection/test/operators/Mappers/certificates/withStakePools.test.ts @@ -53,7 +53,8 @@ describe('withStakePools', () => { pointer: {} as Cardano.Pointer } ], - epochNo + epochNo, + stakeCredentialsByTx: {} }; const result = await firstValueFrom( Mappers.withStakePools()(of(data as UnifiedExtChainSyncEvent)) @@ -91,7 +92,8 @@ describe('withStakePools', () => { pointer: {} as Cardano.Pointer } ], - epochNo + epochNo, + stakeCredentialsByTx: {} }; const result = await firstValueFrom( Mappers.withStakePools()(of(data as UnifiedExtChainSyncEvent)) diff --git a/packages/projection/test/operators/Mappers/withAddresses.test.ts b/packages/projection/test/operators/Mappers/withAddresses.test.ts index 2aadb8b5790..43aeefbdea9 100644 --- a/packages/projection/test/operators/Mappers/withAddresses.test.ts +++ b/packages/projection/test/operators/Mappers/withAddresses.test.ts @@ -1,18 +1,26 @@ import { Cardano } from '@cardano-sdk/core'; +import { ProducedUtxo, WithUtxo } from '../../../src/operators/Mappers'; import { ProjectionEvent } from '../../../src'; -import { WithUtxo } from '../../../src/operators/Mappers'; import { cip19TestVectors, generateRandomHexString } from '@cardano-sdk/util-dev'; import { firstValueFrom, of } from 'rxjs'; import { withAddresses } from '../../../src/operators/Mappers/withAddresses'; const projectEvent = async (addresses: Cardano.PaymentAddress[]) => { + const producedOutputs = addresses.map( + (address): ProducedUtxo => [ + { index: 0, txId: Cardano.TransactionId(generateRandomHexString(64)) }, + { address, value: { coins: 123n } } + ] + ); const event = { utxo: { - produced: addresses.map((address) => [ - { index: 0, txId: generateRandomHexString(64) }, - { address, value: { coins: 123n } } - ]) - } as WithUtxo['utxo'] + produced: producedOutputs + }, + utxoByTx: { + [Cardano.TransactionId(generateRandomHexString(64))]: { + produced: producedOutputs + } + } } as ProjectionEvent; return firstValueFrom(of(event).pipe(withAddresses())); }; diff --git a/packages/projection/test/operators/Mappers/withUtxo.test.ts b/packages/projection/test/operators/Mappers/withUtxo.test.ts index 98234ddb86d..32c5c003002 100644 --- a/packages/projection/test/operators/Mappers/withUtxo.test.ts +++ b/packages/projection/test/operators/Mappers/withUtxo.test.ts @@ -43,6 +43,7 @@ export const validTxSource$ = of({ } ] }, + id: Cardano.TransactionId('1'.repeat(64)), inputSource: Cardano.InputSource.inputs }, { @@ -71,6 +72,7 @@ export const validTxSource$ = of({ } ] }, + id: Cardano.TransactionId('2'.repeat(64)), inputSource: Cardano.InputSource.inputs }, { @@ -108,6 +110,7 @@ export const validTxSource$ = of({ } ] }, + id: Cardano.TransactionId('3'.repeat(64)), inputSource: Cardano.InputSource.inputs } ] @@ -151,6 +154,7 @@ describe('withUtxo', () => { } ] }, + id: Cardano.TransactionId('1'.repeat(64)), inputSource: Cardano.InputSource.collaterals }, { @@ -200,6 +204,7 @@ describe('withUtxo', () => { } ] }, + id: Cardano.TransactionId('2'.repeat(64)), inputSource: Cardano.InputSource.collaterals } ] @@ -208,26 +213,55 @@ describe('withUtxo', () => { it('maps all produced and consumed utxo into flat arrays', async () => { const { - utxo: { consumed, produced } + utxo: { consumed, produced }, + utxoByTx } = await firstValueFrom(validTxSource$.pipe(withUtxo())); expect(consumed).toHaveLength(4); expect(produced).toHaveLength(5); + + expect(Object.keys(utxoByTx)).toHaveLength(3); + + const tx1 = utxoByTx[Cardano.TransactionId('1'.repeat(64))]; + expect(tx1.consumed).toHaveLength(2); + expect(tx1.produced).toHaveLength(1); + + const tx2 = utxoByTx[Cardano.TransactionId('2'.repeat(64))]; + expect(tx2.consumed).toHaveLength(1); + expect(tx2.produced).toHaveLength(2); + + const tx3 = utxoByTx[Cardano.TransactionId('3'.repeat(64))]; + expect(tx3.consumed).toHaveLength(1); + expect(tx3.produced).toHaveLength(2); }); it('when inputSource is collateral: maps consumed/produced utxo from collateral/collateralReturn', async () => { const { - utxo: { consumed, produced } + utxo: { consumed, produced }, + utxoByTx } = await firstValueFrom(failedTxSource$.pipe(withUtxo())); expect(consumed).toHaveLength(2); expect(produced).toHaveLength(1); expect(consumed[0].index).toBe(2); expect(produced[0][1].address).toBe('addr_test1vptwv4jvaqt635jvthpa29lww3vkzypm8l6vk4lv4tqfhhgajdgwf'); + + expect(Object.keys(utxoByTx)).toHaveLength(2); + + const tx1 = utxoByTx[Cardano.TransactionId('1'.repeat(64))]; + expect(tx1.consumed).toHaveLength(1); + expect(tx1.consumed[0].index).toBe(2); + expect(tx1.produced).toHaveLength(0); + + const tx2 = utxoByTx[Cardano.TransactionId('2'.repeat(64))]; + expect(tx2.consumed).toHaveLength(1); + expect(tx2.produced[0][1].address).toBe('addr_test1vptwv4jvaqt635jvthpa29lww3vkzypm8l6vk4lv4tqfhhgajdgwf'); + expect(tx2.produced).toHaveLength(1); }); describe('filterProducedUtxoByAddresses', () => { it('keeps only utxo produced for supplied addresses', async () => { const { - utxo: { produced } + utxo: { produced }, + utxoByTx } = await firstValueFrom( validTxSource$.pipe( withUtxo(), @@ -242,6 +276,14 @@ describe('withUtxo', () => { ); expect(produced).toHaveLength(2); + expect(Object.keys(utxoByTx)).toHaveLength(3); + expect(Object.values(utxoByTx).filter((utxos) => utxos.produced.length > 0)).toHaveLength(2); + + const tx1 = utxoByTx[Cardano.TransactionId('2'.repeat(64))]; + expect(tx1.produced).toHaveLength(1); + + const tx2 = utxoByTx[Cardano.TransactionId('3'.repeat(64))]; + expect(tx2.produced).toHaveLength(1); }); }); From 2d388fa48d3ac69ed9c1e1a2dd431ae0be772947 Mon Sep 17 00:00:00 2001 From: William Wolff Date: Wed, 21 Aug 2024 18:41:37 +0200 Subject: [PATCH 03/29] refactor: add transactions for projector --- compose/common.yml | 2 +- .../cardano-services/src/Projection/prepareTypeormProjection.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/common.yml b/compose/common.yml index b1c5d7d8144..02df8c7eed7 100644 --- a/compose/common.yml +++ b/compose/common.yml @@ -312,7 +312,7 @@ services: - *projector-environment - *sdk-environment POSTGRES_DB_FILE: /run/secrets/postgres_db_wallet_api - PROJECTION_NAMES: protocol-parameters + PROJECTION_NAMES: protocol-parameters, transactions ports: - ${WALLET_API_PROJECTOR_PORT:-4005}:3000 diff --git a/packages/cardano-services/src/Projection/prepareTypeormProjection.ts b/packages/cardano-services/src/Projection/prepareTypeormProjection.ts index 3a4751ebd25..7a6f28dd25c 100644 --- a/packages/cardano-services/src/Projection/prepareTypeormProjection.ts +++ b/packages/cardano-services/src/Projection/prepareTypeormProjection.ts @@ -67,7 +67,7 @@ export enum ProjectionName { StakePoolMetadataJob = 'stake-pool-metadata-job', StakePoolMetricsJob = 'stake-pool-metrics-job', StakePoolRewardsJob = 'stake-pool-rewards-job', - Transaction = 'transaction', + Transactions = 'transactions', UTXO = 'utxo' } From 0e9f51c669ff2e9f2cabed329aa1fb549a96c234 Mon Sep 17 00:00:00 2001 From: William Wolff Date: Thu, 22 Aug 2024 10:00:38 +0200 Subject: [PATCH 04/29] fix: rename projection --- .../cardano-services/src/Projection/prepareTypeormProjection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cardano-services/src/Projection/prepareTypeormProjection.ts b/packages/cardano-services/src/Projection/prepareTypeormProjection.ts index 7a6f28dd25c..6b5ac97c5e5 100644 --- a/packages/cardano-services/src/Projection/prepareTypeormProjection.ts +++ b/packages/cardano-services/src/Projection/prepareTypeormProjection.ts @@ -307,7 +307,7 @@ const projectionStoreDependencies: Record = { 'stake-pool-metadata-job': ['storeStakePoolMetadataJob'], 'stake-pool-metrics-job': ['storePoolMetricsUpdateJob'], 'stake-pool-rewards-job': ['storeStakePoolRewardsJob'], - transaction: ['storeCredentials', 'storeTransactions'], + transactions: ['storeCredentials', 'storeTransactions'], utxo: ['storeUtxo'] }; From e434b5a2b1a734f788c8956f21927910783cbab3 Mon Sep 17 00:00:00 2001 From: William Wolff Date: Thu, 22 Aug 2024 11:03:10 +0200 Subject: [PATCH 05/29] fix: whitespace for projections --- compose/common.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/common.yml b/compose/common.yml index 02df8c7eed7..f51d7ba1816 100644 --- a/compose/common.yml +++ b/compose/common.yml @@ -312,7 +312,7 @@ services: - *projector-environment - *sdk-environment POSTGRES_DB_FILE: /run/secrets/postgres_db_wallet_api - PROJECTION_NAMES: protocol-parameters, transactions + PROJECTION_NAMES: protocol-parameters,transactions ports: - ${WALLET_API_PROJECTOR_PORT:-4005}:3000 From e9eab1eb2379ad0b7c44da0b1877ad2bb0713997 Mon Sep 17 00:00:00 2001 From: William Wolff Date: Wed, 21 Aug 2024 18:29:18 +0200 Subject: [PATCH 06/29] feat: add transaction projection indexed by credentials --- .../src/Projection/prepareTypeormProjection.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cardano-services/src/Projection/prepareTypeormProjection.ts b/packages/cardano-services/src/Projection/prepareTypeormProjection.ts index 6b5ac97c5e5..3a4751ebd25 100644 --- a/packages/cardano-services/src/Projection/prepareTypeormProjection.ts +++ b/packages/cardano-services/src/Projection/prepareTypeormProjection.ts @@ -67,7 +67,7 @@ export enum ProjectionName { StakePoolMetadataJob = 'stake-pool-metadata-job', StakePoolMetricsJob = 'stake-pool-metrics-job', StakePoolRewardsJob = 'stake-pool-rewards-job', - Transactions = 'transactions', + Transaction = 'transaction', UTXO = 'utxo' } @@ -307,7 +307,7 @@ const projectionStoreDependencies: Record = { 'stake-pool-metadata-job': ['storeStakePoolMetadataJob'], 'stake-pool-metrics-job': ['storePoolMetricsUpdateJob'], 'stake-pool-rewards-job': ['storeStakePoolRewardsJob'], - transactions: ['storeCredentials', 'storeTransactions'], + transaction: ['storeCredentials', 'storeTransactions'], utxo: ['storeUtxo'] }; From e244cf145ce404d48c46e5c2c5726250bd6b3bc9 Mon Sep 17 00:00:00 2001 From: William Wolff Date: Wed, 21 Aug 2024 18:41:37 +0200 Subject: [PATCH 07/29] refactor: add transactions for projector --- .../cardano-services/src/Projection/prepareTypeormProjection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cardano-services/src/Projection/prepareTypeormProjection.ts b/packages/cardano-services/src/Projection/prepareTypeormProjection.ts index 3a4751ebd25..7a6f28dd25c 100644 --- a/packages/cardano-services/src/Projection/prepareTypeormProjection.ts +++ b/packages/cardano-services/src/Projection/prepareTypeormProjection.ts @@ -67,7 +67,7 @@ export enum ProjectionName { StakePoolMetadataJob = 'stake-pool-metadata-job', StakePoolMetricsJob = 'stake-pool-metrics-job', StakePoolRewardsJob = 'stake-pool-rewards-job', - Transaction = 'transaction', + Transactions = 'transactions', UTXO = 'utxo' } From 67ef970918aecaff32e4da22239367eb9330b0df Mon Sep 17 00:00:00 2001 From: William Wolff Date: Thu, 22 Aug 2024 10:00:38 +0200 Subject: [PATCH 08/29] fix: rename projection --- .../cardano-services/src/Projection/prepareTypeormProjection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cardano-services/src/Projection/prepareTypeormProjection.ts b/packages/cardano-services/src/Projection/prepareTypeormProjection.ts index 7a6f28dd25c..6b5ac97c5e5 100644 --- a/packages/cardano-services/src/Projection/prepareTypeormProjection.ts +++ b/packages/cardano-services/src/Projection/prepareTypeormProjection.ts @@ -307,7 +307,7 @@ const projectionStoreDependencies: Record = { 'stake-pool-metadata-job': ['storeStakePoolMetadataJob'], 'stake-pool-metrics-job': ['storePoolMetricsUpdateJob'], 'stake-pool-rewards-job': ['storeStakePoolRewardsJob'], - transaction: ['storeCredentials', 'storeTransactions'], + transactions: ['storeCredentials', 'storeTransactions'], utxo: ['storeUtxo'] }; From df16cd23294d8adcf940e0d8cb7cb1b6c04fb37f Mon Sep 17 00:00:00 2001 From: William Wolff Date: Thu, 22 Aug 2024 10:28:50 +0200 Subject: [PATCH 09/29] refactor: allow reading invalid entropy protocol parameter update --- .../Serialization/Update/ProtocolParamUpdate.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/core/src/Serialization/Update/ProtocolParamUpdate.ts b/packages/core/src/Serialization/Update/ProtocolParamUpdate.ts index f352cb13421..c852f8ed1c5 100644 --- a/packages/core/src/Serialization/Update/ProtocolParamUpdate.ts +++ b/packages/core/src/Serialization/Update/ProtocolParamUpdate.ts @@ -321,13 +321,19 @@ export class ProtocolParamUpdate { case 12n: params.#d = UnitInterval.fromCbor(HexBlob.fromBytes(reader.readEncodedValue())); break; - case 13n: + case 13n: { // entropy is encoded as an array of two elements, where the second elements is the entropy value - reader.readStartArray(); - reader.readEncodedValue(); - params.#extraEntropy = HexBlob.fromBytes(reader.readByteString()); - reader.readEndArray(); + const size = reader.readStartArray(); + if (size === 1) { + reader.readEncodedValue(); + reader.readEndArray(); + } else { + reader.readEncodedValue(); + params.#extraEntropy = HexBlob.fromBytes(reader.readByteString()); + reader.readEndArray(); + } break; + } case 14n: params.#protocolVersion = ProtocolVersion.fromCbor(HexBlob.fromBytes(reader.readEncodedValue())); break; From ae022be304e7ec80397fd83726f87567e230ee61 Mon Sep 17 00:00:00 2001 From: William Wolff Date: Thu, 22 Aug 2024 16:02:07 +0200 Subject: [PATCH 10/29] fix: add mapper for tx cbor --- packages/ogmios/src/ogmiosToCore/tx.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ogmios/src/ogmiosToCore/tx.ts b/packages/ogmios/src/ogmiosToCore/tx.ts index d49050ca9e1..166472fb479 100644 --- a/packages/ogmios/src/ogmiosToCore/tx.ts +++ b/packages/ogmios/src/ogmiosToCore/tx.ts @@ -464,6 +464,7 @@ export const mapBlockBody = (block: CommonBlock | Schema.BlockBFT): Cardano.Bloc type !== 'bft' && transaction.cbor ? { ...Serialization.Transaction.fromCbor(transaction.cbor as Serialization.TxCBOR).toCore(), + cbor: transaction.cbor as Serialization.TxCBOR, inputSource: mapInputSource(transaction.spends) } : mapCommonTx(transaction) From 60ecfc3576ea0eece9ed6d417b6106662d092d48 Mon Sep 17 00:00:00 2001 From: William Wolff Date: Thu, 22 Aug 2024 16:09:21 +0200 Subject: [PATCH 11/29] fix: add tx mapper cbor --- packages/ogmios/src/ogmiosToCore/tx.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ogmios/src/ogmiosToCore/tx.ts b/packages/ogmios/src/ogmiosToCore/tx.ts index 166472fb479..3e8791f7cd6 100644 --- a/packages/ogmios/src/ogmiosToCore/tx.ts +++ b/packages/ogmios/src/ogmiosToCore/tx.ts @@ -436,6 +436,7 @@ const mapCommonTx = (tx: Schema.Transaction): Cardano.OnChainTx => { withdrawals: mapWithdrawals(tx.withdrawals) }) }, + cbor: tx.cbor ? Serialization.TxCBOR(tx.cbor) : undefined, id: Cardano.TransactionId(tx.id), // At the time of writing Byron transactions didn't set this property inputSource: mapInputSource(tx.spends), From f73fecb4d6c286ce1d901ddfbf20a6c1207b1f4c Mon Sep 17 00:00:00 2001 From: William Wolff Date: Tue, 27 Aug 2024 18:45:44 +0200 Subject: [PATCH 12/29] fix(LW-11312): temporary fix to not validate too long shelley addresses --- packages/core/src/Cardano/Address/BaseAddress.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/Cardano/Address/BaseAddress.ts b/packages/core/src/Cardano/Address/BaseAddress.ts index a0b3aa74ecb..2ad24fe5ab7 100644 --- a/packages/core/src/Cardano/Address/BaseAddress.ts +++ b/packages/core/src/Cardano/Address/BaseAddress.ts @@ -121,7 +121,7 @@ export class BaseAddress { * @param data The serialized address data. */ static unpackParts(type: number, data: Uint8Array): Address { - if (data.length !== 57) throw new InvalidArgumentError('data', 'Base address data length should be 57 bytes long.'); + // if (data.length !== 57) throw new InvalidArgumentError('data', 'Base address data length should be 57 bytes long.'); const network = data[0] & 0b0000_1111; const paymentCredential = Hash28ByteBase16(Buffer.from(data.slice(1, 29)).toString('hex')); From a1b59c3ed744e0710000c0112b438abcd51c1c82 Mon Sep 17 00:00:00 2001 From: William Wolff Date: Tue, 27 Aug 2024 18:46:36 +0200 Subject: [PATCH 13/29] fix(LW-11296): temporary fix for protocol parameter entropy reset --- packages/core/src/Serialization/Update/ProtocolParamUpdate.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/src/Serialization/Update/ProtocolParamUpdate.ts b/packages/core/src/Serialization/Update/ProtocolParamUpdate.ts index c852f8ed1c5..ce1bbdc8d4c 100644 --- a/packages/core/src/Serialization/Update/ProtocolParamUpdate.ts +++ b/packages/core/src/Serialization/Update/ProtocolParamUpdate.ts @@ -324,11 +324,11 @@ export class ProtocolParamUpdate { case 13n: { // entropy is encoded as an array of two elements, where the second elements is the entropy value const size = reader.readStartArray(); + reader.readEncodedValue(); + if (size === 1) { - reader.readEncodedValue(); reader.readEndArray(); } else { - reader.readEncodedValue(); params.#extraEntropy = HexBlob.fromBytes(reader.readByteString()); reader.readEndArray(); } From fc6c0f35e36b5b9a05ba1fc27f445b8a9a51ec0b Mon Sep 17 00:00:00 2001 From: William Wolff Date: Thu, 29 Aug 2024 16:03:40 +0200 Subject: [PATCH 14/29] feat: add log duration operator --- packages/projection/src/operators/index.ts | 15 ++++++++------- .../src/operators/logOperatorDuration.ts | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 packages/projection/src/operators/logOperatorDuration.ts diff --git a/packages/projection/src/operators/index.ts b/packages/projection/src/operators/index.ts index 33c118dfa03..284d8ea1b31 100644 --- a/packages/projection/src/operators/index.ts +++ b/packages/projection/src/operators/index.ts @@ -1,10 +1,11 @@ -export * from './withStaticContext'; -export * from './withEventContext'; -export * from './withRolledBackBlock'; -export * from './withEpochNo'; -export * from './withEpochBoundary'; -export * from './withNetworkInfo'; -export * as Mappers from './Mappers'; +export * from './logOperatorDuration'; export * from './logProjectionProgress'; +export * as Mappers from './Mappers'; export * from './requestNext'; export * from './utils'; +export * from './withEpochBoundary'; +export * from './withEpochNo'; +export * from './withEventContext'; +export * from './withNetworkInfo'; +export * from './withRolledBackBlock'; +export * from './withStaticContext'; diff --git a/packages/projection/src/operators/logOperatorDuration.ts b/packages/projection/src/operators/logOperatorDuration.ts new file mode 100644 index 00000000000..a2b38773ac8 --- /dev/null +++ b/packages/projection/src/operators/logOperatorDuration.ts @@ -0,0 +1,19 @@ +import { Logger } from 'ts-log'; +import { Observable, defer, finalize, tap } from 'rxjs'; + +export const logOperatorDuration = + (name: string, operator: (source: Observable) => Observable, logger: Logger) => + (source: Observable) => + defer(() => { + const start = Date.now(); + let count = 0; + + return source.pipe( + operator, + tap(() => count++), + finalize(() => { + const duration = Date.now() - start; + logger.info(`Operator ${name} processed ${count} items in ${duration}ms`); + }) + ); + }); From 60114c401c700e9ef16415e4a44586237df1dc62 Mon Sep 17 00:00:00 2001 From: Martynas Kazlauskas Date: Thu, 29 Aug 2024 18:09:37 +0300 Subject: [PATCH 15/29] chore(core): remove unused import --- packages/core/src/Cardano/Address/BaseAddress.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/core/src/Cardano/Address/BaseAddress.ts b/packages/core/src/Cardano/Address/BaseAddress.ts index 2ad24fe5ab7..b7e2dfce736 100644 --- a/packages/core/src/Cardano/Address/BaseAddress.ts +++ b/packages/core/src/Cardano/Address/BaseAddress.ts @@ -1,7 +1,6 @@ /* eslint-disable no-bitwise */ import { Address, AddressProps, AddressType, Credential, CredentialType } from './Address'; import { Hash28ByteBase16 } from '@cardano-sdk/crypto'; -import { InvalidArgumentError } from '@cardano-sdk/util'; import { NetworkId } from '../ChainId'; /** From 19dc05b86c46191402343f9ddd15ab2fd3c8ea4c Mon Sep 17 00:00:00 2001 From: Martynas Kazlauskas Date: Thu, 29 Aug 2024 18:10:39 +0300 Subject: [PATCH 16/29] feat: implement operator duration measure --- .../src/Projection/createTypeormProjection.ts | 45 +++++++++++++++---- packages/projection/src/operators/index.ts | 2 +- .../src/operators/logOperatorDuration.ts | 19 -------- .../src/operators/logProjectionProgress.ts | 20 ++++++--- .../src/operators/withOperatorDuration.ts | 38 ++++++++++++++++ 5 files changed, 91 insertions(+), 33 deletions(-) delete mode 100644 packages/projection/src/operators/logOperatorDuration.ts create mode 100644 packages/projection/src/operators/withOperatorDuration.ts diff --git a/packages/cardano-services/src/Projection/createTypeormProjection.ts b/packages/cardano-services/src/Projection/createTypeormProjection.ts index 2ea17287b48..345bc447852 100644 --- a/packages/cardano-services/src/Projection/createTypeormProjection.ts +++ b/packages/cardano-services/src/Projection/createTypeormProjection.ts @@ -1,6 +1,12 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable prefer-spread */ -import { Bootstrap, ProjectionEvent, logProjectionProgress, requestNext } from '@cardano-sdk/projection'; +import { + Bootstrap, + ProjectionEvent, + logProjectionProgress, + requestNext, + withOperatorDuration +} from '@cardano-sdk/projection'; import { Cardano, ObservableCardanoNode } from '@cardano-sdk/core'; import { Logger } from 'ts-log'; import { Observable, concat, defer, groupBy, mergeMap, take, takeWhile } from 'rxjs'; @@ -41,14 +47,35 @@ export interface CreateTypeormProjectionProps { projectionOptions?: ProjectionOptions; } +type TrackDurationProps = { + operatorNames: Array; +}; + const applyMappers = - (selectedMappers: PreparedProjection['mappers']) => + (selectedMappers: PreparedProjection['mappers'], trackDurationProps?: TrackDurationProps) => (evt$: Observable) => - evt$.pipe.apply(evt$, selectedMappers as any) as Observable>; + evt$.pipe.apply( + evt$, + trackDurationProps + ? selectedMappers.map((mapper, i) => + withOperatorDuration(trackDurationProps.operatorNames[i] || '', mapper as any) + ) + : (selectedMappers as any) + ) as Observable>; const applyStores = - (selectedStores: PreparedProjection['stores']) => + ( + selectedStores: PreparedProjection['stores'], + trackDurationProps?: TrackDurationProps + ) => (evt$: Observable) => - evt$.pipe.apply(evt$, selectedStores as any) as Observable; + evt$.pipe.apply( + evt$, + trackDurationProps + ? selectedStores.map((mapper, i) => + withOperatorDuration(trackDurationProps.operatorNames[i] || '', mapper as any) + ) + : (selectedStores as any) + ) as Observable; /** * Creates a projection observable that applies a sequence of operators @@ -72,7 +99,7 @@ export const createTypeormProjection = ({ logger.debug(`Creating projection with policyIds ${JSON.stringify(handlePolicyIds)}`); logger.debug(`Using a ${blocksBufferLength} blocks buffer`); - const { mappers, entities, stores, extensions, willStore } = prepareTypeormProjection( + const { mappers, entities, stores, extensions, willStore, __debug } = prepareTypeormProjection( { options: projectionOptions, projections @@ -117,7 +144,9 @@ export const createTypeormProjection = ({ ).pipe(take(1), toEmpty), defer(() => projectionSource$.pipe( - applyMappers(mappers), + // TODO: only pass {operatorNames} if debugging; + // we should pass some cli argument here + applyMappers(mappers, { operatorNames: __debug.mappers }), // if there are any relevant data to write into db groupBy((evt) => willStore(evt)), mergeMap((group$) => @@ -127,7 +156,7 @@ export const createTypeormProjection = ({ (evt$) => evt$.pipe( withTypeormTransaction({ connection$: connect() }), - applyStores(stores), + applyStores(stores, { operatorNames: __debug.stores }), buffer.storeBlockData(), typeormTransactionCommit() ), diff --git a/packages/projection/src/operators/index.ts b/packages/projection/src/operators/index.ts index 284d8ea1b31..7c9c1af98b3 100644 --- a/packages/projection/src/operators/index.ts +++ b/packages/projection/src/operators/index.ts @@ -1,4 +1,4 @@ -export * from './logOperatorDuration'; +export * from './withOperatorDuration'; export * from './logProjectionProgress'; export * as Mappers from './Mappers'; export * from './requestNext'; diff --git a/packages/projection/src/operators/logOperatorDuration.ts b/packages/projection/src/operators/logOperatorDuration.ts deleted file mode 100644 index a2b38773ac8..00000000000 --- a/packages/projection/src/operators/logOperatorDuration.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Logger } from 'ts-log'; -import { Observable, defer, finalize, tap } from 'rxjs'; - -export const logOperatorDuration = - (name: string, operator: (source: Observable) => Observable, logger: Logger) => - (source: Observable) => - defer(() => { - const start = Date.now(); - let count = 0; - - return source.pipe( - operator, - tap(() => count++), - finalize(() => { - const duration = Date.now() - start; - logger.info(`Operator ${name} processed ${count} items in ${duration}ms`); - }) - ); - }); diff --git a/packages/projection/src/operators/logProjectionProgress.ts b/packages/projection/src/operators/logProjectionProgress.ts index a8f99c37ea2..fc2dbf7c6f1 100644 --- a/packages/projection/src/operators/logProjectionProgress.ts +++ b/packages/projection/src/operators/logProjectionProgress.ts @@ -1,7 +1,8 @@ -import { Cardano, ChainSyncEventType, TipOrOrigin } from '@cardano-sdk/core'; +import { Cardano, ChainSyncEventType, Milliseconds, TipOrOrigin } from '@cardano-sdk/core'; import { Logger } from 'ts-log'; import { Observable, defer, finalize, tap } from 'rxjs'; import { UnifiedExtChainSyncEvent } from '../types'; +import { WithOperatorDuration } from './withOperatorDuration'; import { contextLogger } from '@cardano-sdk/util'; import { pointDescription } from '../util'; @@ -26,8 +27,9 @@ const logSyncLine = (params: { numEvt: number; startedAt: number; tip: Cardano.Tip; + operatorDuration?: WithOperatorDuration['operatorDuration']; }) => { - const { blocksTime, header, logger, numEvt, startedAt, tip } = params; + const { blocksTime, header, logger, numEvt, startedAt, tip, operatorDuration } = params; const syncPercentage = ((header.blockNo * 100) / tip.blockNo).toFixed(2); const now = Date.now(); @@ -49,6 +51,14 @@ const logSyncLine = (params: { logger.info(`Initializing ${syncPercentage}% at block #${header.blockNo} ${speeds.join(' - ')}`); + if (operatorDuration) { + for (const [operatorName, { numCalls, totalTime }] of Object.entries(operatorDuration)) { + logger.info( + `"${operatorName}": Total: ${Milliseconds.toSeconds(totalTime)}s, Avg: ${(totalTime / numCalls).toFixed(1)}ms` + ); + } + } + const pruneOldTimes = (upTo: number) => { for (const block of blocksTime.keys()) if (block <= upTo) blocksTime.delete(block); @@ -59,7 +69,7 @@ const logSyncLine = (params: { }; export const logProjectionProgress = - , 'requestNext'>>(baseLogger: Logger) => + >, 'requestNext'>>(baseLogger: Logger) => (evt$: Observable) => defer(() => { const logger = contextLogger(baseLogger, 'Projector'); @@ -69,7 +79,7 @@ export const logProjectionProgress = const startedAt = Date.now(); logger.info('Started'); return evt$.pipe( - tap(({ block: { header }, eventType, tip }) => { + tap(({ block: { header }, eventType, tip, operatorDuration }) => { numEvt++; if (isAtTheTipOrHigher(header, tip)) { logger.info( @@ -78,7 +88,7 @@ export const logProjectionProgress = } ${pointDescription(header)}` ); } else if (numEvt % logFrequency === 0 && tip !== 'origin') - logSyncLine({ blocksTime, header, logger, numEvt, startedAt, tip }); + logSyncLine({ blocksTime, header, logger, numEvt, operatorDuration, startedAt, tip }); }), finalize(() => logger.info(`Stopped after ${Math.round((Date.now() - startedAt) / 1000)} s`)) ); diff --git a/packages/projection/src/operators/withOperatorDuration.ts b/packages/projection/src/operators/withOperatorDuration.ts new file mode 100644 index 00000000000..49bb2bafa80 --- /dev/null +++ b/packages/projection/src/operators/withOperatorDuration.ts @@ -0,0 +1,38 @@ +import { Milliseconds } from '@cardano-sdk/core'; +import { Observable, map, tap } from 'rxjs'; +import { ProjectionEvent } from '../types'; + +type OperatorStats = { + totalTime: Milliseconds; + numCalls: number; +}; +export type WithOperatorDuration = { + operatorDuration: Record; +}; + +const operatorDuration = {} as Record; + +export const withOperatorDuration = + (name: string, operator: (source: Observable) => Observable) => + (source: Observable): Observable => { + let start: number; + let totalTime = 0; + let numCalls = 0; + + return source.pipe( + tap(() => (start = Date.now())), + operator, + tap(() => { + totalTime += Date.now() - start; + numCalls++; + }), + map((evt) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + operatorDuration[name] = { numCalls, totalTime: totalTime as Milliseconds }; + return { + ...evt, + operatorDuration + }; + }) + ); + }; From 62901b6a80bdcc953d6238fa6fcc30d10cd5ddf0 Mon Sep 17 00:00:00 2001 From: William Wolff Date: Mon, 2 Sep 2024 14:19:22 +0200 Subject: [PATCH 17/29] refactor(credential-projection): add inmemory cache for hydrating txIn --- .../src/operators/storeCredentials.ts | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/packages/projection-typeorm/src/operators/storeCredentials.ts b/packages/projection-typeorm/src/operators/storeCredentials.ts index be442a9f1e8..648c7000907 100644 --- a/packages/projection-typeorm/src/operators/storeCredentials.ts +++ b/packages/projection-typeorm/src/operators/storeCredentials.ts @@ -10,6 +10,8 @@ export interface WithTxCredentials { credentialsByTx: Record; } +const addressByTxInCache = {} as Record; + export const willStoreCredentials = ({ utxoByTx }: Mappers.WithUtxo) => Object.keys(utxoByTx).length > 0; const addInputCredentials = async ( @@ -18,21 +20,26 @@ const addInputCredentials = async ( addCredentialFromAddress: (txId: Cardano.TransactionId, address: Mappers.Address) => void ) => { for (const txHash of Object.keys(utxoByTx) as Cardano.TransactionId[]) { - const txInLookups = utxoByTx[txHash].consumed.map(({ txId, index: outputIndex }) => ({ - outputIndex, - txId - })); + const txInLookups: { outputIndex: number; txId: Cardano.TransactionId }[] = []; + for (const txIn of utxoByTx[txHash]!.consumed) { + const cacheKey = `${txIn.txId}#${txIn.index}`; + if (addressByTxInCache[cacheKey] === undefined) { + txInLookups.push({ outputIndex: txIn.index, txId: txIn.txId }); + } else { + addCredentialFromAddress(txHash, addressByTxInCache[cacheKey]); + } + } const outputEntities = await utxoRepository.find({ select: { address: true, outputIndex: true, txId: true }, - where: txInLookups.map(({ txId, outputIndex }) => ({ - outputIndex, - txId - })) + where: txInLookups }); for (const hydratedTxIn of outputEntities) { if (hydratedTxIn.address) { + addressByTxInCache[`${hydratedTxIn.txId!}#${hydratedTxIn.outputIndex!}`] = Mappers.credentialsFromAddress( + hydratedTxIn.address! + ); addCredentialFromAddress(txHash, Mappers.credentialsFromAddress(hydratedTxIn.address)); } } From 1725716823dfecbb188f2698c549cec33f04d0e4 Mon Sep 17 00:00:00 2001 From: William Wolff Date: Mon, 2 Sep 2024 14:35:20 +0200 Subject: [PATCH 18/29] refactor(credential-projection): remove consumed txIns from cache --- packages/projection-typeorm/src/operators/storeCredentials.ts | 4 ++++ packages/projection-typeorm/src/operators/storeUtxo.ts | 2 ++ 2 files changed, 6 insertions(+) diff --git a/packages/projection-typeorm/src/operators/storeCredentials.ts b/packages/projection-typeorm/src/operators/storeCredentials.ts index 648c7000907..8ac40d9c8b9 100644 --- a/packages/projection-typeorm/src/operators/storeCredentials.ts +++ b/packages/projection-typeorm/src/operators/storeCredentials.ts @@ -12,6 +12,10 @@ export interface WithTxCredentials { const addressByTxInCache = {} as Record; +export const removeTxInFromCache = (txIn: string) => { + delete addressByTxInCache[txIn]; +}; + export const willStoreCredentials = ({ utxoByTx }: Mappers.WithUtxo) => Object.keys(utxoByTx).length > 0; const addInputCredentials = async ( diff --git a/packages/projection-typeorm/src/operators/storeUtxo.ts b/packages/projection-typeorm/src/operators/storeUtxo.ts index 1a4f043d0d5..87fe00fa033 100644 --- a/packages/projection-typeorm/src/operators/storeUtxo.ts +++ b/packages/projection-typeorm/src/operators/storeUtxo.ts @@ -2,6 +2,7 @@ import { Cardano, ChainSyncEventType, Serialization } from '@cardano-sdk/core'; import { Mappers } from '@cardano-sdk/projection'; import { ObjectLiteral } from 'typeorm'; import { OutputEntity, TokensEntity } from '../entity'; +import { removeTxInFromCache } from './storeCredentials'; import { typeormOperator } from './util'; const serializeDatumIfExists = (datum: Cardano.PlutusData | undefined) => @@ -62,6 +63,7 @@ export const storeUtxo = typeormOperator Date: Tue, 3 Sep 2024 13:40:47 +0200 Subject: [PATCH 19/29] refactor(credential-projection-cache): made cache mutation private to credentials projection --- .../projection-typeorm/src/operators/storeCredentials.ts | 9 +++++++-- packages/projection-typeorm/src/operators/storeUtxo.ts | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/projection-typeorm/src/operators/storeCredentials.ts b/packages/projection-typeorm/src/operators/storeCredentials.ts index 8ac40d9c8b9..1ba2f8359e8 100644 --- a/packages/projection-typeorm/src/operators/storeCredentials.ts +++ b/packages/projection-typeorm/src/operators/storeCredentials.ts @@ -12,7 +12,7 @@ export interface WithTxCredentials { const addressByTxInCache = {} as Record; -export const removeTxInFromCache = (txIn: string) => { +const removeTxInFromCache = (txIn: string) => { delete addressByTxInCache[txIn]; }; @@ -111,7 +111,8 @@ export const storeCredentials = typeormOperator< eventType, queryRunner, stakeCredentialsByTx, - utxoByTx + utxoByTx, + utxo: { consumed: consumedUTxOs } } = evt; const txToCredentials = new Map(); @@ -171,5 +172,9 @@ export const storeCredentials = typeormOperator< .orIgnore() .execute(); + for (const consumed of consumedUTxOs) { + removeTxInFromCache(`${consumed.txId}#${consumed.index}`); + } + return { credentialsByTx: Object.fromEntries(txToCredentials) }; }); diff --git a/packages/projection-typeorm/src/operators/storeUtxo.ts b/packages/projection-typeorm/src/operators/storeUtxo.ts index 87fe00fa033..39c8813d1d4 100644 --- a/packages/projection-typeorm/src/operators/storeUtxo.ts +++ b/packages/projection-typeorm/src/operators/storeUtxo.ts @@ -63,7 +63,6 @@ export const storeUtxo = typeormOperator Date: Wed, 4 Sep 2024 12:57:57 +0200 Subject: [PATCH 20/29] refactor(credential-projection): introduced manager to simplify collecting credentials by addresses; added cache; remove unneeded indices and utxo projection --- packages/projection-typeorm/package.json | 1 + .../src/CredentialManager.ts | 102 ++++++++++++++ .../src/entity/Credential.entity.ts | 2 +- .../src/entity/Output.entity.ts | 2 +- .../src/operators/storeCredentials.ts | 128 +++++------------- .../src/operators/storeUtxo.ts | 63 +++++---- .../src/operators/Mappers/withAddresses.ts | 3 +- 7 files changed, 169 insertions(+), 132 deletions(-) create mode 100644 packages/projection-typeorm/src/CredentialManager.ts diff --git a/packages/projection-typeorm/package.json b/packages/projection-typeorm/package.json index 1f5c4f8a9ab..bbbbf685591 100644 --- a/packages/projection-typeorm/package.json +++ b/packages/projection-typeorm/package.json @@ -44,6 +44,7 @@ "@cardano-sdk/util-rxjs": "workspace:~", "backoff-rxjs": "^7.0.0", "lodash": "^4.17.21", + "lru-cache": "^11.0.0", "pg": "^8.9.0", "pg-boss": "8.4.2", "reflect-metadata": "^0.1.13", diff --git a/packages/projection-typeorm/src/CredentialManager.ts b/packages/projection-typeorm/src/CredentialManager.ts new file mode 100644 index 00000000000..1702a33b836 --- /dev/null +++ b/packages/projection-typeorm/src/CredentialManager.ts @@ -0,0 +1,102 @@ +import { Cardano } from '@cardano-sdk/core'; +import { CredentialEntity, CredentialType, credentialEntityComparator } from './entity'; +import { Hash28ByteBase16 } from '@cardano-sdk/crypto'; +import { LRUCache } from 'lru-cache'; +import { Mappers } from '@cardano-sdk/projection'; +import uniqWith from 'lodash/uniqWith.js'; + +export interface Credential { + hash: Hash28ByteBase16; + type?: CredentialType; +} + +type AddressPart = 'payment' | 'stake'; +const credentialTypeMap: { [key: number]: { payment: CredentialType | null; stake: CredentialType } } = { + [Cardano.AddressType.BasePaymentKeyStakeKey]: { payment: CredentialType.PaymentKey, stake: CredentialType.StakeKey }, + [Cardano.AddressType.EnterpriseKey]: { payment: CredentialType.PaymentKey, stake: CredentialType.StakeKey }, + [Cardano.AddressType.BasePaymentKeyStakeScript]: { + payment: CredentialType.PaymentKey, + stake: CredentialType.StakeScript + }, + [Cardano.AddressType.BasePaymentScriptStakeKey]: { + payment: CredentialType.PaymentScript, + stake: CredentialType.StakeKey + }, + [Cardano.AddressType.BasePaymentScriptStakeScript]: { + payment: CredentialType.PaymentScript, + stake: CredentialType.StakeScript + }, + [Cardano.AddressType.EnterpriseScript]: { payment: CredentialType.PaymentScript, stake: CredentialType.StakeScript }, + [Cardano.AddressType.RewardKey]: { payment: null, stake: CredentialType.StakeKey }, + [Cardano.AddressType.RewardScript]: { payment: null, stake: CredentialType.StakeScript } +}; + +const credentialsByTxInCache = new LRUCache({ + max: 1_000_000 +}); + +export class CredentialManager { + txToCredentials = new Map(); + + getCachedCredential(txIn: Cardano.TxIn): Credential[] { + return credentialsByTxInCache.get(`${txIn.txId}#${txIn.index}`) ?? []; + } + + deleteCachedCredential(txIn: Cardano.TxIn) { + credentialsByTxInCache.delete(`${txIn.txId}#${txIn.index}`); + } + + addCredential(txId: Cardano.TransactionId, { hash: credentialHash, type: credentialType }: Credential) { + this.txToCredentials.set( + txId, + uniqWith( + [...(this.txToCredentials.get(txId) || []), { credentialHash, credentialType }], + credentialEntityComparator + ) + ); + } + + // This function caches credentials only if outputIndex is set. + addCredentialFromAddress( + txId: Cardano.TransactionId, + { paymentCredentialHash, stakeCredential, type }: Mappers.Address, + outputIndex?: number + ) { + const cacheKey = `${txId}#${outputIndex}`; + const paymentCredentialType = this.credentialTypeFromAddressType(type, 'payment'); + + if (paymentCredentialHash && paymentCredentialType) { + this.addCredential(txId, { hash: paymentCredentialHash, type: paymentCredentialType }); + if (outputIndex) { + credentialsByTxInCache.set(cacheKey, [ + ...(credentialsByTxInCache.get(cacheKey) || []), + { hash: paymentCredentialHash, type: paymentCredentialType } + ]); + } + } + + if (stakeCredential) { + const stakeCredentialType = this.credentialTypeFromAddressType(type, 'stake'); + // FIXME: support pointers + if (stakeCredentialType && typeof stakeCredential === 'string') { + this.addCredential(txId, { hash: stakeCredential, type: stakeCredentialType }); + + if (outputIndex) { + credentialsByTxInCache.set(cacheKey, [ + ...(credentialsByTxInCache.get(cacheKey) || []), + { hash: stakeCredential, type: stakeCredentialType } + ]); + } + } + } + } + + credentialTypeFromAddressType(type: Cardano.AddressType, part: AddressPart) { + const credential = credentialTypeMap[type]; + if (!credential) { + // FIXME: map byron address, pointer script, pointer key type + return null; + } + return credential[part]; + } +} diff --git a/packages/projection-typeorm/src/entity/Credential.entity.ts b/packages/projection-typeorm/src/entity/Credential.entity.ts index 024eb1ceae5..04014de0035 100644 --- a/packages/projection-typeorm/src/entity/Credential.entity.ts +++ b/packages/projection-typeorm/src/entity/Credential.entity.ts @@ -15,7 +15,7 @@ export class CredentialEntity { @PrimaryColumn('varchar') credentialHash?: Hash28ByteBase16; - @Column('enum', { enum: CredentialType, nullable: false }) + @Column('enum', { enum: CredentialType, nullable: true }) credentialType?: CredentialType; @ManyToMany(() => TransactionEntity, (transaction) => transaction.credentials, { onDelete: 'CASCADE' }) diff --git a/packages/projection-typeorm/src/entity/Output.entity.ts b/packages/projection-typeorm/src/entity/Output.entity.ts index 00e9ecd0af1..1482d3df524 100644 --- a/packages/projection-typeorm/src/entity/Output.entity.ts +++ b/packages/projection-typeorm/src/entity/Output.entity.ts @@ -9,7 +9,7 @@ import { TokensEntity } from './Tokens.entity'; export class OutputEntity { @PrimaryGeneratedColumn() id?: number; - @Index() + // @Index() @Column('varchar') address?: Cardano.PaymentAddress; @Index() diff --git a/packages/projection-typeorm/src/operators/storeCredentials.ts b/packages/projection-typeorm/src/operators/storeCredentials.ts index 1ba2f8359e8..9ae8952bc94 100644 --- a/packages/projection-typeorm/src/operators/storeCredentials.ts +++ b/packages/projection-typeorm/src/operators/storeCredentials.ts @@ -1,36 +1,35 @@ +/* eslint-disable sonarjs/cognitive-complexity */ import { Cardano, ChainSyncEventType } from '@cardano-sdk/core'; -import { CredentialEntity, CredentialType, OutputEntity, credentialEntityComparator } from '../entity'; -import { Hash28ByteBase16 } from '@cardano-sdk/crypto'; +import { CredentialEntity, CredentialType, OutputEntity } from '../entity'; import { Mappers } from '@cardano-sdk/projection'; import { Repository } from 'typeorm'; import { typeormOperator } from './util'; -import uniqWith from 'lodash/uniqWith.js'; + +import { CredentialManager } from '../CredentialManager'; export interface WithTxCredentials { credentialsByTx: Record; } -const addressByTxInCache = {} as Record; - -const removeTxInFromCache = (txIn: string) => { - delete addressByTxInCache[txIn]; -}; - export const willStoreCredentials = ({ utxoByTx }: Mappers.WithUtxo) => Object.keys(utxoByTx).length > 0; const addInputCredentials = async ( utxoByTx: Record, utxoRepository: Repository, - addCredentialFromAddress: (txId: Cardano.TransactionId, address: Mappers.Address) => void + manager: CredentialManager ) => { for (const txHash of Object.keys(utxoByTx) as Cardano.TransactionId[]) { const txInLookups: { outputIndex: number; txId: Cardano.TransactionId }[] = []; + for (const txIn of utxoByTx[txHash]!.consumed) { - const cacheKey = `${txIn.txId}#${txIn.index}`; - if (addressByTxInCache[cacheKey] === undefined) { - txInLookups.push({ outputIndex: txIn.index, txId: txIn.txId }); + const cachedCredentials = manager.getCachedCredential(txIn); + if (cachedCredentials.length > 0) { + for (const credential of cachedCredentials) { + manager.addCredential(txHash, { hash: credential.hash, type: credential.type ?? undefined }); + } + manager.deleteCachedCredential(txIn); // can only be consumed once so chances are it won't have to be resolved again } else { - addCredentialFromAddress(txHash, addressByTxInCache[cacheKey]); + txInLookups.push({ outputIndex: txIn.index, txId: txIn.txId }); } } @@ -41,10 +40,7 @@ const addInputCredentials = async ( for (const hydratedTxIn of outputEntities) { if (hydratedTxIn.address) { - addressByTxInCache[`${hydratedTxIn.txId!}#${hydratedTxIn.outputIndex!}`] = Mappers.credentialsFromAddress( - hydratedTxIn.address! - ); - addCredentialFromAddress(txHash, Mappers.credentialsFromAddress(hydratedTxIn.address)); + manager.addCredentialFromAddress(txHash, Mappers.credentialsFromAddress(hydratedTxIn.address!)); } } } @@ -52,55 +48,29 @@ const addInputCredentials = async ( const addOutputCredentials = ( addressesByTx: Record, - addCredentialFromAddress: (txId: Cardano.TransactionId, address: Mappers.Address) => void + manager: CredentialManager ) => { for (const txId of Object.keys(addressesByTx) as Cardano.TransactionId[]) { - for (const address of addressesByTx[txId]) { - addCredentialFromAddress(txId, address); + for (const [index, address] of addressesByTx[txId].entries()) { + manager.addCredentialFromAddress(txId, address, index); } } }; const addCertificateCredentials = ( credentialsByTx: Record, - addCredential: ( - txId: Cardano.TransactionId, - credentialHash: Hash28ByteBase16, - credentialType: CredentialType - ) => Map + manager: CredentialManager ) => { for (const txId of Object.keys(credentialsByTx) as Cardano.TransactionId[]) { for (const credential of credentialsByTx[txId]) { - addCredential( - txId, - credential.hash, - credential.type === 0 ? CredentialType.StakeKey : CredentialType.StakeScript - ); + manager.addCredential(txId, { + hash: credential.hash, + type: credential.type === 0 ? CredentialType.StakeKey : CredentialType.StakeScript + }); } } }; -type AddressPart = 'payment' | 'stake'; -const credentialTypeMap: { [key: number]: { payment: CredentialType | null; stake: CredentialType } } = { - [Cardano.AddressType.BasePaymentKeyStakeKey]: { payment: CredentialType.PaymentKey, stake: CredentialType.StakeKey }, - [Cardano.AddressType.EnterpriseKey]: { payment: CredentialType.PaymentKey, stake: CredentialType.StakeKey }, - [Cardano.AddressType.BasePaymentKeyStakeScript]: { - payment: CredentialType.PaymentKey, - stake: CredentialType.StakeScript - }, - [Cardano.AddressType.BasePaymentScriptStakeKey]: { - payment: CredentialType.PaymentScript, - stake: CredentialType.StakeKey - }, - [Cardano.AddressType.BasePaymentScriptStakeScript]: { - payment: CredentialType.PaymentScript, - stake: CredentialType.StakeScript - }, - [Cardano.AddressType.EnterpriseScript]: { payment: CredentialType.PaymentScript, stake: CredentialType.StakeScript }, - [Cardano.AddressType.RewardKey]: { payment: null, stake: CredentialType.StakeKey }, - [Cardano.AddressType.RewardScript]: { payment: null, stake: CredentialType.StakeScript } -}; - export const storeCredentials = typeormOperator< Mappers.WithUtxo & Mappers.WithAddresses & Mappers.WithCertificates, WithTxCredentials @@ -115,66 +85,30 @@ export const storeCredentials = typeormOperator< utxo: { consumed: consumedUTxOs } } = evt; - const txToCredentials = new Map(); + const manager = new CredentialManager(); // produced credentials will be automatically deleted via block cascade if (txs.length === 0 || eventType !== ChainSyncEventType.RollForward) { - return { credentialsByTx: Object.fromEntries(txToCredentials) }; + return { credentialsByTx: Object.fromEntries(manager.txToCredentials) }; } - const utxoRepository = queryRunner.manager.getRepository(OutputEntity); - const addCredential = ( - txId: Cardano.TransactionId, - credentialHash: Hash28ByteBase16, - credentialType: CredentialType - ) => - txToCredentials.set( - txId, - uniqWith([...(txToCredentials.get(txId) || []), { credentialHash, credentialType }], credentialEntityComparator) - ); - - const credentialTypeFromAddressType = (type: Cardano.AddressType, part: AddressPart) => { - const credential = credentialTypeMap[type]; - if (!credential) { - // FIXME: map byron address, pointer script, pointer key type - return null; - } - return credential[part]; - }; - - const addCredentialFromAddress = ( - txId: Cardano.TransactionId, - { paymentCredentialHash, stakeCredential, type }: Mappers.Address - ) => { - const paymentCredentialType = credentialTypeFromAddressType(type, 'payment'); - if (paymentCredentialHash && paymentCredentialType) { - addCredential(txId, paymentCredentialHash, paymentCredentialType); - } - if (stakeCredential) { - const stakeCredentialType = credentialTypeFromAddressType(type, 'stake'); - // FIXME: support pointers - if (stakeCredentialType && typeof stakeCredential === 'string') { - addCredential(txId, stakeCredential, stakeCredentialType); - } - } - }; - - await addInputCredentials(utxoByTx, utxoRepository, addCredentialFromAddress); - addOutputCredentials(addressesByTx, addCredentialFromAddress); - addCertificateCredentials(stakeCredentialsByTx, addCredential); + const utxoRepository = queryRunner.manager.getRepository(OutputEntity); + await addInputCredentials(utxoByTx, utxoRepository, manager); + addOutputCredentials(addressesByTx, manager); + addCertificateCredentials(stakeCredentialsByTx, manager); // insert new credentials & ignore conflicts of existing ones await queryRunner.manager .createQueryBuilder() .insert() .into(CredentialEntity) - .values([...txToCredentials.values()].flat()) + .values([...manager.txToCredentials.values()].flat()) .orIgnore() .execute(); for (const consumed of consumedUTxOs) { - removeTxInFromCache(`${consumed.txId}#${consumed.index}`); + manager.deleteCachedCredential(consumed); } - return { credentialsByTx: Object.fromEntries(txToCredentials) }; + return { credentialsByTx: Object.fromEntries(manager.txToCredentials) }; }); diff --git a/packages/projection-typeorm/src/operators/storeUtxo.ts b/packages/projection-typeorm/src/operators/storeUtxo.ts index 39c8813d1d4..e1ec1f7b60b 100644 --- a/packages/projection-typeorm/src/operators/storeUtxo.ts +++ b/packages/projection-typeorm/src/operators/storeUtxo.ts @@ -1,12 +1,11 @@ -import { Cardano, ChainSyncEventType, Serialization } from '@cardano-sdk/core'; +import { ChainSyncEventType } from '@cardano-sdk/core'; import { Mappers } from '@cardano-sdk/projection'; import { ObjectLiteral } from 'typeorm'; -import { OutputEntity, TokensEntity } from '../entity'; -import { removeTxInFromCache } from './storeCredentials'; +import { OutputEntity } from '../entity'; import { typeormOperator } from './util'; -const serializeDatumIfExists = (datum: Cardano.PlutusData | undefined) => - datum ? Serialization.PlutusData.fromCore(datum).toCbor() : undefined; +// const serializeDatumIfExists = (datum: Cardano.PlutusData | undefined) => +// datum ? Serialization.PlutusData.fromCore(datum).toCbor() : undefined; export interface WithStoredProducedUtxo { storedProducedUtxo: Map; @@ -18,20 +17,20 @@ export const willStoreUtxo = ({ utxo: { produced, consumed } }: Mappers.WithUtxo export const storeUtxo = typeormOperator( async ({ utxo: { consumed, produced }, block: { header }, eventType, queryRunner }) => { const utxoRepository = queryRunner.manager.getRepository(OutputEntity); - const tokensRepository = queryRunner.manager.getRepository(TokensEntity); + // const tokensRepository = queryRunner.manager.getRepository(TokensEntity); const storedProducedUtxo = new Map(); if (eventType === ChainSyncEventType.RollForward) { if (produced.length > 0) { const { identifiers } = await utxoRepository.insert( produced.map( - ([{ index, txId }, { scriptReference, address, value, datum, datumHash }]): OutputEntity => ({ + ([{ index, txId }, { address }]): OutputEntity => ({ address, block: { slot: header.slot }, - coins: value.coins, - datum: serializeDatumIfExists(datum), - datumHash, + // coins: value.coins, + // datum: serializeDatumIfExists(datum), + // datumHash, outputIndex: index, - scriptReference, + // scriptReference, txId }) ) @@ -39,27 +38,27 @@ export const storeUtxo = typeormOperator - [...(assets?.entries() || [])].map( - ([assetId, quantity]): TokensEntity => ({ - asset: { id: assetId }, - output: identifiers[producedIndex], - quantity - }) - ) - ); - if (tokens.length > 0) { - await tokensRepository.insert(tokens); - } + // const tokens = produced.flatMap( + // ( + // [ + // _, + // { + // value: { assets } + // } + // ], + // producedIndex + // ) => + // [...(assets?.entries() || [])].map( + // ([assetId, quantity]): TokensEntity => ({ + // asset: { id: assetId }, + // output: identifiers[producedIndex], + // quantity + // }) + // ) + // ); + // if (tokens.length > 0) { + // await tokensRepository.insert(tokens); + // } } for (const { index, txId } of consumed) { await utxoRepository.update({ outputIndex: index, txId }, { consumedAtSlot: header.slot }); diff --git a/packages/projection/src/operators/Mappers/withAddresses.ts b/packages/projection/src/operators/Mappers/withAddresses.ts index ed87db96e6f..fc809863c87 100644 --- a/packages/projection/src/operators/Mappers/withAddresses.ts +++ b/packages/projection/src/operators/Mappers/withAddresses.ts @@ -25,7 +25,8 @@ export const withAddresses = unifiedProjectorOperator(( ...Object.entries(evt.utxoByTx).reduce( (map, [txId, utxo]) => ({ ...map, - [txId]: uniq(utxo.produced.map(([_, txOut]) => txOut.address)).map(credentialsFromAddress) + // no use of uniq to preserve output index via the order of the array + [txId]: utxo.produced.map(([_, txOut]) => txOut.address).map(credentialsFromAddress) }), new Map() ) From fac2747188995d7d3b773fa3dec4e192bf169d51 Mon Sep 17 00:00:00 2001 From: William Wolff Date: Wed, 4 Sep 2024 13:09:47 +0200 Subject: [PATCH 21/29] refactor: add back coin which is not nullable --- packages/projection-typeorm/src/operators/storeUtxo.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/projection-typeorm/src/operators/storeUtxo.ts b/packages/projection-typeorm/src/operators/storeUtxo.ts index e1ec1f7b60b..c46e3a8ac5e 100644 --- a/packages/projection-typeorm/src/operators/storeUtxo.ts +++ b/packages/projection-typeorm/src/operators/storeUtxo.ts @@ -23,10 +23,10 @@ export const storeUtxo = typeormOperator 0) { const { identifiers } = await utxoRepository.insert( produced.map( - ([{ index, txId }, { address }]): OutputEntity => ({ + ([{ index, txId }, { value, address }]): OutputEntity => ({ address, block: { slot: header.slot }, - // coins: value.coins, + coins: value.coins, // datum: serializeDatumIfExists(datum), // datumHash, outputIndex: index, From eef04b04e2f59cfee7e094a91ffef7fb12a3604b Mon Sep 17 00:00:00 2001 From: William Wolff Date: Wed, 18 Sep 2024 14:21:14 +0200 Subject: [PATCH 22/29] fix: add byron credential mapping --- .../src/CredentialManager.ts | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/packages/projection-typeorm/src/CredentialManager.ts b/packages/projection-typeorm/src/CredentialManager.ts index 1702a33b836..681ec649abb 100644 --- a/packages/projection-typeorm/src/CredentialManager.ts +++ b/packages/projection-typeorm/src/CredentialManager.ts @@ -11,7 +11,7 @@ export interface Credential { } type AddressPart = 'payment' | 'stake'; -const credentialTypeMap: { [key: number]: { payment: CredentialType | null; stake: CredentialType } } = { +const credentialTypeMap: { [key: number]: { payment: CredentialType | null; stake: CredentialType | null } } = { [Cardano.AddressType.BasePaymentKeyStakeKey]: { payment: CredentialType.PaymentKey, stake: CredentialType.StakeKey }, [Cardano.AddressType.EnterpriseKey]: { payment: CredentialType.PaymentKey, stake: CredentialType.StakeKey }, [Cardano.AddressType.BasePaymentKeyStakeScript]: { @@ -28,22 +28,23 @@ const credentialTypeMap: { [key: number]: { payment: CredentialType | null; stak }, [Cardano.AddressType.EnterpriseScript]: { payment: CredentialType.PaymentScript, stake: CredentialType.StakeScript }, [Cardano.AddressType.RewardKey]: { payment: null, stake: CredentialType.StakeKey }, - [Cardano.AddressType.RewardScript]: { payment: null, stake: CredentialType.StakeScript } + [Cardano.AddressType.RewardScript]: { payment: null, stake: CredentialType.StakeScript }, + [Cardano.AddressType.Byron]: { payment: CredentialType.PaymentKey, stake: null } }; -const credentialsByTxInCache = new LRUCache({ - max: 1_000_000 -}); - export class CredentialManager { + static credentialsByTxInCache = new LRUCache({ + max: 50_000 + }); + txToCredentials = new Map(); getCachedCredential(txIn: Cardano.TxIn): Credential[] { - return credentialsByTxInCache.get(`${txIn.txId}#${txIn.index}`) ?? []; + return CredentialManager.credentialsByTxInCache.get(`${txIn.txId}#${txIn.index}`) ?? []; } deleteCachedCredential(txIn: Cardano.TxIn) { - credentialsByTxInCache.delete(`${txIn.txId}#${txIn.index}`); + CredentialManager.credentialsByTxInCache.delete(`${txIn.txId}#${txIn.index}`); } addCredential(txId: Cardano.TransactionId, { hash: credentialHash, type: credentialType }: Credential) { @@ -68,8 +69,8 @@ export class CredentialManager { if (paymentCredentialHash && paymentCredentialType) { this.addCredential(txId, { hash: paymentCredentialHash, type: paymentCredentialType }); if (outputIndex) { - credentialsByTxInCache.set(cacheKey, [ - ...(credentialsByTxInCache.get(cacheKey) || []), + CredentialManager.credentialsByTxInCache.set(cacheKey, [ + ...(CredentialManager.credentialsByTxInCache.get(cacheKey) || []), { hash: paymentCredentialHash, type: paymentCredentialType } ]); } @@ -82,8 +83,8 @@ export class CredentialManager { this.addCredential(txId, { hash: stakeCredential, type: stakeCredentialType }); if (outputIndex) { - credentialsByTxInCache.set(cacheKey, [ - ...(credentialsByTxInCache.get(cacheKey) || []), + CredentialManager.credentialsByTxInCache.set(cacheKey, [ + ...(CredentialManager.credentialsByTxInCache.get(cacheKey) || []), { hash: stakeCredential, type: stakeCredentialType } ]); } From 7e27e0db4acac7886db4626565b5a27ef9efc44c Mon Sep 17 00:00:00 2001 From: William Wolff Date: Wed, 18 Sep 2024 14:21:32 +0200 Subject: [PATCH 23/29] refactor: add back output entity index --- packages/projection-typeorm/src/entity/Output.entity.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/projection-typeorm/src/entity/Output.entity.ts b/packages/projection-typeorm/src/entity/Output.entity.ts index 1482d3df524..00e9ecd0af1 100644 --- a/packages/projection-typeorm/src/entity/Output.entity.ts +++ b/packages/projection-typeorm/src/entity/Output.entity.ts @@ -9,7 +9,7 @@ import { TokensEntity } from './Tokens.entity'; export class OutputEntity { @PrimaryGeneratedColumn() id?: number; - // @Index() + @Index() @Column('varchar') address?: Cardano.PaymentAddress; @Index() From d28ff85c54fae3b054dbbb21470d342caa78c77e Mon Sep 17 00:00:00 2001 From: William Wolff Date: Wed, 18 Sep 2024 14:21:59 +0200 Subject: [PATCH 24/29] fix: add check if resolvable txins are not empty --- .../src/operators/storeCredentials.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/projection-typeorm/src/operators/storeCredentials.ts b/packages/projection-typeorm/src/operators/storeCredentials.ts index 9ae8952bc94..9fbc51e3ab6 100644 --- a/packages/projection-typeorm/src/operators/storeCredentials.ts +++ b/packages/projection-typeorm/src/operators/storeCredentials.ts @@ -33,14 +33,16 @@ const addInputCredentials = async ( } } - const outputEntities = await utxoRepository.find({ - select: { address: true, outputIndex: true, txId: true }, - where: txInLookups - }); - - for (const hydratedTxIn of outputEntities) { - if (hydratedTxIn.address) { - manager.addCredentialFromAddress(txHash, Mappers.credentialsFromAddress(hydratedTxIn.address!)); + if (txInLookups.length > 0) { + const outputEntities = await utxoRepository.find({ + select: { address: true, outputIndex: true, txId: true }, + where: txInLookups + }); + + for (const hydratedTxIn of outputEntities) { + if (hydratedTxIn.address) { + manager.addCredentialFromAddress(txHash, Mappers.credentialsFromAddress(hydratedTxIn.address!)); + } } } } From 89331db5aaba27bf31f306ab43949cda017162ca Mon Sep 17 00:00:00 2001 From: William Wolff Date: Wed, 18 Sep 2024 14:22:09 +0200 Subject: [PATCH 25/29] fix: add export of credential manager --- packages/projection-typeorm/src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/projection-typeorm/src/index.ts b/packages/projection-typeorm/src/index.ts index 5a7e5ff488e..b58fda4219e 100644 --- a/packages/projection-typeorm/src/index.ts +++ b/packages/projection-typeorm/src/index.ts @@ -5,3 +5,4 @@ export * from './entity'; export * from './isRecoverableTypeormError'; export * from './operators'; export * from './pgBoss'; +export * from './CredentialManager'; From 8347299a886fa5b52b71eb4a96f4e0f92b507299 Mon Sep 17 00:00:00 2001 From: Martynas Kazlauskas Date: Wed, 18 Sep 2024 16:19:39 +0300 Subject: [PATCH 26/29] fix(cardano-services): remove storeAssets from transaction projection --- .../src/Projection/prepareTypeormProjection.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cardano-services/src/Projection/prepareTypeormProjection.ts b/packages/cardano-services/src/Projection/prepareTypeormProjection.ts index 6b5ac97c5e5..407a781e11a 100644 --- a/packages/cardano-services/src/Projection/prepareTypeormProjection.ts +++ b/packages/cardano-services/src/Projection/prepareTypeormProjection.ts @@ -293,7 +293,7 @@ const storeInterDependencies: Partial> = { storeStakePoolRewardsJob: ['storeBlock'], storeStakePools: ['storeBlock'], storeTransactions: ['storeCredentials', 'storeBlock', 'storeUtxo'], - storeUtxo: ['storeBlock', 'storeAssets'] + storeUtxo: ['storeBlock'] }; const projectionStoreDependencies: Record = { @@ -308,7 +308,7 @@ const projectionStoreDependencies: Record = { 'stake-pool-metrics-job': ['storePoolMetricsUpdateJob'], 'stake-pool-rewards-job': ['storeStakePoolRewardsJob'], transactions: ['storeCredentials', 'storeTransactions'], - utxo: ['storeUtxo'] + utxo: ['storeUtxo', 'storeAssets'] }; const registerMapper = ( From e96fb94b8136283860c17f86afc8ab20fd97f27c Mon Sep 17 00:00:00 2001 From: Martynas Kazlauskas Date: Wed, 18 Sep 2024 16:20:51 +0300 Subject: [PATCH 27/29] feat: set projection postgres tx isolation level to default --- .../src/Projection/createTypeormProjection.ts | 9 ++++++--- .../src/operators/withTypeormTransaction.ts | 7 +++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/cardano-services/src/Projection/createTypeormProjection.ts b/packages/cardano-services/src/Projection/createTypeormProjection.ts index 345bc447852..310d9d8ba85 100644 --- a/packages/cardano-services/src/Projection/createTypeormProjection.ts +++ b/packages/cardano-services/src/Projection/createTypeormProjection.ts @@ -155,10 +155,13 @@ export const createTypeormProjection = ({ shareRetryBackoff( (evt$) => evt$.pipe( - withTypeormTransaction({ connection$: connect() }), + withOperatorDuration( + 'withTypeormTransaction', + withTypeormTransaction({ connection$: connect() }) + ), applyStores(stores, { operatorNames: __debug.stores }), - buffer.storeBlockData(), - typeormTransactionCommit() + withOperatorDuration('storeBlockData', buffer.storeBlockData()), + withOperatorDuration('typeormTransactionCommit', typeormTransactionCommit()) ), { shouldRetry: isRecoverableTypeormError } ) diff --git a/packages/projection-typeorm/src/operators/withTypeormTransaction.ts b/packages/projection-typeorm/src/operators/withTypeormTransaction.ts index 3cab0fd163f..5332e3798ff 100644 --- a/packages/projection-typeorm/src/operators/withTypeormTransaction.ts +++ b/packages/projection-typeorm/src/operators/withTypeormTransaction.ts @@ -11,9 +11,11 @@ import { import { QueryRunner } from 'typeorm'; import { TypeormConnection } from '../createDataSource'; import omit from 'lodash/omit.js'; +import type { IsolationLevel } from 'typeorm/driver/types/IsolationLevel'; export interface WithTypeormTransactionDependencies { connection$: Observable; + isolationLevel?: IsolationLevel; } export interface WithTypeormContext { @@ -37,7 +39,8 @@ export function withTypeormTransaction( /** Start a PostgreSQL transaction for each event. {pgBoss: true} also adds {@link WithPgBoss} context. */ export function withTypeormTransaction({ - connection$ + connection$, + isolationLevel: transactionType }: WithTypeormTransactionDependencies & { pgBoss?: boolean }): UnifiedExtChainSyncOperator< Props, Props & WithTypeormContext & Partial @@ -53,7 +56,7 @@ export function withTypeormTransaction({ // - might be possible to optimize by setting a different isolation level, // but we're using the safest one until there's a need to optimize // https://www.postgresql.org/docs/current/transaction-iso.html - queryRunner.startTransaction('SERIALIZABLE').then(() => ({ transactionCommitted$: new Subject() })) + queryRunner.startTransaction(transactionType).then(() => ({ transactionCommitted$: new Subject() })) ) ) ); From 81565c9eef86ef1de3d5b3b8f7bfd14093b71c12 Mon Sep 17 00:00:00 2001 From: William Wolff Date: Mon, 23 Sep 2024 18:50:13 +0200 Subject: [PATCH 28/29] fix: add index for foreign relationship transaction to block id --- packages/projection-typeorm/src/entity/Transaction.entity.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/projection-typeorm/src/entity/Transaction.entity.ts b/packages/projection-typeorm/src/entity/Transaction.entity.ts index 19bc0664fc2..51294a551a7 100644 --- a/packages/projection-typeorm/src/entity/Transaction.entity.ts +++ b/packages/projection-typeorm/src/entity/Transaction.entity.ts @@ -13,6 +13,7 @@ export class TransactionEntity { @Column('varchar', { nullable: false }) cbor?: Serialization.TxCBOR; + @Index() @ManyToOne(() => BlockEntity, OnDeleteCascadeRelationOptions) @JoinColumn({ name: 'block_id' }) block?: BlockEntity; From 7eb46bbb70e1f5950fa964af6aea33037f51f48b Mon Sep 17 00:00:00 2001 From: William Wolff Date: Thu, 10 Oct 2024 16:53:04 +0300 Subject: [PATCH 29/29] feat: substitute tx input indexing by witness pub keys --- .../src/operators/storeCredentials.ts | 84 +++++++++++-------- 1 file changed, 49 insertions(+), 35 deletions(-) diff --git a/packages/projection-typeorm/src/operators/storeCredentials.ts b/packages/projection-typeorm/src/operators/storeCredentials.ts index 9fbc51e3ab6..19771736135 100644 --- a/packages/projection-typeorm/src/operators/storeCredentials.ts +++ b/packages/projection-typeorm/src/operators/storeCredentials.ts @@ -1,10 +1,11 @@ /* eslint-disable sonarjs/cognitive-complexity */ import { Cardano, ChainSyncEventType } from '@cardano-sdk/core'; -import { CredentialEntity, CredentialType, OutputEntity } from '../entity'; +import { CredentialEntity, CredentialType } from '../entity'; import { Mappers } from '@cardano-sdk/projection'; -import { Repository } from 'typeorm'; +// import { Repository } from 'typeorm'; import { typeormOperator } from './util'; +import * as Crypto from '@cardano-sdk/crypto'; import { CredentialManager } from '../CredentialManager'; export interface WithTxCredentials { @@ -13,37 +14,50 @@ export interface WithTxCredentials { export const willStoreCredentials = ({ utxoByTx }: Mappers.WithUtxo) => Object.keys(utxoByTx).length > 0; -const addInputCredentials = async ( - utxoByTx: Record, - utxoRepository: Repository, - manager: CredentialManager -) => { - for (const txHash of Object.keys(utxoByTx) as Cardano.TransactionId[]) { - const txInLookups: { outputIndex: number; txId: Cardano.TransactionId }[] = []; - - for (const txIn of utxoByTx[txHash]!.consumed) { - const cachedCredentials = manager.getCachedCredential(txIn); - if (cachedCredentials.length > 0) { - for (const credential of cachedCredentials) { - manager.addCredential(txHash, { hash: credential.hash, type: credential.type ?? undefined }); - } - manager.deleteCachedCredential(txIn); // can only be consumed once so chances are it won't have to be resolved again - } else { - txInLookups.push({ outputIndex: txIn.index, txId: txIn.txId }); - } - } - - if (txInLookups.length > 0) { - const outputEntities = await utxoRepository.find({ - select: { address: true, outputIndex: true, txId: true }, - where: txInLookups +// const addInputCredentials = async ( +// utxoByTx: Record, +// utxoRepository: Repository, +// manager: CredentialManager +// ) => { +// for (const txHash of Object.keys(utxoByTx) as Cardano.TransactionId[]) { +// const txInLookups: { outputIndex: number; txId: Cardano.TransactionId }[] = []; + +// for (const txIn of utxoByTx[txHash]!.consumed) { +// const cachedCredentials = manager.getCachedCredential(txIn); +// if (cachedCredentials.length > 0) { +// for (const credential of cachedCredentials) { +// manager.addCredential(txHash, { hash: credential.hash, type: credential.type ?? undefined }); +// } +// manager.deleteCachedCredential(txIn); // can only be consumed once so chances are it won't have to be resolved again +// } else { +// txInLookups.push({ outputIndex: txIn.index, txId: txIn.txId }); +// } +// } + +// if (txInLookups.length > 0) { +// const outputEntities = await utxoRepository.find({ +// select: { address: true, outputIndex: true, txId: true }, +// where: txInLookups +// }); + +// for (const hydratedTxIn of outputEntities) { +// if (hydratedTxIn.address) { +// manager.addCredentialFromAddress(txHash, Mappers.credentialsFromAddress(hydratedTxIn.address!)); +// } +// } +// } +// } +// }; + +const addWitnessCredentials = async (txs: Cardano.OnChainTx[], manager: CredentialManager) => { + for (const tx of txs) { + const pubKeys = Object.keys(tx.witness.signatures) as Crypto.Ed25519PublicKeyHex[]; + for (const pubKey of pubKeys) { + const credential = await Crypto.Ed25519PublicKey.fromHex(pubKey).hash(); + manager.addCredential(tx.id, { + hash: Crypto.Hash28ByteBase16(credential.hex()), + type: CredentialType.PaymentKey }); - - for (const hydratedTxIn of outputEntities) { - if (hydratedTxIn.address) { - manager.addCredentialFromAddress(txHash, Mappers.credentialsFromAddress(hydratedTxIn.address!)); - } - } } } }; @@ -83,7 +97,6 @@ export const storeCredentials = typeormOperator< eventType, queryRunner, stakeCredentialsByTx, - utxoByTx, utxo: { consumed: consumedUTxOs } } = evt; @@ -94,10 +107,11 @@ export const storeCredentials = typeormOperator< return { credentialsByTx: Object.fromEntries(manager.txToCredentials) }; } - const utxoRepository = queryRunner.manager.getRepository(OutputEntity); - await addInputCredentials(utxoByTx, utxoRepository, manager); + // const utxoRepository = queryRunner.manager.getRepository(OutputEntity); + // await addInputCredentials(utxoByTx, utxoRepository, manager); addOutputCredentials(addressesByTx, manager); addCertificateCredentials(stakeCredentialsByTx, manager); + addWitnessCredentials(txs, manager); // insert new credentials & ignore conflicts of existing ones await queryRunner.manager