refactor(api-http): adapt to new transaction format (#730)

* tx refactor

* use RLP encoding for signature / hash calculation

* regenerate genesis block

* style: resolve style guide violations

* fix some tests

* style: resolve style guide violations

* update imports

* set default network

* fix crypto-block fixtures

* style: resolve style guide violations

* unit test fixes

* update e2e configs

* remove obsolete deps

* style: resolve style guide violations

* remove obsolete type field

* regenerate genesis

* update e2e

* style: resolve style guide violations

* wip api tx adjustments

* index transaction function sig

* fix null

* update query builder

* integration test fixes

* style: resolve style guide violations

---------

Co-authored-by: oXtxNt9U <[email protected]>
Co-authored-by: sebastijankuzner <[email protected]>

packages/api-database/source/contracts.ts

@@ -65,8 +65,6 @@ export type TransactionTypeRepositoryExtension = {};
 export type TransactionTypeRepository = ExtendedRepository<TransactionType> & TransactionTypeRepositoryExtension;
 
 export type FeeStatistics = {
-	type: number;
-	typeGroup: number;
 	avg: string;
 	min: string;
 	max: string;
@@ -82,7 +80,7 @@ export type TransactionRepositoryExtension = {
 		options?: Options,
 	): Promise<ResultsPage<Transaction>>;
 
-	getFeeStatistics(genesisTimestamp: number, days?: number, minFee?: number): Promise<FeeStatistics[]>;
+	getFeeStatistics(genesisTimestamp: number, days?: number, minFee?: number): Promise<FeeStatistics | undefined>;
 };
 export type TransactionRepository = ExtendedRepository<Transaction> & TransactionRepositoryExtension;
 

packages/api-database/source/migrations/1697617471901-CreateIndexes.ts

@@ -5,44 +5,36 @@ export class CreateIndexes1697617471901 implements MigrationInterface {
 		// language=postgresql
 		await queryRunner.query(`
             CREATE UNIQUE INDEX transactions_sender_nonce ON transactions(sender_public_key, nonce);
-            CREATE INDEX transactions_recipient_id ON transactions(recipient_id);
+            CREATE INDEX transactions_recipient_address ON transactions(recipient_address);
             CREATE INDEX transactions_sender ON transactions(sender_public_key);
+            CREATE INDEX transactions_sender_address ON transactions(sender_address);
 
             CREATE INDEX transactions_block_id ON transactions(block_id);
             CREATE INDEX transactions_block_height_sequence ON transactions(block_height, sequence);
 
-            CREATE INDEX transactions_asset ON transactions USING GIN(asset) WITH (fastupdate = off);
-            CREATE INDEX transactions_asset_payments ON transactions using gin ((asset -> 'payments'::text)) WITH (fastupdate = off);
-
             CREATE INDEX transactions_amount ON transactions(amount);
-            CREATE INDEX transactions_fee ON transactions(fee);
+            CREATE INDEX transactions_gas_price ON transactions(gas_price);
             CREATE INDEX transactions_nonce ON transactions(nonce);
-            CREATE INDEX transactions_vendor_field ON transactions(vendor_field);
-            CREATE INDEX transactions_version ON transactions(version);
 
             CREATE INDEX transactions_amount_sequence ON transactions(amount, sequence);
-            CREATE INDEX transactions_fee_sequence ON transactions(fee, sequence);
+            CREATE INDEX transactions_gas_price_sequence ON transactions(gas_price, sequence);
             CREATE INDEX transactions_nonce_sequence ON transactions(nonce, sequence);
             CREATE INDEX transactions_timestamp_sequence ON transactions(timestamp, sequence);
-            CREATE INDEX transactions_type_sequence ON transactions(type, sequence);
-            CREATE INDEX transactions_type_group_sequence ON transactions(type_group, sequence);
-            CREATE INDEX transactions_vendor_field_sequence ON transactions(vendor_field, sequence);
-            CREATE INDEX transactions_version_sequence ON transactions(version, sequence);
 
             CREATE INDEX transactions_amount_asc_sequence_desc ON transactions(amount ASC, sequence DESC);
-            CREATE INDEX transactions_fee_asc_sequence_desc ON transactions(fee ASC, sequence DESC);
+            CREATE INDEX transactions_gas_price_asc_sequence_desc ON transactions(gas_price ASC, sequence DESC);
             CREATE INDEX transactions_nonce_asc_sequence_desc ON transactions(nonce ASC, sequence DESC);
             CREATE INDEX transactions_timestamp_asc_sequence_desc ON transactions(timestamp ASC, sequence DESC);
-            CREATE INDEX transactions_type_asc_sequence_desc ON transactions(type ASC, sequence DESC);
-            CREATE INDEX transactions_type_group_asc_sequence_desc ON transactions(type_group ASC, sequence DESC);
-            CREATE INDEX transactions_vendor_field_asc_sequence_desc ON transactions(vendor_field ASC, sequence DESC);
-            CREATE INDEX transactions_version_asc_sequence_desc ON transactions(version ASC, sequence DESC);
+
+            CREATE INDEX transactions_function_sig_address ON transactions(
+                  SUBSTRING(data FROM 1 FOR 4),
+                  recipient_address
+            );
 
             CREATE INDEX blocks_number_of_transactions ON blocks(number_of_transactions);
             CREATE INDEX blocks_reward ON blocks(reward);
             CREATE INDEX blocks_total_amount ON blocks(total_amount);
             CREATE INDEX blocks_total_fee ON blocks(total_fee);
-            CREATE INDEX blocks_version ON blocks(version);
             CREATE INDEX blocks_validator_round ON blocks(validator_round);
 
             CREATE INDEX receipts_block_height ON receipts(block_height);
@@ -58,46 +50,36 @@ export class CreateIndexes1697617471901 implements MigrationInterface {
 		// language=postgresql
 		await queryRunner.query(`
             DROP INDEX transactions_sender_nonce;
-            DROP INDEX transactions_recipient_id;
+            DROP INDEX transactions_recipient_address;
             DROP INDEX transactions_sender;
+            DROP INDEX transactions_sender_address;
 
             DROP INDEX transactions_block_id;
             DROP INDEX transactions_block_height_sequence;
 
-            DROP INDEX transactions_asset;
-            DROP INDEX transactions_asset_payments;
-
             DROP INDEX transactions_amount;
-            DROP INDEX transactions_fee;
+            DROP INDEX transactions_gas_price;
             DROP INDEX transactions_nonce;
-            DROP INDEX transactions_vendor_field;
-            DROP INDEX transactions_version;
 
             DROP INDEX transactions_amount_sequence;
-            DROP INDEX transactions_fee_sequence;
+            DROP INDEX transactions_gas_price_sequence;
             DROP INDEX transactions_nonce_sequence;
             DROP INDEX transactions_timestamp_sequence;
-            DROP INDEX transactions_type_sequence;
-            DROP INDEX transactions_type_group_sequence;
-            DROP INDEX transactions_vendor_field_sequence;
-            DROP INDEX transactions_version_sequence;
 
             DROP INDEX transactions_amount_asc_sequence_desc;
-            DROP INDEX transactions_fee_asc_sequence_desc;
+            DROP INDEX transactions_gas_price_asc_sequence_desc;
             DROP INDEX transactions_nonce_asc_sequence_desc;
             DROP INDEX transactions_timestamp_asc_sequence_desc;
-            DROP INDEX transactions_type_asc_sequence_desc;
-            DROP INDEX transactions_type_group_asc_sequence_desc;
-            DROP INDEX transactions_vendor_field_asc_sequence_desc;
-            DROP INDEX transactions_version_asc_sequence_desc;
+            DROP INDEX transactions_function_sig_address;
 
             DROP INDEX blocks_number_of_transactions;
             DROP INDEX blocks_reward;
             DROP INDEX blocks_total_amount;
             DROP INDEX blocks_total_fee;
-            DROP INDEX blocks_version;
             DROP INDEX blocks_validator_round;
 
+            DROP INDEX receipts_block_height;
+
             DROP INDEX wallets_balance;
             DROP INDEX wallets_attributes;
             DROP INDEX wallets_validators;

packages/api-database/source/models/transaction-type.ts

@@ -6,24 +6,6 @@ import { Column, Entity } from "typeorm";
 export class TransactionType {
 	@Column({
 		primary: true,
-		type: "smallint",
-	})
-	public type!: number;
-
-	@Column({
-		primary: true,
-		type: "integer",
-	})
-	public typeGroup!: number;
-
-	@Column({
-		primary: true,
-		type: "smallint",
-	})
-	public version!: number;
-
-	@Column({
-		nullable: false,
 		type: "varchar",
 	})
 	public key!: string;

packages/api-database/source/models/transaction.ts

@@ -1,6 +1,6 @@
 import { Column, Entity } from "typeorm";
 
-import { vendorFieldTransformer } from "../transformers/vendor-field.js";
+import { bufferTransformer } from "../transformers/buffer.js";
 
 @Entity({
 	name: "transactions",
@@ -10,97 +10,82 @@ export class Transaction {
 		primary: true,
 		type: "varchar",
 	})
-	public id!: string;
-
-	@Column({
-		nullable: false,
-		type: "smallint",
-	})
-	public version!: number;
-
-	@Column({
-		nullable: false,
-		type: "smallint",
-	})
-	public type!: number;
-
-	@Column({
-		default: 1,
-		nullable: false,
-		type: "integer",
-	})
-	public typeGroup!: number;
+	public readonly id!: string;
 
 	@Column({
 		nullable: false,
 		type: "varchar",
 	})
-	public blockId!: string;
+	public readonly blockId!: string;
 
 	@Column({
 		nullable: false,
 		type: "bigint",
 	})
-	public blockHeight!: string;
+	public readonly blockHeight!: string;
 
 	@Column({
 		nullable: false,
 		type: "smallint",
 	})
-	public sequence!: number;
+	public readonly sequence!: number;
 
 	@Column({
 		nullable: false,
 		type: "bigint",
 	})
-	public timestamp!: string;
+	public readonly timestamp!: string;
 
 	@Column({
 		nullable: false,
 		type: "bigint",
 	})
-	public nonce!: string;
+	public readonly nonce!: string;
 
 	@Column({
 		nullable: false,
 		type: "varchar",
 	})
-	public senderPublicKey!: string;
+	public readonly senderPublicKey!: string;
 
 	@Column({
-		default: undefined,
-		nullable: true,
+		nullable: false,
 		type: "varchar",
 	})
-	public recipientId!: string | undefined;
+	public readonly senderAddress!: string;
 
 	@Column({
 		default: undefined,
 		nullable: true,
-		transformer: vendorFieldTransformer,
-		type: "bytea",
+		type: "varchar",
 	})
-	public vendorField: string | undefined;
+	public readonly recipientAddress!: string | undefined;
 
 	@Column({
 		nullable: false,
 		type: "numeric",
 	})
-	public amount!: string;
+	public readonly amount!: string;
 
 	@Column({
 		nullable: false,
-		type: "numeric",
+		type: "bigint",
 	})
-	public fee!: string;
+	public readonly gasPrice!: number;
+
+	@Column({
+		nullable: false,
+		type: "bigint",
+	})
+	public readonly gasLimit!: number;
 
 	@Column({
 		default: undefined,
 		nullable: true,
-		type: "jsonb",
-		// TODO: separate tables for 1:n assets
+		transformer: bufferTransformer,
+		type: "bytea",
 	})
-	public asset: Record<string, any> | undefined;
+	public readonly data: string | undefined;
 
 	@Column({
 		default: undefined,

packages/api-database/source/repositories/transaction-repository.ts

@@ -25,43 +25,41 @@ export const makeTransactionRepository = (dataSource: RepositoryDataSource): Tra
 			return this.listByExpression(transactionExpression, sorting, pagination, options);
 		},
 
-		async getFeeStatistics(genesisTimestamp: number, days?: number, minFee = 0): Promise<FeeStatistics[]> {
+		async getFeeStatistics(
+			genesisTimestamp: number,
+			days?: number,
+			minGasPrice = 0,
+		): Promise<FeeStatistics | undefined> {
 			if (days) {
 				const age = Math.max(dayjs().subtract(days, "day").valueOf() - 1, genesisTimestamp);
 
 				return this.createQueryBuilder()
-					.select(['type_group AS "typeGroup"', "type"])
-					.addSelect("TRUNC(COALESCE(AVG(fee), 0)::numeric)", "avg")
-					.addSelect("TRUNC(COALESCE(MIN(fee), 0)::numeric)", "min")
-					.addSelect("TRUNC(COALESCE(MAX(fee), 0)::numeric)", "max")
-					.addSelect("TRUNC(COALESCE(SUM(fee), 0)::numeric)", "sum")
-					.where("timestamp > :age AND fee >= :minFee", { age, minFee })
-					.groupBy("type_group")
-					.addGroupBy("type")
-					.orderBy("type_group")
-					.addOrderBy("type")
-					.getRawMany();
+					.select("TRUNC(COALESCE(AVG(gas_price), 0)::numeric)", "avg")
+					.addSelect("TRUNC(COALESCE(MIN(gas_price), 0)::numeric)", "min")
+					.addSelect("TRUNC(COALESCE(MAX(gas_price), 0)::numeric)", "max")
+					.addSelect("TRUNC(COALESCE(SUM(gas_price), 0)::numeric)", "sum")
+					.where("timestamp > :age AND gas_price >= :minGasPrice", { age, minGasPrice })
+					.getRawOne();
 			}
 
 			// no days parameter, take the stats from each type for its last 20 txs
-			return this.manager.query<FeeStatistics[]>(
+			const result = await this.manager.query<FeeStatistics>(
 				`
-				select t_outer.type_group as "typeGroup", t_outer.type as "type",
-					TRUNC(COALESCE(AVG(fee), 0)::numeric) AS "avg",
-					TRUNC(COALESCE(MIN(fee), 0)::numeric) AS "min",
-					TRUNC(COALESCE(MIN(fee), 0)::numeric) AS "max",
-					TRUNC(COALESCE(MAX(fee), 0)::numeric) AS "sum"
-				from transactions t_outer
-				join lateral (
-					select 1 from transactions t_inner
-					where t_inner.timestamp > $1 and t_inner.type_group = t_outer.type_group and t_inner.type = t_outer.type and fee >= $2
-					order by t_inner.timestamp desc
+				select 
+					TRUNC(COALESCE(AVG(txs.gas_price), 0)::numeric) AS "avg",
+					TRUNC(COALESCE(MIN(txs.gas_price), 0)::numeric) AS "min",
+					TRUNC(COALESCE(MAX(txs.gas_price), 0)::numeric) AS "max",
+					TRUNC(COALESCE(SUM(txs.gas_price), 0)::numeric) AS "sum"
+				from (
+					select gas_price from transactions
+					where timestamp > $1 and gas_price >= $2
+					order by timestamp desc
 					limit $3
-				) t_limit on true
-				group by t_outer.type_group, t_outer.type
-				order by t_outer.type_group, t_outer.type;
+				) txs;
 			`,
-				[genesisTimestamp, minFee, 20],
+				[genesisTimestamp, minGasPrice, 20],
 			);
+
+			return result?.[0] ?? undefined;
 		},
 	});

packages/api-database/source/search/criteria.ts

@@ -46,12 +46,11 @@ export type TransactionCriteria = {
 	timestamp?: OrNumericCriteria<number>;
 	nonce?: OrNumericCriteria<string>;
 	senderPublicKey?: OrEqualCriteria<string>;
-	type?: OrEqualCriteria<number>;
-	typeGroup?: OrEqualCriteria<number>;
-	vendorField?: OrLikeCriteria<string>;
+	senderAddress?: OrEqualCriteria<string>;
 	amount?: OrNumericCriteria<string>;
-	fee?: OrNumericCriteria<string>;
-	asset?: OrContainsCriteria<Record<string, any>>;
+	gasPrice?: OrNumericCriteria<number>;
+	gasFee?: OrNumericCriteria<number>;
+	data?: OrEqualCriteria<string>;
 };
 
 export type OrTransactionCriteria = OrCriteria<TransactionCriteria>;

packages/api-database/source/search/expressions.ts

@@ -65,6 +65,12 @@ export type JsonbAttributeExists<TEntity> = {
 	attribute: string;
 };
 
+export type FunctionSigExpression<TEntity> = {
+	property: keyof TEntity;
+	op: "functionSig";
+	value: string;
+};
+
 export type Expression<TEntity> =
 	| TrueExpression
 	| FalseExpression
@@ -76,7 +82,8 @@ export type Expression<TEntity> =
 	| ContainsExpression<TEntity>
 	| AndExpression<TEntity>
 	| OrExpression<TEntity>
-	| JsonbAttributeExists<TEntity>;
+	| JsonbAttributeExists<TEntity>
+	| FunctionSigExpression<TEntity>;
 
 export type JsonFieldOperator = "->>";
 export type JsonFieldCastType = "bigint" | "numeric";