Skip to content

Commit

Permalink
fix getTableRows use case for better identity columns handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@Artuomka
Artuomka committed Feb 12, 2025
1 parent 5c170ef commit 851e80d
Show file tree
Hide file tree
Showing 2 changed files with 111 additions and 44 deletions.
91 changes: 63 additions & 28 deletions backend/src/entities/table/use-cases/get-table-rows.use.case.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import Sentry from '@sentry/minimal';
import { processRowsUtil } from '../utils/process-found-rows-util.js';
import JSON5 from 'json5';
import { buildActionEventDto } from '../../table-actions/table-action-rules-module/utils/build-found-action-event-dto.util.js';
import { TableSettingsDS } from '@rocketadmin/shared-code/dist/src/data-access-layer/shared/data-structures/table-settings.ds.js';

@Injectable()
export class GetTableRowsUseCase extends AbstractUseCase<GetTableRowsDs, FoundTableRowsDs> implements IGetTableRows {
Expand Down Expand Up @@ -231,34 +232,38 @@ export class GetTableRowsUseCase extends AbstractUseCase<GetTableRowsDs, FoundTa
allow_csv_import: allowCsvImport,
};

let identities = [];
const identities = [];

if (tableForeignKeys?.length > 0) {
identities = await Promise.all(
tableForeignKeys.map(async (foreignKey) => {
const foreignKeysValuesCollection = rowsRO.rows
.filter((row) => row[foreignKey.column_name])
.map((row) => row[foreignKey.column_name]) as (string | number)[];

const foreignTableSettings = await this._dbContext.tableSettingsRepository.findTableSettings(
connectionId,
foreignKey.referenced_table_name,
);

const identityColumns = await dao.getIdentityColumns(
foreignKey.referenced_table_name,
foreignKey.referenced_column_name,
foreignTableSettings?.identity_column,
foreignKeysValuesCollection,
userEmail,
);

return {
for (const foreignKey of tableForeignKeys) {
const foreignKeysValuesCollection = rowsRO.rows
.filter((row) => row[foreignKey.column_name])
.map((row) => row[foreignKey.column_name]) as (string | number)[];

const foreignTableSettings = await this._dbContext.tableSettingsRepository.findTableSettings(
connectionId,
foreignKey.referenced_table_name,
);

const identityColumns = await this.getBatchedIdentityColumns(
foreignKeysValuesCollection,
foreignKey,
dao,
foreignTableSettings,
userEmail,
);

if (identities.findIndex((el) => el.referenced_table_name === foreignKey.referenced_table_name) > -1) {
identities
.find((el) => el.referenced_table_name === foreignKey.referenced_table_name)
.identity_columns.push(...identityColumns);
} else {
identities.push({
referenced_table_name: foreignKey.referenced_table_name,
identity_columns: identityColumns,
};
}),
);
});
}
}
}

const foreignKeysConformity = tableForeignKeys.map((key) => ({
Expand All @@ -276,15 +281,11 @@ export class GetTableRowsUseCase extends AbstractUseCase<GetTableRowsDs, FoundTa
const foundIdentityForCurrentValue = foundIdentityForCurrentTable?.identity_columns.find(
(el) => el[element.realFKeyName] === row[element.currentFKeyName],
);

row[element.currentFKeyName] = foundIdentityForCurrentValue ? { ...foundIdentityForCurrentValue } : {};
});
});

operationResult = OperationResultStatusEnum.successfully;
if (connectionId === 'JYCEZYqk' && tableName === 'followers') {
console.log('PROCESS ROWS ON LAST STAGE BEFORE RETURNING TO USER ', JSON.stringify(rowsRO.rows));
}
return rowsRO;
} catch (e) {
Sentry.captureException(e);
Expand Down Expand Up @@ -344,4 +345,38 @@ export class GetTableRowsUseCase extends AbstractUseCase<GetTableRowsDs, FoundTa
};
}
}

private chunkArray<T>(array: T[], chunkSize: number): T[][] {
const results = [];
for (let i = 0; i < array.length; i += chunkSize) {
results.push(array.slice(i, i + chunkSize));
}
return results;
}

private async getBatchedIdentityColumns(
foreignKeysValuesCollection: Array<string | number>,
foreignKey: ForeignKeyDS,
dao: IDataAccessObject | IDataAccessObjectAgent,
foreignTableSettings: TableSettingsDS,
userEmail: string,
): Promise<Array<Record<string, unknown>>> {
foreignKeysValuesCollection = [...new Set(foreignKeysValuesCollection)];
const batchSize = 50;
const chunkedValues = this.chunkArray(foreignKeysValuesCollection, batchSize);
let identityColumns = [];

for (const chunk of chunkedValues) {
const result = await dao.getIdentityColumns(
foreignKey.referenced_table_name,
foreignKey.referenced_column_name,
foreignTableSettings?.identity_column,
chunk,
userEmail,
);
identityColumns = identityColumns.concat(result);
}

return identityColumns;
}
}
64 changes: 48 additions & 16 deletions backend/test/ava-tests/saas-tests/different-tables-structures-e2e-test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import { fileURLToPath } from 'url';
import { join } from 'path';
import { Cacher } from '../../../src/helpers/cache/cacher.js';
import { clearAllTestKnex, getTestKnex } from '../../utils/get-test-knex.js';
import { Knex } from 'knex';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

Expand Down Expand Up @@ -92,15 +93,48 @@ const createTransactionsTableRawQuery = `CREATE TABLE transactions (
FOREIGN KEY (reviewerId) REFERENCES users(userId)
);`;

const fillUsersTableRawQuery = `INSERT INTO users (username) VALUES
('Alice'),
('Bob'),
('Charlie');`;
const testEntitiesCount = 500;

const fillTransactionsRawQuery = `INSERT INTO transactions (buyerId, reviewerId, transaction_date, description, amount) VALUES
(1, 2, '2023-10-01', 'Purchase of office supplies', 150.00),
(2, 3, '2023-10-02', 'Consulting services payment', 200.50),
(3, 1, '2023-10-03', 'Design contract', 750.75);`;
async function fillUsersTable(knex: Knex<any, any[]>): Promise<void> {
for (let i = 0; i < testEntitiesCount; i++) {
await knex('users').insert({ username: faker.person.firstName() });
}
}

const insertedIdCombos: Array<{ buyerid: number; reviewerid: number }> = [];

async function fillTransactionsTable(knex: Knex<any, any[]>): Promise<void> {
for (let i = 0; i < testEntitiesCount; i++) {
const { buyerid, reviewerid } = getUniqueIdsComboForInsert();
await knex('transactions').insert({
buyerid: buyerid,
reviewerid: reviewerid,
transaction_date: faker.date.past(),
description: faker.lorem.sentence(),
amount: faker.number.int({ min: 100, max: 10000 }),
});
}
}

function getUniqueIdsCombo(): { buyerid: number; reviewerid: number } {
return {
buyerid: faker.number.int({ min: 1, max: testEntitiesCount }),
reviewerid: faker.number.int({ min: 1, max: testEntitiesCount }),
};
}

function getUniqueIdsComboForInsert(): { buyerid: number; reviewerid: number } {
if (insertedIdCombos.length >= testEntitiesCount) {
return insertedIdCombos.shift();
}
while (true) {
const { buyerid, reviewerid } = getUniqueIdsCombo();
if (!insertedIdCombos.find((idCombo) => idCombo.buyerid === buyerid && idCombo.reviewerid === reviewerid)) {
insertedIdCombos.push({ buyerid, reviewerid });
return { buyerid, reviewerid };
}
}
}

async function loadTestData(): Promise<void> {
const connectionToTestDB = getTestData(mockFactory).connectionToPostgres;
Expand All @@ -109,8 +143,8 @@ async function loadTestData(): Promise<void> {
await testKnex.raw(`DROP TABLE IF EXISTS users;`);
await testKnex.raw(createUsersTableRawQuery);
await testKnex.raw(createTransactionsTableRawQuery);
await testKnex.raw(fillUsersTableRawQuery);
await testKnex.raw(fillTransactionsRawQuery);
await fillUsersTable(testKnex);
await fillTransactionsTable(testKnex);
await clearAllTestKnex();
}

Expand Down Expand Up @@ -161,7 +195,7 @@ test.serial(`${currentTest} should return list of rows of the tables`, async (t)
t.is(createConnectionResponse.status, 201);

const foundRowsFromUsersTableResponse = await request(app.getHttpServer())
.get(`/table/rows/${createConnectionRO.id}?tableName=${testUsersTableName}`)
.get(`/table/rows/${createConnectionRO.id}?tableName=${testUsersTableName}&page=1&perPage=500`)
.set('Cookie', firstUserToken)
.set('Content-Type', 'application/json')
.set('Accept', 'application/json');
Expand All @@ -171,14 +205,14 @@ test.serial(`${currentTest} should return list of rows of the tables`, async (t)
t.is(foundRowsFromUsersTableRO.hasOwnProperty('rows'), true);
t.is(foundRowsFromUsersTableRO.hasOwnProperty('primaryColumns'), true);
t.is(foundRowsFromUsersTableRO.hasOwnProperty('pagination'), true);
t.is(foundRowsFromUsersTableRO.rows.length, 3);
t.is(foundRowsFromUsersTableRO.rows.length, 500);
t.is(Object.keys(foundRowsFromUsersTableRO.rows[1]).length, 4);
t.is(foundRowsFromUsersTableRO.rows[0].hasOwnProperty('userid'), true);
t.is(foundRowsFromUsersTableRO.rows[1].hasOwnProperty('username'), true);
t.is(foundRowsFromUsersTableRO.rows[2].hasOwnProperty('created_at'), true);

const foundRowsFromTransactionsTableResponse = await request(app.getHttpServer())
.get(`/table/rows/${createConnectionRO.id}?tableName=${testTransactionsTableName}`)
.get(`/table/rows/${createConnectionRO.id}?tableName=${testTransactionsTableName}&page=1&perPage=500`)
.set('Cookie', firstUserToken)
.set('Content-Type', 'application/json')
.set('Accept', 'application/json');
Expand All @@ -188,7 +222,7 @@ test.serial(`${currentTest} should return list of rows of the tables`, async (t)
t.is(foundRowsFromTransactionsTableRO.hasOwnProperty('rows'), true);
t.is(foundRowsFromTransactionsTableRO.hasOwnProperty('primaryColumns'), true);
t.is(foundRowsFromTransactionsTableRO.hasOwnProperty('pagination'), true);
t.is(foundRowsFromTransactionsTableRO.rows.length, 3);
t.is(foundRowsFromTransactionsTableRO.rows.length, 500);
t.is(Object.keys(foundRowsFromTransactionsTableRO.rows[1]).length, 7);
t.is(foundRowsFromTransactionsTableRO.rows[0].hasOwnProperty('buyerid'), true);
t.is(foundRowsFromTransactionsTableRO.rows[1].hasOwnProperty('reviewerid'), true);
Expand All @@ -205,8 +239,6 @@ test.serial(`${currentTest} should return list of rows of the tables`, async (t)
t.is(reviewerId.hasOwnProperty('userid'), true);
t.is(typeof reviewerId.userid, 'number');
}


} catch (error) {
console.error(error);
throw error;
Expand Down

0 comments on commit 851e80d

Please sign in to comment.