diff --git a/api/src/modules/import-data/eudr/eudr.import.service.ts b/api/src/modules/import-data/eudr/eudr.import.service.ts index 4f6119412..16643a36e 100644 --- a/api/src/modules/import-data/eudr/eudr.import.service.ts +++ b/api/src/modules/import-data/eudr/eudr.import.service.ts @@ -66,7 +66,7 @@ export class EudrImportService { title: 'Sourcing Records import from EUDR input file', }); - await this.cleanDataBeforeImport(); + //await this.cleanDataBeforeImport(); // TODO: Check what do we need to do with indicators and specially materials: // Do we need to ingest new materials? Activate some through the import? Activate all? @@ -193,26 +193,26 @@ export class EudrImportService { throw new BadRequestException(validationErrorArray); } - /** - * @note: Deletes DB content from required entities - * to ensure DB is prune prior loading a XLSX dataset - */ - async cleanDataBeforeImport(): Promise { - this.logger.log('Cleaning database before import...'); - try { - await this.indicatorService.deactivateAllIndicators(); - await this.materialService.deactivateAllMaterials(); - await this.scenarioService.clearTable(); - await this.indicatorRecordService.clearTable(); - await this.businessUnitService.clearTable(); - await this.supplierService.clearTable(); - await this.sourcingLocationService.clearTable(); - await this.sourcingRecordService.clearTable(); - await this.geoRegionsService.deleteGeoRegionsCreatedByUser(); - } catch (e: any) { - throw new Error( - `Database could not been cleaned before loading new dataset: ${e.message}`, - ); - } - } + // /** + // * @note: Deletes DB content from required entities + // * to ensure DB is prune prior loading a XLSX dataset + // */ + // async cleanDataBeforeImport(): Promise { + // this.logger.log('Cleaning database before import...'); + // try { + // await this.indicatorService.deactivateAllIndicators(); + // await this.materialService.deactivateAllMaterials(); + // await this.scenarioService.clearTable(); + // await this.indicatorRecordService.clearTable(); + // await this.businessUnitService.clearTable(); + // await this.supplierService.clearTable(); + // await this.sourcingLocationService.clearTable(); + // await this.sourcingRecordService.clearTable(); + // await this.geoRegionsService.deleteGeoRegionsCreatedByUser(); + // } catch (e: any) { + // throw new Error( + // `Database could not been cleaned before loading new dataset: ${e.message}`, + // ); + // } + // } } diff --git a/api/src/modules/import-data/import-data.module.ts b/api/src/modules/import-data/import-data.module.ts index d5dae5113..fe01a8744 100644 --- a/api/src/modules/import-data/import-data.module.ts +++ b/api/src/modules/import-data/import-data.module.ts @@ -3,15 +3,11 @@ import { ImportDataController } from 'modules/import-data/import-data.controller import { MaterialsModule } from 'modules/materials/materials.module'; import { BusinessUnitsModule } from 'modules/business-units/business-units.module'; import { SuppliersModule } from 'modules/suppliers/suppliers.module'; -import { AdminRegionsModule } from 'modules/admin-regions/admin-regions.module'; import { SourcingLocationsModule } from 'modules/sourcing-locations/sourcing-locations.module'; -import { SourcingRecordsModule } from 'modules/sourcing-records/sourcing-records.module'; -import { SourcingLocationGroupsModule } from 'modules/sourcing-location-groups/sourcing-location-groups.module'; import { FileService } from 'modules/import-data/file.service'; import { SourcingDataImportService } from 'modules/import-data/sourcing-data/sourcing-data-import.service'; import { SourcingRecordsDtoProcessorService } from 'modules/import-data/sourcing-data/dto-processor.service'; import { GeoCodingModule } from 'modules/geo-coding/geo-coding.module'; -import { GeoRegionsModule } from 'modules/geo-regions/geo-regions.module'; import { IndicatorRecordsModule } from 'modules/indicator-records/indicator-records.module'; import { BullModule } from '@nestjs/bull'; import { ImportDataProducer } from 'modules/import-data/workers/import-data.producer'; @@ -19,18 +15,16 @@ import { ImportDataConsumer } from 'modules/import-data/workers/import-data.cons import { ImportDataService } from 'modules/import-data/import-data.service'; import { TasksModule } from 'modules/tasks/tasks.module'; import { importQueueName } from 'modules/import-data/workers/import-queue.name'; -import { ScenariosModule } from 'modules/scenarios/scenarios.module'; import { IndicatorsModule } from 'modules/indicators/indicators.module'; import { MulterModule } from '@nestjs/platform-express'; import * as config from 'config'; import MulterConfigService from 'modules/import-data/multer-config.service'; import { ImpactModule } from 'modules/impact/impact.module'; import { WebSocketsModule } from 'modules/notifications/websockets/websockets.module'; -import { EudrImportService } from 'modules/import-data/eudr/eudr.import.service'; -import { EUDRDTOProcessor } from 'modules/import-data/eudr/eudr.dto-processor.service'; import { ImportMailService } from 'modules/import-data/import-mail/import-mail.service'; import { NotificationsModule } from 'modules/notifications/notifications.module'; import { ExcelValidatorService } from 'modules/import-data/sourcing-data/validation/excel-validator.service'; +import { SourcingDataDbCleaner } from 'modules/import-data/sourcing-data/sourcing-data.db-cleaner'; // TODO: Move EUDR related stuff to EUDR modules @@ -46,21 +40,13 @@ import { ExcelValidatorService } from 'modules/import-data/sourcing-data/validat BullModule.registerQueue({ name: 'eudr', }), - BullModule.registerQueue({ - name: 'eudr', - }), MaterialsModule, BusinessUnitsModule, SuppliersModule, - AdminRegionsModule, SourcingLocationsModule, - SourcingRecordsModule, - SourcingLocationGroupsModule, GeoCodingModule, - GeoRegionsModule, IndicatorRecordsModule, TasksModule, - ScenariosModule, IndicatorsModule, ImpactModule, WebSocketsModule, @@ -74,10 +60,9 @@ import { ExcelValidatorService } from 'modules/import-data/sourcing-data/validat ImportDataProducer, ImportDataConsumer, ImportDataService, - EudrImportService, - EUDRDTOProcessor, ImportMailService, ExcelValidatorService, + SourcingDataDbCleaner, { provide: 'FILE_UPLOAD_SIZE_LIMIT', useValue: config.get('fileUploads.sizeLimit'), diff --git a/api/src/modules/import-data/import-data.service.ts b/api/src/modules/import-data/import-data.service.ts index acd7ac1ce..ebe029296 100644 --- a/api/src/modules/import-data/import-data.service.ts +++ b/api/src/modules/import-data/import-data.service.ts @@ -21,7 +21,6 @@ export class ImportDataService { constructor( private readonly importDataProducer: ImportDataProducer, private readonly sourcingDataImportService: SourcingDataImportService, - private readonly eudrImport: EudrImportService, private readonly tasksService: TasksService, ) {} @@ -51,43 +50,10 @@ export class ImportDataService { } } - async loadEudrFile( - userId: string, - xlsxFileData: Express.Multer.File, - ): Promise { - const { filename, path } = xlsxFileData; - const task: Task = await this.tasksService.createTask({ - data: { filename, path }, - userId, - }); - try { - await this.importDataProducer.addEudrImportJob(xlsxFileData, task.id); - return task; - } catch (error: any) { - this.logger.error( - `Job for file: ${ - xlsxFileData.filename - } sent by user: ${userId} could not been added to queue: ${error.toString()}`, - ); - - await this.tasksService.remove(task.id); - throw new ServiceUnavailableException( - `File: ${xlsxFileData.filename} could not have been loaded. Please try again later or contact the administrator`, - ); - } - } - async processImportJob(job: Job): Promise { await this.sourcingDataImportService.importSourcingData( job.data.xlsxFileData.path, job.data.taskId, ); } - - async processEudrJob(job: Job): Promise { - await this.eudrImport.importEudr( - job.data.xlsxFileData.path, - job.data.taskId, - ); - } } diff --git a/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts b/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts index d7d0315bc..c8aa6edcf 100644 --- a/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts +++ b/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts @@ -39,6 +39,8 @@ import { } from 'modules/import-data/sourcing-data/validation/excel-validator.service'; import { ExcelValidationError } from 'modules/import-data/sourcing-data/validation/validators/excel-validation.error'; import { GeoCodingError } from 'modules/geo-coding/errors/geo-coding.error'; +import { SourcingDataDbCleaner } from './sourcing-data.db-cleaner'; +import { SourcingLocation } from 'modules/sourcing-locations/sourcing-location.entity'; export interface LocationData { locationAddressInput?: string; @@ -75,21 +77,15 @@ export class SourcingDataImportService { protected readonly materialService: MaterialsService, protected readonly businessUnitService: BusinessUnitsService, protected readonly supplierService: SuppliersService, - protected readonly adminRegionService: AdminRegionsService, - protected readonly geoRegionsService: GeoRegionsService, protected readonly sourcingLocationService: SourcingLocationsService, - protected readonly sourcingRecordService: SourcingRecordsService, - protected readonly sourcingLocationGroupService: SourcingLocationGroupsService, protected readonly fileService: FileService, - protected readonly dtoProcessor: SourcingRecordsDtoProcessorService, protected readonly geoCodingService: GeoCodingAbstractClass, protected readonly tasksService: TasksService, - protected readonly scenarioService: ScenariosService, protected readonly indicatorService: IndicatorsService, - protected readonly indicatorRecordService: IndicatorRecordsService, protected readonly impactService: ImpactService, protected readonly impactCalculator: ImpactCalculator, protected readonly excelValidator: ExcelValidatorService, + protected readonly dbCleaner: SourcingDataDbCleaner, ) {} async importSourcingData(filePath: string, taskId: string): Promise { @@ -99,11 +95,6 @@ export class SourcingDataImportService { const parsedXLSXDataset: SourcingRecordsSheets = await this.fileService.transformToJson(filePath, SHEETS_MAP); - const sourcingLocationGroup: SourcingLocationGroup = - await this.sourcingLocationGroupService.create({ - title: 'Sourcing Records import from XLSX file', - }); - const { data: dtoMatchedData, validationErrors } = await this.excelValidator.validate( parsedXLSXDataset as unknown as SourcingDataSheet, @@ -114,7 +105,7 @@ export class SourcingDataImportService { //TODO: Implement transactional import. Move geocoding to first step - await this.cleanDataBeforeImport(); + await this.dbCleaner.cleanDataBeforeImport(); const materials: Material[] = await this.materialService.findAllUnpaginated(); @@ -171,7 +162,7 @@ export class SourcingDataImportService { newLogs: warnings, })); - const sourcingDataWithOrganizationalEntities: any = + const sourcingDataWithOrganizationalEntities: SourcingLocation[] = await this.relateSourcingDataWithOrganizationalEntities( suppliers, businessUnits, @@ -209,24 +200,24 @@ export class SourcingDataImportService { * @note: Deletes DB content from required entities * to ensure DB is prune prior loading a XLSX dataset */ - async cleanDataBeforeImport(): Promise { - this.logger.log('Cleaning database before import...'); - try { - await this.indicatorService.deactivateAllIndicators(); - await this.materialService.deactivateAllMaterials(); - await this.scenarioService.clearTable(); - await this.indicatorRecordService.clearTable(); - await this.businessUnitService.clearTable(); - await this.supplierService.clearTable(); - await this.sourcingLocationService.clearTable(); - await this.sourcingRecordService.clearTable(); - await this.geoRegionsService.deleteGeoRegionsCreatedByUser(); - } catch ({ message }) { - throw new Error( - `Database could not been cleaned before loading new dataset: ${message}`, - ); - } - } + // async cleanDataBeforeImport(): Promise { + // this.logger.log('Cleaning database before import...'); + // try { + // await this.indicatorService.deactivateAllIndicators(); + // await this.materialService.deactivateAllMaterials(); + // await this.scenarioService.clearTable(); + // await this.indicatorRecordService.clearTable(); + // await this.businessUnitService.clearTable(); + // await this.supplierService.clearTable(); + // await this.sourcingLocationService.clearTable(); + // await this.sourcingRecordService.clearTable(); + // await this.geoRegionsService.deleteGeoRegionsCreatedByUser(); + // } catch ({ message }) { + // throw new Error( + // `Database could not been cleaned before loading new dataset: ${message}`, + // ); + // } + // } /** * @note: Type hack as mpath property does not exist on Materials and BusinessUnits, but its created @@ -239,7 +230,7 @@ export class SourcingDataImportService { businessUnits: Record[], materials: Material[], sourcingData: SourcingData[], - ): Promise { + ): Promise { this.logger.log(`Relating sourcing data with organizational entities`); this.logger.log(`Supplier count: ${suppliers.length}`); this.logger.log(`Business Units count: ${businessUnits.length}`); @@ -268,9 +259,7 @@ export class SourcingDataImportService { sourcingLocation.businessUnitId = businessUnit.id; } } - if (typeof sourcingLocation.materialId === 'undefined') { - return; - } + const sourcingLocationMaterialId: string = sourcingLocation.materialId; if (!(sourcingLocationMaterialId in materialMap)) { @@ -280,6 +269,6 @@ export class SourcingDataImportService { } sourcingLocation.materialId = materialMap[sourcingLocationMaterialId]; } - return sourcingData; + return sourcingData as SourcingLocation[]; } } diff --git a/api/src/modules/import-data/sourcing-data/sourcing-data.db-cleaner.ts b/api/src/modules/import-data/sourcing-data/sourcing-data.db-cleaner.ts new file mode 100644 index 000000000..6dad182c5 --- /dev/null +++ b/api/src/modules/import-data/sourcing-data/sourcing-data.db-cleaner.ts @@ -0,0 +1,40 @@ +import { GeoRegion } from 'modules/geo-regions/geo-region.entity'; +import { Scenario } from 'modules/scenarios/scenario.entity'; +import { DataSource, EntityManager } from 'typeorm'; +import { BusinessUnit } from 'modules/business-units/business-unit.entity'; +import { SourcingLocation } from 'modules/sourcing-locations/sourcing-location.entity'; +import { Supplier } from 'modules/suppliers/supplier.entity'; + +export class SourcingDataDbCleaner { + constructor(private readonly dataSource: DataSource) {} + + async cleanDataBeforeImport(): Promise { + await this.dataSource.transaction(async (manager: EntityManager) => { + await this.deleteExistingUserData(manager); + await this.deactivateAllIndicators(manager); + await this.deactivateAllMaterials(manager); + }); + } + + private async deactivateAllIndicators(manager: EntityManager): Promise { + await manager.query('UPDATE indicators SET active = false'); + } + + private async deactivateAllMaterials(manager: EntityManager): Promise { + await manager.query('UPDATE materials SET active = false'); + } + + private async deleteExistingUserData(manager: EntityManager): Promise { + const entities: any = [Scenario, SourcingLocation, BusinessUnit, Supplier]; + for (const entity of entities) { + await manager.getRepository(entity).delete({}); + } + await this.deleteGeoRegionsCreatedByUser(manager); + } + + private async deleteGeoRegionsCreatedByUser( + manager: EntityManager, + ): Promise { + await manager.getRepository(GeoRegion).delete({ isCreatedByUser: true }); + } +} diff --git a/api/src/modules/import-data/workers/eudr.consumer.ts b/api/src/modules/import-data/workers/eudr.consumer.ts index 6b668cbbd..55cc469ae 100644 --- a/api/src/modules/import-data/workers/eudr.consumer.ts +++ b/api/src/modules/import-data/workers/eudr.consumer.ts @@ -49,8 +49,8 @@ export class ImportDataConsumer { // TODO: Handle eudr-alerts import completion, updating async tasks } - @Process('eudr') - async readImportDataJob(job: Job): Promise { - await this.importDataService.processEudrJob(job); - } + // @Process('eudr') + // async readImportDataJob(job: Job): Promise { + // await this.importDataService.processEudrJob(job); + // } } diff --git a/api/src/modules/indicators/indicators.service.ts b/api/src/modules/indicators/indicators.service.ts index 3427570b6..bbd1f25df 100644 --- a/api/src/modules/indicators/indicators.service.ts +++ b/api/src/modules/indicators/indicators.service.ts @@ -53,15 +53,6 @@ export class IndicatorsService extends AppBaseService< }; } - /** - * Returns all available valid indicators - */ - async getAllIndicators(): Promise { - // It is assumed that the indicators that are enabled/valid, are the ones that are present on the DB since - // the initial seeding import. So a simple getAll is sufficient - return this.findAllIndicators(); - } - async getIndicatorById(id: string): Promise { const found: Indicator | null = await this.indicatorRepository.findOne({ where: { id }, @@ -74,36 +65,6 @@ export class IndicatorsService extends AppBaseService< return found; } - async getDeforestationH3Data(): Promise { - /** - * @note: For at least 2 types of risk maps, retrieving a fixed Indicator's data - * is required to perform the query, and no data to retrieve this Indicator is provided - * in the client's request - */ - - const deforestationIndicator: Indicator | null = - await this.indicatorRepository.findOne({ - where: { nameCode: INDICATOR_NAME_CODES.DF_SLUC }, - }); - if (!deforestationIndicator) - throw new NotFoundException( - 'No Deforestation Indicator data found in database', - ); - const deforestationH3Data: any = await this.indicatorRepository - .createQueryBuilder() - .select() - .from('h3_data', 'h3_data') - .where('"indicatorId" = :indicatorId', { - indicatorId: deforestationIndicator.id, - }) - .getRawOne(); - if (!deforestationH3Data) - throw new NotFoundException( - 'No Deforestation Indicator H3 data found in database, required to retrieve Biodiversity Loss and Carbon Risk-Maps', - ); - return deforestationH3Data; - } - async getIndicatorsById(ids: string[]): Promise { const indicators: Indicator[] = await this.indicatorRepository.findBy({ id: In(ids), @@ -127,28 +88,6 @@ export class IndicatorsService extends AppBaseService< return this.indicatorRepository.find(findOptions); } - async getIndicatorsAndRelatedH3DataIds(): Promise< - IndicatorNameCodeWithRelatedH3[] - > { - const indicators: IndicatorNameCodeWithRelatedH3[] = - await this.indicatorRepository - .createQueryBuilder('indicator') - .select([ - 'indicator.id as id', - 'indicator.nameCode as "nameCode"', - 'h3.id as "h3DataId"', - ]) - .innerJoin(H3Data, 'h3', 'h3.indicatorId = indicator.id') - .getRawMany(); - - if (!indicators.length) { - throw new NotFoundException( - `No Indicators with related H3 Data could be found`, - ); - } - return indicators; - } - async activateIndicators( indicatorsFromSheet: CreateIndicatorDto[], ): Promise { diff --git a/api/test/README.md b/api/test/README.md new file mode 100644 index 000000000..e6b5d4c94 --- /dev/null +++ b/api/test/README.md @@ -0,0 +1,46 @@ +# Testing Approach for LandGriffon + +This document describes the testing approach for LandGriffon. The testing approach is divided into two parts. + +## Aim + +We aim to improve the way we write tests in our app, especially e2e tests. Most of the time, we need to create several +preconditions in the DB before we can actually test the feature. This is a problem because it makes the tests not very +readable and also there is a lot of duplicated code. + +We aim to slowly transition to encapsulate all tests using some sort of Gherkin syntax. This will provide a better +understanding of what is being tested and also help to avoid duplicated code. + +## TestManager Class + +We have a base `TestManager` class, which encapsulates most of the common logic needed for tests. The testing app is +injected in the class and implements several methods for loading it, making requests, teardown the app, and clean the +DB. + +## Test Execution + +For specific precondition generation (ARRANGE), execution (ACT), and assertions (ASSERT), we have a class for each +feature, module, or concept. This class extends the `TestManager` class and implements the specific methods for that +feature. We are slowly transitioning to this approach. + +Example: + +```typescript +describe('GeoRegions Filters (e2e)', () => { + let testManager: EUDRTestManager; + + beforeAll(async () => { + testManager = await TestManager.load(EUDRTestManager); + }); + beforeEach(async () => { + await testManager.refreshState(); + }); + + afterEach(async () => { + await testManager.clearDatabase(); + }); + + afterAll(async () => { + await testManager.close(); + }); +``` diff --git a/api/test/integration/import-data/xlsx-uploads/import-data-controller-validations.spec.ts b/api/test/integration/import-data/xlsx-uploads/import-data-controller-validations.spec.ts index 4c2ef18c6..9ac783fe6 100644 --- a/api/test/integration/import-data/xlsx-uploads/import-data-controller-validations.spec.ts +++ b/api/test/integration/import-data/xlsx-uploads/import-data-controller-validations.spec.ts @@ -59,37 +59,4 @@ describe('XLSX Upload Feature Validation Tests', () => { ); }); }); - - // TODO: Move this to integration tests sets as all types of validations will be moved to a async queue - describe.skip('XLSX Upload Feature File Content Validation Tests', () => { - test('When file with invalid content is sent to the API it should return 400 "Bad Request" error', async () => { - await request(testApplication.getHttpServer()) - .post('/api/v1/import/sourcing-data') - .set('Authorization', `Bearer ${jwtToken}`) - .attach('file', __dirname + '/files/bad-dataset.xlsx') - .expect(HttpStatus.BAD_REQUEST); - - const folderContent = await readdir( - config.get('fileUploads.storagePath'), - ); - expect(folderContent.length).toEqual(0); - }, 100000); - - test('When file with incorrect or missing inputs for upload is sent to API, proper error messages should be received', async () => { - await request(testApplication.getHttpServer()) - .post('/api/v1/import/sourcing-data') - .set('Authorization', `Bearer ${jwtToken}`) - .attach('file', __dirname + '/files/base-dataset-location-errors.xlsx'); - - expect(HttpStatus.BAD_REQUEST); - // TODO: Double check excel used for this test, it has REF errors in it - // expect(response.body.errors[0].meta.rawError.response.message).toEqual( - // sourcingDataValidationErrorResponse, - // ); - const folderContent = await readdir( - config.get('fileUploads.storagePath'), - ); - expect(folderContent.length).toEqual(0); - }, 100000); - }); }); diff --git a/api/test/integration/import-data/xlsx-uploads/validations/api_test_validation_errors_datasheet.xlsx b/api/test/integration/import-data/xlsx-uploads/validations/api_test_validation_errors_datasheet.xlsx new file mode 100644 index 000000000..a480623da Binary files /dev/null and b/api/test/integration/import-data/xlsx-uploads/validations/api_test_validation_errors_datasheet.xlsx differ diff --git a/api/test/integration/import-data/xlsx-uploads/validations/fixtures.ts b/api/test/integration/import-data/xlsx-uploads/validations/fixtures.ts new file mode 100644 index 000000000..d2103c0bd --- /dev/null +++ b/api/test/integration/import-data/xlsx-uploads/validations/fixtures.ts @@ -0,0 +1,269 @@ +import { SourcingDataImportService } from 'modules/import-data/sourcing-data/sourcing-data-import.service'; +import { TestManager } from '../../../../utils/test-manager'; +import { ExcelValidationError } from '../../../../../src/modules/import-data/sourcing-data/validation/validators/excel-validation.error'; +import { ImportTaskError } from '../../../../../src/modules/tasks/types/import-task-error.type'; +import { GeoCodingError } from 'modules/geo-coding/errors/geo-coding.error'; +import { createTask } from '../../../../entity-mocks'; + +export class ImportValidationTestManager extends TestManager { + url: string = '/api/v1/import/sourcing-data'; + + constructor(manager: TestManager) { + super(manager.testApp, manager.jwtToken, manager.dataSource); + } + + WhenIImportAFileWithValidationErrors = async (): Promise => { + const sourcingDataImportService = this.testApp.get( + SourcingDataImportService, + ); + + try { + await sourcingDataImportService.importSourcingData( + __dirname + '/api_test_validation_errors_datasheet.xlsx', + '', + ); + } catch (error) { + return error; + } + }; + + WhenIImportAFileWithGeoCodingErrors = async (): Promise => { + const task = await createTask(); + const sourcingDataImportService = this.testApp.get( + SourcingDataImportService, + ); + + try { + await sourcingDataImportService.importSourcingData( + __dirname + '/api_test_geocoding_errors_datasheet.xlsx', + task.id, + ); + } catch (error) { + console.log(error); + return error; + } + }; + TheErrorShouldBeAnExcelValidationError = (error: any) => { + expect(error).toBeInstanceOf(ExcelValidationError); + }; + + ThenErrorShouldBEAnGeoCodingError = (error: any) => { + expect(error).toBeInstanceOf(GeoCodingError); + }; + + AndItShouldContainAllTheValidationErrorsOfTheFile = (error: any) => { + const expectedValidationErrors: ImportTaskError[] = [ + { + row: 2, + column: 'name', + error: 'Material Name must not be empty', + sheet: 'materials', + type: 'validation-error', + }, + { + row: 3, + column: 'hs_2017_code', + error: 'Material hs_2017_code must not be empty', + sheet: 'materials', + type: 'validation-error', + }, + { + row: 3, + column: 'hs_2017_code', + error: 'Material hs_2017_code is too short', + sheet: 'materials', + type: 'validation-error', + }, + { + row: 3, + column: 'hs_2017_code', + error: 'hs_2017_code must be a string', + sheet: 'materials', + type: 'validation-error', + }, + { + row: 4, + column: 'status', + error: 'Material status must be active or inactive', + sheet: 'materials', + type: 'validation-error', + }, + { + row: 4, + column: 'status', + error: 'status must be a string', + sheet: 'materials', + type: 'validation-error', + }, + { + row: 2, + column: 'path_id', + error: 'Business Unit path_id must be a string', + sheet: 'businessUnits', + type: 'validation-error', + }, + { + row: 2, + column: 'path_id', + error: 'Business Unit path_id must not be empty', + sheet: 'businessUnits', + type: 'validation-error', + }, + { + row: 3, + column: 'name', + error: 'Business Unit Name must not be empty', + sheet: 'businessUnits', + type: 'validation-error', + }, + { + row: 3, + column: 'path_id', + error: 'Supplier path_id must not be empty', + sheet: 'suppliers', + type: 'validation-error', + }, + { + row: 3, + column: 'path_id', + error: 'path_id must be a string', + sheet: 'suppliers', + type: 'validation-error', + }, + { + row: 4, + column: 'name', + error: 'Supplier Name must not be empty', + sheet: 'suppliers', + type: 'validation-error', + }, + { + row: 5, + column: 'material.hsCode', + error: 'Material hs code cannot be empty', + sheet: 'sourcingData', + type: 'validation-error', + }, + { + row: 7, + column: 'business_unit.path', + error: 'business_unit.path must be a string', + sheet: 'sourcingData', + type: 'validation-error', + }, + { + row: 7, + column: 'business_unit.path', + error: 'Business Unit path cannot be empty', + sheet: 'sourcingData', + type: 'validation-error', + }, + { + row: 7, + column: 'location_country_input', + error: 'location_country_input must be a string', + sheet: 'sourcingData', + type: 'validation-error', + }, + { + row: 7, + column: 'location_country_input', + error: 'Location country input is required', + sheet: 'sourcingData', + type: 'validation-error', + }, + { + row: 8, + column: 'location_address_input', + error: + 'Address must be empty for locations of type country-of-production', + sheet: 'sourcingData', + type: 'validation-error', + }, + { + row: 9, + column: 'location_latitude_input', + error: + 'Address input or coordinates are required for locations of type point-of-production. Latitude values must be min: -90, max: 90', + sheet: 'sourcingData', + type: 'validation-error', + }, + { + row: 9, + column: 'location_longitude_input', + error: + 'Address input or coordinates are required for locations of type point-of-production. Longitude values must be min: -180, max: 180', + sheet: 'sourcingData', + type: 'validation-error', + }, + { + row: 10, + column: 'location_address_input', + error: + 'Address input OR coordinates are required for locations of type production-aggregation-point. Address must be empty if coordinates are provided', + sheet: 'sourcingData', + type: 'validation-error', + }, + { + row: 10, + column: 'location_latitude_input', + error: + 'Address input OR coordinates must be provided for locations of type production-aggregation-point. Latitude must be empty if address is provided', + sheet: 'sourcingData', + type: 'validation-error', + }, + { + row: 10, + column: 'location_longitude_input', + error: + 'Address input OR coordinates must be provided for locations of type production-aggregation-point. Latitude must be empty if address is provided', + sheet: 'sourcingData', + type: 'validation-error', + }, + { + row: 2, + column: 'nameCode', + error: 'nameCode must be longer than or equal to 1 characters', + sheet: 'indicators', + type: 'validation-error', + }, + { + row: 2, + column: 'nameCode', + error: 'nameCode should not be empty', + sheet: 'indicators', + type: 'validation-error', + }, + { + row: 2, + column: 'nameCode', + error: 'nameCode must be a string', + sheet: 'indicators', + type: 'validation-error', + }, + { + row: 3, + column: 'status', + error: 'status must be one of the following values: ', + sheet: 'indicators', + type: 'validation-error', + }, + { + row: 3, + column: 'status', + error: 'status should not be empty', + sheet: 'indicators', + type: 'validation-error', + }, + { + row: 3, + column: 'status', + error: 'status must be a string', + sheet: 'indicators', + type: 'validation-error', + }, + ]; + + expect(error.validationErrors).toEqual(expectedValidationErrors); + }; +} diff --git a/api/test/integration/import-data/xlsx-uploads/validations/sourcing-data-import-validations.spec.ts b/api/test/integration/import-data/xlsx-uploads/validations/sourcing-data-import-validations.spec.ts new file mode 100644 index 000000000..93b6a04fd --- /dev/null +++ b/api/test/integration/import-data/xlsx-uploads/validations/sourcing-data-import-validations.spec.ts @@ -0,0 +1,46 @@ +import { ImportValidationTestManager } from './fixtures'; +import { TestManager } from '../../../../utils/test-manager'; +import { FileService } from '../../../../../src/modules/import-data/file.service'; +import { MockFileService } from '../../../../utils/service-mocks'; +import { MaterialsService } from '../../../../../src/modules/materials/materials.service'; +import { SourcingDataDbCleaner } from '../../../../../src/modules/import-data/sourcing-data/sourcing-data.db-cleaner'; +import { IndicatorsService } from '../../../../../src/modules/indicators/indicators.service'; + +describe('Sourcing Data Import Validations', () => { + let importValidationTestManager: ImportValidationTestManager; + + beforeAll(async () => { + importValidationTestManager = await TestManager.load( + ImportValidationTestManager, + TestManager.buildCustomTestModule() + .overrideProvider(FileService) + .useClass(MockFileService), + ); + }); + + beforeEach(async () => { + await importValidationTestManager.refreshState(); + }); + + afterEach(async () => { + await importValidationTestManager.clearDatabase(); + }); + + afterAll(async () => { + await importValidationTestManager.close(); + }); + + describe('Input validations', () => { + test('should catch all validation errors and return them', async () => { + const errors: any = + await importValidationTestManager.WhenIImportAFileWithValidationErrors(); + + importValidationTestManager.TheErrorShouldBeAnExcelValidationError( + errors, + ); + importValidationTestManager.AndItShouldContainAllTheValidationErrorsOfTheFile( + errors, + ); + }); + }); +}); diff --git a/api/test/utils/application-manager.ts b/api/test/utils/application-manager.ts index 61dd9fb57..b9c374377 100644 --- a/api/test/utils/application-manager.ts +++ b/api/test/utils/application-manager.ts @@ -6,15 +6,36 @@ import { useContainer } from 'class-validator'; import * as config from 'config'; import { AppModule } from 'app.module'; import { TestingModuleBuilder } from '@nestjs/testing/testing-module.builder'; -import { Type } from '@nestjs/common/interfaces'; +import { + ClassProvider, + FactoryProvider, + Type, + ValueProvider, +} from '@nestjs/common/interfaces'; import { TestingModule } from '@nestjs/testing/testing-module'; import { isUndefined } from 'lodash'; import { MockAlertRepository, MockEmailService } from './service-mocks'; import { IEmailServiceToken } from '../../src/modules/notifications/notifications.module'; -import { AlertsRepository } from 'modules/eudr-alerts/alerts.repository'; + +type Overrides = { + classes: ClassProvider[]; + values: ValueProvider[]; + factories: FactoryProvider[]; +}; export default class ApplicationManager { static readonly regenerateResourcesOnEachTest: boolean = false; + static readonly defaultOverriders: Overrides = { + classes: [ + { provide: IEmailServiceToken, useClass: MockEmailService }, + { + provide: 'IEUDRAlertsRepository', + useClass: MockAlertRepository, + }, + ], + values: [], + factories: [], + }; static testApplication?: TestApplication; static isCustomApplication: boolean; @@ -44,11 +65,11 @@ export default class ApplicationManager { initTestingModuleBuilder || Test.createTestingModule({ imports: [AppModule], - }) - .overrideProvider(IEmailServiceToken) - .useClass(MockEmailService) - .overrideProvider('IEUDRAlertsRepository') - .useClass(MockAlertRepository); + }); + overrideProviders( + testingModuleBuilder, + ApplicationManager.defaultOverriders, + ); ApplicationManager.testApplication.moduleFixture = await testingModuleBuilder.compile(); @@ -119,3 +140,22 @@ export class TestApplication { ApplicationManager.testApplication = undefined; } } + +const overrideProviders = ( + module: TestingModuleBuilder, + overrides: Overrides, +) => { + const { classes, values, factories } = overrides; + + classes.forEach(({ provide, useClass }) => + module.overrideProvider(provide).useClass(useClass), + ); + values.forEach(({ provide, useValue }) => + module.overrideProvider(provide).useValue(useValue), + ); + factories.forEach(({ provide, useFactory }) => + module + .overrideProvider(provide) + .useFactory({ factory: (args) => useFactory(...args) }), + ); +}; diff --git a/api/test/utils/service-mocks.ts b/api/test/utils/service-mocks.ts index 1dfaeae4c..56776a469 100644 --- a/api/test/utils/service-mocks.ts +++ b/api/test/utils/service-mocks.ts @@ -10,6 +10,7 @@ import { GetAlertSummary, IEUDRAlertsRepository, } from 'modules/eudr-alerts/eudr.repositoty.interface'; +import { FileService } from '../../src/modules/import-data/file.service'; export class MockEmailService implements IEmailService { logger: Logger = new Logger(MockEmailService.name); @@ -48,3 +49,9 @@ export class MockAlertRepository implements IEUDRAlertsRepository { return Promise.resolve([]); } } + +export class MockFileService extends FileService { + async deleteDataFromFS(): Promise { + return; + } +} diff --git a/api/test/utils/test-manager.ts b/api/test/utils/test-manager.ts index c780046a9..eb24bbba8 100644 --- a/api/test/utils/test-manager.ts +++ b/api/test/utils/test-manager.ts @@ -3,19 +3,18 @@ import ApplicationManager, { TestApplication } from './application-manager'; import { clearTestDataFromDatabase } from './database-test-helper'; import { setupTestUser } from './userAuth'; import * as request from 'supertest'; -import { Material } from 'modules/materials/material.entity'; -import { Supplier } from 'modules/suppliers/supplier.entity'; -import { GeoRegion } from 'modules/geo-regions/geo-region.entity'; import { generateRandomName } from './generate-random-name'; +import { ModuleMetadata } from '@nestjs/common/interfaces/modules/module-metadata.interface'; +import { TestingModuleBuilder } from '@nestjs/testing/testing-module.builder'; +import { Type } from '@nestjs/common/interfaces'; +import { Test } from '@nestjs/testing'; +import { AppModule } from '../../src/app.module'; export class TestManager { testApp: TestApplication; jwtToken: string; dataSource: DataSource; response?: request.Response; - materials?: Material[]; - suppliers?: Supplier[]; - geoRegions?: GeoRegion[]; constructor(app: TestApplication, jwtToken: string, dataSource: DataSource) { this.testApp = app; @@ -23,12 +22,16 @@ export class TestManager { this.dataSource = dataSource; } - static async load(manager: any) { - return new manager(await this.createManager()); + static async load(manager: any, testingModuleBuilder?: TestingModuleBuilder) { + return new manager(await this.createManager(testingModuleBuilder)); } - static async createManager() { - const testApplication = await ApplicationManager.init(); + static buildCustomTestModule(): TestingModuleBuilder { + return Test.createTestingModule({ imports: [AppModule] }); + } + + static async createManager(testingModuleBuilder?: TestingModuleBuilder) { + const testApplication = await ApplicationManager.init(testingModuleBuilder); const dataSource = testApplication.get(DataSource); const { jwtToken } = await setupTestUser(testApplication); return new TestManager(testApplication, jwtToken, dataSource); @@ -37,9 +40,6 @@ export class TestManager { async refreshState() { const { jwtToken } = await setupTestUser(this.testApp); this.jwtToken = jwtToken; - this.materials = undefined; - this.suppliers = undefined; - this.geoRegions = undefined; this.response = undefined; }