From e6a697e6d36d683c6fdd7d285e9a6d4b36e47488 Mon Sep 17 00:00:00 2001 From: alexeh Date: Thu, 25 Apr 2024 09:16:24 +0300 Subject: [PATCH 1/8] send import success mail --- .../import-data/import-progress.emitter.ts | 2 +- .../modules/import-data/import-data.module.ts | 4 ++ .../import-mail/import-mail.service.ts | 45 +++++++++++++++++++ .../workers/import-data.consumer.ts | 20 ++++++--- api/src/modules/tasks/tasks.service.ts | 6 ++- 5 files changed, 69 insertions(+), 8 deletions(-) create mode 100644 api/src/modules/import-data/import-mail/import-mail.service.ts diff --git a/api/src/modules/events/import-data/import-progress.emitter.ts b/api/src/modules/events/import-data/import-progress.emitter.ts index 17daa625f..56da609a0 100644 --- a/api/src/modules/events/import-data/import-progress.emitter.ts +++ b/api/src/modules/events/import-data/import-progress.emitter.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; import { EventBus } from '@nestjs/cqrs'; import { ImportProgressUpdateEvent } from 'modules/events/import-data/import-progress.event'; -import { ImportProgressSteps } from './types'; +import { ImportProgressSteps } from 'modules/events/import-data/types'; /** * @note: We use eventBus instead of commandBus because even tho broadcasting via websockets can be considered a command, it is not a command in the context of events. (apparently) diff --git a/api/src/modules/import-data/import-data.module.ts b/api/src/modules/import-data/import-data.module.ts index 57454ab0e..75d95f75c 100644 --- a/api/src/modules/import-data/import-data.module.ts +++ b/api/src/modules/import-data/import-data.module.ts @@ -28,6 +28,8 @@ import { ImpactModule } from 'modules/impact/impact.module'; import { WebSocketsModule } from 'modules/notifications/websockets/websockets.module'; import { EudrImportService } from 'modules/import-data/eudr/eudr.import.service'; import { EUDRDTOProcessor } from 'modules/import-data/eudr/eudr.dto-processor.service'; +import { ImportMailService } from 'modules/import-data/import-mail/import-mail.service'; +import { NotificationsModule } from 'modules/notifications/notifications.module'; // TODO: Move EUDR related stuff to EUDR modules @@ -61,6 +63,7 @@ import { EUDRDTOProcessor } from 'modules/import-data/eudr/eudr.dto-processor.se IndicatorsModule, ImpactModule, WebSocketsModule, + NotificationsModule, ], providers: [ MulterConfigService, @@ -72,6 +75,7 @@ import { EUDRDTOProcessor } from 'modules/import-data/eudr/eudr.dto-processor.se ImportDataService, EudrImportService, EUDRDTOProcessor, + ImportMailService, { provide: 'FILE_UPLOAD_SIZE_LIMIT', useValue: config.get('fileUploads.sizeLimit'), diff --git a/api/src/modules/import-data/import-mail/import-mail.service.ts b/api/src/modules/import-data/import-mail/import-mail.service.ts new file mode 100644 index 000000000..c4da25831 --- /dev/null +++ b/api/src/modules/import-data/import-mail/import-mail.service.ts @@ -0,0 +1,45 @@ +import { Inject, Injectable, Logger } from '@nestjs/common'; +import { IEmailService } from 'modules/notifications/email/email.service.interface'; + +type ImportEmailDTO = { + email: string; + fileName: string; + importDate: string; +}; + +@Injectable() +export class ImportMailService { + logger: Logger = new Logger(ImportMailService.name); + + constructor(@Inject('IEmailService') private emailService: IEmailService) {} + + async sendImportSuccessMail(dto: ImportEmailDTO): Promise { + const htmlContent: string = ` +

Dear ${dto.email || 'User'},

+

Your import of file ${ + dto.fileName + } has been successfully processed.

+

Import date: ${dto.importDate}

+ `; + await this.emailService.sendMail({ + to: dto.email, + subject: 'Import success', + html: htmlContent, + }); + this.logger.debug(`Sent import success email to ${dto.email}`); + } + + async sendImportFailureMail(dto: ImportEmailDTO): Promise { + const htmlContent: string = ` +

Dear ${dto.email || 'User'},

+

Your import of file ${dto.fileName} has failed.

+

Import date: ${dto.importDate}

+ `; + await this.emailService.sendMail({ + to: dto.email, + subject: 'Import failure', + html: htmlContent, + }); + this.logger.debug(`Sent import failure email to ${dto.email}`); + } +} diff --git a/api/src/modules/import-data/workers/import-data.consumer.ts b/api/src/modules/import-data/workers/import-data.consumer.ts index 7ea4796c7..93ae1fcea 100644 --- a/api/src/modules/import-data/workers/import-data.consumer.ts +++ b/api/src/modules/import-data/workers/import-data.consumer.ts @@ -12,8 +12,8 @@ import { ExcelImportJob } from 'modules/import-data/workers/import-data.producer import { TasksService } from 'modules/tasks/tasks.service'; import { Task, TASK_STATUS } from 'modules/tasks/task.entity'; import { importQueueName } from 'modules/import-data/workers/import-queue.name'; -import { ImportProgressEmitter } from 'modules/events/import-data/import-progress.emitter'; -import { ImportProgressSocket } from '../../events/import-data/import-progress.socket'; +import { ImportProgressSocket } from 'modules/events/import-data/import-progress.socket'; +import { ImportMailService } from 'modules/import-data/import-mail/import-mail.service'; @Processor(importQueueName) export class ImportDataConsumer { @@ -23,6 +23,7 @@ export class ImportDataConsumer { public readonly importDataService: ImportDataService, public readonly tasksService: TasksService, public readonly importSocket: ImportProgressSocket, + public readonly importMail: ImportMailService, ) {} @OnQueueError() @@ -42,6 +43,7 @@ export class ImportDataConsumer { message: err.message, }); this.importSocket.emitImportFailureToSocket({ error: err }); + this.logger.error( `Import Failed for file: ${job.data.xlsxFileData.filename} for task: ${task.id}: ${err}`, ); @@ -49,14 +51,20 @@ export class ImportDataConsumer { @OnQueueCompleted() async onJobComplete(job: Job): Promise { - this.logger.log( - `Import XLSX with TASK ID: ${job.data.taskId} completed successfully`, - ); - await this.tasksService.updateImportTask({ + const task: Task = await this.tasksService.updateImportTask({ taskId: job.data.taskId, newStatus: TASK_STATUS.COMPLETED, }); + this.importSocket.emitImportCompleteToSocket({ status: 'completed' }); + await this.importMail.sendImportSuccessMail({ + email: task.user.email, + fileName: job.data.xlsxFileData.originalname, + importDate: task.createdAt, + }); + this.logger.log( + `Import Completed for file: ${job.data.xlsxFileData.filename} for task: ${task.id}`, + ); } @Process('excel-import-job') diff --git a/api/src/modules/tasks/tasks.service.ts b/api/src/modules/tasks/tasks.service.ts index e30eec2cf..378f34303 100644 --- a/api/src/modules/tasks/tasks.service.ts +++ b/api/src/modules/tasks/tasks.service.ts @@ -116,7 +116,11 @@ export class TasksService extends AppBaseService< } if (newStatus) task.status = newStatus; - return task.save(); + await task.save(); + return this.taskRepository.findOneOrFail({ + where: { id: taskId }, + relations: ['user'], + }); } async cleanStalledTasks(): Promise { From 71cfe5934fa7e2f5e0d1325e2aac74903268f2ee Mon Sep 17 00:00:00 2001 From: alexeh Date: Mon, 29 Apr 2024 08:29:29 +0300 Subject: [PATCH 2/8] Create excel sheet class validators --- .../eudr/eudr.dto-processor.service.ts | 2 +- .../import-mail/import-mail.service.ts | 12 +++ .../sourcing-data/dto-processor.service.ts | 2 +- .../sourcing-data-import.service.ts | 2 +- .../address-input.custom.validator.ts | 3 +- .../latitude-input.custom.validator.ts | 3 +- .../longitude-input.custom.validator.ts | 3 +- .../sourcing-data-excel.validator.ts | 10 +++ .../sourcing-data.class.validator.ts | 6 +- .../business-units.sheet-validator.ts | 14 +++ .../validators/indicators.sheet-validator.ts | 18 ++++ .../validators/material.sheet-validator.ts | 32 +++++++ .../sourcing-data.sheet-validator.ts | 88 +++++++++++++++++++ .../validators/supplier.sheet-validator.ts | 15 ++++ .../workers/import-data.consumer.ts | 16 +++- .../email/email.service.interface.ts | 8 ++ 16 files changed, 224 insertions(+), 10 deletions(-) rename api/src/modules/import-data/sourcing-data/{validators => validation}/address-input.custom.validator.ts (98%) rename api/src/modules/import-data/sourcing-data/{validators => validation}/latitude-input.custom.validator.ts (98%) rename api/src/modules/import-data/sourcing-data/{validators => validation}/longitude-input.custom.validator.ts (97%) create mode 100644 api/src/modules/import-data/sourcing-data/validation/sourcing-data-excel.validator.ts rename api/src/modules/import-data/sourcing-data/{validators => validation}/sourcing-data.class.validator.ts (91%) create mode 100644 api/src/modules/import-data/sourcing-data/validation/validators/business-units.sheet-validator.ts create mode 100644 api/src/modules/import-data/sourcing-data/validation/validators/indicators.sheet-validator.ts create mode 100644 api/src/modules/import-data/sourcing-data/validation/validators/material.sheet-validator.ts create mode 100644 api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.sheet-validator.ts create mode 100644 api/src/modules/import-data/sourcing-data/validation/validators/supplier.sheet-validator.ts diff --git a/api/src/modules/import-data/eudr/eudr.dto-processor.service.ts b/api/src/modules/import-data/eudr/eudr.dto-processor.service.ts index 529d2aada..96b308907 100644 --- a/api/src/modules/import-data/eudr/eudr.dto-processor.service.ts +++ b/api/src/modules/import-data/eudr/eudr.dto-processor.service.ts @@ -6,7 +6,7 @@ import { } from '@nestjs/common'; import { CreateSourcingLocationDto } from 'modules/sourcing-locations/dto/create.sourcing-location.dto'; import { SourcingRecord } from 'modules/sourcing-records/sourcing-record.entity'; -import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validators/sourcing-data.class.validator'; +import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validation/sourcing-data.class.validator'; import { validateOrReject } from 'class-validator'; import { plainToClass } from 'class-transformer'; import { diff --git a/api/src/modules/import-data/import-mail/import-mail.service.ts b/api/src/modules/import-data/import-mail/import-mail.service.ts index c4da25831..751a7fddd 100644 --- a/api/src/modules/import-data/import-mail/import-mail.service.ts +++ b/api/src/modules/import-data/import-mail/import-mail.service.ts @@ -5,6 +5,7 @@ type ImportEmailDTO = { email: string; fileName: string; importDate: string; + errorContent?: string; }; @Injectable() @@ -30,15 +31,26 @@ export class ImportMailService { } async sendImportFailureMail(dto: ImportEmailDTO): Promise { + const base64Csv = Buffer.from(dto.errorContent || '').toString('base64'); + const htmlContent: string = `

Dear ${dto.email || 'User'},

Your import of file ${dto.fileName} has failed.

Import date: ${dto.importDate}

+

Please find the error report attached.

`; await this.emailService.sendMail({ to: dto.email, subject: 'Import failure', html: htmlContent, + attachments: [ + { + content: base64Csv, + filename: `${dto.fileName}_error_report.csv`, + type: 'text/csv', + disposition: 'attachment', + }, + ], }); this.logger.debug(`Sent import failure email to ${dto.email}`); } diff --git a/api/src/modules/import-data/sourcing-data/dto-processor.service.ts b/api/src/modules/import-data/sourcing-data/dto-processor.service.ts index e015053e5..e6a0325ad 100644 --- a/api/src/modules/import-data/sourcing-data/dto-processor.service.ts +++ b/api/src/modules/import-data/sourcing-data/dto-processor.service.ts @@ -13,7 +13,7 @@ import { CreateSourcingRecordDto } from 'modules/sourcing-records/dto/create.sou import { CreateSourcingLocationDto } from 'modules/sourcing-locations/dto/create.sourcing-location.dto'; import { WorkSheet } from 'xlsx'; import { SourcingRecord } from 'modules/sourcing-records/sourcing-record.entity'; -import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validators/sourcing-data.class.validator'; +import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validation/sourcing-data.class.validator'; import { validateOrReject } from 'class-validator'; import { plainToClass } from 'class-transformer'; import { CreateIndicatorDto } from 'modules/indicators/dto/create.indicator.dto'; diff --git a/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts b/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts index 5d7ee74a7..05f48375a 100644 --- a/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts +++ b/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts @@ -106,7 +106,7 @@ export class SourcingDataImportService { // TODO: The worker handler also updates the task, check if this is redundant await this.tasksService.updateImportTask({ taskId, - newErrors: err.message, + newErrors: err.response.message, }); throw new BadRequestException( 'Import failed. There are constraint errors present in the file', diff --git a/api/src/modules/import-data/sourcing-data/validators/address-input.custom.validator.ts b/api/src/modules/import-data/sourcing-data/validation/address-input.custom.validator.ts similarity index 98% rename from api/src/modules/import-data/sourcing-data/validators/address-input.custom.validator.ts rename to api/src/modules/import-data/sourcing-data/validation/address-input.custom.validator.ts index b61aac7c6..e98b6d137 100644 --- a/api/src/modules/import-data/sourcing-data/validators/address-input.custom.validator.ts +++ b/api/src/modules/import-data/sourcing-data/validation/address-input.custom.validator.ts @@ -4,7 +4,7 @@ import { ValidatorConstraintInterface, } from 'class-validator'; import { LOCATION_TYPES } from 'modules/sourcing-locations/sourcing-location.entity'; -import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validators/sourcing-data.class.validator'; +import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validation/sourcing-data.class.validator'; @ValidatorConstraint({ name: 'location_address', async: false }) export class LocationAddressInputValidator @@ -40,6 +40,7 @@ export class LocationAddressInputValidator return true; } } + defaultMessage(args: ValidationArguments): string { if ( (args.object as SourcingDataExcelValidator).location_type === diff --git a/api/src/modules/import-data/sourcing-data/validators/latitude-input.custom.validator.ts b/api/src/modules/import-data/sourcing-data/validation/latitude-input.custom.validator.ts similarity index 98% rename from api/src/modules/import-data/sourcing-data/validators/latitude-input.custom.validator.ts rename to api/src/modules/import-data/sourcing-data/validation/latitude-input.custom.validator.ts index b2a505d6a..eb66e1aa1 100644 --- a/api/src/modules/import-data/sourcing-data/validators/latitude-input.custom.validator.ts +++ b/api/src/modules/import-data/sourcing-data/validation/latitude-input.custom.validator.ts @@ -4,7 +4,7 @@ import { ValidatorConstraintInterface, } from 'class-validator'; import { LOCATION_TYPES } from 'modules/sourcing-locations/sourcing-location.entity'; -import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validators/sourcing-data.class.validator'; +import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validation/sourcing-data.class.validator'; @ValidatorConstraint({ name: 'latitude', async: false }) export class LocationLatitudeInputValidator @@ -38,6 +38,7 @@ export class LocationLatitudeInputValidator return true; } } + defaultMessage(args: ValidationArguments): string { if ( (args.object as SourcingDataExcelValidator).location_type === diff --git a/api/src/modules/import-data/sourcing-data/validators/longitude-input.custom.validator.ts b/api/src/modules/import-data/sourcing-data/validation/longitude-input.custom.validator.ts similarity index 97% rename from api/src/modules/import-data/sourcing-data/validators/longitude-input.custom.validator.ts rename to api/src/modules/import-data/sourcing-data/validation/longitude-input.custom.validator.ts index 271ab98c9..cfedf062a 100644 --- a/api/src/modules/import-data/sourcing-data/validators/longitude-input.custom.validator.ts +++ b/api/src/modules/import-data/sourcing-data/validation/longitude-input.custom.validator.ts @@ -4,7 +4,7 @@ import { ValidatorConstraintInterface, } from 'class-validator'; import { LOCATION_TYPES } from 'modules/sourcing-locations/sourcing-location.entity'; -import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validators/sourcing-data.class.validator'; +import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validation/sourcing-data.class.validator'; @ValidatorConstraint({ name: 'longitude', async: false }) export class LocationLongitudeInputValidator @@ -38,6 +38,7 @@ export class LocationLongitudeInputValidator return true; } } + defaultMessage(args: ValidationArguments): string { if ( (args.object as SourcingDataExcelValidator).location_type === diff --git a/api/src/modules/import-data/sourcing-data/validation/sourcing-data-excel.validator.ts b/api/src/modules/import-data/sourcing-data/validation/sourcing-data-excel.validator.ts new file mode 100644 index 000000000..2421bc8e4 --- /dev/null +++ b/api/src/modules/import-data/sourcing-data/validation/sourcing-data-excel.validator.ts @@ -0,0 +1,10 @@ +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class SourcingDataExcelValidator { + constructor() {} + + async validateSheet(sheet: any, sheetName: string): Promise { + return null as any; + } +} diff --git a/api/src/modules/import-data/sourcing-data/validators/sourcing-data.class.validator.ts b/api/src/modules/import-data/sourcing-data/validation/sourcing-data.class.validator.ts similarity index 91% rename from api/src/modules/import-data/sourcing-data/validators/sourcing-data.class.validator.ts rename to api/src/modules/import-data/sourcing-data/validation/sourcing-data.class.validator.ts index 2b19ed9f8..25a6d6259 100644 --- a/api/src/modules/import-data/sourcing-data/validators/sourcing-data.class.validator.ts +++ b/api/src/modules/import-data/sourcing-data/validation/sourcing-data.class.validator.ts @@ -12,9 +12,9 @@ import { ValidateNested, } from 'class-validator'; import { LOCATION_TYPES } from 'modules/sourcing-locations/sourcing-location.entity'; -import { LocationAddressInputValidator } from 'modules/import-data/sourcing-data/validators/address-input.custom.validator'; -import { LocationLatitudeInputValidator } from 'modules/import-data/sourcing-data/validators/latitude-input.custom.validator'; -import { LocationLongitudeInputValidator } from 'modules/import-data/sourcing-data/validators/longitude-input.custom.validator'; +import { LocationAddressInputValidator } from 'modules/import-data/sourcing-data/validation/address-input.custom.validator'; +import { LocationLatitudeInputValidator } from 'modules/import-data/sourcing-data/validation/latitude-input.custom.validator'; +import { LocationLongitudeInputValidator } from 'modules/import-data/sourcing-data/validation/longitude-input.custom.validator'; import { Type } from 'class-transformer'; const MAX_INT32_VALUE: number = 2147483647; diff --git a/api/src/modules/import-data/sourcing-data/validation/validators/business-units.sheet-validator.ts b/api/src/modules/import-data/sourcing-data/validation/validators/business-units.sheet-validator.ts new file mode 100644 index 000000000..b7db878be --- /dev/null +++ b/api/src/modules/import-data/sourcing-data/validation/validators/business-units.sheet-validator.ts @@ -0,0 +1,14 @@ +import { IsNotEmpty, IsOptional, IsString } from 'class-validator'; + +export class BusinessUnitsSheetValidator { + @IsNotEmpty({ message: 'Business Unit path_id must not be empty' }) + @IsString() + path_id: string; + + @IsNotEmpty({ message: 'Supplier Name must not be empty' }) + name: string; + + @IsOptional() + @IsString() + description: string; +} diff --git a/api/src/modules/import-data/sourcing-data/validation/validators/indicators.sheet-validator.ts b/api/src/modules/import-data/sourcing-data/validation/validators/indicators.sheet-validator.ts new file mode 100644 index 000000000..49c76f816 --- /dev/null +++ b/api/src/modules/import-data/sourcing-data/validation/validators/indicators.sheet-validator.ts @@ -0,0 +1,18 @@ +import { IsEnum, IsNotEmpty, IsString, MinLength } from 'class-validator'; + +export class IndicatorsSheetValidator { + @IsString() + @IsNotEmpty() + @MinLength(1) + name!: string; + + @IsString() + @IsNotEmpty() + @IsEnum(['active', 'inactive']) + status!: string; + + @IsString() + @IsNotEmpty() + @MinLength(1) + nameCode!: string; +} diff --git a/api/src/modules/import-data/sourcing-data/validation/validators/material.sheet-validator.ts b/api/src/modules/import-data/sourcing-data/validation/validators/material.sheet-validator.ts new file mode 100644 index 000000000..333fb173f --- /dev/null +++ b/api/src/modules/import-data/sourcing-data/validation/validators/material.sheet-validator.ts @@ -0,0 +1,32 @@ +import { + IsEnum, + IsJSON, + IsNotEmpty, + IsOptional, + IsString, + MinLength, +} from 'class-validator'; + +//TODO: double check if we only use the material sheet in the excel to activate materials + +export class SheetValidatorMaterial { + @IsString() + @MinLength(1, { message: 'Material hs_2017_code is too short' }) + @IsNotEmpty({ message: 'Material hs_2017_code must not be empty' }) + hs_2017_code: string; + + @IsNotEmpty({ message: 'Material Name must not be empty' }) + name: string; + + @IsOptional() + @IsString() + description: string; + + @IsString() + @IsEnum(['active', 'inactive']) + status: string; + + @IsOptional() + @IsJSON() + metadata: string; +} diff --git a/api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.sheet-validator.ts b/api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.sheet-validator.ts new file mode 100644 index 000000000..6824d3eb3 --- /dev/null +++ b/api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.sheet-validator.ts @@ -0,0 +1,88 @@ +import { + IsArray, + IsEnum, + IsNotEmpty, + IsNumber, + IsOptional, + IsString, + Max, + Min, + MinLength, + Validate, + ValidateNested, +} from 'class-validator'; +import { LOCATION_TYPES } from 'modules/sourcing-locations/sourcing-location.entity'; +import { LocationAddressInputValidator } from 'modules/import-data/sourcing-data/validation/address-input.custom.validator'; +import { LocationLatitudeInputValidator } from 'modules/import-data/sourcing-data/validation/latitude-input.custom.validator'; +import { LocationLongitudeInputValidator } from 'modules/import-data/sourcing-data/validation/longitude-input.custom.validator'; +import { Type } from 'class-transformer'; + +const MAX_INT32_VALUE: number = 2147483647; + +export class SourcingDataSheetValidator { + @IsNotEmpty({ + message: 'Material hs code cannot be empty', + }) + @IsString() + @MinLength(2) + 'material.hsCode': string; + + @IsNotEmpty({ + message: 'Business Unit path cannot be empty', + }) + @IsString() + 'business_unit.path': string; + + @IsString() + @IsOptional() + 't1_supplier.name': string; + + @IsString() + @IsOptional() + 'producer.name': string; + + @IsNotEmpty({ + message: + 'New location type input is required for the selected intervention type', + }) + @IsEnum( + Object.values(LOCATION_TYPES).map((s: string) => s.replace(/-/g, ' ')), + { + message: `Available columns for new location type: ${Object.values( + LOCATION_TYPES, + ).join(', ')}`, + }, + ) + 'location_type': LOCATION_TYPES; + + @IsNotEmpty({ + message: 'Location country input is required', + }) + @IsString() + 'location_country_input': string; + + @Validate(LocationAddressInputValidator) + 'location_address_input': string; + + @Validate(LocationLatitudeInputValidator) + 'location_latitude_input': number; + + @Validate(LocationLongitudeInputValidator) + 'location_longitude_input': number; + + @IsArray() + @ValidateNested({ each: true }) + @Type(() => SourcingRecordExcelValidator) + 'sourcingRecords': SourcingRecordExcelValidator[]; +} + +class SourcingRecordExcelValidator { + @IsNumber() + @Min(0) + @Max(MAX_INT32_VALUE) + tonnage: number; + + @IsNumber() + @IsNotEmpty() + year: number; +} diff --git a/api/src/modules/import-data/sourcing-data/validation/validators/supplier.sheet-validator.ts b/api/src/modules/import-data/sourcing-data/validation/validators/supplier.sheet-validator.ts new file mode 100644 index 000000000..b0ac3a08f --- /dev/null +++ b/api/src/modules/import-data/sourcing-data/validation/validators/supplier.sheet-validator.ts @@ -0,0 +1,15 @@ +import { IsNotEmpty, IsOptional, IsString, MinLength } from 'class-validator'; + +export class SheetValidatorSupplier { + @IsString() + @MinLength(1, { message: 'Supplier Name is too short' }) + @IsNotEmpty({ message: 'Supplier Name must not be empty' }) + path_id: string; + + @IsNotEmpty({ message: 'Supplier Name must not be empty' }) + name: string; + + @IsOptional() + @IsString() + description: string; +} diff --git a/api/src/modules/import-data/workers/import-data.consumer.ts b/api/src/modules/import-data/workers/import-data.consumer.ts index 93ae1fcea..581731d4d 100644 --- a/api/src/modules/import-data/workers/import-data.consumer.ts +++ b/api/src/modules/import-data/workers/import-data.consumer.ts @@ -10,7 +10,7 @@ import { Logger, ServiceUnavailableException } from '@nestjs/common'; import { ImportDataService } from 'modules/import-data/import-data.service'; import { ExcelImportJob } from 'modules/import-data/workers/import-data.producer'; import { TasksService } from 'modules/tasks/tasks.service'; -import { Task, TASK_STATUS } from 'modules/tasks/task.entity'; +import { Task, TASK_STATUS, TASK_TYPE } from 'modules/tasks/task.entity'; import { importQueueName } from 'modules/import-data/workers/import-queue.name'; import { ImportProgressSocket } from 'modules/events/import-data/import-progress.socket'; import { ImportMailService } from 'modules/import-data/import-mail/import-mail.service'; @@ -47,6 +47,20 @@ export class ImportDataConsumer { this.logger.error( `Import Failed for file: ${job.data.xlsxFileData.filename} for task: ${task.id}: ${err}`, ); + + const errorReport: string = await this.tasksService.getTaskErrorReport( + task.id, + { + type: TASK_TYPE.SOURCING_DATA_IMPORT, + }, + ); + + await this.importMail.sendImportFailureMail({ + email: task.user.email, + fileName: job.data.xlsxFileData.originalname, + importDate: task.createdAt, + errorContent: errorReport, + }); } @OnQueueCompleted() diff --git a/api/src/modules/notifications/email/email.service.interface.ts b/api/src/modules/notifications/email/email.service.interface.ts index 26b8a30f9..54dca351d 100644 --- a/api/src/modules/notifications/email/email.service.interface.ts +++ b/api/src/modules/notifications/email/email.service.interface.ts @@ -5,6 +5,14 @@ export class SendMailDTO { subject: string; html: string; text?: string; + attachments?: MailAttachment[]; +} + +interface MailAttachment { + content: string; + filename: string; + type: string; + disposition: string; } export interface IEmailService { From f8ad8bb084750210c297c4dd1332e9c64c7b790b Mon Sep 17 00:00:00 2001 From: alexeh Date: Mon, 29 Apr 2024 08:33:54 +0300 Subject: [PATCH 3/8] Add custom excel validation error --- .../validation/validators/excel-validation.error.ts | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 api/src/modules/import-data/sourcing-data/validation/validators/excel-validation.error.ts diff --git a/api/src/modules/import-data/sourcing-data/validation/validators/excel-validation.error.ts b/api/src/modules/import-data/sourcing-data/validation/validators/excel-validation.error.ts new file mode 100644 index 000000000..4c1eec5f9 --- /dev/null +++ b/api/src/modules/import-data/sourcing-data/validation/validators/excel-validation.error.ts @@ -0,0 +1,9 @@ +export class ExcelValidationError extends Error { + public validationErrors: any[] = []; + + constructor(message: string, validationErrors: any[] = []) { + super(message); + this.name = 'Validation Error in Excel File'; + this.validationErrors = validationErrors; + } +} From 10e6c15afdde8a00b362aa56ed05e6e776e59bea Mon Sep 17 00:00:00 2001 From: alexeh Date: Mon, 29 Apr 2024 11:10:56 +0300 Subject: [PATCH 4/8] sourcing location group non mandatory --- api/src/modules/sourcing-locations/sourcing-location.entity.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/modules/sourcing-locations/sourcing-location.entity.ts b/api/src/modules/sourcing-locations/sourcing-location.entity.ts index 863e881f4..695bada90 100644 --- a/api/src/modules/sourcing-locations/sourcing-location.entity.ts +++ b/api/src/modules/sourcing-locations/sourcing-location.entity.ts @@ -199,7 +199,7 @@ export class SourcingLocation extends TimestampedBaseEntity { }, ) @JoinColumn({ name: 'sourcingLocationGroupId' }) - sourcingLocationGroup: SourcingLocationGroup; + sourcingLocationGroup?: SourcingLocationGroup; @Column({ nullable: true }) @ApiPropertyOptional() From 9a5fdb2a123300256955b6f8d213a2a7f7e7c1eb Mon Sep 17 00:00:00 2001 From: alexeh Date: Tue, 30 Apr 2024 11:24:25 +0300 Subject: [PATCH 5/8] Improve excel validation error handling and report --- .../modules/import-data/import-data.module.ts | 2 + .../sourcing-data/dto-processor.service.ts | 38 +++---- .../sourcing-data-import.service.ts | 32 +++--- .../validation/excel-validator.service.ts | 101 ++++++++++++++++++ .../sourcing-data-excel.validator.ts | 10 -- .../business-units.sheet-validator.ts | 4 +- .../validators/excel-validation.error.ts | 3 +- .../validators/material.sheet-validator.ts | 4 +- .../sourcing-data.sheet-validator.ts | 5 +- .../validators/supplier.sheet-validator.ts | 3 +- .../workers/import-data.consumer.ts | 3 +- api/src/modules/tasks/task-report.service.ts | 6 +- 12 files changed, 149 insertions(+), 62 deletions(-) create mode 100644 api/src/modules/import-data/sourcing-data/validation/excel-validator.service.ts delete mode 100644 api/src/modules/import-data/sourcing-data/validation/sourcing-data-excel.validator.ts diff --git a/api/src/modules/import-data/import-data.module.ts b/api/src/modules/import-data/import-data.module.ts index 75d95f75c..d5dae5113 100644 --- a/api/src/modules/import-data/import-data.module.ts +++ b/api/src/modules/import-data/import-data.module.ts @@ -30,6 +30,7 @@ import { EudrImportService } from 'modules/import-data/eudr/eudr.import.service' import { EUDRDTOProcessor } from 'modules/import-data/eudr/eudr.dto-processor.service'; import { ImportMailService } from 'modules/import-data/import-mail/import-mail.service'; import { NotificationsModule } from 'modules/notifications/notifications.module'; +import { ExcelValidatorService } from 'modules/import-data/sourcing-data/validation/excel-validator.service'; // TODO: Move EUDR related stuff to EUDR modules @@ -76,6 +77,7 @@ import { NotificationsModule } from 'modules/notifications/notifications.module' EudrImportService, EUDRDTOProcessor, ImportMailService, + ExcelValidatorService, { provide: 'FILE_UPLOAD_SIZE_LIMIT', useValue: config.get('fileUploads.sizeLimit'), diff --git a/api/src/modules/import-data/sourcing-data/dto-processor.service.ts b/api/src/modules/import-data/sourcing-data/dto-processor.service.ts index e6a0325ad..9e742f7e1 100644 --- a/api/src/modules/import-data/sourcing-data/dto-processor.service.ts +++ b/api/src/modules/import-data/sourcing-data/dto-processor.service.ts @@ -91,14 +91,14 @@ export class SourcingRecordsDtoProcessorService { importData.indicators, ); - const processedSourcingData: Record = - await this.parseSourcingDataFromSheet(importData.sourcingData); - - /** - * Validating parsed and cleaned from Sourcing Data sheet - */ - - await this.validateSourcingData(processedSourcingData.sourcingData); + // const processedSourcingData: Record = + // await this.parseSourcingDataFromSheet(importData.sourcingData); + // + // /** + // * Validating parsed and cleaned from Sourcing Data sheet + // */ + // + // await this.validateSourcingData(processedSourcingData.sourcingData); /** * Builds SourcingData from parsed XLSX */ @@ -126,7 +126,7 @@ export class SourcingRecordsDtoProcessorService { return regexMatch ? parseInt(regexMatch[0]) : null; } - private async parseSourcingDataFromSheet(customData: WorkSheet[]): Promise<{ + async parseSourcingDataFromSheet(customData: WorkSheet[]): Promise<{ sourcingData: SourcingData[]; }> { this.logger.debug(`Cleaning ${customData.length} custom data rows`); @@ -135,18 +135,13 @@ export class SourcingRecordsDtoProcessorService { /** * Clean all hashmaps that are empty therefore useless */ - const nonEmptyData: WorkSheet[] = customData.filter( - (row: WorkSheet) => - row['material.hsCode'] && row['material.hsCode'] !== '', - ); - this.logger.debug(`Found ${customData.length} non-empty custom data rows`); /** * Separate base properties common for each sourcing-location row * Separate metadata properties to metadata object common for each sourcing-location row * Check if current key contains a year ('2018_tonnage') if so, get the year and its value */ - for (const eachRecordOfCustomData of nonEmptyData) { + for (const eachRecordOfCustomData of customData) { const sourcingRecords: Record[] = []; const years: Record = {}; const baseProps: Record = {}; @@ -163,9 +158,6 @@ export class SourcingRecordsDtoProcessorService { } else if (!this.isSourcingLocationData(field)) { metadata[field] = eachRecordOfCustomData[field]; } else { - eachRecordOfCustomData[field] === ' ' - ? (baseProps[field] = null) - : (baseProps[field] = eachRecordOfCustomData[field]); } } /** @@ -289,13 +281,13 @@ export class SourcingRecordsDtoProcessorService { this.logger.debug( `Creating sourcing data DTOs from ${sourcingData.length} data rows`, ); - const processedSourcingData: Record = - await this.parseSourcingDataFromSheet(sourcingData); - - await this.validateSourcingData(processedSourcingData.sourcingData); + // const processedSourcingData: Record = + // await this.parseSourcingDataFromSheet(sourcingData); + // + // await this.validateSourcingData(processedSourcingData.sourcingData); const sourcingLocationDtos: any[] = []; - for (const importRow of processedSourcingData.sourcingData) { + for (const importRow of sourcingData) { const sourcingLocationDto: CreateSourcingLocationDto = await this.createSourcingLocationDTOFromData( importRow, diff --git a/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts b/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts index 05f48375a..9d3e77c91 100644 --- a/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts +++ b/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts @@ -33,6 +33,13 @@ import { ImpactService } from 'modules/impact/impact.service'; import { ImpactCalculator } from 'modules/indicator-records/services/impact-calculator.service'; import { ImportProgressTrackerFactory } from 'modules/events/import-data/import-progress.tracker.factory'; import { ValidationProgressTracker } from 'modules/import-data/progress-tracker/validation.progress-tracker'; +import { SourcingDataExcelValidator } from './validation/sourcing-data.class.validator'; +import { + ExcelValidatorService, + Sheet, +} from './validation/excel-validator.service'; +import { ExcelValidationError } from './validation/validators/excel-validation.error'; +import { GeoCodingError } from '../../geo-coding/errors/geo-coding.error'; export interface LocationData { locationAddressInput?: string; @@ -84,6 +91,7 @@ export class SourcingDataImportService { protected readonly impactService: ImpactService, protected readonly impactCalculator: ImpactCalculator, protected readonly importProgressTrackerFactory: ImportProgressTrackerFactory, + protected readonly excelValidator: ExcelValidatorService, ) {} async importSourcingData(filePath: string, taskId: string): Promise { @@ -98,20 +106,13 @@ export class SourcingDataImportService { title: 'Sourcing Records import from XLSX file', }); - const dtoMatchedData: SourcingRecordsDtos = - await this.validateAndCreateDTOs( - parsedXLSXDataset, - sourcingLocationGroup.id, - ).catch(async (err: any) => { - // TODO: The worker handler also updates the task, check if this is redundant - await this.tasksService.updateImportTask({ - taskId, - newErrors: err.response.message, - }); - throw new BadRequestException( - 'Import failed. There are constraint errors present in the file', - ); - }); + const { data: dtoMatchedData, validationErrors } = + await this.excelValidator.validate( + parsedXLSXDataset as unknown as Sheet, + ); + if (validationErrors.length) { + throw new ExcelValidationError('Validation Errors', validationErrors); + } //TODO: Implement transactional import. Move geocoding to first step @@ -157,8 +158,7 @@ export class SourcingDataImportService { dtoMatchedData.sourcingData, ); if (errors.length) { - await this.tasksService.updateImportTask({ taskId, newErrors: errors }); - throw new BadRequestException( + throw new GeoCodingError( 'Import failed. There are GeoCoding errors present in the file', ); } diff --git a/api/src/modules/import-data/sourcing-data/validation/excel-validator.service.ts b/api/src/modules/import-data/sourcing-data/validation/excel-validator.service.ts new file mode 100644 index 000000000..f58274e5a --- /dev/null +++ b/api/src/modules/import-data/sourcing-data/validation/excel-validator.service.ts @@ -0,0 +1,101 @@ +import { Injectable } from '@nestjs/common'; +import { SourcingRecordsDtoProcessorService } from '../dto-processor.service'; +import { SheetValidatorMaterial } from './validators/material.sheet-validator'; +import { BusinessUnitsSheetValidator } from './validators/business-units.sheet-validator'; +import { SheetValidatorSupplier } from './validators/supplier.sheet-validator'; +import { SourcingDataSheetValidator } from './validators/sourcing-data.sheet-validator'; +import { IndicatorsSheetValidator } from './validators/indicators.sheet-validator'; +import { plainToInstance } from 'class-transformer'; +import { validate, ValidationError } from 'class-validator'; +import { SourcingRecordsSheets } from '../sourcing-records-sheets.interface'; + +export type Sheet = { + materials: SheetValidatorMaterial[]; + businessUnits: BusinessUnitsSheetValidator[]; + suppliers: SheetValidatorSupplier[]; + sourcingData: SourcingDataSheetValidator[]; + indicators: IndicatorsSheetValidator[]; +}; + +type SheetName = + | 'materials' + | 'businessUnits' + | 'suppliers' + | 'sourcingData' + | 'indicators'; + +export const SHEET_NAMES: SheetName[] = [ + 'materials', + 'businessUnits', + 'suppliers', + 'sourcingData', + 'indicators', +]; + +@Injectable() +export class ExcelValidatorService { + private readonly validators = { + materials: SheetValidatorMaterial, + businessUnits: BusinessUnitsSheetValidator, + suppliers: SheetValidatorSupplier, + sourcingData: SourcingDataSheetValidator, + indicators: IndicatorsSheetValidator, + }; + + constructor( + private readonly dtoProcessor: SourcingRecordsDtoProcessorService, + ) {} + + async validate(sheet: Sheet): Promise { + const validationErrors: any[] = []; + const { sourcingData } = await this.dtoProcessor.parseSourcingDataFromSheet( + sheet.sourcingData, + ); + + sheet.sourcingData = + sourcingData as unknown as SourcingDataSheetValidator[]; + + for (const sheetName of SHEET_NAMES) { + const sheetData = sheet[sheetName]; + for (const [index, record] of sheetData.entries()) { + const validator = this.getValidator(sheetName); + const instance: any = plainToInstance(validator, record); + if ( + instance.location_address_input === + 'country of production should not have address' + ) { + console.log('stop here'); + } + const errors: ValidationError[] = await validate(instance); + if (errors.length) { + errors.forEach((error: ValidationError) => { + if (error.constraints) { + Object.values(error.constraints).forEach((constraint) => { + validationErrors.push({ + line: this.setLineNumber(index, sheetName), + column: error.property, + error: constraint, + sheet: sheetName, + type: 'validation-error', + }); + }); + } + }); + } + } + } + + const data = await this.dtoProcessor.createDTOsFromSourcingRecordsSheets( + sheet as unknown as SourcingRecordsSheets, + ); + return { data, validationErrors }; + } + + private getValidator(sheetName: SheetName): any { + return this.validators[sheetName]; + } + + private setLineNumber(index: number, sheetName: SheetName): number { + return sheetName === 'sourcingData' ? index + 5 : index + 1; + } +} diff --git a/api/src/modules/import-data/sourcing-data/validation/sourcing-data-excel.validator.ts b/api/src/modules/import-data/sourcing-data/validation/sourcing-data-excel.validator.ts deleted file mode 100644 index 2421bc8e4..000000000 --- a/api/src/modules/import-data/sourcing-data/validation/sourcing-data-excel.validator.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Injectable } from '@nestjs/common'; - -@Injectable() -export class SourcingDataExcelValidator { - constructor() {} - - async validateSheet(sheet: any, sheetName: string): Promise { - return null as any; - } -} diff --git a/api/src/modules/import-data/sourcing-data/validation/validators/business-units.sheet-validator.ts b/api/src/modules/import-data/sourcing-data/validation/validators/business-units.sheet-validator.ts index b7db878be..ff8bfa232 100644 --- a/api/src/modules/import-data/sourcing-data/validation/validators/business-units.sheet-validator.ts +++ b/api/src/modules/import-data/sourcing-data/validation/validators/business-units.sheet-validator.ts @@ -2,10 +2,10 @@ import { IsNotEmpty, IsOptional, IsString } from 'class-validator'; export class BusinessUnitsSheetValidator { @IsNotEmpty({ message: 'Business Unit path_id must not be empty' }) - @IsString() + @IsString({ message: 'Business Unit path_id must be a string' }) path_id: string; - @IsNotEmpty({ message: 'Supplier Name must not be empty' }) + @IsNotEmpty({ message: 'Business Unit Name must not be empty' }) name: string; @IsOptional() diff --git a/api/src/modules/import-data/sourcing-data/validation/validators/excel-validation.error.ts b/api/src/modules/import-data/sourcing-data/validation/validators/excel-validation.error.ts index 4c1eec5f9..35d3d5386 100644 --- a/api/src/modules/import-data/sourcing-data/validation/validators/excel-validation.error.ts +++ b/api/src/modules/import-data/sourcing-data/validation/validators/excel-validation.error.ts @@ -1,7 +1,8 @@ export class ExcelValidationError extends Error { + // TODO: type this, should be equal to the geocoding validation error shape public validationErrors: any[] = []; - constructor(message: string, validationErrors: any[] = []) { + constructor(message: string, validationErrors: any[]) { super(message); this.name = 'Validation Error in Excel File'; this.validationErrors = validationErrors; diff --git a/api/src/modules/import-data/sourcing-data/validation/validators/material.sheet-validator.ts b/api/src/modules/import-data/sourcing-data/validation/validators/material.sheet-validator.ts index 333fb173f..ca40cce1e 100644 --- a/api/src/modules/import-data/sourcing-data/validation/validators/material.sheet-validator.ts +++ b/api/src/modules/import-data/sourcing-data/validation/validators/material.sheet-validator.ts @@ -23,7 +23,9 @@ export class SheetValidatorMaterial { description: string; @IsString() - @IsEnum(['active', 'inactive']) + @IsEnum(['active', 'inactive'], { + message: 'Material status must be active or inactive', + }) status: string; @IsOptional() diff --git a/api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.sheet-validator.ts b/api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.sheet-validator.ts index 6824d3eb3..47e2bc93d 100644 --- a/api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.sheet-validator.ts +++ b/api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.sheet-validator.ts @@ -23,8 +23,6 @@ export class SourcingDataSheetValidator { @IsNotEmpty({ message: 'Material hs code cannot be empty', }) - @IsString() - @MinLength(2) 'material.hsCode': string; @IsNotEmpty({ @@ -42,8 +40,7 @@ export class SourcingDataSheetValidator { 'producer.name': string; @IsNotEmpty({ - message: - 'New location type input is required for the selected intervention type', + message: 'location type input is required', }) @IsEnum( Object.values(LOCATION_TYPES).map((s: string) => s.replace(/-/g, ' ')), diff --git a/api/src/modules/import-data/sourcing-data/validation/validators/supplier.sheet-validator.ts b/api/src/modules/import-data/sourcing-data/validation/validators/supplier.sheet-validator.ts index b0ac3a08f..b1bad79ac 100644 --- a/api/src/modules/import-data/sourcing-data/validation/validators/supplier.sheet-validator.ts +++ b/api/src/modules/import-data/sourcing-data/validation/validators/supplier.sheet-validator.ts @@ -2,8 +2,7 @@ import { IsNotEmpty, IsOptional, IsString, MinLength } from 'class-validator'; export class SheetValidatorSupplier { @IsString() - @MinLength(1, { message: 'Supplier Name is too short' }) - @IsNotEmpty({ message: 'Supplier Name must not be empty' }) + @IsNotEmpty({ message: 'Supplier path_id must not be empty' }) path_id: string; @IsNotEmpty({ message: 'Supplier Name must not be empty' }) diff --git a/api/src/modules/import-data/workers/import-data.consumer.ts b/api/src/modules/import-data/workers/import-data.consumer.ts index 581731d4d..dc205fb96 100644 --- a/api/src/modules/import-data/workers/import-data.consumer.ts +++ b/api/src/modules/import-data/workers/import-data.consumer.ts @@ -36,11 +36,12 @@ export class ImportDataConsumer { // TODO: Handle events finished and failed cases @OnQueueFailed() - async onJobFailed(job: Job, err: Error): Promise { + async onJobFailed(job: Job, err: any): Promise { const task: Task | undefined = await this.tasksService.updateImportTask({ taskId: job.data.taskId, newStatus: TASK_STATUS.FAILED, message: err.message, + newErrors: err.validationErrors, }); this.importSocket.emitImportFailureToSocket({ error: err }); diff --git a/api/src/modules/tasks/task-report.service.ts b/api/src/modules/tasks/task-report.service.ts index fc80ab3d8..492c432f9 100644 --- a/api/src/modules/tasks/task-report.service.ts +++ b/api/src/modules/tasks/task-report.service.ts @@ -14,8 +14,10 @@ export class TaskReportService { ) {} async createImportErrorReport(errors: ErrorRecord[]): Promise { - const parserOptions: { fields: ['line', 'error'] } = { - fields: ['line', 'error'], + const parserOptions: { + fields: ['line', 'error', 'column', 'sheet', 'type']; + } = { + fields: ['line', 'error', 'column', 'sheet', 'type'], }; return this.reportService.generateCSVReport(errors, parserOptions); } From 7c90eb8a705c5fc811adf34cd910e6940cd448a1 Mon Sep 17 00:00:00 2001 From: alexeh Date: Wed, 1 May 2024 12:34:09 +0300 Subject: [PATCH 6/8] WIP - geocoding error management --- api/src/modules/geo-coding/errors/geo-coding.error.ts | 10 ++++++++-- api/src/modules/geo-coding/geo-coding.service.ts | 8 +++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/api/src/modules/geo-coding/errors/geo-coding.error.ts b/api/src/modules/geo-coding/errors/geo-coding.error.ts index d3f6cdd6a..4cc7305be 100644 --- a/api/src/modules/geo-coding/errors/geo-coding.error.ts +++ b/api/src/modules/geo-coding/errors/geo-coding.error.ts @@ -1,3 +1,9 @@ -import { BadRequestException } from '@nestjs/common'; +export class GeoCodingError extends Error { + validationErrors: any[] = []; -export class GeoCodingError extends BadRequestException {} + constructor(message: string, validationErrors?: any[]) { + super(message); + this.name = 'GeoCoding Error'; + this.validationErrors = validationErrors ?? []; + } +} diff --git a/api/src/modules/geo-coding/geo-coding.service.ts b/api/src/modules/geo-coding/geo-coding.service.ts index 134f78ed1..da513aab4 100644 --- a/api/src/modules/geo-coding/geo-coding.service.ts +++ b/api/src/modules/geo-coding/geo-coding.service.ts @@ -92,7 +92,13 @@ export class GeoCodingService extends GeoCodingAbstractClass { } progressTracker.trackProgress(); } catch (e: any) { - errors.push({ line: i + 5, error: e.message }); + errors.push({ + line: i + 5, + error: e.message, + type: 'geo-coding-error', + sheet: 'sourcingData', + column: null, + }); progressTracker.trackProgress(); } } From c2d694f03473c26591e42876ed894398b152c64d Mon Sep 17 00:00:00 2001 From: alexeh Date: Thu, 2 May 2024 08:41:28 +0300 Subject: [PATCH 7/8] refactor and cleaning import validation step --- .../eudr/eudr.dto-processor.service.ts | 2 +- .../sourcing-data/dto-processor.service.ts | 2 +- .../sourcing-data-import.service.ts | 89 +------------------ .../validation/excel-validator.service.ts | 24 +++-- .../address-input.custom.validator.ts | 2 +- .../latitude-input.custom.validator.ts | 2 +- .../longitude-input.custom.validator.ts | 2 +- .../sourcing-data.class.validator.ts | 6 +- .../sourcing-data.sheet-validator.ts | 6 +- 9 files changed, 31 insertions(+), 104 deletions(-) rename api/src/modules/import-data/sourcing-data/validation/{ => validators}/address-input.custom.validator.ts (97%) rename api/src/modules/import-data/sourcing-data/validation/{ => validators}/latitude-input.custom.validator.ts (97%) rename api/src/modules/import-data/sourcing-data/validation/{ => validators}/longitude-input.custom.validator.ts (97%) rename api/src/modules/import-data/sourcing-data/validation/{ => validators}/sourcing-data.class.validator.ts (89%) diff --git a/api/src/modules/import-data/eudr/eudr.dto-processor.service.ts b/api/src/modules/import-data/eudr/eudr.dto-processor.service.ts index 96b308907..1c98d92c8 100644 --- a/api/src/modules/import-data/eudr/eudr.dto-processor.service.ts +++ b/api/src/modules/import-data/eudr/eudr.dto-processor.service.ts @@ -6,7 +6,7 @@ import { } from '@nestjs/common'; import { CreateSourcingLocationDto } from 'modules/sourcing-locations/dto/create.sourcing-location.dto'; import { SourcingRecord } from 'modules/sourcing-records/sourcing-record.entity'; -import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validation/sourcing-data.class.validator'; +import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validation/validators/sourcing-data.class.validator'; import { validateOrReject } from 'class-validator'; import { plainToClass } from 'class-transformer'; import { diff --git a/api/src/modules/import-data/sourcing-data/dto-processor.service.ts b/api/src/modules/import-data/sourcing-data/dto-processor.service.ts index 9e742f7e1..984f8493c 100644 --- a/api/src/modules/import-data/sourcing-data/dto-processor.service.ts +++ b/api/src/modules/import-data/sourcing-data/dto-processor.service.ts @@ -13,7 +13,7 @@ import { CreateSourcingRecordDto } from 'modules/sourcing-records/dto/create.sou import { CreateSourcingLocationDto } from 'modules/sourcing-locations/dto/create.sourcing-location.dto'; import { WorkSheet } from 'xlsx'; import { SourcingRecord } from 'modules/sourcing-records/sourcing-record.entity'; -import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validation/sourcing-data.class.validator'; +import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validation/validators/sourcing-data.class.validator'; import { validateOrReject } from 'class-validator'; import { plainToClass } from 'class-transformer'; import { CreateIndicatorDto } from 'modules/indicators/dto/create.indicator.dto'; diff --git a/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts b/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts index 9d3e77c91..7916ab527 100644 --- a/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts +++ b/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts @@ -33,13 +33,12 @@ import { ImpactService } from 'modules/impact/impact.service'; import { ImpactCalculator } from 'modules/indicator-records/services/impact-calculator.service'; import { ImportProgressTrackerFactory } from 'modules/events/import-data/import-progress.tracker.factory'; import { ValidationProgressTracker } from 'modules/import-data/progress-tracker/validation.progress-tracker'; -import { SourcingDataExcelValidator } from './validation/sourcing-data.class.validator'; import { ExcelValidatorService, Sheet, -} from './validation/excel-validator.service'; -import { ExcelValidationError } from './validation/validators/excel-validation.error'; -import { GeoCodingError } from '../../geo-coding/errors/geo-coding.error'; +} from 'modules/import-data/sourcing-data/validation/excel-validator.service'; +import { ExcelValidationError } from 'modules/import-data/sourcing-data/validation/validators/excel-validation.error'; +import { GeoCodingError } from 'modules/geo-coding/errors/geo-coding.error'; export interface LocationData { locationAddressInput?: string; @@ -90,7 +89,6 @@ export class SourcingDataImportService { protected readonly indicatorRecordService: IndicatorRecordsService, protected readonly impactService: ImpactService, protected readonly impactCalculator: ImpactCalculator, - protected readonly importProgressTrackerFactory: ImportProgressTrackerFactory, protected readonly excelValidator: ExcelValidatorService, ) {} @@ -206,41 +204,6 @@ export class SourcingDataImportService { } } - private async validateDTOs( - dtoLists: SourcingRecordsDtos, - ): Promise> { - const validationErrorArray: { - line: number; - property: string; - message: any; - }[] = []; - for (const parsedSheet in dtoLists) { - if (dtoLists.hasOwnProperty(parsedSheet)) { - for (const [i, dto] of dtoLists[ - parsedSheet as keyof SourcingRecordsDtos - ].entries()) { - try { - await validateOrReject(dto); - } catch (err: any) { - validationErrorArray.push({ - line: i + 5, - property: err[0].property, - message: err[0].constraints, - }); - } - } - } - } - - /** - * @note If errors are thrown, we should bypass all-exceptions.exception.filter.ts - * in order to return the array containing errors in a more readable way - * Or add a function per entity to validate - */ - if (validationErrorArray.length) - throw new BadRequestException(validationErrorArray); - } - /** * @note: Deletes DB content from required entities * to ensure DB is prune prior loading a XLSX dataset @@ -318,50 +281,4 @@ export class SourcingDataImportService { } return sourcingData; } - - async validateAndCreateDTOs( - parsedXLSXDataset: SourcingRecordsSheets, - sourcingLocationGroupId: string, - ): Promise { - const processingMap: Record = { - materials: this.dtoProcessor.createMaterialDtos.bind(this.dtoProcessor), - businessUnits: this.dtoProcessor.createBusinessUnitDtos.bind( - this.dtoProcessor, - ), - suppliers: this.dtoProcessor.createSupplierDtos.bind(this.dtoProcessor), - countries: this.dtoProcessor.createAdminRegionDtos.bind( - this.dtoProcessor, - ), - sourcingData: this.dtoProcessor.createSourcingDataDTOs.bind( - this.dtoProcessor, - ), - indicators: this.dtoProcessor.createIndicatorDtos.bind(this.dtoProcessor), - sourcingLocationGroupId, - }; - const results: any = {} as SourcingRecordsDtos; - - const totalSteps: number = Object.keys(parsedXLSXDataset).length + 1; // +1 for final validation step - - const tracker: ValidationProgressTracker = - this.importProgressTrackerFactory.createValidationProgressTracker({ - totalSteps: totalSteps, - }); - - for (const [sheetName, sheetEntities] of Object.entries( - parsedXLSXDataset, - )) { - if (sheetName in processingMap) { - const result: any = await processingMap[sheetName]( - sheetEntities, - sourcingLocationGroupId, - ); - results[sheetName] = result; - } else { - await this.validateDTOs(results); - } - tracker.trackProgress(); - } - - return results; - } } diff --git a/api/src/modules/import-data/sourcing-data/validation/excel-validator.service.ts b/api/src/modules/import-data/sourcing-data/validation/excel-validator.service.ts index f58274e5a..e24f6870a 100644 --- a/api/src/modules/import-data/sourcing-data/validation/excel-validator.service.ts +++ b/api/src/modules/import-data/sourcing-data/validation/excel-validator.service.ts @@ -8,6 +8,8 @@ import { IndicatorsSheetValidator } from './validators/indicators.sheet-validato import { plainToInstance } from 'class-transformer'; import { validate, ValidationError } from 'class-validator'; import { SourcingRecordsSheets } from '../sourcing-records-sheets.interface'; +import { ValidationProgressTracker } from '../../progress-tracker/validation.progress-tracker'; +import { ImportProgressTrackerFactory } from 'modules/events/import-data/import-progress.tracker.factory'; export type Sheet = { materials: SheetValidatorMaterial[]; @@ -44,14 +46,16 @@ export class ExcelValidatorService { constructor( private readonly dtoProcessor: SourcingRecordsDtoProcessorService, + private readonly importProgressTrackerFactory: ImportProgressTrackerFactory, ) {} async validate(sheet: Sheet): Promise { + const progressTracker: ValidationProgressTracker = + this.getProgressTracker(sheet); const validationErrors: any[] = []; const { sourcingData } = await this.dtoProcessor.parseSourcingDataFromSheet( sheet.sourcingData, ); - sheet.sourcingData = sourcingData as unknown as SourcingDataSheetValidator[]; @@ -60,12 +64,6 @@ export class ExcelValidatorService { for (const [index, record] of sheetData.entries()) { const validator = this.getValidator(sheetName); const instance: any = plainToInstance(validator, record); - if ( - instance.location_address_input === - 'country of production should not have address' - ) { - console.log('stop here'); - } const errors: ValidationError[] = await validate(instance); if (errors.length) { errors.forEach((error: ValidationError) => { @@ -82,6 +80,7 @@ export class ExcelValidatorService { } }); } + progressTracker.trackProgress(); } } @@ -98,4 +97,15 @@ export class ExcelValidatorService { private setLineNumber(index: number, sheetName: SheetName): number { return sheetName === 'sourcingData' ? index + 5 : index + 1; } + + private getProgressTracker(sheet: Sheet): ValidationProgressTracker { + const totalSteps: number = + SHEET_NAMES.reduce( + (acc: number, sheetName: SheetName) => acc + sheet[sheetName].length, + 0, + ) + 1; + return this.importProgressTrackerFactory.createValidationProgressTracker({ + totalSteps: totalSteps, + }); + } } diff --git a/api/src/modules/import-data/sourcing-data/validation/address-input.custom.validator.ts b/api/src/modules/import-data/sourcing-data/validation/validators/address-input.custom.validator.ts similarity index 97% rename from api/src/modules/import-data/sourcing-data/validation/address-input.custom.validator.ts rename to api/src/modules/import-data/sourcing-data/validation/validators/address-input.custom.validator.ts index e98b6d137..4ef76533d 100644 --- a/api/src/modules/import-data/sourcing-data/validation/address-input.custom.validator.ts +++ b/api/src/modules/import-data/sourcing-data/validation/validators/address-input.custom.validator.ts @@ -4,7 +4,7 @@ import { ValidatorConstraintInterface, } from 'class-validator'; import { LOCATION_TYPES } from 'modules/sourcing-locations/sourcing-location.entity'; -import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validation/sourcing-data.class.validator'; +import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validation/validators/sourcing-data.class.validator'; @ValidatorConstraint({ name: 'location_address', async: false }) export class LocationAddressInputValidator diff --git a/api/src/modules/import-data/sourcing-data/validation/latitude-input.custom.validator.ts b/api/src/modules/import-data/sourcing-data/validation/validators/latitude-input.custom.validator.ts similarity index 97% rename from api/src/modules/import-data/sourcing-data/validation/latitude-input.custom.validator.ts rename to api/src/modules/import-data/sourcing-data/validation/validators/latitude-input.custom.validator.ts index eb66e1aa1..ed7149a20 100644 --- a/api/src/modules/import-data/sourcing-data/validation/latitude-input.custom.validator.ts +++ b/api/src/modules/import-data/sourcing-data/validation/validators/latitude-input.custom.validator.ts @@ -4,7 +4,7 @@ import { ValidatorConstraintInterface, } from 'class-validator'; import { LOCATION_TYPES } from 'modules/sourcing-locations/sourcing-location.entity'; -import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validation/sourcing-data.class.validator'; +import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validation/validators/sourcing-data.class.validator'; @ValidatorConstraint({ name: 'latitude', async: false }) export class LocationLatitudeInputValidator diff --git a/api/src/modules/import-data/sourcing-data/validation/longitude-input.custom.validator.ts b/api/src/modules/import-data/sourcing-data/validation/validators/longitude-input.custom.validator.ts similarity index 97% rename from api/src/modules/import-data/sourcing-data/validation/longitude-input.custom.validator.ts rename to api/src/modules/import-data/sourcing-data/validation/validators/longitude-input.custom.validator.ts index cfedf062a..645973ac4 100644 --- a/api/src/modules/import-data/sourcing-data/validation/longitude-input.custom.validator.ts +++ b/api/src/modules/import-data/sourcing-data/validation/validators/longitude-input.custom.validator.ts @@ -4,7 +4,7 @@ import { ValidatorConstraintInterface, } from 'class-validator'; import { LOCATION_TYPES } from 'modules/sourcing-locations/sourcing-location.entity'; -import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validation/sourcing-data.class.validator'; +import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validation/validators/sourcing-data.class.validator'; @ValidatorConstraint({ name: 'longitude', async: false }) export class LocationLongitudeInputValidator diff --git a/api/src/modules/import-data/sourcing-data/validation/sourcing-data.class.validator.ts b/api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.class.validator.ts similarity index 89% rename from api/src/modules/import-data/sourcing-data/validation/sourcing-data.class.validator.ts rename to api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.class.validator.ts index 25a6d6259..acd9b3f39 100644 --- a/api/src/modules/import-data/sourcing-data/validation/sourcing-data.class.validator.ts +++ b/api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.class.validator.ts @@ -12,9 +12,9 @@ import { ValidateNested, } from 'class-validator'; import { LOCATION_TYPES } from 'modules/sourcing-locations/sourcing-location.entity'; -import { LocationAddressInputValidator } from 'modules/import-data/sourcing-data/validation/address-input.custom.validator'; -import { LocationLatitudeInputValidator } from 'modules/import-data/sourcing-data/validation/latitude-input.custom.validator'; -import { LocationLongitudeInputValidator } from 'modules/import-data/sourcing-data/validation/longitude-input.custom.validator'; +import { LocationAddressInputValidator } from 'modules/import-data/sourcing-data/validation/validators/address-input.custom.validator'; +import { LocationLatitudeInputValidator } from 'modules/import-data/sourcing-data/validation/validators/latitude-input.custom.validator'; +import { LocationLongitudeInputValidator } from 'modules/import-data/sourcing-data/validation/validators/longitude-input.custom.validator'; import { Type } from 'class-transformer'; const MAX_INT32_VALUE: number = 2147483647; diff --git a/api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.sheet-validator.ts b/api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.sheet-validator.ts index 47e2bc93d..9d1e1ee16 100644 --- a/api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.sheet-validator.ts +++ b/api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.sheet-validator.ts @@ -12,9 +12,9 @@ import { ValidateNested, } from 'class-validator'; import { LOCATION_TYPES } from 'modules/sourcing-locations/sourcing-location.entity'; -import { LocationAddressInputValidator } from 'modules/import-data/sourcing-data/validation/address-input.custom.validator'; -import { LocationLatitudeInputValidator } from 'modules/import-data/sourcing-data/validation/latitude-input.custom.validator'; -import { LocationLongitudeInputValidator } from 'modules/import-data/sourcing-data/validation/longitude-input.custom.validator'; +import { LocationAddressInputValidator } from 'modules/import-data/sourcing-data/validation/validators/address-input.custom.validator'; +import { LocationLatitudeInputValidator } from 'modules/import-data/sourcing-data/validation/validators/latitude-input.custom.validator'; +import { LocationLongitudeInputValidator } from 'modules/import-data/sourcing-data/validation/validators/longitude-input.custom.validator'; import { Type } from 'class-transformer'; const MAX_INT32_VALUE: number = 2147483647; From 2f71a11ed041795be2b78b11004fe1775eca7b27 Mon Sep 17 00:00:00 2001 From: alexeh Date: Thu, 2 May 2024 11:14:06 +0300 Subject: [PATCH 8/8] Add geocoding errors to task --- .../sourcing-data/sourcing-data-import.service.ts | 1 + .../validation/excel-validator.service.ts | 2 +- .../validators/sourcing-data.sheet-validator.ts | 13 +++++-------- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts b/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts index 7916ab527..3a23b297c 100644 --- a/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts +++ b/api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts @@ -158,6 +158,7 @@ export class SourcingDataImportService { if (errors.length) { throw new GeoCodingError( 'Import failed. There are GeoCoding errors present in the file', + errors, ); } const warnings: string[] = []; diff --git a/api/src/modules/import-data/sourcing-data/validation/excel-validator.service.ts b/api/src/modules/import-data/sourcing-data/validation/excel-validator.service.ts index e24f6870a..efae6e508 100644 --- a/api/src/modules/import-data/sourcing-data/validation/excel-validator.service.ts +++ b/api/src/modules/import-data/sourcing-data/validation/excel-validator.service.ts @@ -95,7 +95,7 @@ export class ExcelValidatorService { } private setLineNumber(index: number, sheetName: SheetName): number { - return sheetName === 'sourcingData' ? index + 5 : index + 1; + return sheetName === 'sourcingData' ? index + 5 : index + 2; } private getProgressTracker(sheet: Sheet): ValidationProgressTracker { diff --git a/api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.sheet-validator.ts b/api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.sheet-validator.ts index 9d1e1ee16..4cfa33674 100644 --- a/api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.sheet-validator.ts +++ b/api/src/modules/import-data/sourcing-data/validation/validators/sourcing-data.sheet-validator.ts @@ -42,14 +42,11 @@ export class SourcingDataSheetValidator { @IsNotEmpty({ message: 'location type input is required', }) - @IsEnum( - Object.values(LOCATION_TYPES).map((s: string) => s.replace(/-/g, ' ')), - { - message: `Available columns for new location type: ${Object.values( - LOCATION_TYPES, - ).join(', ')}`, - }, - ) + @IsEnum(Object.values(LOCATION_TYPES), { + message: `Available columns for new location type: ${Object.values( + LOCATION_TYPES, + ).join(', ')}`, + }) 'location_type': LOCATION_TYPES; @IsNotEmpty({