Chore/import progress testing and mail notificacion

api/src/modules/events/import-data/import-progress.emitter.ts

@@ -1,7 +1,7 @@
 import { Injectable } from '@nestjs/common';
 import { EventBus } from '@nestjs/cqrs';
 import { ImportProgressUpdateEvent } from 'modules/events/import-data/import-progress.event';
-import { ImportProgressSteps } from './types';
+import { ImportProgressSteps } from 'modules/events/import-data/types';
 
 /**
  * @note: We use eventBus instead of commandBus because even tho broadcasting via websockets can be considered a command, it is not a command in the context of events. (apparently)

api/src/modules/geo-coding/errors/geo-coding.error.ts

@@ -1,3 +1,9 @@
-import { BadRequestException } from '@nestjs/common';
+export class GeoCodingError extends Error {
+  validationErrors: any[] = [];
 
-export class GeoCodingError extends BadRequestException {}
+  constructor(message: string, validationErrors?: any[]) {
+    super(message);
+    this.name = 'GeoCoding Error';
+    this.validationErrors = validationErrors ?? [];
+  }
+}

api/src/modules/geo-coding/geo-coding.service.ts

@@ -92,7 +92,13 @@ export class GeoCodingService extends GeoCodingAbstractClass {
         }
         progressTracker.trackProgress();
       } catch (e: any) {
-        errors.push({ line: i + 5, error: e.message });
+        errors.push({
+          line: i + 5,
+          error: e.message,
+          type: 'geo-coding-error',
+          sheet: 'sourcingData',
+          column: null,
+        });
         progressTracker.trackProgress();
       }
     }

api/src/modules/import-data/eudr/eudr.dto-processor.service.ts

@@ -6,7 +6,7 @@ import {
 } from '@nestjs/common';
 import { CreateSourcingLocationDto } from 'modules/sourcing-locations/dto/create.sourcing-location.dto';
 import { SourcingRecord } from 'modules/sourcing-records/sourcing-record.entity';
-import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validators/sourcing-data.class.validator';
+import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validation/validators/sourcing-data.class.validator';
 import { validateOrReject } from 'class-validator';
 import { plainToClass } from 'class-transformer';
 import {

api/src/modules/import-data/import-data.module.ts

@@ -28,6 +28,9 @@ import { ImpactModule } from 'modules/impact/impact.module';
 import { WebSocketsModule } from 'modules/notifications/websockets/websockets.module';
 import { EudrImportService } from 'modules/import-data/eudr/eudr.import.service';
 import { EUDRDTOProcessor } from 'modules/import-data/eudr/eudr.dto-processor.service';
+import { ImportMailService } from 'modules/import-data/import-mail/import-mail.service';
+import { NotificationsModule } from 'modules/notifications/notifications.module';
+import { ExcelValidatorService } from 'modules/import-data/sourcing-data/validation/excel-validator.service';
 
 // TODO: Move EUDR related stuff to EUDR modules
 
@@ -61,6 +64,7 @@ import { EUDRDTOProcessor } from 'modules/import-data/eudr/eudr.dto-processor.se
     IndicatorsModule,
     ImpactModule,
     WebSocketsModule,
+    NotificationsModule,
   ],
   providers: [
     MulterConfigService,
@@ -72,6 +76,8 @@ import { EUDRDTOProcessor } from 'modules/import-data/eudr/eudr.dto-processor.se
     ImportDataService,
     EudrImportService,
     EUDRDTOProcessor,
+    ImportMailService,
+    ExcelValidatorService,
     {
       provide: 'FILE_UPLOAD_SIZE_LIMIT',
       useValue: config.get('fileUploads.sizeLimit'),

api/src/modules/import-data/import-mail/import-mail.service.ts

@@ -0,0 +1,57 @@
+import { Inject, Injectable, Logger } from '@nestjs/common';
+import { IEmailService } from 'modules/notifications/email/email.service.interface';
+
+type ImportEmailDTO = {
+  email: string;
+  fileName: string;
+  importDate: string;
+  errorContent?: string;
+};
+
+@Injectable()
+export class ImportMailService {
+  logger: Logger = new Logger(ImportMailService.name);
+
+  constructor(@Inject('IEmailService') private emailService: IEmailService) {}
+
+  async sendImportSuccessMail(dto: ImportEmailDTO): Promise<void> {
+    const htmlContent: string = `
+      <p>Dear ${dto.email || 'User'},</p>
+      <p>Your import of file ${
+        dto.fileName
+      } has been successfully processed.</p>
+      <p>Import date: ${dto.importDate}</p>
+    `;
+    await this.emailService.sendMail({
+      to: dto.email,
+      subject: 'Import success',
+      html: htmlContent,
+    });
+    this.logger.debug(`Sent import success email to ${dto.email}`);
+  }
+
+  async sendImportFailureMail(dto: ImportEmailDTO): Promise<void> {
+    const base64Csv = Buffer.from(dto.errorContent || '').toString('base64');
+
+    const htmlContent: string = `
+      <p>Dear ${dto.email || 'User'},</p>
+      <p>Your import of file ${dto.fileName} has failed.</p>
+      <p>Import date: ${dto.importDate}</p>
+      <p>Please find the error report attached.</p>
+    `;
+    await this.emailService.sendMail({
+      to: dto.email,
+      subject: 'Import failure',
+      html: htmlContent,
+      attachments: [
+        {
+          content: base64Csv,
+          filename: `${dto.fileName}_error_report.csv`,
+          type: 'text/csv',
+          disposition: 'attachment',
+        },
+      ],
+    });
+    this.logger.debug(`Sent import failure email to ${dto.email}`);
+  }
+}

api/src/modules/import-data/sourcing-data/dto-processor.service.ts

@@ -13,7 +13,7 @@ import { CreateSourcingRecordDto } from 'modules/sourcing-records/dto/create.sou
 import { CreateSourcingLocationDto } from 'modules/sourcing-locations/dto/create.sourcing-location.dto';
 import { WorkSheet } from 'xlsx';
 import { SourcingRecord } from 'modules/sourcing-records/sourcing-record.entity';
-import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validators/sourcing-data.class.validator';
+import { SourcingDataExcelValidator } from 'modules/import-data/sourcing-data/validation/validators/sourcing-data.class.validator';
 import { validateOrReject } from 'class-validator';
 import { plainToClass } from 'class-transformer';
 import { CreateIndicatorDto } from 'modules/indicators/dto/create.indicator.dto';
@@ -91,14 +91,14 @@ export class SourcingRecordsDtoProcessorService {
       importData.indicators,
     );
 
-    const processedSourcingData: Record<string, any> =
-      await this.parseSourcingDataFromSheet(importData.sourcingData);
-
-    /**
-     * Validating parsed and cleaned from Sourcing Data sheet
-     */
-
-    await this.validateSourcingData(processedSourcingData.sourcingData);
+    // const processedSourcingData: Record<string, any> =
+    //   await this.parseSourcingDataFromSheet(importData.sourcingData);
+    //
+    // /**
+    //  * Validating parsed and cleaned from Sourcing Data sheet
+    //  */
+    //
+    // await this.validateSourcingData(processedSourcingData.sourcingData);
     /**
      * Builds SourcingData from parsed XLSX
      */
@@ -126,7 +126,7 @@ export class SourcingRecordsDtoProcessorService {
     return regexMatch ? parseInt(regexMatch[0]) : null;
   }
 
-  private async parseSourcingDataFromSheet(customData: WorkSheet[]): Promise<{
+  async parseSourcingDataFromSheet(customData: WorkSheet[]): Promise<{
     sourcingData: SourcingData[];
   }> {
     this.logger.debug(`Cleaning ${customData.length} custom data rows`);
@@ -135,18 +135,13 @@ export class SourcingRecordsDtoProcessorService {
     /**
      * Clean all hashmaps that are empty therefore useless
      */
-    const nonEmptyData: WorkSheet[] = customData.filter(
-      (row: WorkSheet) =>
-        row['material.hsCode'] && row['material.hsCode'] !== '',
-    );
-    this.logger.debug(`Found ${customData.length} non-empty custom data rows`);
     /**
      * Separate base properties common for each sourcing-location row
      * Separate metadata properties to metadata object common for each sourcing-location row
      * Check if current key contains a year ('2018_tonnage') if so, get the year and its value
      */
 
-    for (const eachRecordOfCustomData of nonEmptyData) {
+    for (const eachRecordOfCustomData of customData) {
       const sourcingRecords: Record<string, any>[] = [];
       const years: Record<string, any> = {};
       const baseProps: Record<string, any> = {};
@@ -163,9 +158,6 @@ export class SourcingRecordsDtoProcessorService {
         } else if (!this.isSourcingLocationData(field)) {
           metadata[field] = eachRecordOfCustomData[field];
         } else {
-          eachRecordOfCustomData[field] === ' '
-            ? (baseProps[field] = null)
-            : (baseProps[field] = eachRecordOfCustomData[field]);
         }
       }
       /**
@@ -289,13 +281,13 @@ export class SourcingRecordsDtoProcessorService {
     this.logger.debug(
       `Creating sourcing data DTOs from ${sourcingData.length} data rows`,
     );
-    const processedSourcingData: Record<string, any> =
-      await this.parseSourcingDataFromSheet(sourcingData);
-
-    await this.validateSourcingData(processedSourcingData.sourcingData);
+    // const processedSourcingData: Record<string, any> =
+    //   await this.parseSourcingDataFromSheet(sourcingData);
+    //
+    // await this.validateSourcingData(processedSourcingData.sourcingData);
 
     const sourcingLocationDtos: any[] = [];
-    for (const importRow of processedSourcingData.sourcingData) {
+    for (const importRow of sourcingData) {
       const sourcingLocationDto: CreateSourcingLocationDto =
         await this.createSourcingLocationDTOFromData(
           importRow,

api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts

@@ -33,6 +33,12 @@ import { ImpactService } from 'modules/impact/impact.service';
 import { ImpactCalculator } from 'modules/indicator-records/services/impact-calculator.service';
 import { ImportProgressTrackerFactory } from 'modules/events/import-data/import-progress.tracker.factory';
 import { ValidationProgressTracker } from 'modules/import-data/progress-tracker/validation.progress-tracker';
+import {
+  ExcelValidatorService,
+  Sheet,
+} from 'modules/import-data/sourcing-data/validation/excel-validator.service';
+import { ExcelValidationError } from 'modules/import-data/sourcing-data/validation/validators/excel-validation.error';
+import { GeoCodingError } from 'modules/geo-coding/errors/geo-coding.error';
 
 export interface LocationData {
   locationAddressInput?: string;
@@ -83,7 +89,7 @@ export class SourcingDataImportService {
     protected readonly indicatorRecordService: IndicatorRecordsService,
     protected readonly impactService: ImpactService,
     protected readonly impactCalculator: ImpactCalculator,
-    protected readonly importProgressTrackerFactory: ImportProgressTrackerFactory,
+    protected readonly excelValidator: ExcelValidatorService,
   ) {}
 
   async importSourcingData(filePath: string, taskId: string): Promise<any> {
@@ -98,20 +104,13 @@ export class SourcingDataImportService {
           title: 'Sourcing Records import from XLSX file',
         });
 
-      const dtoMatchedData: SourcingRecordsDtos =
-        await this.validateAndCreateDTOs(
-          parsedXLSXDataset,
-          sourcingLocationGroup.id,
-        ).catch(async (err: any) => {
-          // TODO: The worker handler also updates the task, check if this is redundant
-          await this.tasksService.updateImportTask({
-            taskId,
-            newErrors: err.message,
-          });
-          throw new BadRequestException(
-            'Import failed. There are constraint errors present in the file',
-          );
-        });
+      const { data: dtoMatchedData, validationErrors } =
+        await this.excelValidator.validate(
+          parsedXLSXDataset as unknown as Sheet,
+        );
+      if (validationErrors.length) {
+        throw new ExcelValidationError('Validation Errors', validationErrors);
+      }
 
       //TODO: Implement transactional import. Move geocoding to first step
 
@@ -157,9 +156,9 @@ export class SourcingDataImportService {
           dtoMatchedData.sourcingData,
         );
       if (errors.length) {
-        await this.tasksService.updateImportTask({ taskId, newErrors: errors });
-        throw new BadRequestException(
+        throw new GeoCodingError(
           'Import failed. There are GeoCoding errors present in the file',
+          errors,
         );
       }
       const warnings: string[] = [];
@@ -206,41 +205,6 @@ export class SourcingDataImportService {
     }
   }
 
-  private async validateDTOs(
-    dtoLists: SourcingRecordsDtos,
-  ): Promise<void | Array<ErrorConstructor>> {
-    const validationErrorArray: {
-      line: number;
-      property: string;
-      message: any;
-    }[] = [];
-    for (const parsedSheet in dtoLists) {
-      if (dtoLists.hasOwnProperty(parsedSheet)) {
-        for (const [i, dto] of dtoLists[
-          parsedSheet as keyof SourcingRecordsDtos
-        ].entries()) {
-          try {
-            await validateOrReject(dto);
-          } catch (err: any) {
-            validationErrorArray.push({
-              line: i + 5,
-              property: err[0].property,
-              message: err[0].constraints,
-            });
-          }
-        }
-      }
-    }
-
-    /**
-     * @note If errors are thrown, we should bypass all-exceptions.exception.filter.ts
-     * in order to return the array containing errors in a more readable way
-     * Or add a function per entity to validate
-     */
-    if (validationErrorArray.length)
-      throw new BadRequestException(validationErrorArray);
-  }
-
   /**
    * @note: Deletes DB content from required entities
    * to ensure DB is prune prior loading a XLSX dataset
@@ -318,50 +282,4 @@ export class SourcingDataImportService {
     }
     return sourcingData;
   }
-
-  async validateAndCreateDTOs(
-    parsedXLSXDataset: SourcingRecordsSheets,
-    sourcingLocationGroupId: string,
-  ): Promise<SourcingRecordsDtos> {
-    const processingMap: Record<string, any> = {
-      materials: this.dtoProcessor.createMaterialDtos.bind(this.dtoProcessor),
-      businessUnits: this.dtoProcessor.createBusinessUnitDtos.bind(
-        this.dtoProcessor,
-      ),
-      suppliers: this.dtoProcessor.createSupplierDtos.bind(this.dtoProcessor),
-      countries: this.dtoProcessor.createAdminRegionDtos.bind(
-        this.dtoProcessor,
-      ),
-      sourcingData: this.dtoProcessor.createSourcingDataDTOs.bind(
-        this.dtoProcessor,
-      ),
-      indicators: this.dtoProcessor.createIndicatorDtos.bind(this.dtoProcessor),
-      sourcingLocationGroupId,
-    };
-    const results: any = {} as SourcingRecordsDtos;
-
-    const totalSteps: number = Object.keys(parsedXLSXDataset).length + 1; // +1 for final validation step
-
-    const tracker: ValidationProgressTracker =
-      this.importProgressTrackerFactory.createValidationProgressTracker({
-        totalSteps: totalSteps,
-      });
-
-    for (const [sheetName, sheetEntities] of Object.entries(
-      parsedXLSXDataset,
-    )) {
-      if (sheetName in processingMap) {
-        const result: any = await processingMap[sheetName](
-          sheetEntities,
-          sourcingLocationGroupId,
-        );
-        results[sheetName] = result;
-      } else {
-        await this.validateDTOs(results);
-      }
-      tracker.trackProgress();
-    }
-
-    return results;
-  }
 }