-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
12 changed files
with
294 additions
and
85 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
42 changes: 42 additions & 0 deletions
42
api/src/modules/events/import-data/import-progress.tracker.factory.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| import { Injectable } from '@nestjs/common'; | ||
| import { ImportProgressEmitter } from 'modules/events/import-data/import-progress.emitter'; | ||
| import { GeoCodingProgressTracker } from 'modules/geo-coding/progress-tracker/geo-coding.progress-tracker'; | ||
| import { ImpactCalculationProgressTracker } from 'modules/impact/progress-tracker/impact-calculation.progress-tracker'; | ||
| import { SourcingDataImportProgressTracker } from 'modules/sourcing-locations/progress-tracker/sourcing-data.progress-tracker'; | ||
|
|
||
| @Injectable() | ||
| export class ImportProgressTrackerFactory { | ||
| constructor(public readonly importProgressEmitter: ImportProgressEmitter) { | ||
| this.importProgressEmitter = importProgressEmitter; | ||
| } | ||
|
|
||
| createGeoCodingTracker(geoCodeTrackingOptions: { | ||
| totalLocations: number; | ||
| }): GeoCodingProgressTracker { | ||
| return new GeoCodingProgressTracker( | ||
| this.importProgressEmitter, | ||
| geoCodeTrackingOptions, | ||
| ); | ||
| } | ||
|
|
||
| createSourcingDataImportTracker(sourcingDataImportOptions: { | ||
| totalRecords: number; | ||
| totalChunks: number; | ||
| }): SourcingDataImportProgressTracker { | ||
| return new SourcingDataImportProgressTracker(this.importProgressEmitter, { | ||
| totalRecords: sourcingDataImportOptions.totalRecords, | ||
| totalChunks: sourcingDataImportOptions.totalChunks, | ||
| }); | ||
| } | ||
|
|
||
| createImpactCalculationProgressTracker(impactCalculationOptions: { | ||
| totalRecords: number; | ||
| totalChunks: number; | ||
| startingPercentage?: number; | ||
| }): ImpactCalculationProgressTracker { | ||
| return new ImpactCalculationProgressTracker( | ||
| this.importProgressEmitter, | ||
| impactCalculationOptions, | ||
| ); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
19 changes: 0 additions & 19 deletions
19
api/src/modules/geo-coding/progress-tracker/geo-coding.progress-tracker.factory.ts
This file was deleted.
Oops, something went wrong.
61 changes: 61 additions & 0 deletions
61
api/src/modules/impact/progress-tracker/impact-calculation.progress-tracker.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| import { ImportProgressEmitter } from 'modules/events/import-data/import-progress.emitter'; | ||
|
|
||
| export class ImpactCalculationProgressTracker { | ||
| totalRecords: number; | ||
| progress: number = 0; | ||
| progressPerChunk: number; | ||
| private interval: NodeJS.Timer | null = null; | ||
|
|
||
| constructor( | ||
| private readonly importProgressEmitter: ImportProgressEmitter, | ||
| private readonly importTrackInfo: { | ||
| totalRecords: number; | ||
| totalChunks: number; | ||
| startingPercentage?: number; | ||
| estimatedTime?: number; | ||
| }, | ||
| ) { | ||
| this.importProgressEmitter = importProgressEmitter; | ||
| this.totalRecords = importTrackInfo.totalRecords; | ||
| const startingPercentage: number = importTrackInfo.startingPercentage ?? 0; | ||
| this.progressPerChunk = | ||
| (100 - startingPercentage) / importTrackInfo.totalChunks; | ||
| } | ||
|
|
||
| trackProgress(): void { | ||
| this.progress += this.progressPerChunk; | ||
|
|
||
| this.importProgressEmitter.emitImpactCalculationProgress({ | ||
| progress: this.getProgress(), | ||
| }); | ||
| } | ||
|
|
||
| private getProgress(): number { | ||
| return this.progress; | ||
| } | ||
|
|
||
| startProgressInterval(progressIncrement: number, maxProgress: number): void { | ||
| if (this.interval) { | ||
| clearInterval(this.interval); | ||
| } | ||
|
|
||
| this.interval = setInterval(() => { | ||
| this.progress += progressIncrement; | ||
| this.progress = Math.min(this.progress, maxProgress); | ||
| this.importProgressEmitter.emitImpactCalculationProgress({ | ||
| progress: this.getProgress(), | ||
| }); | ||
|
|
||
| if (this.progress >= maxProgress) { | ||
| this.stopProgressInterval(); | ||
| } | ||
| }, 1000); | ||
| } | ||
|
|
||
| stopProgressInterval(): void { | ||
| if (this.interval) { | ||
| clearInterval(this.interval); | ||
| this.interval = null; | ||
| } | ||
| } | ||
| } |
61 changes: 59 additions & 2 deletions
61
api/src/modules/indicator-records/indicator-record.repository.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,13 +1,70 @@ | ||
| import { DataSource } from 'typeorm'; | ||
| import { DataSource, QueryRunner } from 'typeorm'; | ||
| import { IndicatorRecord } from 'modules/indicator-records/indicator-record.entity'; | ||
| import { Injectable, Logger } from '@nestjs/common'; | ||
| import { AppBaseRepository } from 'utils/app-base.repository'; | ||
| import { SaveOptions } from 'typeorm/repository/SaveOptions'; | ||
| import { chunk } from 'lodash'; | ||
| import { AppConfig } from 'utils/app.config'; | ||
| import { ImportProgressTrackerFactory } from 'modules/events/import-data/import-progress.tracker.factory'; | ||
| import { ImpactCalculationProgressTracker } from 'modules/impact/progress-tracker/impact-calculation.progress-tracker'; | ||
|
|
||
| const dbConfig: any = AppConfig.get('db'); | ||
| const batchChunkSize: number = parseInt(`${dbConfig.batchChunkSize}`, 10); | ||
|
|
||
| @Injectable() | ||
| export class IndicatorRecordRepository extends AppBaseRepository<IndicatorRecord> { | ||
| constructor(protected dataSource: DataSource) { | ||
| constructor( | ||
| protected dataSource: DataSource, | ||
| private readonly importProgressTrackerFactory: ImportProgressTrackerFactory, | ||
| ) { | ||
| super(IndicatorRecord, dataSource.createEntityManager()); | ||
| } | ||
|
|
||
| logger: Logger = new Logger(IndicatorRecordRepository.name); | ||
|
|
||
| async saveChunks<IndicatorRecord>( | ||
| entities: IndicatorRecord[], | ||
| options?: SaveOptions, | ||
| ): Promise<IndicatorRecord[]> { | ||
| const queryRunner: QueryRunner = this.dataSource.createQueryRunner(); | ||
| await queryRunner.connect(); | ||
| await queryRunner.startTransaction(); | ||
| const result: IndicatorRecord[][] = []; | ||
| const totalEntities: number = entities.length; | ||
| const totalChunks: number = Math.ceil(totalEntities / batchChunkSize); | ||
| const tracker: ImpactCalculationProgressTracker = | ||
| this.importProgressTrackerFactory.createImpactCalculationProgressTracker({ | ||
| totalRecords: totalEntities, | ||
| totalChunks: totalChunks, | ||
| startingPercentage: 50, | ||
| }); | ||
|
|
||
| try { | ||
| for (const [index, dataChunk] of chunk( | ||
| entities, | ||
| batchChunkSize, | ||
| ).entries()) { | ||
| this.logger.debug( | ||
| `Inserting chunk #${index} (${dataChunk.length} items) from a total of ${totalChunks}...`, | ||
| ); | ||
| const promises: Promise<IndicatorRecord>[] = dataChunk.map( | ||
| (row: IndicatorRecord) => queryRunner.manager.save(row, options), | ||
| ); | ||
| const saved: IndicatorRecord[] = await Promise.all(promises); | ||
| result.push(saved); | ||
| tracker.trackProgress(); | ||
| } | ||
|
|
||
| // commit transaction if every chunk was saved successfully | ||
| await queryRunner.commitTransaction(); | ||
| } catch (err) { | ||
| // rollback changes before throwing error | ||
| await queryRunner.rollbackTransaction(); | ||
| throw err; | ||
| } finally { | ||
| // release query runner which is manually created | ||
| await queryRunner.release(); | ||
| } | ||
| return result.flat(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
31 changes: 31 additions & 0 deletions
31
api/src/modules/sourcing-locations/progress-tracker/sourcing-data.progress-tracker.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| import { ImportProgressEmitter } from 'modules/events/import-data/import-progress.emitter'; | ||
|
|
||
| export class SourcingDataImportProgressTracker { | ||
| totalRecords: number; | ||
| progress: number = 0; | ||
| progressPerChunk: number; | ||
|
|
||
| constructor( | ||
| private readonly importProgressEmitter: ImportProgressEmitter, | ||
| private readonly importTrackInfo: { | ||
| totalRecords: number; | ||
| totalChunks: number; | ||
| }, | ||
| ) { | ||
| this.importProgressEmitter = importProgressEmitter; | ||
| this.totalRecords = importTrackInfo.totalRecords; | ||
| this.progressPerChunk = (100 - 50) / importTrackInfo.totalChunks; | ||
| } | ||
|
|
||
| trackProgress(): void { | ||
| this.progress += this.progressPerChunk; | ||
|
|
||
| this.importProgressEmitter.emitGeocodingProgress({ | ||
| progress: this.getProgress(), | ||
| }); | ||
| } | ||
|
|
||
| private getProgress(): number { | ||
| return this.progress; | ||
| } | ||
| } |
Oops, something went wrong.