Merge remote-tracking branch 'upstream/master'

.gitignore

@@ -41,3 +41,14 @@ cypress/fixtures/
 
 #vscode
 .vscode/*
+
+#script outputs
+src/scripts/products_max_min_calculated_tons.json
+src/scripts/substances_max_min_calculated_tons.json
+src/scripts/error_response_report.json
+src/scripts/d2_tracker_events_updated.json
+
+# unzip files
+glass-amc-recalculate-server
+glass-async-deletions-server
+

i18n/en.pot

@@ -5,8 +5,8 @@ msgstr ""
 "Content-Type: text/plain; charset=utf-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=(n != 1)\n"
-"POT-Creation-Date: 2024-08-07T18:05:19.338Z\n"
-"PO-Revision-Date: 2024-08-07T18:05:19.338Z\n"
+"POT-Creation-Date: 2024-12-12T19:21:44.978Z\n"
+"PO-Revision-Date: 2024-12-12T19:21:44.978Z\n"
 
 msgid "Template {{id}} not loaded"
 msgstr ""
@@ -345,7 +345,7 @@ msgstr ""
 
 msgid ""
 "IMPORTED - The data has been imported, but validations(if applicable) were "
-"not run successfully, in Step 2"
+"not run successfully, in Step 2."
 msgstr ""
 
 msgid ""
@@ -365,7 +365,7 @@ msgstr ""
 msgid "Complete"
 msgstr ""
 
-msgid "Failed to set completed status"
+msgid "Error occurred when setting data submission status to COMPLETED"
 msgstr ""
 
 msgid "Deleting Files"
@@ -377,6 +377,9 @@ msgstr ""
 msgid "This might take several minutes, do not refresh the page or press back."
 msgstr ""
 
+msgid "Marked to be deleted"
+msgstr ""
+
 msgid "Error downloading data"
 msgstr ""
 
@@ -518,6 +521,9 @@ msgstr ""
 msgid "You don't have read access to this page"
 msgstr ""
 
+msgid "Save Questionnaire"
+msgstr ""
+
 msgid "Set as incomplete"
 msgstr ""
 
@@ -557,6 +563,9 @@ msgstr ""
 msgid "Failed to set upload status"
 msgstr ""
 
+msgid "Failed to calculate consumption data"
+msgstr ""
+
 msgid "Importing data and applying validation rules"
 msgstr ""
 

i18n/es.po

@@ -1,7 +1,7 @@
 msgid ""
 msgstr ""
 "Project-Id-Version: i18next-conv\n"
-"POT-Creation-Date: 2024-08-07T18:05:19.338Z\n"
+"POT-Creation-Date: 2024-12-12T19:21:44.978Z\n"
 "PO-Revision-Date: 2018-10-25T09:02:35.143Z\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
@@ -346,7 +346,7 @@ msgstr ""
 
 msgid ""
 "IMPORTED - The data has been imported, but validations(if applicable) were "
-"not run successfully, in Step 2"
+"not run successfully, in Step 2."
 msgstr ""
 
 msgid ""
@@ -367,7 +367,7 @@ msgstr ""
 msgid "Complete"
 msgstr ""
 
-msgid "Failed to set completed status"
+msgid "Error occurred when setting data submission status to COMPLETED"
 msgstr ""
 
 msgid "Deleting Files"
@@ -379,6 +379,9 @@ msgstr ""
 msgid "This might take several minutes, do not refresh the page or press back."
 msgstr ""
 
+msgid "Marked to be deleted"
+msgstr ""
+
 msgid "Error downloading data"
 msgstr ""
 
@@ -520,6 +523,9 @@ msgstr ""
 msgid "You don't have read access to this page"
 msgstr ""
 
+msgid "Save Questionnaire"
+msgstr ""
+
 msgid "Set as incomplete"
 msgstr ""
 
@@ -559,6 +565,9 @@ msgstr ""
 msgid "Failed to set upload status"
 msgstr ""
 
+msgid "Failed to calculate consumption data"
+msgstr ""
+
 msgid "Importing data and applying validation rules"
 msgstr ""
 

package.json

@@ -1,7 +1,7 @@
 {
     "name": "glass",
     "description": "DHIS2 Glass App",
-    "version": "1.6.20",
+    "version": "1.6.23",
     "license": "GPL-3.0",
     "author": "EyeSeeTea team",
     "homepage": ".",
@@ -117,10 +117,14 @@
         "script-example": "npx ts-node src/scripts/index.ts example getExample",
         "start-amc-recalculate": "npx ts-node src/scripts/cliAMC.ts  --debug",
         "build-amc-recalculate": "ncc build src/scripts/cliAMC.ts -m -o $npm_package_name-amc-recalculate-server && zip -r $npm_package_name-amc-recalculate-server.zip $npm_package_name-amc-recalculate-server && npx rimraf $npm_package_name-amc-recalculate-server/",
+        "start-async-deletions": "npx ts-node -r dotenv/config src/scripts/cliAsyncDeletions.ts",
+        "build-async-deletions": "ncc build src/scripts/cliAsyncDeletions.ts -m -o $npm_package_name-async-deletions-server && zip -r $npm_package_name-async-deletions-server.zip $npm_package_name-async-deletions-server && npx rimraf $npm_package_name-async-deletions-server/",
         "amr-agg-data-validation-ris": "npx ts-node -r dotenv/config src/scripts/amr_agg_data_validation.ts",
         "amr-agg-data-reset-ris": "npx ts-node -r dotenv/config src/scripts/amr_agg_data_reset.ts",
         "amr-agg-data-validation-sample": "npx ts-node -r dotenv/config src/scripts/amr_agg_data_validation_sample.ts",
-        "amr-agg-data-reset-sample": "npx ts-node -r dotenv/config src/scripts/amr_agg_data_reset_sample.ts"
+        "amr-agg-data-reset-sample": "npx ts-node -r dotenv/config src/scripts/amr_agg_data_reset_sample.ts",
+        "get-max-min-amc-calculated-tons": "npx ts-node -r dotenv/config src/scripts/get_max_min_amc_calculated_tons.ts",
+        "update-tons-to-kilograms": "npx ts-node -r dotenv/config src/scripts/update_autocalculated_and_manual_tons_to_kilograms.ts"
     },
     "manifest.webapp": {
         "name": "glass",

src/CompositionRoot.ts

@@ -93,6 +93,12 @@ import { DownloadEmptyTemplateUseCase } from "./domain/usecases/DownloadEmptyTem
 import { DownloadPopulatedTemplateUseCase } from "./domain/usecases/DownloadPopulatedTemplateUseCase";
 import { CountryDefaultRepository } from "./data/repositories/CountryDefaultRepository";
 import { GetAllCountriesUseCase } from "./domain/usecases/GetAllCountriesUseCase";
+import { SetToAsyncDeletionsUseCase } from "./domain/usecases/SetToAsyncDeletionsUseCase";
+import { GetAsyncDeletionsUseCase } from "./domain/usecases/GetAsyncDeletionsUseCase";
+import { DeletePrimaryFileDataUseCase } from "./domain/usecases/data-entry/DeletePrimaryFileDataUseCase";
+import { DeleteSecondaryFileDataUseCase } from "./domain/usecases/data-entry/DeleteSecondaryFileDataUseCase";
+import { DownloadDocumentAsArrayBufferUseCase } from "./domain/usecases/DownloadDocumentAsArrayBufferUseCase";
+import { GetGlassUploadByIdUseCase } from "./domain/usecases/GetGlassUploadByIdUseCase";
 
 export function getCompositionRoot(instance: Instance) {
     const api = getD2APiFromInstance(instance);
@@ -156,6 +162,7 @@ export function getCompositionRoot(instance: Instance) {
         }),
         glassUploads: getExecute({
             getAll: new GetGlassUploadsUseCase(glassUploadsRepository),
+            getById: new GetGlassUploadByIdUseCase(glassUploadsRepository),
             setStatus: new SetUploadStatusUseCase(glassUploadsRepository),
             getAMRUploadsForCurrentDataSubmission: new GetGlassUploadsByDataSubmissionUseCase(
                 glassUploadsRepository,
@@ -166,12 +173,15 @@ export function getCompositionRoot(instance: Instance) {
             setBatchId: new SetUploadBatchIdUseCase(glassUploadsRepository),
             saveImportSummaryErrorsOfFiles: new SaveImportSummaryErrorsOfFilesInUploadsUseCase(glassUploadsRepository),
             getCurrentDataSubmissionFileType: new GetUploadsByDataSubmissionUseCase(glassUploadsRepository),
+            setToAsyncDeletions: new SetToAsyncDeletionsUseCase(glassUploadsRepository),
+            getAsyncDeletions: new GetAsyncDeletionsUseCase(glassUploadsRepository),
         }),
         glassDocuments: getExecute({
             getAll: new GetGlassDocumentsUseCase(glassDocumentsRepository),
             upload: new UploadDocumentUseCase(glassDocumentsRepository, glassUploadsRepository),
             deleteByUploadId: new DeleteDocumentInfoByUploadIdUseCase(glassDocumentsRepository, glassUploadsRepository),
             download: new DownloadDocumentUseCase(glassDocumentsRepository),
+            downloadAsArrayBuffer: new DownloadDocumentAsArrayBufferUseCase(glassDocumentsRepository),
             updateSecondaryFileWithPrimaryId: new UpdateSampleUploadWithRisIdUseCase(glassUploadsRepository),
         }),
         fileSubmission: getExecute({
@@ -180,7 +190,6 @@ export function getCompositionRoot(instance: Instance) {
                 risIndividualFungalRepository,
                 metadataRepository,
                 dataValuesRepository,
-                glassModuleRepository,
                 dhis2EventsDefaultRepository,
                 excelRepository,
                 glassDocumentsRepository,
@@ -231,6 +240,29 @@ export function getCompositionRoot(instance: Instance) {
                 egaspProgramRepository,
                 metadataRepository
             ),
+            deletePrimaryFile: new DeletePrimaryFileDataUseCase({
+                risDataRepository,
+                metadataRepository,
+                dataValuesRepository,
+                dhis2EventsDefaultRepository,
+                excelRepository,
+                glassDocumentsRepository,
+                instanceRepository,
+                glassUploadsRepository,
+                trackerRepository,
+                amcSubstanceDataRepository,
+            }),
+            deleteSecondaryFile: new DeleteSecondaryFileDataUseCase({
+                sampleDataRepository,
+                dataValuesRepository,
+                dhis2EventsDefaultRepository,
+                excelRepository,
+                glassDocumentsRepository,
+                metadataRepository,
+                instanceRepository,
+                glassUploadsRepository,
+                trackerRepository,
+            }),
         }),
         questionnaires: getExecute({
             get: new GetQuestionnaireUseCase(questionnaireD2Repository),

src/data/data-store/DataStoreKeys.ts

@@ -8,4 +8,5 @@ export const DataStoreKeys = {
     SIGNALS: "signals",
     ATC_CLASSIFICATION: "ATCs",
     AMC_RECALCULATION: "amc-recalculation",
+    ASYNC_DELETIONS: "async-deletions",
 };

src/data/repositories/ExcelPopulateDefaultRepository.ts

@@ -1,4 +1,4 @@
-import i18n from "@eyeseetea/d2-ui-components/locales";
+import _ from "lodash";
 import { Sheet } from "../../domain/entities/Sheet";
 import { ExcelRepository, ExcelValue, ReadCellOptions } from "../../domain/repositories/ExcelRepository";
 import XLSX, {
@@ -19,6 +19,7 @@ import {
     AMC_SUBSTANCE_CALCULATED_CONSUMPTION_PROGRAM_ID,
 } from "../../domain/usecases/data-entry/amc/ImportAMCSubstanceLevelData";
 import { removeCharacters } from "./utils/string";
+import i18n from "../../locales";
 
 type RowWithCells = XLSX.Row & { _cells: XLSX.Cell[] };
 
@@ -47,6 +48,15 @@ export class ExcelPopulateDefaultRepository extends ExcelRepository {
         });
     }
 
+    public loadTemplateFromArrayBuffer(buffer: ArrayBuffer, programId: Id): FutureData<string> {
+        const templateId = getTemplateId(programId);
+        return Future.fromPromise(this.parseFromArrayBuffer(buffer)).map(workbook => {
+            const id = templateId;
+            this.workbooks[id] = workbook;
+            return id;
+        });
+    }
+
     public async toBlob(id: string): Promise<Blob> {
         const data = await this.toBuffer(id);
         return new Blob([data], {
@@ -62,6 +72,11 @@ export class ExcelPopulateDefaultRepository extends ExcelRepository {
     private async parseFile(file: Blob): Promise<ExcelWorkbook> {
         return XLSX.fromDataAsync(file);
     }
+
+    private async parseFromArrayBuffer(buffer: ArrayBuffer): Promise<ExcelWorkbook> {
+        return XLSX.fromDataAsync(buffer);
+    }
+
     public async findRelativeCell(id: string, location?: SheetRef, cellRef?: CellRef): Promise<CellRef | undefined> {
         const workbook = await this.getWorkbook(id);
 

src/data/repositories/GlassUploadsDefaultRepository.ts

@@ -15,6 +15,17 @@ export class GlassUploadsDefaultRepository implements GlassUploadsRepository {
         return this.dataStoreClient.listCollection<GlassUploads>(DataStoreKeys.UPLOADS);
     }
 
+    getById(uploadId: Id): FutureData<GlassUploads> {
+        return this.dataStoreClient.listCollection<GlassUploads>(DataStoreKeys.UPLOADS).flatMap(uploads => {
+            const upload = uploads?.find(upload => upload.id === uploadId);
+            if (upload) {
+                return Future.success(upload);
+            } else {
+                return Future.error("Upload does not exist");
+            }
+        });
+    }
+
     save(upload: GlassUploads): FutureData<void> {
         return this.dataStoreClient.listCollection(DataStoreKeys.UPLOADS).flatMap(uploads => {
             const newUploads = [...uploads, upload];
@@ -46,6 +57,36 @@ export class GlassUploadsDefaultRepository implements GlassUploadsRepository {
         });
     }
 
+    setEventListDataDeleted(uploadId: string): FutureData<void> {
+        return this.dataStoreClient.listCollection<GlassUploads>(DataStoreKeys.UPLOADS).flatMap(uploads => {
+            const upload = uploads.find(el => el.id === uploadId);
+            if (upload) {
+                const restUploads = uploads.filter(upload => upload.id !== uploadId);
+                return this.dataStoreClient.saveObject(DataStoreKeys.UPLOADS, [
+                    ...restUploads,
+                    { ...upload, eventListDataDeleted: true },
+                ]);
+            } else {
+                return Future.error("Upload not found");
+            }
+        });
+    }
+
+    setCalculatedEventListDataDeleted(uploadId: string): FutureData<void> {
+        return this.dataStoreClient.listCollection<GlassUploads>(DataStoreKeys.UPLOADS).flatMap(uploads => {
+            const upload = uploads.find(el => el.id === uploadId);
+            if (upload) {
+                const restUploads = uploads.filter(upload => upload.id !== uploadId);
+                return this.dataStoreClient.saveObject(DataStoreKeys.UPLOADS, [
+                    ...restUploads,
+                    { ...upload, calculatedEventListDataDeleted: true },
+                ]);
+            } else {
+                return Future.error("Upload not found");
+            }
+        });
+    }
+
     delete(id: string): FutureData<{
         fileId: string;
         eventListFileId: string | undefined;
@@ -180,4 +221,28 @@ export class GlassUploadsDefaultRepository implements GlassUploadsRepository {
             }
         });
     }
+
+    setAsyncDeletions(uploadIdsToDelete: Id[]): FutureData<Id[]> {
+        return this.dataStoreClient.listCollection<Id>(DataStoreKeys.ASYNC_DELETIONS).flatMap(asyncDeletionsArray => {
+            const newAsyncDeletions = [...asyncDeletionsArray, ...uploadIdsToDelete];
+            return this.dataStoreClient.saveObject(DataStoreKeys.ASYNC_DELETIONS, newAsyncDeletions).flatMap(() => {
+                return Future.success(uploadIdsToDelete);
+            });
+        });
+    }
+
+    getAsyncDeletions(): FutureData<Id[]> {
+        return this.dataStoreClient.listCollection<Id>(DataStoreKeys.ASYNC_DELETIONS);
+    }
+
+    removeAsyncDeletions(uploadIdToRemove: Id[]): FutureData<Id[]> {
+        return this.dataStoreClient.listCollection<Id>(DataStoreKeys.ASYNC_DELETIONS).flatMap(asyncDeletionsArray => {
+            const restAsyncDeletions = asyncDeletionsArray.filter(
+                uploadIdToBeDeleted => !uploadIdToRemove.includes(uploadIdToBeDeleted)
+            );
+            return this.dataStoreClient.saveObject(DataStoreKeys.ASYNC_DELETIONS, restAsyncDeletions).flatMap(() => {
+                return Future.success(uploadIdToRemove);
+            });
+        });
+    }
 }

src/data/repositories/QuestionnaireD2DefaultRepository.ts

@@ -179,8 +179,8 @@ export class QuestionnaireD2DefaultRepository implements QuestionnaireRepository
         }
     }
 
-    saveResponse(questionnaire: QuestionnaireSelector, question: Question): FutureData<void> {
-        const dataValues = this.getDataValuesForQuestion(questionnaire, question);
+    saveResponse(questionnaire: QuestionnaireSelector, questions: Question[]): FutureData<void> {
+        const dataValues = this.getDataValuesForQuestions(questionnaire, questions);
         return this.postDataValues(dataValues);
     }
 
@@ -288,6 +288,10 @@ export class QuestionnaireD2DefaultRepository implements QuestionnaireRepository
         }
     }
 
+    private getDataValuesForQuestions(questionnaire: QuestionnaireSelector, questions: Question[]): D2DataValue[] {
+        return questions.flatMap(question => this.getDataValuesForQuestion(questionnaire, question));
+    }
+
     private getDataValuesForQuestion(questionnaire: QuestionnaireSelector, question: Question): D2DataValue[] {
         const { type } = question;
         const { orgUnitId, year } = questionnaire;