From 476735772eedd93ccbeee8555ffb0a8d5e23a6b0 Mon Sep 17 00:00:00 2001 From: Eduardo Peredo Rivero Date: Sat, 8 Jun 2024 11:59:48 -0500 Subject: [PATCH 1/3] exclude default categories from dataElements --- .../metadata/usecases/MetadataSyncUseCase.ts | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/domain/metadata/usecases/MetadataSyncUseCase.ts b/src/domain/metadata/usecases/MetadataSyncUseCase.ts index 2c08376ad..f7c219132 100644 --- a/src/domain/metadata/usecases/MetadataSyncUseCase.ts +++ b/src/domain/metadata/usecases/MetadataSyncUseCase.ts @@ -5,11 +5,19 @@ import { ExportBuilder } from "../../../types/synchronization"; import { promiseMap } from "../../../utils/common"; import { debug } from "../../../utils/debug"; import { Ref } from "../../common/entities/Ref"; +import { Id } from "../../common/entities/Schemas"; import { Instance } from "../../instance/entities/Instance"; import { MappingMapper } from "../../mapping/helpers/MappingMapper"; import { SynchronizationResult } from "../../reports/entities/SynchronizationResult"; import { GenericSyncUseCase } from "../../synchronization/usecases/GenericSyncUseCase"; -import { Document, MetadataEntities, MetadataPackage, Program } from "../entities/MetadataEntities"; +import { + DataElement, + DataSet, + Document, + MetadataEntities, + MetadataPackage, + Program, +} from "../entities/MetadataEntities"; import { NestedRules } from "../entities/MetadataExcludeIncludeRules"; import { buildNestedRules, cleanObject, cleanReferences, getAllReferences } from "../utils"; @@ -41,6 +49,7 @@ export class MetadataSyncUseCase extends GenericSyncUseCase { const metadataRepository = await this.getMetadataRepository(); const syncMetadata = await metadataRepository.getMetadataByIds(ids); const elements = syncMetadata[collectionName] || []; + const defaultIds = await metadataRepository.getDefaultIds(); for (const element of elements) { //ProgramRules is not included in programs items in the response by the dhis2 API @@ -48,11 +57,19 @@ export class MetadataSyncUseCase extends GenericSyncUseCase { const fixedElement = type === "programs" ? await this.requestAndIncludeProgramRules(element as Program) : element; + // In DHIS 2.38.5 and above the defaults parameter is not working + // /api/metadata.json?fields=id&filter=id:eq:data_element_id&categoryCombo&defaults=INCLUDE/EXCLUDE; this is not working + // so manually removing the default categoryCombo + const elementWithoutDefaults = + type === "dataElements" || type === "dataSets" + ? this.excludeDefaultsFromElement(fixedElement as DataElement, defaultIds) + : fixedElement; + // Store metadata object in result const object = cleanObject( this.api, schema.name, - fixedElement, + elementWithoutDefaults, excludeRules, includeSharingSettings, removeOrgUnitReferences, @@ -87,6 +104,12 @@ export class MetadataSyncUseCase extends GenericSyncUseCase { return recursiveExport(originalBuilder); } + private excludeDefaultsFromElement(fixedElement: DataElement | DataSet, defaultIds: Id[]): Partial { + return fixedElement.categoryCombo && defaultIds.includes(fixedElement.categoryCombo.id) + ? _(fixedElement).omit("categoryCombo").value() + : fixedElement; + } + public buildPayload = memoize(async () => { const { metadataIds, syncParams, filterRules = [] } = this.builder; const { From 962fad0d7b43ecb0722644cf6526fa6abebc31d7 Mon Sep 17 00:00:00 2001 From: Eduardo Peredo Rivero Date: Mon, 10 Jun 2024 09:35:58 -0500 Subject: [PATCH 2/3] move logic into data layer --- i18n/en.pot | 4 +- src/data/metadata/MetadataD2ApiRepository.ts | 27 ++++------- .../metadata/usecases/MetadataSyncUseCase.ts | 27 +---------- src/utils/d2-utils.ts | 47 +++++++++++++++++++ src/utils/synchronization.ts | 5 +- 5 files changed, 63 insertions(+), 47 deletions(-) diff --git a/i18n/en.pot b/i18n/en.pot index 5102f45ca..1fcaf189b 100644 --- a/i18n/en.pot +++ b/i18n/en.pot @@ -5,8 +5,8 @@ msgstr "" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n != 1)\n" -"POT-Creation-Date: 2024-05-08T09:54:22.766Z\n" -"PO-Revision-Date: 2024-05-08T09:54:22.767Z\n" +"POT-Creation-Date: 2024-06-10T12:10:57.544Z\n" +"PO-Revision-Date: 2024-06-10T12:10:57.544Z\n" msgid "" "THIS NEW RELEASE INCLUDES SHARING SETTINGS PER INSTANCES. FOR THIS VERSION " diff --git a/src/data/metadata/MetadataD2ApiRepository.ts b/src/data/metadata/MetadataD2ApiRepository.ts index 2614560ee..877113b3c 100644 --- a/src/data/metadata/MetadataD2ApiRepository.ts +++ b/src/data/metadata/MetadataD2ApiRepository.ts @@ -33,7 +33,7 @@ import { D2Api, D2Model, Id, MetadataResponse, Model, Stats } from "../../types/ import { Dictionary, isNotEmpty, Maybe } from "../../types/utils"; import { cache } from "../../utils/cache"; import { promiseMap } from "../../utils/common"; -import { getD2APiFromInstance } from "../../utils/d2-utils"; +import { D2MetadataUtils, getD2APiFromInstance } from "../../utils/d2-utils"; import { debug } from "../../utils/debug"; import { paginate } from "../../utils/pagination"; import { metadataTransformations } from "../transformations/PackageTransformations"; @@ -158,23 +158,8 @@ export class MetadataD2ApiRepository implements MetadataRepository { @cache() public async getDefaultIds(filter?: string): Promise { - const response = (await this.api - .get("/metadata", { - filter: "identifiable:eq:default", - fields: "id", - }) - .getData()) as { - [key: string]: { id: string }[]; - }; - - const metadata = _.pickBy(response, (_value, type) => !filter || type === filter); - - return _(metadata) - .omit(["system"]) - .values() - .flatten() - .map(({ id }) => id) - .value(); + const metadata = D2MetadataUtils.getDefaultIds(this.api, filter); + return metadata; } @cache() @@ -557,7 +542,11 @@ export class MetadataD2ApiRepository implements MetadataRepository { if (results.system) delete results.system; const metadata = await this.validateEventVisualizationsByIds(results); - return metadata; + const defaultIds = await this.getDefaultIds(); + const metadataExcludeDefaults = includeDefaults + ? metadata + : await D2MetadataUtils.excludeDefaults(metadata, defaultIds); + return metadataExcludeDefaults; } private async validateEventVisualizationsByIds(metadata: any) { diff --git a/src/domain/metadata/usecases/MetadataSyncUseCase.ts b/src/domain/metadata/usecases/MetadataSyncUseCase.ts index f7c219132..2c08376ad 100644 --- a/src/domain/metadata/usecases/MetadataSyncUseCase.ts +++ b/src/domain/metadata/usecases/MetadataSyncUseCase.ts @@ -5,19 +5,11 @@ import { ExportBuilder } from "../../../types/synchronization"; import { promiseMap } from "../../../utils/common"; import { debug } from "../../../utils/debug"; import { Ref } from "../../common/entities/Ref"; -import { Id } from "../../common/entities/Schemas"; import { Instance } from "../../instance/entities/Instance"; import { MappingMapper } from "../../mapping/helpers/MappingMapper"; import { SynchronizationResult } from "../../reports/entities/SynchronizationResult"; import { GenericSyncUseCase } from "../../synchronization/usecases/GenericSyncUseCase"; -import { - DataElement, - DataSet, - Document, - MetadataEntities, - MetadataPackage, - Program, -} from "../entities/MetadataEntities"; +import { Document, MetadataEntities, MetadataPackage, Program } from "../entities/MetadataEntities"; import { NestedRules } from "../entities/MetadataExcludeIncludeRules"; import { buildNestedRules, cleanObject, cleanReferences, getAllReferences } from "../utils"; @@ -49,7 +41,6 @@ export class MetadataSyncUseCase extends GenericSyncUseCase { const metadataRepository = await this.getMetadataRepository(); const syncMetadata = await metadataRepository.getMetadataByIds(ids); const elements = syncMetadata[collectionName] || []; - const defaultIds = await metadataRepository.getDefaultIds(); for (const element of elements) { //ProgramRules is not included in programs items in the response by the dhis2 API @@ -57,19 +48,11 @@ export class MetadataSyncUseCase extends GenericSyncUseCase { const fixedElement = type === "programs" ? await this.requestAndIncludeProgramRules(element as Program) : element; - // In DHIS 2.38.5 and above the defaults parameter is not working - // /api/metadata.json?fields=id&filter=id:eq:data_element_id&categoryCombo&defaults=INCLUDE/EXCLUDE; this is not working - // so manually removing the default categoryCombo - const elementWithoutDefaults = - type === "dataElements" || type === "dataSets" - ? this.excludeDefaultsFromElement(fixedElement as DataElement, defaultIds) - : fixedElement; - // Store metadata object in result const object = cleanObject( this.api, schema.name, - elementWithoutDefaults, + fixedElement, excludeRules, includeSharingSettings, removeOrgUnitReferences, @@ -104,12 +87,6 @@ export class MetadataSyncUseCase extends GenericSyncUseCase { return recursiveExport(originalBuilder); } - private excludeDefaultsFromElement(fixedElement: DataElement | DataSet, defaultIds: Id[]): Partial { - return fixedElement.categoryCombo && defaultIds.includes(fixedElement.categoryCombo.id) - ? _(fixedElement).omit("categoryCombo").value() - : fixedElement; - } - public buildPayload = memoize(async () => { const { metadataIds, syncParams, filterRules = [] } = this.builder; const { diff --git a/src/utils/d2-utils.ts b/src/utils/d2-utils.ts index 47bfeab31..f32b7a965 100644 --- a/src/utils/d2-utils.ts +++ b/src/utils/d2-utils.ts @@ -1,6 +1,9 @@ import _ from "lodash"; import { D2Api } from "../types/d2-api"; import { Instance } from "../domain/instance/entities/Instance"; +import { Id } from "../domain/common/entities/Schemas"; +import { DataElement, DataSet, Program } from "../domain/metadata/entities/MetadataEntities"; +import { cache } from "./cache"; export function getMajorVersion(version: string): number { const apiVersion = _.get(version.split("."), 1); @@ -22,3 +25,47 @@ export function getD2APiFromInstance(instance: Instance) { */ return new D2Api({ baseUrl: instance.url, auth: instance.auth, backend: "fetch" }); } + +export class D2MetadataUtils { + static async excludeDefaults(metadata: GenericMetadata, defaultIds: Id[]) { + // In DHIS 2.38.5 and above the defaults parameter is not working + // /api/metadata.json?fields=id,categoryCombo&filter=id:eq:data_element_id&defaults=INCLUDE/EXCLUDE; this is not working + // so manually removing the default categoryCombo + const result = _(metadata) + .mapValues(elements => { + return elements.map(element => this.excludeDefaultsFromModels(element, defaultIds)); + }) + .value(); + return result; + } + + @cache() + static async getDefaultIds(api: D2Api, filter?: string): Promise { + const response = (await api + .get("/metadata", { + filter: "identifiable:eq:default", + fields: "id", + }) + .getData()) as { + [key: string]: { id: string }[]; + }; + + const metadata = _.pickBy(response, (_value, type) => !filter || type === filter); + + return _(metadata) + .omit(["system"]) + .values() + .flatten() + .map(({ id }) => id) + .value(); + } + + private static excludeDefaultsFromModels(fixedElement: MetadataDefaultCategoryCombo, defaultIds: Id[]) { + return fixedElement.categoryCombo && defaultIds.includes(fixedElement.categoryCombo.id) + ? _(fixedElement).omit("categoryCombo").value() + : fixedElement; + } +} + +type GenericMetadata = { dataElements: DataElement[]; dataSets: DataSet[]; programs: Program[] }; +type MetadataDefaultCategoryCombo = DataElement | DataSet | Program; diff --git a/src/utils/synchronization.ts b/src/utils/synchronization.ts index fe9d4122a..ab8279160 100644 --- a/src/utils/synchronization.ts +++ b/src/utils/synchronization.ts @@ -9,6 +9,7 @@ import { MAPPED_BY_VALUE_KEY } from "../presentation/react/core/components/mappi import { D2Api } from "../types/d2-api"; import { buildObject } from "../types/utils"; import "../utils/lodash-mixins"; +import { D2MetadataUtils } from "./d2-utils"; //TODO: when all request to metadata using metadataRepository.getMetadataByIds // this function should be removed @@ -29,7 +30,9 @@ export async function getMetadata(api: D2Api, elements: string[], fields = ":all const response = await Promise.all(promises); const results = _.deepMerge({}, ...response); if (results.system) delete results.system; - return results; + const defaultIds = await D2MetadataUtils.getDefaultIds(api); + const metadataExcludeDefaults = await D2MetadataUtils.excludeDefaults(results, defaultIds); + return metadataExcludeDefaults; } export const availablePeriods = buildObject<{ From 37b0dab173e4bcc1b841ed2ddcabefde4f6aa0fa Mon Sep 17 00:00:00 2001 From: Eduardo Peredo Rivero Date: Tue, 11 Jun 2024 07:23:49 -0500 Subject: [PATCH 3/3] move class into data layer --- src/data/metadata/D2MetadataUtils.ts | 49 ++++++++++++++++++++ src/data/metadata/MetadataD2ApiRepository.ts | 3 +- src/utils/d2-utils.ts | 47 ------------------- src/utils/synchronization.ts | 2 +- 4 files changed, 52 insertions(+), 49 deletions(-) create mode 100644 src/data/metadata/D2MetadataUtils.ts diff --git a/src/data/metadata/D2MetadataUtils.ts b/src/data/metadata/D2MetadataUtils.ts new file mode 100644 index 000000000..59213c380 --- /dev/null +++ b/src/data/metadata/D2MetadataUtils.ts @@ -0,0 +1,49 @@ +import _ from "lodash"; +import { Id } from "../../domain/common/entities/Schemas"; +import { DataElement, DataSet, Program } from "../../domain/metadata/entities/MetadataEntities"; +import { D2Api } from "../../types/d2-api"; +import { cache } from "../../utils/cache"; + +export class D2MetadataUtils { + static async excludeDefaults(metadata: GenericMetadata, defaultIds: Id[]) { + // In DHIS 2.38.5 and above the defaults parameter is not working + // /api/metadata.json?fields=id,categoryCombo&filter=id:eq:data_element_id&defaults=INCLUDE/EXCLUDE; this is not working + // so manually removing the default categoryCombo + const result = _(metadata) + .mapValues(elements => { + return elements.map(element => this.excludeDefaultsFromModels(element, defaultIds)); + }) + .value(); + return result; + } + + @cache() + static async getDefaultIds(api: D2Api, filter?: string): Promise { + const response = (await api + .get("/metadata", { + filter: "identifiable:eq:default", + fields: "id", + }) + .getData()) as { + [key: string]: { id: string }[]; + }; + + const metadata = _.pickBy(response, (_value, type) => !filter || type === filter); + + return _(metadata) + .omit(["system"]) + .values() + .flatten() + .map(({ id }) => id) + .value(); + } + + private static excludeDefaultsFromModels(fixedElement: MetadataDefaultCategoryCombo, defaultIds: Id[]) { + return fixedElement.categoryCombo && defaultIds.includes(fixedElement.categoryCombo.id) + ? _(fixedElement).omit("categoryCombo").value() + : fixedElement; + } +} + +type GenericMetadata = { dataElements: DataElement[]; dataSets: DataSet[]; programs: Program[] }; +type MetadataDefaultCategoryCombo = DataElement | DataSet | Program; diff --git a/src/data/metadata/MetadataD2ApiRepository.ts b/src/data/metadata/MetadataD2ApiRepository.ts index 877113b3c..801e449f7 100644 --- a/src/data/metadata/MetadataD2ApiRepository.ts +++ b/src/data/metadata/MetadataD2ApiRepository.ts @@ -33,10 +33,11 @@ import { D2Api, D2Model, Id, MetadataResponse, Model, Stats } from "../../types/ import { Dictionary, isNotEmpty, Maybe } from "../../types/utils"; import { cache } from "../../utils/cache"; import { promiseMap } from "../../utils/common"; -import { D2MetadataUtils, getD2APiFromInstance } from "../../utils/d2-utils"; +import { getD2APiFromInstance } from "../../utils/d2-utils"; import { debug } from "../../utils/debug"; import { paginate } from "../../utils/pagination"; import { metadataTransformations } from "../transformations/PackageTransformations"; +import { D2MetadataUtils } from "./D2MetadataUtils"; export class MetadataD2ApiRepository implements MetadataRepository { private api: D2Api; diff --git a/src/utils/d2-utils.ts b/src/utils/d2-utils.ts index f32b7a965..47bfeab31 100644 --- a/src/utils/d2-utils.ts +++ b/src/utils/d2-utils.ts @@ -1,9 +1,6 @@ import _ from "lodash"; import { D2Api } from "../types/d2-api"; import { Instance } from "../domain/instance/entities/Instance"; -import { Id } from "../domain/common/entities/Schemas"; -import { DataElement, DataSet, Program } from "../domain/metadata/entities/MetadataEntities"; -import { cache } from "./cache"; export function getMajorVersion(version: string): number { const apiVersion = _.get(version.split("."), 1); @@ -25,47 +22,3 @@ export function getD2APiFromInstance(instance: Instance) { */ return new D2Api({ baseUrl: instance.url, auth: instance.auth, backend: "fetch" }); } - -export class D2MetadataUtils { - static async excludeDefaults(metadata: GenericMetadata, defaultIds: Id[]) { - // In DHIS 2.38.5 and above the defaults parameter is not working - // /api/metadata.json?fields=id,categoryCombo&filter=id:eq:data_element_id&defaults=INCLUDE/EXCLUDE; this is not working - // so manually removing the default categoryCombo - const result = _(metadata) - .mapValues(elements => { - return elements.map(element => this.excludeDefaultsFromModels(element, defaultIds)); - }) - .value(); - return result; - } - - @cache() - static async getDefaultIds(api: D2Api, filter?: string): Promise { - const response = (await api - .get("/metadata", { - filter: "identifiable:eq:default", - fields: "id", - }) - .getData()) as { - [key: string]: { id: string }[]; - }; - - const metadata = _.pickBy(response, (_value, type) => !filter || type === filter); - - return _(metadata) - .omit(["system"]) - .values() - .flatten() - .map(({ id }) => id) - .value(); - } - - private static excludeDefaultsFromModels(fixedElement: MetadataDefaultCategoryCombo, defaultIds: Id[]) { - return fixedElement.categoryCombo && defaultIds.includes(fixedElement.categoryCombo.id) - ? _(fixedElement).omit("categoryCombo").value() - : fixedElement; - } -} - -type GenericMetadata = { dataElements: DataElement[]; dataSets: DataSet[]; programs: Program[] }; -type MetadataDefaultCategoryCombo = DataElement | DataSet | Program; diff --git a/src/utils/synchronization.ts b/src/utils/synchronization.ts index ab8279160..34effe51a 100644 --- a/src/utils/synchronization.ts +++ b/src/utils/synchronization.ts @@ -1,6 +1,7 @@ import FileSaver from "file-saver"; import _ from "lodash"; import moment, { unitOfTime } from "moment"; +import { D2MetadataUtils } from "../data/metadata/D2MetadataUtils"; import { MetadataMapping, MetadataMappingDictionary } from "../domain/mapping/entities/MetadataMapping"; import { CategoryOptionCombo } from "../domain/metadata/entities/MetadataEntities"; import { SynchronizationRule } from "../domain/rules/entities/SynchronizationRule"; @@ -9,7 +10,6 @@ import { MAPPED_BY_VALUE_KEY } from "../presentation/react/core/components/mappi import { D2Api } from "../types/d2-api"; import { buildObject } from "../types/utils"; import "../utils/lodash-mixins"; -import { D2MetadataUtils } from "./d2-utils"; //TODO: when all request to metadata using metadataRepository.getMetadataByIds // this function should be removed