From 9b59ced78fe287ae15081de4188d9b487bf47ce0 Mon Sep 17 00:00:00 2001 From: Eduardo Peredo Rivero Date: Wed, 31 Jul 2024 19:47:57 -0500 Subject: [PATCH] add dataStore support --- src/data/common/D2ApiDataStore.ts | 65 +++++++++++ .../DataStoreMetadataD2Repository.ts | 109 ++++++++++++++++++ src/data/metadata/MetadataD2ApiRepository.ts | 26 +++-- src/data/storage/StorageDataStoreClient.ts | 4 +- src/domain/common/entities/Struct.ts | 45 ++++++++ .../common/factories/RepositoryFactory.ts | 7 ++ src/domain/data-store/DataStoreMetadata.ts | 98 ++++++++++++++++ .../data-store/DataStoreMetadataRepository.ts | 15 +++ .../metadata/entities/MetadataEntities.ts | 15 +++ .../metadata/usecases/MetadataSyncUseCase.ts | 62 +++++++++- .../packages/entities/MetadataPackageDiff.ts | 2 +- src/domain/reports/entities/Stats.ts | 27 +++++ .../usecases/GenericSyncUseCase.ts | 6 + src/models/dhis/factory.ts | 1 + src/models/dhis/metadata.ts | 19 +++ src/presentation/CompositionRoot.ts | 2 + .../common/MetadataSelectionStep.tsx | 2 +- .../sync-wizard/common/SummaryStep.tsx | 32 +++++ 18 files changed, 523 insertions(+), 14 deletions(-) create mode 100644 src/data/common/D2ApiDataStore.ts create mode 100644 src/data/data-store/DataStoreMetadataD2Repository.ts create mode 100644 src/domain/common/entities/Struct.ts create mode 100644 src/domain/data-store/DataStoreMetadata.ts create mode 100644 src/domain/data-store/DataStoreMetadataRepository.ts create mode 100644 src/domain/reports/entities/Stats.ts diff --git a/src/data/common/D2ApiDataStore.ts b/src/data/common/D2ApiDataStore.ts new file mode 100644 index 000000000..2515deff5 --- /dev/null +++ b/src/data/common/D2ApiDataStore.ts @@ -0,0 +1,65 @@ +import { D2Api } from "../../types/d2-api"; +import { DataSource, isDhisInstance } from "../../domain/instance/entities/DataSource"; +import { getD2APiFromInstance } from "../../utils/d2-utils"; +import { DataStore, DataStoreKey } from "../../domain/metadata/entities/MetadataEntities"; +import { promiseMap } from "../../utils/common"; +import { DataStoreMetadata } from "../../domain/data-store/DataStoreMetadata"; + +export class D2ApiDataStore { + private api: D2Api; + + constructor(instance: DataSource) { + if (!isDhisInstance(instance)) { + throw new Error("Invalid instance type for MetadataD2ApiRepository"); + } + this.api = getD2APiFromInstance(instance); + } + + async getDataStore(): Promise { + const response = await this.api.request({ method: "get", url: "/dataStore" }).getData(); + const namespacesWithKeys = await this.getAllKeysFromNamespaces(response); + return namespacesWithKeys; + } + + private async getAllKeysFromNamespaces(namespaces: string[]): Promise { + const result = await promiseMap(namespaces, async namespace => { + const keys = await this.getKeysPaginated([], namespace); + return { + code: namespace, + displayName: namespace, + externalAccess: false, + favorites: [], + id: `${namespace}${DataStoreMetadata.NS_SEPARATOR}`, + keys: keys, + name: namespace, + translations: [], + }; + }); + return result; + } + + private async getKeysPaginated(keysState: DataStoreKey[], namespace: string): Promise { + const keyResponse = await this.getKeysByNameSpace(namespace); + const newKeys = [...keysState, ...keyResponse]; + return newKeys; + } + + private async getKeysByNameSpace(namespace: string): Promise { + const response = await this.api + .request({ + method: "get", + url: `/dataStore/${namespace}`, + // Since v38 we can use the fields parameter to get keys and values in the same request + // Empty fields returns a paginated response + // https://docs.dhis2.org/en/full/develop/dhis-core-version-240/developer-manual.html#query-api + // params: { fields: "", page: page, pageSize: 200 }, + }) + .getData(); + + return this.buildArrayDataStoreKey(response, namespace); + } + + private buildArrayDataStoreKey(keys: string[], namespace: string): DataStoreKey[] { + return keys.map(key => ({ id: `${namespace}${DataStoreMetadata.NS_SEPARATOR}${key}`, displayName: key })); + } +} diff --git a/src/data/data-store/DataStoreMetadataD2Repository.ts b/src/data/data-store/DataStoreMetadataD2Repository.ts new file mode 100644 index 000000000..fc5969ee2 --- /dev/null +++ b/src/data/data-store/DataStoreMetadataD2Repository.ts @@ -0,0 +1,109 @@ +import _ from "lodash"; +import { DataStoreMetadata } from "../../domain/data-store/DataStoreMetadata"; +import { DataStoreMetadataRepository, SaveOptions } from "../../domain/data-store/DataStoreMetadataRepository"; +import { Instance } from "../../domain/instance/entities/Instance"; +import { Stats } from "../../domain/reports/entities/Stats"; +import { SynchronizationResult } from "../../domain/reports/entities/SynchronizationResult"; +import { promiseMap } from "../../utils/common"; +import { StorageDataStoreClient } from "../storage/StorageDataStoreClient"; + +export class DataStoreMetadataD2Repository implements DataStoreMetadataRepository { + private instance: Instance; + + constructor(instance: Instance) { + this.instance = instance; + } + + async get(dataStores: DataStoreMetadata[]): Promise { + const result = await promiseMap(dataStores, async dataStore => { + const dataStoreClient = new StorageDataStoreClient(this.instance, dataStore.namespace); + const dataStoreWithValue = this.getValuesByDataStore(dataStoreClient, dataStore); + return dataStoreWithValue; + }); + return result; + } + + private async getValuesByDataStore( + dataStoreClient: StorageDataStoreClient, + dataStore: DataStoreMetadata + ): Promise { + const keys = await this.getAllKeys(dataStoreClient, dataStore); + const keyWithValue = await promiseMap(keys, async key => { + const keyValue = await dataStoreClient.getObject(key.id); + return { id: key.id, value: keyValue }; + }); + const keyInNamespace = _(dataStore.keys).first()?.id; + const sharing = keyInNamespace ? await dataStoreClient.getObjectSharing(keyInNamespace) : undefined; + return new DataStoreMetadata({ + namespace: dataStore.namespace, + keys: keyWithValue, + sharing, + }); + } + + private async getAllKeys( + dataStoreClient: StorageDataStoreClient, + dataStore: DataStoreMetadata + ): Promise { + if (dataStore.keys.length > 0) return dataStore.keys; + const keys = await dataStoreClient.listKeys(); + return keys.map(key => ({ id: key, value: "" })); + } + + async save( + dataStores: DataStoreMetadata[], + options: SaveOptions = { mergeMode: "MERGE" } + ): Promise { + const keysIdsToDelete = await this.getKeysToDelete(dataStores, options); + + const resultStats = await promiseMap(dataStores, async dataStore => { + const dataStoreClient = new StorageDataStoreClient(this.instance, dataStore.namespace); + const stats = await promiseMap(dataStore.keys, async key => { + const exist = await dataStoreClient.getObject(key.id); + await dataStoreClient.saveObject(key.id, key.value); + if (dataStore.sharing) { + await dataStoreClient.saveObjectSharing(key.id, dataStore.sharing); + } + return exist ? Stats.createOrEmpty({ updated: 1 }) : Stats.createOrEmpty({ imported: 1 }); + }); + return stats; + }); + + const deleteStats = await promiseMap(keysIdsToDelete, async keyId => { + const [namespace, key] = keyId.split(DataStoreMetadata.NS_SEPARATOR); + const dataStoreClient = new StorageDataStoreClient(this.instance, namespace); + await dataStoreClient.removeObject(key); + return Stats.createOrEmpty({ deleted: 1 }); + }); + + const allStats = resultStats.flatMap(result => result).concat(deleteStats); + const dataStoreStats = { ...Stats.combine(allStats.map(stat => Stats.create(stat))), type: "DataStore Keys" }; + + const result: SynchronizationResult = { + date: new Date(), + instance: this.instance, + status: "SUCCESS", + type: "metadata", + stats: dataStoreStats, + typeStats: [dataStoreStats], + }; + + return result; + } + + private async getKeysToDelete(dataStores: DataStoreMetadata[], options: SaveOptions) { + if (options.mergeMode === "MERGE") return []; + + const existingRecords = await this.get(dataStores.map(x => ({ ...x, keys: [] }))); + const existingKeysIds = existingRecords.flatMap(dataStore => { + return dataStore.keys.map(key => `${dataStore.namespace}[NS]${key.id}`); + }); + + const keysIdsToSave = dataStores.flatMap(dataStore => { + return dataStore.keys.map(key => `${dataStore.namespace}[NS]${key.id}`); + }); + + const keysIdsToDelete = existingKeysIds.filter(id => !keysIdsToSave.includes(id)); + return keysIdsToDelete; + } +} diff --git a/src/data/metadata/MetadataD2ApiRepository.ts b/src/data/metadata/MetadataD2ApiRepository.ts index 801e449f7..b84546f35 100644 --- a/src/data/metadata/MetadataD2ApiRepository.ts +++ b/src/data/metadata/MetadataD2ApiRepository.ts @@ -38,6 +38,7 @@ import { debug } from "../../utils/debug"; import { paginate } from "../../utils/pagination"; import { metadataTransformations } from "../transformations/PackageTransformations"; import { D2MetadataUtils } from "./D2MetadataUtils"; +import { D2ApiDataStore } from "../common/D2ApiDataStore"; export class MetadataD2ApiRepository implements MetadataRepository { private api: D2Api; @@ -101,17 +102,24 @@ export class MetadataD2ApiRepository implements MetadataRepository { const filter = this.buildListFilters(params); const { apiVersion } = this.instance; const options = { type, fields, filter, order, page, pageSize, rootJunction }; - const { objects: baseObjects, pager } = await this.getListPaginated(options); - // Prepend parent objects (if option enabled) as virtual rows, keep pagination unmodified. - const objects = _.concat(await this.getParentObjects(listParams), baseObjects); + if (type === "dataStores") { + const d2ApiDataStore = new D2ApiDataStore(this.instance); + const response = await d2ApiDataStore.getDataStore(); + // Hardcoded pagination since DHIS2 does not support pagination for namespaces + return { objects: response, pager: { page: 1, total: response.length, pageSize: 100 } }; + } else { + const { objects: baseObjects, pager } = await this.getListPaginated(options); + // Prepend parent objects (if option enabled) as virtual rows, keep pagination unmodified. + const objects = _.concat(await this.getParentObjects(listParams), baseObjects); - const metadataPackage = this.transformationRepository.mapPackageFrom( - apiVersion, - { [type]: objects }, - metadataTransformations - ); + const metadataPackage = this.transformationRepository.mapPackageFrom( + apiVersion, + { [type]: objects }, + metadataTransformations + ); - return { objects: metadataPackage[type as keyof MetadataEntities] ?? [], pager }; + return { objects: metadataPackage[type as keyof MetadataEntities] ?? [], pager }; + } } @cache() diff --git a/src/data/storage/StorageDataStoreClient.ts b/src/data/storage/StorageDataStoreClient.ts index 7d97e77ba..92b8df1d7 100644 --- a/src/data/storage/StorageDataStoreClient.ts +++ b/src/data/storage/StorageDataStoreClient.ts @@ -14,10 +14,10 @@ export class StorageDataStoreClient extends StorageClient { private api: D2Api; private dataStore: DataStore; - constructor(instance: Instance) { + constructor(instance: Instance, namespace: string = dataStoreNamespace) { super(); this.api = getD2APiFromInstance(instance); - this.dataStore = this.api.dataStore(dataStoreNamespace); + this.dataStore = this.api.dataStore(namespace); } public async getObject(key: string): Promise { diff --git a/src/domain/common/entities/Struct.ts b/src/domain/common/entities/Struct.ts new file mode 100644 index 000000000..8b9a69275 --- /dev/null +++ b/src/domain/common/entities/Struct.ts @@ -0,0 +1,45 @@ +/** + * Base class for typical classes with attributes. Features: create, update. + * + * ``` + * class Counter extends Struct<{ id: Id; value: number }>() { + * add(value: number): Counter { + * return this._update({ value: this.value + value }); + * } + * } + * + * const counter1 = Counter.create({ id: "some-counter", value: 1 }); + * const counter2 = counter1._update({ value: 2 }); + * ``` + */ + +export function Struct() { + abstract class Base { + constructor(_attributes: Attrs) { + Object.assign(this, _attributes); + } + + _getAttributes(): Attrs { + const entries = Object.getOwnPropertyNames(this).map(key => [key, (this as any)[key]]); + return Object.fromEntries(entries) as Attrs; + } + + protected _update(partialAttrs: Partial): this { + const ParentClass = this.constructor as new (values: Attrs) => typeof this; + return new ParentClass({ ...this._getAttributes(), ...partialAttrs }); + } + + static create(this: new (attrs: Attrs) => U, attrs: Attrs): U { + return new this(attrs); + } + } + + return Base as { + new (values: Attrs): Attrs & Base; + create: typeof Base["create"]; + }; +} + +const GenericStruct = Struct(); + +export type GenericStructInstance = InstanceType; diff --git a/src/domain/common/factories/RepositoryFactory.ts b/src/domain/common/factories/RepositoryFactory.ts index d94179c01..b54916b61 100644 --- a/src/domain/common/factories/RepositoryFactory.ts +++ b/src/domain/common/factories/RepositoryFactory.ts @@ -5,6 +5,7 @@ import { } from "../../aggregated/repositories/AggregatedRepository"; import { ConfigRepositoryConstructor } from "../../config/repositories/ConfigRepository"; import { CustomDataRepositoryConstructor } from "../../custom-data/repository/CustomDataRepository"; +import { DataStoreMetadataRepositoryConstructor } from "../../data-store/DataStoreMetadataRepository"; import { EventsRepository, EventsRepositoryConstructor } from "../../events/repositories/EventsRepository"; import { FileRepositoryConstructor } from "../../file/repositories/FileRepository"; import { DataSource } from "../../instance/entities/DataSource"; @@ -120,6 +121,11 @@ export class RepositoryFactory { return this.get(Repositories.EventsRepository, [instance]); } + @cache() + public dataStoreMetadataRepository(instance: Instance) { + return this.get(Repositories.DataStoreMetadataRepository, [instance]); + } + @cache() public teisRepository(instance: Instance): TEIRepository { return this.get(Repositories.TEIsRepository, [instance]); @@ -209,4 +215,5 @@ export const Repositories = { MappingRepository: "mappingRepository", SettingsRepository: "settingsRepository", SchedulerRepository: "schedulerRepository", + DataStoreMetadataRepository: "dataStoreMetadataRepository", } as const; diff --git a/src/domain/data-store/DataStoreMetadata.ts b/src/domain/data-store/DataStoreMetadata.ts new file mode 100644 index 000000000..e1b62f389 --- /dev/null +++ b/src/domain/data-store/DataStoreMetadata.ts @@ -0,0 +1,98 @@ +import _ from "lodash"; +import { Maybe } from "../../types/utils"; +import { MetadataImportParams } from "../metadata/entities/MetadataSynchronizationParams"; +import { ObjectSharing } from "../storage/repositories/StorageClient"; + +export type DataStoreNamespace = string; +export type DataStoreKey = { id: string; value: any }; + +export type DataStoreAttrs = { namespace: DataStoreNamespace; keys: DataStoreKey[]; sharing: Maybe }; + +export class DataStoreMetadata { + public readonly namespace: DataStoreNamespace; + public readonly keys: DataStoreKey[]; + public readonly sharing: Maybe; + + static NS_SEPARATOR = "[NS]"; + + constructor(data: DataStoreAttrs) { + this.keys = data.keys; + this.namespace = data.namespace; + this.sharing = data.sharing; + } + + static buildFromKeys(keysWithNamespaces: string[]): DataStoreMetadata[] { + const dataStoreIds = keysWithNamespaces.filter(id => id.includes(DataStoreMetadata.NS_SEPARATOR)); + + const namespaceAndKey = dataStoreIds.map(dataStoreId => { + const match = dataStoreId.split(DataStoreMetadata.NS_SEPARATOR); + if (!match) { + throw new Error(`dataStore value does not match expected format: ${dataStoreId}`); + } + const [namespace, key] = match; + return { namespace, key }; + }); + + const groupByNamespace = _(namespaceAndKey) + .groupBy(record => record.namespace) + .value(); + + const result = _(groupByNamespace) + .map((keys, namespace) => { + return new DataStoreMetadata({ + namespace, + + keys: _(keys) + .map(key => { + if (!key.key) return undefined; + return { id: key.key, value: "" }; + }) + .compact() + .value(), + sharing: undefined, + }); + }) + .value(); + + return result; + } + + static combine( + origin: DataStoreMetadata[], + destination: DataStoreMetadata[], + action: MetadataImportParams["mergeMode"] = "MERGE" + ): DataStoreMetadata[] { + const destinationDataStore = _.keyBy(destination, "namespace"); + + return _(origin) + .map(originItem => { + const destItem = destinationDataStore[originItem.namespace]; + + if (!destItem) return originItem; + + const combinedKeys = + action === "MERGE" + ? _(originItem.keys) + .unionBy(destItem.keys, record => record.id) + .value() + : originItem.keys; + + return new DataStoreMetadata({ + namespace: originItem.namespace, + keys: combinedKeys, + sharing: originItem.sharing, + }); + }) + .value(); + } + + static removeSharingSettings(dataStores: DataStoreMetadata[]): DataStoreMetadata[] { + return dataStores.map(dataStore => { + return new DataStoreMetadata({ + namespace: dataStore.namespace, + keys: dataStore.keys, + sharing: undefined, + }); + }); + } +} diff --git a/src/domain/data-store/DataStoreMetadataRepository.ts b/src/domain/data-store/DataStoreMetadataRepository.ts new file mode 100644 index 000000000..236b4a09b --- /dev/null +++ b/src/domain/data-store/DataStoreMetadataRepository.ts @@ -0,0 +1,15 @@ +import { Instance } from "../instance/entities/Instance"; +import { MetadataImportParams } from "../metadata/entities/MetadataSynchronizationParams"; +import { SynchronizationResult } from "../reports/entities/SynchronizationResult"; +import { DataStoreMetadata } from "./DataStoreMetadata"; + +export interface DataStoreMetadataRepositoryConstructor { + new (instance: Instance): DataStoreMetadataRepository; +} + +export interface DataStoreMetadataRepository { + get(namespaces: DataStoreMetadata[]): Promise; + save(namespaces: DataStoreMetadata[], options: SaveOptions): Promise; +} + +export type SaveOptions = { mergeMode: MetadataImportParams["mergeMode"] }; diff --git a/src/domain/metadata/entities/MetadataEntities.ts b/src/domain/metadata/entities/MetadataEntities.ts index 26cd6cff0..d09da7c15 100644 --- a/src/domain/metadata/entities/MetadataEntities.ts +++ b/src/domain/metadata/entities/MetadataEntities.ts @@ -4992,6 +4992,19 @@ export type Visualization = { yearlySeries: string[]; }; +export type DataStore = { + externalAccess: boolean; + id: Id; + code: string; + name: string; + displayName: string; + favorites: string[]; + translations: Translation[]; + keys: DataStoreKey[]; +}; + +export type DataStoreKey = { id: Id; displayName: string }; + export type MetadataEntity = | UserAuthorityGroup | Attribute @@ -5025,6 +5038,7 @@ export type MetadataEntity = | Section | DataApprovalLevel | DataApprovalWorkflow + | DataStore | ValidationRule | ValidationRuleGroup | ValidationNotificationTemplate @@ -5076,6 +5090,7 @@ export type MetadataEntities = { dataEntryForms: DataEntryForm[]; dataSets: DataSet[]; dataSetNotificationTemplates: DataSetNotificationTemplate[]; + dataStores: DataStore[]; documents: Document[]; eventCharts: EventChart[]; eventReports: EventReport[]; diff --git a/src/domain/metadata/usecases/MetadataSyncUseCase.ts b/src/domain/metadata/usecases/MetadataSyncUseCase.ts index 0628d61aa..5b49c10db 100644 --- a/src/domain/metadata/usecases/MetadataSyncUseCase.ts +++ b/src/domain/metadata/usecases/MetadataSyncUseCase.ts @@ -2,15 +2,18 @@ import _ from "lodash"; import memoize from "nano-memoize"; import { defaultName, modelFactory } from "../../../models/dhis/factory"; import { ExportBuilder } from "../../../types/synchronization"; +import { Maybe } from "../../../types/utils"; import { promiseMap } from "../../../utils/common"; import { debug } from "../../../utils/debug"; import { Ref } from "../../common/entities/Ref"; +import { DataStoreMetadata } from "../../data-store/DataStoreMetadata"; import { Instance } from "../../instance/entities/Instance"; import { MappingMapper } from "../../mapping/helpers/MappingMapper"; import { SynchronizationResult } from "../../reports/entities/SynchronizationResult"; import { GenericSyncUseCase } from "../../synchronization/usecases/GenericSyncUseCase"; import { Document, MetadataEntities, MetadataPackage, Program } from "../entities/MetadataEntities"; import { NestedRules } from "../entities/MetadataExcludeIncludeRules"; +import { MetadataImportParams } from "../entities/MetadataSynchronizationParams"; import { buildNestedRules, cleanObject, cleanReferences, getAllReferences } from "../utils"; export class MetadataSyncUseCase extends GenericSyncUseCase { @@ -163,13 +166,70 @@ export class MetadataSyncUseCase extends GenericSyncUseCase { debug("Metadata package", { originalPayload, payload }); + const dataStorePayload = await this.buildDataStorePayload(instance); + const dataStoreResult = + dataStorePayload.length > 0 + ? await this.saveDataStorePayload(instance, dataStorePayload, syncParams?.mergeMode) + : undefined; + const remoteMetadataRepository = await this.getMetadataRepository(instance); - const syncResult = await remoteMetadataRepository.save(payload, syncParams); + const metadataResult = await remoteMetadataRepository.save(payload, syncParams); const origin = await this.getOriginInstance(); + const syncResult = this.generateSyncResults(metadataResult, dataStoreResult); return [{ ...syncResult, origin: origin.toPublicObject(), payload }]; } + private generateSyncResults( + metadataResult: SynchronizationResult, + dataStoreResult: Maybe + ): SynchronizationResult { + if (!dataStoreResult) return metadataResult; + + return { + ...metadataResult, + typeStats: _(metadataResult.typeStats) + .concat(dataStoreResult.typeStats || []) + .value(), + stats: metadataResult.stats + ? { + deleted: metadataResult.stats.deleted + (dataStoreResult.stats?.deleted || 0), + ignored: metadataResult.stats.ignored + (dataStoreResult.stats?.ignored || 0), + imported: metadataResult.stats.imported + (dataStoreResult.stats?.imported || 0), + updated: metadataResult.stats.updated + (dataStoreResult.stats?.updated || 0), + total: (metadataResult.stats?.total || 0) + (dataStoreResult.stats?.total || 0), + } + : undefined, + }; + } + + private async buildDataStorePayload(instance: Instance): Promise { + const { metadataIds, syncParams } = this.builder; + const dataStore = DataStoreMetadata.buildFromKeys(metadataIds); + if (dataStore.length === 0) return []; + + const dataStoreRepository = await this.getDataStoreMetadataRepository(); + const dataStoreRemoteRepository = await this.getDataStoreMetadataRepository(instance); + + const dataStoreLocal = await dataStoreRepository.get(dataStore); + const dataStoreRemote = await dataStoreRemoteRepository.get(dataStore); + + const dataStorePayload = DataStoreMetadata.combine(dataStoreLocal, dataStoreRemote, syncParams?.mergeMode); + return syncParams?.includeSharingSettings + ? dataStorePayload + : DataStoreMetadata.removeSharingSettings(dataStorePayload); + } + + private async saveDataStorePayload( + instance: Instance, + dataStores: DataStoreMetadata[], + mergeMode: MetadataImportParams["mergeMode"] + ): Promise { + const dataStoreRemoteRepository = await this.getDataStoreMetadataRepository(instance); + const result = await dataStoreRemoteRepository.save(dataStores, { mergeMode: mergeMode }); + return result; + } + public async buildDataStats() { return undefined; } diff --git a/src/domain/packages/entities/MetadataPackageDiff.ts b/src/domain/packages/entities/MetadataPackageDiff.ts index 241284483..9dc08e102 100644 --- a/src/domain/packages/entities/MetadataPackageDiff.ts +++ b/src/domain/packages/entities/MetadataPackageDiff.ts @@ -110,7 +110,7 @@ function getStringValue(value: AttributeValue): string { // On arrays, remove the ignored fields and apply a simple, best-effort sorting // so the same items in different order are considered equal. return stringify( - _(value as Obj[]) + _(value as unknown as Obj[]) .map(obj => _.omit(obj, ignoredFields)) .sortBy(obj => obj.id || stringify(obj)) .value() diff --git a/src/domain/reports/entities/Stats.ts b/src/domain/reports/entities/Stats.ts new file mode 100644 index 000000000..c7892e91f --- /dev/null +++ b/src/domain/reports/entities/Stats.ts @@ -0,0 +1,27 @@ +import { Struct } from "../../common/entities/Struct"; + +type StatsAttrs = { + imported: number; + ignored: number; + updated: number; + deleted: number; + total: number; +}; + +export class Stats extends Struct() { + static combine(stats: Stats[]): Stats { + return stats.reduce((acum, stat) => { + return Stats.create({ + imported: acum.imported + stat.imported, + ignored: acum.ignored + stat.ignored, + updated: acum.updated + stat.updated, + deleted: acum.deleted + stat.deleted, + total: acum.total + stat.imported + stat.deleted + stat.ignored + stat.updated, + }); + }, Stats.createOrEmpty()); + } + + static createOrEmpty(stats?: Partial): Stats { + return Stats.create({ imported: 0, ignored: 0, updated: 0, total: 0, deleted: 0, ...stats }); + } +} diff --git a/src/domain/synchronization/usecases/GenericSyncUseCase.ts b/src/domain/synchronization/usecases/GenericSyncUseCase.ts index 376b7dbcd..1008f518e 100644 --- a/src/domain/synchronization/usecases/GenericSyncUseCase.ts +++ b/src/domain/synchronization/usecases/GenericSyncUseCase.ts @@ -94,6 +94,12 @@ export abstract class GenericSyncUseCase { return this.repositoryFactory.eventsRepository(remoteInstance ?? defaultInstance); } + @cache() + protected async getDataStoreMetadataRepository(remoteInstance?: Instance) { + const defaultInstance = await this.getOriginInstance(); + return this.repositoryFactory.dataStoreMetadataRepository(remoteInstance ?? defaultInstance); + } + @cache() protected async getTeisRepository(remoteInstance?: Instance) { const defaultInstance = await this.getOriginInstance(); diff --git a/src/models/dhis/factory.ts b/src/models/dhis/factory.ts index f66f48b8f..74e3a3189 100644 --- a/src/models/dhis/factory.ts +++ b/src/models/dhis/factory.ts @@ -20,6 +20,7 @@ export const metadataModels = [ metadataClasses.DataElementGroupSetModel, metadataClasses.DataEntryFormModel, metadataClasses.DataSetModel, + metadataClasses.DataStoreModel, metadataClasses.DocumentsModel, metadataClasses.EventChartModel, metadataClasses.EventReportModel, diff --git a/src/models/dhis/metadata.ts b/src/models/dhis/metadata.ts index f0e1bdeb1..316ede929 100644 --- a/src/models/dhis/metadata.ts +++ b/src/models/dhis/metadata.ts @@ -1,3 +1,4 @@ +import { DataStore } from "../../domain/metadata/entities/MetadataEntities"; import { categoryOptionColumns, categoryOptionFields, @@ -780,3 +781,21 @@ export class SqlView extends D2Model { protected static includeRules = ["attributes"]; } + +export class DataStoreModel extends D2Model { + protected static metadataType = "dataStore"; + protected static modelName = "Data Store"; + protected static collectionName = "dataStores" as const; + protected static childrenKeys = ["keys"]; + + protected static modelTransform = (dataStores: DataStore[]) => { + return dataStores.map(({ keys = [], ...rest }) => ({ + ...rest, + keys: keys.map(keyItem => ({ ...keyItem, model: DataStoreKeysModel })), + })); + }; +} + +export class DataStoreKeysModel extends D2Model { + protected static modelName = "Keys"; +} diff --git a/src/presentation/CompositionRoot.ts b/src/presentation/CompositionRoot.ts index 499b7419a..a432b8dfe 100644 --- a/src/presentation/CompositionRoot.ts +++ b/src/presentation/CompositionRoot.ts @@ -119,6 +119,7 @@ import { GetSystemInfoUseCase } from "../domain/system-info/usecases/GetSystemIn import { ListTEIsUseCase } from "../domain/tracked-entity-instances/usecases/ListTEIsUseCase"; import { GetCurrentUserUseCase } from "../domain/user/usecases/GetCurrentUserUseCase"; import { cache } from "../utils/cache"; +import { DataStoreMetadataD2Repository } from "../data/data-store/DataStoreMetadataD2Repository"; export class CompositionRoot { private repositoryFactory: RepositoryFactory; @@ -148,6 +149,7 @@ export class CompositionRoot { this.repositoryFactory.bind(Repositories.MappingRepository, MappingD2ApiRepository); this.repositoryFactory.bind(Repositories.SchedulerRepository, SchedulerD2ApiRepository); this.repositoryFactory.bind(Repositories.SettingsRepository, SettingsD2ApiRepository); + this.repositoryFactory.bind(Repositories.DataStoreMetadataRepository, DataStoreMetadataD2Repository); } @cache() diff --git a/src/presentation/react/core/components/sync-wizard/common/MetadataSelectionStep.tsx b/src/presentation/react/core/components/sync-wizard/common/MetadataSelectionStep.tsx index da13ac46e..378f32fde 100644 --- a/src/presentation/react/core/components/sync-wizard/common/MetadataSelectionStep.tsx +++ b/src/presentation/react/core/components/sync-wizard/common/MetadataSelectionStep.tsx @@ -28,7 +28,7 @@ import { SyncWizardStepProps } from "../Steps"; const config = { metadata: { models: metadataModels, - childrenKeys: undefined, + childrenKeys: ["keys"], }, aggregated: { models: [ diff --git a/src/presentation/react/core/components/sync-wizard/common/SummaryStep.tsx b/src/presentation/react/core/components/sync-wizard/common/SummaryStep.tsx index 752c2d92f..9c2f526a5 100644 --- a/src/presentation/react/core/components/sync-wizard/common/SummaryStep.tsx +++ b/src/presentation/react/core/components/sync-wizard/common/SummaryStep.tsx @@ -4,6 +4,7 @@ import _ from "lodash"; import moment from "moment"; import React, { useEffect, useMemo, useState } from "react"; import { useHistory } from "react-router-dom"; +import { DataStoreMetadata } from "../../../../../../domain/data-store/DataStoreMetadata"; import { filterRuleToString } from "../../../../../../domain/metadata/entities/FilterRule"; import { MetadataEntities, @@ -272,6 +273,8 @@ export const SummaryStepContent = (props: SummaryStepContentProps) => { ); })} + + {syncRule.filterRules.length > 0 && ( { ); }; + +export const DataStoreSectionContent = (props: { metadataIds: string[] }) => { + const { metadataIds } = props; + + const dataStoreInfo = React.useMemo(() => { + return _(metadataIds) + .map(metadataId => { + return metadataId.includes(DataStoreMetadata.NS_SEPARATOR) ? metadataId : undefined; + }) + .compact() + .value(); + }, [metadataIds]); + + if (dataStoreInfo.length === 0) return null; + + return ( + <> + +
    + {dataStoreInfo.map(dataStore => { + const [namespace, key] = dataStore.split(DataStoreMetadata.NS_SEPARATOR); + const keyName = key ? `${key}` : "All Keys"; + return ; + })} +
+ + + ); +};