add dataStore support

src/data/common/D2ApiDataStore.ts

@@ -0,0 +1,65 @@
+import { D2Api } from "../../types/d2-api";
+import { DataSource, isDhisInstance } from "../../domain/instance/entities/DataSource";
+import { getD2APiFromInstance } from "../../utils/d2-utils";
+import { DataStore, DataStoreKey } from "../../domain/metadata/entities/MetadataEntities";
+import { promiseMap } from "../../utils/common";
+import { DataStoreMetadata } from "../../domain/data-store/DataStoreMetadata";
+
+export class D2ApiDataStore {
+    private api: D2Api;
+
+    constructor(instance: DataSource) {
+        if (!isDhisInstance(instance)) {
+            throw new Error("Invalid instance type for MetadataD2ApiRepository");
+        }
+        this.api = getD2APiFromInstance(instance);
+    }
+
+    async getDataStore(): Promise<DataStore[]> {
+        const response = await this.api.request<string[]>({ method: "get", url: "/dataStore" }).getData();
+        const namespacesWithKeys = await this.getAllKeysFromNamespaces(response);
+        return namespacesWithKeys;
+    }
+
+    private async getAllKeysFromNamespaces(namespaces: string[]): Promise<DataStore[]> {
+        const result = await promiseMap<string, DataStore>(namespaces, async namespace => {
+            const keys = await this.getKeysPaginated([], namespace);
+            return {
+                code: namespace,
+                displayName: namespace,
+                externalAccess: false,
+                favorites: [],
+                id: `${namespace}${DataStoreMetadata.NS_SEPARATOR}`,
+                keys: keys,
+                name: namespace,
+                translations: [],
+            };
+        });
+        return result;
+    }
+
+    private async getKeysPaginated(keysState: DataStoreKey[], namespace: string): Promise<DataStoreKey[]> {
+        const keyResponse = await this.getKeysByNameSpace(namespace);
+        const newKeys = [...keysState, ...keyResponse];
+        return newKeys;
+    }
+
+    private async getKeysByNameSpace(namespace: string): Promise<DataStoreKey[]> {
+        const response = await this.api
+            .request<string[]>({
+                method: "get",
+                url: `/dataStore/${namespace}`,
+                // Since v38 we can use the fields parameter to get keys and values in the same request
+                // Empty fields returns a paginated response
+                // https://docs.dhis2.org/en/full/develop/dhis-core-version-240/developer-manual.html#query-api
+                // params: { fields: "", page: page, pageSize: 200 },
+            })
+            .getData();
+
+        return this.buildArrayDataStoreKey(response, namespace);
+    }
+
+    private buildArrayDataStoreKey(keys: string[], namespace: string): DataStoreKey[] {
+        return keys.map(key => ({ id: `${namespace}${DataStoreMetadata.NS_SEPARATOR}${key}`, displayName: key }));
+    }
+}

src/data/data-store/DataStoreMetadataD2Repository.ts

@@ -0,0 +1,109 @@
+import _ from "lodash";
+import { DataStoreMetadata } from "../../domain/data-store/DataStoreMetadata";
+import { DataStoreMetadataRepository, SaveOptions } from "../../domain/data-store/DataStoreMetadataRepository";
+import { Instance } from "../../domain/instance/entities/Instance";
+import { Stats } from "../../domain/reports/entities/Stats";
+import { SynchronizationResult } from "../../domain/reports/entities/SynchronizationResult";
+import { promiseMap } from "../../utils/common";
+import { StorageDataStoreClient } from "../storage/StorageDataStoreClient";
+
+export class DataStoreMetadataD2Repository implements DataStoreMetadataRepository {
+    private instance: Instance;
+
+    constructor(instance: Instance) {
+        this.instance = instance;
+    }
+
+    async get(dataStores: DataStoreMetadata[]): Promise<DataStoreMetadata[]> {
+        const result = await promiseMap(dataStores, async dataStore => {
+            const dataStoreClient = new StorageDataStoreClient(this.instance, dataStore.namespace);
+            const dataStoreWithValue = this.getValuesByDataStore(dataStoreClient, dataStore);
+            return dataStoreWithValue;
+        });
+        return result;
+    }
+
+    private async getValuesByDataStore(
+        dataStoreClient: StorageDataStoreClient,
+        dataStore: DataStoreMetadata
+    ): Promise<DataStoreMetadata> {
+        const keys = await this.getAllKeys(dataStoreClient, dataStore);
+        const keyWithValue = await promiseMap(keys, async key => {
+            const keyValue = await dataStoreClient.getObject(key.id);
+            return { id: key.id, value: keyValue };
+        });
+        const keyInNamespace = _(dataStore.keys).first()?.id;
+        const sharing = keyInNamespace ? await dataStoreClient.getObjectSharing(keyInNamespace) : undefined;
+        return new DataStoreMetadata({
+            namespace: dataStore.namespace,
+            keys: keyWithValue,
+            sharing,
+        });
+    }
+
+    private async getAllKeys(
+        dataStoreClient: StorageDataStoreClient,
+        dataStore: DataStoreMetadata
+    ): Promise<DataStoreMetadata["keys"]> {
+        if (dataStore.keys.length > 0) return dataStore.keys;
+        const keys = await dataStoreClient.listKeys();
+        return keys.map(key => ({ id: key, value: "" }));
+    }
+
+    async save(
+        dataStores: DataStoreMetadata[],
+        options: SaveOptions = { mergeMode: "MERGE" }
+    ): Promise<SynchronizationResult> {
+        const keysIdsToDelete = await this.getKeysToDelete(dataStores, options);
+
+        const resultStats = await promiseMap(dataStores, async dataStore => {
+            const dataStoreClient = new StorageDataStoreClient(this.instance, dataStore.namespace);
+            const stats = await promiseMap(dataStore.keys, async key => {
+                const exist = await dataStoreClient.getObject(key.id);
+                await dataStoreClient.saveObject(key.id, key.value);
+                if (dataStore.sharing) {
+                    await dataStoreClient.saveObjectSharing(key.id, dataStore.sharing);
+                }
+                return exist ? Stats.createOrEmpty({ updated: 1 }) : Stats.createOrEmpty({ imported: 1 });
+            });
+            return stats;
+        });
+
+        const deleteStats = await promiseMap(keysIdsToDelete, async keyId => {
+            const [namespace, key] = keyId.split(DataStoreMetadata.NS_SEPARATOR);
+            const dataStoreClient = new StorageDataStoreClient(this.instance, namespace);
+            await dataStoreClient.removeObject(key);
+            return Stats.createOrEmpty({ deleted: 1 });
+        });
+
+        const allStats = resultStats.flatMap(result => result).concat(deleteStats);
+        const dataStoreStats = { ...Stats.combine(allStats.map(stat => Stats.create(stat))), type: "DataStore Keys" };
+
+        const result: SynchronizationResult = {
+            date: new Date(),
+            instance: this.instance,
+            status: "SUCCESS",
+            type: "metadata",
+            stats: dataStoreStats,
+            typeStats: [dataStoreStats],
+        };
+
+        return result;
+    }
+
+    private async getKeysToDelete(dataStores: DataStoreMetadata[], options: SaveOptions) {
+        if (options.mergeMode === "MERGE") return [];
+
+        const existingRecords = await this.get(dataStores.map(x => ({ ...x, keys: [] })));
+        const existingKeysIds = existingRecords.flatMap(dataStore => {
+            return dataStore.keys.map(key => `${dataStore.namespace}[NS]${key.id}`);
+        });
+
+        const keysIdsToSave = dataStores.flatMap(dataStore => {
+            return dataStore.keys.map(key => `${dataStore.namespace}[NS]${key.id}`);
+        });
+
+        const keysIdsToDelete = existingKeysIds.filter(id => !keysIdsToSave.includes(id));
+        return keysIdsToDelete;
+    }
+}

src/data/metadata/MetadataD2ApiRepository.ts

@@ -38,6 +38,7 @@ import { debug } from "../../utils/debug";
 import { paginate } from "../../utils/pagination";
 import { metadataTransformations } from "../transformations/PackageTransformations";
 import { D2MetadataUtils } from "./D2MetadataUtils";
+import { D2ApiDataStore } from "../common/D2ApiDataStore";
 
 export class MetadataD2ApiRepository implements MetadataRepository {
     private api: D2Api;
@@ -101,17 +102,24 @@ export class MetadataD2ApiRepository implements MetadataRepository {
         const filter = this.buildListFilters(params);
         const { apiVersion } = this.instance;
         const options = { type, fields, filter, order, page, pageSize, rootJunction };
-        const { objects: baseObjects, pager } = await this.getListPaginated(options);
-        // Prepend parent objects (if option enabled) as virtual rows, keep pagination unmodified.
-        const objects = _.concat(await this.getParentObjects(listParams), baseObjects);
+        if (type === "dataStores") {
+            const d2ApiDataStore = new D2ApiDataStore(this.instance);
+            const response = await d2ApiDataStore.getDataStore();
+            // Hardcoded pagination since DHIS2 does not support pagination for namespaces
+            return { objects: response, pager: { page: 1, total: response.length, pageSize: 100 } };
+        } else {
+            const { objects: baseObjects, pager } = await this.getListPaginated(options);
+            // Prepend parent objects (if option enabled) as virtual rows, keep pagination unmodified.
+            const objects = _.concat(await this.getParentObjects(listParams), baseObjects);
 
-        const metadataPackage = this.transformationRepository.mapPackageFrom(
-            apiVersion,
-            { [type]: objects },
-            metadataTransformations
-        );
+            const metadataPackage = this.transformationRepository.mapPackageFrom(
+                apiVersion,
+                { [type]: objects },
+                metadataTransformations
+            );
 
-        return { objects: metadataPackage[type as keyof MetadataEntities] ?? [], pager };
+            return { objects: metadataPackage[type as keyof MetadataEntities] ?? [], pager };
+        }
     }
 
     @cache()

src/data/storage/StorageDataStoreClient.ts

@@ -14,10 +14,10 @@ export class StorageDataStoreClient extends StorageClient {
     private api: D2Api;
     private dataStore: DataStore;
 
-    constructor(instance: Instance) {
+    constructor(instance: Instance, namespace: string = dataStoreNamespace) {
         super();
         this.api = getD2APiFromInstance(instance);
-        this.dataStore = this.api.dataStore(dataStoreNamespace);
+        this.dataStore = this.api.dataStore(namespace);
     }
 
     public async getObject<T extends object>(key: string): Promise<T | undefined> {

src/domain/common/entities/Struct.ts

@@ -0,0 +1,45 @@
+/**
+ * Base class for typical classes with attributes. Features: create, update.
+ *
+ * ```
+ * class Counter extends Struct<{ id: Id; value: number }>() {
+ *     add(value: number): Counter {
+ *         return this._update({ value: this.value + value });
+ *     }
+ * }
+ *
+ * const counter1 = Counter.create({ id: "some-counter", value: 1 });
+ * const counter2 = counter1._update({ value: 2 });
+ * ```
+ */
+
+export function Struct<Attrs>() {
+    abstract class Base {
+        constructor(_attributes: Attrs) {
+            Object.assign(this, _attributes);
+        }
+
+        _getAttributes(): Attrs {
+            const entries = Object.getOwnPropertyNames(this).map(key => [key, (this as any)[key]]);
+            return Object.fromEntries(entries) as Attrs;
+        }
+
+        protected _update(partialAttrs: Partial<Attrs>): this {
+            const ParentClass = this.constructor as new (values: Attrs) => typeof this;
+            return new ParentClass({ ...this._getAttributes(), ...partialAttrs });
+        }
+
+        static create<U extends Base>(this: new (attrs: Attrs) => U, attrs: Attrs): U {
+            return new this(attrs);
+        }
+    }
+
+    return Base as {
+        new (values: Attrs): Attrs & Base;
+        create: typeof Base["create"];
+    };
+}
+
+const GenericStruct = Struct<unknown>();
+
+export type GenericStructInstance = InstanceType<typeof GenericStruct>;

src/domain/common/factories/RepositoryFactory.ts

@@ -5,6 +5,7 @@ import {
 } from "../../aggregated/repositories/AggregatedRepository";
 import { ConfigRepositoryConstructor } from "../../config/repositories/ConfigRepository";
 import { CustomDataRepositoryConstructor } from "../../custom-data/repository/CustomDataRepository";
+import { DataStoreMetadataRepositoryConstructor } from "../../data-store/DataStoreMetadataRepository";
 import { EventsRepository, EventsRepositoryConstructor } from "../../events/repositories/EventsRepository";
 import { FileRepositoryConstructor } from "../../file/repositories/FileRepository";
 import { DataSource } from "../../instance/entities/DataSource";
@@ -120,6 +121,11 @@ export class RepositoryFactory {
         return this.get<EventsRepositoryConstructor>(Repositories.EventsRepository, [instance]);
     }
 
+    @cache()
+    public dataStoreMetadataRepository(instance: Instance) {
+        return this.get<DataStoreMetadataRepositoryConstructor>(Repositories.DataStoreMetadataRepository, [instance]);
+    }
+
     @cache()
     public teisRepository(instance: Instance): TEIRepository {
         return this.get<TEIRepositoryConstructor>(Repositories.TEIsRepository, [instance]);
@@ -209,4 +215,5 @@ export const Repositories = {
     MappingRepository: "mappingRepository",
     SettingsRepository: "settingsRepository",
     SchedulerRepository: "schedulerRepository",
+    DataStoreMetadataRepository: "dataStoreMetadataRepository",
 } as const;