Merge pull request #286 from docknetwork/DCKM-591-new-wallet-sdk-edv-…

…storage-package-for-edv-integration

edv sync implementation

.github/workflows/integration-tests.yaml

@@ -84,6 +84,9 @@ jobs:
           PR_TITLE: ${{ steps.pr_info.outputs.title }}
           GITHUB_ACTION_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
           TESTING_API_URL: ${{secrets.TESTING_API_URL}}
+          EDV_URL: ${{secrets.EDV_URL}}
+          EDV_AUTH_KEY: ${{secrets.EDV_AUTH_KEY}}
+
         run: |
           export NODE_OPTIONS="--max_old_space_size=12096"
           export CI=true

integration-tests/helpers/wallet-helpers.ts

@@ -27,6 +27,7 @@ import {
 } from '@docknetwork/wallet-sdk-core/src/credential-provider';
 import {dockService} from '@docknetwork/wallet-sdk-wasm/src/services/dock';
 import {createDataStore} from '@docknetwork/wallet-sdk-data-store-typeorm/src';
+import {DataStore} from '@docknetwork/wallet-sdk-data-store/src/types';
 
 let wallet: IWallet;
 let didProvider: IDIDProvider;
@@ -35,17 +36,18 @@ let messageProvider: IMessageProvider;
 
 export async function createNewWallet({
   dontWaitForNetwork,
-  otherProps,
+  dataStore,
 }: {
   dontWaitForNetwork?: boolean;
-  otherProps?: CreateWalletProps;
+  dataStore?: DataStore;
 } = {}): Promise<IWallet> {
-  const dataStore = await createDataStore({
-    databasePath: ':memory:',
-    dbType: 'sqlite',
-    defaultNetwork: 'testnet',
-    ...(otherProps ? otherProps : {}),
-  });
+  if (!dataStore) {
+    dataStore = await createDataStore({
+      databasePath: ':memory:',
+      dbType: 'sqlite',
+      defaultNetwork: 'testnet',
+    });
+  }
 
   wallet = await createWallet({
     dataStore,

jest.config.e2e.js

@@ -26,6 +26,12 @@ module.exports = {
   globalTeardown: './scripts/integration-test-teardown.js',
   setupFiles: ['jest-localstorage-mock'],
   moduleNameMapper: {
+    '@digitalbazaar/edv-client': require.resolve(
+      '@digitalbazaar/edv-client/main.js',
+    ),
+    '@digitalbazaar/http-signature-zcap-invoke': require.resolve(
+      '@digitalbazaar/http-signature-zcap-invoke/main.js',
+    ),
     '@digitalbazaar/x25519-key-agreement-key-2020':
       '@digitalbazaar/x25519-key-agreement-key-2020/lib/X25519KeyAgreementKey2020',
     '@digitalbazaar/ed25519-verification-key-2020':

package.json

@@ -25,7 +25,7 @@
     "@babel/core": "^7.16.0",
     "@babel/preset-env": "^7.22.10",
     "@docknetwork/sdk": "8.6.0",
-    "@docknetwork/universal-wallet": "^2.0.1",
+    "@docknetwork/universal-wallet": "2.1.1",
     "@polkadot/api": "9.7.1",
     "@polkadot/keyring": "10.1.11",
     "@polkadot/rpc-core": "9.7.1",

packages/core/src/cloud-wallet.ts

@@ -0,0 +1,183 @@
+import {
+  DataStore,
+  DataStoreEvents,
+} from '@docknetwork/wallet-sdk-data-store/src/types';
+import {logger} from '@docknetwork/wallet-sdk-data-store/src/logger';
+import {edvService} from '@docknetwork/wallet-sdk-wasm/src/services/edv';
+
+export const SYNC_MARKER_TYPE = 'SyncMarkerDocument';
+
+export async function initializeCloudWallet({
+  dataStore,
+  edvUrl,
+  agreementKey,
+  verificationKey,
+  hmacKey,
+  authKey,
+}: {
+  dataStore?: DataStore;
+  edvUrl: string;
+  agreementKey: any;
+  verificationKey: any;
+  hmacKey: any;
+  authKey: string;
+}) {
+
+  await edvService.initialize({
+    hmacKey,
+    agreementKey,
+    verificationKey,
+    edvUrl,
+    authKey
+  });
+
+  let pendingOperations = 0;
+  let pendingOperationsResolvers = [];
+
+  function waitForEdvIdle() {
+    if (pendingOperations === 0) {
+      return Promise.resolve();
+    }
+    return new Promise(resolve => {
+      pendingOperationsResolvers.push(resolve);
+    });
+  }
+
+  async function findDocumentByContentId(id) {
+    const result = await edvService.find({
+      equals: {
+        'content.id': id,
+      },
+    });
+
+    return result.documents[0];
+  }
+
+  async function updateDocumentByContentId(documentContent) {
+    const edvDocument = await findDocumentByContentId(documentContent.id);
+
+    if (!edvDocument) {
+      throw new Error('Document not found in EDV');
+    }
+
+    logger.debug(`Updating document ${documentContent.id} in EDV`);
+
+    await edvService.update({
+      document: {
+        id: edvDocument.id,
+        content: documentContent,
+      },
+    });
+
+    logger.debug(`Document ${documentContent.id} updated in EDV`);
+  }
+
+  async function addDocumentHandler(content) {
+    pendingOperations++;
+    try {
+      logger.debug(`Adding document to EDV: ${content.id}`);
+      await edvService.insert({
+        document: {
+          content: content,
+        },
+      });
+      logger.debug(`Document added to EDV: ${content.id}`);
+    } finally {
+      pendingOperations--;
+      if (pendingOperations === 0) {
+        pendingOperationsResolvers.forEach(resolve => resolve());
+        pendingOperationsResolvers = [];
+      }
+    }
+  }
+
+  async function removeDocumentHandler(documentId) {
+    pendingOperations++;
+    try {
+      logger.debug(`Removing document from EDV: ${documentId}`);
+      const edvDocument = await findDocumentByContentId(documentId);
+      await edvService.delete({document: edvDocument});
+      logger.debug(`Document removed from EDV: ${documentId}`);
+    } finally {
+      pendingOperations--;
+      if (pendingOperations === 0) {
+        pendingOperationsResolvers.forEach(resolve => resolve());
+        pendingOperationsResolvers = [];
+      }
+    }
+  }
+
+  async function updateDocumentHandler(documentContent) {
+    pendingOperations++;
+    try {
+      await updateDocumentByContentId(documentContent);
+    } finally {
+      pendingOperations--;
+      if (pendingOperations === 0) {
+        pendingOperationsResolvers.forEach(resolve => resolve());
+        pendingOperationsResolvers = [];
+      }
+    }
+  }
+
+  dataStore.events.on(DataStoreEvents.DocumentCreated, addDocumentHandler);
+  dataStore.events.on(DataStoreEvents.DocumentDeleted, removeDocumentHandler);
+  dataStore.events.on(DataStoreEvents.DocumentUpdated, updateDocumentHandler);
+
+  async function getSyncMarkerDiff() {
+    const edvSyncMaker = await findDocumentByContentId(SYNC_MARKER_TYPE);
+    const localSyncMarker = await dataStore.documents.getDocumentById(
+      SYNC_MARKER_TYPE,
+    );
+
+    return edvSyncMaker?.content?.updatedAt - localSyncMarker?.updatedAt;
+  }
+
+  async function pushSyncMarker() {
+    const edvSyncMarker = await findDocumentByContentId(SYNC_MARKER_TYPE);
+    const syncMarker = {
+      id: SYNC_MARKER_TYPE,
+      type: SYNC_MARKER_TYPE,
+      updatedAt: Date.now(),
+    };
+
+    if (edvSyncMarker) {
+      await dataStore.documents.updateDocument(syncMarker);
+    } else {
+      await dataStore.documents.addDocument(syncMarker);
+    }
+  }
+
+  async function pullDocuments() {
+    const allDocs = await edvService.find({});
+
+    for (const doc of allDocs.documents) {
+      const edvDoc = doc.content;
+      const walletDoc = await dataStore.documents.getDocumentById(edvDoc.id);
+
+      if (!walletDoc) {
+        const result = await dataStore.documents.addDocument(edvDoc, {
+          stopPropagation: true,
+        });
+      }
+    }
+  }
+
+  async function clearEdvDocuments() {
+    const allDocs = await edvService.find({});
+
+    for (const doc of allDocs.documents) {
+      await edvService.delete({document: doc});
+    }
+  }
+
+  return {
+    clearEdvDocuments,
+    pushSyncMarker,
+    getSyncMarkerDiff,
+    findDocumentByContentId,
+    updateDocumentByContentId,
+    waitForEdvIdle,
+    pullDocuments,
+  };
+}

packages/core/src/v1-helpers.ts

@@ -80,7 +80,7 @@ export type KeypairType = 'sr25519' | 'ed25519' | 'ecdsa';
 export function toV1WalletService(wallet: IWallet) {
   return {
     getDocumentById: id => {
-      return wallet.getDocumentById(id);
+      return wallet.dataStore.documents.getDocumentById(id);
     },
     // accounts are not required in a wallet
     // Ideally should move this code to the accounts provider file

packages/core/src/wallet.ts

@@ -64,10 +64,10 @@ export async function createWallet({
     getNetworkId: () => {
       return dataStore.networkId;
     },
-    getDocumentById: dataStore.documents.getDocumentById,
-    getAllDocuments: dataStore.documents.getAllDocuments,
-    getDocumentsById: dataStore.documents.getDocumentsById,
-    getDocumentsByType: dataStore.documents.getDocumentsByType,
+    getDocumentById: (id) => dataStore.documents.getDocumentById(id),
+    getAllDocuments: () => dataStore.documents.getAllDocuments(),
+    getDocumentsById: (idList) => dataStore.documents.getDocumentsById(idList),
+    getDocumentsByType: (type) => dataStore.documents.getDocumentsByType(type),
     addDocument: (json: any) => {
       return dataStore.documents.addDocument(json).then(result => {
         eventEmitter.emit(WalletEvents.documentAdded, result);

packages/data-store-typeorm/src/migration/migration1/v1-data-store.ts

@@ -1,11 +1,6 @@
+import {LocalStorage} from '@docknetwork/wallet-sdk-data-store/src/types';
 import assert from 'assert';
 
-export type LocalStorage = {
-  getItem: (key: string) => Promise<string | null>;
-  setItem: (key: string, value: string) => Promise<void>;
-  removeItem: (key: string) => Promise<void>;
-};
-
 let _localStorage: LocalStorage;
 
 /**

packages/data-store-typeorm/src/migrations-data-source.ts

@@ -1,6 +1,6 @@
-import {getDataSource} from './helpers';
+import {createDataSource} from './helpers';
 
-export default getDataSource({
+export default createDataSource({
   dbType: 'sqlite',
   databasePath: 'data-store.sqlite',
 });

packages/data-store-web/.gitignore

@@ -0,0 +1,3 @@
+*.sqlite
+lib/
+tsconfig.build.tsbuildinfo

packages/data-store-web/.npmignore

@@ -0,0 +1,2 @@
+node_modules/
+sqlite/

packages/data-store-web/jest.config.ts

@@ -0,0 +1,20 @@
+module.exports = {
+  testEnvironment: 'node',
+  testTimeout: 30000,
+  testMatch: [
+    "<rootDir>/src/**/!(*.e2e).test.ts"
+  ],
+  coverageThreshold: {
+    global: {
+      branches: 10,
+      functions: 10,
+      lines: 10,
+      statements: 10,
+    },
+  },
+  transform: {
+    '^.+\\.ts$': 'ts-jest',
+  },
+  resetMocks: false,
+  setupFiles: ['jest-localstorage-mock'],
+};

packages/data-store-web/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@docknetwork/wallet-sdk-data-store-web",
+  "version": "0.4.19",
+  "license": "https://github.com/docknetwork/react-native-sdk/LICENSE",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/docknetwork/react-native-sdk",
+    "directory": "packages/data-store"
+  },
+  "scripts": {
+    "test": "jest",
+    "build": "tsc -p tsconfig.build.json"
+  },
+  "peerDependencies": {
+    "typeorm": "^0.3.15"
+  },
+  "dependencies": {
+    "@docknetwork/wallet-sdk-wasm": "^0.4.19",
+    "uuid": "^8.3.2"
+  },
+  "devDependencies": {
+    "jest": "29.1.0",
+    "ts-jest": "29.1.0",
+    "ts-node": "^10.9.1",
+    "typescript": "^5.0.4",
+    "typeorm": "^0.3.15",
+    "sqlite3": "^5.0.2",
+    "reflect-metadata": "^0.1.13"
+  }
+}

packages/data-store-web/scripts/publish.sh

@@ -0,0 +1,3 @@
+yarn build
+npm publish --access public
+