Backend integration with Nordigen - account sync (actualbudget#74)

* Add nordigen integration * Move normalizatoin of accounts to the backend side * Remove .idea from git * Move normalization of transactions to the backend side * Fix some edgecases * Move nordigen to separate directory * Partial refactor of nordigen and e2e test * WIP refactor * Refactoring * Refactoring * Add more tests * Update get accounts path * Rm not needed import * Fix after merge * Fix AnimatedLoading * Fix coverage, jest config, linter * Code review changes * Upgrade to ESM nordigen * Upgrade to ESM nordigen * Remove e2e tests and cleanup packages * Move env vars to config * Rollback prettierrc config * Move nordigen app behind to src * Revert supertest lib * Fixing specs * fixes linter * Fix build errors * Fix linter * Update nordigen-node lib * remove snapshot * remove babel * Fix spec * fix linter * Revert "remove babel" This reverts commit 07ce9fc. * Fix coverage * Add supertest * Add sortByBookingDate as default sort option for integration bank * Add comment with explanation of client const --------- Co-authored-by: Filip Stybel <[email protected]>
lelemm · Mar 4, 2023 · 19cd163 · 19cd163
1 parent 06b687e
commit 19cd163
Show file tree

Hide file tree

Showing 30 changed files with 3,171 additions and 5 deletions.
diff --git a/.gitignore b/.gitignore
@@ -23,3 +23,8 @@ build/
 !.yarn/releases
 !.yarn/sdks
 !.yarn/versions
+
+dist
+.idea
+/coverage
+/coverage-e2e
diff --git a/babel.config.json b/babel.config.json
@@ -0,0 +1,3 @@
+{
+  "presets": ["@babel/preset-flow"]
+}
diff --git a/jest.config.json b/jest.config.json
@@ -1,4 +1,14 @@
 {
-	"setupFiles": ["./jest.setup.js"],
-	"testPathIgnorePatterns": ["/node_modules/", "/build/"]
+  "setupFiles": ["./jest.setup.js"],
+  "testPathIgnorePatterns": ["dist", "/node_modules/", "/build/"],
+  "roots": ["<rootDir>"],
+  "testMatch": ["<rootDir>/**/*.spec.js"],
+  "moduleFileExtensions": ["ts", "js", "json"],
+  "testEnvironment": "node",
+  "collectCoverage": true,
+  "collectCoverageFrom": ["**/*.{js,ts,tsx}"],
+  "coveragePathIgnorePatterns": ["dist", "/node_modules/", "/build/"],
+  "coverageReporters": ["html", "lcov", "text", "text-summary"],
+  "resetMocks": true,
+  "restoreMocks": true
 }
diff --git a/package.json b/package.json
@@ -8,7 +8,7 @@
     "start": "node app",
     "lint": "eslint .",
     "build": "tsc",
-    "test": "NODE_OPTIONS='--experimental-vm-modules --trace-warnings' jest",
+    "test": "NODE_ENV=test NODE_OPTIONS='--experimental-vm-modules --trace-warnings' jest  --coverage",
     "types": "tsc --noEmit --incremental",
     "verify": "yarn -s lint && yarn types"
   },
@@ -22,9 +22,11 @@
     "express": "4.18.2",
     "express-actuator": "1.8.4",
     "express-response-size": "^0.0.3",
+    "nordigen-node": "^1.2.3",
     "uuid": "^9.0.0"
   },
   "devDependencies": {
+    "@babel/preset-flow": "^7.18.6",
     "@types/bcrypt": "^5.0.0",
     "@types/better-sqlite3": "^7.5.0",
     "@types/cors": "^2.8.13",

diff --git a/src/app-nordigen/README.md b/src/app-nordigen/README.md
@@ -0,0 +1,61 @@
+# Integration new bank
+
+Find in [doc](https://docs.google.com/spreadsheets/d/1ogpzydzotOltbssrc3IQ8rhBLlIZbQgm5QCiiNJrkyA/edit#gid=489769432) what is id of bank which you want to integrate
+
+Add the `institution_id` and your name to list of possible options in the frontend
+project `actual/packages/loot-design/src/components/modals/NordigenExternalMsg.js`
+
+```jsx
+<Strong>Choose your banks:</Strong>
+<CustomSelect
+  options={[
+    ['default', 'Choose your bank'],
+    ['ING_PL_INGBPLPW', 'ING PL'],
+    ['MBANK_RETAIL_BREXPLPW', 'MBANK'],
+    ['SANDBOXFINANCE_SFIN0000', 'DEMO - TEST']
+  ]}
+```
+
+Launch frontend and backend server
+
+Create new linked account selecting the institution which you added recently.
+
+In the server logs you can find all required information to create class for
+your bank.
+
+Create new a bank class based on `app-nordigen/banks/sandboxfinance-sfin0000.js`. Name of the file and class should be
+created based on the ID of the integrated institution.
+
+Fill the logic of `normalizeAccount`, `sortTransactions`, and `calculateStartingBalance` functions.
+You should do it based on the data which you found in the logs.
+
+Example logs which help you to fill:
+
+- `normalizeAccount` function:
+
+```log
+Available account properties for new institution integration {
+  account: '{"iban":"PL00000000000000000987654321","currency":"PLN","ownerName":"John Example","displayName":"Product name","product":"Daily account","usage":"PRIV","ownerAddressUnstructured":["POL","UL. Example 1","00-000 Warsaw"],"id":"XXXXXXXX-XXXX-XXXXX-XXXXXX-XXXXXXXXX","created":"2023-01-18T12:15:16.502446Z","last_accessed":null,"institution_id":"MBANK_RETAIL_BREXPLPW","status":"READY","owner_name":"","institution":{"id":"MBANK_RETAIL_BREXPLPW","name":"mBank Retail","bic":"BREXPLPW","transaction_total_days":"90","countries":["PL"],"logo":"https://cdn.nordigen.com/ais/MBANK_RETAIL_BREXCZPP.png","supported_payments":{},"supported_features":["access_scopes","business_accounts","card_accounts","corporate_accounts","pending_transactions","private_accounts"]}}'
+}
+```
+
+- `sortTransactions` function:
+
+```log
+Available (first 10) transactions properties for new integration of institution in sortTransactions function {
+  top10SortedTransactions: '[{"transactionId":"20220101001","bookingDate":"2022-01-01","valueDate":"2022-01-01","transactionAmount":{"amount":"5.01","currency":"EUR"},"creditorName":"JOHN EXAMPLE","creditorAccount":{"iban":"PL00000000000000000987654321"},"debtorName":"CHRIS EXAMPLE","debtorAccount":{"iban":"PL12345000000000000987654321"},"remittanceInformationUnstructured":"TEST BANK TRANSFER","remittanceInformationUnstructuredArray":["TEST BANK TRANSFER"],"balanceAfterTransaction":{"balanceAmount":{"amount":"448.52","currency":"EUR"},"balanceType":"interimBooked"},"internalTransactionId":"casfib7720c2a02c0331cw2"}]'
+}
+```
+
+- `calculateStartingBalance` function:
+
+```log
+Available (first 10) transactions properties for new integration of institution in calculateStartingBalance function {
+  balances: '[{"balanceAmount":{"amount":"448.52","currency":"EUR"},"balanceType":"forwardAvailable"},{"balanceAmount":{"amount":"448.52","currency":"EUR"},"balanceType":"interimBooked"}]',
+    top10SortedTransactions: '[{"transactionId":"20220101001","bookingDate":"2022-01-01","valueDate":"2022-01-01","transactionAmount":{"amount":"5.01","currency":"EUR"},"creditorName":"JOHN EXAMPLE","creditorAccount":{"iban":"PL00000000000000000987654321"},"debtorName":"CHRIS EXAMPLE","debtorAccount":{"iban":"PL12345000000000000987654321"},"remittanceInformationUnstructured":"TEST BANK TRANSFER","remittanceInformationUnstructuredArray":["TEST BANK TRANSFER"],"balanceAfterTransaction":{"balanceAmount":{"amount":"448.52","currency":"EUR"},"balanceType":"interimBooked"},"internalTransactionId":"casfib7720c2a02c0331cw2"}]'
+}
+```
+
+Add new bank integration to `BankFactory` class in file `actual-server/app-nordigen/bank-factory.js`
+
+Remember to add tests for new bank integration in
diff --git a/src/app-nordigen/app-nordigen.js b/src/app-nordigen/app-nordigen.js
@@ -0,0 +1,166 @@
+import express from 'express';
+
+import { nordigenService } from './services/nordigen-service.js';
+import {
+  RequisitionNotLinked,
+  AccountNotLinedToRequisition,
+  GenericNordigenError
+} from './errors.js';
+import { handleError } from './util/handle-error.js';
+import validateUser from '../util/validate-user.js';
+
+const app = express();
+export { app as handlers };
+app.use(express.json());
+app.use(async (req, res, next) => {
+  let user = await validateUser(req, res);
+  if (!user) {
+    return;
+  }
+  next();
+});
+
+app.post(
+  '/create-web-token',
+  handleError(async (req, res) => {
+    const { accessValidForDays, institutionId } = req.body;
+    const { origin } = req.headers;
+
+    const { link, requisitionId } = await nordigenService.createRequisition({
+      accessValidForDays,
+      institutionId,
+      host: origin
+    });
+
+    res.send({
+      status: 'ok',
+      data: {
+        link,
+        requisitionId
+      }
+    });
+  })
+);
+
+app.post(
+  '/get-accounts',
+  handleError(async (req, res) => {
+    const { requisitionId } = req.body;
+
+    try {
+      const { requisition, accounts } =
+        await nordigenService.getRequisitionWithAccounts(requisitionId);
+
+      res.send({
+        status: 'ok',
+        data: {
+          ...requisition,
+          accounts
+        }
+      });
+    } catch (error) {
+      if (error instanceof RequisitionNotLinked) {
+        res.send({
+          status: 'ok',
+          requisitionStatus: error.details.requisitionStatus
+        });
+      } else {
+        throw error;
+      }
+    }
+  })
+);
+
+app.post(
+  '/remove-account',
+  handleError(async (req, res) => {
+    let { requisitionId } = req.body;
+
+    const data = await nordigenService.deleteRequisition(requisitionId);
+    if (data.summary === 'Requisition deleted') {
+      res.send({
+        status: 'ok',
+        data
+      });
+    } else {
+      res.send({
+        status: 'error',
+        data: {
+          data,
+          reason: 'Can not delete requisition'
+        }
+      });
+    }
+  })
+);
+
+app.post(
+  '/transactions',
+  handleError(async (req, res) => {
+    const { requisitionId, startDate, endDate, accountId } = req.body;
+
+    try {
+      const {
+        balances,
+        institutionId,
+        startingBalance,
+        transactions: { booked, pending }
+      } = await nordigenService.getTransactionsWithBalance(
+        requisitionId,
+        accountId,
+        startDate,
+        endDate
+      );
+
+      res.send({
+        status: 'ok',
+        data: {
+          balances,
+          institutionId,
+          startingBalance,
+          transactions: {
+            booked,
+            pending
+          }
+        }
+      });
+    } catch (error) {
+      const sendErrorResponse = (data) =>
+        res.send({ status: 'ok', data: { ...data, details: error.details } });
+
+      switch (true) {
+        case error instanceof RequisitionNotLinked:
+          sendErrorResponse({
+            error_type: 'ITEM_ERROR',
+            error_code: 'ITEM_LOGIN_REQUIRED',
+            status: 'expired',
+            reason: 'Access to account has expired as set in End User Agreement'
+          });
+          break;
+        case error instanceof AccountNotLinedToRequisition:
+          sendErrorResponse({
+            error_type: 'INVALID_INPUT',
+            error_code: 'INVALID_ACCESS_TOKEN',
+            status: 'rejected',
+            reason: 'Account not linked with this requisition'
+          });
+          break;
+        case error instanceof GenericNordigenError:
+          console.log({ message: 'Something went wrong', error });
+          sendErrorResponse({
+            error_type: 'SYNC_ERROR',
+            error_code: 'NORDIGEN_ERROR'
+          });
+          break;
+        default:
+          console.log({ message: 'Something went wrong', error });
+          sendErrorResponse({
+            error_type: 'UNKNOWN',
+            error_code: 'UNKNOWN',
+            reason: 'Something went wrong'
+          });
+          break;
+      }
+    }
+  })
+);
diff --git a/src/app-nordigen/bank-factory.js b/src/app-nordigen/bank-factory.js
@@ -0,0 +1,9 @@
+import IngPlIngbplpw from './banks/ing-pl-ingbplpw.js';
+import IntegrationBank from './banks/integration-bank.js';
+import MbankRetailBrexplpw from './banks/mbank-retail-brexplpw.js';
+import SandboxfinanceSfin0000 from './banks/sandboxfinance-sfin0000.js';
+
+const banks = [MbankRetailBrexplpw, SandboxfinanceSfin0000, IngPlIngbplpw];
+
+export default (institutionId) =>
+  banks.find((b) => b.institutionId === institutionId) || IntegrationBank;
diff --git a/src/app-nordigen/banks/bank.interface.ts b/src/app-nordigen/banks/bank.interface.ts
@@ -0,0 +1,28 @@
+import {
+  DetailedAccountWithInstitution,
+  NormalizedAccountDetails
+} from '../nordigen.types.js';
+import { Transaction, Balance } from '../nordigen-node.types.js';
+
+export interface IBank {
+  institutionId: string;
+  /**
+   * Returns normalized object with required data for the frontend
+   */
+  normalizeAccount: (
+    account: DetailedAccountWithInstitution
+  ) => NormalizedAccountDetails;
+
+  /**
+   * Function sorts an array of transactions from newest to oldest
+   */
+  sortTransactions: (transactions: Transaction[]) => Transaction[];
+
+  /**
+   * Calculates account balance before which was before transactions provided in sortedTransactions param
+   */
+  calculateStartingBalance: (
+    sortedTransactions: Transaction[],
+    balances: Balance[]
+  ) => number;
+}
diff --git a/src/app-nordigen/banks/ing-pl-ingbplpw.js b/src/app-nordigen/banks/ing-pl-ingbplpw.js
@@ -0,0 +1,45 @@
+import { printIban, amountToInteger } from '../utils.js';
+
+/** @type {import('./bank.interface.js').IBank} */
+export default {
+  institutionId: 'ING_PL_INGBPLPW',
+
+  normalizeAccount(account) {
+    return {
+      account_id: account.id,
+      institution: account.institution,
+      mask: account.iban.slice(-4),
+      name: [account.product, printIban(account)].join(' ').trim(),
+      official_name: account.product,
+      type: 'checking'
+    };
+  },
+
+  sortTransactions(transactions = []) {
+    return transactions.sort((a, b) => {
+      return (
+        Number(b.transactionId.substr(2)) - Number(a.transactionId.substr(2))
+      );
+    });
+  },
+
+  calculateStartingBalance(sortedTransactions = [], balances = []) {
+    if (sortedTransactions.length) {
+      const oldestTransaction =
+        sortedTransactions[sortedTransactions.length - 1];
+      const oldestKnownBalance = amountToInteger(
+        oldestTransaction.balanceAfterTransaction.balanceAmount.amount
+      );
+      const oldestTransactionAmount = amountToInteger(
+        oldestTransaction.transactionAmount.amount
+      );
+
+      return oldestKnownBalance - oldestTransactionAmount;
+    } else {
+      return amountToInteger(
+        balances.find((balance) => 'interimBooked' === balance.balanceType)
+          .balanceAmount.amount
+      );
+    }
+  }
+};