Support MySQL & PostgreSQL stores

examples/graphql/package.json

@@ -14,6 +14,7 @@
   },
   "dependencies": {
     "@neuledge/engine": "^0.2.1",
+    "@neuledge/postgresql-store": "*",
     "@neuledge/mongodb-store": "^0.2.0",
     "dotenv": "^16.0.3",
     "fastify": "^4.14.1",

packages/engine/README.md

@@ -12,8 +12,9 @@
 </p>
 <p align=center>
   <a href="https://www.npmjs.com/package/@neuledge/mongodb-store">MongoDB</a> ⇄ 
-  MySQL (<a href="https://github.com/neuledge/engine-js/pull/20">soon</a>) ⇄ 
-  PostgreSQL (<a href="https://github.com/neuledge/engine-js/pull/20">soon</a>)
+  <a href="https://www.npmjs.com/package/@neuledge/mysql-store">MySQL</a> ⇄ 
+  <a href="https://www.npmjs.com/package/@neuledge/postgresql-store">PostgreSQL</a> ⇄ 
+  Your DB (<a href="https://github.com/neuledge/engine-js/issues/new">request</a>)
 </p>
 <br>
 <p align="center">

packages/engine/src/engine/exec/alter.test.ts

@@ -63,13 +63,11 @@ describe('engine/exec/alter', () => {
         collection: metadata['collections']['categories'],
         where: { id: { $eq: 1 }, __h: { $eq: hash }, __v: { $eq: 0 } },
         set: { name: 'foo', description: 'bar', __v: 1 },
-        limit: 1,
       });
       expect(update).toHaveBeenNthCalledWith(2, {
         collection: metadata['collections']['categories'],
         where: { id: { $eq: 2 }, __h: { $eq: hash }, __v: { $eq: 0 } },
         set: { name: 'foo', description: 'bar', __v: 1 },
-        limit: 1,
       });
     });
 
@@ -129,13 +127,11 @@ describe('engine/exec/alter', () => {
         collection: metadata['collections']['categories'],
         where: { id: { $eq: 1 }, __h: { $eq: hash }, __v: { $eq: 0 } },
         set: { name: 'foo', description: 'bar', __v: 1 },
-        limit: 1,
       });
       expect(update).toHaveBeenNthCalledWith(2, {
         collection: metadata['collections']['categories'],
         where: { id: { $eq: 2 }, __h: { $eq: hash }, __v: { $eq: 0 } },
         set: { name: 'foo', description: 'bar', __v: 1 },
-        limit: 1,
       });
     });
 
@@ -182,13 +178,11 @@ describe('engine/exec/alter', () => {
         collection: metadata['collections']['categories'],
         where: { id: { $eq: 1 }, __h: { $eq: hash }, __v: { $eq: 0 } },
         set: { name: 'foo', description: 'bar', __v: 1 },
-        limit: 1,
       });
       expect(update).toHaveBeenNthCalledWith(2, {
         collection: metadata['collections']['categories'],
         where: { id: { $eq: 2 }, __h: { $eq: hash }, __v: { $eq: 0 } },
         set: { name: 'foo', description: 'bar', __v: 1 },
-        limit: 1,
       });
     });
 
@@ -264,13 +258,11 @@ describe('engine/exec/alter', () => {
         collection: metadata['collections']['posts'],
         where: { id: { $eq: 1 }, __h: { $eq: hash }, __v: { $eq: 0 } },
         set: { title: 'foo', content: 'bar', category_id: 1, __v: 1 },
-        limit: 1,
       });
       expect(update).toHaveBeenNthCalledWith(2, {
         collection: metadata['collections']['posts'],
         where: { id: { $eq: 2 }, __h: { $eq: hash }, __v: { $eq: 0 } },
         set: { title: 'foo', content: 'bar', category_id: 1, __v: 1 },
-        limit: 1,
       });
     });
   });
@@ -376,7 +368,6 @@ describe('engine/exec/alter', () => {
         collection: metadata['collections']['categories'],
         where: { id: { $eq: 1 }, __h: { $eq: hash }, __v: { $eq: 0 } },
         set: { name: 'foo', description: 'bar', __v: 1 },
-        limit: 1,
       });
     });
 
@@ -454,7 +445,6 @@ describe('engine/exec/alter', () => {
         collection: metadata['collections']['categories'],
         where: { id: { $eq: 1 }, __h: { $eq: hash }, __v: { $eq: 0 } },
         set: { name: 'foo', description: 'bar', __v: 1 },
-        limit: 1,
       });
     });
 

packages/engine/src/engine/metadata/collections.ts

@@ -19,6 +19,8 @@ const ensureStoreCollection = async (
   store: Store,
   collection: MetadataCollection,
 ): Promise<void> => {
+  // FIXME how we handle fields or indexes changes? (e.g. a field changed to be nullable)
+
   await store.ensureCollection({
     collection,
     indexes: Object.values(collection.indexes),

packages/engine/src/engine/metadata/load.ts

@@ -4,10 +4,10 @@ import { Store } from '@neuledge/store';
 import { ensureStoreCollections } from './collections';
 import {
   ensureMetadataCollection,
-  getMetadataCollection,
   getStoreMetadataSnapshot,
   syncStoreMetadata,
 } from './store';
+import { getMetadataCollection } from './state';
 
 const DEFAULT_METADATA_COLLECTION_NAME = '__neuledge_metadata';
 

packages/engine/src/engine/metadata/state.ts

@@ -1,12 +1,14 @@
+import { StoreCollection, StoreField, StorePrimaryKey } from '@neuledge/store';
 import { NeuledgeError } from '@/error';
 import {
   StateSnapshot,
   StateFieldSnapshot,
   StateRelationSnapshot,
+  METADATA_HASH_BYTES,
 } from '@/metadata';
 
 export interface StoreMetadataState {
-  collectionName: string;
+  collection_name: string;
   name: string;
   hash: Buffer;
   fields: StoreMetadataStateField[];
@@ -32,6 +34,38 @@ interface StoreMetadataStateRelation {
   index: number;
 }
 
+// FIXME we can't save buffers on json fields. We need to encode them somehow or use relations for `fields` and `relations` fields.
+
+export const getMetadataCollection = (
+  metadataCollectionName: string,
+): StoreCollection => {
+  const hash: StoreField = {
+    name: 'hash',
+    type: 'binary',
+    size: METADATA_HASH_BYTES,
+  };
+
+  const primaryKey: StorePrimaryKey = {
+    name: 'hash',
+    fields: { [hash.name]: { sort: 'asc' } },
+    unique: 'primary',
+  };
+
+  return {
+    name: metadataCollectionName,
+    primaryKey,
+    indexes: { [primaryKey.name]: primaryKey },
+    fields: {
+      [hash.name]: hash,
+      collection_name: { name: 'collection_name', type: 'string' },
+      name: { name: 'name', type: 'string' },
+      fields: { name: 'fields', type: 'json', list: true },
+      relations: { name: 'relations', type: 'json', list: true },
+      v: { name: 'v', type: 'number', unsigned: true, scale: 0, precision: 4 },
+    },
+  };
+};
+
 export const fromStoreMetadataState = (
   getState: (hash: Buffer) => StateSnapshot,
   getType: (key: string) => StateFieldSnapshot['type'],
@@ -45,7 +79,7 @@ export const fromStoreMetadataState = (
   }
 
   return getState(doc.hash).assign({
-    collectionName: doc.collectionName,
+    collectionName: doc.collection_name,
     name: doc.name,
     hash: doc.hash,
     fields: doc.fields.map((field) =>
@@ -60,7 +94,7 @@ export const fromStoreMetadataState = (
 export const toStoreMetadataState = (
   state: StateSnapshot,
 ): StoreMetadataState => ({
-  collectionName: state.collectionName,
+  collection_name: state.collectionName,
   name: state.name,
   hash: state.hash,
   fields: state.fields.map((field) => toStoreMetadataStateField(field)),

packages/engine/src/engine/metadata/store.ts

@@ -1,13 +1,7 @@
 import { NeuledgeError } from '@/error';
-import { MetadataChange, StateSnapshot, METADATA_HASH_BYTES } from '@/metadata';
+import { MetadataChange, StateSnapshot } from '@/metadata';
 import { MetadataSnapshot } from '@/metadata/snapshot';
-import {
-  Store,
-  StoreCollection,
-  StoreField,
-  StoreList,
-  StorePrimaryKey,
-} from '@neuledge/store';
+import { Store, StoreCollection, StoreList } from '@neuledge/store';
 import pLimit from 'p-limit';
 import {
   fromStoreMetadataState,
@@ -16,34 +10,7 @@ import {
 } from './state';
 
 const HASH_ENCODING = 'base64url';
-const COLLECTION_FIND_LIMIT = 1000;
-
-export const getMetadataCollection = (
-  metadataCollectionName: string,
-): StoreCollection => {
-  const hash: StoreField = {
-    name: 'hash',
-    type: 'binary',
-    size: METADATA_HASH_BYTES,
-  };
-
-  const primaryKey: StorePrimaryKey = {
-    name: 'hash',
-    fields: { [hash.name]: { sort: 'asc' } },
-    unique: 'primary',
-  };
-
-  return {
-    name: metadataCollectionName,
-    primaryKey,
-    indexes: { [primaryKey.name]: primaryKey },
-    fields: {
-      [hash.name]: hash,
-      key: { name: 'key', type: 'string' },
-      payload: { name: 'payload', type: 'json' },
-    },
-  };
-};
+const COLLECTION_FIND_LIMIT = 100;
 
 export const ensureMetadataCollection = async (
   store: Store,
@@ -140,7 +107,6 @@ export const syncStoreMetadata = async (
             collection: metadataCollection,
             where: { hash: { $eq: hash } },
             set: set as never,
-            limit: 1,
           }),
         ),
       ),

packages/engine/src/engine/mutations/store.ts

@@ -58,7 +58,6 @@ const updateStoreDocument = async (
     collection,
     where: getWhereRecordByPrimaryKeys(collection, document),
     set: Object.fromEntries(setEntries),
-    limit: 1,
   });
 
   return !!res.affectedCount;
@@ -74,7 +73,6 @@ const deleteStoreDocuments = async (
   await store.delete({
     collection,
     where: getWhereByPrimaryKeys(collection, documents),
-    limit: documents.length,
   });
 };
 
@@ -86,7 +84,6 @@ const deleteStoreDocuments = async (
 //   await store.delete({
 //     collectionName: collection.name,
 //     where: getWhereRecord(collection.primaryKeys, document),
-//     limit: 1,
 //   });
 // };
 

packages/mongodb-store/README.md

@@ -14,7 +14,7 @@ npm install @neuledge/mongodb-store
 import { Engine } from '@neuledge/engine';
 import { MongoDBStore } from '@neuledge/mongodb-store';
 
-const store = store: new MongoDBStore({
+const store = new MongoDBStore({
   url: process.env.MONGODB_URL ?? 'mongodb://localhost:27017',
   name: process.env.MONGODB_DATABASE ?? 'my-database',
 });

packages/mongodb-store/src/indexes.ts

@@ -1,20 +1,26 @@
-import { StoreIndex, StorePrimaryKey } from '@neuledge/store';
+import { StoreIndex, StorePrimaryKey, throwStoreError } from '@neuledge/store';
 import { Collection } from 'mongodb';
 import { escapeFieldName } from './fields';
 
 export const dropIndexes = async (
   collection: Collection,
   indexes: string[],
 ): Promise<void> => {
-  await Promise.all(indexes.map((index) => collection.dropIndex(index)));
+  await Promise.all(indexes.map((index) => collection.dropIndex(index))).catch(
+    throwStoreError,
+  );
 };
 
 export const ensureIndexes = async (
   primaryKey: StorePrimaryKey,
   collection: Collection,
   indexes: StoreIndex[],
 ): Promise<void> => {
-  const exists = await collection.listIndexes().toArray();
+  const exists = await collection
+    .listIndexes()
+    .toArray()
+    .catch(throwStoreError);
+
   const existMap = new Map(exists.map((item) => [item.name, item]));
 
   for (const index of indexes) {
@@ -39,11 +45,13 @@ export const ensureIndexes = async (
     // documents that don't have the indexed fields. This maintains the same
     // behavior with relational databases where NULL values are not indexed.
 
-    await collection.createIndex(indexSpec, {
-      name: index.name,
-      unique: !!index.unique,
-      sparse: true,
-      background: true,
-    });
+    await collection
+      .createIndex(indexSpec, {
+        name: index.name,
+        unique: !!index.unique,
+        sparse: true,
+        background: true,
+      })
+      .catch(throwStoreError);
   }
 };

packages/mongodb-store/src/inserted-ids.ts

@@ -1,4 +1,4 @@
-import { StoreCollection, StoreError } from '@neuledge/store';
+import { StoreCollection, StoreError, throwStoreError } from '@neuledge/store';
 import { Collection } from 'mongodb';
 
 export interface AutoIncrementDocument {
@@ -56,17 +56,19 @@ const autoIncrementPrimaryKey = async (
   autoIncrement: Collection<AutoIncrementDocument>,
   collectionName: string,
 ): Promise<number> => {
-  const { value: doc } = await autoIncrement.findOneAndUpdate(
-    {
-      _id: collectionName,
-    },
-    {
-      $inc: { value: 1 },
-    },
-    {
-      upsert: true,
-    },
-  );
+  const { value: doc } = await autoIncrement
+    .findOneAndUpdate(
+      {
+        _id: collectionName,
+      },
+      {
+        $inc: { value: 1 },
+      },
+      {
+        upsert: true,
+      },
+    )
+    .catch(throwStoreError);
 
   return (doc?.value ?? 0) + 1;
 };