collections: updates

packages/collections/package.json

@@ -12,10 +12,9 @@
     "./package.json": "./package.json"
   },
   "scripts": {
-    "tmp": "node test.js",
     "build": "pnpm clean && build-adaptor collections",
-    "test": "mocha --experimental-specifier-resolution=node --no-warnings **/*.test.js",
-    "test:watch": "mocha -w --experimental-specifier-resolution=node --no-warnings *.test.js",
+    "test": "mocha --experimental-specifier-resolution=node --no-warnings --recursive",
+    "test:watch": "mocha -w --experimental-specifier-resolution=node --no-warnings --recursive",
     "clean": "rimraf dist types docs",
     "pack": "pnpm pack --pack-destination ../../dist",
     "lint": "eslint src"

packages/collections/src/collections.js

@@ -58,8 +58,11 @@ export const setMockClient = mockClient => {
  */
 export function get(name, query = {}) {
   return async state => {
-    const [resolvedName, resolvedQuery] = expandReferences(state, name, query);
-
+    let [resolvedName, resolvedQuery] = expandReferences(state, name, query);
+    // TODO need a test on this
+    if (typeof resolvedQuery === 'string') {
+      resolvedQuery = { key: resolvedQuery };
+    }
     const { key, ...rest } = expandQuery(resolvedQuery);
 
     // TODO maybe add query options here
@@ -72,17 +75,23 @@ export function get(name, query = {}) {
     );
 
     let data;
+    // TODO unit tests on json encoding please
     if (!key.match(/\*/) || Object.keys(resolvedQuery).length === 0) {
       // If one specific item was requested, write it straight to state.data
-      [data] = (await response.body.json()).items;
-      data = JSON.parse(data);
+      const body = await response.body.json();
+      const item = body.items[0];
+      if (item) {
+        item.value = JSON.parse(item.value);
+      }
+      data = item;
       console.log(`Fetched "${key}" from collection "${name}"`);
     } else {
       // build a response array
       data = [];
       console.log(`Downloading data from collection "${name}"...`);
       await streamResponse(response, item => {
-        data.push(JSON.parse(item));
+        item.value = JSON.parse(item.value);
+        data.push(item);
       });
       console.log(`Fetched "${data.length}" values from collection "${name}"`);
     }
@@ -149,7 +158,7 @@ export function set(name, keyGen, values) {
     }
 
     const result = await response.body.json();
-    console.log(`Set ${result.upserts} values in collection "${name}"`);
+    console.log(`Set ${result.upserted} values in collection "${name}"`);
     if (result.error) {
       console.log(`Errors reported on set:`, result.error);
     }
@@ -190,9 +199,7 @@ export function remove(name, query = {}, options = {}) {
         query: rest,
       }
     );
-    console.log(response.statusCode);
     const result = await response.body.json();
-    console.log(result);
     console.log(`Set ${result.upserts} values in collection "${name}"`);
 
     return state;
@@ -229,8 +236,7 @@ export function each(name, query = {}, callback = () => {}) {
     const response = await request(
       state,
       getClient(state),
-      //`${resolvedName}/${key}`,
-      resolvedName,
+      `${resolvedName}/${key}`,
       { query: rest }
     );
 

packages/collections/src/mock.js

@@ -16,6 +16,8 @@ export function API() {
     collections = api.collections = {};
   };
 
+  // Note that the mock allows data in any format,
+  // but the real API only takes strings
   const upsert = (name, key, value) => {
     if (!(name in collections)) {
       throw new Error(COLLECTION_NOT_FOUND);
@@ -49,6 +51,9 @@ export function API() {
   const byKey = (name, key) => {
     return collections[name][key];
   };
+  const asJSON = (name, key) => {
+    return JSON.parse(collections[name][key]);
+  };
 
   // TODO strictly speaking this should support patterns
   // but keeping it super simple in the mock for now
@@ -59,6 +64,8 @@ export function API() {
     const col = collections[name];
 
     delete col[key];
+
+    return [key];
   };
 
   const api = {
@@ -69,13 +76,17 @@ export function API() {
     fetch,
     remove,
     byKey,
+    asJSON,
   };
 
   return api;
 }
 
 // naive little path parser
 const parsePath = path => {
+  if (!path.startsWith('/')) {
+    path = `/${path}`;
+  }
   let [_, _collections, name, key] = path.split('/');
   return { name, key };
 };
@@ -89,8 +100,6 @@ const assertAuth = req => {
 
 // This creates a mock lightning server
 // It should present the same rest API as lightning, but can be implemented however we like
-// TODO add mock auth here
-// basically it needs to see if there is a jwt in the header for each request
 export function createServer(url = 'https://app.openfn.org') {
   const agent = new MockAgent();
   agent.disableNetConnect();
@@ -134,17 +143,26 @@ export function createServer(url = 'https://app.openfn.org') {
       return { statusCode: 403 };
     }
 
+    let upserted = 0;
+    const errors = [];
+
     try {
       const { name, key } = parsePath(req.path);
       const body = JSON.parse(req.body);
 
       for (const { key, value } of body.items) {
         // TODO error if key or value not set
         api.upsert(name, key, value);
+        upserted++;
       }
 
-      // TODO return upserted summary and errors
-      return { statusCode: 200 };
+      return {
+        statusCode: 200,
+        responseOptions: {
+          headers: { 'Content-Type': 'application/json' },
+        },
+        data: JSON.stringify({ upserted, errors }),
+      };
     } catch (e) {
       if (e.message === COLLECTION_NOT_FOUND) {
         return { statusCode: 404 };
@@ -162,9 +180,15 @@ export function createServer(url = 'https://app.openfn.org') {
     try {
       const { name, key } = parsePath(req.path);
 
-      api.remove(name, key);
+      const keys = api.remove(name, key);
 
-      return { statusCode: 200 };
+      return {
+        statusCode: 200,
+        responseOptions: {
+          headers: { 'Content-Type': 'application/json' },
+        },
+        data: JSON.stringify({ deleted: keys.length, keys }),
+      };
     } catch (e) {
       if (e.message === COLLECTION_NOT_FOUND) {
         return { statusCode: 404 };

packages/collections/test/Adaptor.test.js

@@ -20,10 +20,10 @@ const init = items => {
 
   if (items) {
     for (const [key, value] of items) {
-      api.upsert(COLLECTION, key, value);
+      api.upsert(COLLECTION, key, JSON.stringify(value));
     }
   } else {
-    api.upsert(COLLECTION, 'x', { id: 'x' });
+    api.upsert(COLLECTION, 'x', JSON.stringify({ id: 'x' }));
   }
 
   const state = {
@@ -62,7 +62,7 @@ describe('each', () => {
       count++;
       expect(state).to.eql(state);
 
-      const item = api.byKey(COLLECTION, key);
+      const item = JSON.parse(api.byKey(COLLECTION, key));
       expect(item).not.to.be.undefined;
       expect(item).to.eql(value);
     })(state);
@@ -221,7 +221,7 @@ describe('set', () => {
 
     await collections.set(COLLECTION, key, item)(state);
 
-    const result = api.byKey(COLLECTION, key);
+    const result = api.asJSON(COLLECTION, key);
     expect(result).to.eql(item);
   });
 
@@ -233,11 +233,11 @@ describe('set', () => {
 
     await collections.set(COLLECTION, keygen, items)(state);
 
-    const x = api.byKey(COLLECTION, items[0].key);
-    expect(x).to.eql(items[0].value);
+    const x = api.asJSON(COLLECTION, items[0].id);
+    expect(x).to.eql(items[0]);
 
-    const y = api.byKey(COLLECTION, items[1].key);
-    expect(y).to.eql(items[1].value);
+    const y = api.asJSON(COLLECTION, items[1].id);
+    expect(y).to.eql(items[1]);
   });
 });
 
@@ -262,7 +262,7 @@ describe('remove', () => {
     await collections.remove(COLLECTION, 'x')(state);
 
     const result = api.byKey(COLLECTION, 'x');
-    expect(result).to.eql(undefined);
+    expect(result).to.be.undefined;
   });
 });
 

packages/collections/test/mock/api.test.js

@@ -76,7 +76,7 @@ describe('API', () => {
 
     const result = api.fetch('a', 'x', {});
 
-    expect(result).to.eql([{ id: 1 }]);
+    expect(result.items).to.eql([{ key: 'x', value: { id: 1 } }]);
   });
 
   it('should fetch from a collection with wildcard', () => {
@@ -87,9 +87,13 @@ describe('API', () => {
     api.upsert('a', 'axb', { id: 3 });
     api.upsert('a', 'yy', { id: 4 });
 
-    const { results } = api.fetch('a', 'x*', {});
+    const { items } = api.fetch('a', 'x*', {});
 
-    expect(results).to.eql([{ id: 1 }, { id: 2 }, { id: 3 }]);
+    expect(items).to.eql([
+      { key: 'x', value: { id: 1 } },
+      { key: 'xx', value: { id: 2 } },
+      { key: 'axb', value: { id: 3 } },
+    ]);
   });
 
   it('should remove from a collection', () => {

packages/collections/test/mock/server.test.js

@@ -1,9 +1,10 @@
 import { expect } from 'chai';
 import { createServer } from '../../src/mock';
-import { streamResponse } from '../../src/utils';
+import { streamResponse } from '../../src/collections';
 
 let request;
 let api;
+
 beforeEach(() => {
   ({ api, request } = createServer());
 });
@@ -106,7 +107,7 @@ describe('POST', () => {
     const response = await request({
       method: 'POST',
       path: 'collections/my-collection',
-      data: [],
+      data: { items: [] },
     });
     expect(response.statusCode).to.equal(200);
   });
@@ -116,7 +117,7 @@ describe('POST', () => {
     const response = await request({
       method: 'POST',
       path: 'collections/my-collection',
-      data: [{}],
+      data: { items: [{}] },
     });
     expect(response.statusCode).to.equal(404);
   });
@@ -140,14 +141,31 @@ describe('POST', () => {
     const response = await request({
       method: 'POST',
       path: 'collections/my-collection',
-      data: [item],
+      data: { items: [item] },
     });
 
     expect(response.statusCode).to.equal(200);
 
     const result = api.byKey('my-collection', item.key);
     expect(result).to.eql(item.value);
   });
+
+  it('should return a JSON summary', async () => {
+    api.createCollection('my-collection');
+
+    const item = { key: 'x', value: { id: 'x' } };
+
+    const response = await request({
+      method: 'POST',
+      path: 'collections/my-collection',
+      data: { items: [item] },
+    });
+
+    const { upserted, errors } = await response.body.json();
+
+    expect(upserted).to.equal(1);
+    expect(errors).to.eql([]);
+  });
 });
 
 describe('DELETE', () => {
@@ -195,4 +213,19 @@ describe('DELETE', () => {
     const result = api.byKey('my-collection', 'x');
     expect(result).to.be.undefined;
   });
+
+  it('should return a JSON summary', async () => {
+    api.createCollection('my-collection');
+    api.upsert('my-collection', 'x', { id: 'x' });
+
+    const response = await request({
+      method: 'DELETE',
+      path: 'collections/my-collection/x',
+    });
+
+    const { deleted, keys } = await response.body.json();
+
+    expect(deleted).to.equal(1);
+    expect(keys).to.eql(['x']);
+  });
 });