Add new endpoints to api-server

modules/demo/api-server/index.js

@@ -29,5 +29,5 @@ const env = {
 };
 
 spawnSync(process.execPath,
-          [fastify, 'start', '-l', 'info', '-P', '-p', '3010', '-w', 'app.js'],
+          [fastify, 'start', '-l', 'info', '--ignore-watch', '-P', '-p', '3010', 'app.js'],
           {env, cwd: __dirname, stdio: 'inherit'});

modules/demo/api-server/package.json

@@ -12,8 +12,8 @@
   "description": "A fastify-based web server that provides browser-access to GPU resources.",
   "scripts": {
     "test": "tap \"test/**/*.test.js\"",
-    "start": "fastify start -l info -P -p 3010 -w app.js",
-    "dev": "fastify start -w -l info -P -p 3010 app.js"
+    "start": "fastify start --ignore-watch -l info -P -p 3010 app.js",
+    "dev": "fastify start --ignore-watch -l info -P -p 3010 app.js"
   },
   "keywords": [
     "rapids.ai",

modules/demo/api-server/routes/gpu/index.js

@@ -0,0 +1,122 @@
+// Copyright (c) 2022, NVIDIA CORPORATION.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+const {Utf8String, Int32, Uint32, Float32, DataFrame, Series, Float64} = require('@rapidsai/cudf');
+const {RecordBatchStreamWriter, Field, Vector, List}                   = require('apache-arrow');
+const Path                                                             = require('path');
+const {promisify}                                                      = require('util');
+const Fs                                                               = require('fs');
+const Stat                                                             = promisify(Fs.stat);
+const fastifyCors                                                      = require('@fastify/cors');
+const fastify = require('fastify')({logger: true});
+
+const arrowPlugin = require('fastify-arrow');
+const gpu_cache   = require('../../util/gpu_cache.js');
+const root_schema = require('../../util/schema.js');
+
+module.exports = async function(fastify, opts) {
+  fastify.register(arrowPlugin);
+  fastify.register(fastifyCors, {origin: '*'});
+  fastify.decorate('cacheObject', gpu_cache.cacheObject);
+  fastify.decorate('getData', gpu_cache.getData);
+  fastify.decorate('readCSV', gpu_cache.readCSV);
+  fastify.decorate('publicPath', gpu_cache.publicPath);
+
+  const get_schema = {
+    logLevel: 'debug',
+    schema: {
+      response: {
+        200: {
+          type: 'object',
+          properties:
+            {success: {type: 'boolean'}, message: {type: 'string'}, params: {type: 'string'}}
+        }
+      }
+    }
+  };
+
+  fastify.get('/', {...get_schema, handler: () => root_schema['gpu']});
+
+  fastify.route({
+    method: 'POST',
+    url: '/DataFrame/readCSV',
+    schema: {},
+    handler: async (request, reply) => {
+      let message = 'Error';
+      let result  = {'params': request.body, success: false, message: message};
+      try {
+        const path             = Path.join(fastify.publicPath(), request.body.filename);
+        const stats            = await Stat(path);
+        const message          = 'File is available';
+        const currentDataFrame = await fastify.getData(request.body.filename);
+        if (currentDataFrame !== undefined) {
+          console.log('Found existing dataframe.');
+          console.log(request.body);
+          console.log(currentDataFrame);
+          currentDataFrame.dispose();
+        }
+        const cacheObject = await fastify.readCSV({
+          header: 0,
+          sourceType: 'files',
+          sources: [path],
+        });
+        const name        = request.body.filename;
+        await fastify.cacheObject(name, cacheObject);
+        result.success    = true;
+        result.message    = 'CSV file in GPU memory.';
+        result.statusCode = 200;
+        await reply.code(200).send(result);
+      } catch (e) {
+        result.message = e.message;
+        if (e.message.search('no such file or directory') !== -1) {
+          await reply.code(404).send(result);
+        } else {
+          await reply.code(500).send(result);
+        }
+      }
+    }
+  });
+
+  fastify.route({
+    method: 'GET',
+    url: '/get_column/:table/:column',
+    schema: {querystring: {table: {type: 'string'}, 'column': {type: 'string'}}},
+    handler: async (request, reply) => {
+      let message = 'Error';
+      let result  = {'params': JSON.stringify(request.params), success: false, message: message};
+      const table = await fastify.getData(request.params.table);
+      if (table == undefined) {
+        result.message = 'Table not found';
+        await reply.code(404).send(result);
+      } else {
+        try {
+          const name        = request.params.column;
+          const column      = table.get(name);
+          const newDfObject = {};
+          newDfObject[name] = column;
+          const result      = new DataFrame(newDfObject);
+          const writer      = RecordBatchStreamWriter.writeAll(result.toArrow());
+          await reply.code(200).send(writer.toNodeStream());
+        } catch (e) {
+          if (e.message.search('Unknown column name') != -1) {
+            result.message = e;
+            await reply.code(404).send(result);
+          } else {
+            await reply.code(500).send(result);
+          }
+        }
+      }
+    }
+  });
+}

modules/demo/api-server/routes/graphology/index.js

@@ -28,13 +28,13 @@ const root_schema = require('../../util/schema.js');
 module.exports = async function(fastify, opts) {
   fastify.register(arrowPlugin);
   fastify.register(fastifyCors, {origin: 'http://localhost:3001'});
-  fastify.decorate('setDataframe', gpu_cache.setDataframe);
-  fastify.decorate('getDataframe', gpu_cache.getDataframe);
+  fastify.decorate('cacheObject', gpu_cache.cacheObject);
+  fastify.decorate('getData', gpu_cache.getData);
   fastify.decorate('listDataframes', gpu_cache.listDataframes);
   fastify.decorate('readGraphology', gpu_cache.readGraphology);
   fastify.decorate('readLargeGraphDemo', gpu_cache.readLargeGraphDemo);
   fastify.decorate('clearDataFrames', gpu_cache.clearDataframes);
-  fastify.get('/', async function(request, reply) { return root_schema; });
+  fastify.get('/', async function(request, reply) { return root_schema['graphology']; });
 
   fastify.route({
     method: 'POST',
@@ -76,9 +76,9 @@ module.exports = async function(fastify, opts) {
           result.success = true;
           try {
             const graphology = await fastify.readLargeGraphDemo(path);
-            await fastify.setDataframe('nodes', graphology['nodes']);
-            await fastify.setDataframe('edges', graphology['edges']);
-            await fastify.setDataframe('options', graphology['options']);
+            await fastify.cacheObject('nodes', graphology['nodes']);
+            await fastify.cacheObject('edges', graphology['edges']);
+            await fastify.cacheObject('options', graphology['options']);
             result.message = 'File read onto GPU.';
           } catch (e) {
             result.success    = false;
@@ -135,10 +135,10 @@ module.exports = async function(fastify, opts) {
           result.success = true;
           try {
             const graphology = await fastify.readGraphology(path);
-            await fastify.setDataframe('nodes', graphology['nodes']);
-            await fastify.setDataframe('edges', graphology['edges']);
-            await fastify.setDataframe('clusters', graphology['clusters']);
-            await fastify.setDataframe('tags', graphology['tags']);
+            await fastify.cacheObject('nodes', graphology['nodes']);
+            await fastify.cacheObject('edges', graphology['edges']);
+            await fastify.cacheObject('clusters', graphology['clusters']);
+            await fastify.cacheObject('tags', graphology['tags']);
             result.message = 'File read onto GPU.';
           } catch (e) {
             result.success    = false;
@@ -173,7 +173,7 @@ module.exports = async function(fastify, opts) {
     handler: async (request, reply) => {
       let message = 'Error';
       let result  = {'params': JSON.stringify(request.params), success: false, message: message};
-      const table = await fastify.getDataframe(request.params.table);
+      const table = await fastify.getData(request.params.table);
       if (table == undefined) {
         result.message = 'Table not found';
         await reply.code(404).send(result);
@@ -208,7 +208,7 @@ module.exports = async function(fastify, opts) {
     handler: async (request, reply) => {
       let message = 'Error';
       let result  = {'params': JSON.stringify(request.params), success: false, message: message};
-      const table = await fastify.getDataframe(request.params.table);
+      const table = await fastify.getData(request.params.table);
       if (table == undefined) {
         result.message = 'Table not found';
         await reply.code(404).send(result);
@@ -225,7 +225,7 @@ module.exports = async function(fastify, opts) {
     handler: async (request, reply) => {
       let message = 'Error';
       let result  = {success: false, message: message};
-      const df    = await fastify.getDataframe('nodes');
+      const df    = await fastify.getData('nodes');
       if (df == undefined) {
         result.message = 'Table not found';
         await reply.code(404).send(result);
@@ -250,7 +250,7 @@ module.exports = async function(fastify, opts) {
     handler: async (request, reply) => {
       let message = 'Error';
       let result  = {success: false, message: message};
-      const df    = await fastify.getDataframe('nodes');
+      const df    = await fastify.getData('nodes');
       if (df == undefined) {
         result.message = 'Table not found';
         await reply.code(404).send(result);
@@ -290,9 +290,9 @@ module.exports = async function(fastify, opts) {
       let message = 'Error';
       let result  = {success: false, message: message};
       /** @type DataFrame<{x: Float32, y: Float32}> */
-      const df = await fastify.getDataframe('nodes');
+      const df = await fastify.getData('nodes');
       /** @type DataFrame<{x: Int32, y: Int32}> */
-      const edges = await fastify.getDataframe('edges');
+      const edges = await fastify.getData('edges');
       if (df == undefined) {
         result.message = 'Table not found';
         await reply.code(404).send(result);

modules/demo/api-server/routes/particles/index.js

@@ -0,0 +1,138 @@
+// Copyright (c) 2022, NVIDIA CORPORATION.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+const {Int32, Float32, DataFrame, Series} = require('@rapidsai/cudf');
+const {RecordBatchStreamWriter}           = require('apache-arrow');
+const fastify                             = require('fastify')({logger: {level: 'debug'}});
+const fastifyCors                         = require('@fastify/cors');
+
+const arrowPlugin = require('fastify-arrow');
+const gpu_cache   = require('../../util/gpu_cache.js');
+const root_schema = require('../../util/schema.js');
+
+module.exports = async function(fastify, opts) {
+  fastify.register(fastifyCors, {origin: '*'});
+  fastify.register(arrowPlugin);
+  fastify.decorate('cacheObject', gpu_cache.cacheObject);
+  fastify.decorate('getData', gpu_cache.getData);
+  fastify.decorate('readCSV', gpu_cache.readCSV);
+
+  const get_schema = {
+    logLevel: 'debug',
+    schema: {
+      response: {
+        200: {
+          type: 'object',
+          properties:
+            {success: {type: 'boolean'}, message: {type: 'string'}, params: {type: 'string'}}
+        }
+      }
+    }
+  };
+
+  fastify.get('/', {...get_schema, handler: () => root_schema['particles']});
+  fastify.post('/', {...get_schema, handler: () => root_schema['particles']});
+
+  const filterPoints =
+    (column, min, max) => {
+      const gt   = column._col.gt(parseInt(min));
+      const lt   = column._col.lt(parseInt(max));
+      const mask = gt.bitwiseAnd(lt);
+      return column.filter(Series.new(mask));
+    }
+
+  const handler = async (request, reply) => {
+    let message = 'Error';
+    let result  = {'params': request.params, success: false, message: message};
+    const table = await fastify.getData(request.params.table);
+    if (table == undefined) {
+      result.message = 'Table not found';
+      await reply.code(404).send(result);
+    } else {
+      try {
+        let x = undefined;
+        let y = undefined;
+        if (request.params.xmin != undefined && request.params.xmax != undefined &&
+            request.params.ymin != undefined && request.params.ymax != undefined) {
+          x = filterPoints(table.get('Longitude'), request.params.xmin, request.params.xmax);
+          y = filterPoints(table.get('Latitude'), request.params.ymin, request.params.ymax);
+        } else {
+          x = table.get('Longitude');
+          y = table.get('Latitude');
+        }
+
+        // Map x, y, r, g, b to offsets for client display
+        let tiled       = Series.sequence({type: new Float32, init: 0.0, size: (2 * x.length)});
+        let base_offset = Series.sequence({type: new Int32, init: 0.0, size: x.length}).mul(2);
+        tiled           = tiled.scatter(x, base_offset.cast(new Int32));
+        x.dispose();
+        tiled = tiled.scatter(y, base_offset.add(1).cast(new Int32));
+        y.dispose();
+        const result = new DataFrame({'gpu_buffer': tiled});
+        const writer = RecordBatchStreamWriter.writeAll(result.toArrow());
+        await reply.code(200).send(writer.toNodeStream());
+        tiled.dispose();
+        result.dispose();
+        writer.close();
+      } catch (e) {
+        result.message = e.message;
+        if (e.message.search('Unknown column name') != -1) {
+          result.message = {
+            error: result.message,
+            message:
+              'Imported CSV file must contain four columns: State, Zip_Code, Longitude, and Latitude'
+          };
+          await reply.code(500).send(result);
+        } else {
+          await reply.code(500).send(result);
+        }
+      }
+    }
+  };
+
+  fastify.route({
+    method: 'GET',
+    url: '/get_shader_column/:table/:xmin/:xmax/:ymin/:ymax',
+    schema: {
+      querystring: {
+        table: {type: 'string'},
+        xmin: {type: 'number'},
+        xmax: {type: 'number'},
+        ymin: {type: 'number'},
+        ymax: {type: 'number'}
+      }
+    },
+    handler: handler
+  });
+  fastify.route({
+    method: 'GET',
+    url: '/get_shader_column/:table',
+    schema: {querystring: {table: {type: 'string'}}},
+    handler: handler
+  });
+  fastify.route({
+    method: 'POST',
+    url: '/quadtree/create/:table',
+    schema: {querystring: {table: {type: 'string'}}},
+    handler: async (request, reply) => {
+      let message = 'Error';
+      let result  = {'params': request.params, success: false, message: message};
+      const table = await fastify.getData(request.params.table);
+      if (table == undefined) {
+        result.message = 'Table not found';
+        await reply.code(404).send(result);
+      }
+    }
+  });
+}