Merge pull request #7 from Quansight/vega-lite

Add preliminary support for vega lite

Dockerfile

@@ -1,7 +1,13 @@
-FROM jupyter/scipy-notebook
+FROM jupyter/scipy-notebook:1085ca054a5f
 
 RUN conda install -y -c conda-forge \
-    jupyterlab=0.32.0 \
-    pymapd
+    pymapd==0.3.2 \
+    altair==2.0.1
 
-RUN pip install vdom altair==2.0.0rc2
+RUN pip install \
+    vdom \
+    git+https://github.com/Quansight/ibis.git@0d1d81400a7a06943f3c99037c348c26942b0ffe
+
+USER root
+RUN echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook
+USER $NB_USER

docker-compose.yml

@@ -4,9 +4,13 @@ services:
     build: .
     environment:
       JUPYTER_ENABLE_LAB: 'true'
+      YARN_CACHE_FOLDER: /yarn-cache
     command: start-notebook.sh --watch
     ports:
       - "8888:8888"
     volumes:
       - ".:/home/jovyan"
       - "./docker-start.sh:/usr/local/bin/start-notebook.d/docker-start.sh"
+      - "yarn-cache:/yarn-cache"
+volumes:
+  yarn-cache:

docker-start.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 set -euxo
 
+sudo chown -R $NB_USER:$NB_GID $YARN_CACHE_FOLDER
+
 jlpm
 jlpm build
 jupyter labextension link --no-build

mapd_renderer.py

@@ -7,6 +7,11 @@
 
 import vdom
 
+try:
+    import altair as alt
+except ImportError:
+    alt = None
+
 from IPython.core.magic import register_cell_magic
 from IPython.display import display
 
@@ -15,11 +20,12 @@ class MapDBackendRenderer:
     Class that produces a mimebundle that the notebook
     mapd renderer can understand.
     """
-    def __init__(self, connection, data):
+    def __init__(self, connection, data=None, vl_data=None):
         """
-        Initialize the backend renderer.
+        Initialize the backend renderer. Either `data` or `vl_data`
+        is required.
 
-        Paramters
+        Parameters
         =========
 
         connection: dict
@@ -29,9 +35,15 @@ def __init__(self, connection, data):
 
         data: dict
             Vega data to render.
+        
+        data: dict
+            Vega lite data to render.
         """
+        if (not (data or vl_data)) or (data and vl_data):
+            raise RuntimeError('Either vega or vega lite data must be specified')
         self.connection = connection
         self.data = data
+        self.vl_data = vl_data
 
     def _repr_mimebundle_(self, include=None, exclude=None):
         """
@@ -40,9 +52,12 @@ def _repr_mimebundle_(self, include=None, exclude=None):
         in Jupyter notebooks.
         """
         bundle = {
-            'connection': self.connection,
-            'vega': self.data
+            'connection': self.connection
         }
+        if self.data:
+            bundle['vega'] = self.data
+        else:
+            bundle['vegalite'] = self.vl_data
         return {
             'application/vnd.mapd.vega+json': bundle
         }
@@ -73,3 +88,39 @@ def mapd(line, cell):
     connection_data = ast.literal_eval(line)
     vega = yaml.load(cell)
     display(MapDBackendRenderer(connection_data, vega))
+
+@register_cell_magic
+def mapd_vl(line, cell):
+    """
+    Cell magic for rendering vega lite produced by the mapd backend.
+
+    Usage: Initiate it with the line `%% mapd $connection_data`,
+    where `connection_data` is the dictionary containing the connection
+    data for the MapD server. The rest of the cell should be yaml-specified
+    vega lite data.
+    """
+    connection_data = ast.literal_eval(line)
+    vl = yaml.load(cell)
+    display(MapDBackendRenderer(connection_data, vl_data=vl))
+
+def mapd_mimetype(spec, conn):
+    """
+    Returns a mapd vega lite mimetype, assuming that the URL
+    for the vega spec is actually the SQL query
+    """
+    data = spec['data']
+    data['sql'] = data.pop('url')
+    return {'application/vnd.mapd.vega+json': {
+        'vegalite': spec,
+        'connection': {
+            'host': conn.host,
+            'protocol': conn.protocol,
+            'port': conn.port,
+            'user': conn.user,
+            'dbName': conn.db_name,
+            'password': conn.password
+        }
+    }}
+
+if alt:
+    alt.renderers.register('mapd', mapd_mimetype)

package.json

@@ -41,7 +41,9 @@
     "@phosphor/coreutils": "^1.3.0",
     "@phosphor/datagrid": "^0.1.5",
     "@phosphor/widgets": "^1.5.0",
-    "mapd-connector": "^1.0.0"
+    "mapd-connector": "^1.0.0",
+    "vega-lite": "2.x",
+    "vega": "3.x"
   },
   "devDependencies": {
     "rimraf": "~2.6.2",

src/mimextension.ts

@@ -18,6 +18,7 @@ import {
   MapDVega
 } from './widget';
 
+import { compileToVega } from './vega-lite';
 
 import '../style/index.css';
 
@@ -62,10 +63,10 @@ class RenderedMapD extends Widget implements IRenderMime.IRenderer {
 
     // Get the data from the mimebundle
     const data = model.data[MIME_TYPE] as IMapDMimeBundle;
-    const { connection, vega } = data;
+    const { connection, vega, vegalite } = data;
 
     // Create a new MapDVega
-    this._widget = new MapDVega(vega, connection);
+    this._widget = new MapDVega(vega || compileToVega(vegalite), connection);
     this.node.appendChild(this._widget.node);
     return this._widget.renderedImage.then(data => {
       // Set the mime data for the png.
@@ -99,7 +100,11 @@ interface IMapDMimeBundle extends JSONObject {
   /**
    * The vega JSON object to render, including the SQL query.
    */
-  vega: JSONObject;
+  vega?: JSONObject;
+  /**
+   * The vega lite JSON object to render, including the SQL query.
+   */
+  vegalite?: JSONObject;
 }
 
 /**

src/vega-lite.ts

@@ -0,0 +1,55 @@
+import { compile } from "vega-lite";
+
+/**
+ * Compiles a Mapd vega lite spec to a Mapd vega spec.
+ * 
+ * The vega lite spec should have an `sql` field in it's data instead of a `values` field.
+ * 
+ * The vega spec has a bunch of differences between mapd and the original spec.
+ * @param spec 
+ */
+export function compileToVega(vlSpec: any): any {
+    const sql = vlSpec.data.sql;
+    delete vlSpec.data.sql;
+    vlSpec.data.name = 'mapd_data' 
+
+    const vSpec = compile(vlSpec, {config: {invalidValues: null}}).spec;
+
+    // manually remove transformation from vega spec
+    // until https://github.com/vega/vega-lite/issues/3665 is merged
+    vSpec.data[0].name = vSpec.data.pop().name;
+    vSpec.data[0].sql = sql;
+
+    for (const mark of vSpec.marks) {
+        // mapd uses mark.properties instead of mark.encode.update
+        const properties = mark.encode.update;
+        mark.properties = properties;
+        delete mark.encode;
+
+        // mapd has an opacity value instead of an object
+        if (properties.opacity) {
+            properties.opacity = properties.opacity.value;
+        }
+
+        // mapd has a fillColor instead of a fill object
+        if (properties.fill) {
+            properties.fillColor = properties.fill.value;
+            delete properties.fill;
+        }
+
+        //mapd has a shape string instead of object
+        if (properties.shape) {
+            properties.shape = properties.shape.value;
+        }
+
+        // the width and height are required in mapd for symbols,
+        // unlike in vega where they are optional, or you can just set the
+        // size
+        if (mark.type === 'symbol') {
+            properties.width = properties.height = properties.size;
+            delete properties.size;
+        }
+    }
+
+    return vSpec;
+}

src/widget.ts

@@ -70,7 +70,7 @@ class MapDVega extends Widget {
             // If there was an error, clear any image data,
             // and set the text content of the error node.
             this._setImageData('');
-            this._error.textContent = error.message;
+            this._error.textContent = `${error.message} \n\n For Vega spec: \n ${JSON.stringify(vega, null, 2)}`;
             this._rendered.reject(error.message);
           } else {
             // Set the image data.