Word_Embeddings_LevelDB.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word Embeddings in LevelDB\n",
    "\n",
    "This example uses the [plyvel](https://plyvel.readthedocs.io/en/latest/) Connector within Python3 to store and retrieve various amounts of Word Embeddings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import plyvel\n",
    "import subprocess\n",
    "import shutil\n",
    "import io\n",
    "import time\n",
    "import numpy\n",
    "import plotly\n",
    "from tqdm import tqdm_notebook as tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dummy Embeddings\n",
    "\n",
    "For testing purposes we will use randomly generated numpy arrays as dummy embbeddings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def embeddings(n=1000, dim=300):\n",
    "    \"\"\"\n",
    "    Yield n tuples of random numpy arrays of *dim* length indexed by *n*\n",
    "    \"\"\"\n",
    "    idx = 0\n",
    "    while idx < n:\n",
    "        yield (str(idx), numpy.random.rand(dim))\n",
    "        idx += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conversion Functions\n",
    "\n",
    "Since we can't just save a NumPy array into the database, we will convert it into a BLOB."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def adapt_array(array):\n",
    "    \"\"\"\n",
    "    Using the numpy.save function to save a binary version of the array,\n",
    "    and BytesIO to catch the stream of data and convert it into a BLOB.\n",
    "    \"\"\"\n",
    "    out = io.BytesIO()\n",
    "    numpy.save(out, array)\n",
    "    out.seek(0)\n",
    "\n",
    "    return out.read()\n",
    "\n",
    "def convert_array(blob):\n",
    "    \"\"\"\n",
    "    Using BytesIO to convert the binary version of the array back into a numpy array.\n",
    "    \"\"\"\n",
    "    out = io.BytesIO(blob)\n",
    "    out.seek(0)\n",
    "\n",
    "    return numpy.load(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "connection = plyvel.DB('./leveldb.embedding.db', create_if_missing=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 142 ms, sys: 11.7 ms, total: 153 ms\n",
      "Wall time: 152 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for key, emb in embeddings():\n",
    "    arr = adapt_array(emb)\n",
    "    connection.put(key.encode(), arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 216 ms, sys: 95 µs, total: 216 ms\n",
      "Wall time: 214 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for key, _ in embeddings():\n",
    "    arr = connection.get(key.encode())\n",
    "    emb = convert_array(arr)\n",
    "    assert(type(emb) is numpy.ndarray)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "connection.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sample some data\n",
    "\n",
    "To test the I/O we will write and read some data from the database. This may take a while."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "779c4fdaf19940379c2efe1f4c5ba9c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a42ce64fa2414ae89d83a3e1d595bc17",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "694ef63923c44cd58b70bf9cc30d2916",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aaff034fc48f44cc90dc7fca3785faa7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ec2e48017b64ecaa0cd732e5e597aa1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=4000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0615da1c57f548358348c7854ee59204",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "DONE\n"
     ]
    }
   ],
   "source": [
    "write_times = []\n",
    "read_times = []\n",
    "db_sizes = []\n",
    "counts = [500, 1000, 2000, 3000, 4000, 5000]\n",
    "\n",
    "for c in counts:\n",
    "    shutil.rmtree('./leveldb.embedding.db', ignore_errors=True)\n",
    "    connection = plyvel.DB('./leveldb.embedding.db', create_if_missing=True)\n",
    "    \n",
    "    start_time_write = time.time()\n",
    "    for key, emb in tqdm(embeddings(c), total=c):\n",
    "        arr = adapt_array(emb)\n",
    "        connection.put(key.encode(), arr)\n",
    "    write_times.append(time.time() - start_time_write)\n",
    "    \n",
    "    start_time_read = time.time()\n",
    "    for key, _ in embeddings(c):\n",
    "        arr = connection.get(key.encode())\n",
    "        emb = convert_array(arr)\n",
    "        assert(type(emb) is numpy.ndarray)\n",
    "    read_times.append(time.time() - start_time_read)\n",
    "\n",
    "    connection.close()\n",
    "    db_sizes.append(subprocess.check_output(['du','-s', './leveldb.embedding.db']).split()[0].decode('utf-8'))\n",
    "    \n",
    "print('DONE')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "data": [
        {
         "mode": "lines+markers",
         "type": "scatter",
         "uid": "07338f3a-af8a-11e8-8bf3-94b86d86a98e",
         "x": [
          500,
          1000,
          2000,
          3000,
          4000,
          5000
         ],
         "y": [
          0.13895416259765625,
          0.17101764678955078,
          0.3788294792175293,
          0.5008127689361572,
          0.7033114433288574,
          1.1339409351348877
         ]
        }
       ],
       "layout": {
        "title": "LevelDB Write Times",
        "xaxis": {
         "title": "Embedding Count"
        },
        "yaxis": {
         "title": "Time in Seconds"
        }
       }
      },
      "text/html": [
       "<div id=\"818a884c-e72b-4e87-af28-e03b866f8936\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";\n",
       "        Plotly.plot(\n",
       "            '818a884c-e72b-4e87-af28-e03b866f8936',\n",
       "            [{\"mode\": \"lines+markers\", \"x\": [500, 1000, 2000, 3000, 4000, 5000], \"y\": [0.13895416259765625, 0.17101764678955078, 0.3788294792175293, 0.5008127689361572, 0.7033114433288574, 1.1339409351348877], \"type\": \"scatter\", \"uid\": \"07338f3a-af8a-11e8-8bf3-94b86d86a98e\"}],\n",
       "            {\"title\": \"LevelDB Write Times\", \"xaxis\": {\"title\": \"Embedding Count\"}, \"yaxis\": {\"title\": \"Time in Seconds\"}},\n",
       "            {\"showLink\": true, \"linkText\": \"Export to plot.ly\"}\n",
       "        ).then(function () {return Plotly.addFrames('818a884c-e72b-4e87-af28-e03b866f8936',{});}).then(function(){Plotly.animate('818a884c-e72b-4e87-af28-e03b866f8936');})\n",
       "        });</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<div id=\"818a884c-e72b-4e87-af28-e03b866f8936\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";\n",
       "        Plotly.plot(\n",
       "            '818a884c-e72b-4e87-af28-e03b866f8936',\n",
       "            [{\"mode\": \"lines+markers\", \"x\": [500, 1000, 2000, 3000, 4000, 5000], \"y\": [0.13895416259765625, 0.17101764678955078, 0.3788294792175293, 0.5008127689361572, 0.7033114433288574, 1.1339409351348877], \"type\": \"scatter\", \"uid\": \"07338f3a-af8a-11e8-8bf3-94b86d86a98e\"}],\n",
       "            {\"title\": \"LevelDB Write Times\", \"xaxis\": {\"title\": \"Embedding Count\"}, \"yaxis\": {\"title\": \"Time in Seconds\"}},\n",
       "            {\"showLink\": true, \"linkText\": \"Export to plot.ly\"}\n",
       "        ).then(function () {return Plotly.addFrames('818a884c-e72b-4e87-af28-e03b866f8936',{});}).then(function(){Plotly.animate('818a884c-e72b-4e87-af28-e03b866f8936');})\n",
       "        });</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotly.offline.init_notebook_mode(connected=True)\n",
    "trace = plotly.graph_objs.Scatter(\n",
    "    y = write_times,\n",
    "    x = counts,\n",
    "    mode = 'lines+markers'\n",
    ")\n",
    "layout = plotly.graph_objs.Layout(title=\"LevelDB Write Times\",\n",
    "                yaxis=dict(title='Time in Seconds'),\n",
    "                xaxis=dict(title='Embedding Count'))\n",
    "data = [trace]\n",
    "fig = plotly.graph_objs.Figure(data=data, layout=layout)\n",
    "plotly.offline.iplot(fig, filename='jupyter-scatter-write')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "data": [
        {
         "mode": "lines+markers",
         "type": "scatter",
         "uid": "084ae044-af8a-11e8-8bf3-94b86d86a98e",
         "x": [
          500,
          1000,
          2000,
          3000,
          4000,
          5000
         ],
         "y": [
          0.0985105037689209,
          0.19457459449768066,
          0.39040303230285645,
          0.5678610801696777,
          0.7731680870056152,
          0.9867994785308838
         ]
        }
       ],
       "layout": {
        "title": "LevelDB Read Times",
        "xaxis": {
         "title": "Embedding Count"
        },
        "yaxis": {
         "title": "Time in Seconds"
        }
       }
      },
      "text/html": [
       "<div id=\"9ad806a7-d226-4fea-9a58-c8e9bc808f01\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";\n",
       "        Plotly.plot(\n",
       "            '9ad806a7-d226-4fea-9a58-c8e9bc808f01',\n",
       "            [{\"mode\": \"lines+markers\", \"x\": [500, 1000, 2000, 3000, 4000, 5000], \"y\": [0.0985105037689209, 0.19457459449768066, 0.39040303230285645, 0.5678610801696777, 0.7731680870056152, 0.9867994785308838], \"type\": \"scatter\", \"uid\": \"084ae044-af8a-11e8-8bf3-94b86d86a98e\"}],\n",
       "            {\"title\": \"LevelDB Read Times\", \"xaxis\": {\"title\": \"Embedding Count\"}, \"yaxis\": {\"title\": \"Time in Seconds\"}},\n",
       "            {\"showLink\": true, \"linkText\": \"Export to plot.ly\"}\n",
       "        ).then(function () {return Plotly.addFrames('9ad806a7-d226-4fea-9a58-c8e9bc808f01',{});}).then(function(){Plotly.animate('9ad806a7-d226-4fea-9a58-c8e9bc808f01');})\n",
       "        });</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<div id=\"9ad806a7-d226-4fea-9a58-c8e9bc808f01\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";\n",
       "        Plotly.plot(\n",
       "            '9ad806a7-d226-4fea-9a58-c8e9bc808f01',\n",
       "            [{\"mode\": \"lines+markers\", \"x\": [500, 1000, 2000, 3000, 4000, 5000], \"y\": [0.0985105037689209, 0.19457459449768066, 0.39040303230285645, 0.5678610801696777, 0.7731680870056152, 0.9867994785308838], \"type\": \"scatter\", \"uid\": \"084ae044-af8a-11e8-8bf3-94b86d86a98e\"}],\n",
       "            {\"title\": \"LevelDB Read Times\", \"xaxis\": {\"title\": \"Embedding Count\"}, \"yaxis\": {\"title\": \"Time in Seconds\"}},\n",
       "            {\"showLink\": true, \"linkText\": \"Export to plot.ly\"}\n",
       "        ).then(function () {return Plotly.addFrames('9ad806a7-d226-4fea-9a58-c8e9bc808f01',{});}).then(function(){Plotly.animate('9ad806a7-d226-4fea-9a58-c8e9bc808f01');})\n",
       "        });</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotly.offline.init_notebook_mode(connected=True)\n",
    "trace = plotly.graph_objs.Scatter(\n",
    "    y = read_times,\n",
    "    x = counts,\n",
    "    mode = 'lines+markers'\n",
    ")\n",
    "layout = plotly.graph_objs.Layout(title=\"LevelDB Read Times\",\n",
    "                yaxis=dict(title='Time in Seconds'),\n",
    "                xaxis=dict(title='Embedding Count'))\n",
    "data = [trace]\n",
    "fig = plotly.graph_objs.Figure(data=data, layout=layout)\n",
    "plotly.offline.iplot(fig, filename='jupyter-scatter-read')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "data": [
        {
         "type": "bar",
         "uid": "1ed6e4de-af8a-11e8-8bf3-94b86d86a98e",
         "x": [
          500,
          1000,
          2000,
          3000,
          4000,
          5000
         ],
         "y": [
          "1264",
          "2512",
          "5012",
          "7508",
          "10008",
          "12512"
         ]
        }
       ],
       "layout": {
        "title": "Database Size",
        "xaxis": {
         "title": "Embedding Count"
        },
        "yaxis": {
         "title": "Size in KB"
        }
       }
      },
      "text/html": [
       "<div id=\"61163ece-872b-4594-987a-395ba3ddbc5b\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";\n",
       "        Plotly.plot(\n",
       "            '61163ece-872b-4594-987a-395ba3ddbc5b',\n",
       "            [{\"x\": [500, 1000, 2000, 3000, 4000, 5000], \"y\": [\"1264\", \"2512\", \"5012\", \"7508\", \"10008\", \"12512\"], \"type\": \"bar\", \"uid\": \"1ed6e4de-af8a-11e8-8bf3-94b86d86a98e\"}],\n",
       "            {\"title\": \"Database Size\", \"xaxis\": {\"title\": \"Embedding Count\"}, \"yaxis\": {\"title\": \"Size in KB\"}},\n",
       "            {\"showLink\": true, \"linkText\": \"Export to plot.ly\"}\n",
       "        ).then(function () {return Plotly.addFrames('61163ece-872b-4594-987a-395ba3ddbc5b',{});}).then(function(){Plotly.animate('61163ece-872b-4594-987a-395ba3ddbc5b');})\n",
       "        });</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<div id=\"61163ece-872b-4594-987a-395ba3ddbc5b\" style=\"height: 525px; width: 100%;\" class=\"plotly-graph-div\"></div><script type=\"text/javascript\">require([\"plotly\"], function(Plotly) { window.PLOTLYENV=window.PLOTLYENV || {};window.PLOTLYENV.BASE_URL=\"https://plot.ly\";\n",
       "        Plotly.plot(\n",
       "            '61163ece-872b-4594-987a-395ba3ddbc5b',\n",
       "            [{\"x\": [500, 1000, 2000, 3000, 4000, 5000], \"y\": [\"1264\", \"2512\", \"5012\", \"7508\", \"10008\", \"12512\"], \"type\": \"bar\", \"uid\": \"1ed6e4de-af8a-11e8-8bf3-94b86d86a98e\"}],\n",
       "            {\"title\": \"Database Size\", \"xaxis\": {\"title\": \"Embedding Count\"}, \"yaxis\": {\"title\": \"Size in KB\"}},\n",
       "            {\"showLink\": true, \"linkText\": \"Export to plot.ly\"}\n",
       "        ).then(function () {return Plotly.addFrames('61163ece-872b-4594-987a-395ba3ddbc5b',{});}).then(function(){Plotly.animate('61163ece-872b-4594-987a-395ba3ddbc5b');})\n",
       "        });</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# DB Size\n",
    "plotly.offline.init_notebook_mode(connected=True)\n",
    "trace = plotly.graph_objs.Bar(\n",
    "    x = counts,\n",
    "    y = db_sizes\n",
    ")\n",
    "layout = plotly.graph_objs.Layout(title=\"Database Size\",\n",
    "                yaxis=dict(title='Size in KB'),\n",
    "                xaxis=dict(title='Embedding Count'))\n",
    "data = [trace]\n",
    "fig = plotly.graph_objs.Figure(data=data, layout=layout)\n",
    "plotly.offline.iplot(fig, filename='jupyter-bar-size')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}