demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Setup the application**\n",
    "\n",
    "Import required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob, sys, threading\n",
    "\n",
    "from omegaconf import OmegaConf\n",
    "from tqdm.auto import tqdm\n",
    "from grams.prelude import GRAMS, ROOT_DIR, DATA_DIR, I, fetch_tables, WikidataSemanticModelHelper\n",
    "\n",
    "from sm_widgets.widgets.annotator import Annotator, BatchAnnotator\n",
    "from sm_widgets_integration.wikidata import get_qnode_db, get_ontclass_db, get_ontprop_db\n",
    "from sm_widgets_integration.with_grams import convert_table\n",
    "from sm_widgets.widgets.slider import DisplayShell, Slider\n",
    "from sm.prelude import M"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create GRAMS instance. `DATA_DIR` is where GRAMS will store entities retrieved from Wikidata, `cfg` contains configuration of GRAMS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "cwd = ROOT_DIR / \"examples/misc\"\n",
    "cfg = OmegaConf.load(ROOT_DIR / \"grams.yaml\")\n",
    "grams = GRAMS(DATA_DIR, cfg, proxy=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Fetch tables from a webpage and select table that we want to annotate**\n",
    "\n",
    "Fetch table from a Wikipedia page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tables = fetch_tables(\"https://en.wikipedia.org/wiki/President_of_the_National_Council_(Austria)\")\n",
    "# tables = fetch_tables(\"https://en.wikipedia.org/wiki/Novartis\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a slider to view fetch tables and select the table we want to annotate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f5e4795eb9f4d82aa82444d84c783c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='<div id=\"e6a74cdd-d7bc-42fc-9883-e7c06fe1064b\"></div>')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b5fb7187c9b54884811bf1359864e4d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "        if (window.IPyApps === undefined) {\n",
       "            window.IPyApps = new Map();\n",
       "        }\n",
       "        window.IPyApps.set('e6a74cdd-d7bc-42fc-9883-e7c06fe1064b', window);\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "990da083e20443c49dd4bc7656b2c270",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "SlowTunnelWidget(js_endpoint=(0, ''), py_endpoint=(0, ''), tunnel_id='fbd1c876-f810-49c9-893f-07afd07add5f')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7983cf19e70c4673a688d88b633b692d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Output()"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            \n",
       "    function repeatUntilSuccess(fn, timeout, maxTry) {\n",
       "        if (fn() === true) {\n",
       "            return;\n",
       "        }\n",
       "        if (maxTry === undefined) {\n",
       "            maxTry = 10;\n",
       "        }\n",
       "        if (maxTry === 0) {\n",
       "            console.error(\"Max retries error\");\n",
       "            alert(\"max retries error\");\n",
       "            throw new Error(\"Max retries error\");\n",
       "        } else {\n",
       "            setTimeout(function () {\n",
       "                repeatUntilSuccess(fn, timeout, maxTry - 1);\n",
       "            }, timeout);\n",
       "        }\n",
       "    }\n",
       "    \n",
       "            function setupApp0c09cab714c24684b59d357da6041eeb() {\n",
       "                if (window.IPyCallback === undefined || window.IPyApps === undefined) {\n",
       "                    return false;\n",
       "                }\n",
       "                let tunnel = window.IPyCallback.get('fbd1c876-f810-49c9-893f-07afd07add5f');\n",
       "                if (tunnel === undefined) {\n",
       "                    return false;\n",
       "                }\n",
       "                \n",
       "                let appwin = window.IPyApps.get('e6a74cdd-d7bc-42fc-9883-e7c06fe1064b');\n",
       "                if (appwin === undefined) {\n",
       "                    return false;\n",
       "                }\n",
       "                \n",
       "                let container = appwin.document.getElementById('e6a74cdd-d7bc-42fc-9883-e7c06fe1064b');\n",
       "                if (container === null) {\n",
       "                    return false;\n",
       "                }\n",
       "                \n",
       "                let div = appwin.document.createElement(\"div\");\n",
       "                div.id = '0c09cab7-14c2-4684-b59d-357da6041eeb';\n",
       "                div.style = \"margin-bottom: 8px\";\n",
       "                container.parentElement.prepend(div);\n",
       "                \n",
       "                // use the tunnel first to send out the code, after the application is rendered, the listening function \n",
       "                // is going to be replaced by the listener in the application, so we don't have to worry.\n",
       "                tunnel.on_receive(function (version, msg) {\n",
       "                    let payload = JSON.parse(msg);\n",
       "                    if (payload.id !== '/init/download_code') {\n",
       "                        alert('invalid calling order. you need to set the source code first');\n",
       "                        console.error(\"invalid call order\", payload);\n",
       "                        return;\n",
       "                    }\n",
       "                    appwin.eval(payload.response);\n",
       "                    appwin.Slider.renderApp('0c09cab7-14c2-4684-b59d-357da6041eeb', tunnel);\n",
       "                    window.IPyApps.set('0c09cab7-14c2-4684-b59d-357da6041eeb', appwin);\n",
       "                    tunnel.send_msg(JSON.stringify({ url: '/init/done', params: null, id: '/init/done' }));\n",
       "                });\n",
       "                tunnel.send_msg(JSON.stringify({ url: '/init/download_code', params: null, id: '/init/download_code' }));\n",
       "                return true;\n",
       "            }\n",
       "            repeatUntilSuccess(setupApp0c09cab714c24684b59d357da6041eeb, 50, 10);\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "slider = Slider(app=DisplayShell(render_fn=lambda df: display(df)), app_render_fn='set_data', embedded_error_log=True)\n",
    "slider.render()\n",
    "\n",
    "threading.Thread(\n",
    "    target=lambda: slider.set_data([\n",
    "        {\"description\": \"<br>\" + \" > \".join([r.header for r in table.context]), \"args\": [table.as_df()]}\n",
    "        for table in tables\n",
    "    ])\n",
    ").start()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Annotate the selected table**\n",
    "\n",
    "To annotate a table, simply call `grams.annotate`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_table = 10\n",
    "table = tables[selected_table]\n",
    "table = table.as_relational_linked_table(table_id=f\"{table.page_url}/{selected_table}\")\n",
    "annotation = grams.annotate(table)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "save table data (optional)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "M.serialize_json(table.to_json(), cwd / \"tables\" / f\"{table.get_friendly_fs_id()}.json\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Display and edit the annotation**\n",
    "\n",
    "Run the following two cells to render an interactive widget that allows to curate the predicted annotation. The former cell renders the widget, and the later cell sends data to the widget."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c9a6c962842746a49c734498ff7e851d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "SlowTunnelWidget(js_endpoint=(0, ''), py_endpoint=(0, ''), tunnel_id='8531e7a7-f23c-4951-8ae3-d51375c0a086')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99d0caf5cf4948d7a549c729a14e6ce4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HTML(value='<div id=\"11927c6b-20c4-4997-997c-0d7fb073880a\"></div>')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "        \n",
       "    function repeatUntilSuccess(fn, timeout, maxTry) {\n",
       "        if (fn() === true) {\n",
       "            return;\n",
       "        }\n",
       "        if (maxTry === undefined) {\n",
       "            maxTry = 10;\n",
       "        }\n",
       "        if (maxTry === 0) {\n",
       "            console.error(\"Max retries error\");\n",
       "            alert(\"max retries error\");\n",
       "            throw new Error(\"Max retries error\");\n",
       "        } else {\n",
       "            setTimeout(function () {\n",
       "                repeatUntilSuccess(fn, timeout, maxTry - 1);\n",
       "            }, timeout);\n",
       "        }\n",
       "    }\n",
       "    \n",
       "        if (window.IPyApps === undefined) {\n",
       "            window.IPyApps = new Map();\n",
       "        }\n",
       "        function setupApp11927c6b20c44997997c0d7fb073880a() {\n",
       "            // get tunnel\n",
       "            if (window.IPyCallback === undefined) {\n",
       "                return false;\n",
       "            }\n",
       "            let tunnel = window.IPyCallback.get('8531e7a7-f23c-4951-8ae3-d51375c0a086');\n",
       "            if (tunnel === undefined) {\n",
       "                return false;\n",
       "            }\n",
       "            \n",
       "            // setup html container\n",
       "            let win = window;\n",
       "            \n",
       "            // use the tunnel first to send out the code, after the application is rendered, the listening function \n",
       "            // is going to be replaced by the listener in the application, so we don't have to worry.\n",
       "            tunnel.on_receive(function (version, msg) {\n",
       "                let payload = JSON.parse(msg);\n",
       "                if (payload.id !== '/init/download_code') {\n",
       "                    alert('invalid calling order. you need to set the source code first');\n",
       "                    console.error(\"invalid call order. waiting for source code but get:\", payload);\n",
       "                    return;\n",
       "                }\n",
       "                win.eval(payload.response);\n",
       "                let shadowDOM = true;\n",
       "                win.Annotator.renderApp('11927c6b-20c4-4997-997c-0d7fb073880a', tunnel, undefined, shadowDOM);\n",
       "                window.IPyApps.set('11927c6b-20c4-4997-997c-0d7fb073880a', win);\n",
       "                tunnel.send_msg(JSON.stringify({ url: '/init/done', params: null, id: '/init/done' }));\n",
       "            });\n",
       "            tunnel.send_msg(JSON.stringify({ url: '/init/download_code', params: null, id: '/init/download_code' }));\n",
       "            return true;\n",
       "        }\n",
       "        repeatUntilSuccess(setupApp11927c6b20c44997997c0d7fb073880a, 50, 10);\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<sm_widgets.widgets.annotator.annotator.Annotator at 0x7f6d1c6368e0>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "annotator = Annotator(\n",
    "    entities=get_qnode_db(grams.qnodes), ontclasses=get_ontclass_db(grams.wdclasses), ontprops=get_ontprop_db(grams.wdprops), \n",
    "    prop_type_uri='http://www.wikidata.org/prop/P31', savedir=cwd / \"ground-truth\", eshost='http://mira.isi.edu:9200', dev=False, assistant=None)\n",
    "annotator.render(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "threading.Thread(\n",
    "    target=lambda: annotator.set_data(table.get_friendly_fs_id(), convert_table(table), sms=[annotation.sm])\n",
    ").start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "grams",
   "language": "python",
   "name": "grams"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}