Search_for_keyword_cluster-bad_scaling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from py2neo import Graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "graph = Graph(auth=('neo4j', 'shekhor'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Graph('bolt://neo4j@localhost:7687')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "phrase = \"myers battery backup sump pump manual\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def construct_query(phrase, min_n_keywords = 3, n_common_urls = 4):\n",
    "    \"\"\"\n",
    "    Iteratively build cliques of min_n_keywords where all edges have more than n_common_urls\n",
    "    \"\"\"\n",
    "    # Find keyword\n",
    "    match_phrase = 'MATCH (k1:Keyword {{keyword: \"{phrase}\"}})-[r1:ASSOCIATED_WITH]-(k2:Keyword)\\n'\\\n",
    "                   'WHERE size(r1.urls) > {n_common_urls}\\n'\\\n",
    "                   'WITH *\\n'\\\n",
    "                    .format(phrase=phrase, n_common_urls=n_common_urls)\n",
    "    find_next_keyword = ''\n",
    "    for n in range(2,min_n_keywords):\n",
    "        # Find k_n+1: k_n's neighbors that share at least n_common_urls urls with k_1, k_2, ..., k_n\n",
    "        keywords_up_to_n = ', '.join(['k'+str(i) for i in range(1,n+1)])\n",
    "        relationships_up_to_n = ', '.join(['r'+str(i) for i in range(1,n+1)])\n",
    "        find_next_keyword += 'MATCH (k{n})-[r{n}:ASSOCIATED_WITH]-(k{nplus1})\\n'\\\n",
    "                             'WHERE NOT k{nplus1} in [{keywords_up_to_n}]\\n'\\\n",
    "                             'WITH *,reduce(intersect = r1.urls, r IN [{relationships_up_to_n}] | apoc.coll.intersection(intersect, r.urls)) AS commonUrls\\n'\\\n",
    "                             'WHERE size(commonUrls) > 4\\n'\\\n",
    "                             .format(n=n,nplus1=n+1,keywords_up_to_n=keywords_up_to_n,relationships_up_to_n=relationships_up_to_n,n_common_urls=n_common_urls)\n",
    "    return_keywords = ', '.join(['k'+str(i) for i in range(1,min_n_keywords+1)])\n",
    "    return_query = 'RETURN [k in [{return_keywords}] | k.keyword] AS group, commonUrls\\n'\\\n",
    "                    .format(return_keywords=return_keywords)\n",
    "    return match_phrase + find_next_keyword + return_query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 16.4 s, sys: 155 ms, total: 16.6 s\n",
      "Wall time: 16.7 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "foo = graph.run(construct_query(phrase, min_n_keywords = 5)).to_table()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "329136"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(foo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['myers battery backup sump pump manual',\n",
       "  'rain barrel sump pump discharge',\n",
       "  'sump pump use basement',\n",
       "  'deko sump pump manual',\n",
       "  'sump pump on switch'],\n",
       " [21219, 27267, 4357, 27162, 3085])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "foo[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5min 28s, sys: 3.01 s, total: 5min 31s\n",
      "Wall time: 5min 33s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "foo = graph.run(construct_query(phrase, min_n_keywords = 6)).to_table()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}