check_words.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9903244/9903244 [16:27<00:00, 10029.28it/s]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from tqdm import *\n",
    "from collections import defaultdict\n",
    "import jieba\n",
    "\n",
    "zh_data = open('/data/xuwenshen/ai_challenge/data/train/train/train.zh', encoding='utf-8').readlines()\n",
    "zh_tokens = []\n",
    "\n",
    "zhl = defaultdict(lambda : 0)\n",
    "\n",
    "for i in tqdm(range(len(zh_data))):\n",
    "    zh_data[i]=zh_data[i].rstrip('\\n')\n",
    "    seg_list = ' '.join(jieba.cut(zh_data[i])).split()\n",
    "    zhl[len(seg_list)] += 1\n",
    "    zh_tokens.append(seg_list)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "train_zh = open('/data/xuwenshen/ai_challenge/data/train/train_words/zh_words.json', 'w')\n",
    "\n",
    "json.dump(zh_tokens, train_zh)\n",
    "\n",
    "train_zh.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAFkCAYAAADYL8pXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3X14VdWd//3352CkBQ+otUKl2KTTGRodtRKfUqzTFgqo\nBe1Nf1Ox3rVPV3VaGgfUPkEHqjC1PoCp0tZWHX/Vmhkr0xFUjKgztcUM3gZrnRqd3/wKtcJAy4hw\niFYCZ91/7H1g55AACefkJDuf13XlSrL39+y9zvIgH9Zeey+FEDAzMzMb6DKVboCZmZlZKTjUmJmZ\nWSo41JiZmVkqONSYmZlZKjjUmJmZWSo41JiZmVkqONSYmZlZKjjUmJmZWSo41JiZmVkqONSYmZlZ\nKvQ41Ej6gKTlkjZIykuanth3mKTvSPq1pB1xzf+W9I6iYwyVtFTSFkk5SfdLOrao5ihJP5G0TdJW\nSbdLGl5UM1bSQ5LaJW2SdL2kTFHNyZKelPSGpN9JurqL9/RBSa2S/iTpPyVd2tN+MTMzs8rqzUjN\ncOBXwBeB4oWjhgHvA74FnAp8DBgHPFBUdzNwPjADOAc4DlhWVHMvUAtMjGvPAW4r7IzDy8PAYcBZ\nwKXAp4FrEjVZoBlYB4wHrgYWSPp8oqYaeBB4HDgFaARul/SRg+gLMzMz6yd0KAtaSsoDF4YQlu+n\n5jRgDfCuEMIrkkYAfwQuCiH8LK4ZB7QBZ4UQnpZUC/wGqAshPBvXTAEeAt4ZQtgk6VxgOfCOEMKW\nuOYy4Drg7SGEXZL+BrgWGB1C2BXXfBu4IIRwQvz7d4BzQwgnJ9rcBIwMIZzX684xMzOzPtUXc2qO\nJBrReS3+vY5odOXxQkEI4SXgZaA+3nQWsLUQaGKPxcc5M1HzfCHQxJqBkcCJiZonC4EmUTNO0shE\nzWNFbW5OtMXMzMwGgMPKeXBJQ4lGTu4NIeyIN48GdoYQtheVb473FWr+kNwZQtgt6dWims1dHKOw\n77n4+2/3U7NtP8cZIWloCOHNLt7X24ApwHrgT8X7zczMrFtvAaqB5hDC/5TywGULNZIOA35KNLry\nxXKdp5d0iK+fAvykFA0xMzMbpD5JNH+2ZMoSahKBZizw4cQoDcAm4HBJI4pGa0bF+wo1xXdDDQGO\nLqo5vejUoxL7Ct9HdVETDqJme1ejNLH1APfccw+1tbXdlFix2bNns2TJkko3Y8Bxv/Wc+6x33G89\n5z7ruba2Ni655BKI/y4tpZKHmkSgeTfwoRDC1qKSVmAX0V1NyYnCxwMtcU0LcKSkUxPzaiYSjbCs\nSdR8Q9IxiXk1k4kuKb2QqFkoaUgIYXei5qUQwrZEzblFbZycaEtX/gRQW1vL+PHj91NmSSNHjnR/\n9YL7refcZ73jfus599khKfn0jd48p2a4pFMkvS/e9O7497FxoFlGdPv0JUCVpFHxVxVAPDpzB7A4\nfj5MHXAnsDqE8HRc8yLRZN0fSTpd0gTgFqAphFAYYXmUKLzcHT+LZgrRnU63hhA64pp7gZ3AnZJO\nkPQJoAG4KfGWfhC/h+9IGifpi8DHgcU97RszMzOrnN6M1JwG/CvRJZzA3oDwv4meTzMt3v6reLvi\n3z8EPBlvmw3sBu4HhgKPAF8qOs/FwK1Edybl49orCjtDCHlJHwW+DzwFtAN3AfMTNdslTQaWAs8A\nW4AFIYQ7EjXrJZ0PLCEKPK8AnwshFN8RZWZmZv1Yj0NNCOHn7H+E54CjP/FclS/HX93VvEY02rO/\n4/we+OgBav4D+KsD1DxJdKu5mZmZDVBe+8n6zMyZMyvdhAHJ/dZz7rPecb/1nPusfzmkJwoPVpLG\nA62tra2eIGZmZtYDa9eupa6uDqJVA9aW8tgeqTEzM7NUcKgxMzOzVHCoMTMzs1RwqDEzM7NUcKgx\nMzOzVHCoMTMzs1RwqDEzM7NUcKgxMzOzVHCoMTMzs1RwqDEzM7NUcKgxMzOzVHCoMTMzs1RwqDEz\nM7NUcKjpA14J3czMrPwcasokl8sxv6GBSTU1XDh2LJNqapjf0EAul6t008zMzFLpsEo3II1yuRwz\n6uuZ09bGgnweAQFoXrqUGU88wbKWFrLZbKWbaWZmlioeqSmDG+fOZU5bG1PjQAMgYGo+z+y2Nm6a\nN6+SzTMzM0slh5oyWL1iBVPy+U7bCrNqpubzrF6+vO8bZWZmlnK+/FRiIQSGd3QgIAfcCKwGhgPt\nwATg8DffJISApP0cyczMzHrCIzUlJon2qiq2AzOAemAV8ED8/Szg/2zZwo4dOyrYSjMzs/RxqCmD\nCdOm0QDMAaZCp3k15wKLd+3yvBozM7MSc6gpg6sWLeKZqiqmdLP//BA8r8bMzKzEHGrK4IgjjqDm\nmGPobsaMgGEdHX4on5mZWQk51JSBJN4YOpTuIksA2quqPFHYzMyshBxqymTCtGk0Z7ru3kcyGc6e\nPr2PW2RmZpZuDjVlctWiRSyurWVlJrNnxCYAKzMZltTWcuXChZVsnpmZWeo41JRJNptlWUsLa2bN\nYnJ1NReMGcPk6mrWzJrlZRLMzMzKwA/fK6NsNsuCxkZobPTD9szMzMrMIzV9xIHGzMysvBxqzMzM\nLBUcaszMzCwVHGrMzMwsFRxqzMzMLBUcaszMzCwVHGrMzMwsFRxqzMzMLBUcaszMzCwVHGrMzMws\nFRxqzMzMLBV6HGokfUDSckkbJOUlTe+i5hpJGyW9LmmVpPcU7R8qaamkLZJyku6XdGxRzVGSfiJp\nm6Stkm6XNLyoZqykhyS1S9ok6XpJmaKakyU9KekNSb+TdHUX7f2gpFZJf5L0n5Iu7Wm/mJmZWWX1\nZqRmOPAr4ItAKN4p6avALOALwBlAO9As6fBE2c3A+cAM4BzgOGBZ0aHuBWqBiXHtOcBtifNkgIeJ\nFuU8C7gU+DRwTaImCzQD64DxwNXAAkmfT9RUAw8CjwOnAI3A7ZI+cpD9YWZmZv1Aj1fpDiE8AjwC\noK5XabwCuDaE8GBc8ylgM3AhcJ+kEcBngYtCCD+Paz4DtEk6I4TwtKRaYApQF0J4Nq75MvCQpKtC\nCJvi/e8FPhRC2AI8L+mbwHWSFoQQdgGXAFXA5+Lf2ySdCswBbo/b+zfAb0MIX4l/f0nS2cBsYFVP\n+8fMzMwqo6RzaiTVAKOJRj0ACCFsB9YA9fGm04jCVLLmJeDlRM1ZwNZCoIk9RjQydGai5vk40BQ0\nAyOBExM1T8aBJlkzTtLIRM1jRW+lOdEWMzMzGwBKPVF4NFHw2Fy0fXO8D2AUsDMOO93VjAb+kNwZ\nQtgNvFpU09V5KFHNCElDMTMzswGhx5efUqKry2Y9Nnv2bEaOHNlp28yZM5k5cyYAIQS6vkJnZmaW\nfk1NTTQ1NXXatm3btrKdr9ShZhNRYBhF59GPUcCziZrDJY0oGq0ZFe8r1BTfDTUEOLqo5vSi849K\n7Ct8H9VFTTiImu0hhDf3fYt7LVmyhPHjx3falsvlmN/QwOoVKxje0UF7VRUTpk3jqkWLyGazDjpm\nZjZoJP+hX7B27Vrq6urKcr6SXn4KIawjCgkTC9viicFnAk/Fm1qBXUU144DjgZZ4UwtwZDypt2Ai\nUWBak6g5SdIxiZrJwDbghUTNOXEgSta8FELYlqiZSGeTE205aLlcjhn19dQvXcqq9et5YMMGVq1f\nzym33srZ1dV8+F3v4sKxY5lUU8P8hgZyuVxPT2FmZmbd6M1zaoZLOkXS++JN745/Hxv/fjMwT9I0\nSScBPwZeAR6APROH7wAWx8+HqQPuBFaHEJ6Oa14kmqz7I0mnS5oA3AI0xXc+ATxKFF7ujp9FMwW4\nFrg1hNAR19wL7ATulHSCpE8ADcBNibf0g/g9fEfSOElfBD4OLO5p39w4dy5z2tqYms/vub61A/hB\nCHz71Vd5/OWX9wSd+qVLmVFf72BjZmZWIr0ZqTmN6FJSK9FlnJuAtcC3AEII1xMFkNuIRlXeCpwb\nQtiZOMZsomfD3A/8G7CR6Jk1SRcDLxLdmfQg8CRwWWFnCCEPfBTYTTQK9GPgLmB+omY70ahLNfAM\ncAOwIIRwR6JmPdFzcCYRPX9nNtEt4MV3RB3Q6hUrmJLPd9p2I9H94+exdyKPgKn5PLPb2rhp3rye\nnsbMzMy6oBD2eX6eHYCk8UBra2vrnjk1IQQuHDuWBzZs6FQ7iehhN13NognA5OpqVq1bV+YWm5mZ\n9Q+JOTV1IYS1pTy2134qEUm0V1V1esRyIHr8cnfTggUM6+jAwdLMzOzQOdSU0IRp02jO7O1SEa0R\n0V1kCUB7VZXvhjIzMysBh5oSumrRIhbX1rIyk9kTZN4PrOym/pFMhrOn77MeqJmZmfXCYH34Xllk\ns1mWtbRw07x5LF6+nGEdHWwfMoTlO3bAa69xbnxXVCAKNEtqa1m2cGGlm21mZpYKDjUlls1mWdDY\nCI2Nex60l8vluGnePJbEQef1qiomTJ/OsoULyWazlW6ymZlZKjjUlFFhrkxXQcfMzMxKy3Nq+pgD\njZmZWXk41JiZmVkqONSYmZlZKjjUmJmZWSo41JiZmVkqONSYmZlZKjjUmJmZWSo41JiZmVkqONSY\nmZlZKjjUmJmZWSo41JiZmVkqONSYmZlZKjjUmJmZWSo41JiZmVkqONSYmZlZKjjUmJmZWSo41JiZ\nmVkqONSYmZlZKjjUmJmZWSo41JiZmVkqONSYmZlZKjjUmJmZWSo41JiZmVkqONSYmZlZKjjUmJmZ\nWSo41FRACKHSTTAzM0sdh5o+ksvlmN/QwKSaGi4cO5ZJNTXMb2ggl8tVumlmZmapcFilGzAY5HI5\nZtTXM6etjQX5PAIC0Lx0KTOeeIJlLS1ks9lKN9PMzGxA80hNH7hx7lzmtLUxNQ40AAKm5vPMbmvj\npnnzKtk8MzOzVHCo6QOrV6xgSj7f5b6p+Tyrly/v4xaZmZmlj0NNmYUQGN7RsWeEppiAYR0dnjxs\nZmZ2iBxqykwS7VVVdBdZAtBeVYXUXewxMzOzg+FQ0wcmTJtGc6brrn4kk+Hs6dP7uEVmZmbp41DT\nB65atIjFtbWszGT2jNgEYGUmw5LaWq5cuLCSzTMzM0uFkocaSRlJ10r6raTXJf2XpH1u75F0jaSN\ncc0qSe8p2j9U0lJJWyTlJN0v6diimqMk/UTSNklbJd0uaXhRzVhJD0lql7RJ0vWSMkU1J0t6UtIb\nkn4n6epS9kk2m2VZSwtrZs1icnU1F4wZw+TqatbMmuXbuc3MzEqkHM+p+RpwGfAp4AXgNOAuSa+F\nEG4FkPRVYFZcsx5YCDRLqg0h7IyPczNwLjAD2A4sBZYBH0ic615gFDAROBy4C7gNuCQ+TwZ4GNgI\nnAUcB9wN7ATmxTVZoBl4NG73ScA/SNoaQri9VJ2SzWZZ0NgIjY2EEDyHxszMrMTKEWrqgQdCCI/E\nv78s6WLgjETNFcC1IYQHASR9CtgMXAjcJ2kE8FngohDCz+OazwBtks4IITwtqRaYAtSFEJ6Na74M\nPCTpqhDCpnj/e4EPhRC2AM9L+iZwnaQFIYRdRAGoCvhc/HubpFOBOUDJQk2SA42ZmVnplWNOzVPA\nREl/DiDpFGAC0YgJkmqA0cDjhReEELYDa4gCEUSjO4cV1bwEvJyoOQvYWgg0sceIpqucmah5Pg40\nBc3ASODERM2TcaBJ1oyTNLKnb97MzMwqoxwjNdcBI4AXJe0mCk5zQwj/GO8fTRQ8Nhe9bnO8D6JL\nSjvjsNNdzWjgD8mdIYTdkl4tqunqPIV9z8Xff7ufmm1dv00zMzPrT8oRaj4BXAxcRDSn5n1Ao6SN\nIYS7y3A+MzMzs7KEmuuBb4cQfhr//htJ1cDXiSbpbiJ6kO4oOo+ijAIKl5I2AYdLGlE0WjMq3leo\nKb4baghwdFHN6UXtG5XYV/g+6gA1XZo9ezYjR3a+QjVz5kxmzpy5v5eZmZkNCk1NTTQ1NXXatm1b\n+S6AlCPUDAN2F23LE8/fCSGsk7SJ6I6lXwPEE4PPJLrDCaAV2BXX/CyuGQccD7TENS3AkZJOTcyr\nmUgUmNYkar4h6ZjEvJrJRJeUXkjULJQ0JISwO1HzUghhvz2/ZMkSxo8ff4DuMDMzG5y6+of+2rVr\nqaurK8v5yjFReAUwT9J5kt4l6WPAbOCfEzU3xzXTJJ0E/Bh4BXgA9kwcvgNYLOmDkuqAO4HVIYSn\n45oXiSb0/kjS6ZImALcATfGdTxDdpv0CcHf8LJopwLXArSGEjrjmXqJbvO+UdIKkTwANwE1l6Bsz\nMzMrk3KM1MwiCg5LiS4PbQS+H28DIIRwvaRhRM+UORL4BXBu4hk1EAWh3cD9wFDgEeBLRee6GLiV\n6K6nfFx7ReI8eUkfjc//FNBO9Cyb+Yma7ZImx+19BtgCLAgh3HEonWBmZmZ9S14duuckjQdaW1tb\nffnJzMysBxKXn+pCCGtLeWyv/WRmZmap4FBjZmZmqeBQY2ZmZqngUGNmZmap4FBjZmZmqeBQY2Zm\nZqngUGNmZmap4FBjZmZmqeBQY2ZmZqngUGNmZmap4FBjZmZmqeBQY2ZmZqngUGNmZmap4FBjZmZm\nqeBQY2ZmZqngUGNmZmap4FBTBiGESjfBzMxs0HGoKZFcLsf8hgYm1dRw4dixTKqpYX5DA7lcrtJN\nMzMzGxQOq3QD0iCXyzGjvp45bW0syOcREIDmpUuZ8cQTLGtpIZvNVrqZZmZmqeaRmhK4ce5c5rS1\nMTUONAACpubzzG5r46Z58yrZPDMzs0HBoaYEVq9YwZR8vst9U/N5Vi9f3sctMjMzG3wcag5RCIHh\nHR17RmiKCRjW0dHl5GFPKDYzMysdh5pDJIn2qiq6iycBaK+qQopijycUm5mZlYdDTQlMmDaN5kzX\nXflIJsPZ06cDeycU1y9dyqr163lgwwZWrV9P/dKlzKivd7AxMzM7BA41JXDVokUsrq1lZSazZ8Qm\nACszGZbU1nLlwoWAJxSbmZmVk0NNCWSzWZa1tLBm1iwmV1dzwZgxTK6uZs2sWZ1u5/aEYjMzs/Lx\nc2pKJJvNsqCxERobCSHsmUNT0JMJxcWvNTMzswPzSE0ZdBVKejqh2MzMzHrGoaYPHeyEYjMzM+s5\nh5o+dLATis3MzKznHGr60MFOKDYzM7Oe80ThPnagCcVmZmbWOx6pqSAHGjMzs9JxqDEzM7NUcKgx\nMzOzVHCoMTMzs1RwqDEzM7NUcKgxMzOzVHCoMTMzs1RwqDEzM7NUcKgxMzOzVChLqJF0nKS7JW2R\n9Lqk5ySNL6q5RtLGeP8qSe8p2j9U0tL4GDlJ90s6tqjmKEk/kbRN0lZJt0saXlQzVtJDktolbZJ0\nvaRMUc3Jkp6U9Iak30m6utR9YmZmZuVV8lAj6UhgNfAmMAWoBa4EtiZqvgrMAr4AnAG0A82SDk8c\n6mbgfGAGcA5wHLCs6HT3xsefGNeeA9yWOE8GeJhoOYizgEuBTwPXJGqyQDOwDhgPXA0skPT5XneC\nmZmZ9blyrP30NeDlEEIyFPyuqOYK4NoQwoMAkj4FbAYuBO6TNAL4LHBRCOHncc1ngDZJZ4QQnpZU\nSxSa6kIIz8Y1XwYeknRVCGFTvP+9wIdCCFuA5yV9E7hO0oIQwi7gEqAK+Fz8e5ukU4E5wO2l7hwz\nMzMrj3JcfpoGPCPpPkmbJa1NjnpIqgFGA48XtoUQtgNrgPp402lEgStZ8xLwcqLmLGBrIdDEHgMC\ncGai5vk40BQ0AyOBExM1T8aBJlkzTtLInr55MzMzq4xyhJp3A38DvARMBr4PfFfS/xvvH00UPDYX\nvW5zvA9gFLAzDjvd1YwG/pDcGULYDbxaVNPVeehhjZmZmfVz5bj8lAGeDiF8M/79OUl/CVwO3F2G\n81XM7NmzGTmy82DOzJkzmTlzZoVaZGZm1n80NTXR1NTUadu2bdvKdr5yhJr/BtqKtrUB/0/88yZA\nRKMxyRGSUcCziZrDJY0oGq0ZFe8r1BTfDTUEOLqo5vSitoxK7Ct8H3WAmi4tWbKE8ePH76/EzMxs\n0OrqH/pr166lrq6uLOcrx+Wn1cC4om3jiCcLhxDWEYWFiYWd8cTgM4Gn4k2twK6imnHA8UBLvKkF\nODKe1FswkSgwrUnUnCTpmETNZGAb8EKi5pw4ECVrXgohlC9OmpmZWUmVI9QsAc6S9HVJfybpYuDz\nwK2JmpuBeZKmSToJ+DHwCvAA7Jk4fAewWNIHJdUBdwKrQwhPxzUvEk3o/ZGk0yVNAG4BmuI7nwAe\nJQovd8fPopkCXAvcGkLoiGvuBXYCd0o6QdIngAbgpjL0jZmZmZVJyS8/hRCekfQx4Drgm0TPf7ki\nhPCPiZrrJQ0jeqbMkcAvgHNDCDsTh5oN7AbuB4YCjwBfKjrdxURh6TEgH9dekThPXtJHiSYrP0X0\nPJy7gPmJmu2SJgNLgWeALcCCEMIdh9YTZmZm1pcUQqh0Gwac+OnIra2trZ5TY2Zm1gOJOTV1IYS1\npTy2134yMzOzVHCoMTMzs1RwqDEzM7NUcKgxMzOzVHCoMTMzs1RwqKkw331mZmZWGg41FZDL5Zjf\n0MCkmhouHDuWSTU1zG9oIJfLVbppZmZmA1Y51n6y/cjlcsyor2dOWxsL8nlEtGR589KlzHjiCZa1\ntJDNZivdTDMzswHHIzV97Ma5c5nT1sbUONBAtFjV1Hye2W1t3DRvXiWbZ2ZmNmA51PSx1StWMCWf\n73Lf1Hye1cuX93GLzMzM0sGhpg+FEBje0bFnhKaYgGEdHZ48bGZm1gsONX1IEu1VVXQXWQLQXlWF\n1F3sMTMzs+441PSxCdOm0ZzputsfyWQ4e/r0Pm6RmZlZOjjU9LGrFi1icW0tKzOZPSM2AViZybCk\ntpYrFy6sZPPMzMwGLIeaPpbNZlnW0sKaWbOYXF3NBWPGMLm6mjWzZvl2bjMzs0Pg59RUQDabZUFj\nIzQ2EkLwHBozM7MS8EhNhTnQmJmZlYZDjZmZmaWCQ42ZmZmlgkONmZmZpYJDjZmZmaWCQ42ZmZml\ngkONmZmZpYJDjZmZmaWCQ42ZmZmlgkONmZmZpYJDjZmZmaWCQ42ZmZmlgkONmZmZpYJDjZmZmaWC\nQ42ZmZmlgkONmZmZpYJDjZmZmaWCQ42ZmZmlgkONmZmZpYJDjZmZmaWCQ00/EUKodBPMzMwGNIea\nCsrlcsxvaGBSTQ0Xjh3LpJoa5jc0kMvlKt00MzOzAeewSjdgsMrlcsyor2dOWxsL8nkEBKB56VJm\nPPEEy1payGazlW6mmZnZgOGRmgq5ce5c5rS1MTUONAACpubzzG5r46Z58yrZPDMzswHHoaZCVq9Y\nwZR8vst9U/N5Vi9f3sctMjMzG9jKHmokfU1SXtLiou3XSNoo6XVJqyS9p2j/UElLJW2RlJN0v6Rj\ni2qOkvQTSdskbZV0u6ThRTVjJT0kqV3SJknXS8oU1Zws6UlJb0j6naSrS90PSSEEhnd07BmhKSZg\nWEeHJw+bmZn1QFlDjaTTgS8AzxVt/yowK953BtAONEs6PFF2M3A+MAM4BzgOWFZ0inuBWmBiXHsO\ncFviPBngYaK5Q2cBlwKfBq5J1GSBZmAdMB64Glgg6fO9fuMHIIn2qiq6iywBaK+qQuou9piZmVmx\nsoUaSUcA9wCfB14r2n0FcG0I4cEQwn8AnyIKLRfGrx0BfBaYHUL4eQjhWeAzwARJZ8Q1tcAU4HMh\nhGdCCE8BXwYukjQ6Ps8U4L3AJ0MIz4cQmoFvAl+SVJgkfQlQFR+nLYRwH/BdYE6p+yRpwrRpNGe6\n7v5HMhnOnj69nKc3MzNLnXKO1CwFVoQQnkhulFQDjAYeL2wLIWwH1gD18abTiEZXkjUvAS8nas4C\ntsaBp+AxooGOMxM1z4cQtiRqmoGRwImJmidDCLuKasZJGtmTN9wTVy1axOLaWlZmMntGbAKwMpNh\nSW0tVy5cWK5Tm5mZpVJZQo2ki4D3AV/vYvdoor+/Nxdt3xzvAxgF7IzDTnc1o4E/JHeGEHYDrxbV\ndHUeelhTctlslmUtLayZNYvJ1dVcMGYMk6urWTNrlm/nNjMz64WSP6dG0juJ5sNMCiF0lPr4/cns\n2bMZObLzYM7MmTOZOXPmQb0+m82yoLERGhsJIXgOjZmZpUpTUxNNTU2dtm3btq1s5yvHw/fqgLcD\na7X3b+khwDmSZhHNcRHRaExyhGQUULiUtAk4XNKIotGaUfG+Qk3x3VBDgKOLak4vat+oxL7C91EH\nqOnSkiVLGD9+fKdtvb1jyYHGzMzSpqt/6K9du5a6urqynK8cl58eA04iuvx0Svz1DNGk4VNCCL8l\nCgsTCy+IJwafCTwVb2oFdhXVjAOOB1riTS3AkZJOTZx7IlFgWpOoOUnSMYmaycA24IVEzTlxIErW\nvBRCOKg46eUOzMzMKq/kIzUhhHb2BgYAJLUD/xNCaIs33QzMk/RfwHrgWuAV4IH4GNsl3QEslrQV\nyBHdkbQ6hPB0XPOipGbgR5L+BjgcuAVoCiEURlgejdtyd3wb+Tvic92auDR2L/B3wJ2SvkMUyBqI\n7tA6IC93YGZm1j/01ROFO12TCSFcTxRAbiMaVXkrcG4IYWeibDbwIHA/8G/ARqJn1iRdDLxINDr0\nIPAkcFniPHngo8BuolGgHwN3AfMTNduJRmaqiUaUbgAWhBDuOJg35uUOzMzM+gf5qbU9J2k80Nra\n2spXZsxg1fr1XT4dOACTq6tZtW5dH7fQzMysf0rMqakLIawt5bG99tMh8HIHZmZm/YdDzSHwcgdm\nZmb9h0PNIfJyB2ZmZv2DQ80h8nIHZmZm/YNDzSHycgdmZmb9QzmeKDzoeLkDMzOzyvNITYk50JiZ\nmVWGQ42ZmZmlgkONmZmZpYJDTT/ih/SZmZn1nkNNhXmFbzMzs9Lw3U8V5BW+zczMSscjNRXkFb7N\nzMxKx6GmglavWMGUfL7LfVPzeVYvX97HLTIzMxu4HGoqxCt8m5mZlZZDTYV4hW8zM7PScqipIK/w\nbWZmVjoONRXkFb7NzMxKx6GmgrzCt5mZWen4OTUV5hW+zczMSsMjNf2IA42ZmVnvOdSYmZlZKjjU\nmJmZWSqsaSjHAAAVZElEQVQ41JiZmVkqONSYmZlZKjjUmJmZWSo41JiZmVkqONSYmZlZKjjU9ENe\nmdvMzKznHGr6iVwux/yGBibV1HDh2LFMqqlhfkMDuVyu0k0zMzMbELxMQj+Qy+WYUV/PnLY2FuTz\niGhhy+alS5nxxBNeB8rMzOwgeKSmH7hx7lzmtLUxNQ40AAKm5vPMbmvjpnnzKtk8MzOzAcGhph9Y\nvWIFU/L5LvdNzedZvXx5H7fIzMxs4HGoqbAQAsM7OuhuKUsBwzo6PHnYzMzsABxqKkwS7VVVdBdZ\nAtBeVeUVvM3MzA7AoaYfmDBtGs2Zrv9TPJLJcPb06X3cIjMzs4HHoaYfuGrRIhbX1rIyk9kzYhOA\nlZkMS2pruXLhwko2z8zMbEBwqOkHstksy1paWDNrFpOrq7lgzBgmV1ezZtYs385tZmZ2kPycmn4i\nm82yoLERGhsJIXgOjZmZWQ95pKYfcqAxMzPruZKHGklfl/S0pO2SNkv6maS/6KLuGkkbJb0uaZWk\n9xTtHyppqaQtknKS7pd0bFHNUZJ+ImmbpK2Sbpc0vKhmrKSHJLVL2iTpekmZopqTJT0p6Q1Jv5N0\ndSn7xMzMzMqvHCM1HwBuAc4EJgFVwKOS3lookPRVYBbwBeAMoB1olnR44jg3A+cDM4BzgOOAZUXn\nuheoBSbGtecAtyXOkwEeJrrMdhZwKfBp4JpETRZoBtYB44GrgQWSPt/7LjAzM7O+VvI5NSGE85K/\nS/o08AegDvhlvPkK4NoQwoNxzaeAzcCFwH2SRgCfBS4KIfw8rvkM0CbpjBDC05JqgSlAXQjh2bjm\ny8BDkq4KIWyK978X+FAIYQvwvKRvAtdJWhBC2AVcQhS8Phf/3ibpVGAOcHup++dgeV6NmZlZz/TF\nnJojie5QfhVAUg0wGni8UBBC2A6sAerjTacRBa5kzUvAy4mas4CthUATeyw+15mJmufjQFPQDIwE\nTkzUPBkHmmTNOEkje/F+e80rdZuZmfVeWe9+UjTUcDPwyxDCC/Hm0UTBY3NR+eZ4H8AoYGccdrqr\nGU00ArRHCGG3pFeLaro6T2Hfc/H33+6nZlt376+UvFK3mZnZoSn3SM33gBOAi8p8ngHPK3WbmZkd\nmrKN1Ei6FTgP+EAI4b8TuzYR/X09is6jKKOAZxM1h0saUTRaMyreV6gpvhtqCHB0Uc3pRU0bldhX\n+D7qADVdmj17NiNHdr5CNXPmTGbOnLm/l3Vp9YoVLNjPSt2Lly+HxsYeH9fMzKxSmpqaaGpq6rRt\n27byXQApS6iJA80FwF+FEF5O7gshrJO0ieiOpV/H9SOI5sEsjctagV1xzc/imnHA8UBLXNMCHCnp\n1MS8molEgWlNouYbko5JzKuZTHRJ6YVEzUJJQ0IIuxM1L4UQ9tvzS5YsYfz48QfTJfvVk5W6PXnY\nzMwGiq7+ob927Vrq6urKcr5yPKfme8AngYuBdkmj4q+3JMpuBuZJmibpJODHwCvAA7Bn4vAdwGJJ\nH5RUB9wJrA4hPB3XvEg0ofdHkk6XNIHoVvKm+M4ngEeJwsvd8bNopgDXAreGEDrimnuBncCdkk6Q\n9AmgAbip1H3THa/UbWZmdujKMafmcmAE8G/AxsTXXxcKQgjXEwWQ24hGVd4KnBtC2Jk4zmzgQeD+\nxLFmFJ3rYuBForueHgSeBC5LnCcPfBTYDTxFFJ7uAuYnarYTjcxUA88ANwALQgh39O7t945X6jYz\nMzs0CqG78QHrjqTxQGtra2tJLj/B3rufZicmCweiQLOkttZ3P5mZWSokLj/VhRDWlvLYXvupn/BK\n3WZmZofGq3T3I16p28zMrPc8UtNPFQKNLw+amZkdHIeafsjLJZiZmfWcLz/1M14uwczMrHc8UtPP\neLkEMzOz3nGo6WdWr1jBlP0sl7B6+fI+bpGZmdnA4FDTj/RkuQQzMzPrzKGmH/FyCWZmZr3nUNPP\neLkEMzOz3nGo6WeuWrSIxbW1rMxk9ozYBOBhiSW1tVy5cGElm2dmZtZvOdT0M8nlEj58/PGMHz6c\nk4YM4YZhw9iVy3Hj3Ll+Xo2ZmVkXHGpKpJSTd7PZLFcuXEhVNsuiN97g+d27+df2dh5/+WXqly5l\nRn29g42ZmVkRh5pD0N7eXrYn/xaeV3Oun1djZmZ2UOTbg3tO0nig9cx3v5sF69czJfnk30yGxbW1\nh/zk30k1Naxav77L27sDMLm6mlXr1vX6+GZmZpWwdu1a6urqAOpCCGtLeWyP1ByCS9atK8uTf7t7\nXk0hfvp5NWZmZvtyqDkE9d2EikN98m/yeTU5YD4wCbgw/v53wLYhQ/y8GjMzswSHmkNQzif/Tpg2\njZ9JzADqgVXAA/H3s4DXduzwZGEzM7MEh5pDUM4n/161aBHfOuoo/haYCp0ucZ0H/P1rr3mysJmZ\nWYJDzSFo6Sa0lOLJv9lslrcdcQTndrEvAOd6cUszM7NODqt0Awaye2pq+LP16/dMFg5EgWZJbS3L\nDvHJvyEEsrt37xmhyQE3AquB4UA78Oof/8j27dsZMWLEIZ3LzMwsDTxScwhuuOsu1syaxeTqai4Y\nM4bJ1dWsmTXrkG/nhn0nC3c1t+ba9nY+/v73e26NmZkZHqk5JMOHD2dBYyM0NhJCKPndSBOmTaN5\n6VJa8nnmEM2tSTofyMS3jy9obCzpuc3MzAYaj9SUSDlury4sbvkYMIWub+9uyef5+b/8S8nPbWZm\nNtA41PRj2WyW+596isOHD2cHXV+Cqgc2btzI9u3bK9hSMzOzynOo6edGjBjBkLe/nRtgzyWoHUQj\nNh8BfgC8bdcu/tc553hujZmZDWoONQPAhGnTeJy9l6CKR2x+CTQ895xX7zYzs0HNoWYAuHLhQjjs\nMER0W/ccYAKwgGhezceAJcDo3/yGv//KVyrWTjMzs0pyqBkARowYwdDjjiMQPafm/ewdrfln4H1x\n3Wbgn3/wA75++eUesTEzs0HHoWaA+MAFF/CIxHDgJvaO1nwcOIUo6OwG3gs8edttTHjXu9i4cWPF\n2mtmZtbXHGoGiKsWLWLJCSewhWi0ZgrRpajLiSYLv5+9ozZvBd6xdSuTxo7la5dd5jujzMxsUHCo\nGSCy2SzLWloYfvLJDCFa2HI18Cs6j9oULkmdChyWz/PgD3/I5JEjqctm+dpll/mylJmZpZZDzQCS\nzWZZ9stfsqGqijzRGlBPsXfUphBuLgR+Dnwnsf+oHTt49oc/5Kyjj/acGzMzSyWHmgEmm80y7bOf\nZaXEDqJgUxi1KYSb44meY3M2+865ec+uXXvm3GzYsKEi78HMzKwcHGoGoG/ccAONJ5zAO4EtsGfU\nphBuXqH7OTe1RM+6yW/dyox3vtOXpczMLDUcagagwvya4y6/nI0SjwDtROFmGJ0DTnLOzcFclvKk\nYjMzG6gcagaobDbLt7//fVa/8grfOPpojgOagdfZG3CK59x0d1nqTWDorl0sv+02phx5JB985zuZ\n39Dg0RszMxtQHGpKIIRQsXMfd9xx/GL9esZcfjlXVFVxHDAGeBT2mXPT1WWp9wGHA/OIHug3PARG\nbtjAY7fcsudZN5V8f2ZmZgfLoeYQ/OD665lUU8OFY8cyqaamYqMbhVGb1v/5H467/HJWH3YY34JO\nc266uyz1HF0/66YK0NatTBwzhg8fcQQfOv545jc0sH37doccMzPrlxxqDsHJ993HqvXreWDDBlat\nX0/90qUVXVQym81y3fe/z7OvvspfXX45zxxxBK8Aj9D9ZamunnVzCtHozXzgfwHh9dd59fe/555b\nbqF+5EgmHHYYp40Y0enBfg46ZmZWaQ41h+D9IaD4ZwFT83lmt7Vx07x5lWzWnnDzm1yOpxJzboov\nS0HXz7pJjt68j+hDMgy4FWgBzsnn2ZnL8cAPf8gZI0dyQiazz2gOREGnEHZCCDQ1NfVNB6SM+63n\n3Ge9437rOfdZ/+JQE5P0JUnrJL0h6d8lnd6b40zN51m9fHmpm9drY8aM2TPnpviyFOw776Z49OY5\nOk8wLtxBNR8YC/w98PEQCK+/zrbf/55/uuUWThs5kj+TOCGT4cRMZk/o+fJnP7vPJazi4JP8+UD7\nS/26/nrupqamQfm+u/u5O8n9hb9ountNT451MPW9PUZfjHD25Bz7+wvao7H7KsU/1tyvpXVYpRvQ\nH0j6BNE6kV8AngZmA82S/iKEsGW/Ly4+FjCso4MQApIOWN8XCnNuvnH99Sz6yldYcc897Nixg5VE\nQeUx9l6Wgr2jN98iGr2BKOwsIAo4FxGN2BRGcy4H/p1osvGtwBvAVcBt8b5fhcCTr7/OTuCeW27h\nn265hTywCxgan7sj/llEDwnMZDLszOc5vGh/dz/39nUD4dx/BN6byQy6952sHSYxMpPhzWHD+MjF\nF/ONG24gm82Sy+W4ce5cVq9YwdA33+T3O3YwJAR++8Yb/OVhh/H2t7wFve1tfOCCC7jsa1/jtuuu\nY/WKFQzv6KC9qooJ06Zx1aJF+xxreEcH24YMYeiRR9Lx2mtkd+/ep74rB3OM06ZMQcD/19zcZTtK\nobgdvT1HqY6TJsV90vLqq8xvaOhRn7hfy6jwL6DB/EX0d3Jj4ncR3Sz0lW7qxwOhFUIo+spDmFhd\nHfq7V155JZxy9NHhnyCcDOEhCBPj9n8YwvT452nxzyHeX6iZCOGbEFZC+LvE90sTPy+D8JH4+4cg\nHAXh/vjns7r4OVnb1f5Sv26gnPttg/R9F35+mOgzV/jz9SCEibW1YcOGDeEjJ54YVmYyYVt8vMJx\nzyx6zf1S+POhQ8PDmUyn7SszmfCRE0/sdKw8hO3xcR4qOk6hfvv27fv8mdq+ffsBj7Etfk8P9uC4\nPVXcjp6cY9q0aSU5Tlp11SfTetgn7tcQWltbAxCA8aHEf58P+stPkqqAOuDxwrYQQiAawKjv6fEe\nyWQ4e/r00jWwTAqXpV5oaGDEO9/JlyXeQTSp+Gz2Xp5KTjAu3EEF+042Lnx/he4vYf0F8Gv2Xs4q\n/rn4ctf+akvxuoFy7rcO0vdd+Plc6DR37Xzgb198kc+cdx5z2tqYms9zU3y8wnGPLXrNr0Pg5jff\n5Nx8vst5cMljib3zy84rOs7+5s3dOHfuAY9xU/yezu/BcXuquB29PUepjpMmXfUJ9KxP3K/l5ctP\ncAwwBNhctH0zMK6b17wF4B+Joqbi778A/rG6mhs+/nHWrl1bntaW2PRLL2X6pZeyY8cO/uG73+Wy\nBx7gi7t28QfgFuDdwH8D3wM2xa9ZG297a/xzR/x9J1FfFLY9AlwArIxf95bEz2/v4ufpB9hf6tcN\nlHPvGKTvu/BzV3+S3hEC//X887w9n2dt0fGmA9uKXlfY3tWxjs3nOx3rYOpX/vSnTL/00k7bV95/\nP9MPcIzeHLenitvRk3Ns27Ztz/+7DuU4adVVnxQ+awfbJ+5XaGtrK/z4llIfW2GQT1KS9A5gA1Af\nQliT2P4d4JwQwj6jNZIuBn7Sd600MzNLnU+GEO4t5QE9UhNdadkNjCraPoq9gxPFmoFPAuuBP5Wt\nZWZmZunzFqCa6O/Skhr0IzUAkv4dWBNCuCL+XcDLwHdDCDdUtHFmZmZ2UDxSE1kM3CWplb23dA8D\n7qpko8zMzOzgOdQAIYT7JB0DXEN02elXwJQQwh8r2zIzMzM7WL78ZGZmZqkw6J9TY2ZmZungUGNm\nZmap4FDTQ6Va+DKNJM2XlC/6eqGo5hpJGyW9LmmVpPdUqr2VIukDkpZL2hD30T6PoD5QP0kaKmmp\npC2ScpLul3Rs372LvnWgPpP0D1189h4uqhlsffZ1SU9L2i5ps6SfSfqLLur8WUs4mH7z560zSZdL\nek7StvjrKUlTi2r65HPmUNMDiYUv5wOnEj3lvTmeZGyR/yCabD06/jq7sEPSV4FZRAuHnkG0AkOz\npMMr0M5KGk40Gf2LRA+j7uQg++lmoqftzwDOAY4DlpW32RW13z6LraTzZ29m0f7B1mcfIHow+JnA\nJKAKeFTSWwsF/qx16YD9FvPnba/fA18lWhexDngCeEBSLfTx56zUi0ml+YseLnw52L6Iwt7a/ezf\nCMxO/D6CaFHvv6502yvYZ3lgek/6Kf79TeBjiZpx8bHOqPR7qlCf/QPwz/t5zaDus/j9HhO/37MT\n2/xZ612/+fN24H77H+Az8c999jnzSM1BKvXClyn25/Elgv8r6R5JYwEk1RD9aybZf9uBNbj/9jjI\nfjqN6HEMyZqXiB4YOZj78oPx5YIXJX1P0tGJfXW4z44kGuV6FfxZ64FO/Zbgz1sXJGUkXUT0rLen\n+vpz5ufUHLzeLHw52Pw78GngJeAdwALgSUl/SfShDnTdf6P7ron93sH00yhgZ/w/hu5qBpuVREPV\n64A/A74NPCypPv7Hx2gGcZ9JEtHw/i9DCIV5bv6sHUA3/Qb+vO0j/v98C9ESCDmiUZeXJNXTh58z\nhxormRBCch2P/5D0NPA74K+BFyvTKhsMQgj3JX79jaTngf8LfBD414o0qn/5HnACMKHSDRlguuw3\nf9669CJwCjAS+DjwY0nn9HUjfPnp4PVm4ctBLYSwDfhP4D1EfSTcfwdyMP20CThc0oj91AxqIYR1\nRH9mC3dYDNo+k3QrcB7wwRDCfyd2+bO2H/vpt3348wYhhF0hhN+GEJ4NIcwlupHmCvr4c+ZQc5BC\nCB1AKzCxsC0empwIPFWpdvVnko4g+kO+Mf5Dv4nO/TeC6A4D91/sIPupFdhVVDMOOJ5o+HfQk/RO\n4G1A4S+jQdln8V/MFwAfCiG8nNznz1r39tdv3dT787avDDC0zz9nlZ4hPZC+iC6jvA58CngvcBvR\nDO+3V7pt/eELuIHoVrx3Ae8HVhFdE31bvP8rcX9NA04C/gX4P8DhlW57H/fTcKJh2vcRze7/2/j3\nsQfbT0TD4uuIhrvrgNXALyr93irRZ/G+6+P/Sb4r/h/jM0AbUDWI++x7wFaiW5RHJb7ekqjxZ62H\n/ebPW5d99vdxf70L+EuiOUa7gA/39ees4p0x0L6InpOxnuh2tBbgtEq3qb98AU1Et7i/QTRr/V6g\npqhmAdHtfa8DzcB7Kt3uCvTTX8V/Me8u+rrzYPsJGEr0LI0tRJPyfgocW+n3Vok+I5qY+AjRvwb/\nBPwW+D5F/9gYhH3WVX/tBj5VVOfPWg/6zZ+3Lvvs9rgf3oj75VHiQNPXnzMvaGlmZmap4Dk1ZmZm\nlgoONWZmZpYKDjVmZmaWCg41ZmZmlgoONWZmZpYKDjVmZmaWCg41ZmZmlgoONWZmZpYKDjVmZmaW\nCg41ZmZmlgoONWZmZpYK/z/XKcbzBWSiKAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f386fa76cc0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "zhl = dict(zhl)\n",
    "\n",
    "zhx = []\n",
    "zhy = []\n",
    "for item in zhl.items():\n",
    "    zhx.append(item[0])\n",
    "    zhy.append(item[1])\n",
    "\n",
    "\n",
    "plt.plot(zhx, zhy, 'ro')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "en_tokens = json.load(open('/data/xuwenshen/ai_challenge/data/train/train_words/en_tokes.json'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9903244/9903244 [00:11<00:00, 875131.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9886910\n",
      "9886910\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import *\n",
    "\n",
    "en_keep = []\n",
    "zh_keep = []\n",
    "\n",
    "for i in tqdm(range(len(en_tokens))):\n",
    "    if len(en_tokens[i]) > 60 or len(en_tokens[i]) == 0:\n",
    "        continue\n",
    "        \n",
    "    if len(zh_tokens[i]) > 50 or len(zh_tokens[i]) == 0:\n",
    "        continue\n",
    "        \n",
    "    en_keep.append(en_tokens[i])\n",
    "    zh_keep.append(zh_tokens[i])\n",
    "    \n",
    "print (len(en_keep))\n",
    "print (len(zh_keep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "_ = open('/data/xuwenshen/ai_challenge/data/train/train_words/en_data.json', 'w')\n",
    "json.dump(en_keep, _)\n",
    "_.close()\n",
    "\n",
    "_ = open('/data/xuwenshen/ai_challenge/data/train/train_words/zh_data.json', 'w')\n",
    "json.dump(zh_keep, _)\n",
    "_.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "en_keep = json.load(open('/data/xuwenshen/ai_challenge/data/train/train_words/en_data.json'))\n",
    "\n",
    "zh_keep = json.load(open('/data/xuwenshen/ai_challenge/data/train/train_words/zh_data.json'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9886910/9886910 [01:42<00:00, 96234.65it/s]\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "from tqdm import * \n",
    "\n",
    "\n",
    "zh_pfc = defaultdict(lambda : 0)\n",
    "en_pfc = defaultdict(lambda : 0)\n",
    "\n",
    "for i in tqdm(range(len(en_keep))):\n",
    "    for j in range(len(en_keep[i])):\n",
    "        en_pfc[en_keep[i][j].lower()] += 1\n",
    "    for j in range(len(zh_keep[i])):\n",
    "        zh_pfc[zh_keep[i][j]] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "669948\n",
      "589781\n"
     ]
    }
   ],
   "source": [
    "zh_pfc = dict(zh_pfc)\n",
    "en_pfc = dict(en_pfc)\n",
    "\n",
    "print (len(zh_pfc))\n",
    "print (len(en_pfc))\n",
    "\n",
    "zh_voc = sorted(zh_pfc.items(), key=lambda x: x[1], reverse=True)\n",
    "en_voc = sorted(en_pfc.items(), key=lambda x: x[1], reverse=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "zh_ = open('/data/xuwenshen/ai_challenge/data/train/train_words/zh_voc_whole.json', 'w')\n",
    "json.dump(zh_voc, zh_)\n",
    "zh_.close()\n",
    "\n",
    "en_ = open('/data/xuwenshen/ai_challenge/data/train/train_words/en_voc_whole.json', 'w')\n",
    "json.dump(en_voc, en_)\n",
    "en_.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50000/50000 [00:00<00:00, 1576995.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9684311685367554\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhYAAAFkCAYAAAB8RXKEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3XuYXXV97/H3d5JwCblxTSIXQQ0xlhZIBEG02iKgRaGC\nRzrIwaJoU6TUHD1qHz2l9XjgoRWoR64K5wQUx1p58IkUDI3VYxGImACKcr9UbgkkhEnIjZD5nT/W\n3syenb1nz97Zs/damffreeaZzNpr7fWbH8Osz3x/v/VbkVJCkiSpHXq63QBJkrTjMFhIkqS2MVhI\nkqS2MVhIkqS2MVhIkqS2MVhIkqS2MVhIkqS2MVhIkqS2MVhIkqS2MVhIkqS2aTpYRMQ7I2JRRDwT\nEQMRcdIIjnl3RCyLiE0R8XBEfLS15kqSpDxrpWKxG3AvcA7Q8EEjEXEgcDPwY+BQ4GvANRFxXAvn\nliRJORbb8xCyiBgA/jSltGiYfS4C3pdS+oOKbX3A1JTSn7R8ckmSlDudmGNxFLCkatti4OgOnFuS\nJHXQ+A6cYwawsmrbSmBKROycUtpcfUBE7AmcADwJbBr1FkqStOPYBTgQWJxSWt3pk3ciWLTiBOCG\nbjdCkqQC+wjwnU6ftBPBYgUwvWrbdGBtrWpFyZMA3/72t5kzZ84oNk2VFixYwKWXXtrtZowp9nnn\n2eedZ5931gMPPMAZZ5wBpWtpp3UiWNwJvK9q2/Gl7fVsApgzZw5z584drXapytSpU+3vDrPPO88+\n7zz7vGu6MpWglXUsdouIQyPisNKmN5S+3r/0+oURcV3FIVeV9rkoImZHxDnAh4BLtrv1kiQpV1q5\nK+StwD3AMrJ1LC4GlgN/X3p9BrB/eeeU0pPAicB7yNa/WAB8PKVUfaeIJEkquKaHQlJK/49hAklK\n6awa234GzGv2XJIkjWVLlizhwQcfJCJIKdX9DLz276effrqrbc7rXSHqgt7e3m43YcyxzzvPPu+8\nPPT5T3/6U+6///6GF+iRfgZG7b3OPffclr7H8WRDBuPa1Get2q6VN0dLRMwFli1btswJP5JUEL/8\n5S+56667gNG76Dbz+a/+6q+62R0dtRMwE7iC7LbLt2ab56WUlne6LVYsJGkH0NfXx+rVqzt+IW/1\nr2vVti8QLR57BfAnZJMeu8lgIUlttmjRIp566qlRL6WPpb/Im7U9F+hu2m07jq1e16FbDBaSxpQr\nr7ySgYGBtv81PxYv8nm+eG/PBbpbHmrxuIOBPcnPfwuDhaTcWbVqFd/97neB7f+rHtghL/p5uKjn\n9eLd6gW62/YA9m7x2BfJ1n/o9s8EGCwktckNN9zAmjVrgNbCwDe+8Q1+9atfdfm7aE03LvLdvqjn\n+eI9GXhdtxvRovXbceytZHMsus1gIY1xDz74IEuWLGkpDBRh4l4nLvqdvsjn4aLeA8zqdiOGsT0X\n6G5qdQWKnYBzgMvJbjntJoOFVHDr169n4cKFTYWCL3zhC6xfn59fvaN58R/ti343LvKzu3DOWvLz\nE7St7i4R1XmvAM8A8+n+hb3b55dU8uSTT3LzzTePKBzcdtttLFq0qKvtbWcYGK2Lfycu+vsCkzpw\nnmp5uKiPtYt3J+1H9nP1YNX2yy67bEQrb1500UWdbvJrDBbSKFizZg033HBDw7sKzj//fFavXt3R\ntrUrELQrDKxi9C5QRwL9o/TelbpxkfeiPjLNXqDzsPJm5efjjjuO2bObq1EtX77cYCEVwXe+8x3W\nrFlT9xfEt771LZYuXTrq7djeYNCOQNDuSsBolfb76cxFvxsX+Z2BzV04by3li/ejwKsV27/+9a93\ndeXNys/vete7OOSQQ7rQO2OPwUJj2r333svPf/7zrkxObDUgbE8waFcgmAMMtOm9YHQv/jvqX/ab\ngQlkyze38y9yaP1CfsQRR/C2t72tsx2h3DFYaIf08ssvs3DhQmDbX5Kf/vSn2bp1a1vP10pIaCUg\nrGf7w0E7qgMDtDcMPAe097/IoJml9++EemX3Wn+5Q3tL6aeffjp77LFHh75TqT6DhQrp6quvZsuW\nLUN+wd5yyy3ccsstbTtHM2Gh2ZDwJK3/Jb29waBdgaDdlYAJwJY2vydkoWIfsqGD3Rh60T/vvPOY\nNWtW2/+a/8AHPsABBxwwCt+NlH8GC+VOvdsn2zEsMRphYQ2tXWRbDQjbGwzaFQh2BTa26b0gCxXl\nv/hfJRuvL9uesXoYGgbmz5/PuHHdfrC0tOMyWKgrbr/9du6777623T450sAw0rDQ7HBDKyGh1YDw\nEvByi8e200ayKsDejLz8P9LPe++9N6eddlrHvydJ289goVHzq1/9ip/97GevXSy253kNjYLDSAJD\nM2Gh2aDQSkjYnsrBFGDtdhwPg9WBFWRhpazVSX9nnHEG06ZN285WSSo6g4W224033shzzz1HRLBg\nwQK2bGlupHwk1YbhgsPLjDw0NBMYmgkLz5OtfNcpaxkMBhOAX1e81mylYPbs2bznPe/pYOsl7cgM\nFhqxxYsX8+ijj7ZUfRguPDSqNowkNIw0MDQTFpqtKOwCbGryGBgMCC+RVQ/KGlUOIJs7MHHiRM46\n66wWzixJ7Wew0DbWrVvH9ddf33SAaCU8jOT2ySNoXPYfaWBoJiw0uwDRJgZDQmLo91UOCVD/roKD\nDjqIE088sYkzSlL+GCzGuBUrVvD973+fH/7wh9x2220jOqZegKgXHhoFh0bVhrU0Dg5rRrBPsxot\nQDTckMMee+zB6aef3uYWSVL+GSzGmMsvv5xLL72Uxx57rOG+zQSI4cLDrsBwd/SPJBCMpNIwGVg3\ngv1gsLLwJEOHL4YLCy5AJEmNGSx2YPfeey+33357w+GMdgWI4SoPw4WHkdw+OR1Y2WCfdQwGhvHA\n/RWv1QsMhx56KO94xzsavLMkaaQMFjuQn/zkJxx33HHDLlddK0Q0EyAOJJt7UMtw4aFRxaHR7ZMr\ngb3IJkhWD0ucddZZzJs3b0homDx5Mh/96EcbnFWS1G4GiwK7++67OfLII4fdZ7+qr6tDxDpqh4i9\ngD3rvGe9ALE96zI0e/vk/PnzGT/eH19Jyht/MxfI2rVrmTp1at3XG1UjVlD74l9vCGM0AsRIHtK0\n2267efukJBWUwSLnent7+e53v1vzteogUV2NqFWJqBUi2h0gyuHhd8CGiu2V6zIcc8wxHHbYYS2e\nQZKUVwaLHIqovRrEcEGiVog4iuw2zEq1QsSLDA0AI1Wv+vCFUoA49NBDeec739nCO0uSispgkRO1\nwkSzQaK6GlFrbYd6VYhJ1L8zo16A+Hxp+OKUU05h5syZdY6WJI0lBosuahQmhgsSU4EZVdtqVSNq\nBYkgWxmy0ssMBohxwG8qXisHiOOPP55Zs2bVeEdJkjIGiw4baZhYy9AwMQ54U9VxI6lG7A88VbUt\nMRgiNpEtElU+/rLLLmPSpEneqilJaonBogOOOuooli5dOmRbrTBRXZWoHtoY6bBGpaeA15GtPbEr\n8NuKYy+44AIOPvhgTj311BG8kyRJjRksRlF1daJRmJhIVmEoayVIwLbLVT9b2n7ZZZdxzjnn1J0c\nKknS9jJYtNlNN93EKaec8trXjcLEgQxdybIyTKwGNo7gnNWTK8sB5Pbbb+eYY44ZUbslSWoHg0Wb\nzJ07l3vuuee1r8uBolaYeCNDO74yTFRXJXZj28pFOUg8AmytOOY3v/kNb3nLW1ppviRJbWGw2E4H\nHHAATz2VTY+srk48w+BFfxrZg7TKhgsTldYzGCR6yOZIlPdfunRpwyW9JUnqJINFiw4++GAeeeQR\nYPjqROUEzJGGCRg6vFG5b0rVN4pKkpQfBosmnXnmmXzrW98ChgaKzQwGiv0YDBn1wsQ4smGMSrXC\nxNSpU3nppZfa1XxJkkaVwaIJ5bspKgPFUwyGgFrViXqVia2l99iXbcPE4YcfzvLly9vWbkmSOsVg\nMQK1AsUjwEDp9XKgGOlQh8MckqQdlcFiGMcccwx33HEHMFhZeAJ4pfR6daB4Gag3aFEOE+sZuhKm\nYUKStCMxWNRRXaWonENRHSgqqw57kD0ttKzWA7wc6pAk7agMFlVOPvlkFi1aNGTYYySBouxFBsPE\nrsA9Fa9ZnZAk7egMFhUqqxSTGAwU88iGORpNyCwHinUMrVAYKCRJY4XBoiQiXqtS9JMtbrU7sA+D\noWIlsKXGseVA8QJOxpQkjW093W5At/3t3/7ta6FiElkwWEc27LEPWaBYX9q+haHP9dgPeHNp+4Nk\nz/aALFAYKiRJY9GYrlhMmzaN/v5+9iV72NczwOuBXdh22GNPsuCwmcEKxQAOeUiSVGnMViwigkn9\n/exHFiheJKtSlEPF0wwd1lhNNjRSeZfHw2TzMX70ox8ZKiRJYoxWLCqHPh4CJgL7M/zkzOrbRvcG\neqZP5+kVK0a5tZIkFceYCxblUJHIQsXvAa+ShYpnGVxNs6wcKDYwdNjjeSsUkiRto6WhkIj4VEQ8\nEREbI+KuiDiiwf4fiYh7I2J9RDwbEddGxB6tNbl15VDxAlmImM1gqHiaLFRUJq3KKsXvyIY93v72\ntzvsIUlSHU1XLCLiNOBi4JPAL4AFwOKIODiltKrG/scA1wF/DdxMdn2+GvgG8KHWm950u9mXbD4F\nZKGi1tDHqwwGivLdHkFW4XjaQCFJ0rBaqVgsAK5OKV2fUnoQmE82UvCxOvsfBTyRUro8pfSfKaU7\nyILFkS21uAX1QsWLZKFit4p9K6sUj5GlICKsUkiSNAJNBYuImEC2EOWPy9tSdsVdAhxd57A7gf0j\n4n2l95gO/BfgX1tpcLMqQ8VODIaKp8nSUPnBYLuy7QTNfYGPfO5zDAxUz7yQJEm1NDsUshcwjmwR\nykorGXyUxhAppTsi4gzgnyNil9I5FwHnNnnuph1yyCGvhYpdyNaoKIeKspcZDBTjgN8Ak8kWyXLo\nQ5Kk5oz6XSER8Rbga8DfAbcBM4Gvkg2HnD3csQsWLGDq1KlDtvX29tLb29vwvIsXL+al3/yGZ8hW\ny6wVKmAwVKwhS0flIOLQhyQp7/r6+ujr6xuyrb+/v0utyUQzF9DSUMgG4NSU0qKK7QuBqSmlD9Y4\n5npgl5TShyu2HQP8BzAzpVRd/SAi5gLLli1bxty5c5v4dgbtF7HNnIp6oeJJYBOGCklS8S1fvpx5\n8+YBzEspLe/0+ZuaY5FS2gIsA44tb4vskaDHAnfUOWwi2c0WlQbIbrSIZs4/UhHButK/G4WKBzFU\nSJLULq3cFXIJ8ImIODMi3gxcRRYeFgJExIURcV3F/j8ETo2I+RFxUKla8TVgaUqp7ctWHnDAAewN\nrGVkoQIMFZIktUvTcyxSSt+LiL2ALwPTgXuBE1JKL5R2mUG2QnZ5/+siYhLwKbK5FS+R3VXyhe1s\n+zaeffZZBp56ihcwVEiS1A0tTd5MKV0BXFHntbNqbLscuLyVczXj8H335Xkah4rHSl8bKiRJaq8d\n5ummBx98MC+S3SoKg6tqlpVDxQqyFTUNFZIktd8OEyxefuQRXgVeRxYq1gB7ll4rh4qNZOMwhgpJ\nkkbHDvF00+zGFHgz2YJXT5MtiLWawVAxkWxexX6l1z/+8Y93o6mSJO3QCl+xuPbaawHYm6HzKjaR\nLeFdtpwseCSyIHLNNdd0sJWSJI0NhQ8W55+dLd65B9nciUr7MHgHyHSyZbqfAZ/9IUnSKCl0sDj7\n7LN5BjiMrFqxgtLTSBkcAlld+no8zquQJGm0FTpY3HTttYwnm5RZHgJ5hsFQMQV4ofT1M8BBBx3U\njWZKkjRmFDZYnH322bwIvJFtby0t+wWD8yoAHn/88U40TZKkMauwwaLv2muZxuCEzd1L2yufVjqB\n7C6RZ4AbbrihG82UJGlMKWSwuOmmm9hANiFzbWnbGqDyAesryaoZ5btATj/99M42UpKkMaiQweKM\nU05hL7JqxVpgWmn7ZAbvAjkQ7wKRJKnTChksNpCtqvl86euXGBwC2bm07VWyUDF9+vSOt0+SpLGq\ncMFi1qxZTCWrVrxCtsJmpfsYOmFzxYq2P5ldkiTVUbhg8eijjzKDwWrFJgarFeUwUZ6w+frXv77z\nDZQkaQwrVLB4+OGHgfrVikcYWq148sknO9U0SZJEwYLF7Nmz+X2yigRsW63oYbBaseeee9Z8D0mS\nNHoKFSwgq1S8RPZskEpPAG9isFqxatWqTjZLkiRRoGBx9913v3aLKcCLDFYrdiULHOVqhSRJ6o7C\nBIsjjzySPclW2ZxS9drDZHMryr761a92rF2SJGlQYYLFOAarFWvJKhVlUfpcHgb5zGc+06lmSZKk\nCoUIFq+++ir7kA13lE0jCxf9ZM8JKQ+D7Lvvvtu+gSRJ6ohCBIs5c+YwhWztiplVr20AdmOwWvH0\n008jSZK6oxDB4tFHH31tGOQ5BidtTiYbHnHSpiRJ+VCIYLEL2ZoV1R4nu8W0bNasWZ1pkCRJqin3\nwWLlypXMAVaRPSa9UvWkzV//+tcda5ckSdpW7oPFEUccwYbSv1cyOAwyiW2HQXbeeedt30CSJHVM\n7oPFU0899dr8ikq/A3zEmCRJ+ZL7YDGJbFGs6iW8qxv+zW9+szMNkiRJdeU6WLzwwgscUvr3S2Rr\nV5RVz684++yzO9YuSZJUW66DxVVXXcWa0r8HGJxbsZnsThFvM5UkKV9yHSx++IMfsB6ovon0FaBy\nmubhhx/euUZJkqS6ch0sdiWbX/EKQ8NFVO136623dqxNkiSpvlwHixmlz/8JbCQbBplG9nyQl8lC\nB8DkyZM73zhJkrSNXAeLLTW2PQtUP2Zs4sSJHWiNJElqJNfBYmPpc+XQR/UwiCRJyo9cB4u1wFTq\nB4sALrjggo62SZIk1ZfrYLEZ2Er2WPSyyvUrEvDJT36y082SJEl15DpYQPZU06lkEzfHkQWNl4EV\nZI33+SCSJOVH7oNFpU1A9TTNSZMmdaMpkiSphlwHi0aNy3XjJUkag3J9bd6Z+g3sAT7/xS92sDWS\nJKmR8d1uwHDGM/QukHLISGRrXHz2s5/teJskSVJ9ua5YbAB2J5u4uSvZuhYvA88DE7rYLkmSVFuu\ng0VlteJlsoBR3h5AhMtlSZKUJ7keChluYSyAqVOndrA1kiSpkcJULFzWW5Kk/Mt1sKhUHSY+9pd/\n2ZV2SJKk+nIdLPaps30TcNFFF3WyKZIkaQRyHSw2k03YnAr0k03gXE12h4gkScqfXAeLsvJTTmFw\nSMQ7QiRJyp9CBIvqCGGkkCQpn1oKFhHxqYh4IiI2RsRdEXFEg/13ioj/FRFPRsSmiHg8Iv68pXMP\nvmcrh0uSpFHU9DoWEXEacDHwSeAXwAJgcUQcnFJaVeewfwH2Bs4CHgNm0kSoqVWx2HVXZ1pIkpQ3\nrSyQtQC4OqV0PUBEzAdOBD4G/EP1zhHxXuCdwBtSSi+VNv+uteZmoSIBPT2FGMWRJGlMaerqHBET\ngHnAj8vbUkoJWAIcXeewDwC/BD4fEU9HxEMR8Y8RscuIz1v19Yy9926m2ZIkqUOarVjsBYwDVlZt\nXwnMrnPMG8gqFpuAPy29x5XAHsDHmzw/Adx2113NHiZJkjqgE88K6QEGgNNTSi8DRMR/A/4lIs5J\nKW2ud2A/2RNOy8MfW0qfZ86cOeqNliQp7/r6+ujr6xuyrb+/v0utyTQbLFYBW4HpVdunAyvqHPMc\n8Ew5VJQ8QJYX9iObzFnTTLL1KzaSBYzudpUkSfnS29tLb2/vkG3Lly9n3rx5XWpRk3MsUkpbgGXA\nseVtkd33eSxwR53Dfg68LiImVmybTVbFeLrROQcqGunj0iVJyrdWbq24BPhERJwZEW8GrgImAgsB\nIuLCiLiuYv/vkK3E/X8jYk5E/CHZ3SPXDjcMUuZTTSVJKo6m51iklL4XEXsBXyYbArkXOCGl9EJp\nlxnA/hX7r4+I44CvA3eThYx/Bv7HSM5XK0xYsZAkKZ9amryZUroCuKLOa2fV2PYwcEIr56pmpJAk\nKb8KtcqUoUKSpHzLfbDwAWSSJBVH7oNFJR9AJklSvuU+WNSKEBMmTOh4OyRJUmO5Dxa1WLGQJCmf\nChcs3nviid1ugiRJqqMwwaJco7jsuuuG3U+SJHVProNFOUwk4JXS5ylTpnSvQZIkaVi5DhY7kz18\nbAPZk00DSCl1tU2SJKm+XAcLqLGOhRM3JUnKrdwHi0pGCkmS8q0QwSLIGuogiCRJ+ZbrYFE5eXOg\n9LVDIZIk5Veug0W5QlGuWICTNyVJyrNcB4tatQkrFpIk5Veug0W1QjVWkqQxqBDX6nFkDR3AoRBJ\nkvKsEMFiK1mo6AF6egrRZEmSxqRCXKXLFQuwYiFJUp4VIliUDXS7AZIkaViFCBZbyW49Hd/thkiS\npGHlPlgEMJFsOGQAbzeVJCnPch0sgqxSsRmHQSRJKoJcB4ta0zSdvClJUn7lOlhUKi/r7VCIJEn5\nletgEaWPncl5QyVJEpDz63VlbaL8hFNJkpRfuQ4WicHJmwlX3pQkKe9yf5WuHAoZh3MsJEnKs1wH\ni+oI8adnnNGVdkiSpJHJdbCoNABc8PWvd7sZkiRpGLkOFgNkVYv1ZHMspkyZ0t0GSZKkYeU6WGwG\npjAYMCRJUr7lOlgMAC8xGCxcdVOSpHzLdbAoG09psSzvCJEkKddyHSzGAdPIeSMlSdJrCnHNfpXa\nDySTJEn5Mr7bDRjOVrI5Fj0YLCRJKoJcVywcCpEkqVi8ZkuSpLYxWEiSpLYxWEiSpLYpVLBwHQtJ\nkvKtUMHClTclScq3wgSLcd1ugCRJaij3wSKAPcnWsXAoRJKkfMt1sNgTmAWsIXsQmUMhkiTlW66D\nxWrgYbJQMQ4rFpIk5V2ugwUMHQqxYiFJUr7lOlhUDoX42HRJkvIv18FiNfAIsDtWLCRJKoKWgkVE\nfCoinoiIjRFxV0QcMcLjjomILRGxfCT7W7GQJKlYmg4WEXEacDFwPnA4cB+wOCL2anDcVOA6YMlI\nz2XFQpKkYmmlYrEAuDqldH1K6UFgPrAB+FiD464CbgDuGumJrFhIklQsTQWLiJgAzAN+XN6WsjLC\nEuDoYY47CzgI+PtmzlddsZAkSfnWbMViL7IlJVZWbV8JzKh1QETMAi4APpJSGmjmZJUVi1zPMpUk\nSQCMH803j4gesuGP81NKj5U3j/T4+8kqFgPAK8DJJ59Mb28vvb29bW+rJElF09fXR19f35Bt/f39\nXWpNJpqZEFkaCtkAnJpSWlSxfSEwNaX0war9p5IVHF5lMFD0lP79KnB8SumnNc4zF1h2JDAH+C3w\nAvDY1q309Fi7kCSpnuXLlzNv3jyAeSmlEd2F2U5NXaVTSluAZcCx5W2Rzag8FrijxiFrgUOAw4BD\nSx9XAQ+W/r10xOfGyZuSJOVdK0MhlwALI2IZ8Auyu0QmAgsBIuJC4HUppY+WJnb+tvLgiHge2JRS\neqCZk25toaGSJKmzmg4WKaXvldas+DIwHbgXOCGl9EJplxnA/u1qYAJW4eRNSZKKoKk5Fp1SnmPx\nRmAfYDPwIvD4wIDDIZIkDaNQcyw6bU/gYLL7WyVJUv7lOlhIkqRiMVhIkqS2MVhIkqS2KUywcB0L\nSZLyrzDBYid8bLokSXlXiGAxiYI0VJKkMS731+sDgXOALV1uhyRJamxUn266va4Anidb4rOp561L\nkqSuyHWwOIfs6aY+J0SSpGLI/VCIJEkqjsIEC283lSQp/woTLLzdVJKk/CtEsPB2U0mSiiH31+sD\n8XZTSZKKItd3hVTebroV51hIkpR3uQ4WlbebludYGC4kScqv3A+FQDbHYly3GyFJkhrKfbA4kKxy\nsRmHQiRJyrtcD4VUzrEYj0MhkiTlXa6DRXmOxa44FCJJUhEUZijkFRwKkSQp73JdsagcCtlp3DiH\nQiRJyrlcB4vKoZCXDBWSJOVeIYZCPgWM6+nxWSGSJOVcrisW5aGQS4GtAwNWLCRJyrlcVyzOAZYC\nfwHsuuuuViwkScq5wlQsJu2+uxULSZJyrhAVi08AE/fcs8utkSRJjRSmYrFlzZout0aSJDVSiIrF\njcCUrVudYyFJUs7lumJxJTAXSMC6ceOcYyFJUs7lumJRdguw0+67d7sZkiSpgVxXLBJwK/A1nGMh\nSVIR5Lpi4RwLSZKKJdcVC+dYSJJULLmuWJQ5x0KSpGLIdcXCORaSJBVLrisWzrGQJKlYcl2xcI6F\nJEnFkuuKRZlzLCRJKoZcVyycYyFJUrHkumLhHAtJkool1xWLyjkW6ydMcI6FJEk5l+uKRdmtPT28\n46STut0MSZLUQK4rFgn4V2DBhAn89POf73ZzJElSA7muWJwD3A1c+MorfOOii7rdHEmS1ECug8WV\nwN8Bp6TEzxct6nJrJElSI7kOFmUBTNyyxbtCJEnKuUIEC+8KkSSpGAoRLH7kXSGSJBVCS8EiIj4V\nEU9ExMaIuCsijhhm3w9GxG0R8XxE9EfEHRFx/EjOk8huNb10zhw+85WvtNJUSZLUQU0Hi4g4DbgY\nOB84HLgPWBwRe9U55A+B24D3ka139RPghxFxaKNznTZuHP90yCEsvO02Jk+e3GxTJUlSh7VSsVgA\nXJ1Suj6l9CAwH9gAfKzWzimlBSmlr6aUlqWUHkspfRF4BPhAoxP989atLLj/fv78+ONZt25dC02V\nJEmd1FSwiIgJwDzgx+VtKbtVYwlw9AjfI4DJwIsN9wXeOzDAggce4OIvfamZpkqSpC5otmKxFzAO\nWFm1fSUwY4Tv8d+B3YDvjfSk7x0YcB0LSZIKoKNLekfE6cD/AE5KKa1qtP8CYGrp379esYKTTjqJ\n3t5eent7R7OZkiQVQl9fH319fUO29ff3d6k1mWhm0anSUMgG4NSU0qKK7QuBqSmlDw5z7J8B1wAf\nSin9qMF55gLLljH4dNPjDjyQJU88MeK2SpI0Fi1fvpx58+YBzEspLe/0+ZsaCkkpbQGWAceWt5Xm\nTBwL3FHvuIjoBa4F/qxRqKjFdSwkSSqGVoZCLgEWRsQy4BdkIxYTgYUAEXEh8LqU0kdLX59eeu08\n4O6ImF7ksRuRAAAJ30lEQVR6n40ppbXDnahyHYsbXcdCkqTca/p205TS94DPAl8G7gH+ADghpfRC\naZcZwP4Vh3yCbMLn5cCzFR//1Ohc58ycydJzz+XGO+90HQtJkgqgpcmbKaUrgCvqvHZW1dd/1Mo5\nIKtY+OAxSZKKI9fPCrnyuec4+vLLOfXoo10gS5KkAsh1sHCBLEmSiiXXwaLMBbIkSSqGQgSLACZu\n2eJ8C0mScq4QwSIB6ydMIFsyQ5Ik5VUhgoULZEmSVAwdfVZIs1wgS5KkYsl1xcIFsiRJKpZcVyyu\nvPlm5s6d2+1mSJKkEcp1xUKSJBWLwUKSJLWNwUKSJLWNwUKSJLWNwUKSJLVNroPF/Pe/n/PPO88n\nm0qSVBC5DhY+Nl2SpGLJdbDwsemSJBVLroNFmY9NlySpGAoRLHxsuiRJxVCIYOFj0yVJKoZCBAsf\nmy5JUjHk+iFkPjZdkqRiyXXFwsemS5JULLmuWPjYdEmSiiXXFQtJklQsBgtJktQ2BgtJktQ2BgtJ\nktQ2BgtJktQ2BgtJktQ2uQ4W89//fs4/7zwfmS5JUkHkOlhc+dxzHH355Zx69NGGC0mSCiDXwSLI\nHpm+4IEHuPhLX+p2cyRJUgO5DhZl7x0Y4OeLFnW7GZIkqYFCBIsAJm7ZQkqp202RJEnDKESwSMD6\nCROIiG43RZIkDaMQweJHPT2846STut0MSZLUQK6fbpqAW3t6uHTOHG78yle63RxJktRArisW58yc\nydJzz+XGO+9k8uTJ3W6OJElqINcViytvvpm5c+d2uxmSJGmEcl2xkCRJxWKwkCRJbWOwkCRJbWOw\nkCRJbWOwkCRJbWOwkCRJbWOwkCRJbWOwkCRJbWOwkCRJbWOwkCRJbWOw0Gv6+vq63YQxxz7vPPu8\n8+zzsaWlYBERn4qIJyJiY0TcFRFHNNj/3RGxLCI2RcTDEfHR1pqr0eT//J1nn3eefd559vnY0nSw\niIjTgIuB84HDgfuAxRGxV539DwRuBn4MHAp8DbgmIo5rrcmSJCmvWqlYLACuTildn1J6EJgPbAA+\nVmf/vwQeTyl9LqX0UErpcuD7pfeRJEk7kKaCRURMAOaRVR8ASCklYAlwdJ3Djiq9XmnxMPtLkqSC\nGt/k/nsB44CVVdtXArPrHDOjzv5TImLnlNLmGsfsAvDAAw802Txtj/7+fpYvX97tZowp9nnn2eed\nZ593VsW1c5dunL/ZYNEpBwKcccYZXW7G2DNv3rxuN2HMsc87zz7vPPu8Kw4E7uj0SZsNFquArcD0\nqu3TgRV1jllRZ/+1daoVkA2VfAR4EtjUZBslSRrLdiELFYu7cfKmgkVKaUtELAOOBRYBRESUvv7f\ndQ67E3hf1bbjS9vrnWc18J1m2iZJkl7T8UpFWSt3hVwCfCIizoyINwNXAROBhQARcWFEXFex/1XA\nGyLiooiYHRHnAB8qvY8kSdqBND3HIqX0vdKaFV8mG9K4FzghpfRCaZcZwP4V+z8ZEScClwLnAU8D\nH08pVd8pIkmSCi6yu0UlSZK2n88KkSRJbWOwkCRJbZO7YNHsA87Gqoh4Z0QsiohnImIgIk6qsc+X\nI+LZiNgQEf8WEW+qen3niLg8IlZFxLqI+H5E7FO1z+4RcUNE9EfEmoi4JiJ2q9pn/4j414hYHxEr\nIuIfIiJ3P1vbIyL+JiJ+ERFrI2JlRNwUEQfX2M8+b6OImB8R95X6oj8i7oiI91btY5+Pkoj4Qun3\nyyVV2+3zNoqI80v9XPnx26p9itPnKaXcfACnka1bcSbwZuBq4EVgr263LW8fwHvJJtCeTLa2yElV\nr3++1HfvBw4BfgA8BuxUsc+VZGuFvIvsgXJ3AP9R9T63AsuBtwJvBx4Gvl3xeg/wa7L7pX8fOAF4\nHvhKt/uozf19C/BfgTml7/PmUt/tap+Par+fWPpZfyPwJuArwGZgjn0+6n1/BPA4cA9wiT/no9rX\n5wO/AvYG9il97FHUPu96h1Z903cBX6v4OsjuIvlct9uW5w9ggG2DxbPAgoqvpwAbgQ9XfL0Z+GDF\nPrNL73Vk6es5pa8Pr9jnBOBVYEbp6/cBW6gIf8BfAGuA8d3um1Hs871KffMO+7zjfb8aOMs+H9U+\nngQ8BPwx8BOGBgv7vP39fT6wfJjXC9XnuSkpRWsPOFMNEXEQ2W2/lX25FljKYF++lex248p9HgJ+\nV7HPUcCalNI9FW+/BEjA2yr2+XVKaVXFPouBqcDvtelbyqNpZP3wItjnnRARPRHxZ2Tr5txhn4+q\ny4EfppT+vXKjfT6qZkU2tP1YRHw7IvaHYvZ5boIFwz/gbEbnm1NoM8h+WIbry+nAK6Uf0Hr7zCAr\ng70mpbSV7GJauU+t88AO+t8tIgL4J+D2lFJ5HNQ+HyURcUhErCP7i+wKsr/KHsI+HxWl8HYY8Dc1\nXrbPR8ddwJ+TVRDmAwcBPyvNfyhcn+f1IWRSnl0BvAU4ptsNGSMeBA4l+6vpQ8D1EfGH3W3Sjiki\n9iMLze9JKW3pdnvGipRS5TM97o+IXwD/CXyY7Oe/UPJUsWjlAWeqbQXZ/JTh+nIFsFNETGmwT/Ws\n4nHAHlX71DoP7ID/3SLiMuBPgHenlJ6reMk+HyUppVdTSo+nlO5JKX0RuA/4a+zz0TCPbALh8ojY\nEhFbyCYD/nVEvEL216t9PspSSv1kEyvfRAF/znMTLErpuPyAM2DIA8669jCVIkopPUH2Q1DZl1PI\nxtHKfbmMbNJO5T6zgQMYfEDcncC0iDi84u2PJfshX1qxz+9Htsx72fFAPzDkdqmiK4WKk4E/Sin9\nrvI1+7yjeoCd7fNRsYTsboDDyKpEhwK/BL4NHJpSehz7fNRFxCSyUPFsIX/Ouz0btmrm64eBDQy9\n3XQ1sHe325a3D2A3sv/pDyOb6fvp0tf7l17/XKnvPkD2i+IHwCMMvT3pCuAJ4N1kf6n8nG1vT7qF\n7BfLEWSl/4eAb1W83kP2F+StwB+QjRGuBP5nt/uozf19BdnM6HeSJfjyxy4V+9jn7e/3C0p9/nqy\n2+wuJPsF+sf2ecf+G1TfFWKft7+P/xH4w9LP+duBfyt9r3sWsc+73qE1OvgcsntxN5Klp7d2u015\n/CArTw6QDR9Vfvyfin3+juw2pQ1kM3vfVPUeOwNfJxuGWgf8C7BP1T7TyP5a6Se7sH4TmFi1z/5k\n6zq8XPohvAjo6XYftbm/a/X1VuDMqv3s8/b2+zVkaylsJPur7TZKocI+79h/g3+nIljY56PSx31k\nSytsJLuT4zvAQUXtcx9CJkmS2iY3cywkSVLxGSwkSVLbGCwkSVLbGCwkSVLbGCwkSVLbGCwkSVLb\nGCwkSVLbGCwkSVLbGCwkSVLbGCwkSVLbGCwkSVLb/H+wretwyh13oQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f84354df208>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import json\n",
    "from tqdm import *\n",
    "\n",
    "zh_voc = json.load(open('/data/xuwenshen/ai_challenge/data/train/train_words/zh_voc_whole.json'))\n",
    "\n",
    "prob = []\n",
    "\n",
    "counlst = list(map(int, np.array(zh_voc)[:,1]))\n",
    "n2 = sum(counlst)\n",
    "\n",
    "prev = 0\n",
    "for i in tqdm(range(50000)):\n",
    "    n1 = counlst[i] + prev\n",
    "    prev = n1\n",
    "    prob.append(n1/n2)\n",
    "plt.plot(range(0, 50000),prob , 'ro')\n",
    "print (prob[-1])\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "\n",
    "zh_voc = json.load(open('/data/xuwenshen/ai_challenge/data/train/train_words/zh_voc_whole.json'))\n",
    "\n",
    "zh = list(np.array(zh_voc)[:50000, 0])\n",
    "\n",
    "zh.append('eos#')\n",
    "zh.append('go#')\n",
    "zh.append('pad#')\n",
    "zh.append('nuk#')\n",
    "\n",
    "ids = [i for i in range(len(zh))]\n",
    "zhvoc = dict(zip(zh, ids))\n",
    " \n",
    "\n",
    "zh_ = open('/data/xuwenshen/ai_challenge/data/train/train_words/zh_voc.json', 'w')\n",
    "json.dump(zhvoc, zh_)\n",
    "zh_.close()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50000/50000 [00:00<00:00, 1554332.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9853794648686306\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhYAAAFkCAYAAAB8RXKEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3XuU3WV97/H3NyE3IEQImotQI5IAaosStVKtN2yAakGF\no0QUi5fKAavmrNXaWly0HlsWepRSTcSKPQHRqYpLFioYBartkZsmolWUm+GSAEFCmCSQe57zx7M3\ns7Mztz3Zs/fzm3m/1pqVmd/+/fbvmSeT/D7zXCOlhCRJUjtM6HYBJEnS2GGwkCRJbWOwkCRJbWOw\nkCRJbWOwkCRJbWOwkCRJbWOwkCRJbWOwkCRJbWOwkCRJbWOwkCRJbdNysIiIP46IayJibUTsjohT\nhnHNayJiZURsjYi7IuJdIyuuJEkq2UhaLA4AbgfOBYbcaCQi5gHfAW4AjgUuAS6LiD8Zwb0lSVLB\nYl82IYuI3cCbUkrXDHLORcDJKaU/aDjWA8xIKf3piG8uSZKK04kxFi8Hrm86tgI4vgP3liRJHbRf\nB+4xG1jXdGwdcFBETEkpbWu+ICJmAicC9wFbR72EkiSNHVOBecCKlNL6Tt+8E8FiJE4EvtLtQkiS\nVGFnAl/t9E07ESweAWY1HZsFbOyvtaLmPoArr7ySY445ZhSLpkZLlizh4osv7nYxxhXrvPPGQ53f\ncsstPPDAA6SUiIiW/gRavmao97jqqqs4/fTTu1ae/t7rk5/8ZHf+ckbRRGAmeYzDI/nQfd0oRyeC\nxc3AyU3HFtWOD2QrwDHHHMNxxx03WuVSkxkzZljfHWadd169zu+55x5WrFhRxEOuHe/1l3/5l12r\n0+G46KKLul2EMW0yMAdYRv7N/SX5cFeGErQcLCLiAOBIIGqHjoiIY4HHU0oPRsSFwNyUUn2tikuB\n82qzQ/4NOAE4HXBGiDRGbd26lcsuu6yIh2/je5x//vn09vYSEYMVX2Pcs+l7gI0ly8gP1lVdLsdI\nWixeAvwHeQ2LBHy6dvxy4N3kwZqH109OKd0XEW8ALgY+CKwB3pNSap4pIqmNfvjDH/KrX/2qYw/w\n2267jS9/+ctd/q7Hp9IflI8Bh3a7EA0O6HYBRklz10C3tBwsUko/YpBpqimls/s59p/AwlbvJY1V\n1157LatXr2b16tUsXbq0bb/Nl94c3k31h29pD7l2KP1BuYFyynhntwswChaQx1aUEi5LnRWiLli8\neHG3i1CkzZs3s3z5cmDffqv/wAc+0O/7D3S8Skr/jRn6Hmy7KOch1w5VeVCWUs5DyQ/hseZxchdC\nCf8ODRZ62lgMFlu3buVLX/pSyyHg2muv5dprr+128VvSzYd76Q/qe8iBou7hbhVkFBwCPLPbhaiY\nJ7tdgFFyHWUMXjRYqBLWrl3Lt771rWGFgltvvbW4vv5OPPS79XDfSjm/jQ7mqG4XYBSN1QflaFnT\n7QKMgsnkDbyWkgc6dpPBQl1x11138YMf/GDQsQMXXXQRDz74YMfLNhohYLQf+t1+sFfhoT2WH75j\n8UGp1mwH1gLn0P0He7fvrzHmG9/4Bo8++mi/geGjH/0omzZtGtX7tyMUtDsEdOKhPx2Y24H7DKQK\nD20fvmp2GHAg8Jum45/97Gcrt6ZJ43usWbOmq+uGGCw0bD09PTz++OPDHpS4L0YaEPY1FGym/UFg\nFvCMNr9nf7r5cN8E9Hbx/sr25UE5Wg+5KrzXokWLmD9/fof+lkbfqlWrDBYqwze/+U0eeeSRUZm+\n2GpQGElAaFdf/2g063fiod/t38gPJk8rLF394ftY7aOuir+lDvRer3/96znqqCp0UGksMliMM1dd\ndRXr1q17+j+jfQ0Oww0MrQSFtYz8IdmO/0pHIwR0+6HfCRvoe2hPBW5veO1zn/tcEU3EjX8+73nP\n46STTupwLUljn8FiDNq5cyeXXnrpPrc6DCc0DCcw3EWeX92KkQaEdoSCDW16n24ZSXM4tPe352nT\npvGe97yno9+3pDIYLCpuxYoV3HPPPdx///186lOfaunaoYLDUKGhlW6HVoPCSB/s7errn157r3Zr\nRx/4cMPAq1/9al74wheOwnchSQMzWFTIt7/9bR544IGWBksOFh4GCw7DCQ1HAJOGWY5Wg8K+dB20\no69/E30hYD/glw2vjSQENIeBk08+mSOOOGIfSylJ5TFYFGrNmjVcffXVw+rGmMvAm7cMFB6GCg4L\nGN7YieEGhk6OMRisr38koWD69Om8613v6vdekqQ9GSwKcccdd7Bo0SLWrl076Hn9tUCMJDzMA6YM\nUabhhIbRDgz1gPAk0LhU1nAHA06dOpX3vve9o1xKSVKdwaJLNm7cyOzZs9myZcuA5wwnROxk4ADx\nXPIyrwMZKjgMJzRMAHYP47y6elB4lLxpTl1jSwLsPXbgsMMO401velMLd5IkdYPBooPe/va309PT\n0+9rwwkRAwWIwQZGDhYehgoOk8nLxA6mHipaHZQ4f/58Fi1aNMS7S5KqxmAxyiL6H6nQHCSGEyIG\n2u53X8LDYLaTf0BmM3RogPy9nn766cyaNWsf7ipJqjKDxSjoL0wMFiTWsHcAmEue8tisvxCxlT1X\nENyjLAy9hsRwWxvOOOMMZs7sL9pIkpQZLNpk4sSJ7N6952iDxjDRGCT6a43orzujvxAxUAvELGBd\nP8froWKg8NA4CPItb3kLc+bMGeAOkiQNzWCxD8466yy+/OUv73GsvzCxkb3DRHOQaCVE9GcdfeHh\nQOCnDa81hofTTjuN2bNnt/DOkiQNn8FiBJq7OvoLE60GiVbHQvTXArGG3H0xadIkfvL+97f4jpIk\n7TuDRQsaA8VQYWIq8JyGr/clSAwUIj73uc+Nue1+JUnVZrAYhv4CRX9h4lnk5aTrGsPEemDgFSv6\n1EPEZvrCRz1EvPGNb+Q5z3nOgNdKktRtBotBDBQoHqfvoT9Yy0Rzq8QB7N1yUQ8SD5HHYtSvufrq\nqzn11FP3ofSSJHWewaIfAwWKxtaJxjETg4WJRk/SFyQmAHc0nH/jjTfy2te+duSFliSpAAaLBtOm\nTWPr1q1A/4HiIKA+GXO4YQL2HCPReG59YSlJksaKgTbFHFeWLl1KRLB161aeTV8QWEMOFc8nt1DM\nIQeKJ8nTO/tb2KruMODo2udr6Bt4mVJ6+kOSpLFm3LdY1Ls9+muheBnQC+yir4WiMUhMAbY1fG3L\nhCRpvBvXwSIi+g0UC8mzMnrpP1DUbaMvTOwG7mp4zTAhSRqPxmWwWLBgAXfffTfPJoeCeqCobzO+\nmb0Dxf7AUw3v0d/aEocccgjr168ftXJLklS6cTfGIiK4++67OYz8zd8JTCSPoZhM3xiK+viJ+mDN\np8hTS+tjJxrHTXzoQx8ipWSokCSNe+OqxaKx66PeElGfNtrcQjGTvKjVw+SFr6aQu0vqYWJ/4Gd3\n3smCBQtGu9iSJFXGuGmxqIeK+myPA8mhormFor6CxXryKpqHAYcAD5JDxWzge9/7Hk+mZKiQJKnJ\nuGixqIeK3wFrgZcDG8iBohfY1HDuQNuMPxP4/de9jhtuuKEjZZYkqYrGfLCoh4q1ta+Poi9U1Ls9\nZpADBvQFirvJ00wnAvtNmcKjtYWzJEnSwMZ0sGgMFVOAefQ/fbSXvkAxGfhFw2s7nTYqSdKwjdlg\n0RgqZpNbJRpbKQ4nj5uAvbs9ng1snjGDJ554onMFliRpDBiTwaIxVNTXpmgMFZBDxVxyS8Y08oZg\nkAPIrzduZPr06Z0rsCRJY8SYCxaNoWI+edpLc6iAvlaKDcBqcsh4CHjCrg9JkkZsTAWLE0444elQ\n8TyGDhWNXR9rcRluSZL21Zhax+LOG29kLTk47EdfqGj8JvsLFb+bPNlQIUlSG4yZFov6LqUHkVfI\nrIeKycD22jmNoWIqeXXNrTNnsu2xxzpdXEmSxqQxESwOO+wwJpLXnZhDX6jYjxwq9iPPDKmHioPJ\nS3LvnDWLxx55pCtlliRpLBoTwYK1a9lF3xLd9TEVO8ndII2h4pnkVozeAw9kk6FCkqS2qnywqHeB\nvAR4hL0Has6lL1TMJIeKtUDatAlJktRelR68eeqppwK5FWITg8/+OJA8rsLZH5IkjZ5KB4uV11wD\n5N1H60t113cnrYeK+2tfz8BQIUnSaKtssDjyyCNZC/whfeMqZpF3J62Hiq3AltrXa4GVK1d2p7CS\nJI0TlQ0WD957LwcBT9DXBbIOOLThnPuAo4HdwJw5czjuuOM6WkZJksabSg7enDdvHtvJ+4A82fTa\nVPrGVRxJHnvxEJAeeqijZZQkaTyqZIvF/fffz3Po6wI5vHa83gXyKDCJ3BWyFti4cWM3iilJ0rhT\nuWBx7LHHArllot4GUd+pdBL5G3ocOILcWnHIIYe4U6kkSR0yomAREedFxOqI2BIRt0TES4c4/8yI\nuD0inoyIhyLiSxFxyEju/Ytf/IIF5NaK3fT15Uwgb4F+B3mhrM1AL7B+/fqR3EaSJI1Ay8EiIt4G\nfBq4AHgx8HNgRUQcOsD5rwAuB74IPB84HXgZ8K+t3vvDH/5wfk/2XF2z3gWyo3bsSXIXyKte9apW\nbyFJkvbBSFoslgBfSCldkVL6DXAO8BTw7gHOfzmwOqW0NKV0f0rpJuAL5HDRkksuuYT57D1gs+5e\n8iyQ+koVP/rRj1q9hSRJ2gctBYuImAQsBG6oH0t5xanrgeMHuOxm4PCIOLn2HrOA/wF8d6QFXkPu\n9oC+1orHyYtgbSa3Vrzzne8cydtLkqR90GqLxaHARPKSEY3Wkff62kutheIdwNciYjvwMLAB+EAr\nN547dy5z6Wut2EYerFn3KHln03prxRVXXNHK20uSpDYY9XUsIuL5wCXA3wPfJz///w+5O+S9g127\nZMkSZsyYAcDDDz/MAeRgMZm8HfoscmvFY8Cz6GutOOOMM0bhO5EkqSw9PT309PTscay3t7dLpcmi\nlb0zal0hTwGnpZSuaTi+HJiRUnpzP9dcAUxNKb214dgrgP8C5qSUmls/iIjjgJUrV658erXMiOAw\n+gZt7kffduj30LcYlvuBSJLGs1WrVrFw4UKAhSmlVZ2+f0tdISmlHcBK4IT6scj7lp8A3DTAZfuT\nJ2802k3utYi9T9/btGnTeBG5ZaKuHio2AgfT11rxghe8YDhvKUmSRsFIZoV8BnhfRJwVEUcDl5LD\nw3KAiLgwIi5vOP/bwGkRcU5EPLfWWnEJcGtK6ZHh3HDr1q1sIa+kuX/Ta48DM+kbW/HLX/5yBN+S\nJElqh5bHWKSUvl5bs+Lj5GEOtwMnppR+VztlNn2rbJNSujwiDgTOI4+teII8q+RvhnvPifQN2nyK\nvpkgQU5G9dYKSZLUXSMavJlSWgYsG+C1s/s5thRYOpJ7HX/88SwEbiN3eWxoeG0N8HvkYAFw5ZVX\njuQWkiSpTYrfK+SWW26hPr51A3mtirp64evdIGeeeWbHyiVJkvZWfLCYyp4rbU4nd4M8wZ6DNoc1\nClSSJI2qooPFE088wXxyl8ezm17bTg4ddXfdfXfHyiVJkvpXdLA4//zz2V77fC15VGi9wPU/668f\neeSRnSyaJEnqR9HB4uabb96jG2Q/8nTTJ4BDyN0gj3ajYJIkqV9FB4v9gUeAg5qOJ/Ys+Mc+9rGO\nlUmSJA2s6GBxGHnJzq3kAZt1zbNBPvrRj3ayWJIkaQBFB4tttT+3A88gh4td5C6RxkWxpk6duvfF\nkiSp44oOFlsGOHZApwsiSZKGpehg8Th5rYpGzetVLFvW7wKgkiSpC4oOFjvJLRSNq23Wg0V9fMXp\np5/e0TJJkqSBFR0sII+vqK+2WZ8Nshl4qPb6zJkzu1QySZLUrOhg0dzt0bza5gRgwoSivwVJksaV\nop/K0xi4gBOAg6ZN62BpJEnSUIoOFgewZwGbC/vjn/60g6WRJElDKTpY1Jfubi7kDvJ4izlz5nS8\nTJIkaWBFB4tE3/4gu4GJ5IGbj9c+j3CzdEmSSlJ0sGiMDY37g0TtY/r06R0vkyRJGlhlgkV/n0+c\nOLGDpZEkSUMpOlg0au70mOY0U0mSilP003mgERQB3PiTn3SyKJIkaRiKDhb70xcu6n/uJu9wOn/+\n/K6USZIkDazoYDGJvJbFfuQwsRnYUPtakiSVpxLP6F3k6aXQ0ILhVFNJkopTdItF3UCzQyRJUlkq\nFyzqprlPiCRJxalEsGgWuIaFJEklqsQYi0YBTJkypdvFkCRJ/ahEi0VzV8gNt97alXJIkqTBFR0s\nmgPF7trHggULulAaSZI0lKKDxTRgKrCNvIbFZgovsCRJ41zxz+n6dumQdzh1uqkkSeUqfvBmavjc\nxbEkSSpb8S0WLo4lSVJ1VCpYPH3MFgtJkopUfLBoFkBKacjzJElS51UqWNTbKSZMqFSxJUkaN4p/\nQjd3egR2hUiSVKrig0WjIM8SmTRpUreLIkmS+lF8sGhsm0hUYH6sJEnjWPHBoj5MM8iF/foNN3Sx\nNJIkaTDFB4t6AXcD24FXvvKVXSyNJEkaTNE9CwcAveRAsQWYhAM3JUkqWdEtFrvIYUKSJFVD0cFC\nkiRVS6WChWtYSJJUtkoEi/qMEHA5b0mSSlZ0sHADMkmSqqXoYNFsAgYLSZJKVolgMZFc0N3YFSJJ\nUskqESx2kUNFJQorSdI4NqJndUScFxGrI2JLRNwSES8d4vzJEfGPEXFfRGyNiN9GxJ8P934T6Wu1\nsCtEkqRytbzyZkS8Dfg08BfAbcASYEVELEgpPTbAZd8AngmcDdwLzGEYoaYeIaaQV97c1WphJUlS\nR41kSe8lwBdSSlcARMQ5wBuAdwOfbD45Ik4C/hg4IqX0RO3wA8O9WQK2kbtCwDEWkiSVrKWukIiY\nBCwEnt5iNOUn/fXA8QNc9mfAT4GPRMSaiLgzIj4VEVOHul9jhKivZWFXiCRJ5Wq1xeJQ8nCHdU3H\n1wFHDXDNEeQWi63Am2rv8XngEOA9g92ssStkK3aFSJJUuk7sblqfKfr2lNJmgIj4X8A3IuLclNK2\ngS58kL5Wix21z//93/+dM888c5SLLElS+Xp6eujp6dnjWG9vb5dKk0UrYxZqXSFPAaellK5pOL4c\nmJFSenM/1ywH/iiltKDh2NHAr4AFKaV7+7nmOGDl88mJZFvtpgl4aOdOJk6cOOwyS5I0nqxatYqF\nCxcCLEwprer0/VsaY5FS2gGsBE6oH4s86OEE4KYBLvsxMDci9m84dhQ5M6wZ6p5B7gpxDQtJkso3\nkuf1Z4D3RcRZtZaHS4H9geUAEXFhRFzecP5XgfXA/42IYyLiVeTZI18arBsE+t8rRJIklavlMRYp\npa9HxKHAx4FZwO3AiSml39VOmQ0c3nD+kxHxJ8BngZ+QQ8bXgI+NpMDOCpEkqVwjGryZUloGLBvg\ntbP7OXYXcOJI7tUocB0LSZJKVrmhC7ZYSJJUrsoEi/3IhbXFQpKkchUdLCaQV+N6BnkKyW5ssZAk\nqWRFB4tEXm3zCXJBiy6sJEkq+1kd7NlikYAJE4ousiRJ41onlvQeseYWi+cdfXR3CyRJkgZV9K//\njS0Wu4Cv/OAH3S2QJEkaVNHBAnKrxVpgJzB37twul0aSJA2m6GCxGZhOLmTRfTaSJAkoPFgUXThJ\nkrQXn92SJKltDBaSJKltDBaSJKltKhUsXM5bkqSyVSpYuAGZJEllq0ywmIgtFpIkla74YBHATPJC\nWbZYSJJUtqKDxUxgPrCBHDAkSVLZig4W64G7gYPJLRZ2hUiSVLaig0Vji8Vu7AqRJKl0RQeL9cBd\n5FDh4E1JkspXdLCAPQdvSpKkshUdLBq7QiZgV4gkSaUrOlg0Dt4M7AqRJKl0RQeLmcAC8viKXV0u\niyRJGlrRwQLy2IrHqEBBJUlS2c/rxq4Qx1hIklS+ooNFY1eIC2RJklS+ooNFI8dYSJJUvuKDhWMs\nJEmqjqKf181jLCRJUtmKfl43jrGQJEnlKzpYSJKkaqlUsHC6qSRJZatMsHC6qSRJ5atMsHC6qSRJ\n5Ss+WDjdVJKk6ij6ee3uppIkVUvRwaJxuulkHLwpSVLpig4WdQdSkYJKkjTOFf+8ngecC+zocjkk\nSdLQ9ut2AQazDHgU+Diwu8tlkSRJQys6WJwLHINTTSVJqoriu0IkSVJ1GCwkSVLbVCZYuKS3JEnl\nq0ywcB0LSZLKV4lgcSB5kSxJklS24oPFPPLskG3YFSJJUumKnm7auI7FfuSuEMOFJEnlKjpY1Nex\nmEYFmlYkSdLIntcRcV5ErI6ILRFxS0S8dJjXvSIidkTEquHeax4u6S1JUlW0HCwi4m3Ap4ELgBcD\nPwdWRMShQ1w3A7gcuH6491oG/GHtZrtwjIUkSaUbSYvFEuALKaUrUkq/Ac4BngLePcR1lwJfAW4Z\n7o3OBb5GDhVON5UkqXwtBYuImAQsBG6oH0v5aX89cPwg150NPBf4h5EU0ummkiRVQ6stFoeSn/Hr\nmo6vA2b3d0FEzAf+CTgzpdTyJqXzcLqpJElVMaqzQiJiArn744KU0r31w8O9fgrwPeBfgO3AKaec\nwuLFi1m8eHHbyypJUtX09PTQ09Ozx7He3t4ulSaLVsYt1LpCngJOSyld03B8OTAjpfTmpvNnABuA\nnfQFigm1z3cCi1JKP+znPscBK19Gnm76APAwcMfu3bZaSJI0iFWrVrFw4UKAhSmlYc/CbJeWukJS\nSjuAlcAJ9WORn/QnADf1c8lG4IXAi4Bjax+XAr+pfX7rUPech10hkiRVxUi6Qj4DLI+IlcBt5Fki\n+wPLASLiQmBuSuldtYGddzReHBGPAltTSr8e6kauvClJUrW0HCxSSl+vrVnxcWAWcDtwYkrpd7VT\nZgOHt6NwjStvOitEkqTyjWjlzZTSspTSvJTStJTS8Smlnza8dnZK6XWDXPsPKaXjhnuveeSAsR27\nQiRJKl3Re4U0doVMnjjRrhBJkgpXdLBo7Ap5fNcuQ4UkSYUrftPQecB55IK6pLckSWUrusWi3hVy\ncbcLIkmShqXoFotzyQtd/AUO3pQkqQoq0WLh4E1Jkqqh6GDROHjzCUOFJEnFK7orBPoGb06cMMHB\nm5IkFa7oFovGwZu73IBMkqTiFd1iUR+8+X5ssZAkqQpssZAkSW1TmRaLaVOn2mIhSVLhKtNikSZM\nsMVCkqTCVaLF4i+AnV0uiyRJGlplWiyesXu3C2RJklS4SrRYXAVs2LbNUCFJUuGKbrH4PHAccB0w\nuTZ403AhSVK5im6xSORQcTFw4MEHGyokSSpc0cGi3hXyPmD/mTO7XBpJkjSUortCGgdv7tiwocul\nkSRJQ6lEi8U3gQN37HCBLEmSCld0i0V98GYCHt682TEWkiQVrugWi7rrgO3dLoQkSRpS0cGiPivk\nn4HDDzzQrhBJkgpXdLBoXCBry6RJdoVIklS4Soyx+C4w+eCDu1waSZI0lKKDRb0r5BKcbipJUhVU\noivkm8BBu3Y5xkKSpMIV3WLRON1008SJjrGQJKlwRbdY1F2LYywkSaqColssHGMhSVK1FN1i4RgL\nSZKqpegWC8dYSJJULUW3WNQ5xkKSpGoousXCMRaSJFVL0S0WjrGQJKlaim6xcIyFJEnVUnSLRZ1j\nLCRJqoaiWywcYyFJUrUU3WLhGAtJkqql6BaLxjEWT06a5BgLSZIKV3SLRd13gZeddFK3iyFJkoZQ\ndLCoj7H4x9rnkiSpbEUHi/oYixXAT7/3vS6XRpIkDaUSYywA9t+xg5SS4ywkSSpY0S0WdQ7elCSp\nGioRLK6bMIFXnnJKt4shSZKGUHRXSCLPCFkyaRI//MhHul0cSZI0hKJbLM4FfgJcuH07/3rRRd0u\njiRJGkLRweLzwN8Db0mJH19zTZdLI0mShlJ0sKgL+maFSJKkco0oWETEeRGxOiK2RMQtEfHSQc59\nc0R8PyIejYjeiLgpIha1cj9nhUiSVA0tB4uIeBvwaeAC4MXAz4EVEXHoAJe8Cvg+cDJ5WYr/AL4d\nEccO957fc1aIJEmVMJJZIUuAL6SUrgCIiHOANwDvBj7ZfHJKaUnTob+LiFOBPyOHkgEl4DsR/PNR\nR/GtT3xiBEWVJEmd1FKLRURMAhYCN9SPpTzw4Xrg+GG+RwDTgceHOvdc4KqU2O3YCkmSKqHVrpBD\ngYnAuqbj64DZw3yPvwIOAL4+1ImfB5YDf3XXXXz6/POHX0pJktQVHV0gKyLeDnwMOCWl9NhQ5y8B\nZgDs3s3NX/wiq1avZvHixSxevHiUSypJUvl6enro6enZ41hvb2+XSpNFK1M4a10hTwGnpZSuaTi+\nHJiRUnrzINeeAVwGnJ5SGnSr0og4Dli5kr5NyE599rO5+sEHnRkiSdIgVq1axcKFCwEWppRWdfr+\nLXWFpJR2ACuBE+rHamMmTgBuGui6iFgMfAk4Y6hQ0e99cbqpJElVMJKukM8AyyNiJXAbucdif/Jw\nCCLiQmBuSuldta/fXnvtg8BPImJW7X22pJQ2DueGTjeVJKkaWg4WKaWv19as+DgwC7gdODGl9Lva\nKbOBwxsueR95wOfS2kfd5eQpqgPfC6ebSpJUJSMavJlSWgYsG+C1s5u+fu1I7gF5uukxTjeVJKky\nit4rxOmmkiRVS9HBou6k3bvd3VSSpAqoRLBwd1NJkqqhEsHC6aaSJFVDJYKF000lSaqGji7p3aoE\nXDdhAhcfcwzfdLqpJEnFKzpYvG3iRJ73ghew/LrrmD59ereLI0mShlB0V8jXdu1iyS9/yZ8vWsSm\nTZu6XRxJkjSEooNFkKeaLvn1r13HQpKkCig6WNS5joUkSdVQiWDhOhaSJFVDJYKF61hIklQNlQgW\nrmMhSVI1FD3d1HUsJEmqlqJbLM6dM4dbP/ABvnnzza5jIUlSBRTdYvH573yH4447rtvFkCRJw1R0\ni4UkSaqWooPFOW98Ixd88IOuuilJUkUUHSw+//DDHL90Kacdf7zhQpKkCig6WLiktyRJ1VJ0sKhz\nSW9JkqqhEsHCJb0lSaqGSgQLl/SWJKkaKhEsXNJbkqRqKHqBLJf0liSpWopusXBJb0mSqqXoFguX\n9JYkqVpXY13aAAAIAElEQVSKbrGQJEnVYrCQJEltY7CQJEltY7CQJEltU3SwcHdTSZKqpehg4e6m\nkiRVS9HBwt1NJUmqlqKDRZ27m0qSVA2VCBbubipJUjVUIli4u6kkSdVQiWDh7qaSJFVD0XuFuLup\nJEnVUnSLhbubSpJULUW3WLi7qSRJ1VJ0i4UkSaoWg4UkSWobg4UkSWobg4UkSWobg4UkSWobg4Uk\nSWobg4UkSWobg4UkSWobg4UkSWobg4We1tPT0+0ijDvWeedZ551nnY8vIwoWEXFeRKyOiC0RcUtE\nvHSI818TESsjYmtE3BUR7xpZcTWa/MffedZ551nnnWedjy8tB4uIeBvwaeAC4MXAz4EVEXHoAOfP\nA74D3AAcC1wCXBYRfzKyIkuSpFKNpMViCfCFlNIVKaXfAOcATwHvHuD8/wn8NqX01ymlO1NKS4Gr\nau8jSZLGkJaCRURMAhaSWx8ASCkl4Hrg+AEue3nt9UYrBjlfkiRVVKvbph8KTATWNR1fBxw1wDWz\nBzj/oIiYklLa1s81UwF+/etft1g87Yve3l5WrVrV7WKMK9Z551nnnWedd1bDs3NqN+7farDolHkA\n73jHO7pcjPFn4cKF3S7CuGOdd5513nnWeVfMA27q9E1bDRaPAbuAWU3HZwGPDHDNIwOcv3GA1grI\nXSVnAvcBW1ssoyRJ49lUcqhY0Y2btxQsUko7ImIlcAJwDUBERO3rfxngspuBk5uOLaodH+g+64Gv\ntlI2SZL0tI63VNSNZFbIZ4D3RcRZEXE0cCmwP7AcICIujIjLG86/FDgiIi6KiKMi4lzg9Nr7SJKk\nMaTlMRYppa/X1qz4OLlL43bgxJTS72qnzAYObzj/voh4A3Ax8EFgDfCelFLzTBFJklRxkWeLSpIk\n7Tv3CpEkSW1jsJAkSW1TXLBodYOz8Soi/jgiromItRGxOyJO6eecj0fEQxHxVET8ICKObHp9SkQs\njYjHImJTRFwVEc9qOufgiPhKRPRGxIaIuCwiDmg65/CI+G5EPBkRj0TEJyOiuJ+tfRERfxsRt0XE\nxohYFxHfiogF/ZxnnbdRRJwTET+v1UVvRNwUESc1nWOdj5KI+Jva/y+faTpunbdRRFxQq+fGjzua\nzqlOnaeUivkA3kZet+Is4GjgC8DjwKHdLltpH8BJ5AG0p5LXFjml6fWP1OrujcALgauBe4HJDed8\nnrxWyKvJG8rdBPxX0/tcB6wCXgL8EXAXcGXD6xOA/ybPl/594ETgUeAT3a6jNtf3tcA7gWNq3+d3\nanU3zTof1Xp/Q+1n/XnAkcAngG3AMdb5qNf9S4HfAj8DPuPP+ajW9QXAL4BnAs+qfRxS1TrveoU2\nfdO3AJc0fB3kWSR/3e2ylfwB7GbvYPEQsKTh64OALcBbG77eBry54Zyjau/1strXx9S+fnHDOScC\nO4HZta9PBnbQEP6A9wMbgP26XTejWOeH1urmldZ5x+t+PXC2dT6qdXwgcCfwOuA/2DNYWOftr+8L\ngFWDvF6pOi+mSSlGtsGZ+hERzyVP+22sy43ArfTV5UvI040bz7kTeKDhnJcDG1JKP2t4++uBBPxh\nwzn/nVJ6rOGcFcAM4AVt+pZK9AxyPTwO1nknRMSEiDiDvG7OTdb5qFoKfDuldGPjQet8VM2P3LV9\nb0RcGRGHQzXrvJhgweAbnM3ufHEqbTb5h2WwupwFbK/9gA50zmxyM9jTUkq7yA/TxnP6uw+M0b+3\niAjgn4H/l1Kq94Na56MkIl4YEZvIv5EtI/9WdifW+aiohbcXAX/bz8vW+ei4BfhzcgvCOcBzgf+s\njX+oXJ2XugmZVLJlwPOBV3S7IOPEb4Bjyb81nQ5cERGv6m6RxqaIOIwcml+fUtrR7fKMFymlxj09\nfhkRtwH3A28l//xXSkktFiPZ4Ez9e4Q8PmWwunwEmBwRBw1xTvOo4onAIU3n9HcfGIN/bxHxOeBP\ngdeklB5ueMk6HyUppZ0ppd+mlH6WUvo74OfAh7DOR8NC8gDCVRGxIyJ2kAcDfigitpN/e7XOR1lK\nqZc8sPJIKvhzXkywqKXj+gZnwB4bnHVtM5UqSimtJv8QNNblQeR+tHpdriQP2mk85yjg9+jbIO5m\n4BkR8eKGtz+B/EN+a8M5vx95mfe6RUAvsMd0qaqrhYpTgdemlB5ofM0676gJwBTrfFRcT54N8CJy\nK9GxwE+BK4FjU0q/xTofdRFxIDlUPFTJn/Nuj4ZtGvn6VuAp9pxuuh54ZrfLVtoHcAD5H/2LyCN9\nP1z7+vDa639dq7s/I/9HcTVwN3tOT1oGrAZeQ/5N5cfsPT3pWvJ/LC8lN/3fCXy54fUJ5N8grwP+\ngNxHuA74392uozbX9zLyyOg/Jif4+sfUhnOs8/bX+z/V6vw55Gl2F5L/A32ddd6xv4PmWSHWefvr\n+FPAq2o/538E/KD2vc6sYp13vUL7qeBzyXNxt5DT00u6XaYSP8jNk7vJ3UeNH//WcM7fk6cpPUUe\n2Xtk03tMAT5L7obaBHwDeFbTOc8g/7bSS36wfhHYv+mcw8nrOmyu/RBeBEzodh21ub77q+tdwFlN\n51nn7a33y8hrKWwh/9b2fWqhwjrv2N/BjTQEC+t8VOq4h7y0whbyTI6vAs+tap27CZkkSWqbYsZY\nSJKk6jNYSJKktjFYSJKktjFYSJKktjFYSJKktjFYSJKktjFYSJKktjFYSJKktjFYSJKktjFYSJKk\ntjFYSJKktvn/znO8/ZkZXvIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f843566ffd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import json\n",
    "from tqdm import *\n",
    "\n",
    "en_voc = json.load(open('/data/xuwenshen/ai_challenge/data/train/train_words/en_voc_whole.json'))\n",
    "prob = []\n",
    "\n",
    "counlst = list(map(int, np.array(en_voc)[:,1]))\n",
    "n2 = sum(counlst)\n",
    "counlst = counlst[:50000]\n",
    "\n",
    "prev = 0\n",
    "for i in tqdm(range(50000)):\n",
    "    n1 = counlst[i] + prev\n",
    "    prev = n1\n",
    "    prob.append(n1/n2)\n",
    "plt.plot(range(0, 50000),prob , 'ro')\n",
    "print (prob[-1])\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "en_voc = json.load(open('/data/xuwenshen/ai_challenge/data/train/train_words/en_voc_whole.json'))\n",
    "en = list(np.array(en_voc)[:50000, 0])\n",
    "\n",
    "en.append('eos#')\n",
    "en.append('go#')\n",
    "en.append('pad#')\n",
    "en.append('nuk#')\n",
    "\n",
    "ids = [i for i in range(len(en))]\n",
    "envoc = dict(zip(en, ids))\n",
    "\n",
    "en_ = open('/data/xuwenshen/ai_challenge/data/train/train_words/en_voc.json', 'w')\n",
    "json.dump(envoc, en_)\n",
    "en_.close()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9886910\n",
      "9886910\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9886910/9886910 [10:08<00:00, 16258.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9886720 9886910\n"
     ]
    }
   ],
   "source": [
    "# 将en和zh都dump到同一个h5py文件，valid.h5py\n",
    "import json\n",
    "import h5py\n",
    "from collections import defaultdict\n",
    "from tqdm import *\n",
    "import random\n",
    "\n",
    "_en = json.load(open('/data/xuwenshen/ai_challenge/data/train/train_words/en_data.json'))\n",
    "en_voc = json.load(open('/data/xuwenshen/ai_challenge/data/train/train_words/en_voc.json'))\n",
    "\n",
    "_zh = json.load(open('/data/xuwenshen/ai_challenge/data/train/train_words/zh_data.json'))\n",
    "zh_voc = json.load(open('/data/xuwenshen/ai_challenge/data/train/train_words/zh_voc.json'))\n",
    "\n",
    "print (len(_en))\n",
    "print (len(_zh))\n",
    "\n",
    "data = [0 for i in range(len(_en))]\n",
    "for i in range(len(data)):\n",
    "    data[i] = [_en[i], _zh[i]]\n",
    "random.shuffle(data)\n",
    "\n",
    "f = h5py.File(\"/data/xuwenshen/ai_challenge/data/train/train_words/train0.h5py\", \"a\")\n",
    "\n",
    "\n",
    "en_set = f.create_dataset(\"en\", (len(data), 60), dtype='i8')\n",
    "en_length_set = f.create_dataset('en_len', (len(data), ), dtype='i8')\n",
    "\n",
    "\n",
    "label_set = f.create_dataset(\"label\", (len(data), 51), dtype='i8')\n",
    "zh_length_set = f.create_dataset('zh_len', (len(data), ), dtype='i8')\n",
    "gtruth_set = f.create_dataset('ground_truth', (len(data), 51), dtype='i8')\n",
    "\n",
    "zh_str_set = f.create_dataset('zh_str', (len(data), ), dtype='S400')\n",
    "en_str_set = f.create_dataset('en_str', (len(data), ), dtype='S400')\n",
    "\n",
    "\n",
    "en = []\n",
    "en_len = []\n",
    "en_str = []\n",
    "\n",
    "labels = []\n",
    "zh_len = []\n",
    "gtruths = []\n",
    "zh_str = []\n",
    "\n",
    "for i in tqdm(range(len(data))):\n",
    "    \n",
    "    #en data\n",
    "    tmp = [en_voc['pad#'] for j in range(60)]\n",
    "    if len(data[i][0]) == 0 or len(data[i][0]) > 60:\n",
    "        print ('en ', len(data[i][0]))\n",
    "        continue\n",
    "    if len(data[i][1]) == 0 or len(data[i][1]) > 50:\n",
    "        print ('zh ', len(data[i][1]))\n",
    "        continue\n",
    "    \n",
    "    for j in range(len(data[i][0])):\n",
    "        if data[i][0][j].lower() not in en_voc:\n",
    "            tmp[j] = en_voc['nuk#']\n",
    "        else:\n",
    "            tmp[j] = en_voc[data[i][0][j].lower()]\n",
    "    en.append(tmp)\n",
    "    en_len.append(len(data[i][0]))\n",
    "    en_str.append(' '.join(data[i][0][:min(400, len(data[i][0]))]).encode('utf8') )\n",
    "    \n",
    "    # zh data\n",
    "    tlabel = [zh_voc['pad#'] for j in range(51)]\n",
    "    tgtruth = [zh_voc['pad#'] for j in range(51)]\n",
    "    for j in range(len(data[i][1])):\n",
    "        if data[i][1][j] not in zh_voc:\n",
    "            tlabel[j] = zh_voc['nuk#']\n",
    "            tgtruth[j+1] = zh_voc['nuk#']\n",
    "        else:\n",
    "            tlabel[j] = zh_voc[data[i][1][j]]\n",
    "            tgtruth[j+1] = zh_voc[data[i][1][j]]\n",
    "            \n",
    "    tlabel[len(data[i][1])] = zh_voc['eos#']\n",
    "    tgtruth[0] = zh_voc['go#']\n",
    "    \n",
    "    \n",
    "    labels.append(tlabel)\n",
    "    gtruths.append(tgtruth)\n",
    "    zh_len.append(len(data[i][1])+1)    \n",
    "    zh_str.append(''.join(data[i][1][:min(200, len(data[i][1]))]).encode('gb2312', 'ignore'))\n",
    "    \n",
    "    if len(en) == 256:\n",
    "        start = i+1-256\n",
    "        end = i+1\n",
    "        # en data\n",
    "        en_set[start : end] = en\n",
    "        en_length_set[start : end] = en_len\n",
    "        en_str_set[start : end] =  en_str\n",
    "        en = []\n",
    "        en_len = []\n",
    "        en_str = []\n",
    "        \n",
    "        #zh data\n",
    "        label_set[start : end] = labels\n",
    "        zh_length_set[start : end] = zh_len\n",
    "        gtruth_set[start : end] = gtruths      \n",
    "        zh_str_set[start:end] = zh_str\n",
    "        labels = []\n",
    "        zh_len = []\n",
    "        gtruths = []\n",
    "        zh_str = []\n",
    "\n",
    "        \n",
    "end = len(data)\n",
    "start = end-len(en)\n",
    "print (start, end)\n",
    "en_set[start : end] = en\n",
    "en_length_set[start : end] = en_len\n",
    "en_str_set[start : end] =  en_str\n",
    "\n",
    "label_set[start : end] = labels\n",
    "zh_length_set[start : end] = zh_len\n",
    "gtruth_set[start : end] = gtruths\n",
    "zh_str_set[start : end] =  zh_str\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 go#打无线电找你。\n",
      "1 go#我来帮你拿啤酒。\n",
      "2 go#不是他，别闹了。是罗杰。布鲁克。\n",
      "3 go#他...我们的所作所为他强迫我做的事。\n",
      "4 go#你怎么能这么说？你是他的朋友啊\n",
      "5 go#我原以为是。但是不是的。\n",
      "6 go#如果你还能活下去，\n",
      "7 go#我不能停在别的车旁边。\n",
      "8 go#我从你姐姐那听说你被开除了？\n",
      "9 go#怎么解决呢？\n",
      "10 go#他们两个都该去死了。\n",
      "11 go#如果你需要我帮忙请登录...\n",
      "12 go#她穿6号的，但她正在长个子我得给她买8号的。\n",
      "13 go#我也有一个？\n",
      "14 go#好吧，他其实是不请自来。我---\n",
      "15 go#如今没有达成类似的共识。\n",
      "16 go#这个家伙是恶魔历史方面的专家。\n",
      "17 go#我准备写一本书，打算将它命名为《nuk#》。\n",
      "18 go#我只知道我的内心浮动着黑暗…\n",
      "19 go#你两小时前在哪里干什么？\n",
      "20 go#选民们厌倦了腐败，从他屡次承诺贯彻改革而又食言中清醒，并逐渐怀疑对他的批评者为西方的代理人、同伙的指责。\n",
      "21 go#假设某人偷了一万美元，\n",
      "22 go#好吧。我们一起干。\n",
      "23 go#今天美国同访客建立了外交关系。\n",
      "24 go#你能好好度过余生。\n",
      "25 go#什么？你不是开玩笑吧？我只是问问。\n",
      "26 go#小小的nuk#根本不能掩饰他巨大的成就。\n",
      "27 go#布兰顿·凯？\n",
      "28 go#哦，太棒了。耶。\n",
      "29 go#但我想这个年轻的想让我加入他们这一伙。\n",
      "30 go#本文主要运用nuk#统计学与nuk#方法对2000年7月至2003年7月nuk#所nuk#价格以及相应的主要影响因素进行实证分析。\n",
      "31 go#为了理解像空气这样nuk#的事物\n",
      "32 go#因为我以前曾经爱过政治。\n",
      "33 go#我正想拿香皂。\n",
      "34 go#实际情况是，目前美联储似乎根本就没有恰当的工具组合来治疗美国经济的nuk#。\n",
      "35 go#我是黑人-我早就知道了。\n",
      "36 go#他长大后，\n",
      "37 go#有时候杀人是nuk#的不杀人倒是有罪。\n",
      "38 go#而把我nuk#叔叔好吗？\n",
      "39 go#因为我以为我们要死了！\n",
      "40 go#喂，我吵到你了吗？别担心？\n",
      "41 go#戴安娜女士最终打开了这个nuk#，nuk#仍裹在原先的报纸里，她将其送至nuk#一家家族nuk#nuk#nuk#拍卖行。\n",
      "42 go#你的朋友呢？你是说他们？\n",
      "43 go#为什么？看看你自己伙计，你又回来了，你很健康。\n",
      "44 go#好啦你要不要讲重点？\n",
      "45 go#nuk#还鼓励女儿不要被狭隘的思维方式所束缚。\n",
      "46 go#不过，我反复查询之后找到个nuk#\n",
      "47 go#好吧，我问你个事-\n",
      "48 go#她nuk#我很久。\n",
      "49 go#在我看来，是非常微不足道的。\n",
      "50 go#你也许会好奇为什么我们\n",
      "51 go#我帮你吧。不要，不要。\n",
      "52 go#你这叫阴魂不散，亲爱的。\n",
      "53 go#中国最大铝生产商nuk#（nuk#）表示，担心信贷吃紧的客户拖欠货款。\n",
      "54 go#你拿到持枪许可证了。\n",
      "55 go#如果我犯了什么过错那都是出于父爱。\n",
      "56 go#我可没这么说！那么，是你？\n",
      "57 go#2010年，整个华尔街甚至金融界人士祖母一辈的人们都对nuk#的黄金充满信心。\n",
      "58 go#哦，好吧。没事。明天怎么样？\n",
      "59 go#当时在索马里，\n",
      "60 go#就像邻家的那些男孩一样，渴望有个温暖的家。\n",
      "61 go#所有检查都做过了，对吗？\n",
      "62 go#新戒毒方法把对贩毒者的惩罚和对吸毒者的治疗区分开来，这是它行之有效的关键。\n",
      "63 go#现在，从它们的藏身之处拿出...\n",
      "64 go#注意，所有驾驶员返回月球基地进行简报。\n",
      "65 go#你的帐户里有一大笔钱\n",
      "66 go#有什么能帮你的？-有什么能帮你的？\n",
      "67 go#我对经济一窍不通\n",
      "68 go#我不想这一辈子就这样完了。\n",
      "69 go#但大规模太阳喷发，像这样，可以形成风暴-\n",
      "70 go#我们的政府不支持。\n",
      "71 go#好的，听着不管怎样\n",
      "72 go#只有nuk#作战因素的改变，一次核武器大屠杀才能避免。\n",
      "73 go#如果这与任何动物测试有关，就让我来试试。\n",
      "74 go#在车辆进站后检查。\n",
      "75 go#“我们看到，欺诈行为正在nuk#，”美国联邦调查局（nuk#）nuk#部门负责人詹姆斯·巴纳克说。\n",
      "76 go#最好表现的尊重点。\n",
      "77 go#不知道他们的交通工具就无法追踪。\n",
      "78 go#泰迪熊总在nuk#。\n",
      "79 go#总统当选人照例将答复所有来自西方八国集团领导人的来电。\n",
      "80 go#倾泻直下nuk#我们的尖顶……\n",
      "81 go#如你所愿，亲爱的。\n",
      "82 go#南北战争时是什么样子？\n",
      "83 go#你的库房被烧是你生意上的损失。\n",
      "84 go#因为我…我没得选\n",
      "85 go#我也快放弃了。\n",
      "86 go#干得不错。我的消息不通过任何种nuk#。\n",
      "87 go#放期中nuk#幼儿园关门了，所以我就带他们两人去看电影。\n",
      "88 go#他是就地身亡的。\n",
      "89 go#我们是神龙大侠和盖世nuk#\n",
      "90 go#吉姆,一名42岁的工程师,他的教学是八岁的儿子怎么飞无线电遥控飞机.\n",
      "91 go#我是开玩笑。这可不好笑。\n",
      "92 go#我刚还在跟安娜贝丝说昨晚多棒，\n",
      "93 go#他花了一个星期消毒关在nuk#上，遍地研读电影游戏，睡在他的办公桌，对计划，可以提供他的球队多一个院子做梦。\n",
      "94 go#卢克，今晚你不上班，是吧？在这里帮帮我们。\n",
      "95 go#你要记念耶和华你的神，因为得nuk#的力量是他给你的，为要坚定他向你列祖起誓nuk#的约，像今日一样。\n",
      "96 go#他们想让我穿着高跟鞋在地铁隧道里步行？\n",
      "97 go#包括通知联邦调查局，去救他呢？\n",
      "98 go#但其宪政文化的国家主义观也体现出他的历史的局限性,并为后世学者所诟病。\n",
      "99 go#nuk#还是保时捷？两种发音我都听过。\n",
      "100 go#我想她们出去，\n",
      "101 go#现实性与理想性的统一具有稳定性。\n",
      "102 go#像是化妆品。\n",
      "103 go#竭尽全力的卷土重来，\n",
      "104 go#既然他不愿意和我们出去购物，就不要nuk#。\n",
      "105 go#我想听你亲耳告诉我。\n",
      "106 go#动物园熊猫小组的负责人唐nuk#说，nuk#周二下午1点15nuk#碎竹子nuk#的窝nuk#下nuk#的。她从周三开始出现感染，但是工作人员现在还不打算介入，除非她有健康危险。\n",
      "107 go#他每天都走东面的免税高速公路去工作。\n",
      "108 go#但年龄这东西..\n",
      "109 go#布巴卡，该你了\n",
      "110 go#我们就能看到任何一个复制者所能看到的东西了。\n",
      "111 go#所以你还是想要浪漫了。\n",
      "112 go#我们就会放下武器...\n",
      "113 go#形态特征：落叶nuk#nuk#,具nuk#nuk#。\n",
      "114 go#如果我在这儿检查你怎么样？\n",
      "115 go#我后面没人啊。\n",
      "116 go#他明白什么意思。我们和好了。\n",
      "117 go#喂，那样很危险！\n",
      "118 go#不过，法特米认为，萨尔科齐在推动改革方面看来还是能够取得成功，并且在7月份完成改革，到时候，法国将开始出任欧洲联盟的轮值主席。\n",
      "119 go#上次你去你的办公室是什么时候？\n",
      "120 go#听我的没错，卡特简直是天赐之物。\n",
      "121 go#我要做很久以来一直想做的事。\n",
      "122 go#好。你知道么？你是对的。你确实帮了我太多的忙。\n",
      "123 go#他长的什么样？他是干什么的？\n",
      "124 go#所以我们才需要\n",
      "125 go#如果你把房间弄得跟你穿衣服的样子一样那可真伤眼睛。\n",
      "126 go#nuk#以一寸代表nuk#。\n",
      "127 go#nuk#招收的大多是二十岁左右的学生和职场新人。\n",
      "128 go#是啊，你们的朋友说他无意中听到。\n",
      "129 go#你能制止他吗？我不知道。\n",
      "130 go#兽医们认为像炒蛋、nuk#那样的熟食对猫来说是一道既美味又营养的菜。\n",
      "131 go#他还是那个，我常常看到在nuk#的nuk#，仍趴在书桌上钻研人们寄来的信件的人。\n",
      "132 go#只不过我们经常处理而已。\n",
      "133 go#我要你明天过来。\n",
      "134 go#她被人下药了。\n",
      "135 go#nuk#，你也去拿一个。\n",
      "136 go#我订了8点钟的桌子。\n",
      "137 go#我们支持你，儿子。\n",
      "138 go#他是在浴缸里洗的吗\n",
      "139 go#现在的问题是奥巴马先生如何处置自己nuk#梦想的破灭。\n",
      "140 go#好的！我来了！\n",
      "141 go#我不想欠那废物的人情。\n",
      "142 go#一张？你说什么呢？\n",
      "143 go#哭得伤心极了。\n",
      "144 go#加深了对你的理解也希望支持你，\n",
      "145 go#包装在爆炸时保护了它。\n",
      "146 go#所以我回到学校\n",
      "147 go#尤达大师是如此的欣赏你。\n",
      "148 go#这样的发现并不仅限于数学，即使在依靠直觉的美学学习上也是如此。\n",
      "149 go#更宽泛的看，在全球范围内，保留死刑国家的数量，和暴力犯罪率，同时都在下降。\n",
      "150 go#嘿，亲爱的。是中国的nuk#了。\n",
      "151 go#对我触动更大的倒是这些菜鸟让我反思自己。\n",
      "152 go#我想我能够对付一个小男孩跟太监的。\n",
      "153 go#她看见公主在河的那一边\n",
      "154 go#我都没说她相貌平平我是说。那个...\n",
      "155 go#把它给你可以吗？\n",
      "156 go#以及泰勒·斯威夫特…\n",
      "157 go#仅仅错在像这样偷偷溜走。\n",
      "158 go#你要做什么？把枪给我。\n",
      "159 go#安全。还有人...\n",
      "160 go#你接收了我们的提议。\n",
      "161 go#当然起初一切看起来都很平常\n",
      "162 go#它能飞。\n",
      "163 go#他手脚太笨\n",
      "164 go#但你……你肯定么？\n",
      "165 go#何况你还这么漂亮。\n",
      "166 go#在精nuk#大头nuk#，采用的定位方式是一面nuk#。\n",
      "167 go#雪:(nuk#说)原来,这才是真正的大难。\n",
      "168 go#她也告诉我你是如何拿到的。\n",
      "169 go#他在它们的脸上看到了难以形容的爱。\n",
      "170 go#我们要看nuk#死亡案的nuk#。\n",
      "171 go#别以为我是那种说，\n",
      "172 go#科特的形象太没说服力了。\n",
      "173 go#在这种情况下，就会忽略他们nuk#的收入。\n",
      "174 go#相亲时就更容易成功。\n",
      "175 go#我知道，对不起。如果我能选择的话-\n",
      "176 go#你得…你得相信我。\n",
      "177 go#不知道。我感觉不疼的时候。\n",
      "178 go#就像吸毒一样。\n",
      "179 go#每次我坐飞机去加州nuk#落基山脉时，\n",
      "180 go#可是，如果nuk#老师问有没有人主动去擦黑板或者nuk#大家的试卷交到她桌上时，我总是会第一个举手。\n",
      "181 go#但是如果我不杀你的话...\n",
      "182 go#地线一种与零电位位置相连接的nuk#物体，如电线。\n",
      "183 go#很好。那就交一篇论文。\n",
      "184 go#好吧，好吧，现在读。\n",
      "185 go#我们都有注意到。\n",
      "186 go#希望你已经nuk#了今晚人很多。所有的评论家都出席了。\n",
      "187 go#因为合唱团nuk#了。\n",
      "188 go#厌倦了nuk#吗?\n",
      "189 go#好吧，我现在急需nuk#。\n",
      "190 go#很高兴你告诉我。上一次你不辞而别。\n",
      "191 go#在nuk#政府nuk#路旁的nuk#部委办公楼里，一个会计或行政人员每个月可以挣到nuk#nuk#，大概是最低工资的20倍。\n",
      "192 go#尽管他们对持续的农村贫穷、腐败与剥削描绘了一副惨淡的图景，他们没有特别指出这种功能障碍潜在的政治后果。\n",
      "193 go#你将检查超过100，000个数据。\n",
      "194 go#希望这个数字不要再增加。\n",
      "195 go#早上起来跳nuk#，\n",
      "196 go#在格雷nuk#旁边！\n",
      "197 go#我把他和其他一切都在岛上忘了。\n",
      "198 go#因为她当时太小，\n",
      "199 go#给你个提示吧，蒂安娜。\n",
      "200 go#你今晚要做它吗？哼。\n",
      "201 go#接收器对手动刺激是没有反应的。\n",
      "202 go#将近年来因遭受巨变与抨击而分崩离析的教会团结起来。\n",
      "203 go#一月一百万能持平。\n",
      "204 go#他的妻子其实没什么了不起，\n",
      "205 go#在某个当地的饭馆吃饭。\n",
      "206 go#nuk#潜水经验并不是必须的。\n",
      "207 go#即使你们分开了。\n",
      "208 go#nuk#神呐我马上过去。\n",
      "209 go#各位可以看到他们的nuk#的圣母礼拜堂的“杰作”。\n",
      "210 go#好的。你确定？已经准备好了么？\n",
      "211 go#如果他们不过来呢？\n",
      "212 go#太漂亮了，伙计。我喜欢你的发型，我喜欢你的眼神。\n",
      "213 go#但世界并非nuk#，记得吗？\n",
      "214 go#去哪里？-走吧。\n",
      "215 go#我读到你让一些12岁的孩子在宣传活动中做帮手。\n",
      "216 go#他们不会把手放在nuk#里了吧？\n",
      "217 go#但那样犯下的过失是nuk#nuk#的。\n",
      "218 go#分布自nuk#、台湾nuk#至日本nuk#群岛。\n",
      "219 go#一定是你的男孩们试着找破绽吧。\n",
      "220 go#比做一个糟糕的演出更难。\n",
      "221 go#这两笔交易都尚未完成。\n",
      "222 go#听我说，我知道这不算，是个，画廊。\n",
      "223 go#喂！请你帮我在茶几上拿nuk#把门开了？\n",
      "224 go#对，但他们会让我们去奶奶在棕榈泉的别墅。\n",
      "225 go#可是，自己nuk#里的车子越来越快。\n",
      "226 go#几位高管表示，汤姆道莱（tomnuk#）未能及时就该行的次贷相关交易风险预警，或是使用了过于专业或晦涩的语言。\n",
      "227 go#我父母帮他收场了。\n",
      "228 go#不，但是你唱的时候就不会面带微笑。\n",
      "229 go#但我们发现签名上有猫腻。\n",
      "230 go#我们鄙视那些nuk#上司的人，这些人通常会恃强凌弱地对待他们的下属。\n",
      "231 go#别让她看这个.少多嘴!\n",
      "232 go#不管是由什么触发的，有一件事是确定的你肯定会不时变的愤怒。\n",
      "233 go#那里供应青岛啤酒，但可口可乐和咖啡就没有了。\n",
      "234 go#你应该具体和你的直接上司讨论这件事，包括你晋升或离职之后的替代人选。\n",
      "235 go#但是，如果你想强调的不只是几个词，你可以用nuk#，nuk#，彩色，或较大nuk#。\n",
      "236 go#一件薄薄的羊毛长外套和一条紫色的头巾为她挡了风。\n",
      "237 go#它们都是nuk#华美，生意兴隆的铺子。\n",
      "238 go#我们半夜就走。\n",
      "239 go#出这种事儿能把我这生意搅黄了。\n",
      "240 go#我有很多朋友是消防员。\n",
      "241 go#已经很辛苦了。\n",
      "242 go#我将与图利乌斯会面。\n",
      "243 go#每件事都模糊又清晰的浮现。\n",
      "244 go#我不想你忧虑这个，\n",
      "245 go#你不好画，画成了就俗\n",
      "246 go#nuk#，我看她nuk#个死了，\n",
      "247 go#我没有印象\n",
      "248 go#不，世上仅此一件。\n",
      "249 go#在混凝土的地上被风nuk#的垃圾。\n",
      "250 go#一会我们去拜访你\n",
      "251 go#大约2000平方英尺的宴会空间，可以举办小型商业会议也可以举办大型接待会。\n",
      "252 go#爱是道中围绕在一切周围的主要振动。\n",
      "253 go#向北奥马哈儿童医院的一个朋友询问\n",
      "254 go#因为那有200公斤的海洛因。\n",
      "255 go#我的手机震动我的口袋里。\n",
      "256 go#我也不想做这事，吉尔。\n",
      "257 go#你能听到我的身体语言么我是个很好的聆听者。\n",
      "258 go#我不想把工作和前一段婚姻\n",
      "259 go#春天意味着更nuk#的好天气，同时往往意味着更多的雨水。\n",
      "260 go#好吧行。\n",
      "261 go#多年劳累，nuk#，只有nuk#一间，得不到片刻休息。\n",
      "262 go#估计，是在他的婚礼上唱的。\n",
      "263 go#巫术并非仅限于南非足球。\n",
      "264 go#我要排多久的队才能领到食物？\n",
      "265 go#待会我打给你我这有另一个电话。\n",
      "266 go#就从这镜子，\n",
      "267 go#你应该先把脚擦干净。\n",
      "268 go#听着，我今天不太舒服。\n",
      "269 go#在我看来，你是恶棍。\n",
      "270 go#嘿！朋友怎么能跟对方\n",
      "271 go#然而，一篇发表在物理评论快报上的文章，讲述了狭义相对论更为一般的应用。\n",
      "272 go#有钱人他妈的越来越有钱。\n",
      "273 go#他们手术前都通知你了吗？\n",
      "274 go#你误解了，基徳。\n",
      "275 go#2008年，以色列长期遭受哈马斯火箭炮攻击的斯德罗nuk#居民试图迫使政府建立能量武器防御设施通过把主要的官员带上庭的方式。\n",
      "276 go#但明天这个时候一切都烟消云散。\n",
      "277 go#你知道吗，我本来是打算在我去加纳的时候，让他们住在我妈那边\n",
      "278 go#我们这位前任将军好朋友急于将商品脱手。\n",
      "279 go#有福共享在这个国家就是句套话。\n",
      "280 go#是的，我看过文件了。\n",
      "281 go#我告诉豪斯我认为那是个错误。\n",
      "282 go#你们今天要去哪里旅行？洛杉矶。\n",
      "283 go#我的心在为你哭泣我心等待我nuk#。\n",
      "284 go#我也爱我的姐姐，朋友...\n",
      "285 go#几个月来，我保护着小镇的安全。\n",
      "286 go#如凡人般伤感。\n",
      "287 go#我认为这主意不太好。\n",
      "288 go#公关负责人也是你的婚礼策划人。\n",
      "289 go#就在那个时候，不知什么原因，我就肆无忌惮地说起来了。\n",
      "290 go#唱片销售是由一国际巡回演出团协助的。在演出中，歌唱家在舞台上表演固定舞蹈动作，这使歌迷为之nuk#起舞。\n",
      "291 go#北盐湖城\n",
      "292 go#你嫁了一个智商卓越的人这不是什么不好的事。\n",
      "293 go#不好意思。我叫nuk#·马修，不动产律师。\n",
      "294 go#几乎是一年前的事了\n",
      "295 go#全nuk#阵容的那种电影。\n",
      "296 go#你们也一直是我的朋友\n",
      "297 go#另一箱啤酒！\n",
      "298 go#其实什么工作都很类似。\n",
      "299 go#你们大学有多少nuk#？\n",
      "300 go#我在耍你呢，老爸。\n",
      "301 go#他那扭伤的脚踝nuk#了。\n",
      "302 go#我来想象成高中生。\n",
      "303 go#所有小朋友都对这装有发条玩具感兴趣。\n",
      "304 go#苹果沉寂了太久，突然nuk#连自己也有些吃惊。\n",
      "305 go#正在考虑的利用方式之一是由nuk#建造“猎户座”的服务模块，该模块包括了航天器的推进器和nuk#。\n",
      "306 go#再给我用一下肥皂好吗？\n",
      "307 go#每天早上一醒来就想，今天会见到他。\n",
      "308 go#多重射击不再以平民为目标。\n",
      "309 go#我们已经加入了，而且我们会找到钱的，你明白吗？\n",
      "310 go#四月末就可以测试。\n",
      "311 go#排名第一的教学医院主任就这么做。\n",
      "312 go#抽出一看它有\n",
      "313 go#她比他更坏。\n",
      "314 go#你刚才那种说法。\n",
      "315 go#好了，站着别动。\n",
      "316 go#千疮百孔。瘟疫nuk#。\n",
      "317 go#如果nuk#脑硬化就是罪犯呢？\n",
      "318 go#我就在门口等着，好吧？\n",
      "319 go#死于nuk#一场车祸。\n",
      "320 go#nuk#博士珍妮特，祝你有一个愉快的旅行？\n",
      "321 go#探讨糖代谢异常高龄孕妇的妊娠结局。\n",
      "322 go#我做好眼睛部分了。我还在画嘴。\n",
      "323 go#既然都想那你为什么还呆在这儿？？\n",
      "324 go#是啊，我跟他们一样都是普通学生。\n",
      "325 go#弗雷德里克nuk#泰勒（nuk#-1915），这个被安葬于宾夕法尼亚的nuk#教友的墓志铭上写着：“现代科学管理之父”。\n",
      "326 go#我的脖子抽筋了。\n",
      "327 go#其旨在帮助参与者感知其环境中的艺术。\n",
      "328 go#可是你会感受到你无法...\n",
      "329 go#我想和他建立心灵上的联系，\n",
      "330 go#担心下个月的冠军赛吧。那才是你该担心的事。\n",
      "331 go#你也许也像我一样，会发现这些观点（首先是关于同性恋权利）令人反感，他的措辞也令人厌恶，但是必须承认，他非常坚持，始终如一。\n",
      "332 go#妈妈唯一的快乐时光\n",
      "333 go#劫车，勒索，强奸。\n",
      "334 go#可能吧，但你得说具体点让我看看是不是真的。\n",
      "335 go#我在你身边。她会回来的。\n",
      "336 go#会很大声的，但是\n",
      "337 go#耶稣nuk#一个全新的你。\n",
      "338 go#餐具\n",
      "339 go#图案匹配:\n",
      "340 go#每天晚上当父亲为我盖上被子时他总会唱那首歌。\n",
      "341 go#我们倾向于认为，一个彻底的健忘症患者，他与以前没有患病时仍然是同一个人。\n",
      "342 go#当然，我发现这个地方非常的多元化是不是阿，达西？\n",
      "343 go#感染最符合。\n",
      "344 go#很晚了。怎么这么多人？\n",
      "345 go#我认为你应该尽可能快的长23年，所以我们可以在一起。\n",
      "346 go#别走开没，接下来，\n",
      "347 go#克里斯！我儿子在里面！克里斯，快出来！\n",
      "348 go#噢，请原谅每一次的顶撞，甘地！\n",
      "349 go#哼？你想要回去吗？\n",
      "350 go#我可不是便宜的荡妇。\n",
      "351 go#呃，她看上了我。\n",
      "352 go#我们去待nuk#看看。\n",
      "353 go#公众和你在心理上都能够接受。\n",
      "354 go#你青春期时患有泛焦虑症。\n",
      "355 go#伤心的时候会对着镜子，\n",
      "356 go#当我第一次遇见年幼的nuk#先生时，\n",
      "357 go#又或者你根本就是个白痴？\n",
      "358 go#安全第一，所以你要最后上去，杰克。\n",
      "359 go#目前，Spotify服务尚未登陆美国，不过该公司称，正在签署登陆美国所需要的一些许可协议。\n",
      "360 go#对我来说最好的事就是避免和他在一起。\n",
      "361 go#你知道吗，我觉得你说的是实话。\n",
      "362 go#学习的主动性和nuk#。\n",
      "363 go#你在这儿我做不了。\n",
      "364 go#当然，照片得是真人，但其他都可以虚构。\n",
      "365 go#你还有脸批评我们的人生选择？你以为你是谁？\n",
      "366 go#你知道如果其中一匹狼决定独自生活会发生什么事吗？\n",
      "367 go#我哥哥有个拖车。他可以来把车拖去。\n",
      "368 go#戈壁地区湍流能量nuk#率和温度脉动nuk#率的分析。\n",
      "369 go#我发誓，我不会伤害他。\n",
      "370 go#所以生命的循环得以延续。\n",
      "371 go#听着，你得向他们证明自己。\n",
      "372 go#他恐怕nuk#下来。\n",
      "373 go#怎么会呢，别傻了\n",
      "374 go#我指的是我对他的误解。\n",
      "375 go#你要来点香槟，好吗？\n",
      "376 go#不用，只需要破坏特定的一个。\n",
      "377 go#但是剪辑根本完成不了，这你也知道！\n",
      "378 go#我来了，杰米。\n",
      "379 go#杜普里先生自诩其在卡特丽娜离开后从未提高税收、削减公共服务以及裁减工人。\n",
      "380 go#我没想到这一大早\n",
      "381 go#我就出去一下。\n",
      "382 go#你不用这样踢我吧，长官。\n",
      "383 go#是酒鬼的栖息之地。\n",
      "384 go#水上乐园\n",
      "385 go#可以参观美国历史上的国家军用飞机博物馆和nuk#玻璃博物馆、或享受宁静的手指湖葡萄酒产区风景。\n",
      "386 go#其中很多人都只是受了很浅的一点伤？\n",
      "387 go#天啊！等一下，别下定论。\n",
      "388 go#右转驶上nuk#nuk#Roadnuk#-73nuk#nuk#Lnuk#高速公路。\n",
      "389 go#很糟糕。-阿尔特曼医生，\n",
      "390 go#我数到3。好的，好的。\n",
      "391 go#已经给了你一个多月的时间了。\n",
      "392 go#当加州理工学院的研究人员把银加入微量合金混合物nuk#，它们就能够将nuk#的厚度提升到六厘米了。\n",
      "393 go#我和鲁卡斯在一起的时候。\n",
      "394 go#她不出门口的，怎会知道呢？\n",
      "395 go#偶尔，这些卵的其中之一允许成为新女王，\n",
      "396 go#而且，他把新闻集团经营得很像家族企业，nuk#拥有的特别投票权让他们可以抵抗季度收益的压力。\n",
      "397 go#他也会那样说一桌nuk#。\n",
      "398 go#你刚刚nuk#啊。\n",
      "399 go#下一回的nuk#中。\n",
      "400 go#现在不来那一套了。应该来的！\n",
      "401 go#从现在开始，禁止你在这里购物。你别想在这买水果了！\n",
      "402 go#你们不妨再比比运算能力。\n",
      "403 go#但是如果你沉迷于此就会无法自拔了。\n",
      "404 go#因此，nuk#博士及其同事能够寻找新的途径从容地观测反物质可能与普通物质存在的差别。\n",
      "405 go#nuk#是位于nuk#nuk#东南方35公里（22英里）的一个小镇，当地居民说这些骨头是1915年数以万计的亚美尼亚人在叙利亚沙漠中行进时被土耳其军队屠杀的人的遗骸。\n",
      "406 go#我想到了个赚钱的好主意。\n",
      "407 go#我们已经很惨了。\n",
      "408 go#会炒掉谁的人。\n",
      "409 go#我只请求你们在回到陪审团室的时候，\n",
      "410 go#我们走，起来！\n",
      "411 go#你的唇好像一条nuk#线，你的嘴也秀美。你的两太阳在nuk#内，如同一块石榴。\n",
      "412 go#中国和美国股市昨日的走势形成了鲜明对比，由於大部分欧洲市场和部分亚洲市场因复活节周一假期nuk#，外汇市场、美国和新兴市场债券的交易均较为nuk#。\n",
      "413 go#三个星期，日期和nuk#显示。\n",
      "414 go#但在这过程中，\n",
      "415 go#立法机关太过软弱。\n",
      "416 go#就为了满足他的杀戮欲。\n",
      "417 go#它们无所不在nuk#网。\n",
      "418 go#他们把nuk#先生也带来了。\n",
      "419 go#很不幸我们找不到这只遗失了的动物，\n",
      "420 go#又去打牌？\n",
      "421 go#你得把他的车废掉。\n",
      "422 go#多谢替我nuk#。\n",
      "423 go#你没告诉他？我当然告诉他了。\n",
      "424 go#我分辨不出简和萨拉，她们长得太像了。\n",
      "425 go#巴林的nuk#比利比亚的规模小得多；况且西方国家也nuk#于与巴林，美国海军第五舰队基地所在地，和其皇室保护者沙特阿拉伯集结的军事结盟。\n",
      "426 go#其结果是，数以百计来自全球各地的藏人本周在印度nuk#nuk#萨拉召开一次史无前例的会议，讨论西藏运动下一步的去向。\n",
      "427 go#你刚才说什么？我们必须走了。\n",
      "428 go#把它靠墙放就行了\n",
      "429 go#我向你保证我们会不惜一切nuk#孩子的。\n",
      "430 go#多半是简单的诊疗程序。\n",
      "431 go#作为剧院管理人，\n",
      "432 go#你自己花钱如流水还说什么nuk#。\n",
      "433 go#主要错误是提升肩膀被称为‘锁骨呼吸’。\n",
      "434 go#尽管你妻子很残忍我还是站在她的那一边！\n",
      "435 go#你知道吗，情况看起来对你不利-\n",
      "436 go#唔。好吧，那就为留在这里做些准备吧\n",
      "437 go#在电子交易中，无行为能力人和限制行为能力人缔结的合同效力确定，适用我国《合同法》第47条规定。\n",
      "438 go#每天姑娘们来到这儿\n",
      "439 go#我相信你有，查理，不过...\n",
      "440 go#而且欧洲央行觉得，如果贷款给银行（即使到时银行再贷给政府），比介入主权债券市场的泥潭会更安全些。\n",
      "441 go#我应该辞掉这份工作么？\n",
      "442 go#我不去想...\n",
      "443 go#带血的唾液。\n",
      "444 go#回去到魔法产生的最初地吧。\n",
      "445 go#在过去的数十年中，妇女无疑表现nuk#男人们更为出色。\n",
      "446 go#我去付钱。要吃东西吗？\n",
      "447 go#嗯……我在找遥控器。\n",
      "448 go#商务管理（nuk#维校区）\n",
      "449 go#即使这不是法例，更何况这是...\n",
      "450 go#小男孩看了看糖果。小男孩在找他的朋友。\n",
      "451 go#可惜我得拒绝你，nuk#座的活\n",
      "452 go#但是你出了名鲁莽nuk#不顾后果\n",
      "453 go#唐纳利被撞飞。倒在了地上。\n",
      "454 go#你觉得是？我觉得不可能nuk#出去。\n",
      "455 go#nuk#也不错啊。\n",
      "456 go#把草图拿给客户看是设计过程中通常的做法，在大型logo设计项目或其它项目中这种做法是普遍的。\n",
      "457 go#我告诉旧金山市长\n",
      "458 go#不是我杀的她！\n",
      "459 go#nuk#烟灰缸或鸟笼之类的东西。\n",
      "460 go#你没听到我老大在和你说话啊\n",
      "461 go#你瞧，火炬木做的就是这事儿。\n",
      "462 go#我们上游泳nuk#迟到了。\n",
      "463 go#距教堂山nuk#英里\n",
      "464 go#该评级机构维持其负面前景不变，表明还有可能进一步下调评级。\n",
      "465 go#对预算控制的非难“nuk#”，一个英国官员这样称呼它们只适用于欧元区。\n",
      "466 go#坚持党的nuk#，解放思想、实事求是、与时俱进，是我们党坚持先进性和增强创造力的决定性因素。\n",
      "467 go#2月，中国nuk#被革职，并因可能的腐败问题接受调查。\n",
      "468 go#向右转，在第一个红绿灯处左转，贝斯特韦斯特nuk#套房酒店就在拐角处。\n",
      "469 go#纽约大学医学院教授兼纽约大学朗格尼医学中心心律疾病治疗中心主任拉里nuk#茨博士说，跟阿司匹林相比，中风风险降低55%，这个数字是引人注目的。\n",
      "470 go#所以，我的处事原则就是\n",
      "471 go#但是你知道是谁杀了你兄弟？\n",
      "472 go#是他们。他们来这儿了。\n",
      "473 go#不，这是对美国教育制度的控诉。\n",
      "474 go#在骨盆旁边有两处阴影。\n",
      "475 go#经济学家普遍偏好透明补助而非暗地里的。\n",
      "476 go#贾尼斯是一个很好的成熟的女士，她是我们的队长，始终激励我们努力工作。\n",
      "477 go#我们说的都是几世纪以来nuk#不分的事情。\n",
      "478 go#距离Thenuk#nuk#餐厅5.5个街区，全套服务，美式，营业范围：早餐、午餐\n",
      "479 go#我给了她好多事情做吗？\n",
      "480 go#新的银行营业照可能授予有政治背景的实业家。\n",
      "481 go#我不希望大家觉得...\n",
      "482 go#从你的留言我听出这点来了。\n",
      "483 go#从一刻钟到几个小时。却一无所获。\n",
      "484 go#你难道能否认这有点吓人吗？\n",
      "485 go#nuk#nuk#nuk#木甲醇萃取nuk#发炎及抗癌活性研究。\n",
      "486 go#哦，他应该拿起一块石头并且…\n",
      "487 go#手还有点味道？\n",
      "488 go#不像生活，探戈很简单，这一点使得探戈很nuk#。\n",
      "489 go#住手！结束了。\n",
      "490 go#对于一场秀来讲这真是前所未有。\n",
      "491 go#他躺在—床上，nuk#不停\n",
      "492 go#很明显这个人想告诉我们的是我们不能想当然地相信闪影中看到的事。\n",
      "493 go#家住nuk#，姓nuk#永\n",
      "494 go#我要多点手术时间。\n",
      "495 go#全心全意地相信\n",
      "496 go#有人看到我的手套吗？\n",
      "497 go#我们像那些玩家和狐狸的那奔跑。\n",
      "498 go#如若各国采取的政策不同，那么公司或许会仅仅凭借国家税率的友好程度选择投资方向。\n",
      "499 go#你不会刚好手边放着\n",
      "500 go#你能再说说其他症状吗？\n",
      "501 go#我觉得袭击我们的东西？\n",
      "502 go#精力就用得差不多了。\n",
      "503 go#我们看看它能不能站起来。\n",
      "504 go#一声有关军官集合响起，我们要集中注意力，\n",
      "505 go#罗姆尼和他的助手曾经考虑让很多女性候选者来担任nuk#州州政府的高级职务，他们确实也雇佣了他们中的大部分人。\n",
      "506 go#因为急救事故，\n",
      "507 go#他们的结论是：叶酸可降低血液nuk#nuk#的浓度。\n",
      "508 go#我想见我妈妈！她在哪？\n",
      "509 go#他放弃了。\n",
      "510 go#好啊。我的王后，nuk#女孩的鲜血。\n",
      "511 go#相当有意思？\n",
      "512 go#我们过去查问她。\n",
      "513 go#从车里出来，弗兰克。\n",
      "514 go#我是说，想和你一起分担。\n",
      "515 go#你带图纸来了吗？\n",
      "516 go#我国近一半老人过nuk#生活,生命质量相对较低。\n",
      "517 go#好的，你可以选nuk#或者巴西nuk#。\n",
      "518 go#当时，这是一种很大胆的行为。\n",
      "519 go#所以这门课的进化部分有两次导论课。\n",
      "520 go#妈妈，你选择安顿下来的决定是对的。\n",
      "521 go#当我们刚在一起的时候，我妈妈说，\n",
      "522 go#研究样本来自七个nuk#各一所小学、10个班级，计nuk#nuk#学生。\n",
      "523 go#再给他们一点时间。\n",
      "524 go#关乎童年的回忆一日胜过一日。\n",
      "525 go#我们还听到工程对两个性质nuk#“反对”和“反对亚历山大的执行人。”\n",
      "526 go#我觉得有些人知道。\n",
      "527 go#大人，我上年纪了，\n",
      "528 go#你知道，他nuk#，\n",
      "529 go#什么同僚？\n",
      "530 go#小姐！史东小姐。\n",
      "531 go#我收到了罗杰和高层的消息，\n",
      "532 go#提示：本文是一篇关于“意见”的系列文章的第一部分。\n",
      "533 go#如果予以足够警惕，很有可能将一个有威胁的巨石推出原先的轨道。\n",
      "534 go#否则我就把这孩子杀了。\n",
      "535 go#性命冒险。但是\n",
      "536 go#nuk#准备用白纸黑字写出事实真相，让这些辱骂就此消除。\n",
      "537 go#在计算时间与成本的应用中，寄出包裹的一天有否计算在nuk#所需的nuk#内？\n",
      "538 go#他一定是把nuk#了，我这个老公啊\n",
      "539 go#他们的称赞理所当然nuk#属于无数菲律宾的扶轮社nuk#，他们从机场到饭店至研讨会会场展现联谊，以菲律宾人的殷勤招待nuk#每一个人。\n",
      "540 go#她要去维加斯了。\n",
      "541 go#别说是空军。就是空军！\n",
      "542 go#他们把这道菜取名为\"nuk#。\n",
      "543 go#是啊，怎么没想到呢。\n",
      "544 go#我们已经练习了一个星期了。\n",
      "545 go#相反，这位纽约大学经济学家认为经济增速将回升，但还是不足以提升美国nuk#的收入。\n",
      "546 go#你好，我是奈德。史布里。\n",
      "547 go#你今天的腰带选的可不怎么样。\n",
      "548 go#可能是团队头目。\n",
      "549 go#粉红色耐克。\n",
      "550 go#不过，我相信，很多年之后\n",
      "551 go#不为任何提案费用指定用途。\n",
      "552 go#她想让你花这些钱，去享受那些她享受不了的快乐。\n",
      "553 go#乔治华盛顿中学离我家大约一英里半，是纽约市最大和最好的公立学校之一。\n",
      "554 go#他就送了我一把吉他。\n",
      "555 go#在哪？半什么？\n",
      "556 go#她的名字也叫艾丽西娅。\n",
      "557 go#他要干啥？-杀你他要来杀你！\n",
      "558 go#是个灰色区域。你确定吗？\n",
      "559 go#我本可以不在意你的需求。\n",
      "560 go#我就不告发你。\n",
      "561 go#你知道，如果你不愿意我们可以取消。\n",
      "562 go#nuk#nuk#nuk#让你的冬天升温无限。\n",
      "563 go#我们得找到新的启动堆。\n",
      "564 go#但是她还是会来看你们的。\n",
      "565 go#喔。我以为nuk#是天主教的事。\n",
      "566 go#我就说一句我认为你的表现很精彩。\n",
      "567 go#一旦钢筋生锈，\n",
      "568 go#三到五年？是啊。\n",
      "569 go#但给我离远点。\n",
      "570 go#不知道您是否知道，我们支行来了位新经理。\n",
      "571 go#去掉那个荧光绿的马桶座\n",
      "572 go#她的化妆品中还有钛，\n",
      "573 go#太爷们儿了。\n",
      "574 go#我要走了不，不，不，你不能走。\n",
      "575 go#好吧，我并不倾向于使用这个词。\n",
      "576 go#但令人垂涎的巨额数字并没有转化成实实在在的盈利。\n",
      "577 go#她给我写了信。\n",
      "578 go#我他妈毙了你。\n",
      "579 go#有幸来此的商务旅客特别享受我们沃特伯里酒店提供的宾至如归的服务，以及酒店配备的会议和活动设施。\n",
      "580 go#从那以后，我就洗心革面了\n",
      "581 go#所以我来了，看看你可否帮我。\n",
      "582 go#你在外面支持下去，我也能在里面支持下去。\n",
      "583 go#致体形、运动训练模式失灵(nuk#不正当肩胛骨肱骨节奏)等等.\n",
      "584 go#但我能感到忧郁\n",
      "585 go#你没说再见。他们，呃...\n",
      "586 go#她老是翻出那件令人尴尬的事。\n",
      "587 go#六个月不知所踪的女朋友发来的。\n",
      "588 go#如果他与你联系，\n",
      "589 go#他们不是出奇勇敢就是出奇傻帽。\n",
      "590 go#我努力想跟世界打赢的仗\n",
      "591 go#年龄上nuk#很多，\n",
      "592 go#近五年来东亚春季nuk#气溶胶分布输送和辐射效应的模拟研究。\n",
      "593 go#他跟我说。每一秒钟他都清醒无比。\n",
      "594 go#他们竟然卖不出去卷饼？\n",
      "595 go#你的拉拉用品套装几周后就会寄到你家\n",
      "596 go#但也存在着很多最远可追溯到7世纪的nuk#，和长城一样也是为了抵御北方的侵略而建造。\n",
      "597 go#她真的喜欢你，查理。\n",
      "598 go#很nuk#。别管那些了。\n",
      "599 go#nuk#？什么？\n",
      "600 go#哪个？-那个吹小号的。他哭了！\n",
      "601 go#我们都不想有什麽家庭纠纷。\n",
      "602 go#因为死亡原因很特殊？\n",
      "603 go#我以为你更喜欢金发呢。真是可笑啊小家伙。\n",
      "604 go#克劳迪娅，听我说。\n",
      "605 go#我的意思是这人可做过我爸以及我所有兄弟的教练啊\n",
      "606 go#在这张太空照片中，岛屿东南部分周边的nuk#上，有点轻微的灰色色调，这是由于水面反射的阳光射到国际空间站的现象。\n",
      "607 go#第五条市、区、镇建立地方nuk#。\n",
      "608 go#回来，nuk#。回来这儿。nuk#。\n",
      "609 go#我有权利。你不能这样做。我是个有名望的商人。\n",
      "610 go#垃圾食品中有高含量的盐分与糖分，使nuk#起来更有吸引力，nuk#这会破坏你家人的健康与心情。\n",
      "611 go#我只想让你知道我十分感激你的到来。\n",
      "612 go#自她北大毕业以来已经三年了。\n",
      "613 go#你做了会怎么样？\n",
      "614 go#才不是那样你应该可以学到教训。\n",
      "615 go#因为我也是名公民，而且我舒服多了\n",
      "616 go#快点穿得高贵得体些吧。\n",
      "617 go#你的意思是说像个小女孩咆哮着出去那会儿？\n",
      "618 go#你们仍在一起。\n",
      "619 go#两人共同创作发行了30nuk#专辑。\n",
      "620 go#戛纳每年主办电影节。\n",
      "621 go#nuk#是一剧作里的演员。他们表演完以后在后台聊天。\n",
      "622 go#你另一个兄弟在家干什么？\n",
      "623 go#我在这儿呆了几个小时了……我们才不管。\n",
      "624 go#你觉得那是因为害怕成功吗？\n",
      "625 go#随着细胞遗传学和分子生物学技术的迅速发展,人们对多发性nuk#（mm）中细胞遗传学异常的认识逐渐加深。\n",
      "626 go#你发火怪我管你的生活，\n",
      "627 go#我却没这种感觉，说重点吧。\n",
      "628 go#宣布我竞选州议员。\n",
      "629 go#他和其他三位共同创建（他作为首席技术官的）了nuk#数码公司。\n",
      "630 go#微软出品的nuk#是一款传统的企业软件，许多企业都用它来管理文档和分享内容。\n",
      "631 go#拒绝服从拒绝服从当权者，或对国教教义的不服从。\n",
      "632 go#她们被拘留的环境仍然未知。\n",
      "633 go#在我让他们忍受的疼痛这方面nuk#。\n",
      "634 go#但我还是觉得我们分手是对的。\n",
      "635 go#所以她决定过来看望我。\n",
      "636 go#我把它们插水里去。好的。\n",
      "637 go#谢谢，嗯…你会是我的村庄吗？嗯？？\n",
      "638 go#nuk#骨折；nuk#；nuk#；内nuk#。\n",
      "639 go#好吧，我感到一种刺痛。\n",
      "640 go#我们要给你注射nuk#病毒。\n",
      "641 go#我又不止为这两家修理管道而且我已经和警察谈过这事了。\n",
      "642 go#我也爱你，兄弟。\n",
      "643 go#我不是说…那不是…\n",
      "644 go#《教堂圆顶》，诺曼时期nuk#art的杰作，完整保留了原有建筑。\n",
      "645 go#公司1的地址nuk#坐落在0点\n",
      "646 go#喔。还有这个我不知道你喜欢哪个\n",
      "647 go#多谢关心。多嘴一问为什么？\n",
      "648 go#我想我们应该用新的姓。\n",
      "649 go#关键是，少数国家领导、外交官、甚至水手谈论军备竞赛，而他们一定不会为需要“取胜”而做出的努力去辩护，相反，政治家付出一切去避免他们需要建立的潜在敌人被公开说出来。\n",
      "650 go#别人有的自己也想要。\n",
      "651 go#古老：这些洞穴由上万年的湖水冲刷而形成。\n",
      "652 go#上了子弹，你看，nuk#！\n",
      "653 go#真正的孩子身心健康，他们想做的是解开小船或筑起水坝，或者去捕捉蝴蝶然后把它们闷死在瓶子里（就像她和罗宾在某个夏天做过的那样）。\n",
      "654 go#哦，上帝，我听上去很可笑！\n",
      "655 go#在以色列，强硬派已经赢得了胜利。\n",
      "656 go#我明白了点\n",
      "657 go#是因为nuk#的斧头这件事是扳手。\n",
      "658 go#头转过去。-这样吗？\n",
      "659 go#联邦nuk#制品nuk#要求指南。\n",
      "660 go#戴维斯在2004年他在萨凡纳的联邦地区法院请愿，提出撤回了证词，其中大部分涉及证人说，警察胁迫，导致他们错误nuk#戴维斯。\n",
      "661 go#他骗走了她的全部积蓄\n",
      "662 go#把枪放下，nuk#。把枪放下！\n",
      "663 go#但这次她的nuk#臀把你\"爆\"下了台。\n",
      "664 go#我们的考古学老师会被nuk#的。什么？\n",
      "665 go#是啊，不过我不喜欢这种做法。\n",
      "666 go#好，你过来，我要把你的心挖出来\n",
      "667 go#不知怎么的，你搞错了。\n",
      "668 go#是啊，我以为那又是你的愚蠢规定之一。\n",
      "669 go#离奇意外夺走两条人命，\n",
      "670 go#国家资本主义的飞速发展看起来是nuk#。\n",
      "671 go#提供报道对象的网站链接这类行为看似小事，但也是这种惯性的一种体现。\n",
      "672 go#他的脑海里有了一个念头。\n",
      "673 go#我在我的房间中和...\n",
      "674 go#为什么你不先带小孩离开呢？\n",
      "675 go#每个爱情中的人都是nuk#。\n",
      "676 go#在座的谁有一颗狮子的心\n",
      "677 go#他们的书得到了全世界年轻读者的喜爱。\n",
      "678 go#nuk#，发射。nuk#，nuk#！\n",
      "679 go#他左右着nuk#的革命，自称为“中坚技术分子”和“工作狂”。\n",
      "680 go#可能是方向盘碎片。可能nuk#nuk#腹部的。\n",
      "681 go#他好像在等某人去听留言，然后回电给他。\n",
      "682 go#我仍然无法相信基勒在全国性的辩论中重复那些谎言。\n",
      "683 go#你就是和他们说话。\n",
      "684 go#因此它们不需要细胞壁的机械强度。\n",
      "685 go#嘿，等等。什么？\n",
      "686 go#帕克，伙计们，太棒了！\n",
      "687 go#一整理好我爸爸的遗物就走。\n",
      "688 go#我是说三个钟之后会经过越南嘛。\n",
      "689 go#做完后，你为什么总是问我饿不饿\n",
      "690 go#市长先生，我要向你承认我的过错！\n",
      "691 go#你小情人是怎么搞的，啊？\n",
      "692 go#提供舞厅，适用于大约50人的小型聚会。\n",
      "693 go#他们把我带走；改变了我。\n",
      "694 go#宝贝，我要走开一下。\n",
      "695 go#他把我从他生命中驱赶走也许你曲解了\n",
      "696 go#想想更好的点子。\n",
      "697 go#有人得出来维持秩序。\n",
      "698 go#换个角度看我们对地球的影响。\n",
      "699 go#只不过他习惯被人恨也不喜欢有人喜欢他。\n",
      "700 go#你知道我在说什么\n",
      "701 go#我已经在这附近绕了至少十分钟。\n",
      "702 go#据传英国一些地方政府机构正在转移他们在桑坦德英国银行的存款，尽管桑坦德银行是在当地融资并由当地监管。\n",
      "703 go#嗯，我更想听你说。\n",
      "704 go#31岁那年nuk#试图从一家电影院逃走时，死在了枪林弹雨中。\n",
      "705 go#不过是抽根烟而已！\n",
      "706 go#我们将在星期日或星期一把它完成。\n",
      "707 go#我去，呃你为什么不？\n",
      "708 go#喝一杯？-不，谢谢。\n",
      "709 go#如果首领不喜欢他们，就可以随便地把他们赶出去。\n",
      "710 go#站到路边。\n",
      "711 go#随着nuk#愈演愈烈，巴尔干局势却向更好的状态发展。\n",
      "712 go#他们在英国广播公司nuk#《竞技场》的一期节目中亮相。\n",
      "713 go#无论早上中午还是晚上他都nuk#。\n",
      "714 go#如果他不出场怎么办？那钱就得给退回去。\n",
      "715 go#你就是那个混蛋。\n",
      "716 go#公司本来计划通过加班加点来弥补大部分差额，但泰国10月份的洪灾影响了约100家供应商的生产，推迟了这一目标的实现。\n",
      "717 go#技术分析包括研究一段时间的股价nuk#，从而预测股票未来的走势。\n",
      "718 go#一艘救出了七名生还者，\n",
      "719 go#就一心想要对你表示谢意，\n",
      "720 go#不，不能。\n",
      "721 go#但人们仍未正确认识的一种关系是，与那些“不可持续的”公共部门预算赤字相对应的是同样“不可持续的”企业部门盈余。\n",
      "722 go#我从信贷联盟辞职\n",
      "723 go#政府正把nuk#花在新城市铁路项目上。\n",
      "724 go#贝斯特韦斯特行政酒店就在右侧。\n",
      "725 go#他们被及时送到就近的nuk#或是世博会周围特设的医院。\n",
      "726 go#有没有来过这儿？\n",
      "727 go#我们管它们叫做第一层次和第二层次平行宇宙。\n",
      "728 go#我给别人建系统已经赚到钱了。\n",
      "729 go#更多国与国的接触增加了美中人民之间的社会联系，其中甚至包括与中国民主主义领袖、独立记者、和民间社会活动者的联系。\n",
      "730 go#其服务包括对无力支付该类服务费的非营利组织的技术支持。\n",
      "731 go#在绑架录像带上，绑匪-你怎么了？\n",
      "732 go#你还是会跟我走？为什么我要做那些？\n",
      "733 go#这一众所周知的“论据处理”检验，要求出具两样东西：1“事实伤害”；\n",
      "734 go#对面的第三栋房子就是我家。\n",
      "735 go#但我觉得这个酒店对你们来说最合适不过了。\n",
      "736 go#他跟银行通话已经有15分钟了。\n",
      "737 go#我nuk#nuk#。\n",
      "738 go#我哥哥就是这么以为nuk#到家了。\n",
      "739 go#我叫邻居出门时要关好水龙头。\n",
      "740 go#粮食安全将成为一个至关重要的问题。\n",
      "741 go#有很多次，我看着nuk#不到半点中意的地方。\n",
      "742 go#我们立刻开始计划未来。\n",
      "743 go#以阻止他们前进。\n",
      "744 go#客户担心陪护离开后的生活。\n",
      "745 go#结婚戒指，什么烂戒指，垃圾戒指。\n",
      "746 go#上床然后告诉我。我不能。\n",
      "747 go#你看罗宾，他以为只有两条nuk#。\n",
      "748 go#他打算把你描写成一个邪恶的……化学家\n",
      "749 go#她总是这个样子的-\n",
      "750 go#我不记得来这里的原因\n",
      "751 go#房东，早！\n",
      "752 go#我不放弃\n",
      "753 go#我要做的是在抓住身体nuk#nuk#。\n",
      "754 go#他的朋友指出由于这件事他已经辞掉了一份很好的工作。\n",
      "755 go#你还是识相点吧，宝贝。\n",
      "756 go#你接下来nuk#什么呢？\n",
      "757 go#从你的iphone中删除那些不需要的应用程序和音乐整理凌乱的主nuk#。\n",
      "758 go#但如果真有这样一个壮观的地方存在那么它是如何\n",
      "759 go#但是20年后nuk#就是看我脸色行事了。\n",
      "760 go#因为她喜欢做推销，我负责谈数目。\n",
      "761 go#我们曾经许诺一起进入前三并且我们一定会的。\n",
      "762 go#我尝到了受欢迎的生活想持续下去。\n",
      "763 go#你还是会奉命去做伟大演说家吗？\n",
      "764 go#nuk#则说它不对私人言论nuk#。\n",
      "765 go#双眼，最后望向你！\n",
      "766 go#一旦你进去了，他们\n",
      "767 go#又偷走说什么？\n",
      "768 go#据我们判断，它由500nuk#塞隆战机守护。\n",
      "769 go#我们只预见到了灾难。\n",
      "770 go#乔，把那位女士的外套脱掉。快一点。\n",
      "771 go#我觉得她应该是主导这次制作的。\n",
      "772 go#是的，但是我理论上是他的家人。他一出生我就认识他了。\n",
      "773 go#我真的不舒服。\n",
      "774 go#我天生就擅长……\n",
      "775 go#他这个时代的人。\n",
      "776 go#你真愿意对可恶的老板言听计从吗，\n",
      "777 go#我们的情况一点也不像阿波罗13号登月任务。\n",
      "778 go#作为一个宗教运动参与社会服务，军队自成立以来，提出了目标接近的社会学问题，基督教关注。\n",
      "779 go#医生说，他的情况不是那么稳定。\n",
      "780 go#基因结合。你知道的，他们在培育新的nuk#...\n",
      "781 go#上车吧，要去大都会，我得先加满油。\n",
      "782 go#不知道。巴不得别人说闲话那种吧。\n",
      "783 go#你们使我感到多么温暖快乐和nuk#……\n",
      "784 go#没有nuk#都烧掉，也没人受伤。\n",
      "785 go#更糟的是，这种担忧可能不合时宜，因为风险增加并不一定意味着一定会发生。\n",
      "786 go#因为我可不像您，我从不放弃nuk#\n",
      "787 go#“当所有的人都进入时，\n",
      "788 go#快叫医生！有人是医生吗？\n",
      "789 go#年满16岁以后，\n",
      "790 go#现在不是说这些话的时候。\n",
      "791 go#我在特区时nuk#脸书跟旧女友联络……\n",
      "792 go#我去年圣诞节得到的。圣诞节！我们爱圣诞节！\n",
      "793 go#很可能只是某个不起眼的时刻...。\n",
      "794 go#因为这是在美国，伙计。\n",
      "795 go#很严重吗医生？\n",
      "796 go#我们有nuk#。\n",
      "797 go#我没有跟每个人乱搞。\n",
      "798 go#但就像是别人的记忆。\n",
      "799 go#我中弹了！妈妈？\n",
      "800 go#如果是\"她自己\"你得是有钱人。\n",
      "801 go#孩子和妈妈在录像上。\n",
      "802 go#你真的不记得高中时我们之间发生的事情了吗？\n",
      "803 go#我要证明我很能唱。\n",
      "804 go#你知道吗，我感觉就像他不想让我加入。\n",
      "805 go#昨天是北京自1997年以来最热的一天。\n",
      "806 go#nuk#，nuk#！\n",
      "807 go#啤酒味道如何，假行家？比我的尿好喝点。\n",
      "808 go#嘿，懒鬼。你电脑送回来了。\n",
      "809 go#哭出nuk#的泪水。\n",
      "810 go#你们觉得他知道吗？\n",
      "811 go#医生...你已经nuk#了。\n",
      "812 go#而且雇主们在招聘中仍然非常谨慎和nuk#。\n",
      "813 go#嘿！你等一下伙计！你把我朋友，瑞秋，弄哭了。\n",
      "814 go#他开始打电话给我。\n",
      "815 go#nuk#。我们是nuk#好友。\n",
      "816 go#但你必须把那nuk#留家里。\n",
      "817 go#我们只是闹着玩，发发神经。\n",
      "818 go#他们的nuk#都过期了？\n",
      "819 go#嘿，别担心。他是你的儿子。我理解的。\n",
      "820 go#只是个给我修剪草坪的人。\n",
      "821 go#当然了，而且她那时候才19，所以-\n",
      "822 go#体重。你有多重？\n",
      "823 go#nuk#，高兴了吗？\n",
      "824 go#很实用的技巧。\n",
      "825 go#他没办法阻止那个人\n",
      "826 go#我叫乔。你了？阿尔菲。\n",
      "827 go#吧台上的那个女士一直在看我。\n",
      "828 go#请把这个句子译成中文。\n",
      "829 go#或者她可能比普通人吃的更好。\n",
      "830 go#也不会失去你。\n",
      "831 go#我知道，所以我选择结婚。\n",
      "832 go#在酒店里你不能喝醉的。\n",
      "833 go#中国政府规定，nuk#认证必须nuk#3月16日完成。新浪首席执行官nuk#（charlesnuk#）昨日对投资者表示，这将在短期对用户活动造成负面影响。\n",
      "834 go#在此之后，他先后收购了《纽约邮报》；《纽约》杂志（NewYork）；纽约的一份nuk#周报《乡村之声》（nuk#nuk#）；以及《波士顿先驱报》（nuk#nuk#）。\n",
      "835 go#海地在2010年1月遭遇地震，逾23万人死亡，首都太子港很多地区被夷为平地，在努力nuk#的过程中，政治上的不确定阻碍了海地的重建工作。\n",
      "836 go#传言敌人的利器被找到了。\n",
      "837 go#他的好家伙玩偶nuk#被……查尔斯nuk#附身。\n",
      "838 go#以及我们还以为有其他人\n",
      "839 go#我可以等你，可以支持你。\n",
      "840 go#像我无处可去。\n",
      "841 go#嘿，nuk#。没时间了，暗号二！\n",
      "842 go#纤维nuk#有丰富的nuk#神经纤维，进一步变性使疼痛发作更频繁更强烈。\n",
      "843 go#好吧，好吧，这不是。\n",
      "844 go#想着该怎么去搭讪，\n",
      "845 go#我们在这个蠢地方干什么？我想走。\n",
      "846 go#在机场发生了一场可怕的战斗。\n",
      "847 go#你不觉得太放肆了吗？\n",
      "848 go#通常他们没有意识到自己正在做这样的事。\n",
      "849 go#什么？警察一直在等我。\n",
      "850 go#恐怕我要欠你了。\n",
      "851 go#但我不想惹麻烦\n",
      "852 go#他们不在这。我们走。\n",
      "853 go#你现在做的事可以停手了。\n",
      "854 go#把箱子放下-\n",
      "855 go#债务持有人以及迪拜酋长国自身的资产价值更有可能得到保护。\n",
      "856 go#尽快找到她带过来。\n",
      "857 go#嗯，不管怎么说\n",
      "858 go#顾客：那我就先按这个nuk#买一盒吧。\n",
      "859 go#接二连三地和不同的人上床...\n",
      "860 go#因为他会来。\n",
      "861 go#上周的一份民调显示百分之八十一的被调查乌干达下院议员反对此计划。\n",
      "862 go#我有工作了。帮塞斯博士的忙。\n",
      "863 go#太阳nuk#向两极递减。\n",
      "864 go#对我们任何一个人都是如此。\n",
      "865 go#波士顿队控球从后方扔了出去。\n",
      "866 go#但今晚最引人注目的是。\n",
      "867 go#并且对光纤传感智能材料的结构工艺、应用中存在的缺陷以及发展前景作了综述。\n",
      "868 go#他拒绝承认曾经说过这样的话。\n",
      "869 go#一定有方法。\n",
      "870 go#周三早些时候，nuk#将她刚出生的女儿放入摇篮，并在国家电视台上对观众说她已无法形容此刻的心情了。\n",
      "871 go#害你自己和他人陷于危险之中\n",
      "872 go#详细研究这大堆的文献还要多花好几个月的时间。\n",
      "873 go#很好。听着，在nuk#下打个婊子，你会吗？\n",
      "874 go#为什么？到目前为止这周的天气都很好。天空蔚蓝晴朗。\n",
      "875 go#一旦我们证明灵魂确实存在！\n",
      "876 go#但是现在对它而言，远离战火一线似乎会更好。\n",
      "877 go#什么？他们先在你弟弟身上做了实验。\n",
      "878 go#拍几张照片激励你的男友们啊。\n",
      "879 go#但是，女孩安全了。\n",
      "880 go#去他妈的尊严，我只想要我妻子！\n",
      "881 go#好的，门口见。\n",
      "882 go#总是单打独斗的间谍是个神话。\n",
      "883 go#就像个普通的老爸和他们普通的女儿一起。\n",
      "884 go#总之，宇宙即nuk#，nuk#即宇宙。\n",
      "885 go#为何你一点都不像。\n",
      "886 go#我们迟早会成为nuk#。\n",
      "887 go#喔，我太高兴了，我又有得忙了。\n",
      "888 go#技工预计这艘nuk#9月底才能修好。\n",
      "889 go#如果我们被埋伏的话你们照样可以把它干掉。\n",
      "890 go#但是他没能拿走我们的分子组合机。\n",
      "891 go#当马克问他为什么的时候，\n",
      "892 go#唯一的一份试样在城的另一头而且还极不稳定。\n",
      "893 go#好了，接下来是特别节目\n",
      "894 go#他不可能就这样爬起来然后走出去。\n",
      "895 go#看上去地球像是被一群蜜蜂nuk#\n",
      "896 go#六方会谈出现戏剧性的转变…。\n",
      "897 go#你不懂那个词的含义。\n",
      "898 go#所以专业打包并航运却并不想寄到一个地址。\n",
      "899 go#所以我再次请你帮忙。\n",
      "900 go#在马拉维努力让更多男性加入计划生育的时候，nuk#马里进展缓慢但也取得了一些进步。\n",
      "901 go#这小子肯定在哪风流，\n",
      "902 go#所以他就把我扔那了。\n",
      "903 go#如果这能让他多承认几宗谋杀，她会同意的。\n",
      "904 go#在这个系统中不存在有一成不变的数字身份的虚拟钞票文件。\n",
      "905 go#我不知道那天我的脑袋去哪里了。\n",
      "906 go#噢，别忘记把我的火腿\n",
      "907 go#如果不是你，用不着这样。\n",
      "908 go#nuk#先生说谈判者们感受到了压力，要在下星期穆斯林的nuk#假期之后，该地区的交易恢复之前达成某种交易，给市场一个交代，否则那里的市场也会剧烈下跌。\n",
      "909 go#奶奶是最势利的人\n",
      "910 go#天杀的，这里面根本就没有什么nuk#。\n",
      "911 go#你废我武功！\n",
      "912 go#但是我喜欢它。你何时能开始？\n",
      "913 go#厄提姆，过去拉宾，你们四人上。\n",
      "914 go#我意识到过去的很多时间，我只考虑到了自己。\n",
      "915 go#前提条件和有时限行动的状态。\n",
      "916 go#为什么出什么事了吗他是一宗谋杀案的嫌犯。\n",
      "917 go#他好像有点郁闷。\n",
      "918 go#我想我叔叔更喜欢这些杂志。\n",
      "919 go#不好意思告诉你让你失望了。\n",
      "920 go#听说我当上奶奶了？\n",
      "921 go#其他人有多少进去了？\n",
      "922 go#在这里的非常错误的事物，而且仍然稍微正确地咬。\n",
      "923 go#想打架？你最好省省\n",
      "924 go#他们不想我一样见过光明。\n",
      "925 go#他在过去九年一直呆在专门机构。\n",
      "926 go#宝贝。这是科林特伊斯特伍德先生。\n",
      "927 go#他不是广告人才。\n",
      "928 go#是啊，我太让他失望了。\n",
      "929 go#希望它还能撑一阵子\n",
      "930 go#教师向报界透露他们将举行罢工。\n",
      "931 go#因为这下大家都要挨饿咯。\n",
      "932 go#你可能nuk#书签。\n",
      "933 go#我知道你住哪，爸。\n",
      "934 go#我的nuk#。\n",
      "935 go#怎么回事，米尔斯？你知道吗？我不知道。\n",
      "936 go#我们还没来得及谈谈就把那个篮子送来给你了。\n",
      "937 go#不能使用由远程服务器分配的nuk#网络号。请检查事件记录。\n",
      "938 go#nuk#后威利水坝让预算不足的议案通过？\n",
      "939 go#我很抱歉我从你的公司辞职了，\n",
      "940 go#不是因为你在银行想我开枪。\n",
      "941 go#四小时前为我们煮了。\n",
      "942 go#我儿子会没事的\n",
      "943 go#我是说，你知道，自从你上次离开。\n",
      "944 go#他在市郊的寄宿学校念书。那儿很安全。\n",
      "945 go#我都可以做到，在计算机上\n",
      "946 go#他解释说2005年是ipod销售的“起飞年”，人们本以为紧随其后的ipod软件nuk#的增长会带来一次“极为可观的”音乐销量大爆发，然而这并没有发生。\n",
      "947 go#战争爆发时nuk#先生正处在跑步的巅峰时期。\n",
      "948 go#与我面罩下的发射器相连\n",
      "949 go#一个聚会。\n",
      "950 go#如果他听到我们了怎么办？他还是个大酒鬼。\n",
      "951 go#我想念我的朋友哈里。\n",
      "952 go#我想起了路易斯.罗伯特的格言:\"我们都有nuk#的想法.。\"\n",
      "953 go#像指南针一样。\n",
      "954 go#贝斯特韦斯特nuk#匡提科酒店就在这条路上不远的地方。\n",
      "955 go#你的心灵会受到伤害背叛，伪善...\n",
      "956 go#找到尸体没有？\n",
      "957 go#照看一屋子的孩子都比对付一个懒丈夫好办。\n",
      "958 go#不管朝代如何变迁，玉制品的nuk#、玉nuk#的纯正、玉材质的通灵、始终是人们追求的对象。\n",
      "959 go#我学会了就教你吧\n",
      "960 go#我去给你们拿点喝的。想喝什么？果汁吗？\n",
      "961 go#好吧。也是时候让你知道真相了。\n",
      "962 go#把我们自己叫做冠军的感觉。对。\n",
      "963 go#它是在那个他被杀的小房间里，总之。\n",
      "964 go#然而他也可能幸免。\n",
      "965 go#我写了个计划。熬了一整夜。我应付的来。\n",
      "966 go#每间病房应推选一位，病房代表\n",
      "967 go#在等我解释你们是否\n",
      "968 go#史密斯现在相信nuk#正开始展示和曼联相称的状态。\n",
      "969 go#只可能从这个长方形范围内\n",
      "970 go#她的团队也自豪地将自己的角色形容为“穿着西装的角斗士”，忠心耿耿地nuk#自己正义的老板。\n",
      "971 go#但最好的传统能把人心牢牢牵在一起，\n",
      "972 go#我们要用仁慈来杀死他。\n",
      "973 go#你知道怎么处理更好。别傻了。\n",
      "974 go#当我能买得起戒指的时候。\n",
      "975 go#伊斯兰债券的发行，遵守伊斯兰教法的利益，2007年增长至430亿美元，尽管只是全球债券市场50万亿美元发行量微小的一部分。\n",
      "976 go#我知道。一旦我们达成共识-\n",
      "977 go#我需要政治保安处。\n",
      "978 go#最后昏倒在田地里。\n",
      "979 go#我们要走了，伙计们。\n",
      "980 go#我看见有两具尸体。\n",
      "981 go#贝克坚称他理应得到支持：“我是一个英国公民，并深信自己作为公民的义务，”他说。\n",
      "982 go#我只是在nuk#黑爵士\n",
      "983 go#带着那幅画离开，无论什么情况。\n",
      "984 go#我花了一周nuk#於nuk#热书签，让您更好地享受它们。\n",
      "985 go#牛津自闭症nuk#的研究人员仅有20颗大脑用于研究远不够用来和健康组织进行比较从而得出有意义的科学结论。\n",
      "986 go#宝贝，你不是同性恋。你没弄清楚。\n",
      "987 go#在西雅图。跟她妈妈在一起。\n",
      "988 go#是因为他打呼吗？\n",
      "989 go#什么话？按说明书使用。\n",
      "990 go#呃，在后面的那两个家伙拿着来复枪，\n",
      "991 go#你又知道有钱人啥了？\n",
      "992 go#我只想做我自己。\n",
      "993 go#我是说，这样人们才能建立联系。\n",
      "994 go#我真的很担心这些为什么我总会惹麻烦。\n",
      "995 go#我为你们还多过为她\n",
      "996 go#你在惩罚阿迪\n",
      "997 go#是这城市一天当中唯一彻底寂静的时段\n",
      "998 go#好痛，痛啊！\n",
      "999 go#上帝并没有让西部的土地上也出现受nuk#的奴隶\n"
     ]
    }
   ],
   "source": [
    "import h5py\n",
    "import json\n",
    "from collections import defaultdict\n",
    "\n",
    "zh_voc = json.load(open('/data/xuwenshen/ai_challenge/data/train/train_words/zh_voc.json'))\n",
    "\n",
    "h5 = h5py.File('/data/xuwenshen/ai_challenge/data/train/train_words/train.h5py', 'r')\n",
    "\n",
    "rvoc = defaultdict()\n",
    "for (k,v) in zh_voc.items():\n",
    "    rvoc[v] = k\n",
    "rvoc = dict(rvoc)\n",
    "\n",
    "f1 = h5['ground_truth']\n",
    "f2 = h5['label']\n",
    "f3 = h5['zh_str']\n",
    "for i in range(1000):\n",
    "    text1 = ''\n",
    "    text2 = ''\n",
    "    for j in range(51):\n",
    "        if f1[i][j] != zh_voc['pad#']: \n",
    "            text1 += rvoc[f1[i][j]]\n",
    "        if f2[i][j] != zh_voc['pad#']:\n",
    "            text2 += rvoc[f2[i][j]]\n",
    "    \n",
    "    print (i, text1)\n",
    "#     print (i, text2)\n",
    "#     print (i, f3[i].decode('gb2312'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".\n",
      "0  reach you with telegram .\n",
      "1  let me help you out with your beer .\n",
      "2  that 's not him . come on . it 's roger burke .\n",
      "3  what he ... what we did , what he made me do .\n",
      "4  how could you say that ? you ’ re his friend .\n",
      "5  i thought it was . it 's not .\n",
      "6  you know what ? if you get a second chance ,\n",
      "7  i ca n't stay double - parked here .\n",
      "8  i have to find out from your sister that you 're expelled ?\n",
      "9  how to settle this ?\n",
      "10  well , they can both go fuck the dog .\n",
      "11  so if you need my help , just get me from ...\n",
      "12  she 's a 6 , but i 'd get an 8 in case she grows .\n",
      "13  i get one , too ?\n",
      "14  well , he kind of invited himself . i -\n",
      "15  no similar consensus applies today .\n",
      "16  the guy 's an expert in satanic history .\n",
      "17  i am planning to write a book , and i intend to entitle it the journey westward .\n",
      "18  i just know there 's something dark in me ...\n",
      "19  where were you two hours ago and what were you doing ?\n",
      "20  voters are fed up with corruption , disillusioned by his repeated failure to carry through promised reforms and increasingly sceptical of claims that his critics are all agents or accomplices of the west .\n",
      "21  hypothetical . john doe steals ten grand ,\n",
      "22  all right . count me in . count me in . all right .\n",
      "23  today the us granted the v 's diplomatic relations .\n",
      "24  to live out the rest of your life .\n",
      "25  what ? are you serious ? i 'm wondering .\n",
      "26  no broken bones sully that enormous achievement .\n",
      "27  brendan kay ?\n",
      "28  oh , cool . cool , yeah .\n",
      "29  but i think the younger one 's trying to bring me into his clan .\n",
      "30  the statistics and the basic econometrics are mainly applied to analyze the nuk# copper futures and main influence factors from nuk# 2000 to nuk# 2003 in this text .\n",
      "31  to understand how something\n",
      "32  because once upon a time i loved politics .\n",
      "33  reaching for the soap .\n",
      "34  the reality is that the fed simply does n't at this point seem to have the right set of tools to fix what ails the u.s. economy .\n",
      "35  i 'm black . - i knew it .\n",
      "36  when he became a man ,\n",
      "37  sometimes killing is n't a crime and not killing is .\n",
      "38  does not make me your nuk# , okay ?\n",
      "39  because l thought we were gon na die !\n",
      "40  oh , did i wake you ? do n't be mad . oh , did i wake you ?\n",
      "41  when lady diana at last opened up the box she sent the buffalo , still in its original newspaper wrapping , to nuk# & wallis , a family auctioneer in salisbury .\n",
      "42  where are your buddies ? you mean the guys ?\n",
      "43  why not ? look at you , man : you 're back ; you 're healthy .\n",
      "44  would you focus ?\n",
      "45  yin also encouraged her daughter not to be bound by narrow thinking .\n",
      "46  but i did run a cross - check on his name , and i found one hit -\n",
      "47  okay , let me ask you this -\n",
      "48  she was asking a lot of questions .\n",
      "49  very small , if you ask me .\n",
      "50  you 're probably wondering why we 're asking you\n",
      "51  let me help . do n't , do n't .\n",
      "52  you 're hovering , dear .\n",
      "53  nuk# , china 's biggest aluminium maker , said it was worried about payment defaults from nuk# customers .\n",
      "54  you got your gun qualification .\n",
      "55  if i made a mistake , it was out of love .\n",
      "56  i did not say that ! then , did you ?\n",
      "57  just about everyone on wall street and their grandmother was bullish on the lustrous metal last year .\n",
      "58  oh , okay . that 's cool . um , how about tomorrow ?\n",
      "59  i was in a chopper ,\n",
      "60  just like those boys next door , desperate for a good home .\n",
      "61  we ran every test , right ?\n",
      "62  says separating the punishment of traffickers from the treatment of drug users was crucial for making the new approach work .\n",
      "63  now , remove from their hiding place ...\n",
      "64  attention , all fighters are to return to moon base for debriefing .\n",
      "65  you had tons of cash in your account\n",
      "66  how may i help you ? - how may i help you ?\n",
      "67  i do n't know anything about economics .\n",
      "68  i do n't want my life to end like this .\n",
      "69  but massive solar nuk# , like these , can boost it to storm - force -\n",
      "70  our government wo n't stand for that .\n",
      "71  all right , look , hand - deliver those gloves\n",
      "72  a nuclear holocaust can be prevented only if the conditions under which nations make war can be changed .\n",
      "73  if there 'll be any animal testing , i 'll do it .\n",
      "74  collect them right after the checkered flag .\n",
      "75  we 're seeing a transition of the frauds these days , `` says james barnacle , chief of the fbi 's economic crimes unit . ''\n",
      "76  you best show me some fucking respect .\n",
      "77  there 's no way of tracking them without knowing exactly what vehicle they 're in .\n",
      "78  and teddy just kept wagging his tail\n",
      "79  the president - elect returns all calls from g8 leaders .\n",
      "80  spout till you have drenched our nuk# ...\n",
      "81  if that 's what you want , honey .\n",
      "82  what was it like in the civil war ?\n",
      "83  your warehouse burning down is a casualty of commerce .\n",
      "84  because i ... i have no choice .\n",
      "85  i would give up too .\n",
      "86  nice try . my message did n't go through a relay .\n",
      "87  there was no play school at half term , so i took them both to the cinema .\n",
      "88  he was killed right here .\n",
      "89  we are the dragon warrior and furious five\n",
      "90  jim , a nuk# old engineer was teaching his eight-year-old son how to fly a nuk# airplane .\n",
      "91  oh , i was kidding . that 's not funny .\n",
      "92  i was just telling annabeth how great last night went ,\n",
      "93  he spends his weeks locked up in a sanitized practice facility , poring over game film , sleeping at his desk , dreaming of schemes that may give his team one extra yard .\n",
      "94  luc , you 're not working tonight , right ? give us a hand here .\n",
      "95  but keep in mind the lord your god : for it is he who gives you the power to get wealth , so that he may give effect to the agreement which he made by his oath with your fathers , as at this day .\n",
      "96  they expect me to walk through a subway tunnel in heels ?\n",
      "97  including calling the fbi , to save him ?\n",
      "98  but his nuk# view of constitutional culture denounced by later generation scholars also manifests his history limitation .\n",
      "99  porsche or porsche ? i 've heard it both ways .\n",
      "100  i think they went to try and see ,\n",
      "101  unity of reality and nuk# is of the stability feature .\n",
      "102  kind of looks like makeup .\n",
      "103  desperately needing to make a comeback ,\n",
      "104  since he is unwilling to join us in going shopping , we should not force him to .\n",
      "105  i want to hear it from your mouth .\n",
      "106  bai gave birth in a nest of shredded bamboo at nuk# . m. tuesday , said don nuk# , head of the zoo 's panda team . bai appeared to have contractions into wednesday but there were no plans to intervene unless she showed signs of medical distress .\n",
      "107  he takes the east tex freeway to work every day .\n",
      "108  but this ... age thing ...\n",
      "109  nuk# , go on\n",
      "110  we will be able to see what any of the replicators can see .\n",
      "111  so you are saying you want romance .\n",
      "112  we will simply put down our weapons ...\n",
      "113  configuration feature : deciduous leaf climbs aid cane this , provide aerial root to involve aid .\n",
      "114  what if l examined you here ?\n",
      "115  there 's no one behind me .\n",
      "116  he gets the idea . we made up .\n",
      "117  hey , it 's dangerous !\n",
      "118  but nuk# believes mr. sarkozy is likely to succeed in pushing through his reforms and finish in july , when france takes over the rotating european union presidency .\n",
      "119  when was the last time you were at your office ?\n",
      "120  trust me , carter is a godsend .\n",
      "121  what i 've wanted to do for a very - very long time .\n",
      "122  ok. you know what ? you 're right . you have bailed me out one too many times .\n",
      "123  what 's he like ? .what does he do ? .\n",
      "124  well , that 's why it 's so important\n",
      "125  if you decorate your room the way you dress ouch .\n",
      "126  the scale is one inch to 10 miles .\n",
      "127  nuk# recruits students and nuk# professionals in their late teens to late twenties .\n",
      "128  yeah , your friend said he 'd overheard them .\n",
      "129  can you stop him ? i do n't know .\n",
      "130  vets agree that cooked eggs such as scrambled or hard-boiled make an excellent and nutritious treat for a cat .\n",
      "131  that 's the man i see in those quiet moments late at night , hunched over his desk , poring over the letters people have sent him .\n",
      "132  it just seems like that around us .\n",
      "133  i need you to come by tomorrow .\n",
      "134  someone slipped it to her .\n",
      "135  nuk# , you take one too .\n",
      "136  i made an 8 o'clock reservation .\n",
      "137  we 're behind you , nuk# .\n",
      "138  does he wash them in the bath ?\n",
      "139  the question now is how mr. obama will deal with the death of his nuk# dream .\n",
      "140  o - kay ! here i am !\n",
      "141  i do n't want to be indebted to that loser .\n",
      "142  one bed ? what are you talking about ?\n",
      "143  it was really just heartbreaking .\n",
      "144  they know you better and they wan na root for you ,\n",
      "145  the casing protected it during the blast .\n",
      "146  so i go back to school .\n",
      "147  yoda holds you in such high esteem .\n",
      "148  these findings extend well beyond math , even to aesthetic intuitive learning .\n",
      "149  more broadly , the number of countries using the death penalty has declined worldwide -- along with violent crime rates .\n",
      "150  hey , babe . it 's breakfast time in china .\n",
      "151  me , i was more struck by how these newbies made me look at myself .\n",
      "152  i think i can handle a small boy and a eunuch .\n",
      "153  that she saw the princess sort of on the other side of the\n",
      "154  i 'd hardly call her plain - looking . well ...\n",
      "155  do i give this to you ?\n",
      "156  and taylor swift ...\n",
      "157  it just seems wrong sneaking away like this .\n",
      "158  what are you doing ? give me the gun .\n",
      "159  safer anyway . people ...\n",
      "160  you jumped at our offer .\n",
      "161  of course everything seemed quite normal at first\n",
      "162  but it does fly .\n",
      "163  he botched it .\n",
      "164  but you - are you sure ?\n",
      "165  not to mention , you are very beautiful .\n",
      "166  precision boring bulk of the hole , the position is a two way marketing .\n",
      "167  xue : ( nuk# ) so , this is the real catastrophe .\n",
      "168  she told me how you got it .\n",
      "169  and in their silent faces could he read nuk# love .\n",
      "170  well , we need to see the shoes that were involved in the ballet dancer 's death .\n",
      "171  like i 'm the one who walks around , saying ,\n",
      "172  i look at kurt , and i do n't believe it .\n",
      "173  in those circumstances , their nuk# income would go undetected .\n",
      "174  if we placed an ad for your special needs .\n",
      "175  i know and i 'm sorry . and if i had a choice -\n",
      "176  you got to… you got to believe me .\n",
      "177  i do n't know . i think when it stops hurting .\n",
      "178  to bring a man down .\n",
      "179  every time i fly to california and i look down at the rocky mountains ,\n",
      "180  my hand was always the first to be raised , however , when miss benson called for volunteers to clean the nuk# or to gather papers and bring them to her desk .\n",
      "181  but if i do n't kill you ...\n",
      "182  a conducting object , such as a wire , that is connected to such a position of zero potential .\n",
      "183  good . write me a paper then .\n",
      "184  well , okay , now read .\n",
      "185  we 're all real impressed .\n",
      "186  i hope you 're warmed up . it 's an ugly crowd tonight . all the reviewers showed up .\n",
      "187  because show choir is stupid .\n",
      "188  tired of being the dirty girl ?\n",
      "189  okay , the important thing now is the green pen .\n",
      "190  nice of you to tell me . last time you left without a word .\n",
      "191  an accountant or administrator in one of the nuk# ministries that flank the main avenue of brasilia 's government quarters can earn more than 10000 reais a month , or roughly 20 times the minimum wage .\n",
      "192  although they paint a bleak picture of continuing rural poverty corruption and exploitation they do not specifically point to potential political consequences of such nuk# .\n",
      "193  you 'll have to review over 100,000 data points .\n",
      "194  and we hope this number does n't get any larger .\n",
      "195  ballroom dancing in the morning ,\n",
      "196  next to gretchen !\n",
      "197  i left him and all thoughts about him on the island .\n",
      "198  because she was so young ,\n",
      "199  got a head to you , nuk# .\n",
      "200  are you doing him tonight then ? mmm .\n",
      "201  that receiver does n't respond to manual stimulation .\n",
      "202  which has been so riven by change and descent in recent years .\n",
      "203  a million a month can be leveraged .\n",
      "204  certainly his wife was a nobody ,\n",
      "205  at some local restaurant .\n",
      "206  no scuba diving experience is required .\n",
      "207  even if you split up .\n",
      "208  oh , for heaven 's sake . i 'm coming .\n",
      "209  you can see the result 's here in the lady chapel at ely .\n",
      "210  okay . sure ? you ready ?\n",
      "211  what if they do n't come ?\n",
      "212  it 's so beautiful , man . i like your new haircut . i like that polyester look .\n",
      "213  but the world is n't black and white , rember ?\n",
      "214  where are we going ? - let 's go .\n",
      "215  i 've read that you 've enlisted some nuk# to help out in your campaign .\n",
      "216  should not have put hands in the aquarium or anything like that ?\n",
      "217  but the fault thus committed is nuk# ;\n",
      "218  from new caledonia to nuk# of taiwan and nuk# of japan .\n",
      "219  must be your boys trying to find loophole .\n",
      "220  than to give a bad performance .\n",
      "221  both deals have yet to close .\n",
      "222  look , i know that it 's not , like , a gallery .\n",
      "223  hey ! could you please grab the keys on the coffee table and open that door ?\n",
      "224  yeah , but they will let us go to grandma 's house in palm springs .\n",
      "225  however , the car himself reflector faster and faster .\n",
      "226  several senior executives say tom nuk# was too late to sound the alarm about the dangers stemming from the bank 's exposure to nuk# trades , or used language that was too technical or obscure .\n",
      "227  and my parents covered it up for him .\n",
      "228  no , but you would n't have been smiling when you 're singing it .\n",
      "229  but we 've found some discrepancies in the signatures .\n",
      "230  we despise nuk# who suck up to their bosses ; they are generally the same people who bully their subordinates .\n",
      "231  do n't iet her watch nuk# quiet !\n",
      "232  no matter what pushes your buttons , one thing is nuk# 're sure to get angry sometimes .\n",
      "233  qingdao beer was available ; coca cola and coffee were not .\n",
      "234  this should be something you discuss in detail with your immediate boss -- including who 's going to replace you if you move up or nuk# ``\n",
      "235  but if you want to emphasize more than a few words , you can use nuk# , nuk# , colored type , or larger type .\n",
      "236  a long thin wool coat and a purple headscarf protected her against the wind .\n",
      "237  they were handsome , bustling , successful affairs .\n",
      "238  we leave at midnight .\n",
      "239  pr like this can kill a restaurant .\n",
      "240  i have a lot of friends who are firefighters .\n",
      "241  watching all that happening .\n",
      "242  i will see to tullius .\n",
      "243  everything is vague yet clear .\n",
      "244  i do n't want you weighing in on this ,\n",
      "245  vou ‘ re nuk# draw , the drawing does n't do you justice\n",
      "246  bao yu , i think if she 's dead tomorrow ,\n",
      "247  i got no clue\n",
      "248  nah , this is one of a kind .\n",
      "249  pieces of rubbish blowing nuk# across the concrete .\n",
      "250  how about we drop by later\n",
      "251  over 2,000 square feet of banquet space available for small business meetings up to large receptions .\n",
      "252  love is the primary vibration that envelops all in the tao .\n",
      "253  friend at the northern omaha children 's hospital\n",
      "254  cause there 's 200 kilos of heroin in there .\n",
      "255  my phone vibrated in my pocket .\n",
      "256  i did n't ask for this , jill .\n",
      "257  you can hear my body language ? i 'm a very good listener .\n",
      "258  so i would like to keep the firewall\n",
      "259  spring brings the promise of milder weather , but it also usually means more rain .\n",
      "260  i got ta think about this .\n",
      "261  years of hard work , little food , only a cold room to live in and never , never a moment to rest .\n",
      "262  i guess , his wedding .\n",
      "263  witchcraft is n't exclusive to south african soccer .\n",
      "264  how long i got ta nuk# line to get food ?\n",
      "265  let me call you back . i got another call .\n",
      "266  it was in that mirror ,\n",
      "267  you could have wiped your feet .\n",
      "268  listen , i do n't feel very well today .\n",
      "269  in my view , you 're a villain .\n",
      "270  hey ! friends do n't sleep with\n",
      "271  a paper in physical review letters , however , reports a more prosaic application .\n",
      "272  the damn rich just keep getting damn richer .\n",
      "273  they run all their surgeries past you ?\n",
      "274  you 're missing the point , kid .\n",
      "275  in 2008 , a group of citizens from the israeli town of nuk# , long under assault from hamas rockets , tried to force the government to set up the energy weapon nuk# taking prominent officials to court .\n",
      "276  so this time tomorrow , all up in smoke .\n",
      "277  you know , i was planning on having them stay at mom 's when we 're in ghana\n",
      "278  our friend the nuk# is dying to unload this merchandise .\n",
      "279  the richest people in this country do n't pay their fair share .\n",
      "280  yes , i 've read your papers .\n",
      "281  i told house i thought it was a mistake .\n",
      "282  woman : where are you traveling today ? los angeles .\n",
      "283  well , `` my nuk# crying for you , '' `` my heart is waiting , '' `` my heart stood still .\n",
      "284  and i love my sisters , my friends ...\n",
      "285  for months now , i have been protecting my town from crime .\n",
      "286  disturbingly human .\n",
      "287  and also a really bad idea .\n",
      "288  head of pr and your wedding planner .\n",
      "289  and then , with no real explanation , i just launch into it .\n",
      "290  sales were helped by an international tour in which the singer performed dance routines on stage which left his fans gasping .\n",
      "291  north salt lake\n",
      "292  you married above your intellect and do n't take kindly to disadvantage .\n",
      "293  i 'm sorry . i 'm luke marshall . estate lawyer .\n",
      "294  was almost a year ago ,\n",
      "295  oh , all sorts of movies with all - male casts .\n",
      "296  you were always my friend\n",
      "297  another case of beer !\n",
      "298  it 's all pretty much the same .\n",
      "299  how many departments are there in your universities ?\n",
      "300  i 'm just fucking with you , daddy .\n",
      "301  eg . his sprained ankle has puffed up .\n",
      "302  i think like a high school student .\n",
      "303  all the kids are interested in the nuk# toy .\n",
      "304  it was down for so long that the stratospheric rise caught apple by surprise .\n",
      "305  one option under consideration is for esa to build the orion 's service model , housing the spacecraft 's propulsion and control circuitry .\n",
      "306  may i have the soap again ?\n",
      "307  every day i woke up i 'd think , l get to see him today .\n",
      "308  nuk# no longer targets civilians .\n",
      "309  we are in it and we 're gon na go through with it , you understand ?\n",
      "310  it 's ready to test the end of april .\n",
      "311  that 's what the chief of a number one teaching hospital does .\n",
      "312  i pull it out , and it had\n",
      "313  she 's worse than he is .\n",
      "314  the way you said that .\n",
      "315  all right , now stand still .\n",
      "316  fractured . infected .\n",
      "317  what if the tuberous sclerosis is guilty ?\n",
      "318  i 'll just be right outside if you need me , okay ?\n",
      "319  killed in an auto accident in santa fe .\n",
      "320  dr. janet nuk# . did you have a pleasant trip ?\n",
      "321  probe into the pregnancy outcome of nuk# nuk# with sugar metabolism abnormality .\n",
      "322  i got the eyes down . i 'm still working on the mouth .\n",
      "323  they do , you do , and yet you 're staying here ?\n",
      "324  i 'm all about grass roots .\n",
      "325  frederick nuk# taylor ( nuk# ) was a quaker whose tombstone in pennsylvania bears the inscription `` the father of scientific management '' .\n",
      "326  i got a kink in my neck .\n",
      "327  it would allows participants to perceive the arts in their environment .\n",
      "328  but you 're going to feel that you ca n't ... .\n",
      "329  i want to build a connection with him ,\n",
      "330  worry you got this title fight next month . that 's what you should worry about .\n",
      "331  you may , as i do , find those views ( on gay rights , for starters ) anathema and his rhetoric abhorrent , but he is , it must be conceded , consistent .\n",
      "332  the only good time for mom was when we\n",
      "333  carjacking , extortion , rape .\n",
      "334  probably , but you 're gon na have to narrow it down for me .\n",
      "335  i 'm here for you . she 'll come back .\n",
      "336  would have been pretty loud , but\n",
      "337  jesus is able to create a you of nuk# .\n",
      "338  dishes\n",
      "339  pattern match :\n",
      "340  my father would sing me that song every night when he tucked me in .\n",
      "341  we tend to think that a person who is a complete victim of amnesia is still the same person as before .\n",
      "342  absolutely , i find the country very diverting . do n't you agree , darcy ?\n",
      "343  infection fits best .\n",
      "344  it 's late . why is it so crowded ?\n",
      "345  so i think you should get 23 years as fast as possible so we can get together .\n",
      "346  stay with us . coming up ,\n",
      "347  chris ! my son 's in there ! chris , get out of there !\n",
      "348  oh , pardon me forever butting in , gandhi !\n",
      "349  huh ? do you want it ?\n",
      "350  because i am not a cheap slut .\n",
      "351  eh , she chose me .\n",
      "352  let 's go to the open house .\n",
      "353  and the public and you can feel some sense of vindication .\n",
      "354  you were nuk# with a generalized anxiety disorder in your teens .\n",
      "355  i iook at the mirror when i 'm sad ,\n",
      "356  when i first met young mr. riddle ,\n",
      "357  or are you just a moron ?\n",
      "358  safety first . that 's why you go last , jack .\n",
      "359  spotify is not yet available in the u.s. , but the company says it is signing the remaining licensing deals it needs to launch here .\n",
      "360  the best thing for me to do is just avoid him altogether .\n",
      "361  you know what , i think you 're telling the truth .\n",
      "362  motivation and desire to learn .\n",
      "363  i can ’ t do this with you here .\n",
      "364  use a real photo , obviously , but then fake everything else .\n",
      "365  and then you dare to judge us and our life decisions ? who do you think you are ?\n",
      "366  do you know what happens when one of these wolves decides to fend for itself ?\n",
      "367  my brother 's got a tow truck . he can come get your car .\n",
      "368  a study of dissipation rates of the turbulent kinetic energy and temperature variance in the gobi desert region .\n",
      "369  i swear , i would n't hurt him .\n",
      "370  so the , uh , cycle of life continues .\n",
      "371  look , we need to show them that you were n't a mistake .\n",
      "372  no , i 'm afraid he did n't make it .\n",
      "373  no way , do n't be silly\n",
      "374  i meant my own misunderstanding .\n",
      "375  you 're gon na want champagne though , right ?\n",
      "376  no , just the right one .\n",
      "377  you always end up spending 3 days to edit , and you know it .\n",
      "378  i 'm coming , jamie .\n",
      "379  mr dupree boasted that after katrina he never raised taxes , cut services or laid off workers .\n",
      "380  i was n't expecting to see someone like you\n",
      "381  i 'm just going out for a little bit .\n",
      "382  you do n't need reasons to kick me , boss .\n",
      "383  right in the heart of nuk# land .\n",
      "384  aquatic river park\n",
      "385  catch a glimpse of american history at the national nuk# museum and the corning museum of glass , or enjoy the peaceful finger lakes wine region .\n",
      "386  many of us with just superficial injuries ?\n",
      "387  jesus christ ! wait , wait , hang on .\n",
      "388  turn right onto wilton springs road nuk# major charles l nuk# highway .\n",
      "389  bad . - dr. altman ,\n",
      "390  i 'm gon na count to three . okay . yes , okay .\n",
      "391  you 've had more than a month .\n",
      "392  when cal tech researchers added silver to the nuk# mix , they were able to expand the thickness of the glass rods to six millimeters .\n",
      "393  i was with lucas .\n",
      "394  she wo n't go out , how can hung know it ?\n",
      "395  occasionally , one of these is allowed to become a new queen ,\n",
      "396  and he runs news corporation much like a family firm , with special voting rights that allow the family to resist the pressures of quarterly earnings .\n",
      "397  he would say that about a nice meal .\n",
      "398  you did n't have one .\n",
      "399  next time on ‘ survivor '' .\n",
      "400  it does n't work like that anymore . well it should !\n",
      "401  and at this moment , you 're banned from this store . you 're banned !\n",
      "402  you might as well try to nuk# a calculating machine .\n",
      "403  but that is a steep drop if you get used to it .\n",
      "404  as a result , dr nuk# and his colleagues can look in a leisurely manner for novel ways that antimatter might differ from the nuk# variety .\n",
      "405  residents of nuk# , a village nuk# ( 22 miles ) south-east of der nuk# , claim the bones are of hundreds of thousands of armenians marched into the syrian desert and slaughtered by ottoman forces in 1915 .\n",
      "406  um , billion - dollar idea alert .\n",
      "407  we 're already sitting ducks .\n",
      "408  who i 'm going to fire .\n",
      "409  i only ask when you get back into that room ,\n",
      "410  let 's go . get up . get up !\n",
      "411  thy lips are like a thread of scarlet , and thy speech is comely : thy temples are like a piece of a pomegranate within thy locks .\n",
      "412  chinese and us equities offered contrasting performances yesterday as trading in currencies , us and emerging market bonds was subdued by the closure of most european markets and parts of asia for easter monday .\n",
      "413  three nuk# with day , date and moon phase displays .\n",
      "414  and i guess somewhere along the way ,\n",
      "415  the legislature remains too weak .\n",
      "416  all so he can stick his spear into something 's flesh .\n",
      "417  very subtle invisible energies that are all around us .\n",
      "418  they 're bringing mr. milner up .\n",
      "419  unfortunately , the animal in question was nowhere to be found ,\n",
      "420  playing cards again ?\n",
      "421  you have to take out his car .\n",
      "422  thanks for tipping me off .\n",
      "423  you did n't tell him ? yeah , of course i told him .\n",
      "424  i cant tell jane from sarah ( tell jane and sarah apart ) nuk# look so alike .\n",
      "425  the violence in bahrain is on a vastly smaller scale than that in libya ; and the west is locked into a military alliance with both nuk# to america 's fifth nuk# its royal family 's protector , saudi arabia .\n",
      "426  as a result , hundreds of tibetans from communities around the world have this week gathered for an unprecedented meeting in the indian hill town of nuk# . they are discussing what course the tibetan movement should take next .\n",
      "427  what did you say ? we got ta go .\n",
      "428  just right up against this\n",
      "429  i assure you we 're doing everything to find your little boy .\n",
      "430  simple procedures for the most part .\n",
      "431  in the capacity of theatre manager ,\n",
      "432  you made yourself less of a man when you spent all the money in your bank account .\n",
      "433  the main mistake is lifting the nuk# as nuk# breathing ' .\n",
      "434  i 'm totally on your wife 's side however ruthless she is !\n",
      "435  you know , this looking kind of bad for you -\n",
      "436  hmm . okay , then be prepared to stay here\n",
      "437  in the electronic deal , the fixation of efficacy of the contract , which is signed by incompetent person and person limited in disposing capacity , is ture of no 47 item of contract law .\n",
      "438  and every day women come in here\n",
      "439  i am sure you do charlie but um ...\n",
      "440  and the ecb feels on much safer ground when it lends to banks ( even if they then lend to governments ) than when it steps into the swamp of nuk# markets .\n",
      "441  should i quit this job ?\n",
      "442  i do n't think it 's ...\n",
      "443  blood mixed with saliva .\n",
      "444  go back to where the magic began .\n",
      "445  women have certainly performed better over the past decade than men .\n",
      "446  i 'll go pay . want something to eat ?\n",
      "447  erm ... i was looking for the remote .\n",
      "448  ba ( nuk# ) business administration ( nuk# campus )\n",
      "449  even if this was not a law ...\n",
      "450  the cherub looks at candy . cherub looks for his friends .\n",
      "451  but this is part of my precious task , as kitchen master\n",
      "452  but you 've got a reputation for recklessness\n",
      "453  what a bone - crushing open - ice hit to donnelly . and he 's down .\n",
      "454  and what do you think ? i think there 's no way we 're walking out of this one .\n",
      "455  kiwi sounds good .\n",
      "456  getting thumbnail approvals from clients is a common part of the illustration process . it is also common on large logo design projects and other projects as well .\n",
      "457  i told the mayor of san francisco\n",
      "458  i did n't kill that bitch !\n",
      "459  in the shop courses making ashtrays and nuk# .\n",
      "460  hey ! do n't you hear my boss ?\n",
      "461  that 's what torchwood does , you see .\n",
      "462  we 'll be late to swimming class .\n",
      "463  65.00 mile ( s ) from chapel hill\n",
      "464  the agency left in place its negative outlook , signalling further downgrades are possible .\n",
      "465  the strictures of budgetary nuk# '' instruments of torture '' , one british official calls nuk# only to the euro zone .\n",
      "466  upholding the nuk# ideological line , emancipating the mind , seeking truth from facts and keeping pace with the times are decisive factors for our party to retain its nuk# and enhance its creativity .\n",
      "467  in february , the rail minister was sacked and placed under investigation for possible corruption .\n",
      "468  make a right turn , at the first light turn left , the best western laramie inn & suites is on corner .\n",
      "469  a 55 percent reduction in stroke risk compared to aspirin is impressive , said dr. larry nuk# , professor of medicine at the new york university school of medicine and director of the heart rhythm center at nyu nuk# medical center .\n",
      "470  so i operate from the principle that\n",
      "471  but the guy who whacked your buddy ?\n",
      "472  oh , my god . it 's them . they 're here .\n",
      "473  no , it 's an indictment of the american education system .\n",
      "474  two dark clumps near the pelvis .\n",
      "475  economists generally prefer transparent to hidden subsidies .\n",
      "476  janice is a nice mature lady , she is our team leader and always motivating us to work hard .\n",
      "477  i mean , we 're talking centuries of truth mixed with fiction .\n",
      "478  5.5 block ( s ) from the running bear , full service , american , open for : breakfast , lunch\n",
      "479  about how i 'm giving her too much to do ?\n",
      "480  new banking licences may be awarded to politically connected industrialists .\n",
      "481  i do n't want the world to think ...\n",
      "482  i got that from your message .\n",
      "483  pumping quarters into him for hours . we got nothing .\n",
      "484  can you deny it 's a little frightening ?\n",
      "485  anti-inflammatory and antitumor activities of nuk# extracts of nuk# chinensis and nuk# nuk# .\n",
      "486  well , he could take a stone and ...\n",
      "487  hands a little smelly ?\n",
      "488  not like life , it 's simple , that makes the tango so great .\n",
      "489  stop ! it 's over .\n",
      "490  this is an amazing location for a fashion show .\n",
      "491  he was in - in bed , inhaling .\n",
      "492  what this guy apparently was trying to show us is we ca n't take what we saw for granted .\n",
      "493  my name is tong yong ; living in dan yang\n",
      "494  i 'm getting some o. r. time .\n",
      "495  i always believed that\n",
      "496  anybody see my catcher 's mitt ?\n",
      "497  we 're like those geeks that run with foxes .\n",
      "498  if countries differ in their approach , firms may simply invest more in those with more congenial rates .\n",
      "499  so , you just happened to have a bamboo dagger\n",
      "500  what else can you tell us about the symptoms ?\n",
      "501  and that thing that attacked us ?\n",
      "502  after a quarter mile at top speed .\n",
      "503  we 'll see if he gets up .\n",
      "504  from fall in the officers we all come to attention ,\n",
      "505  romney and his staff did consider a lot of female candidates for senior jobs in massachusetts , and they employed a lot of them , too .\n",
      "506  during a medical emergency ,\n",
      "507  they concluded that folic acid lowers the concentrations nuk# in the blood .\n",
      "508  i wan na see my mom ! where is she ?\n",
      "509  he gave it up .\n",
      "510  yes . the blood of the twelfth , my queen .\n",
      "511  um , really interesting . you know ?\n",
      "512  let 's question her .\n",
      "513  out of the car , frank .\n",
      "514  i was going to say i 'd like to be included .\n",
      "515  do you have the plans ?\n",
      "516  nearly half of the elderly in china have empty nest life , quality of life is relatively low .\n",
      "517  okay , it 's between your maple and your old - growth brazilian cherry .\n",
      "518  at the time , that was daring .\n",
      "519  so the evolution part of the course is set up basically with two introductory lectures .\n",
      "520  mom , you were right to settle .\n",
      "521  when you and i first got together , my mother said ,\n",
      "522  this research specimen comes from seven elementary schools of different seven counties , including 10 classes and nuk# sixth graders .\n",
      "523  give 'em a few more minutes .\n",
      "524  that part of my book is getting bigger and bigger .\n",
      "525  we also hear of works on the two natures `` against nuk# '' , and `` against the nuk# of alexander '' .\n",
      "526  i think some of them did .\n",
      "527  sir , i am an old man ,\n",
      "528  you know , he went all floppy ,\n",
      "529  what colleague ?\n",
      "530  miss ! miss stone .\n",
      "531  and i get these nuk# from roger and the `` higher powers ” ,\n",
      "532  note : this is part 1 of a nuk# series about nuk# ' .\n",
      "533  given enough warning , it should be possible to push a threatening boulder out of the way .\n",
      "534  if i feel you behind me , i will kill the kid .\n",
      "535  at risk . but\n",
      "536  nuk# xie ready to write the truth in black and white , this elimination of these insults .\n",
      "537  does the calculation of days in transit by the calculate time and cost application include the day the package is shipped on ?\n",
      "538  he must have sold the watch , my old man\n",
      "539  their praises rightly belong to the countless philippine rotarians who , from the airport to the hotel to the sessions to the fellowship , showered everyone with the famous filipino hospitality .\n",
      "540  she 's going to las vegas .\n",
      "541  do n't say the air force . the air force !\n",
      "542  he calls that meal the sacred and the profane .\n",
      "543  ah , yeah , had n't thought of that .\n",
      "544  we have practiced it every night for a week .\n",
      "545  instead , the nyu economist says growth will pick up , but not enough to produce raises for average american workers .\n",
      "546  hello , this is ned nuk# .\n",
      "547  yeah , i 'm not really loving the belt .\n",
      "548  believed to be the ringleader .\n",
      "549  a pink nike 's .\n",
      "550  but i truly believe that years from now\n",
      "551  no earmarks for any spending proposals .\n",
      "552  she wants you to spend the money , to have the fun that she ca n't have .\n",
      "553  george washington high school , about a mile and a half from our apartment , was one of the city 's largest and best public schools .\n",
      "554  and he gave me a guitar .\n",
      "555  where is he ? half what ?\n",
      "556  her name 's alicia too .\n",
      "557  he 's what ? - kill you ! he 's coming to kill you !\n",
      "558  it 's a gray area . are you sure about this ?\n",
      "559  i could care less what you need .\n",
      "560  and i will say nothing of your behavior .\n",
      "561  you know , we do n't still have to do that if you do n't want to .\n",
      "562  beautiful girl nuk# spoils the body giving way to you going over litre in winter boundless .\n",
      "563  we need to go get the new initiator .\n",
      "564  but she 'll come and see you again .\n",
      "565  wow . i thought guilt was a catholic thing .\n",
      "566  so i 'll just say i thought you did a great job with it .\n",
      "567  as the steel rebar nuk# ,\n",
      "568  three to five years ? yes .\n",
      "569  but stand the hell back .\n",
      "570  i do n't know if you 're aware of it , but we have a new manager in the branch .\n",
      "571  are you not glad you gave up\n",
      "572  her makeup contained titanium ,\n",
      "573  it 's too rugged .\n",
      "574  i 'm leaving . no , no , no , you ca n't .\n",
      "575  well , that 's not a term i would personally use .\n",
      "576  but impressive nuk# numbers did n't translate into a profitable business .\n",
      "577  she writes to me .\n",
      "578  i will fucking shoot you in the fucking eye .\n",
      "579  business travelers who are lucky enough to come to the area particularly enjoy the homey feel of our nuk# hotel , along with our convenient meeting and event facilities .\n",
      "580  after that , l could have been washed up\n",
      "581  so , here i am , seeing if maybe you can help me .\n",
      "582  if you can outside , l can inside .\n",
      "583  nuk# induced , training faulty movement nuk# ( ex . improper scapula nuk# rhythm ) so on .\n",
      "584  but i sure can feel blue\n",
      "585  well , you did n't say good - bye . they , uh ...\n",
      "586  she always nuk# up that embarrassing story .\n",
      "587  who disappeared without a trace six months ago .\n",
      "588  if he were to contact you ,\n",
      "589  they 're either incredibly brave or incredibly stupid .\n",
      "590  the battle i was trying to win against the world\n",
      "591  there was an age differences ,\n",
      "592  simulation research of distribution transportation and nuk# effects of black carbon aerosol in recent five spring seasons over east asia region .\n",
      "593  well , he told me . he was aware of every moment .\n",
      "594  how could they not sell tacos ?\n",
      "595  you 'll be receiving your kit in a few weeks\n",
      "596  but there are many older walls dating from the 7th century that served the common purpose of defending china from invasion from the north .\n",
      "597  oh ! she does like you , charlie .\n",
      "598  clever . do n't bother with that .\n",
      "599  etienne ? what ?\n",
      "600  which one ? - the nuk# . he 's crying !\n",
      "601  we did n't want to start a family feud .\n",
      "602  because what is the cause of death ?\n",
      "603  i thought you preferred blondes . such a droll little fellow .\n",
      "604  hey , claude , okay , listen to me .\n",
      "605  i mean , the man coached my dad , all my brothers .\n",
      "606  in this astronaut photograph , the ocean surface near the southeastern half of the island has a slight grey tinge due to nuk# , or light reflecting off the water surface back towards the international space station .\n",
      "607  article 5 a city , district or town shall establish a local federation of trade unions .\n",
      "608  get back , fiona . get back here . fiona .\n",
      "609  i have rights . you ca n't do this . i 'm a respectable businessman .\n",
      "610  create healthy nuk# food high in salt and sugar may taste appealing , but it will play havoc with your family 's health and moods .\n",
      "611  i just want you to know i 'm real grateful you 're here .\n",
      "612  it has been three years since she graduated from beijing university .\n",
      "613  what if you did ?\n",
      "614  it 's never a complete waste of time .\n",
      "615  because i 'm a citizen too , and i 'm a lot more comfortable\n",
      "616  put on something elegant and be quick about it .\n",
      "617  you mean stormed out like a little girl ?\n",
      "618  we 're still together .\n",
      "619  the two have written and composed more than 30 albums .\n",
      "620  cannes hosts the annual film festival .\n",
      "621  dan and julie are actors in a play . they are talking together backstage after a performance .\n",
      "622  and what 's this other brother of yours doing at home ?\n",
      "623  i 've been here for hours ... like we care .\n",
      "624  do you think it 's a fear of success ?\n",
      "625  with the advent of new nuk# and molecular techniques , our knowledge of the nuk# in multiple nuk# ( mm ) becomes more and more clear .\n",
      "626  you got all fired up about taking charge of your life ,\n",
      "627  it 's not mutual , so get to it .\n",
      "628  to announce my run for state assembly .\n",
      "629  he and three others nuk# blue state digital , where he is chief technology officer .\n",
      "630  microsoft ( msft ) sharepoint is traditional enterprise software used by organizations to manage documents and share content .\n",
      "631  the refusal to conform to the authority or doctrine of an established church ; nonconformity .\n",
      "632  the circumstances surrounding their capture remain unclear .\n",
      "633  in how they respond to the pain i put them through .\n",
      "634  but i still think we made the right decision to break up .\n",
      "635  so she 's decided to come and visit .\n",
      "636  so they - i 'm gon na go put these in water . okay .\n",
      "637  thanks , um ... will you be my village ? huh ?\n",
      "638  nuk# fracture ; nuk# fracture ; bone transplantation ; internal fixation .\n",
      "639  well , i 've had this terrific pain .\n",
      "640  we 're going to inject you with hepatitis a .\n",
      "641  i work on a lot of houses . i 've been over this with the cops .\n",
      "642  i love you , too , bro .\n",
      "643  i am not talking - that 's not -\n",
      "644  the dome , a masterpiece of the sicilian art in the normans ' period , preserves intact the original structure .\n",
      "645  let 's call it firm 1 sits at point 0\n",
      "646  nuk# . i did n't know what you like\n",
      "647  how nice . may i ask why ?\n",
      "648  i think we should assume a new surname .\n",
      "649  crucially , few national leaders , diplomats , or even sailors talk in arms race terms , and they certainly do not justify their efforts by the need to `` get ahead . '' on the contrary policymakers make every effort to avoid nuk# naming possible adversaries that they need to build against .\n",
      "650  one wants what the other has .\n",
      "651  ancient : the caves have been formed by flowing water over thousands of years\n",
      "652  i 'm loaded . see how cool it is !\n",
      "653  real children , somewhere , were nuk# intent on nuk# boats or building dams or collecting butterflies to as nuk# in jars ( as she and robin had done one summer ) .\n",
      "654  oh , god , i 'm sounding pretty ridiculous !\n",
      "655  in israel , nuk# have won the day .\n",
      "656  i got it figured out then\n",
      "657  over this whole blood on the ax thing . wrench .\n",
      "658  turn your head .- like this ?\n",
      "659  a guide to federal labeling nuk# for meat and poultry products .\n",
      "660  in his 2004 petition to the federal district court in savannah , davis presented recanted testimony , most of which involves witnesses who say police coercion caused them to wrongly implicate davis .\n",
      "661  he took her life savings .\n",
      "662  put the gun down , errol . put it down !\n",
      "663  but this time , she butted you off the stage .\n",
      "664  if we can translate this . what ?\n",
      "665  yeah , but i do n't like the whole business .\n",
      "666  come here , nuk# dig your heart out\n",
      "667  somehow , you 're mistaken .\n",
      "668  yeah , yeah , i thought it was one of your stupid rules .\n",
      "669  a bizarre accident has left two people dead ,\n",
      "670  state capitalism increasingly looks like the coming trend .\n",
      "671  seemingly small things like linking to the sites that are the subjects of articles is but one manifestation of this inertia .\n",
      "672  an idea formed in his mind .\n",
      "673  i was in my room and ...\n",
      "674  why could n't you have just taken the kid and left ?\n",
      "675  every man is a poet when he is in love .\n",
      "676  nuk# is there a lion heart who here sits\n",
      "677  their books are loved by young readers the world over .\n",
      "678  jeff : three , two , one , launch . short , short !\n",
      "679  he has dominated the revolution in personal computing and is a self-described `` hard-core nuk# , '' and a `` workaholic . ''\n",
      "680  part of the steering column , maybe . might have shattered inside the abdomen .\n",
      "681  sounded like he 's waiting for someone else to check the message , call him back .\n",
      "682  i still ca n't believe keeler would repeat those lies in a national debate .\n",
      "683  and you just talk to them .\n",
      "684  they do n't need that kind of mechanical strength .\n",
      "685  hey , wait a second . what ?\n",
      "686  puck , guys , spectacular !\n",
      "687  as soon as i get my dad 's stuff squared away .\n",
      "688  cause according to calculations it 's where we 're going .\n",
      "689  after sex , why do you always ask\n",
      "690  monsieur le mayor , i have a crime to declare !\n",
      "691  what is this about , your lady friend ? huh ?\n",
      "692  ballroom , for small gathering approximately 50 people\n",
      "693  they took me ; they changed me .\n",
      "694  honey , i 'm gon na excuse myself .\n",
      "695  he 's pushing me out of his life or maybe you 're nuk#\n",
      "696  that you could do even better .\n",
      "697  someone has to act as guard .\n",
      "698  to see our influence on the planet from a very different perspective .\n",
      "699  except he loves to be hated and maybe would hate to know every once in a while he 's loved .\n",
      "700  and you know what i 'm saying\n",
      "701  i been here for ten minutes .\n",
      "702  in britain some local-government bodies are reportedly moving their deposits from santander 's british bank , even though it is locally capitalised and supervised .\n",
      "703  mm , i 'd like to hear it from you .\n",
      "704  dillinger died , aged 31 , in a hail of bullets as he fled a chicago cinema .\n",
      "705  it was just a cigarette !\n",
      "706  we will finish it within sunday or monday .\n",
      "707  i 'll , er why do n't you ?\n",
      "708  would you like a drink ? - no , thank you .\n",
      "709  the chief can turf them out unceremoniously if he takes a dislike to them .\n",
      "710  and get off the road .\n",
      "711  while the middle east was exploding , the balkans was getting better .\n",
      "712  they appeared on an edition of nuk# 's arena .\n",
      "713  he was available to me morning , noon , and night .\n",
      "714  and what if he does n't show up ? the money will have to be returned .\n",
      "715  you 're that asshole .\n",
      "716  plans to make up most of the shortfall through overtime work were pushed back by october floods in thailand that affected about 100 suppliers .\n",
      "717  technical analysis involves studying graphs of stock movement over time as a way to predict future moves .\n",
      "718  a police boat , responding to the sos ,\n",
      "719  i 've been most anxious to tell you how grateful i am ,\n",
      "720  no , ca n't\n",
      "721  but the link that nuk# be properly recognised is that the counterparts to those `` unsustainable '' public-sector budget deficits are equally `` unsustainable '' nuk# surpluses .\n",
      "722  my resignation from the credit union had nothing to do\n",
      "723  the government is spending billions of dollars on new urban rail projects .\n",
      "724  the best western executive inn is on the right .\n",
      "725  they were sent to the clinics or designated hospitals around the expo site in time .\n",
      "726  before he met up with speedy ?\n",
      "727  which we call level - one and level - two parallel universes .\n",
      "728  i already make money setting up systems for people .\n",
      "729  greater nuk# engagement has increased societal contacts between americans and chinese including even some contacts with chinese democratic leaders independent journalists and civil society activists .\n",
      "730  services include technical support to nonprofit organizations that do not have the resources to afford such services .\n",
      "731  on the kidnapping video , the assailants - what is wrong with you ?\n",
      "732  you 're still coming with me . why would i do that ?\n",
      "733  this test , known as the data processing test , requires that the plaintiff seeking judicial review of agency action show two things : 1 an `` injury in fact '' ;\n",
      "734  i 'm just ... ... i 'm three houses down , across .\n",
      "735  but i think that this is the hotel that 's going to work best for you .\n",
      "736  he 's been on the phone for 15 minutes with the bank .\n",
      "737  i just made that .\n",
      "738  that 's what my brother believed and he was a fool .\n",
      "739  i told the neighbors to shut the water when they leave .\n",
      "740  food security will become a crucial issue .\n",
      "741  there are times when i look at people and i see nothing wo n't h liking .\n",
      "742  we started making plans right away .\n",
      "743  in order to stop their nuk# .\n",
      "744  clients start to worry about what their lives are going to be like after their companion leaves .\n",
      "745  wedding rings , toe rings , nose rings .\n",
      "746  get into bed and tell me . i ca n't .\n",
      "747  you see hood , he thinks there are two tunnels that are clear .\n",
      "748  and he 's prepared to write you regular nuk# as a nuk# ... chemist\n",
      "749  the way that it had always been done to her -\n",
      "750  i ca n't remember why i came\n",
      "751  morning , landlord !\n",
      "752  well , i wo n't back down\n",
      "753  what i am gon na do is after get this body , just twisting around quite lot .\n",
      "754  his friends point out that he has already resigned one plum role over the affair .\n",
      "755  do yourself a favor , sweetheart .\n",
      "756  so what are you gon na do next ?\n",
      "757  delete those unwanted apps and music from your iphone , and reorganize those messy home screens .\n",
      "758  but if such a magnificent place does exist , how has it eluded\n",
      "759  but in 20 years , quentin here will be working for me .\n",
      "760  because that 's what she likes to do , and l talked numbers .\n",
      "761  we vowed we will go to the final three together and we will .\n",
      "762  i had tasted the good life , and i wanted more .\n",
      "763  are you still to be a great orator in spite of yourself ?\n",
      "764  jobbik says it will not comment on private correspondence .\n",
      "765  eyes , look your last !\n",
      "766  once you 're in , they\n",
      "767  again . excuse me ?\n",
      "768  from what we can tell , it 's guarded by about a half dozen raiders .\n",
      "769  we foresee nothing but disaster .\n",
      "770  hey joe , take the lady 's coat . come on .\n",
      "771  i think she 's the brains of the operation .\n",
      "772  yeah , but i 'm practically his family . i 've known him since he was born .\n",
      "773  i had to go really bad .\n",
      "774  i was born doing ...\n",
      "775  of his generation here in newport .\n",
      "776  do you really want to listen to your insufferable boss another day ,\n",
      "777  our situation is nothing like the apollo 13 mission to the moon .\n",
      "778  as a religious movement engaged in social services , the army has , since its inception , made a goal of approaching sociological problems with christian concern .\n",
      "779  the doctor said , his situation is still critical .\n",
      "780  gene - splicing . you know , they 're breeding chickens ...\n",
      "781  get in . i have to fill up if we 're going to metropolis .\n",
      "782  i do n't know . scandalous women .\n",
      "783  it was so kind and thoughtful of you to make my first moments here ...\n",
      "784  damn lucky the place is still here , and no one was hurt .\n",
      "785  worse , that worry may be misplaced ; an increased chance is not a certainty .\n",
      "786  because unlike you , i never abandon my weak students\n",
      "787  and when all were into the very last ,\n",
      "788  get a doctor ! is anybody a doctor ?\n",
      "789  when they 're the age of sixteen ,\n",
      "790  it is n't the right time to say this .\n",
      "791  when i 'm in d. c. , i mostly facebook old girlfriends ...\n",
      "792  i got it for christmas last year . christmas ! we love christmas .\n",
      "793  it 'll most likely be an anonymous moment ...\n",
      "794  cause this is america , dude ...\n",
      "795  is he very bad , doctor ?\n",
      "796  we control the sky .\n",
      "797  i 'm not throwing my cat at everybody .\n",
      "798  but it just feels like somebody else 's memories .\n",
      "799  i 've been hit ! mom ?\n",
      "800  if it was `` herself , '' then you 'd have a fortune .\n",
      "801  so , the baby and the mom is on tape .\n",
      "802  you really do n't remember what happened between us in high school , do you ?\n",
      "803  i will prove to you i can sing .\n",
      "804  you know , i feel like he 's trying to cut me out .\n",
      "805  beijing had its hottest day yesterday since 1997 .\n",
      "806  chunhyang , hear me out !\n",
      "807  how 's that beer , beer - snob ? better than my own piss .\n",
      "808  hey , slacker . your computer 's back .\n",
      "809  sobbing buckets of tears .\n",
      "810  you guys think he knows ?\n",
      "811  doc ... you 've already done that .\n",
      "812  and employers are still being very cautious and choosy when it comes to hiring .\n",
      "813  hey ! you hold on pal ! now you made my friend , rachel , cry .\n",
      "814  he started calling me .\n",
      "815  no intention . great friends .\n",
      "816  but only if you promise to keep that detector thing at home .\n",
      "817  we were just having fun , doing something crazy .\n",
      "818  their account 's past due ?\n",
      "819  hey , do n't worry about it . it 's your kid . i get it .\n",
      "820  is just a guy who cuts my lawn .\n",
      "821  of course , and for the record , she was 19 , so -\n",
      "822  your weight . how much do you weigh ?\n",
      "823  nuk# , are you happy ?\n",
      "824  very handy nuk# to have .\n",
      "825  he ca n't do shit to stop this guy\n",
      "826  i 'm joe . what 's your name ? alfie .\n",
      "827  that lady at the bar keeps looking at me .\n",
      "828  translate this sentence into chinese please .\n",
      "829  or she may have eaten a richer diet than commoners did .\n",
      "830  i 'm not gon na lose you , too .\n",
      "831  i know that . that 's why i was getting married .\n",
      "832  you do not get drunk on hotel property .\n",
      "833  the nuk# registration requirement , which the chinese government has said must be completed by march 16 , `` will have a negative impact on user activity in the short term , '' charles chao , chief executive , told investors yesterday .\n",
      "834  then came the acquisitions of the new york post ; new york magazine ; the village voice , a nuk# new york weekly ; and the boston herald .\n",
      "835  the political uncertainty has hindered reconstruction efforts as haiti struggles to recover from an earthquake in january 2010 which killed more than nuk# people and leveled much of the capital .\n",
      "836  it is rumored that the weapon of the enemy has been found .\n",
      "837  claiming his good guy doll chucky was possessed by ... ... charles lee ray .\n",
      "838  and we just realized that somebody else\n",
      "839  i can be there for you , i can support you .\n",
      "840  like i had nowhere to go .\n",
      "841  hey , rach . no time , code two !\n",
      "842  the annulus is richly nuk# with nuk# , and further nuk# tends to be associated with more frequent and more intense episodes of pain .\n",
      "843  okay , well , it 's not .\n",
      "844  trying to see how i was going to meet him ,\n",
      "845  what are we doing in this stupid place ? i want to get going .\n",
      "846  there was a terrible battle at the airport , but the hostages ...\n",
      "847  that 's rather presumptuous ?\n",
      "848  often , they are unaware of what they are doing .\n",
      "849  what ? the police were waiting .\n",
      "850  i 'm afraid i 'll have to owe you .\n",
      "851  but i did n't want to get in trouble\n",
      "852  they 're not here . let 's ride .\n",
      "853  you can stop what you 're doing here .\n",
      "854  put the suitcase down -\n",
      "855  value is more likely to be preserved for nuk# and for the emirate itself .\n",
      "856  find her and get her here as soon as possible .\n",
      "857  well , it was a good place for us\n",
      "858  client : ok , please give me one casket of nuk# .\n",
      "859  with guy after guy after guy ...\n",
      "860  because he will want to be here .\n",
      "861  a poll last week found that 81 percent of ugandan mps surveyed opposed the plan .\n",
      "862  i have a job already . with dr nuk# .\n",
      "863  solar nuk# decreases towards the poles .\n",
      "864  from any of us .\n",
      "865  boston grabs control and dumps it out of their end .\n",
      "866  but the real story tonight .\n",
      "867  the structural nuk# , existing problems and prospect of fiber-optic smart materials are described .\n",
      "868  he denied ever saying it .\n",
      "869  there has to be a way that .\n",
      "870  earlier wednesday , nuk# nuk# cradled her newborn daughter and told a national tv audience she could n't describe the feeling .\n",
      "871  you put yourself and others at risk\n",
      "872  it will take several more months to pore through the volumes of documents .\n",
      "873  yeah , great . listen , flip a bitch at this iight , will you ?\n",
      "874  b why ? it 's been a nice week so far . lots of blue skies .\n",
      "875  once we prove the other side really exists !\n",
      "876  but maybe it 's better for him to stay out of the firing line for now .\n",
      "877  what ? they tested it on your brother first to see what would happen to you .\n",
      "878  shooting your picture to inspire your boyfriends .\n",
      "879  but the girl is safe .\n",
      "880  screw pride , l want my wife !\n",
      "881  yeah , sure . we 'll meet you outside .\n",
      "882  the lone spy who always works solo is a myth .\n",
      "883  just like any normal dad , you know , with their normal daughter .\n",
      "884  in all , the universe and nuk# combine in one .\n",
      "885  you seem to have escaped all of that .\n",
      "886  sooner or later , we 're going to end up on opposite sides .\n",
      "887  oh , i 'm so happy 'cause i 'm bus - bus - bus - busy .\n",
      "888  the operator expects the ship to be out of commission until the end of september .\n",
      "889  if they ambush us , you can still take 'em out from down there .\n",
      "890  but he was n't able to take our molecular assembler .\n",
      "891  when mark asked him why ,\n",
      "892  the only sample is all the way across town and extremely unstable .\n",
      "893  alright , coming at ya , special request .\n",
      "894  well , yeah , he could n't have just gotten up and walked out of there .\n",
      "895  it looks like the earth is surrounded by a hive of bees .\n",
      "896  six - party talks takes a dramatic turn ...\n",
      "897  you do n't know the meaning of the word .\n",
      "898  so , professionally packed and shipped but never intended to reach a destination .\n",
      "899  so i 'm asking you to do it again .\n",
      "900  as malawi strives to involve more men in family planning , the west african nation of mali is slowly but surely making progress .\n",
      "901  that guy must be flirting with gals ,\n",
      "902  so he left me there .\n",
      "903  if it means he confesses more murder , she may be persuaded .\n",
      "904  there are no virtual nuk# files with an immutable digital identity flitting around the system .\n",
      "905  i do n't know where my head was that day .\n",
      "906  oh , and do n't forget to take my ham\n",
      "907  you 're not going to her in a panic .\n",
      "908  mr. nuk# said negotiators would feel pressure to reach some kind of deal to present to the markets before trading in the region resumes next week after the eid holiday , because otherwise markets there might fall precipitously too .\n",
      "909  gran is the biggest snob i know\n",
      "910  god dammit , there is no deeper meaning in this book .\n",
      "911  you 're eradicating my power !\n",
      "912  but i like it . when can you start ?\n",
      "913  nuk# , over there . nuk# , you four go .\n",
      "914  i noticed yesterday that most times i only think of myself .\n",
      "915  status of conditions precedent and nuk# actions .\n",
      "916  why ? why ? what 's going on ? he 's a suspect in a murder case .\n",
      "917  that guy seems a little down .\n",
      "918  i think my uncle would prefer one of these magazines .\n",
      "919  and i think i should tell you you 're in for a terrible disappointment .\n",
      "920  while i was getting my nails done ?\n",
      "921  how many of your men have you gone in after ?\n",
      "922  something 's very wrong here , and yet a little bit right .\n",
      "923  you wanted to battle now it 's the last thing you wan na do\n",
      "924  who have not seen the light as i have .\n",
      "925  he 's been institutionalized for the past 9 years .\n",
      "926  it 's okay , will . this here is mr. clint eastwood visiting .\n",
      "927  he just does n't cut it .\n",
      "928  yeah . i was a big disappointment to him .\n",
      "929  hope it stays up there\n",
      "930  eg . teachers revealed to the press that they were going on strike .\n",
      "931  because i think everybody 's going to starve .\n",
      "932  you might need a bookmark .\n",
      "933  i know where you live , dad .\n",
      "934  my wild child .\n",
      "935  what 's going on , mr. mears ? do you know ? because i do n't .\n",
      "936  i sent that basket before we had our conversation .\n",
      "937  can not use the nuk# net number assigned by the remote server . check the event log .\n",
      "938  to pull that nuk# dam out and let the deficiency bill goes through ?\n",
      "939  i 'm sorry that when i quit working for you ,\n",
      "940  for not shooting me in the bank .\n",
      "941  for us , four hours ago .\n",
      "942  my son is gon na be all right\n",
      "943  i mean , you know , after the last time .\n",
      "944  he is at boarding school outside the city . it is safer there .\n",
      "945  however big i want to , and on a computer\n",
      "946  he explains that 2005 was the `` liftoff '' year for ipod sales , and for the installed base to grow beyond that would require a `` massive '' sales surge which is not happening .\n",
      "947  mr nuk# was on the cusp of reaching his running prime when war intervened .\n",
      "948  slaved to the communicator in my mask .\n",
      "949  a gathering .\n",
      "950  what if he hears us ? he 's also a very heavy drinker .\n",
      "951  i missed my friend harry .\n",
      "952  i recall the robert louis stevenson nuk# , `` we all have thoughts that would shame hell . ''\n",
      "953  as the needle of the compass .\n",
      "954  the best western nuk# quantico inn is down the road .\n",
      "955  you get your heart broke - treachery , hypocrisy ...\n",
      "956  they find the body yet ?\n",
      "957  a houseful of children is easier than dealing with one lazy husband .\n",
      "958  no matter how the dynasties change , the solemnity of jade products , the nuk# of jade ornaments and the spirituality of jade materials are always what people pursue .\n",
      "959  teach you after i learn it\n",
      "960  i was going to get you drinks . what ? fruit juice ?\n",
      "961  okay . it 's , it 's time that you know the truth .\n",
      "962  the right to call ourselves champions . yeah .\n",
      "963  it was in the box where he was killed , after all .\n",
      "964  yet he too may survive .\n",
      "965  i wrote a plan . i was up all night . i handled it .\n",
      "966  each ward shall elect a captain , or ward representative\n",
      "967  is waiting for my explanation of what you were doing\n",
      "968  smith now believes fletcher is beginning to show the same kind of form for united .\n",
      "969  they could only have been made from this\n",
      "970  her team proudly owns their role as `` gladiators in suits , '' loyally following their nuk# leader .\n",
      "971  but the best kinds of traditions are the ones that bring people together ,\n",
      "972  we 're gon na kill him with kindness .\n",
      "973  you know better than that . do n't be so stupid .\n",
      "974  as soon as i can afford the ring .\n",
      "975  issuance of nuk# , which comply with sharia law prohibitions on interest , grew to $ 43 billion in 2007 , though that is a sliver of the $ 50 nuk# global bond market .\n",
      "976  i know . so , once we come up with something -\n",
      "977  i want special branch .\n",
      "978  eventually passed out in this field .\n",
      "979  well , we got to go , you guys .\n",
      "980  i see two bodies .\n",
      "981  mr baker insists he deserves support : `` i am a british citizen , i believe in civic responsibility , '' he says .\n",
      "982  just doing my job , blackadder .\n",
      "983  with that painting , no matter what .\n",
      "984  i spent all week slaving over a hot nuk# stove , so you better enjoy them .\n",
      "985  at the oxford brain bank for autism researchers have just 20 brains to work nuk# where near enough to draw meaningful scientific conclusions from comparisons with healthy tissue .\n",
      "986  honey , no , you 're not gay . you are just confused .\n",
      "987  she 's up in seattle . with her mother .\n",
      "988  is this about his snoring ?\n",
      "989  what words ? use as directed .\n",
      "990  well , the two guys in the back had rifles ,\n",
      "991  and what do you know about rich people ?\n",
      "992  i just wan na be me .\n",
      "993  i mean , that 's how people relate .\n",
      "994  i 'm really afraid that this is how it 's always gon na be .\n",
      "995  i 've done more for you than her\n",
      "996  you 're punishing adi\n",
      "997  the only time in this city 's day when it is completely deserted\n",
      "998  it hurts , it hurts , it hurts , it hurts !\n",
      "999  the lord did not intend for these western\n"
     ]
    }
   ],
   "source": [
    "import h5py\n",
    "import json\n",
    "from collections import defaultdict\n",
    "\n",
    "en_voc = json.load(open('/data/xuwenshen/ai_challenge/data/train/train_words/en_voc.json'))\n",
    "\n",
    "h5 = h5py.File('/data/xuwenshen/ai_challenge/data/train/train_words/train.h5py', 'r')\n",
    "\n",
    "rvoc = defaultdict()\n",
    "for (k,v) in en_voc.items():\n",
    "    rvoc[v] = k\n",
    "rvoc = dict(rvoc)\n",
    "print (rvoc[0])\n",
    "\n",
    "f1 = h5['en_len']\n",
    "f2 = h5['en']\n",
    "f3 = h5['en_str']\n",
    "for i in range(1000):\n",
    "    text = ''\n",
    "    for j in range(len(f2[i])):\n",
    "        if rvoc[f2[i][j]] == 'pad#':\n",
    "            break\n",
    "        text += ' ' + rvoc[f2[i][j]]\n",
    "    print (i, text)\n",
    "#     print (i, f3[i].decode('utf8'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n",
      "\n",
      "<mteval>\n",
      "\n",
      "<srcset setid=\"setid\" srclang=\"en\" trglang=\"zh\">\n",
      "\n",
      "<doc sysid=\"sysid\" docid=\"docid\" genre=\"talk\">\n",
      "\n",
      "</doc>\n",
      "\n",
      "</srcset>\n",
      "\n",
      "</mteval>\n",
      "\n",
      "28\n",
      "6\n"
     ]
    }
   ],
   "source": [
    "# 解析valid en.sgm 和 test en.sgm\n",
    "import json\n",
    "import xml\n",
    "import re\n",
    "import nltk\n",
    "\n",
    "fin = open('/data/xuwenshen/ai_challenge/data/test/test.sgm').readlines()\n",
    "\n",
    "start = re.compile(r'\\d+\\\"> ')\n",
    "end = re.compile(r' </seg>')\n",
    "\n",
    "en = []\n",
    "for i in range(len(fin)):\n",
    "    \n",
    "    lst = start.split(fin[i])\n",
    "    if len(lst) != 2:\n",
    "        print (fin[i])\n",
    "        continue\n",
    "    \n",
    "    lst = end.split(lst[-1])\n",
    "    \n",
    "    if len(lst) != 2:\n",
    "        print (fin[i])\n",
    "        continue\n",
    "    \n",
    "    en.append(lst[0])\n",
    "\n",
    "    \n",
    "    \n",
    "en_tokens = []\n",
    "lens = []\n",
    "for i in range(len(en)):\n",
    "    tokens = nltk.word_tokenize(en[i])\n",
    "    en_tokens.append(tokens)\n",
    "    lens.append(len(tokens))\n",
    "    \n",
    "fout = open('/data/xuwenshen/ai_challenge/data/test/test_words/en.json', 'w')\n",
    "\n",
    "json.dump(en_tokens, fout)\n",
    "\n",
    "fout.close()\n",
    "print (max(lens))\n",
    "print (min(lens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n",
      "\n",
      "<mteval>\n",
      "\n",
      "<refset setid=\"setid\" srclang=\"en\" trglang=\"zh\" refid=\"ref0\">\n",
      "\n",
      "<doc sysid=\"sysid\" docid=\"docid\" genre=\"talk\">\n",
      "\n",
      "</doc>\n",
      "\n",
      "</refset>\n",
      "\n",
      "</mteval>\n",
      "\n",
      "35\n",
      "4\n"
     ]
    }
   ],
   "source": [
    "# 解析valid zh.sgm\n",
    "import json\n",
    "import xml\n",
    "import re\n",
    "import nltk\n",
    "import jieba\n",
    "\n",
    "\n",
    "fin = open('/data/xuwenshen/ai_challenge/data/valid/valid/valid.en-zh.zh.sgm').readlines()\n",
    "\n",
    "start = re.compile(r'\\d+\\\"> ')\n",
    "end = re.compile(r' </seg>')\n",
    "\n",
    "zh = []\n",
    "for i in range(len(fin)):\n",
    "    \n",
    "    lst = start.split(fin[i])\n",
    "    if len(lst) != 2:\n",
    "        print (fin[i])\n",
    "        continue\n",
    "    \n",
    "    lst = end.split(lst[-1])\n",
    "    \n",
    "    if len(lst) != 2:\n",
    "        print (fin[i])\n",
    "        continue\n",
    "    \n",
    "    zh.append(lst[0])\n",
    "\n",
    "zh_tokens = []\n",
    "lens = []\n",
    "\n",
    "for i in range(len(zh)):\n",
    "    \n",
    "    zh[i]=zh[i].rstrip('\\n')\n",
    "    seg_list = ' '.join(jieba.cut(zh[i])).split()\n",
    "    zh_tokens.append(seg_list)\n",
    "    lens.append(len(seg_list))\n",
    "    \n",
    "fout = open('/data/xuwenshen/ai_challenge/data/valid/valid_words/zh.json', 'w')\n",
    "\n",
    "json.dump(zh_tokens, fout)\n",
    "\n",
    "fout.close()\n",
    "print (max(lens))\n",
    "print (min(lens))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 1831/8000 [00:00<00:00, 18303.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8000\n",
      "8000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8000/8000 [00:00<00:00, 18501.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7936 8000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# 将en和zh都dump到同一个h5py文件，valid.h5py\n",
    "import json\n",
    "import h5py\n",
    "from collections import defaultdict\n",
    "from tqdm import *\n",
    "\n",
    "_en = json.load(open('/data/xuwenshen/ai_challenge/data/valid/valid_words/en.json'))\n",
    "en_voc = json.load(open('/data/xuwenshen/ai_challenge/data/train/train_words/en_voc.json'))\n",
    "\n",
    "_zh = json.load(open('/data/xuwenshen/ai_challenge/data/valid/valid_words/zh.json'))\n",
    "zh_voc = json.load(open('/data/xuwenshen/ai_challenge/data/train/train_words/zh_voc.json'))\n",
    "\n",
    "print (len(_en))\n",
    "print (len(_zh))\n",
    "\n",
    "f = h5py.File(\"/data/xuwenshen/ai_challenge/data/valid/valid_words/valid.h5py\", \"a\")\n",
    "\n",
    "\n",
    "en_set = f.create_dataset(\"en\", (len(_en), 60), dtype='i8')\n",
    "en_length_set = f.create_dataset('en_len', (len(_en), ), dtype='i8')\n",
    "\n",
    "\n",
    "label_set = f.create_dataset(\"label\", (len(_zh), 51), dtype='i8')\n",
    "zh_length_set = f.create_dataset('zh_len', (len(_zh), ), dtype='i8')\n",
    "gtruth_set = f.create_dataset('ground_truth', (len(_zh), 51), dtype='i8')\n",
    "\n",
    "zh_str_set = f.create_dataset('zh_str', (len(_zh), ), dtype='S400')\n",
    "en_str_set = f.create_dataset('en_str', (len(_en), ), dtype='S400')\n",
    "\n",
    "\n",
    "en = []\n",
    "en_len = []\n",
    "en_str = []\n",
    "\n",
    "labels = []\n",
    "zh_len = []\n",
    "gtruths = []\n",
    "zh_str = []\n",
    "\n",
    "for i in tqdm(range(len(_en))):\n",
    "    \n",
    "    #en data\n",
    "    tmp = [en_voc['pad#'] for j in range(60)]\n",
    "    if len(_en[i]) == 0 or len(_en[i]) > 60:\n",
    "        print ('en ', len(_en[i]))\n",
    "        continue\n",
    "    if len(_zh[i]) == 0 or len(_zh[i]) > 50:\n",
    "        print ('zh ', len(_zh[i]))\n",
    "        continue\n",
    "    \n",
    "    for j in range(len(_en[i])):\n",
    "        if _en[i][j].lower() not in en_voc:\n",
    "            tmp[j] = en_voc['nuk#']\n",
    "        else:\n",
    "            tmp[j] = en_voc[_en[i][j].lower()]\n",
    "    en.append(tmp)\n",
    "    en_len.append(len(_en[i]))\n",
    "    en_str.append(' '.join(_en[i][:min(400, len(_en[i]))]).encode('utf8') )\n",
    "    \n",
    "    # zh data\n",
    "    tlabel = [zh_voc['pad#'] for j in range(51)]\n",
    "    tgtruth = [zh_voc['pad#'] for j in range(51)]\n",
    "    for j in range(len(_zh[i])):\n",
    "        if _zh[i][j] not in zh_voc:\n",
    "            tlabel[j] = zh_voc['nuk#']\n",
    "            tgtruth[j+1] = zh_voc['nuk#']\n",
    "        else:\n",
    "            tlabel[j] = zh_voc[_zh[i][j]]\n",
    "            tgtruth[j+1] = zh_voc[_zh[i][j]]\n",
    "            \n",
    "    tlabel[len(_zh[i])] = zh_voc['eos#']\n",
    "    tgtruth[0] = zh_voc['go#']\n",
    "    \n",
    "    \n",
    "    labels.append(tlabel)\n",
    "    gtruths.append(tgtruth)\n",
    "    zh_len.append(len(_zh[i])+1)    \n",
    "    zh_str.append(''.join(_zh[i][:min(200, len(_zh[i]))]).encode('gb2312', 'ignore'))\n",
    "    \n",
    "    if len(en) == 256:\n",
    "        start = i+1-256\n",
    "        end = i+1\n",
    "        # en data\n",
    "        en_set[start : end] = en\n",
    "        en_length_set[start : end] = en_len\n",
    "        en_str_set[start : end] =  en_str\n",
    "        en = []\n",
    "        en_len = []\n",
    "        en_str = []\n",
    "        \n",
    "        #zh data\n",
    "        label_set[start : end] = labels\n",
    "        zh_length_set[start : end] = zh_len\n",
    "        gtruth_set[start : end] = gtruths      \n",
    "        zh_str_set[start:end] = zh_str\n",
    "        labels = []\n",
    "        zh_len = []\n",
    "        gtruths = []\n",
    "        zh_str = []\n",
    "\n",
    "        \n",
    "end = len(_en)\n",
    "start = end-len(en)\n",
    "print (start, end)\n",
    "en_set[start : end] = en\n",
    "en_length_set[start : end] = en_len\n",
    "en_str_set[start : end] =  en_str\n",
    "\n",
    "label_set[start : end] = labels\n",
    "zh_length_set[start : end] = zh_len\n",
    "gtruth_set[start : end] = gtruths\n",
    "zh_str_set[start : end] =  zh_str\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# dump test\n",
    "import json\n",
    "import h5py\n",
    "from collections import defaultdict\n",
    "from tqdm import *\n",
    "\n",
    "_en = json.load(open('/data/xuwenshen/ai_challenge/data/test/test_words/en.json'))\n",
    "en_voc = json.load(open('/data/xuwenshen/ai_challenge/data/train/train_words/en_voc.json'))\n",
    "\n",
    "print (len(_en))\n",
    "\n",
    "f = h5py.File(\"/data/xuwenshen/ai_challenge/data/test/test_words/test.h5py\", \"a\")\n",
    "\n",
    "en_set = f.create_dataset(\"en\", (len(_en), 60), dtype='i8')\n",
    "en_length_set = f.create_dataset('en_len', (len(_en), ), dtype='i8')\n",
    "\n",
    "\n",
    "en = []\n",
    "en_len = []\n",
    "\n",
    "for i in tqdm(range(len(_en))):\n",
    "    \n",
    "    #en data\n",
    "    tmp = [en_voc['pad#'] for j in range(60)]\n",
    "    if len(_en[i]) == 0 or len(_en[i]) > 60:\n",
    "        print ('en ', len(_en[i]))\n",
    "        _en[i] = _en[i][:min(60, len(_en[i]))]\n",
    "    \n",
    "    for j in range(len(_en[i])):\n",
    "        if _en[i][j].lower() not in en_voc:\n",
    "            tmp[j] = en_voc['nuk#']\n",
    "        else:\n",
    "            tmp[j] = en_voc[_en[i][j].lower()]\n",
    "    en.append(tmp)\n",
    "    en_len.append(len(_en[i]))\n",
    "    \n",
    "    \n",
    "       \n",
    "    if len(en) == 256:\n",
    "        start = i+1-256\n",
    "        end = i+1\n",
    "        # en data\n",
    "        en_set[start : end] = en\n",
    "        en_length_set[start : end] = en_len\n",
    "        en = []\n",
    "        en_len = []\n",
    "\n",
    "        \n",
    "end = len(_en)\n",
    "start = end-len(en)\n",
    "print (start, end)\n",
    "en_set[start : end] = en\n",
    "en_length_set[start : end] = en_len\n",
    "\n",
    "f.close()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}