make_pretrained_embedding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-27T02:46:34.769056Z",
     "start_time": "2019-04-27T02:46:34.607992Z"
    }
   },
   "outputs": [],
   "source": [
    "import _pickle as pickle\n",
    "import numpy as np\n",
    "import torch\n",
    "from random import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-27T02:46:34.866765Z",
     "start_time": "2019-04-27T02:46:34.858647Z"
    }
   },
   "outputs": [],
   "source": [
    "dic_src = pickle.load(open(\"./data/id2word_src.dat\", \"rb\"))\n",
    "dic_tgt = pickle.load(open(\"./data/id2word_tgt.dat\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-27T02:46:35.162039Z",
     "start_time": "2019-04-27T02:46:35.155861Z"
    }
   },
   "outputs": [],
   "source": [
    "vector_dict = dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-27T02:46:40.098137Z",
     "start_time": "2019-04-27T02:46:40.096215Z"
    }
   },
   "outputs": [],
   "source": [
    "dict_size_src = 40000\n",
    "dict_size_tgt = 10000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# model_cnndm_lem.vec 128维预训练词嵌入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-24T11:46:15.566537Z",
     "start_time": "2019-04-24T11:46:10.759562Z"
    }
   },
   "outputs": [],
   "source": [
    "count = 0\n",
    "with open(\"./fastText/model_cnndm_128.vec\", \"r\") as f:\n",
    "    for line in f:\n",
    "        count += 1\n",
    "        if count == 1:\n",
    "            continue\n",
    "        vector = line.split(\" \")\n",
    "        key = vector[0]\n",
    "        vector = [float(num) for num in vector[1:-1]]\n",
    "        vector_dict[key] = vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# model_cnn_256.vec 256维预训练词嵌入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-24T11:48:48.689813Z",
     "start_time": "2019-04-24T11:48:40.676497Z"
    }
   },
   "outputs": [],
   "source": [
    "count = 0\n",
    "with open(\"./fastText/model_cnndm_256.vec\", \"r\") as f:\n",
    "    for line in f:\n",
    "        count += 1\n",
    "        if count == 1:\n",
    "            continue\n",
    "        vector = line.split(\" \")\n",
    "        key = vector[0]\n",
    "        vector = [float(num) for num in vector[1:-1]]\n",
    "        vector_dict[key] = vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-15T00:53:32.678819Z",
     "start_time": "2019-03-15T00:53:32.672638Z"
    }
   },
   "source": [
    "# model_cnndm_512.vec 512维预训练词嵌入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-27T02:47:03.034899Z",
     "start_time": "2019-04-27T02:46:45.340345Z"
    }
   },
   "outputs": [],
   "source": [
    "count = 0\n",
    "with open(\"./fastText/model_cnndm_512.vec\", \"r\") as f:\n",
    "    for line in f:\n",
    "        count += 1\n",
    "        if count == 1:\n",
    "            continue\n",
    "        vector = line.split(\" \")\n",
    "        key = vector[0]\n",
    "        vector = [float(num) for num in vector[1:-1]]\n",
    "        vector_dict[key] = vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 选择语料中单词的向量，构造embedding矩阵\n",
    "- Encoder和Decoder分别用不同的大小的嵌入矩阵\n",
    "- 手动更改保存的dat文件中的维度名"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-24T11:49:48.095598Z",
     "start_time": "2019-04-24T11:49:48.093617Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "223469\n"
     ]
    }
   ],
   "source": [
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-24T11:49:48.508710Z",
     "start_time": "2019-04-24T11:49:48.503186Z"
    }
   },
   "outputs": [],
   "source": [
    "special_token_random_vector = vector_dict['.']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-24T11:49:50.444126Z",
     "start_time": "2019-04-24T11:49:49.031250Z"
    }
   },
   "outputs": [],
   "source": [
    "embedding = [special_token_random_vector for _ in range(4)]\n",
    "for i in range(4, dict_size_src):\n",
    "    embedding.append(vector_dict[dic_src[i]])\n",
    "embedding = np.asarray(embedding)\n",
    "pickle.dump(\n",
    "    torch.from_numpy(embedding),\n",
    "    open(\"./data/pretrained_embedding_512_src.dat\", \"wb\"), True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-24T11:49:51.505806Z",
     "start_time": "2019-04-24T11:49:51.148630Z"
    }
   },
   "outputs": [],
   "source": [
    "embedding = [special_token_random_vector for _ in range(4)]\n",
    "for i in range(4, dict_size_tgt):\n",
    "    embedding.append(vector_dict[dic_tgt[i]])\n",
    "embedding = np.asarray(embedding)\n",
    "pickle.dump(\n",
    "    torch.from_numpy(embedding),\n",
    "    open(\"./data/pretrained_embedding_512_tgt.dat\", \"wb\"), True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}