TF-IDF.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TF-IDF\n",
    "***\n",
    "***\n",
    "Time: 2020-09-16<br>\n",
    "Author: dsy\n",
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus=[\"我 来到 北京 清华大学\",#第一类文本切词后的结果，词之间以空格隔开\n",
    "\"他 来到 了 网易 杭研 大厦\",#第二类文本的切词结果\n",
    "\"小明 硕士 毕业 与 中国 科学院\",#第三类文本的切词结果\n",
    "\"我 爱 北京 天安门\"]#第四类文本的切词结果"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`TfidfTransformer and CountVectorizer`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer=CountVectorizer()#该类会将文本中的词语转换为词频矩阵，矩阵元素a[i][j] 表示j词在i类文本下的词频\n",
    "transformer=TfidfTransformer()#该类会统计每个词语的tf-idf权值\n",
    "tfidf=transformer.fit_transform(vectorizer.fit_transform(corpus))#第一个fit_transform是计算tf-idf，第二个fit_transform是将文本转为词频矩阵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['中国', '北京', '大厦', '天安门', '小明', '来到', '杭研', '毕业', '清华大学', '硕士', '科学院', '网易']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word=vectorizer.get_feature_names()#获取词袋模型中的所有词语\n",
    "word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.52640543, 0.        , 0.        , 0.        ,\n",
       "        0.52640543, 0.        , 0.        , 0.66767854, 0.        ,\n",
       "        0.        , 0.        ],\n",
       "       [0.        , 0.        , 0.52547275, 0.        , 0.        ,\n",
       "        0.41428875, 0.52547275, 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.52547275],\n",
       "       [0.4472136 , 0.        , 0.        , 0.        , 0.4472136 ,\n",
       "        0.        , 0.        , 0.4472136 , 0.        , 0.4472136 ,\n",
       "        0.4472136 , 0.        ],\n",
       "       [0.        , 0.6191303 , 0.        , 0.78528828, 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        ]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weight=tfidf.toarray()#将tf-idf矩阵抽取出来，元素a[i][j]表示j词在i类文本中的tf-idf权重\n",
    "weight"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`TfidfVectorizer`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer()\n",
    "X = vectorizer.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['中国', '北京', '大厦', '天安门', '小明', '来到', '杭研', '毕业', '清华大学', '硕士', '科学院', '网易']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.52640543, 0.        , 0.        , 0.        ,\n",
       "        0.52640543, 0.        , 0.        , 0.66767854, 0.        ,\n",
       "        0.        , 0.        ],\n",
       "       [0.        , 0.        , 0.52547275, 0.        , 0.        ,\n",
       "        0.41428875, 0.52547275, 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.52547275],\n",
       "       [0.4472136 , 0.        , 0.        , 0.        , 0.4472136 ,\n",
       "        0.        , 0.        , 0.4472136 , 0.        , 0.4472136 ,\n",
       "        0.4472136 , 0.        ],\n",
       "       [0.        , 0.6191303 , 0.        , 0.78528828, 0.        ,\n",
       "        0.        , 0.        , 0.        , 0.        , 0.        ,\n",
       "        0.        , 0.        ]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.toarray()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}