2 Data Collection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Python IO与档案处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f = open('tmp.txt', 'w')\n",
    "f.write('hello world')\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('tmp.txt', 'w') as f:\n",
    "    f.write('hello\\nworld')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello\n",
      "world\n"
     ]
    }
   ],
   "source": [
    "with open('tmp.txt', 'r') as f:\n",
    "    print(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello\n",
      "world\n"
     ]
    }
   ],
   "source": [
    "with open('tmp.txt', 'r') as f:\n",
    "    for line in f.readlines():\n",
    "        print(line.strip())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 处理CSV, Excel格式资料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "﻿指标,2015年,2014年,2013年,2012年,2011年,2010年,2009年,2008年,2007年,2006年\n",
      "\n",
      "年末总人口(万人),137462,136782,136072,135404,134735,134091,133450,132802,132129,131448\n",
      "\n",
      "男性人口(万人),70414,70079,69728,69395,69068,68748,68647,68357,68048,67728\n",
      "\n",
      "女性人口(万人),67048,66703,66344,66009,65667,65343,64803,64445,64081,63720\n",
      "\n",
      "城镇人口(万人),77116,74916,73111,71182,69079,66978,64512,62403,60633,58288\n",
      "\n",
      "乡村人口(万人),60346,61866,62961,64222,65656,67113,68938,70399,71496,73160\n"
     ]
    }
   ],
   "source": [
    "with open('Data\\Population.csv', 'r', encoding='UTF-8') as f:\n",
    "    for line in f.readlines():\n",
    "        print(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!pip install pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>指标</th>\n",
       "      <th>2015年</th>\n",
       "      <th>2014年</th>\n",
       "      <th>2013年</th>\n",
       "      <th>2012年</th>\n",
       "      <th>2011年</th>\n",
       "      <th>2010年</th>\n",
       "      <th>2009年</th>\n",
       "      <th>2008年</th>\n",
       "      <th>2007年</th>\n",
       "      <th>2006年</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>年末总人口(万人)</td>\n",
       "      <td>137462</td>\n",
       "      <td>136782</td>\n",
       "      <td>136072</td>\n",
       "      <td>135404</td>\n",
       "      <td>134735</td>\n",
       "      <td>134091</td>\n",
       "      <td>133450</td>\n",
       "      <td>132802</td>\n",
       "      <td>132129</td>\n",
       "      <td>131448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>男性人口(万人)</td>\n",
       "      <td>70414</td>\n",
       "      <td>70079</td>\n",
       "      <td>69728</td>\n",
       "      <td>69395</td>\n",
       "      <td>69068</td>\n",
       "      <td>68748</td>\n",
       "      <td>68647</td>\n",
       "      <td>68357</td>\n",
       "      <td>68048</td>\n",
       "      <td>67728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>女性人口(万人)</td>\n",
       "      <td>67048</td>\n",
       "      <td>66703</td>\n",
       "      <td>66344</td>\n",
       "      <td>66009</td>\n",
       "      <td>65667</td>\n",
       "      <td>65343</td>\n",
       "      <td>64803</td>\n",
       "      <td>64445</td>\n",
       "      <td>64081</td>\n",
       "      <td>63720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>城镇人口(万人)</td>\n",
       "      <td>77116</td>\n",
       "      <td>74916</td>\n",
       "      <td>73111</td>\n",
       "      <td>71182</td>\n",
       "      <td>69079</td>\n",
       "      <td>66978</td>\n",
       "      <td>64512</td>\n",
       "      <td>62403</td>\n",
       "      <td>60633</td>\n",
       "      <td>58288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>乡村人口(万人)</td>\n",
       "      <td>60346</td>\n",
       "      <td>61866</td>\n",
       "      <td>62961</td>\n",
       "      <td>64222</td>\n",
       "      <td>65656</td>\n",
       "      <td>67113</td>\n",
       "      <td>68938</td>\n",
       "      <td>70399</td>\n",
       "      <td>71496</td>\n",
       "      <td>73160</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          指标   2015年   2014年   2013年   2012年   2011年   2010年   2009年   2008年  \\\n",
       "0  年末总人口(万人)  137462  136782  136072  135404  134735  134091  133450  132802   \n",
       "1   男性人口(万人)   70414   70079   69728   69395   69068   68748   68647   68357   \n",
       "2   女性人口(万人)   67048   66703   66344   66009   65667   65343   64803   64445   \n",
       "3   城镇人口(万人)   77116   74916   73111   71182   69079   66978   64512   62403   \n",
       "4   乡村人口(万人)   60346   61866   62961   64222   65656   67113   68938   70399   \n",
       "\n",
       "    2007年   2006年  \n",
       "0  132129  131448  \n",
       "1   68048   67728  \n",
       "2   64081   63720  \n",
       "3   60633   58288  \n",
       "4   71496   73160  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas\n",
    "df = pandas.read_csv('Data/Population.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>指标</th>\n",
       "      <th>2015年</th>\n",
       "      <th>2014年</th>\n",
       "      <th>2013年</th>\n",
       "      <th>2012年</th>\n",
       "      <th>2011年</th>\n",
       "      <th>2010年</th>\n",
       "      <th>2009年</th>\n",
       "      <th>2008年</th>\n",
       "      <th>2007年</th>\n",
       "      <th>2006年</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>国民总收入(亿元)</td>\n",
       "      <td>686181.5</td>\n",
       "      <td>644791.1</td>\n",
       "      <td>590422.4</td>\n",
       "      <td>539116.5</td>\n",
       "      <td>484753.2</td>\n",
       "      <td>411265.2</td>\n",
       "      <td>348498.5</td>\n",
       "      <td>321500.5</td>\n",
       "      <td>270844.0</td>\n",
       "      <td>219028.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>国内生产总值(亿元)</td>\n",
       "      <td>689052.1</td>\n",
       "      <td>643974.0</td>\n",
       "      <td>595244.4</td>\n",
       "      <td>540367.4</td>\n",
       "      <td>489300.6</td>\n",
       "      <td>413030.3</td>\n",
       "      <td>349081.4</td>\n",
       "      <td>319515.5</td>\n",
       "      <td>270232.3</td>\n",
       "      <td>219438.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>第一产业增加值(亿元)</td>\n",
       "      <td>60862.1</td>\n",
       "      <td>58343.5</td>\n",
       "      <td>55329.1</td>\n",
       "      <td>50902.3</td>\n",
       "      <td>46163.1</td>\n",
       "      <td>39362.6</td>\n",
       "      <td>34161.8</td>\n",
       "      <td>32753.2</td>\n",
       "      <td>27788.0</td>\n",
       "      <td>23317.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>第二产业增加值(亿元)</td>\n",
       "      <td>282040.3</td>\n",
       "      <td>277571.8</td>\n",
       "      <td>261956.1</td>\n",
       "      <td>244643.3</td>\n",
       "      <td>227038.8</td>\n",
       "      <td>191629.8</td>\n",
       "      <td>160171.7</td>\n",
       "      <td>149956.6</td>\n",
       "      <td>126633.6</td>\n",
       "      <td>104361.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>第三产业增加值(亿元)</td>\n",
       "      <td>346149.7</td>\n",
       "      <td>308058.6</td>\n",
       "      <td>277959.3</td>\n",
       "      <td>244821.9</td>\n",
       "      <td>216098.6</td>\n",
       "      <td>182038.0</td>\n",
       "      <td>154747.9</td>\n",
       "      <td>136805.8</td>\n",
       "      <td>115810.7</td>\n",
       "      <td>91759.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>人均国内生产总值(元)</td>\n",
       "      <td>50251.0</td>\n",
       "      <td>47203.0</td>\n",
       "      <td>43852.0</td>\n",
       "      <td>40007.0</td>\n",
       "      <td>36403.0</td>\n",
       "      <td>30876.0</td>\n",
       "      <td>26222.0</td>\n",
       "      <td>24121.0</td>\n",
       "      <td>20505.0</td>\n",
       "      <td>16738.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            指标     2015年     2014年     2013年     2012年     2011年     2010年  \\\n",
       "0    国民总收入(亿元)  686181.5  644791.1  590422.4  539116.5  484753.2  411265.2   \n",
       "1   国内生产总值(亿元)  689052.1  643974.0  595244.4  540367.4  489300.6  413030.3   \n",
       "2  第一产业增加值(亿元)   60862.1   58343.5   55329.1   50902.3   46163.1   39362.6   \n",
       "3  第二产业增加值(亿元)  282040.3  277571.8  261956.1  244643.3  227038.8  191629.8   \n",
       "4  第三产业增加值(亿元)  346149.7  308058.6  277959.3  244821.9  216098.6  182038.0   \n",
       "5  人均国内生产总值(元)   50251.0   47203.0   43852.0   40007.0   36403.0   30876.0   \n",
       "\n",
       "      2009年     2008年     2007年     2006年  \n",
       "0  348498.5  321500.5  270844.0  219028.5  \n",
       "1  349081.4  319515.5  270232.3  219438.5  \n",
       "2   34161.8   32753.2   27788.0   23317.0  \n",
       "3  160171.7  149956.6  126633.6  104361.8  \n",
       "4  154747.9  136805.8  115810.7   91759.7  \n",
       "5   26222.0   24121.0   20505.0   16738.0  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas\n",
    "df = pandas.read_excel('Data/GDP.xls')\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 处理JSON, XML格式资料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('Data/jd.json', 'r') as f:\n",
    "    jd = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "华为官方旗舰店 华为官方旗舰店\n",
      "牧申手机旗舰店 官方正品；售后无忧；良心经营！\n",
      "新松联手机旗舰店 正品行货  全国联保 全国包邮\n",
      "爱优创杰手机专营店 爱优创杰只售正品行货，支持官方渠道验货\n",
      "佳沪数码手机旗舰店 官方正品丨货票同行丨快速物流丨无忧售后\n",
      "中恒国信手机旗舰店 正品保障·全国联保·正规发票\n",
      "众盛手机专营店 诚信，务实，以顾客为本，服务造就未来。\n",
      "环球名品海外专营店 香港倉儲，境外直郵，誠信經營，正品保證\n",
      "培新数码手机专营店 全新国行！全国联保！\n",
      "GIT通讯数码旗舰店 GIT全球购，港行正品，品质保障。\n",
      "麦考利手机专营店 全国联保·正品保障·正规发票\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "dic = json.loads(jd)\n",
    "for shop in dic:\n",
    "    print(shop.get('shop_name'), shop.get('shop_brief'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#json.dumps(dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_category</th>\n",
       "      <th>cid1</th>\n",
       "      <th>icon</th>\n",
       "      <th>industry_effective_score</th>\n",
       "      <th>industry_service_score</th>\n",
       "      <th>industry_total_score</th>\n",
       "      <th>main_brand</th>\n",
       "      <th>pic_url</th>\n",
       "      <th>shop_brief</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>...</th>\n",
       "      <th>shop_type</th>\n",
       "      <th>summary</th>\n",
       "      <th>txt_weight</th>\n",
       "      <th>vender_effective_score</th>\n",
       "      <th>vender_id</th>\n",
       "      <th>vender_service_score</th>\n",
       "      <th>vender_total_score</th>\n",
       "      <th>vender_type</th>\n",
       "      <th>vender_ware_score</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>车载配件 电脑、办公 路由器 手机配件 手机保护套 电脑整机 蓝牙耳机 交换机 手机存储卡 ...</td>\n",
       "      <td>9987</td>\n",
       "      <td>0</td>\n",
       "      <td>963</td>\n",
       "      <td>971</td>\n",
       "      <td>974</td>\n",
       "      <td>华为（HUAWEI）</td>\n",
       "      <td></td>\n",
       "      <td>华为官方旗舰店</td>\n",
       "      <td>1000004259</td>\n",
       "      <td>...</td>\n",
       "      <td>28</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000004259</td>\n",
       "      <td>1000</td>\n",
       "      <td>1000</td>\n",
       "      <td>1</td>\n",
       "      <td>1000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>手机通讯 手机配件 创意配件 手机</td>\n",
       "      <td>9987</td>\n",
       "      <td>0</td>\n",
       "      <td>954</td>\n",
       "      <td>954</td>\n",
       "      <td>954</td>\n",
       "      <td>摩托罗拉（Motorola） 酷派（Coolpad） HALOVE SHARP 纽曼（New...</td>\n",
       "      <td></td>\n",
       "      <td>官方正品；售后无忧；良心经营！</td>\n",
       "      <td>178287</td>\n",
       "      <td>...</td>\n",
       "      <td>11</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>936</td>\n",
       "      <td>174907</td>\n",
       "      <td>943</td>\n",
       "      <td>944</td>\n",
       "      <td>0</td>\n",
       "      <td>948</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>手机通讯 手机配件 创意配件 手机</td>\n",
       "      <td>9987</td>\n",
       "      <td>0</td>\n",
       "      <td>954</td>\n",
       "      <td>954</td>\n",
       "      <td>954</td>\n",
       "      <td>摩托罗拉（Motorola） 酷派（Coolpad） SHARP HALOVE 诺基亚（NO...</td>\n",
       "      <td></td>\n",
       "      <td>正品行货  全国联保 全国包邮</td>\n",
       "      <td>179483</td>\n",
       "      <td>...</td>\n",
       "      <td>11</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>949</td>\n",
       "      <td>185259</td>\n",
       "      <td>953</td>\n",
       "      <td>955</td>\n",
       "      <td>0</td>\n",
       "      <td>954</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>手机通讯 手机配件 创意配件 手机</td>\n",
       "      <td>9987</td>\n",
       "      <td>0</td>\n",
       "      <td>954</td>\n",
       "      <td>954</td>\n",
       "      <td>954</td>\n",
       "      <td>酷派（Coolpad） HALOVE 诺基亚（NOKIA） 三星（SAMSUNG） ivvi...</td>\n",
       "      <td></td>\n",
       "      <td>爱优创杰只售正品行货，支持官方渠道验货</td>\n",
       "      <td>178637</td>\n",
       "      <td>...</td>\n",
       "      <td>24</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>935</td>\n",
       "      <td>185329</td>\n",
       "      <td>942</td>\n",
       "      <td>945</td>\n",
       "      <td>0</td>\n",
       "      <td>948</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>手机通讯 对讲机 手机配件 创意配件 特殊商品 手机</td>\n",
       "      <td>9987</td>\n",
       "      <td>0</td>\n",
       "      <td>954</td>\n",
       "      <td>954</td>\n",
       "      <td>954</td>\n",
       "      <td>酷派（Coolpad） 传奇（SAGA） HALOVE 华星 三星（SAMSUNG） TCL...</td>\n",
       "      <td>http://img11.360buyimg.com/cms/jfs/t1591/100/3...</td>\n",
       "      <td>官方正品丨货票同行丨快速物流丨无忧售后</td>\n",
       "      <td>113406</td>\n",
       "      <td>...</td>\n",
       "      <td>11</td>\n",
       "      <td>官方正品保障，货票同行无忧。</td>\n",
       "      <td>0</td>\n",
       "      <td>943</td>\n",
       "      <td>116240</td>\n",
       "      <td>946</td>\n",
       "      <td>946</td>\n",
       "      <td>0</td>\n",
       "      <td>946</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 23 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   business_category  cid1  icon  \\\n",
       "0  车载配件 电脑、办公 路由器 手机配件 手机保护套 电脑整机 蓝牙耳机 交换机 手机存储卡 ...  9987     0   \n",
       "1                                  手机通讯 手机配件 创意配件 手机  9987     0   \n",
       "2                                  手机通讯 手机配件 创意配件 手机  9987     0   \n",
       "3                                  手机通讯 手机配件 创意配件 手机  9987     0   \n",
       "4                         手机通讯 对讲机 手机配件 创意配件 特殊商品 手机  9987     0   \n",
       "\n",
       "   industry_effective_score  industry_service_score  industry_total_score  \\\n",
       "0                       963                     971                   974   \n",
       "1                       954                     954                   954   \n",
       "2                       954                     954                   954   \n",
       "3                       954                     954                   954   \n",
       "4                       954                     954                   954   \n",
       "\n",
       "                                          main_brand  \\\n",
       "0                                         华为（HUAWEI）   \n",
       "1  摩托罗拉（Motorola） 酷派（Coolpad） HALOVE SHARP 纽曼（New...   \n",
       "2  摩托罗拉（Motorola） 酷派（Coolpad） SHARP HALOVE 诺基亚（NO...   \n",
       "3  酷派（Coolpad） HALOVE 诺基亚（NOKIA） 三星（SAMSUNG） ivvi...   \n",
       "4  酷派（Coolpad） 传奇（SAGA） HALOVE 华星 三星（SAMSUNG） TCL...   \n",
       "\n",
       "                                             pic_url           shop_brief  \\\n",
       "0                                                                 华为官方旗舰店   \n",
       "1                                                         官方正品；售后无忧；良心经营！   \n",
       "2                                                         正品行货  全国联保 全国包邮   \n",
       "3                                                     爱优创杰只售正品行货，支持官方渠道验货   \n",
       "4  http://img11.360buyimg.com/cms/jfs/t1591/100/3...  官方正品丨货票同行丨快速物流丨无忧售后   \n",
       "\n",
       "      shop_id   ...   shop_type         summary  txt_weight  \\\n",
       "0  1000004259   ...          28                           0   \n",
       "1      178287   ...          11                           0   \n",
       "2      179483   ...          11                           0   \n",
       "3      178637   ...          24                           0   \n",
       "4      113406   ...          11  官方正品保障，货票同行无忧。           0   \n",
       "\n",
       "   vender_effective_score   vender_id  vender_service_score  \\\n",
       "0                    1000  1000004259                  1000   \n",
       "1                     936      174907                   943   \n",
       "2                     949      185259                   953   \n",
       "3                     935      185329                   942   \n",
       "4                     943      116240                   946   \n",
       "\n",
       "   vender_total_score  vender_type  vender_ware_score  weight  \n",
       "0                1000            1               1000       0  \n",
       "1                 944            0                948       0  \n",
       "2                 955            0                954       0  \n",
       "3                 945            0                948       0  \n",
       "4                 946            0                946       0  \n",
       "\n",
       "[5 rows x 23 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas\n",
    "df = pandas.read_json('Data/jd.json')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import xml.etree.ElementTree as ET\n",
    "tree = ET.parse('Data\\weather.xml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Element 'china' at 0x00000000047B5638>"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "root = tree.getroot()\n",
    "root"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "哈尔滨 7\n",
      "长春 9\n",
      "沈阳 8\n",
      "海口 19\n",
      "呼和浩特 7\n",
      "乌鲁木齐 2\n",
      "拉萨 2\n",
      "西宁 0\n",
      "银川 8\n",
      "兰州 7\n",
      "石家庄 12\n",
      "郑州 13\n",
      "武汉 16\n",
      "长沙 18\n",
      "济南 14\n",
      "南京 13\n",
      "合肥 14\n",
      "太原 10\n",
      "西安 11\n",
      "成都 12\n",
      "昆明 11\n",
      "贵阳 14\n",
      "杭州 12\n",
      "福州 12\n",
      "南昌 15\n",
      "广州 16\n",
      "南宁 19\n",
      "北京 12\n",
      "天津 14\n",
      "上海 13\n",
      "重庆 17\n",
      "香港 18\n",
      "澳门 18\n",
      "台北 17\n",
      "西沙 24\n",
      "南沙 25\n",
      "钓鱼岛 18\n"
     ]
    }
   ],
   "source": [
    "for city in root.iter('city'):\n",
    "    print(city.get('cityname'), city.get('tem1'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 撰写网路爬虫搜集网路资料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "newsurl = 'http://news.qq.com/'\n",
    "res = requests.get(newsurl)\n",
    "#print(res.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'   Hello World This is link1 This is link2  '"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "html_sample = ' \\\n",
    "<html> \\\n",
    " <body> \\\n",
    " <h1 id=\"title\">Hello World</h1> \\\n",
    " <a href=\"#\" class=\"link\">This is link1</a> \\\n",
    " <a href=\"# link2\" class=\"link\">This is link2</a> \\\n",
    " </body> \\\n",
    " </html>'\n",
    "soup = BeautifulSoup(html_sample, 'html.parser')\n",
    "soup.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Hello World'"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.select('h1')[0].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'This is link1'"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.select('a')[0].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'This is link2'"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.select('a')[1].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is link1\n",
      "This is link2\n"
     ]
    }
   ],
   "source": [
    "for alink in soup.select('a'):\n",
    "    print(alink.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Hello World'"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.select('#title')[0].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'This is link1'"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.select('.link')[0].text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#\n",
      "# link2\n"
     ]
    }
   ],
   "source": [
    "for link in soup.select('a'):\n",
    "    print(link['href'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "俄圣彼得堡地铁恐袭致11死 嫌犯为中亚人 http://news.qq.com/a/20170404/003976.htm\n",
      "新任河北省委副书记许勤今日调研安新县 http://news.qq.com/a/20170404/013012.htm\n",
      "巴西众院弹劾总统案获通过 罗塞夫政党承认落败 http://news.qq.com/a/20160418/023091.htm\n",
      "人民日报：雄安新区不是迁都 也非再建首都副中心 http://finance.qq.com/a/20170404/004444.htm\n",
      "特朗普女婿访伊拉克 频涉外交如“影子国务卿” http://news.qq.com/a/20170404/015986.htm\n",
      "朴槿惠在拘留所的首个周末：午饭吃土豆辣酱锅辣白菜 http://news.qq.com/a/20170404/011267.htm\n",
      "中国留学生在美身亡：飙车飞起撞隧道顶 女乘客受伤 http://news.qq.com/a/20170404/010762.htm\n",
      "容城官方发公开信：市场上所有预售楼盘均为违规销售 http://news.qq.com/a/20170404/010053.htm\n",
      "北京半个月发12项楼市调控政策 限购“再无空可钻” http://news.qq.com/a/20170404/005873.htm\n",
      "香港查获101箱日本核灾区牛肉 已经通知日本 http://news.qq.com/a/20170404/009136.htm\n",
      "朝鲜外务省发言人：世界很快将见证对朝制裁被粉碎 http://news.qq.com/a/20170404/011198.htm\n",
      "韩国海警派遣12艘舰艇打击中国渔船 配重火力武器 http://news.qq.com/a/20170404/013382.htm\n",
      "特斯拉股价创历史新高 CEO马斯克发推文嘲讽做空者 http://tech.qq.com/a/20170404/002822.htm\n",
      "联通千兆宽带就是噱头货：比硬盘还快 依旧卡顿 http://tech.qq.com/a/20170404/003766.htm\n",
      "52岁港星意外去世 疑因头部受伤不治身亡 http://ent.qq.com/a/20170404/015329.htm\n",
      "日本中学恢复“拼刺刀”课程 引军国复辟恐慌 http://news.qq.com/a/20170404/010067.htm\n",
      "韩国检方今将到看守所讯问朴槿惠 或在大选前公诉 http://news.qq.com/a/20170404/003866.htm\n",
      "面对横空出世的雄安新区 周边城市怎么说？ http://news.qq.com/a/20170404/000631.htm\n",
      "韩国最大在野党提名文在寅为总统候选人 http://news.qq.com/a/20170404/001051.htm\n",
      "日本驻韩大使时隔85天将重返韩国 被指暴露权宜外交 http://news.qq.com/a/20170404/008809.htm\n",
      "雄安新区召开三县干部会议：强调防止炒房影响投资 http://news.qq.com/a/20170403/020498.htm\n",
      "非洲南部国家博茨瓦纳发生6.6级地震 http://news.qq.com/a/20170404/001069.htm\n",
      "解读圣彼得堡爆炸：车臣组织嫌疑更大 系有组织行动 http://news.qq.com/a/20170404/000687.htm\n",
      "雄安新区白洋淀移民搬迁每亩补偿6万？村民很淡定 http://news.qq.com/a/20170404/000745.htm\n",
      "俄华人讲地铁恐袭经历：200米外听到爆炸声音 http://news.qq.com/a/20170404/001160.htm\n",
      "北京市住建委：国有土地上住宅平房纳入限购范围 http://news.qq.com/a/20170403/025107.htm\n",
      "雄安亲历记：市民称去年有外地炒房客一下买两栋楼 http://news.qq.com/a/20170404/001489.htm\n",
      "环保部：京冀有企业蒙骗执法人员 拒不接受检查 http://news.qq.com/a/20170404/001743.htm\n",
      "7旬老翁婚后20年出轨当爹 被追问振振有词：续香火 http://news.qq.com/a/20170404/000732.htm\n",
      "人社局官员支招企业伪造失业人数 骗税1544万余元 http://news.qq.com/a/20170404/001732.htm\n",
      "湖南一天发生三起事故 翻车火灾等事故致19死20伤 http://news.qq.com/a/20170403/031776.htm\n",
      "俄罗斯地铁发生爆炸 已致10死50伤 http://news.qq.com/a/20170403/026216.htm\n",
      "韩国防部：可考虑为世越号派遗骸挖掘鉴定小组 http://news.qq.com/a/20170403/017000.htm\n",
      "雄县居民：激动得一夜没睡 我们已经是特区人民了 http://news.qq.com/a/20170403/001731.htm\n",
      "人民日报海外版：部署“萨德” 日本凑什么热闹 http://news.qq.com/a/20170403/001649.htm\n",
      "日本外务省：中日副外长级磋商将于明日举行 http://news.qq.com/a/20170403/012498.htm\n",
      "北方多地再现重污染 环保部称部分政府部门不作为 http://news.qq.com/a/20170403/015965.htm\n",
      "雄县已全面停止房产交易 专人24小时值守劝离购房者 http://news.qq.com/a/20170403/014084.htm\n",
      "韩国检方正式对朴槿惠进行后续调查 将派专案组讯问 http://news.qq.com/a/20170403/009808.htm\n",
      "3个月40城楼市调控：北京等多地有望进入低迷调整期 http://news.qq.com/a/20170403/005648.htm\n",
      "安徽安庆一油品公司发生闪爆事故致5死3伤 http://news.qq.com/a/20170403/006649.htm\n",
      "南京宝马肇事案：二次鉴定为何删除“急性短暂性”表述 http://news.qq.com/a/20170403/009867.htm\n",
      "河北安新县开发商非法占地建37栋楼外售 老板被刑拘 http://news.qq.com/a/20170403/007883.htm\n",
      "俄地铁爆炸系恐袭 已致11死45伤 http://news.qq.com/a/20170404/000747.htm\n",
      "河北雄安新区临时党委租下整栋酒店办公 http://news.qq.com/a/20170403/016227.htm\n",
      "天津市长：坚决拥护设立雄安新区重大决策部署 http://news.qq.com/a/20170403/013419.htm\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "res = requests.get('http://news.qq.com/')\n",
    "soup = BeautifulSoup(res.text, 'html.parser')\n",
    "for news in soup.select('.Q-tpWrap .text'):\n",
    "    print(news.select('a')[0].text, news.select('a')[0]['href'])\n",
    "    #print(\"================================\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "res = requests.get('http://news.qq.com/')\n",
    "soup = BeautifulSoup(res.text, 'html.parser')\n",
    "\n",
    "newsary = []\n",
    "for news in soup.select('.Q-tpWrap .text'):\n",
    "    newsary.append({'title':news.select('a')[0].text, 'url':news.select('a')[0]['href']})\n",
    "    #print(\"================================\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>俄圣彼得堡地铁恐袭致11死 嫌犯为中亚人</td>\n",
       "      <td>http://news.qq.com/a/20170404/003976.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>新任河北省委副书记许勤今日调研安新县</td>\n",
       "      <td>http://news.qq.com/a/20170404/013012.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>巴西众院弹劾总统案获通过 罗塞夫政党承认落败</td>\n",
       "      <td>http://news.qq.com/a/20160418/023091.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>人民日报：雄安新区不是迁都 也非再建首都副中心</td>\n",
       "      <td>http://finance.qq.com/a/20170404/004444.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>特朗普女婿访伊拉克 频涉外交如“影子国务卿”</td>\n",
       "      <td>http://news.qq.com/a/20170404/015986.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>朴槿惠在拘留所的首个周末：午饭吃土豆辣酱锅辣白菜</td>\n",
       "      <td>http://news.qq.com/a/20170404/011267.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>中国留学生在美身亡：飙车飞起撞隧道顶 女乘客受伤</td>\n",
       "      <td>http://news.qq.com/a/20170404/010762.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>容城官方发公开信：市场上所有预售楼盘均为违规销售</td>\n",
       "      <td>http://news.qq.com/a/20170404/010053.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>北京半个月发12项楼市调控政策 限购“再无空可钻”</td>\n",
       "      <td>http://news.qq.com/a/20170404/005873.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>香港查获101箱日本核灾区牛肉 已经通知日本</td>\n",
       "      <td>http://news.qq.com/a/20170404/009136.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>朝鲜外务省发言人：世界很快将见证对朝制裁被粉碎</td>\n",
       "      <td>http://news.qq.com/a/20170404/011198.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>韩国海警派遣12艘舰艇打击中国渔船 配重火力武器</td>\n",
       "      <td>http://news.qq.com/a/20170404/013382.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>特斯拉股价创历史新高 CEO马斯克发推文嘲讽做空者</td>\n",
       "      <td>http://tech.qq.com/a/20170404/002822.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>联通千兆宽带就是噱头货：比硬盘还快 依旧卡顿</td>\n",
       "      <td>http://tech.qq.com/a/20170404/003766.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>52岁港星意外去世 疑因头部受伤不治身亡</td>\n",
       "      <td>http://ent.qq.com/a/20170404/015329.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>日本中学恢复“拼刺刀”课程 引军国复辟恐慌</td>\n",
       "      <td>http://news.qq.com/a/20170404/010067.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>韩国检方今将到看守所讯问朴槿惠 或在大选前公诉</td>\n",
       "      <td>http://news.qq.com/a/20170404/003866.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>面对横空出世的雄安新区 周边城市怎么说？</td>\n",
       "      <td>http://news.qq.com/a/20170404/000631.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>韩国最大在野党提名文在寅为总统候选人</td>\n",
       "      <td>http://news.qq.com/a/20170404/001051.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>日本驻韩大使时隔85天将重返韩国 被指暴露权宜外交</td>\n",
       "      <td>http://news.qq.com/a/20170404/008809.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>雄安新区召开三县干部会议：强调防止炒房影响投资</td>\n",
       "      <td>http://news.qq.com/a/20170403/020498.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>非洲南部国家博茨瓦纳发生6.6级地震</td>\n",
       "      <td>http://news.qq.com/a/20170404/001069.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>解读圣彼得堡爆炸：车臣组织嫌疑更大 系有组织行动</td>\n",
       "      <td>http://news.qq.com/a/20170404/000687.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>雄安新区白洋淀移民搬迁每亩补偿6万？村民很淡定</td>\n",
       "      <td>http://news.qq.com/a/20170404/000745.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>俄华人讲地铁恐袭经历：200米外听到爆炸声音</td>\n",
       "      <td>http://news.qq.com/a/20170404/001160.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>北京市住建委：国有土地上住宅平房纳入限购范围</td>\n",
       "      <td>http://news.qq.com/a/20170403/025107.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>雄安亲历记：市民称去年有外地炒房客一下买两栋楼</td>\n",
       "      <td>http://news.qq.com/a/20170404/001489.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>环保部：京冀有企业蒙骗执法人员 拒不接受检查</td>\n",
       "      <td>http://news.qq.com/a/20170404/001743.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>7旬老翁婚后20年出轨当爹 被追问振振有词：续香火</td>\n",
       "      <td>http://news.qq.com/a/20170404/000732.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>人社局官员支招企业伪造失业人数 骗税1544万余元</td>\n",
       "      <td>http://news.qq.com/a/20170404/001732.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>湖南一天发生三起事故 翻车火灾等事故致19死20伤</td>\n",
       "      <td>http://news.qq.com/a/20170403/031776.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>俄罗斯地铁发生爆炸 已致10死50伤</td>\n",
       "      <td>http://news.qq.com/a/20170403/026216.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>韩国防部：可考虑为世越号派遗骸挖掘鉴定小组</td>\n",
       "      <td>http://news.qq.com/a/20170403/017000.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>雄县居民：激动得一夜没睡 我们已经是特区人民了</td>\n",
       "      <td>http://news.qq.com/a/20170403/001731.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>人民日报海外版：部署“萨德” 日本凑什么热闹</td>\n",
       "      <td>http://news.qq.com/a/20170403/001649.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>日本外务省：中日副外长级磋商将于明日举行</td>\n",
       "      <td>http://news.qq.com/a/20170403/012498.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>北方多地再现重污染 环保部称部分政府部门不作为</td>\n",
       "      <td>http://news.qq.com/a/20170403/015965.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>雄县已全面停止房产交易 专人24小时值守劝离购房者</td>\n",
       "      <td>http://news.qq.com/a/20170403/014084.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>韩国检方正式对朴槿惠进行后续调查 将派专案组讯问</td>\n",
       "      <td>http://news.qq.com/a/20170403/009808.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>3个月40城楼市调控：北京等多地有望进入低迷调整期</td>\n",
       "      <td>http://news.qq.com/a/20170403/005648.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>安徽安庆一油品公司发生闪爆事故致5死3伤</td>\n",
       "      <td>http://news.qq.com/a/20170403/006649.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>南京宝马肇事案：二次鉴定为何删除“急性短暂性”表述</td>\n",
       "      <td>http://news.qq.com/a/20170403/009867.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>河北安新县开发商非法占地建37栋楼外售 老板被刑拘</td>\n",
       "      <td>http://news.qq.com/a/20170403/007883.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>俄地铁爆炸系恐袭 已致11死45伤</td>\n",
       "      <td>http://news.qq.com/a/20170404/000747.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>河北雄安新区临时党委租下整栋酒店办公</td>\n",
       "      <td>http://news.qq.com/a/20170403/016227.htm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>天津市长：坚决拥护设立雄安新区重大决策部署</td>\n",
       "      <td>http://news.qq.com/a/20170403/013419.htm</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        title                                          url\n",
       "0        俄圣彼得堡地铁恐袭致11死 嫌犯为中亚人     http://news.qq.com/a/20170404/003976.htm\n",
       "1          新任河北省委副书记许勤今日调研安新县     http://news.qq.com/a/20170404/013012.htm\n",
       "2      巴西众院弹劾总统案获通过 罗塞夫政党承认落败     http://news.qq.com/a/20160418/023091.htm\n",
       "3     人民日报：雄安新区不是迁都 也非再建首都副中心  http://finance.qq.com/a/20170404/004444.htm\n",
       "4      特朗普女婿访伊拉克 频涉外交如“影子国务卿”     http://news.qq.com/a/20170404/015986.htm\n",
       "5    朴槿惠在拘留所的首个周末：午饭吃土豆辣酱锅辣白菜     http://news.qq.com/a/20170404/011267.htm\n",
       "6    中国留学生在美身亡：飙车飞起撞隧道顶 女乘客受伤     http://news.qq.com/a/20170404/010762.htm\n",
       "7    容城官方发公开信：市场上所有预售楼盘均为违规销售     http://news.qq.com/a/20170404/010053.htm\n",
       "8   北京半个月发12项楼市调控政策 限购“再无空可钻”     http://news.qq.com/a/20170404/005873.htm\n",
       "9      香港查获101箱日本核灾区牛肉 已经通知日本     http://news.qq.com/a/20170404/009136.htm\n",
       "10    朝鲜外务省发言人：世界很快将见证对朝制裁被粉碎     http://news.qq.com/a/20170404/011198.htm\n",
       "11   韩国海警派遣12艘舰艇打击中国渔船 配重火力武器     http://news.qq.com/a/20170404/013382.htm\n",
       "12  特斯拉股价创历史新高 CEO马斯克发推文嘲讽做空者     http://tech.qq.com/a/20170404/002822.htm\n",
       "13     联通千兆宽带就是噱头货：比硬盘还快 依旧卡顿     http://tech.qq.com/a/20170404/003766.htm\n",
       "14       52岁港星意外去世 疑因头部受伤不治身亡      http://ent.qq.com/a/20170404/015329.htm\n",
       "15      日本中学恢复“拼刺刀”课程 引军国复辟恐慌     http://news.qq.com/a/20170404/010067.htm\n",
       "16    韩国检方今将到看守所讯问朴槿惠 或在大选前公诉     http://news.qq.com/a/20170404/003866.htm\n",
       "17       面对横空出世的雄安新区 周边城市怎么说？     http://news.qq.com/a/20170404/000631.htm\n",
       "18         韩国最大在野党提名文在寅为总统候选人     http://news.qq.com/a/20170404/001051.htm\n",
       "19  日本驻韩大使时隔85天将重返韩国 被指暴露权宜外交     http://news.qq.com/a/20170404/008809.htm\n",
       "20    雄安新区召开三县干部会议：强调防止炒房影响投资     http://news.qq.com/a/20170403/020498.htm\n",
       "21         非洲南部国家博茨瓦纳发生6.6级地震     http://news.qq.com/a/20170404/001069.htm\n",
       "22   解读圣彼得堡爆炸：车臣组织嫌疑更大 系有组织行动     http://news.qq.com/a/20170404/000687.htm\n",
       "23    雄安新区白洋淀移民搬迁每亩补偿6万？村民很淡定     http://news.qq.com/a/20170404/000745.htm\n",
       "24     俄华人讲地铁恐袭经历：200米外听到爆炸声音     http://news.qq.com/a/20170404/001160.htm\n",
       "25     北京市住建委：国有土地上住宅平房纳入限购范围     http://news.qq.com/a/20170403/025107.htm\n",
       "26    雄安亲历记：市民称去年有外地炒房客一下买两栋楼     http://news.qq.com/a/20170404/001489.htm\n",
       "27     环保部：京冀有企业蒙骗执法人员 拒不接受检查     http://news.qq.com/a/20170404/001743.htm\n",
       "28  7旬老翁婚后20年出轨当爹 被追问振振有词：续香火     http://news.qq.com/a/20170404/000732.htm\n",
       "29  人社局官员支招企业伪造失业人数 骗税1544万余元     http://news.qq.com/a/20170404/001732.htm\n",
       "30  湖南一天发生三起事故 翻车火灾等事故致19死20伤     http://news.qq.com/a/20170403/031776.htm\n",
       "31         俄罗斯地铁发生爆炸 已致10死50伤     http://news.qq.com/a/20170403/026216.htm\n",
       "32      韩国防部：可考虑为世越号派遗骸挖掘鉴定小组     http://news.qq.com/a/20170403/017000.htm\n",
       "33    雄县居民：激动得一夜没睡 我们已经是特区人民了     http://news.qq.com/a/20170403/001731.htm\n",
       "34     人民日报海外版：部署“萨德” 日本凑什么热闹     http://news.qq.com/a/20170403/001649.htm\n",
       "35       日本外务省：中日副外长级磋商将于明日举行     http://news.qq.com/a/20170403/012498.htm\n",
       "36    北方多地再现重污染 环保部称部分政府部门不作为     http://news.qq.com/a/20170403/015965.htm\n",
       "37  雄县已全面停止房产交易 专人24小时值守劝离购房者     http://news.qq.com/a/20170403/014084.htm\n",
       "38   韩国检方正式对朴槿惠进行后续调查 将派专案组讯问     http://news.qq.com/a/20170403/009808.htm\n",
       "39  3个月40城楼市调控：北京等多地有望进入低迷调整期     http://news.qq.com/a/20170403/005648.htm\n",
       "40       安徽安庆一油品公司发生闪爆事故致5死3伤     http://news.qq.com/a/20170403/006649.htm\n",
       "41  南京宝马肇事案：二次鉴定为何删除“急性短暂性”表述     http://news.qq.com/a/20170403/009867.htm\n",
       "42  河北安新县开发商非法占地建37栋楼外售 老板被刑拘     http://news.qq.com/a/20170403/007883.htm\n",
       "43          俄地铁爆炸系恐袭 已致11死45伤     http://news.qq.com/a/20170404/000747.htm\n",
       "44         河北雄安新区临时党委租下整栋酒店办公     http://news.qq.com/a/20170403/016227.htm\n",
       "45      天津市长：坚决拥护设立雄安新区重大决策部署     http://news.qq.com/a/20170403/013419.htm"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas\n",
    "newsdf = pandas.DataFrame(newsary)\n",
    "newsdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "newsdf.to_excel('news.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}