梯度下降.ipynb

{"cells":[{"cell_type":"markdown","metadata":{"graffitiCellId":"id_on4sre8","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"7572198A886042739E2AD4ABD4B8B745","mdEditEnable":false},"source":"# 梯度下降 \n（[Boyd & Vandenberghe, 2004](https://d2l.ai/chapter_references/zreferences.html#boyd-vandenberghe-2004)）"},{"cell_type":"code","execution_count":1,"metadata":{"graffitiCellId":"id_nl4rnln","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"BF05AB89B8E640C281B914AAFA206BB0","collapsed":false,"scrolled":false},"outputs":[],"source":"%matplotlib inline\nimport numpy as np\nimport torch\nimport time\nfrom torch import nn, optim\nimport math\nimport sys\nsys.path.append('/home/kesci/input')\nimport d2lzh1981 as d2l"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_t1q0ber","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"BCD65556302D4CCF8FF96FE8ED6623D2","mdEditEnable":false},"source":"## 一维梯度下降"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_pppxq5z","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"94EC994AB87740408746EC4E07170286","mdEditEnable":false},"source":"**证明：沿梯度反方向移动自变量可以减小函数值**\n\n泰勒展开：\n\n$$\nf(x+\\epsilon)=f(x)+\\epsilon f^{\\prime}(x)+\\mathcal{O}\\left(\\epsilon^{2}\\right)\n$$\n\n代入沿梯度方向的移动量 $\\eta f^{\\prime}(x)$：\n\n$$\nf\\left(x-\\eta f^{\\prime}(x)\\right)=f(x)-\\eta f^{\\prime 2}(x)+\\mathcal{O}\\left(\\eta^{2} f^{\\prime 2}(x)\\right)\n$$\n\n$$\nf\\left(x-\\eta f^{\\prime}(x)\\right) \\lesssim f(x)\n$$\n\n\n$$\nx \\leftarrow x-\\eta f^{\\prime}(x)\n$$\n"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_mhywc3s","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"E74B71FF3F134AFE89F6C3DD34026F32","mdEditEnable":false},"source":"e.g.\n\n$$\nf(x) = x^2\n$$\n"},{"cell_type":"code","execution_count":2,"metadata":{"graffitiCellId":"id_6xl9gas","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"842E5A0DE2274C24A6AF587E490517FD","collapsed":false,"scrolled":false},"outputs":[{"output_type":"stream","text":"epoch 10, x: 0.06046617599999997\n","name":"stdout"}],"source":"def f(x):\n    return x**2  # Objective function\n\ndef gradf(x):\n    return 2 * x  # Its derivative\n\ndef gd(eta):\n    x = 10\n    results = [x]\n    for i in range(10):\n        x -= eta * gradf(x)\n        results.append(x)\n    print('epoch 10, x:', x)\n    return results\n\nres = gd(0.2)"},{"cell_type":"code","execution_count":3,"metadata":{"graffitiCellId":"id_bpk707t","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"474742EF0BCD428780FEBB6BA93E1785","collapsed":false,"scrolled":false},"outputs":[{"output_type":"display_data","metadata":{"needs_background":"light"},"data":{"text/plain":"<Figure size 252x180 with 1 Axes>","text/html":"<img src=\"https://cdn.kesci.com/rt_upload/474742EF0BCD428780FEBB6BA93E1785/q5oim9v02y.svg\">"},"transient":{}}],"source":"def show_trace(res):\n    n = max(abs(min(res)), abs(max(res)))\n    f_line = np.arange(-n, n, 0.01)\n    d2l.set_figsize((3.5, 2.5))\n    d2l.plt.plot(f_line, [f(x) for x in f_line],'-')\n    d2l.plt.plot(res, [f(x) for x in res],'-o')\n    d2l.plt.xlabel('x')\n    d2l.plt.ylabel('f(x)')\n    \n\nshow_trace(res)"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_mhgaljc","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"3A083FD82CE549C38C1B76F74672A80C","mdEditEnable":false},"source":"### 学习率"},{"cell_type":"code","execution_count":4,"metadata":{"graffitiCellId":"id_8nd5ipw","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"63F2E327184549D38E1C1323766EBA35","collapsed":false,"scrolled":false},"outputs":[{"output_type":"stream","text":"epoch 10, x: 3.4867844009999995\n","name":"stdout"},{"output_type":"display_data","metadata":{"needs_background":"light"},"data":{"text/plain":"<Figure size 252x180 with 1 Axes>","text/html":"<img src=\"https://cdn.kesci.com/rt_upload/63F2E327184549D38E1C1323766EBA35/q5oim918r4.svg\">"},"transient":{}}],"source":"show_trace(gd(0.05))"},{"cell_type":"code","execution_count":5,"metadata":{"graffitiCellId":"id_lqgn6wo","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"540302EC5E434318809B895B453AABE4","collapsed":false,"scrolled":false},"outputs":[{"output_type":"stream","text":"epoch 10, x: 61.917364224000096\n","name":"stdout"},{"output_type":"display_data","metadata":{"needs_background":"light"},"data":{"text/plain":"<Figure size 252x180 with 1 Axes>","text/html":"<img src=\"https://cdn.kesci.com/rt_upload/540302EC5E434318809B895B453AABE4/q5oim98mjm.svg\">"},"transient":{}}],"source":"show_trace(gd(1.1))"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_8hf2rdv","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"2EC285A41CCE4994BB9AFDEAFF9A7283","mdEditEnable":false},"source":"### 局部极小值"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_32accll","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"1D6CF0E790DF412B8BEF0C105B28E908","mdEditEnable":false},"source":"e.g.\n\n$$\nf(x) = x\\cos cx\n$$\n"},{"cell_type":"code","execution_count":6,"metadata":{"graffitiCellId":"id_rq7w3yx","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"8372CFD30F534D63B847DED05E6BC1E7","collapsed":false,"scrolled":false},"outputs":[{"output_type":"stream","text":"epoch 10, x: -1.528165927635083\n","name":"stdout"},{"output_type":"display_data","metadata":{"needs_background":"light"},"data":{"text/plain":"<Figure size 252x180 with 1 Axes>","text/html":"<img src=\"https://cdn.kesci.com/rt_upload/8372CFD30F534D63B847DED05E6BC1E7/q5oim9yozt.svg\">"},"transient":{}}],"source":"c = 0.15 * np.pi\n\ndef f(x):\n    return x * np.cos(c * x)\n\ndef gradf(x):\n    return np.cos(c * x) - c * x * np.sin(c * x)\n\nshow_trace(gd(2))"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_nf23exa","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"534F5C33B2D14E7083455E6F9189AED1","mdEditEnable":false},"source":"## 多维梯度下降"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_e2tln4o","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"99DE5DDC9BFB4A638E07224320A1F21F","mdEditEnable":false},"source":"\n$$\n\\nabla f(\\mathbf{x})=\\left[\\frac{\\partial f(\\mathbf{x})}{\\partial x_{1}}, \\frac{\\partial f(\\mathbf{x})}{\\partial x_{2}}, \\dots, \\frac{\\partial f(\\mathbf{x})}{\\partial x_{d}}\\right]^{\\top}\n$$\n\n$$\nf(\\mathbf{x}+\\epsilon)=f(\\mathbf{x})+\\epsilon^{\\top} \\nabla f(\\mathbf{x})+\\mathcal{O}\\left(\\|\\epsilon\\|^{2}\\right)\n$$\n\n$$\n\\mathbf{x} \\leftarrow \\mathbf{x}-\\eta \\nabla f(\\mathbf{x})\n$$"},{"cell_type":"code","execution_count":7,"metadata":{"graffitiCellId":"id_kxz1kl7","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"D2A69BABDFD04AF48847E925A8DB8351","collapsed":false,"scrolled":false},"outputs":[],"source":"def train_2d(trainer, steps=20):\n    x1, x2 = -5, -2\n    results = [(x1, x2)]\n    for i in range(steps):\n        x1, x2 = trainer(x1, x2)\n        results.append((x1, x2))\n    print('epoch %d, x1 %f, x2 %f' % (i + 1, x1, x2))\n    return results\n\ndef show_trace_2d(f, results): \n    d2l.plt.plot(*zip(*results), '-o', color='#ff7f0e')\n    x1, x2 = np.meshgrid(np.arange(-5.5, 1.0, 0.1), np.arange(-3.0, 1.0, 0.1))\n    d2l.plt.contour(x1, x2, f(x1, x2), colors='#1f77b4')\n    d2l.plt.xlabel('x1')\n    d2l.plt.ylabel('x2')"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_843firi","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"1319758B74AC452588639895F40862F3","mdEditEnable":false},"source":"\n$$\nf(x) = x_1^2 + 2x_2^2\n$$\n"},{"cell_type":"code","execution_count":8,"metadata":{"graffitiCellId":"id_rbr2v5f","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"7C4F69BAE0E2424D80BC99B0867DB110","collapsed":false,"scrolled":false},"outputs":[{"output_type":"stream","text":"epoch 20, x1 -0.057646, x2 -0.000073\n","name":"stdout"},{"output_type":"display_data","metadata":{"needs_background":"light"},"data":{"text/plain":"<Figure size 252x180 with 1 Axes>","text/html":"<img src=\"https://cdn.kesci.com/rt_upload/7C4F69BAE0E2424D80BC99B0867DB110/q5oim9cca5.svg\">"},"transient":{}}],"source":"eta = 0.1\n\ndef f_2d(x1, x2):  # 目标函数\n    return x1 ** 2 + 2 * x2 ** 2\n\ndef gd_2d(x1, x2):\n    return (x1 - eta * 2 * x1, x2 - eta * 4 * x2)\n\nshow_trace_2d(f_2d, train_2d(gd_2d))"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_ojs9x5n","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"F1AFA19516FE42F3827F10E5F5B4BDD0","mdEditEnable":false},"source":"## 自适应方法"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_ybiw4cv","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"98F16E9A992D4D189DE9B5E21F70B750","mdEditEnable":false},"source":"### 牛顿法"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_s76eff5","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"62C9EFD5B9D94D058B395F93EF9162A7","mdEditEnable":false},"source":"在 $x + \\epsilon$ 处泰勒展开：\n\n$$\nf(\\mathbf{x}+\\epsilon)=f(\\mathbf{x})+\\epsilon^{\\top} \\nabla f(\\mathbf{x})+\\frac{1}{2} \\epsilon^{\\top} \\nabla \\nabla^{\\top} f(\\mathbf{x}) \\epsilon+\\mathcal{O}\\left(\\|\\epsilon\\|^{3}\\right)\n$$\n\n最小值点处满足: $\\nabla f(\\mathbf{x})=0$, 即我们希望 $\\nabla f(\\mathbf{x} + \\epsilon)=0$, 对上式关于 $\\epsilon$ 求导，忽略高阶无穷小，有：\n\n$$\n\\nabla f(\\mathbf{x})+\\boldsymbol{H}_{f} \\boldsymbol{\\epsilon}=0 \\text { and hence } \\epsilon=-\\boldsymbol{H}_{f}^{-1} \\nabla f(\\mathbf{x})\n$$\n"},{"cell_type":"code","execution_count":9,"metadata":{"graffitiCellId":"id_ne0bb4i","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"93AB8E85F6264E129E9F622747D059FD","collapsed":false,"scrolled":false},"outputs":[{"output_type":"stream","text":"epoch 10, x: 0.0\n","name":"stdout"},{"output_type":"display_data","metadata":{"needs_background":"light"},"data":{"text/plain":"<Figure size 252x180 with 1 Axes>","text/html":"<img src=\"https://cdn.kesci.com/rt_upload/93AB8E85F6264E129E9F622747D059FD/q5oima7f51.svg\">"},"transient":{}}],"source":"c = 0.5\n\ndef f(x):\n    return np.cosh(c * x)  # Objective\n\ndef gradf(x):\n    return c * np.sinh(c * x)  # Derivative\n\ndef hessf(x):\n    return c**2 * np.cosh(c * x)  # Hessian\n\n# Hide learning rate for now\ndef newton(eta=1):\n    x = 10\n    results = [x]\n    for i in range(10):\n        x -= eta * gradf(x) / hessf(x)\n        results.append(x)\n    print('epoch 10, x:', x)\n    return results\n\nshow_trace(newton())"},{"cell_type":"code","execution_count":10,"metadata":{"graffitiCellId":"id_0zttur6","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"6F40E126C3FD4E059463B6B5E1B809E5","collapsed":false,"scrolled":false},"outputs":[{"output_type":"stream","text":"epoch 10, x: 26.83413291324767\n","name":"stdout"},{"output_type":"display_data","metadata":{"needs_background":"light"},"data":{"text/plain":"<Figure size 252x180 with 1 Axes>","text/html":"<img src=\"https://cdn.kesci.com/rt_upload/6F40E126C3FD4E059463B6B5E1B809E5/q5oimahwv7.svg\">"},"transient":{}}],"source":"c = 0.15 * np.pi\n\ndef f(x):\n    return x * np.cos(c * x)\n\ndef gradf(x):\n    return np.cos(c * x) - c * x * np.sin(c * x)\n\ndef hessf(x):\n    return - 2 * c * np.sin(c * x) - x * c**2 * np.cos(c * x)\n\nshow_trace(newton())"},{"cell_type":"code","execution_count":11,"metadata":{"graffitiCellId":"id_x3tkks0","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"5FE9DDAA901B422181ECFBFB171DDDF0","collapsed":false,"scrolled":false},"outputs":[{"output_type":"stream","text":"epoch 10, x: 7.269860168684531\n","name":"stdout"},{"output_type":"display_data","metadata":{"needs_background":"light"},"data":{"text/plain":"<Figure size 252x180 with 1 Axes>","text/html":"<img src=\"https://cdn.kesci.com/rt_upload/5FE9DDAA901B422181ECFBFB171DDDF0/q5oimau6rq.svg\">"},"transient":{}}],"source":"show_trace(newton(0.5))"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_abhcxnu","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"585574DCA7F24BF682E485EDB1EE6B3E","mdEditEnable":false},"source":"### 收敛性分析"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_p7m23hf","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"BF48B399382743CABB1D0ABCD94B3144","mdEditEnable":false},"source":"只考虑在函数为凸函数, 且最小值点上 $f''(x^*) > 0$ 时的收敛速度：\n\n令 $x_k$ 为第 $k$ 次迭代后 $x$ 的值， $e_{k}:=x_{k}-x^{*}$ 表示 $x_k$ 到最小值点 $x^{*}$ 的距离，由 $f'(x^{*}) = 0$:\n\n$$\n0=f^{\\prime}\\left(x_{k}-e_{k}\\right)=f^{\\prime}\\left(x_{k}\\right)-e_{k} f^{\\prime \\prime}\\left(x_{k}\\right)+\\frac{1}{2} e_{k}^{2} f^{\\prime \\prime \\prime}\\left(\\xi_{k}\\right) \\text{for some } \\xi_{k} \\in\\left[x_{k}-e_{k}, x_{k}\\right]\n$$\n\n两边除以 $f''(x_k)$, 有：\n\n$$\ne_{k}-f^{\\prime}\\left(x_{k}\\right) / f^{\\prime \\prime}\\left(x_{k}\\right)=\\frac{1}{2} e_{k}^{2} f^{\\prime \\prime \\prime}\\left(\\xi_{k}\\right) / f^{\\prime \\prime}\\left(x_{k}\\right)\n$$\n\n代入更新方程 $x_{k+1} = x_{k} - f^{\\prime}\\left(x_{k}\\right) / f^{\\prime \\prime}\\left(x_{k}\\right)$, 得到：\n\n$$\nx_k - x^{*} - f^{\\prime}\\left(x_{k}\\right) / f^{\\prime \\prime}\\left(x_{k}\\right) =\\frac{1}{2} e_{k}^{2} f^{\\prime \\prime \\prime}\\left(\\xi_{k}\\right) / f^{\\prime \\prime}\\left(x_{k}\\right)\n$$\n\n\n$$\nx_{k+1} - x^{*} = e_{k+1} = \\frac{1}{2} e_{k}^{2} f^{\\prime \\prime \\prime}\\left(\\xi_{k}\\right) / f^{\\prime \\prime}\\left(x_{k}\\right)\n$$\n\n当 $\\frac{1}{2} f^{\\prime \\prime \\prime}\\left(\\xi_{k}\\right) / f^{\\prime \\prime}\\left(x_{k}\\right) \\leq c$ 时，有:\n\n$$\ne_{k+1} \\leq c e_{k}^{2}\n$$\n"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_3uu8q0a","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"650317A0356D4B23A2A9D3B22464D7CC","mdEditEnable":false},"source":"### 预处理 （Heissan阵辅助梯度下降）"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_vqv7thj","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"12635895E5334461A0ED724BFD885B25","mdEditEnable":false},"source":"\n$$\n\\mathbf{x} \\leftarrow \\mathbf{x}-\\eta \\operatorname{diag}\\left(H_{f}\\right)^{-1} \\nabla \\mathbf{x}\n$$\n"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_8khl2xc","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"34301E3820F5435795E8DC14A663FD87","mdEditEnable":false},"source":"### 梯度下降与线性搜索（共轭梯度法）"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_cvo62el","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"882E8525B5E64B159A284225BB043DAF","mdEditEnable":false},"source":"# 随机梯度下降"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_5myc4ii","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"BE0C412BB3A9439BA8C064ECA4239857","mdEditEnable":false},"source":"## 随机梯度下降参数更新"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_vccc2dg","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"CEDA7E740EC34520859761F73D38F3C8","mdEditEnable":false},"source":"对于有 $n$ 个样本对训练数据集，设 $f_i(x)$ 是第 $i$ 个样本的损失函数, 则目标函数为:\n\n$$\nf(\\mathbf{x})=\\frac{1}{n} \\sum_{i=1}^{n} f_{i}(\\mathbf{x})\n$$\n\n其梯度为:\n\n$$\n\\nabla f(\\mathbf{x})=\\frac{1}{n} \\sum_{i=1}^{n} \\nabla f_{i}(\\mathbf{x})\n$$\n\n使用该梯度的一次更新的时间复杂度为 $\\mathcal{O}(n)$\n\n随机梯度下降更新公式 $\\mathcal{O}(1)$:\n\n$$\n\\mathbf{x} \\leftarrow \\mathbf{x}-\\eta \\nabla f_{i}(\\mathbf{x})\n$$\n\n且有：\n\n$$\n\\mathbb{E}_{i} \\nabla f_{i}(\\mathbf{x})=\\frac{1}{n} \\sum_{i=1}^{n} \\nabla f_{i}(\\mathbf{x})=\\nabla f(\\mathbf{x})\n$$\n"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_xa873bz","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"9F69CFFA97964916B90CD2BB23B01ABE","mdEditEnable":false},"source":"e.g. \n\n$$\nf(x_1, x_2) = x_1^2 + 2 x_2^2\n$$\n"},{"cell_type":"code","execution_count":12,"metadata":{"graffitiCellId":"id_dccr2u1","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"4CB5541B156E4F0AB7E145B54C8E6EA0","collapsed":false,"scrolled":false},"outputs":[{"output_type":"stream","text":"epoch 50, x1 -0.027566, x2 0.137605\n","name":"stdout"},{"output_type":"display_data","metadata":{"needs_background":"light"},"data":{"text/plain":"<Figure size 252x180 with 1 Axes>","text/html":"<img src=\"https://cdn.kesci.com/rt_upload/4CB5541B156E4F0AB7E145B54C8E6EA0/q5oimaluhh.svg\">"},"transient":{}}],"source":"def f(x1, x2):\n    return x1 ** 2 + 2 * x2 ** 2  # Objective\n\ndef gradf(x1, x2):\n    return (2 * x1, 4 * x2)  # Gradient\n\ndef sgd(x1, x2):  # Simulate noisy gradient\n    global lr  # Learning rate scheduler\n    (g1, g2) = gradf(x1, x2)  # Compute gradient\n    (g1, g2) = (g1 + np.random.normal(0.1), g2 + np.random.normal(0.1))\n    eta_t = eta * lr()  # Learning rate at time t\n    return (x1 - eta_t * g1, x2 - eta_t * g2)  # Update variables\n\neta = 0.1\nlr = (lambda: 1)  # Constant learning rate\nshow_trace_2d(f, train_2d(sgd, steps=50))\n"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_u5xwp88","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"B969ED17218C4BA887090B4B1C41D0B6","mdEditEnable":false},"source":"## 动态学习率"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_dh8duv0","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"68DC0E0518E64F20915A7FAE1489A72B","mdEditEnable":false},"source":"\n$$\n\\begin{array}{ll}{\\eta(t)=\\eta_{i} \\text { if } t_{i} \\leq t \\leq t_{i+1}} & {\\text { piecewise constant }} \\\\ {\\eta(t)=\\eta_{0} \\cdot e^{-\\lambda t}} & {\\text { exponential }} \\\\ {\\eta(t)=\\eta_{0} \\cdot(\\beta t+1)^{-\\alpha}} & {\\text { polynomial }}\\end{array}\n$$\n"},{"cell_type":"code","execution_count":13,"metadata":{"graffitiCellId":"id_d78lmwl","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"7E2E2CF3EB3A4A2D90F41A070E1D008F","collapsed":false,"scrolled":false},"outputs":[{"output_type":"stream","text":"epoch 1000, x1 -0.677947, x2 -0.089379\n","name":"stdout"},{"output_type":"display_data","metadata":{"needs_background":"light"},"data":{"text/plain":"<Figure size 252x180 with 1 Axes>","text/html":"<img src=\"https://cdn.kesci.com/rt_upload/7E2E2CF3EB3A4A2D90F41A070E1D008F/q5oimas9qm.svg\">"},"transient":{}}],"source":"def exponential():\n    global ctr\n    ctr += 1\n    return math.exp(-0.1 * ctr)\n\nctr = 1\nlr = exponential  # Set up learning rate\nshow_trace_2d(f, train_2d(sgd, steps=1000))"},{"cell_type":"code","execution_count":14,"metadata":{"graffitiCellId":"id_4sql34e","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"B3DF53CD6E1A458B836D2308F8263B58","collapsed":false,"scrolled":false},"outputs":[{"output_type":"stream","text":"epoch 50, x1 -0.095244, x2 -0.041674\n","name":"stdout"},{"output_type":"display_data","metadata":{"needs_background":"light"},"data":{"text/plain":"<Figure size 252x180 with 1 Axes>","text/html":"<img src=\"https://cdn.kesci.com/rt_upload/B3DF53CD6E1A458B836D2308F8263B58/q5oimapo7u.svg\">"},"transient":{}}],"source":"def polynomial():\n    global ctr\n    ctr += 1\n    return (1 + 0.1 * ctr)**(-0.5)\n\nctr = 1\nlr = polynomial  # Set up learning rate\nshow_trace_2d(f, train_2d(sgd, steps=50))"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_zwblf6k","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"45094D70C75448CE8483D4ECC50DCECF","mdEditEnable":false},"source":"# 小批量随机梯度下降"},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_jero8pd","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"CCED44DA669F40EB83770D4FCE98C82A","mdEditEnable":false},"source":"## 读取数据\n[读取数据](https://archive.ics.uci.edu/ml/datasets/Airfoil+Self-Noise)"},{"cell_type":"code","execution_count":16,"metadata":{"graffitiCellId":"id_f3mc1re","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"0C848C4C23DC489FACA6EC84A8EDE071","collapsed":false,"scrolled":false},"outputs":[{"output_type":"execute_result","metadata":{},"data":{"text/plain":"torch.Size([1500, 5])"},"transient":{},"execution_count":16}],"source":"def get_data_ch7():  # 本函数已保存在d2lzh_pytorch包中方便以后使用\n    data = np.genfromtxt('/home/kesci/input/airfoil4755/airfoil_self_noise.dat', delimiter='\\t')\n    data = (data - data.mean(axis=0)) / data.std(axis=0) # 标准化\n    return torch.tensor(data[:1500, :-1], dtype=torch.float32), \\\n           torch.tensor(data[:1500, -1], dtype=torch.float32) # 前1500个样本(每个样本5个特征)\n\nfeatures, labels = get_data_ch7()\nfeatures.shape"},{"cell_type":"code","metadata":{"graffitiCellId":"id_t2q026w","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"E7B13A69CE92494088087B62DB85D091","collapsed":false,"scrolled":false},"outputs":[{"output_type":"execute_result","metadata":{},"data":{"text/plain":"      0    1       2     3         4        5\n0   800  0.0  0.3048  71.3  0.002663  126.201\n1  1000  0.0  0.3048  71.3  0.002663  125.201\n2  1250  0.0  0.3048  71.3  0.002663  125.951\n3  1600  0.0  0.3048  71.3  0.002663  127.591\n4  2000  0.0  0.3048  71.3  0.002663  127.461\n5  2500  0.0  0.3048  71.3  0.002663  125.571\n6  3150  0.0  0.3048  71.3  0.002663  125.201\n7  4000  0.0  0.3048  71.3  0.002663  123.061\n8  5000  0.0  0.3048  71.3  0.002663  121.301\n9  6300  0.0  0.3048  71.3  0.002663  119.541","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n      <th>5</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <td>0</td>\n      <td>800</td>\n      <td>0.0</td>\n      <td>0.3048</td>\n      <td>71.3</td>\n      <td>0.002663</td>\n      <td>126.201</td>\n    </tr>\n    <tr>\n      <td>1</td>\n      <td>1000</td>\n      <td>0.0</td>\n      <td>0.3048</td>\n      <td>71.3</td>\n      <td>0.002663</td>\n      <td>125.201</td>\n    </tr>\n    <tr>\n      <td>2</td>\n      <td>1250</td>\n      <td>0.0</td>\n      <td>0.3048</td>\n      <td>71.3</td>\n      <td>0.002663</td>\n      <td>125.951</td>\n    </tr>\n    <tr>\n      <td>3</td>\n      <td>1600</td>\n      <td>0.0</td>\n      <td>0.3048</td>\n      <td>71.3</td>\n      <td>0.002663</td>\n      <td>127.591</td>\n    </tr>\n    <tr>\n      <td>4</td>\n      <td>2000</td>\n      <td>0.0</td>\n      <td>0.3048</td>\n      <td>71.3</td>\n      <td>0.002663</td>\n      <td>127.461</td>\n    </tr>\n    <tr>\n      <td>5</td>\n      <td>2500</td>\n      <td>0.0</td>\n      <td>0.3048</td>\n      <td>71.3</td>\n      <td>0.002663</td>\n      <td>125.571</td>\n    </tr>\n    <tr>\n      <td>6</td>\n      <td>3150</td>\n      <td>0.0</td>\n      <td>0.3048</td>\n      <td>71.3</td>\n      <td>0.002663</td>\n      <td>125.201</td>\n    </tr>\n    <tr>\n      <td>7</td>\n      <td>4000</td>\n      <td>0.0</td>\n      <td>0.3048</td>\n      <td>71.3</td>\n      <td>0.002663</td>\n      <td>123.061</td>\n    </tr>\n    <tr>\n      <td>8</td>\n      <td>5000</td>\n      <td>0.0</td>\n      <td>0.3048</td>\n      <td>71.3</td>\n      <td>0.002663</td>\n      <td>121.301</td>\n    </tr>\n    <tr>\n      <td>9</td>\n      <td>6300</td>\n      <td>0.0</td>\n      <td>0.3048</td>\n      <td>71.3</td>\n      <td>0.002663</td>\n      <td>119.541</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"transient":{},"execution_count":17}],"source":"import pandas as pd\ndf = pd.read_csv('/home/kesci/input/airfoil4755/airfoil_self_noise.dat', delimiter='\\t', header=None)\ndf.head(10)","execution_count":17},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_ryefk8w","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"521D2D54C45848008F28AC827A14071C","mdEditEnable":false},"source":"## 从零开始实现"},{"cell_type":"code","metadata":{"graffitiCellId":"id_wpr9jbt","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"5D8CD5C713ED4A0C828092D8E6CEE3D2","collapsed":false,"scrolled":false},"outputs":[],"source":"def sgd(params, states, hyperparams):\n    for p in params:\n        p.data -= hyperparams['lr'] * p.grad.data","execution_count":18},{"cell_type":"code","metadata":{"graffitiCellId":"id_iqqi7qi","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"3A8DB851F2CC4CA98C398C64DA8C0084","collapsed":false,"scrolled":false},"outputs":[],"source":"# 本函数已保存在d2lzh_pytorch包中方便以后使用\ndef train_ch7(optimizer_fn, states, hyperparams, features, labels,\n              batch_size=10, num_epochs=2):\n    # 初始化模型\n    net, loss = d2l.linreg, d2l.squared_loss\n    \n    w = torch.nn.Parameter(torch.tensor(np.random.normal(0, 0.01, size=(features.shape[1], 1)), dtype=torch.float32),\n                           requires_grad=True)\n    b = torch.nn.Parameter(torch.zeros(1, dtype=torch.float32), requires_grad=True)\n\n    def eval_loss():\n        return loss(net(features, w, b), labels).mean().item()\n\n    ls = [eval_loss()]\n    data_iter = torch.utils.data.DataLoader(\n        torch.utils.data.TensorDataset(features, labels), batch_size, shuffle=True)\n    \n    for _ in range(num_epochs):\n        start = time.time()\n        for batch_i, (X, y) in enumerate(data_iter):\n            l = loss(net(X, w, b), y).mean()  # 使用平均损失\n            \n            # 梯度清零\n            if w.grad is not None:\n                w.grad.data.zero_()\n                b.grad.data.zero_()\n                \n            l.backward()\n            optimizer_fn([w, b], states, hyperparams)  # 迭代模型参数\n            if (batch_i + 1) * batch_size % 100 == 0:\n                ls.append(eval_loss())  # 每100个样本记录下当前训练误差\n    # 打印结果和作图\n    print('loss: %f, %f sec per epoch' % (ls[-1], time.time() - start))\n    d2l.set_figsize()\n    d2l.plt.plot(np.linspace(0, num_epochs, len(ls)), ls)\n    d2l.plt.xlabel('epoch')\n    d2l.plt.ylabel('loss')","execution_count":19},{"cell_type":"code","metadata":{"graffitiCellId":"id_de54suf","scrolled":false,"jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"092890CC6C4941B38908FAA19887696A","collapsed":false},"outputs":[],"source":"def train_sgd(lr, batch_size, num_epochs=2):\n    train_ch7(sgd, None, {'lr': lr}, features, labels, batch_size, num_epochs)","execution_count":20},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_0juox2v","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"0F05E97E82964FF08131F509074DE433","mdEditEnable":false},"source":"对比"},{"cell_type":"code","metadata":{"graffitiCellId":"id_de54suf","scrolled":false,"jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"855ACC5ACA5A472F9BE384E668B84EB2","collapsed":false},"outputs":[{"output_type":"stream","text":"loss: 0.244373, 0.009881 sec per epoch\n","name":"stdout"},{"output_type":"display_data","metadata":{"needs_background":"light"},"data":{"text/plain":"<Figure size 252x180 with 1 Axes>","text/html":"<img src=\"https://cdn.kesci.com/rt_upload/855ACC5ACA5A472F9BE384E668B84EB2/q5oj6bxc03.svg\">"},"transient":{}}],"source":"train_sgd(1, 1500, 6)","execution_count":21},{"cell_type":"code","metadata":{"graffitiCellId":"id_d7d5ym4","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"223ADA73CFAB42EE87A9BFC9E6B709C2","collapsed":false,"scrolled":false},"outputs":[{"output_type":"stream","text":"loss: 0.245968, 0.463836 sec per epoch\n","name":"stdout"},{"output_type":"display_data","metadata":{"needs_background":"light"},"data":{"text/plain":"<Figure size 252x180 with 1 Axes>","text/html":"<img src=\"https://cdn.kesci.com/rt_upload/223ADA73CFAB42EE87A9BFC9E6B709C2/q5oj6ckl5j.svg\">"},"transient":{}}],"source":"train_sgd(0.005, 1)","execution_count":22},{"cell_type":"code","metadata":{"graffitiCellId":"id_g8kfyuu","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"BBA66FD2F352410F88CD4DB1FA37FCD7","collapsed":false,"scrolled":false},"outputs":[{"output_type":"stream","text":"loss: 0.243900, 0.065017 sec per epoch\n","name":"stdout"},{"output_type":"display_data","metadata":{"needs_background":"light"},"data":{"text/plain":"<Figure size 252x180 with 1 Axes>","text/html":"<img src=\"https://cdn.kesci.com/rt_upload/BBA66FD2F352410F88CD4DB1FA37FCD7/q5oj6cw1e3.svg\">"},"transient":{}}],"source":"train_sgd(0.05, 10)","execution_count":23},{"cell_type":"markdown","metadata":{"graffitiCellId":"id_majp1kb","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"AB1E52EA6CEB4124800BBD99B4F21553","mdEditEnable":false},"source":"## 简洁实现"},{"cell_type":"code","metadata":{"graffitiCellId":"id_adqg864","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"60C9FC5C629D49C9857BC163E384843A","collapsed":false,"scrolled":false},"outputs":[],"source":"# 本函数与原书不同的是这里第一个参数优化器函数而不是优化器的名字\n# 例如: optimizer_fn=torch.optim.SGD, optimizer_hyperparams={\"lr\": 0.05}\ndef train_pytorch_ch7(optimizer_fn, optimizer_hyperparams, features, labels,\n                    batch_size=10, num_epochs=2):\n    # 初始化模型\n    net = nn.Sequential(\n        nn.Linear(features.shape[-1], 1)\n    )\n    loss = nn.MSELoss()\n    optimizer = optimizer_fn(net.parameters(), **optimizer_hyperparams)\n\n    def eval_loss():\n        return loss(net(features).view(-1), labels).item() / 2\n\n    ls = [eval_loss()]\n    data_iter = torch.utils.data.DataLoader(\n        torch.utils.data.TensorDataset(features, labels), batch_size, shuffle=True)\n\n    for _ in range(num_epochs):\n        start = time.time()\n        for batch_i, (X, y) in enumerate(data_iter):\n            # 除以2是为了和train_ch7保持一致, 因为squared_loss中除了2\n            l = loss(net(X).view(-1), y) / 2 \n            \n            optimizer.zero_grad()\n            l.backward()\n            optimizer.step()\n            if (batch_i + 1) * batch_size % 100 == 0:\n                ls.append(eval_loss())\n    # 打印结果和作图\n    print('loss: %f, %f sec per epoch' % (ls[-1], time.time() - start))\n    d2l.set_figsize()\n    d2l.plt.plot(np.linspace(0, num_epochs, len(ls)), ls)\n    d2l.plt.xlabel('epoch')\n    d2l.plt.ylabel('loss')","execution_count":24},{"cell_type":"code","metadata":{"graffitiCellId":"id_c6ndach","jupyter":{},"tags":[],"slideshow":{"slide_type":"slide"},"id":"0CD81A787CAA435C9EF0F8ED1208FFC4","collapsed":false,"scrolled":false},"outputs":[{"output_type":"stream","text":"loss: 0.243770, 0.047664 sec per epoch\n","name":"stdout"},{"output_type":"display_data","metadata":{"needs_background":"light"},"data":{"text/plain":"<Figure size 252x180 with 1 Axes>","text/html":"<img src=\"https://cdn.kesci.com/rt_upload/0CD81A787CAA435C9EF0F8ED1208FFC4/q5oj6c1ds2.svg\">"},"transient":{}}],"source":"train_pytorch_ch7(optim.SGD, {\"lr\": 0.05}, features, labels, 10)","execution_count":25}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.7.3","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":2}