pytorch01.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pytorch01.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMJZUA94yhHgqevr+IFYbaM",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dlguswn3659/2020_Image_Lab/blob/master/pytorch01.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "443btBjOHpNB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from __future__ import print_function\n",
        "import torch"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-JvWj5YGIsRt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "1e2f1610-c463-4eea-ca8b-80da2071c958"
      },
      "source": [
        "x = torch.empty(5, 3)\n",
        "print(x)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[9.5786e-36, 0.0000e+00, 3.3631e-44],\n",
            "        [0.0000e+00,        nan, 0.0000e+00],\n",
            "        [1.1578e+27, 1.1362e+30, 7.1547e+22],\n",
            "        [4.5828e+30, 1.2121e+04, 7.1846e+22],\n",
            "        [9.2198e-39, 7.0374e+22, 0.0000e+00]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k5qcv14lJCfF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "ee4094b4-ce11-4c13-fa4c-c6966b965f0b"
      },
      "source": [
        "x = torch.rand(5, 3)\n",
        "print(x)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[0.8514, 0.3889, 0.4393],\n",
            "        [0.5985, 0.2058, 0.9969],\n",
            "        [0.4126, 0.1251, 0.8850],\n",
            "        [0.8253, 0.0971, 0.6880],\n",
            "        [0.5220, 0.4585, 0.4803]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w-k9ijjeJM6r",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "e4773b1d-02c2-4bf4-cb3d-eac4b7c0a41d"
      },
      "source": [
        "x = torch.zeros(5, 3, dtype=torch.long)\n",
        "print(x)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[0, 0, 0],\n",
            "        [0, 0, 0],\n",
            "        [0, 0, 0],\n",
            "        [0, 0, 0],\n",
            "        [0, 0, 0]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lst59HEDJU-s",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "254062d3-1043-452a-a9ce-f0c502577048"
      },
      "source": [
        "x = torch.tensor([5.5, 3])\n",
        "print(x)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([5.5000, 3.0000])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cUBHa7l4Jakl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        },
        "outputId": "0a7a5f5b-8d1b-4d49-b1eb-0b98b0ae0dd9"
      },
      "source": [
        "x = x.new_ones(5, 3, dtype=torch.double)\n",
        "print(x)\n",
        "\n",
        "x = torch.randn_like(x, dtype=torch.float)\n",
        "print(x)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[1., 1., 1.],\n",
            "        [1., 1., 1.],\n",
            "        [1., 1., 1.],\n",
            "        [1., 1., 1.],\n",
            "        [1., 1., 1.]], dtype=torch.float64)\n",
            "tensor([[-0.1705,  0.9976, -0.1905],\n",
            "        [ 0.7184, -0.4969,  0.3535],\n",
            "        [ 0.4331,  0.4513,  0.3819],\n",
            "        [-0.1865,  0.4812,  0.0924],\n",
            "        [-0.2184, -0.1768, -2.3957]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x23AquPaJxlN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "70335e1e-490b-49be-d0fa-f516c7586089"
      },
      "source": [
        "print(x.size())"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([5, 3])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vwdjaXnRJ7Kk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "bc2e835e-0d4e-4829-e773-6c5ae7ec587b"
      },
      "source": [
        "y = torch.rand(5, 3)\n",
        "print(x + y)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[ 0.4638,  1.4383,  0.5900],\n",
            "        [ 1.5766,  0.3434,  1.0760],\n",
            "        [ 1.2004,  1.2411,  0.9590],\n",
            "        [ 0.8025,  0.5829,  0.3277],\n",
            "        [-0.1449, -0.0080, -1.9296]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HhM2SmJFKCAC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "ed3e1264-d760-4138-d755-e0267cfc69ff"
      },
      "source": [
        "print(torch.add(x, y))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[ 0.4638,  1.4383,  0.5900],\n",
            "        [ 1.5766,  0.3434,  1.0760],\n",
            "        [ 1.2004,  1.2411,  0.9590],\n",
            "        [ 0.8025,  0.5829,  0.3277],\n",
            "        [-0.1449, -0.0080, -1.9296]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gm65-7l4KHmF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "892cd7b8-e94e-4ab3-db85-a8d757f63afb"
      },
      "source": [
        "result = torch.empty(5, 3)\n",
        "torch.add(x, y, out=result)\n",
        "print(result)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[ 0.4638,  1.4383,  0.5900],\n",
            "        [ 1.5766,  0.3434,  1.0760],\n",
            "        [ 1.2004,  1.2411,  0.9590],\n",
            "        [ 0.8025,  0.5829,  0.3277],\n",
            "        [-0.1449, -0.0080, -1.9296]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uXqlEzFWKTbT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "79da4d33-2bca-45ee-ffdb-cfe329d866e9"
      },
      "source": [
        "y.add_(x)\n",
        "print(y)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[ 0.4638,  1.4383,  0.5900],\n",
            "        [ 1.5766,  0.3434,  1.0760],\n",
            "        [ 1.2004,  1.2411,  0.9590],\n",
            "        [ 0.8025,  0.5829,  0.3277],\n",
            "        [-0.1449, -0.0080, -1.9296]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zNCTGCV4XWeY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "a86943bb-6d7d-44a2-bdcb-309c2c9907c2"
      },
      "source": [
        "print(x[:, 1])"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([ 0.9976, -0.4969,  0.4513,  0.4812, -0.1768])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CRGZI8gSXeVn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "bd35ec68-1863-4a08-f884-24bcfc977bdc"
      },
      "source": [
        "x = torch.randn(4, 4)\n",
        "y = x.view(16)\n",
        "z = x.view(-1, 8)\n",
        "print(x.size(), y.size(), z.size())"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([4, 4]) torch.Size([16]) torch.Size([2, 8])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FKmEwDh2XyNt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "d59a25f5-1c45-49b4-e998-56fd39de88f1"
      },
      "source": [
        "x = torch.randn(1)\n",
        "print(x)\n",
        "print(x.item())"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([2.4096])\n",
            "2.4095749855041504\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3QCPj3VvX8Zv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "3088d671-a54e-48e2-da6e-5691741fbbcf"
      },
      "source": [
        "a = torch.ones(5)\n",
        "print(a)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([1., 1., 1., 1., 1.])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FbOWIwH8YZTE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "0a92ba04-88f0-45f2-d4c9-d94c0f2135e4"
      },
      "source": [
        "import numpy as np\n",
        "a = np.ones(5)\n",
        "b = torch.from_numpy(a)\n",
        "np.add(a, 1, out=a)\n",
        "print(a)\n",
        "print(b)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[2. 2. 2. 2. 2.]\n",
            "tensor([2., 2., 2., 2., 2.], dtype=torch.float64)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LYUlkeCOYyWN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "e5f6e018-084c-4235-fd6f-ee3107745c37"
      },
      "source": [
        "print(torch.cuda.is_available())\n",
        "if torch.cuda.is_available():\n",
        "  device = torch.device(\"cuda\")\n",
        "  y = torch.ones_like(x, device=device)\n",
        "  x = x.to(device)\n",
        "  z = x + y\n",
        "  print(z)\n",
        "  print(z.to(\"cpu\", torch.double))"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "False\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ynx0wbLgaXX5",
        "colab_type": "text"
      },
      "source": [
        "# **오토 그라드 : 자동 미분**\n",
        "PyTorch의 모든 신경망의 중심은 autograd패키지이다. 먼저 이것을 간단히 방문해 보자. 그러면 우리는 첫 신경망을 훈련시키는 것으로 간다.\n",
        "\n",
        "이 autograd패키지는 Tensor의 모든 작업에 대한 자동 차별화를 제공한다. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4rmKaUSNarEv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        },
        "outputId": "8f8e2352-a907-4b32-c453-5758f80a3eb2"
      },
      "source": [
        "import torch\n",
        "\n",
        "x = torch.ones(2, 2, requires_grad=True)\n",
        "print(x)\n",
        "\n",
        "y = x + 2\n",
        "print(y)\n",
        "\n",
        "print(y.grad_fn)\n",
        "\n",
        "z = y*y*3\n",
        "out = z.mean()\n",
        "\n",
        "print(z, out)\n",
        "\n",
        "a = torch.randn(2, 2)\n",
        "a = ((a*3)/(a-1))\n",
        "print(a.requires_grad)\n",
        "a.requires_grad_(True)\n",
        "print(a.requires_grad)\n",
        "b = (a*a).sum()\n",
        "print(b.grad_fn)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[1., 1.],\n",
            "        [1., 1.]], requires_grad=True)\n",
            "tensor([[3., 3.],\n",
            "        [3., 3.]], grad_fn=<AddBackward0>)\n",
            "<AddBackward0 object at 0x7f67a1e05e48>\n",
            "tensor([[27., 27.],\n",
            "        [27., 27.]], grad_fn=<MulBackward0>) tensor(27., grad_fn=<MeanBackward0>)\n",
            "False\n",
            "True\n",
            "<SumBackward0 object at 0x7f67a1e05e48>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yFqPIxdvYRtx",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "285f7ff2-1ba4-4b11-d8ce-0073da6238fe"
      },
      "source": [
        "# print(x)\n",
        "# print(out)\n",
        "\n",
        "out.backward()\n",
        "print(x.grad)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[4.5000, 4.5000],\n",
            "        [4.5000, 4.5000]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IWzdyspDd8d9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "9436ab92-0792-4c5d-bacd-3d051f1deac9"
      },
      "source": [
        "x = torch.randn(3, requires_grad=True)\n",
        "\n",
        "y = x*2\n",
        "while y.data.norm() < 1000:\n",
        "  y = y*2\n",
        "\n",
        "print(y)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([ 588.0714,   71.6271, 1181.8999], grad_fn=<MulBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K2U4BVkeecrw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "0f477681-dea2-4127-a067-bee9256f04ad"
      },
      "source": [
        "v = torch.tensor([0.1, 1.0, 0.0001], dtype=torch.float)\n",
        "y.backward(v)\n",
        "\n",
        "print(x.grad)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([5.1200e+01, 5.1200e+02, 5.1200e-02])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wi_X8PO1evJI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        },
        "outputId": "309f3ea0-d6e6-4608-cca9-9dd4d672a555"
      },
      "source": [
        "print(x.requires_grad)\n",
        "print((x**2).requires_grad)\n",
        "\n",
        "with torch.no_grad():\n",
        "  print((x ** 2).requires_grad)\n",
        "\n",
        "print(x.requires_grad)\n",
        "y = x.detach()\n",
        "print(y.requires_grad)\n",
        "print(x.eq(y).all())"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "True\n",
            "True\n",
            "False\n",
            "True\n",
            "False\n",
            "tensor(True)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VsNMRJEtqmf3",
        "colab_type": "text"
      },
      "source": [
        "# **예제로 PYTORCH 학습**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5rzb70W3qqFx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "# N is batch size; D_in is input dimension;\n",
        "# H is hidden dimension; D_out is output dimension.\n",
        "N, D_in, H, D_out = 64, 1000, 100, 10\n",
        "\n",
        "# Create random input and output data\n",
        "x = np.random.randn(N, D_in)\n",
        "y = np.random.randn(N, D_out)\n",
        "\n",
        "# Randomly initialize weights\n",
        "w1 = np.random.randn(D_in, H)\n",
        "w2 = np.random.randn(H, D_out)\n",
        "\n",
        "learning_rate = 1e-6\n",
        "for t in range(500):\n",
        "    # Forward pass: compute predicted y\n",
        "    h = x.dot(w1)\n",
        "    h_relu = np.maximum(h, 0)\n",
        "    y_pred = h_relu.dot(w2)\n",
        "\n",
        "    # Compute and print loss\n",
        "    loss = np.square(y_pred - y).sum()\n",
        "    print(t, loss)\n",
        "\n",
        "    # Backprop to compute gradients of w1 and w2 with respect to loss\n",
        "    grad_y_pred = 2.0 * (y_pred - y)\n",
        "    grad_w2 = h_relu.T.dot(grad_y_pred)\n",
        "    grad_h_relu = grad_y_pred.dot(w2.T)\n",
        "    grad_h = grad_h_relu.copy()\n",
        "    grad_h[h < 0] = 0\n",
        "    grad_w1 = x.T.dot(grad_h)\n",
        "\n",
        "    # Update weights\n",
        "    w1 -= learning_rate * grad_w1\n",
        "    w2 -= learning_rate * grad_w2"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IZar1ZbfJ45f",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "97a99350-0905-4b33-e118-78f8cff2e990"
      },
      "source": [
        "\n",
        "import torch\n",
        "\n",
        "\n",
        "dtype = torch.float\n",
        "device = torch.device(\"cpu\")\n",
        "# device = torch.device(\"cuda:0\") # Uncomment this to run on GPU\n",
        "\n",
        "# N is batch size; D_in is input dimension;\n",
        "# H is hidden dimension; D_out is output dimension.\n",
        "N, D_in, H, D_out = 64, 1000, 100, 10\n",
        "\n",
        "# Create random input and output data\n",
        "x = torch.randn(N, D_in, device=device, dtype=dtype)\n",
        "y = torch.randn(N, D_out, device=device, dtype=dtype)\n",
        "\n",
        "# Randomly initialize weights\n",
        "w1 = torch.randn(D_in, H, device=device, dtype=dtype)\n",
        "w2 = torch.randn(H, D_out, device=device, dtype=dtype)\n",
        "\n",
        "learning_rate = 1e-6\n",
        "for t in range(500):\n",
        "    # Forward pass: compute predicted y\n",
        "    h = x.mm(w1)\n",
        "    h_relu = h.clamp(min=0)\n",
        "    y_pred = h_relu.mm(w2)\n",
        "\n",
        "    # Compute and print loss\n",
        "    loss = (y_pred - y).pow(2).sum().item()\n",
        "    if t % 100 == 99:\n",
        "        print(t, loss)\n",
        "\n",
        "    # Backprop to compute gradients of w1 and w2 with respect to loss\n",
        "    grad_y_pred = 2.0 * (y_pred - y)\n",
        "    grad_w2 = h_relu.t().mm(grad_y_pred)\n",
        "    grad_h_relu = grad_y_pred.mm(w2.t())\n",
        "    grad_h = grad_h_relu.clone()\n",
        "    grad_h[h < 0] = 0\n",
        "    grad_w1 = x.t().mm(grad_h)\n",
        "\n",
        "    # Update weights using gradient descent\n",
        "    w1 -= learning_rate * grad_w1\n",
        "    w2 -= learning_rate * grad_w2"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "99 618.5730590820312\n",
            "199 4.1626763343811035\n",
            "299 0.04314004257321358\n",
            "399 0.0007671025232411921\n",
            "499 7.498993363697082e-05\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3l8f3B-DRpG4",
        "colab_type": "text"
      },
      "source": [
        "# **오토 그라드**\n",
        "## **PyTorch : 텐서 및 오토 그라드**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XMBHkOYARvs7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "9f4e560f-18d8-4673-d9f9-60b32a49d16f"
      },
      "source": [
        "import torch\n",
        "\n",
        "dtype = torch.float\n",
        "device = torch.device(\"cpu\")\n",
        "# device = torch.device(\"cuda:0\") # Uncomment this to run on GPU\n",
        "\n",
        "# N is batch size; D_in is input dimension;\n",
        "# H is hidden dimension; D_out is output dimension.\n",
        "N, D_in, H, D_out = 64, 1000, 100, 10\n",
        "\n",
        "# Create random Tensors to hold input and outputs.\n",
        "# Setting requires_grad=False indicates that we do not need to compute gradients\n",
        "# with respect to these Tensors during the backward pass.\n",
        "x = torch.randn(N, D_in, device=device, dtype=dtype)\n",
        "y = torch.randn(N, D_out, device=device, dtype=dtype)\n",
        "\n",
        "# Create random Tensors for weights.\n",
        "# Setting requires_grad=True indicates that we want to compute gradients with\n",
        "# respect to these Tensors during the backward pass.\n",
        "w1 = torch.randn(D_in, H, device=device, dtype=dtype, requires_grad=True)\n",
        "w2 = torch.randn(H, D_out, device=device, dtype=dtype, requires_grad=True)\n",
        "\n",
        "learning_rate = 1e-6\n",
        "for t in range(500):\n",
        "    # Forward pass: compute predicted y using operations on Tensors; these\n",
        "    # are exactly the same operations we used to compute the forward pass using\n",
        "    # Tensors, but we do not need to keep references to intermediate values since\n",
        "    # we are not implementing the backward pass by hand.\n",
        "    y_pred = x.mm(w1).clamp(min=0).mm(w2)\n",
        "\n",
        "    # Compute and print loss using operations on Tensors.\n",
        "    # Now loss is a Tensor of shape (1,)\n",
        "    # loss.item() gets the scalar value held in the loss.\n",
        "    loss = (y_pred - y).pow(2).sum()\n",
        "    if t % 100 == 99:\n",
        "        print(t, loss.item())\n",
        "\n",
        "    # Use autograd to compute the backward pass. This call will compute the\n",
        "    # gradient of loss with respect to all Tensors with requires_grad=True.\n",
        "    # After this call w1.grad and w2.grad will be Tensors holding the gradient\n",
        "    # of the loss with respect to w1 and w2 respectively.\n",
        "    loss.backward()\n",
        "\n",
        "    # Manually update weights using gradient descent. Wrap in torch.no_grad()\n",
        "    # because weights have requires_grad=True, but we don't need to track this\n",
        "    # in autograd.\n",
        "    # An alternative way is to operate on weight.data and weight.grad.data.\n",
        "    # Recall that tensor.data gives a tensor that shares the storage with\n",
        "    # tensor, but doesn't track history.\n",
        "    # You can also use torch.optim.SGD to achieve this.\n",
        "    with torch.no_grad():\n",
        "        w1 -= learning_rate * w1.grad\n",
        "        w2 -= learning_rate * w2.grad\n",
        "\n",
        "        # Manually zero the gradients after updating weights\n",
        "        w1.grad.zero_()\n",
        "        w2.grad.zero_()"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "99 364.3855285644531\n",
            "199 1.231141448020935\n",
            "299 0.008226457051932812\n",
            "399 0.000243069400312379\n",
            "499 4.9274993216386065e-05\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}