TabularDataVisualizationUsingPandaProfiler.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TabularDataVisualizationUsingPandaProfiler.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPEpvOpPD/DNf+mBbb1DSZt",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ShaswataJash/kfpcomponent/blob/main/TabularDataVisualizationUsingPandaProfiler.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "This notebook is the development workflow for kubeflow pipeline component of the same name as this notebook. Refer https://github.com/ShaswataJash/kfpcomponent"
      ],
      "metadata": {
        "id": "_fMb4fVRFsPj"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Install required softwares"
      ],
      "metadata": {
        "id": "HYI5LKdFLCTN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!uname -a"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JoSae8JMvVgq",
        "outputId": "1d4150f4-92c7-4c70-c172-78d908f59544"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Linux e751bd1d8376 5.4.188+ #1 SMP Sun Apr 24 10:03:06 PDT 2022 x86_64 x86_64 x86_64 GNU/Linux\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!lsb_release -a"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZZTBdke9vme4",
        "outputId": "04a1e634-b2cf-4156-b051-258c37324f49"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "No LSB modules are available.\n",
            "Distributor ID:\tUbuntu\n",
            "Description:\tUbuntu 18.04.5 LTS\n",
            "Release:\t18.04\n",
            "Codename:\tbionic\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!python --version"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P4orJiv6orBy",
        "outputId": "39fd0605-e2bd-447c-f213-9f77da0a14ca"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Python 3.7.13\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lus4TEd-8DbB",
        "outputId": "f5a36e04-334c-494e-df49-82ec33b14e84"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pandas-profiling==3.2.0\n",
            "  Downloading pandas_profiling-3.2.0-py2.py3-none-any.whl (262 kB)\n",
            "\u001b[K     |████████████████████████████████| 262 kB 5.1 MB/s \n",
            "\u001b[?25hCollecting htmlmin>=0.1.12\n",
            "  Downloading htmlmin-0.1.12.tar.gz (19 kB)\n",
            "Collecting pydantic>=1.8.1\n",
            "  Downloading pydantic-1.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 11.1 MB 24.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: joblib~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from pandas-profiling==3.2.0) (1.1.0)\n",
            "Requirement already satisfied: scipy>=1.4.1 in /usr/local/lib/python3.7/dist-packages (from pandas-profiling==3.2.0) (1.4.1)\n",
            "Collecting visions[type_image_path]==0.7.4\n",
            "  Downloading visions-0.7.4-py3-none-any.whl (102 kB)\n",
            "\u001b[K     |████████████████████████████████| 102 kB 7.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: missingno>=0.4.2 in /usr/local/lib/python3.7/dist-packages (from pandas-profiling==3.2.0) (0.5.1)\n",
            "Collecting tangled-up-in-unicode==0.2.0\n",
            "  Downloading tangled_up_in_unicode-0.2.0-py3-none-any.whl (4.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.7 MB 38.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: jinja2>=2.11.1 in /usr/local/lib/python3.7/dist-packages (from pandas-profiling==3.2.0) (2.11.3)\n",
            "Collecting requests>=2.24.0\n",
            "  Downloading requests-2.28.0-py3-none-any.whl (62 kB)\n",
            "\u001b[K     |████████████████████████████████| 62 kB 1.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.48.2 in /usr/local/lib/python3.7/dist-packages (from pandas-profiling==3.2.0) (4.64.0)\n",
            "Collecting PyYAML>=5.0.0\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 44.2 MB/s \n",
            "\u001b[?25hCollecting phik>=0.11.1\n",
            "  Downloading phik-0.12.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (690 kB)\n",
            "\u001b[K     |████████████████████████████████| 690 kB 48.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas!=1.0.0,!=1.0.1,!=1.0.2,!=1.1.0,>=0.25.3 in /usr/local/lib/python3.7/dist-packages (from pandas-profiling==3.2.0) (1.3.5)\n",
            "Requirement already satisfied: numpy>=1.16.0 in /usr/local/lib/python3.7/dist-packages (from pandas-profiling==3.2.0) (1.21.6)\n",
            "Requirement already satisfied: matplotlib>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from pandas-profiling==3.2.0) (3.2.2)\n",
            "Requirement already satisfied: seaborn>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from pandas-profiling==3.2.0) (0.11.2)\n",
            "Collecting multimethod>=1.4\n",
            "  Downloading multimethod-1.8-py3-none-any.whl (9.8 kB)\n",
            "Collecting markupsafe~=2.1.1\n",
            "  Downloading MarkupSafe-2.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (25 kB)\n",
            "Requirement already satisfied: networkx>=2.4 in /usr/local/lib/python3.7/dist-packages (from visions[type_image_path]==0.7.4->pandas-profiling==3.2.0) (2.6.3)\n",
            "Requirement already satisfied: attrs>=19.3.0 in /usr/local/lib/python3.7/dist-packages (from visions[type_image_path]==0.7.4->pandas-profiling==3.2.0) (21.4.0)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.7/dist-packages (from visions[type_image_path]==0.7.4->pandas-profiling==3.2.0) (7.1.2)\n",
            "Collecting imagehash\n",
            "  Downloading ImageHash-4.2.1.tar.gz (812 kB)\n",
            "\u001b[K     |████████████████████████████████| 812 kB 54.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.2.0->pandas-profiling==3.2.0) (0.11.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.2.0->pandas-profiling==3.2.0) (2.8.2)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.2.0->pandas-profiling==3.2.0) (3.0.9)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.2.0->pandas-profiling==3.2.0) (1.4.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from kiwisolver>=1.0.1->matplotlib>=3.2.0->pandas-profiling==3.2.0) (4.2.0)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas!=1.0.0,!=1.0.1,!=1.0.2,!=1.1.0,>=0.25.3->pandas-profiling==3.2.0) (2022.1)\n",
            "Collecting scipy>=1.4.1\n",
            "  Downloading scipy-1.7.3-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (38.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 38.1 MB 1.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.1->matplotlib>=3.2.0->pandas-profiling==3.2.0) (1.15.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.24.0->pandas-profiling==3.2.0) (1.24.3)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.7/dist-packages (from requests>=2.24.0->pandas-profiling==3.2.0) (2.0.12)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.24.0->pandas-profiling==3.2.0) (2022.5.18.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.24.0->pandas-profiling==3.2.0) (2.10)\n",
            "Requirement already satisfied: PyWavelets in /usr/local/lib/python3.7/dist-packages (from imagehash->visions[type_image_path]==0.7.4->pandas-profiling==3.2.0) (1.3.0)\n",
            "Building wheels for collected packages: htmlmin, imagehash\n",
            "  Building wheel for htmlmin (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for htmlmin: filename=htmlmin-0.1.12-py3-none-any.whl size=27098 sha256=d846de11057b142347a139a8928b1cd56d02107d11d170fc8dbea29d45a7d273\n",
            "  Stored in directory: /root/.cache/pip/wheels/70/e1/52/5b14d250ba868768823940c3229e9950d201a26d0bd3ee8655\n",
            "  Building wheel for imagehash (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for imagehash: filename=ImageHash-4.2.1-py2.py3-none-any.whl size=295206 sha256=224de9754ec7b72a75b70d935101579054d79b23cd86c2e3620d7ed68d0d4b2f\n",
            "  Stored in directory: /root/.cache/pip/wheels/4c/d5/59/5e3e297533ddb09407769762985d134135064c6831e29a914e\n",
            "Successfully built htmlmin imagehash\n",
            "Installing collected packages: tangled-up-in-unicode, scipy, multimethod, visions, markupsafe, imagehash, requests, PyYAML, pydantic, phik, htmlmin, pandas-profiling\n",
            "  Attempting uninstall: scipy\n",
            "    Found existing installation: scipy 1.4.1\n",
            "    Uninstalling scipy-1.4.1:\n",
            "      Successfully uninstalled scipy-1.4.1\n",
            "  Attempting uninstall: markupsafe\n",
            "    Found existing installation: MarkupSafe 2.0.1\n",
            "    Uninstalling MarkupSafe-2.0.1:\n",
            "      Successfully uninstalled MarkupSafe-2.0.1\n",
            "  Attempting uninstall: requests\n",
            "    Found existing installation: requests 2.23.0\n",
            "    Uninstalling requests-2.23.0:\n",
            "      Successfully uninstalled requests-2.23.0\n",
            "  Attempting uninstall: PyYAML\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "  Attempting uninstall: pandas-profiling\n",
            "    Found existing installation: pandas-profiling 1.4.1\n",
            "    Uninstalling pandas-profiling-1.4.1:\n",
            "      Successfully uninstalled pandas-profiling-1.4.1\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "google-colab 1.0.0 requires requests~=2.23.0, but you have requests 2.28.0 which is incompatible.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\n",
            "albumentations 0.1.12 requires imgaug<0.2.7,>=0.2.5, but you have imgaug 0.2.9 which is incompatible.\u001b[0m\n",
            "Successfully installed PyYAML-6.0 htmlmin-0.1.12 imagehash-4.2.1 markupsafe-2.1.1 multimethod-1.8 pandas-profiling-3.2.0 phik-0.12.2 pydantic-1.9.1 requests-2.28.0 scipy-1.7.3 tangled-up-in-unicode-0.2.0 visions-0.7.4\n"
          ]
        }
      ],
      "source": [
        "!pip install pandas-profiling==3.2.0"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install markupsafe==2.0.1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p94sNSClI-h-",
        "outputId": "176af3d0-8d46-41ff-b106-1d07670edced"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting markupsafe==2.0.1\n",
            "  Downloading MarkupSafe-2.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (31 kB)\n",
            "Installing collected packages: markupsafe\n",
            "  Attempting uninstall: markupsafe\n",
            "    Found existing installation: MarkupSafe 2.1.1\n",
            "    Uninstalling MarkupSafe-2.1.1:\n",
            "      Successfully uninstalled MarkupSafe-2.1.1\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "pandas-profiling 3.2.0 requires markupsafe~=2.1.1, but you have markupsafe 2.0.1 which is incompatible.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "Successfully installed markupsafe-2.0.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#needed for test validation\n",
        "!pip install beautifulsoup4==4.11.1"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d8mzfUIu2bbP",
        "outputId": "94b5b134-0a04-4a74-ecf3-c1a83f187480"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting beautifulsoup4==4.11.1\n",
            "  Downloading beautifulsoup4-4.11.1-py3-none-any.whl (128 kB)\n",
            "\u001b[K     |████████████████████████████████| 128 kB 5.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.7/dist-packages (from beautifulsoup4==4.11.1) (2.3.2.post1)\n",
            "Installing collected packages: beautifulsoup4\n",
            "  Attempting uninstall: beautifulsoup4\n",
            "    Found existing installation: beautifulsoup4 4.6.3\n",
            "    Uninstalling beautifulsoup4-4.6.3:\n",
            "      Successfully uninstalled beautifulsoup4-4.6.3\n",
            "Successfully installed beautifulsoup4-4.11.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!apt-get install ca-certificates fuse tzdata curl unzip && \\\n",
        "  echo \"user_allow_other\" >> /etc/fuse.conf"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3p89zTHfYqwc",
        "outputId": "eeb57eec-3866-4929-cdd5-ea5f2ac8fcaa"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "fuse is already the newest version (2.9.7-1ubuntu1).\n",
            "ca-certificates is already the newest version (20210119~18.04.2).\n",
            "curl is already the newest version (7.58.0-2ubuntu3.18).\n",
            "tzdata is already the newest version (2022a-0ubuntu0.18.04).\n",
            "tzdata set to manually installed.\n",
            "unzip is already the newest version (6.0-21ubuntu1.1).\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-460\n",
            "Use 'apt autoremove' to remove it.\n",
            "0 upgraded, 0 newly installed, 0 to remove and 45 not upgraded.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!curl https://rclone.org/install.sh | bash"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YGe2hZo0U7DL",
        "outputId": "036df565-943b-44f4-f01e-a8d0b465695f"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100  4497  100  4497    0     0   5714      0 --:--:-- --:--:-- --:--:--  5706\n",
            "Archive:  rclone-current-linux-amd64.zip\n",
            "   creating: tmp_unzip_dir_for_rclone/rclone-v1.58.1-linux-amd64/\n",
            "  inflating: tmp_unzip_dir_for_rclone/rclone-v1.58.1-linux-amd64/rclone  [binary]\n",
            "  inflating: tmp_unzip_dir_for_rclone/rclone-v1.58.1-linux-amd64/README.html  [text]  \n",
            "  inflating: tmp_unzip_dir_for_rclone/rclone-v1.58.1-linux-amd64/README.txt  [text]  \n",
            "  inflating: tmp_unzip_dir_for_rclone/rclone-v1.58.1-linux-amd64/git-log.txt  [text]  \n",
            "  inflating: tmp_unzip_dir_for_rclone/rclone-v1.58.1-linux-amd64/rclone.1  [text]  \n",
            "Purging old database entries in /usr/share/man...\n",
            "Processing manual pages under /usr/share/man...\n",
            "Purging old database entries in /usr/share/man/cs...\n",
            "Processing manual pages under /usr/share/man/cs...\n",
            "Purging old database entries in /usr/share/man/zh_CN...\n",
            "Processing manual pages under /usr/share/man/zh_CN...\n",
            "Purging old database entries in /usr/share/man/fr...\n",
            "Processing manual pages under /usr/share/man/fr...\n",
            "Purging old database entries in /usr/share/man/da...\n",
            "Processing manual pages under /usr/share/man/da...\n",
            "Purging old database entries in /usr/share/man/ko...\n",
            "Processing manual pages under /usr/share/man/ko...\n",
            "Purging old database entries in /usr/share/man/ja...\n",
            "Processing manual pages under /usr/share/man/ja...\n",
            "Purging old database entries in /usr/share/man/nl...\n",
            "Processing manual pages under /usr/share/man/nl...\n",
            "Purging old database entries in /usr/share/man/zh_TW...\n",
            "Processing manual pages under /usr/share/man/zh_TW...\n",
            "Purging old database entries in /usr/share/man/pl...\n",
            "Processing manual pages under /usr/share/man/pl...\n",
            "Purging old database entries in /usr/share/man/ru...\n",
            "Processing manual pages under /usr/share/man/ru...\n",
            "Purging old database entries in /usr/share/man/it...\n",
            "Processing manual pages under /usr/share/man/it...\n",
            "Purging old database entries in /usr/share/man/de...\n",
            "Processing manual pages under /usr/share/man/de...\n",
            "Purging old database entries in /usr/share/man/sv...\n",
            "Processing manual pages under /usr/share/man/sv...\n",
            "Purging old database entries in /usr/share/man/tr...\n",
            "Processing manual pages under /usr/share/man/tr...\n",
            "Purging old database entries in /usr/share/man/id...\n",
            "Processing manual pages under /usr/share/man/id...\n",
            "Purging old database entries in /usr/share/man/hu...\n",
            "Processing manual pages under /usr/share/man/hu...\n",
            "Purging old database entries in /usr/share/man/fi...\n",
            "Processing manual pages under /usr/share/man/fi...\n",
            "Purging old database entries in /usr/share/man/es...\n",
            "Processing manual pages under /usr/share/man/es...\n",
            "Purging old database entries in /usr/share/man/pt...\n",
            "Processing manual pages under /usr/share/man/pt...\n",
            "Purging old database entries in /usr/share/man/pt_BR...\n",
            "Processing manual pages under /usr/share/man/pt_BR...\n",
            "Purging old database entries in /usr/share/man/sr...\n",
            "Processing manual pages under /usr/share/man/sr...\n",
            "Processing manual pages under /usr/local/man...\n",
            "Updating index cache for path `/usr/local/man/man1'. Wait...done.\n",
            "Checking for stray cats under /usr/local/man...\n",
            "Checking for stray cats under /var/cache/man/oldlocal...\n",
            "1 man subdirectory contained newer manual pages.\n",
            "3 manual pages were added.\n",
            "0 stray cats were added.\n",
            "19 old database entries were purged.\n",
            "\n",
            "rclone v1.58.1 has successfully installed.\n",
            "Now run \"rclone config\" for setup. Check https://rclone.org/docs/ for more details.\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!rclone --version"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Byk9g7UWUtJO",
        "outputId": "c1d9edc5-41f4-45b7-a109-9a53e8af7c2a"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "rclone v1.58.1\n",
            "- os/version: ubuntu 18.04 (64 bit)\n",
            "- os/kernel: 5.4.188+ (x86_64)\n",
            "- os/type: linux\n",
            "- os/arch: amd64\n",
            "- go/version: go1.17.9\n",
            "- go/linking: static\n",
            "- go/tags: none\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Develop source code files"
      ],
      "metadata": {
        "id": "EMVyzkX_LIoG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile data_visualization.py\n",
        "#!/usr/bin/env python3\n",
        "import argparse\n",
        "\n",
        "# Defining and parsing the command-line arguments\n",
        "parser = argparse.ArgumentParser(description='kubeflow pipeline component to read csv file and visualize the data')\n",
        "parser.add_argument('--input-datasource-directory-mountable', default=False, action=\"store_true\", help='whether input csv file is present in mountable remote location')\n",
        "parser.add_argument('--input-datasource-directory-to-be-mounted', type=str, help='if input-datasource-directory-mountable=True, name of the mountable directory (e.g. bucket name for s3)')\n",
        "parser.add_argument('--input-datasource-file-name', type=str, help='name of the csv file including file extension (if any)')\n",
        "parser.add_argument('--additional-options-csv-parsing', type=str, default= '{}', help='json formatted key-value pairs of strings which will be passed to pandas.read_csv()')\n",
        "parser.add_argument('--panda-profiler-options', type=str, default= '{}', help='json formatted key-value pairs of strings which will be passed to ProfileReport()')\n",
        "parser.add_argument('--sensitive-data-present', default=False, action=\"store_true\", help='whether some sensitive data present (if so, panda-profiler will not show sample data etc.)')\n",
        "parser.add_argument('--output-visualization-absolute-path', type=str, help='html filepath having visualization report')\n",
        "args = parser.parse_args()\n",
        "\n",
        "import tempfile\n",
        "local_datastore_read_dir = tempfile.mkdtemp(prefix=\"my_local_read-\")\n",
        "print('local_datastore_read_dir:',local_datastore_read_dir)\n",
        "\n",
        "#input file handling\n",
        "import subprocess\n",
        "import os\n",
        "import sys\n",
        "if args.input_datasource_directory_mountable:\n",
        "    input_data_read_cmd = \"rclone -v mount remoteread:\" + args.input_datasource_directory_to_be_mounted + ' ' + local_datastore_read_dir + ' --daemon'\n",
        "else:\n",
        "    input_data_read_cmd = \"rclone -v copy remoteread:\" + args.input_datasource_file_name + ' ' + local_datastore_read_dir\n",
        "input_data_read_call = subprocess.run(input_data_read_cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)\n",
        "print(input_data_read_call.stdout)\n",
        "if input_data_read_call.returncode != 0:\n",
        "    print(\"Error in rclone, errorcode=\", input_data_read_call.returncode)\n",
        "    sys.stdout.flush()\n",
        "    sys.exit(\"Forceful exit as rclone returned error in context of reading\")\n",
        "\n",
        "#handling input csv file reading\n",
        "import pandas\n",
        "import json\n",
        "\n",
        "try:\n",
        "    parse_config = json.loads(args.additional_options_csv_parsing)\n",
        "    print('parse_config = (', type(parse_config), ')', parse_config)\n",
        "    parse_config['filepath_or_buffer'] = os.path.join(local_datastore_read_dir,args.input_datasource_file_name)\n",
        "    my_data = pandas.read_csv(**parse_config)\n",
        "    print(my_data)\n",
        "    \n",
        "except BaseException as err:\n",
        "    print(\"Error=\", err, ' ', type(err))\n",
        "    sys.stdout.flush()\n",
        "    sys.exit(\"Forceful exit as exception encountered while parsing input csv file\")\n",
        "\n",
        "#generating output html report\n",
        "from pandas_profiling import ProfileReport\n",
        "try: \n",
        "    profile_config = json.loads(args.panda_profiler_options)\n",
        "    print('profile_config = (', type(profile_config), ')', profile_config)\n",
        "    profile_config['progress_bar'] = False\n",
        "    profile = ProfileReport(my_data, **profile_config)\n",
        "    profile.to_file(args.output_visualization_absolute_path)\n",
        "except BaseException as err:\n",
        "    print(\"Error=\", err, ' ', type(err))\n",
        "    sys.stdout.flush()\n",
        "    sys.exit(\"Forceful exit as exception encountered while profiling the data\")\n",
        "\n",
        "#dumping kubeflow pipeline compatible meta data for visualization\n",
        "#refer: https://www.kubeflow.org/docs/components/pipelines/sdk/output-viewer/#web-app\n",
        "\n",
        "with open(args.output_visualization_absolute_path, 'r') as html_rep:\n",
        "    html_content = html_rep.read()\n",
        "\n",
        "metadata = {\n",
        "    'outputs' : [{\n",
        "      'type': 'web-app',\n",
        "      'storage': 'inline',\n",
        "      'source': html_content,\n",
        "    }]\n",
        "}\n",
        "with open('/tmp/mlpipeline-ui-metadata.json', 'w') as f:\n",
        "    json.dump(metadata, f)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WU9Vh7P4P4_h",
        "outputId": "c0e772ae-f5ef-4322-d945-4c1654ae35b6"
      },
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Overwriting data_visualization.py\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Docker size reduction tips:\n",
        "\n",
        "\n",
        "*   https://devopscube.com/reduce-docker-image-size/\n",
        "*   https://www.ecloudcontrol.com/best-practices-to-reduce-docker-images-size/\n",
        "\n"
      ],
      "metadata": {
        "id": "eQXf07hoPzxM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile Dockerfile\n",
        "FROM python:3.7.13-slim\n",
        "\n",
        "RUN python3 -m pip install pandas-profiling==3.2.0\n",
        "RUN python3 -m pip install markupsafe==2.0.1\n",
        "RUN python3 -m pip install beautifulsoup4==4.11.1\n",
        "\n",
        "#install fuse as dependency for rclone. Additionally, install curl, unzip for rclone installer to work\n",
        "RUN apt-get update \\\n",
        "    && apt-get install --no-install-recommends -y curl fuse unzip \\\n",
        "    && echo \"user_allow_other\" >> /etc/fuse.conf \\\n",
        "    && curl https://rclone.org/install.sh | bash \\\n",
        "    && apt-get -y remove --purge curl unzip \\\n",
        "    && rm -rf /var/lib/apt/lists/* \\\n",
        "    && rclone --version\n",
        "\n",
        "COPY src/data_visualization.py /tmp\n",
        "COPY tests/test_validation.py /tmp\n",
        "COPY run_tests.sh /tmp\n",
        "RUN chmod 544 /tmp/run_tests.sh"
      ],
      "metadata": {
        "id": "hk00hb780Qo_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c66e6127-b33e-4e44-d022-040ece0b044c"
      },
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Writing Dockerfile\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile run_tests.sh\n",
        "#!/bin/bash\n",
        "\n",
        "export RCLONE_CONFIG_REMOTEREAD_TYPE='http'\n",
        "#Dataset description: https://archive.ics.uci.edu/ml/datasets/adult\n",
        "export RCLONE_CONFIG_REMOTEREAD_URL='https://archive.ics.uci.edu/ml/machine-learning-databases/adult/'\n",
        "\n",
        "mkdir /tmp/my_local_dir_for_test\n",
        "\n",
        "#refer: https://github.com/ydataai/pandas-profiling/blob/master/src/pandas_profiling/config_default.yaml\n",
        "#override panda-profiler options on top of config_default in --panda-profiler-options below.\n",
        "#Test: visualization of csv file, rclone read in copy mode\n",
        "python /tmp/data_visualization.py --input-datasource-file-name 'adult.data' \\\n",
        "    --additional-options-csv-parsing '{\"names\":[\"age\", \"workclass\", \"fnlwgt\", \"education\", \"education-num\", \"marital-status\", \"occupation\", \"relationship\", \"race\", \"sex\", \"capital-gain\", \"capital-loss\", \"hours-per-week\", \"native-country\", \"income-class\"]}' \\\n",
        "    --panda-profiler-options '{\"title\":\"US Adult Census Income prediction\", \"memory_deep\":true, \"html\":{\"full_width\": true}}' \\\n",
        "    --output-visualization-absolute-path '/tmp/my_local_dir_for_test/adult_data_report.html'\n",
        "\n",
        "python /tmp/test_validation.py"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jzJg29vTVGcf",
        "outputId": "16582f8d-27a1-4e46-b0ef-1ac3fc356231"
      },
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Overwriting run_tests.sh\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile test_validation.py\n",
        "#!/usr/bin/env python3\n",
        "from bs4 import BeautifulSoup\n",
        "with open('/tmp/my_local_dir_for_test/adult_data_report.html') as fp:\n",
        "    soup = BeautifulSoup(fp, \"html.parser\")\n",
        "assert soup.title.get_text() == 'US Adult Census Income prediction'\n",
        "\n",
        "table = soup.find('table', attrs={'class':'dataframe duplicate table table-striped'})\n",
        "\n",
        "import pandas as pd\n",
        "duplicated_row_info_df = pd.read_html(str(table))[0]\n",
        "assert len(duplicated_row_info_df.index) == 10 #there are 10 uniquely identifyable duplicated rows\n",
        "print(\"Test validation is done\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7maMc-RVqDV5",
        "outputId": "9abc2e73-c143-43ec-e70f-2d56dc3ee333"
      },
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Overwriting test_validation.py\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile docker-compose.test.yml\n",
        "services:\n",
        "  sut:\n",
        "    build: .\n",
        "    command: /tmp/run_tests.sh"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cVrXyQZhSIm_",
        "outputId": "13d6764c-6d7f-4fa2-857c-3c1f11f9670e"
      },
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Writing docker-compose.test.yml\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "*   https://www.kubeflow.org/docs/components/pipelines/sdk/component-development/#designing-a-pipeline-component\n",
        "*   https://github.com/kubeflow/pipelines/blob/sdk/release-1.8/sdk/python/kfp/dsl/types.py\n",
        "\n"
      ],
      "metadata": {
        "id": "W7uyWGT6Sccr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile component.yaml\n",
        "name: TabularDataVisualizationUsingPandaProfiler\n",
        "description: |\n",
        "    Visualize tabular data (csv file) using pandas-profiler library. (Refer: https://github.com/ydataai/pandas-profiling)\n",
        "    Refer panda-profiler-options in command line arguments. \n",
        "    pandas-profiler internally uses pandas dataframe to read csv file. You can utilize options exposed by panda's read_csv(). \n",
        "    Refer additional-options-csv-parsing in command line arguments\n",
        "    Input csv file are stored in rclone compatible storage. Both mount and copy mode are supported. (refer: https://rclone.org/)\n",
        "    rclone configurations have to be shared through environment variables (refer: https://rclone.org/docs/#environment-variables). \n",
        "    Thus, before using this component in kubeflow pipeline, those environment variables have to be set from pipeline.\n",
        "    Create rclone read configuration file name as 'REMOTEREAD'. Because the same is used within code.\n",
        "    So convention for creating any environment variables related to rclone should start with 'RCLONE_CONFIG_REMOTEREAD'.\n",
        "    Visualization data is emitted as self-sufficient single html file. \n",
        "\n",
        "inputs:\n",
        "- {name: input-datasource-directory-mountable, optional, description: 'whether input csv file is present in mountable remote location'}\n",
        "- {name: input-datasource-directory-to-be-mounted, type: String, description: 'if input-datasource-directory-mountable=True, name of the mountable directory (e.g. bucket name for s3)'}\n",
        "- {name: input-datasource-file-name, type: String, description: 'name of the csv file including file extension (if any)'}\n",
        "- {name: additional-options-csv-parsing, type: String, default= '{}', description: 'json formatted key-value pairs of strings which will be passed to pandas.read_csv()'}\n",
        "- {name: panda-profiler-options, type: String, default= '{}', description:'json formatted key-value pairs of strings which will be passed to ProfileReport()'}\n",
        "- {name: sensitive-data-present, optional, description: 'whether some sensitive data present (if so, panda-profiler will not show sample data etc.)'}\n",
        "- {name: output-visualization-absolute-path, type: String, description: 'html filepath having visualization report'}\n",
        "\n",
        "outputs:\n",
        "- {name: MLPipeline UI metadata, type: UI metadata}\n",
        "\n",
        "implementation:\n",
        "  container:\n",
        "    image: hub.docker.com/shasjash/kfpcomponents/TabularDataVisualizationUsingPandaProfiler_devlatest\n",
        "    # command is a list of strings (command-line arguments). \n",
        "    # The YAML language has two syntaxes for lists and you can use either of them. \n",
        "    # Here we use the \"flow syntax\" - comma-separated strings inside square brackets.\n",
        "    command: [\n",
        "      python3, \n",
        "      # Path of the program inside the container\n",
        "      /tmp/data_visualization.py,\n",
        "      --input-datasource-directory-mountable,\n",
        "      {inputValue: input-datasource-directory-mountable},\n",
        "      --input-datasource-directory-to-be-mounted, \n",
        "      {inputValue: input-datasource-directory-to-be-mounted},\n",
        "      --input-datasource-file-name, \n",
        "      {inputValue: input-datasource-file-name},\n",
        "      --additional-options-csv-parsing, \n",
        "      {inputValue: additional-options-csv-parsing},\n",
        "      --panda-profiler-options, \n",
        "      {inputValue: panda-profiler-options},\n",
        "      --sensitive-data-present, \n",
        "      {inputValue: sensitive-data-present},\n",
        "      --output-visualization-absolute-path, \n",
        "      {inputValue: output-visualization-absolute-path},\n",
        "    ]\n",
        "    fileOutputs:\n",
        "      MLPipeline UI metadata:  /tmp/mlpipeline-ui-metadata.json"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dhXccnmvIo7u",
        "outputId": "23ccabe3-e546-4f8b-dcbb-819137e7dd4e"
      },
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Writing component.yaml\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Software testing"
      ],
      "metadata": {
        "id": "9uiUWKu4LWUU"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now let us simulate testing what will be done by docker-hub infrastructure as part of auto-testing by using docker-compose.test.yml present in github source repository."
      ],
      "metadata": {
        "id": "i3TK5UnMvfMS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!rm -rf /tmp/my_local_dir_for_test\n",
        "!chmod 544 run_tests.sh\n",
        "!cp data_visualization.py /tmp\n",
        "!cp test_validation.py /tmp\n",
        "!./run_tests.sh"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5i9h_W3T4H87",
        "outputId": "6e1003ff-7b06-4934-e40c-cfa7ee110801"
      },
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "local_datastore_read_dir: /tmp/my_local_read-upk_dk56\n",
            "2022/06/11 20:09:09 NOTICE: Config file \"/root/.config/rclone/rclone.conf\" not found - using defaults\n",
            "2022/06/11 20:09:10 INFO  : adult.data: Copied (new)\n",
            "2022/06/11 20:09:10 INFO  : \n",
            "Transferred:   \t    3.790 MiB / 3.790 MiB, 100%, 0 B/s, ETA -\n",
            "Transferred:            1 / 1, 100%\n",
            "Elapsed time:         1.1s\n",
            "\n",
            "\n",
            "parse_config = ( <class 'dict'> ) {'names': ['age', 'workclass', 'fnlwgt', 'education', 'education-num', 'marital-status', 'occupation', 'relationship', 'race', 'sex', 'capital-gain', 'capital-loss', 'hours-per-week', 'native-country', 'income-class']}\n",
            "       age  ... income-class\n",
            "0       39  ...        <=50K\n",
            "1       50  ...        <=50K\n",
            "2       38  ...        <=50K\n",
            "3       53  ...        <=50K\n",
            "4       28  ...        <=50K\n",
            "...    ...  ...          ...\n",
            "32556   27  ...        <=50K\n",
            "32557   40  ...         >50K\n",
            "32558   58  ...        <=50K\n",
            "32559   22  ...        <=50K\n",
            "32560   52  ...         >50K\n",
            "\n",
            "[32561 rows x 15 columns]\n",
            "profile_config = ( <class 'dict'> ) {'title': 'US Adult Census Income prediction', 'memory_deep': True, 'html': {'full_width': True}}\n",
            "Test validation is done\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Push the code to github"
      ],
      "metadata": {
        "id": "a62pERh-Lasc"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Before commiting code to github, install github client (gh) by following instruction mentioned in https://github.com/cli/cli/blob/trunk/docs/install_linux.md (Choose Debian, Ubuntu Linux way of installation) \n",
        "\n",
        "Use the colab's 'Terminal' icon present in left vertical pane to open linux terminal to type commands. Once 'gh' is installed, type **$gh auth login** (refer https://docs.github.com/en/get-started/getting-started-with-git/caching-your-github-credentials-in-git) to follow onscreen prompts. For colab, use **Paste an authentication token** option. Personal tokens can be generated in https://github.com/settings/tokens\n",
        "\n",
        "You can use Shift+Ctrl+v shortcut to paste any string in colab console"
      ],
      "metadata": {
        "id": "2Nix3R2joPdP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pwd"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G12wQfVT1ann",
        "outputId": "b38c5263-160e-43af-cd05-1ff021983f0e"
      },
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!rm -Rf kfpcomponent"
      ],
      "metadata": {
        "id": "JVc_RmC9MrgG"
      },
      "execution_count": 63,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/ShaswataJash/kfpcomponent.git"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S5LCKX0SXUxs",
        "outputId": "d3499024-9ba3-489a-f958-c319e02a8d8a"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'kfpcomponent'...\n",
            "remote: Enumerating objects: 150, done.\u001b[K\n",
            "remote: Counting objects: 100% (150/150), done.\u001b[K\n",
            "remote: Compressing objects: 100% (103/103), done.\u001b[K\n",
            "remote: Total 150 (delta 77), reused 99 (delta 42), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (150/150), 59.73 KiB | 5.43 MiB/s, done.\n",
            "Resolving deltas: 100% (77/77), done.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Follow directory structure according to https://www.kubeflow.org/docs/components/pipelines/sdk/component-development/#organizing-the-component-files"
      ],
      "metadata": {
        "id": "AIwZrmfnCXPW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!mkdir kfpcomponent/TabularDataVisualizationUsingPandaProfiler\n",
        "!mkdir kfpcomponent/TabularDataVisualizationUsingPandaProfiler/src\n",
        "!mkdir kfpcomponent/TabularDataVisualizationUsingPandaProfiler/tests"
      ],
      "metadata": {
        "id": "PWT-29j1XYhi"
      },
      "execution_count": 65,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#it will ensure file is coped in git repo only if file content is changed by checking checksum of file content\n",
        "!rsync -c data_visualization.py kfpcomponent/TabularDataVisualizationUsingPandaProfiler/src\n",
        "!rsync -c component.yaml kfpcomponent/TabularDataVisualizationUsingPandaProfiler/component.yaml\n",
        "!rsync -c test_validation.py kfpcomponent/TabularDataVisualizationUsingPandaProfiler/tests\n",
        "!rsync -c Dockerfile kfpcomponent/TabularDataVisualizationUsingPandaProfiler/\n",
        "!rsync -c run_tests.sh kfpcomponent/TabularDataVisualizationUsingPandaProfiler/\n",
        "!rsync -c docker-compose.test.yml kfpcomponent/TabularDataVisualizationUsingPandaProfiler/"
      ],
      "metadata": {
        "id": "n3b7gcGhXde6"
      },
      "execution_count": 66,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%cd kfpcomponent"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1RUDYMnmXlIW",
        "outputId": "63ecc65e-5efd-4b7c-c9b2-0d03be5a50e7"
      },
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/kfpcomponent\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!git add -A"
      ],
      "metadata": {
        "id": "iAyaTjClX9S0"
      },
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!git status"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hW4fzKBsYCGO",
        "outputId": "59b3203f-db2f-4858-ec62-8a062b240b03"
      },
      "execution_count": 69,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "On branch main\n",
            "Your branch is up to date with 'origin/main'.\n",
            "\n",
            "Changes to be committed:\n",
            "  (use \"git reset HEAD <file>...\" to unstage)\n",
            "\n",
            "\t\u001b[32mnew file:   TabularDataVisualizationUsingPandaProfiler/Dockerfile\u001b[m\n",
            "\t\u001b[32mnew file:   TabularDataVisualizationUsingPandaProfiler/component.yaml\u001b[m\n",
            "\t\u001b[32mnew file:   TabularDataVisualizationUsingPandaProfiler/docker-compose.test.yml\u001b[m\n",
            "\t\u001b[32mnew file:   TabularDataVisualizationUsingPandaProfiler/run_tests.sh\u001b[m\n",
            "\t\u001b[32mnew file:   TabularDataVisualizationUsingPandaProfiler/src/data_visualization.py\u001b[m\n",
            "\t\u001b[32mnew file:   TabularDataVisualizationUsingPandaProfiler/tests/test_validation.py\u001b[m\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "For git-user who has set their email visibility as private, git provides alternate email address to use in web-based Git operations, e.g., edits and merges. The alias email can be viewed in https://github.com/settings/emails"
      ],
      "metadata": {
        "id": "bmHCKK7CzXNU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!git config --global user.email \"29448766+ShaswataJash@users.noreply.github.com\""
      ],
      "metadata": {
        "id": "FBLk0UVR04lj"
      },
      "execution_count": 70,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!git commit -a -m \"first commit\""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VbaNqafyYHV7",
        "outputId": "d92cbe98-f84b-4d2b-f1fd-2ff10fdb0665"
      },
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[main 1cef073] first commit\n",
            " 6 files changed, 181 insertions(+)\n",
            " create mode 100644 TabularDataVisualizationUsingPandaProfiler/Dockerfile\n",
            " create mode 100644 TabularDataVisualizationUsingPandaProfiler/component.yaml\n",
            " create mode 100644 TabularDataVisualizationUsingPandaProfiler/docker-compose.test.yml\n",
            " create mode 100755 TabularDataVisualizationUsingPandaProfiler/run_tests.sh\n",
            " create mode 100644 TabularDataVisualizationUsingPandaProfiler/src/data_visualization.py\n",
            " create mode 100644 TabularDataVisualizationUsingPandaProfiler/tests/test_validation.py\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!git push origin main"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uLZfkrleaqQ3",
        "outputId": "9e8bd1c6-4498-4641-c498-25d0c86f9f36"
      },
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Counting objects: 11, done.\n",
            "Delta compression using up to 2 threads.\n",
            "Compressing objects:  11% (1/9)   \rCompressing objects:  22% (2/9)   \rCompressing objects:  33% (3/9)   \rCompressing objects:  44% (4/9)   \rCompressing objects:  55% (5/9)   \rCompressing objects:  66% (6/9)   \rCompressing objects:  77% (7/9)   \rCompressing objects:  88% (8/9)   \rCompressing objects: 100% (9/9)   \rCompressing objects: 100% (9/9), done.\n",
            "Writing objects:   9% (1/11)   \rWriting objects:  18% (2/11)   \rWriting objects:  27% (3/11)   \rWriting objects:  36% (4/11)   \rWriting objects:  45% (5/11)   \rWriting objects:  54% (6/11)   \rWriting objects:  63% (7/11)   \rWriting objects:  72% (8/11)   \rWriting objects:  81% (9/11)   \rWriting objects:  90% (10/11)   \rWriting objects: 100% (11/11)   \rWriting objects: 100% (11/11), 4.52 KiB | 4.52 MiB/s, done.\n",
            "Total 11 (delta 1), reused 0 (delta 0)\n",
            "remote: Resolving deltas:   0% (0/1)\u001b[K\rremote: Resolving deltas: 100% (1/1)\u001b[K\rremote: Resolving deltas: 100% (1/1), completed with 1 local object.\u001b[K\n",
            "To https://github.com/ShaswataJash/kfpcomponent.git\n",
            "   55daea2..1cef073  main -> main\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%cd .."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LwhyFmmz5bnu",
        "outputId": "20055fe2-e51f-466a-cc66-e52b9c94ddf9"
      },
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content\n"
          ]
        }
      ]
    }
  ]
}