research_amp/cc/notebooks/Master_single_vendor_qa.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ed40a8d3",
   "metadata": {},
   "source": [
    "# Description"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97ca1304",
   "metadata": {},
   "source": [
    "This notebooks performs QA checks for a single vendor:\n",
    "   - Number of NaN data points as % of total\n",
    "   - Number of data points where `volume=0` as % of total"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8de376c7",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7f2d0ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import os\n",
    "\n",
    "import core.config.config_ as cconconf\n",
    "import helpers.hdbg as hdbg\n",
    "import helpers.henv as henv\n",
    "import helpers.hprint as hprint\n",
    "import helpers.hs3 as hs3\n",
    "import im_v2.ccxt.data.client as icdcl\n",
    "import research_amp.cc.qa as ramccqa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ddd9f212",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "hdbg.init_logger(verbosity=logging.INFO)\n",
    "\n",
    "_LOG = logging.getLogger(__name__)\n",
    "\n",
    "_LOG.info(\"%s\", henv.get_system_signature()[0])\n",
    "\n",
    "hprint.config_notebook()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "59c7d7d4",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "source": [
    "# Configs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d64180a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_master_single_vendor_qa_config() -> cconconf.Config:\n",
    "    \"\"\"\n",
    "    Get task1866-specific config.\n",
    "    \"\"\"\n",
    "    config = cconconf.Config()\n",
    "    param_dict = {\n",
    "        \"data\": {\n",
    "            # Parameters for client initialization.\n",
    "            \"im_client\": {\n",
    "                \"universe_version\": \"v3\",\n",
    "                \"resample_1min\": False,\n",
    "                \"root_dir\": os.path.join(\n",
    "                    hs3.get_s3_bucket_path(\"ck\"), \"reorg\", \"historical.manual.pq\"\n",
    "                ),\n",
    "                \"partition_mode\": \"by_year_month\",\n",
    "                \"dataset\": \"ohlcv\",\n",
    "                \"contract_type\": \"spot\",\n",
    "                \"download_universe_version\": \"v7_3\",\n",
    "                \"aws_profile\": \"ck\",\n",
    "            },\n",
    "            # Parameters for data query.\n",
    "            \"read_data\": {\n",
    "                \"start_ts\": None,\n",
    "                \"end_ts\": None,\n",
    "                \"columns\": [\"full_symbol\", \"close\", \"volume\"],\n",
    "                \"filter_data_mode\": \"assert\",\n",
    "            },\n",
    "        },\n",
    "        \"column_names\": {\n",
    "            \"full_symbol\": \"full_symbol\",\n",
    "            \"close_price\": \"close\",\n",
    "        },\n",
    "        \"stats\": {\n",
    "            \"threshold\": 30,\n",
    "        },\n",
    "    }\n",
    "    config = cconfig.Config.from_dict(param_dict)\n",
    "    return config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b467b67",
   "metadata": {},
   "outputs": [],
   "source": [
    "config = get_master_single_vendor_qa_config()\n",
    "print(config)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca5e8cf9",
   "metadata": {},
   "source": [
    "# QA checks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "605a3808",
   "metadata": {},
   "source": [
    "Major metric for a QA check is `\"bad data [%]\"` which is the sum of `\"volume=0 [%]\"` and `\"NaNs [%]\"`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e41af470",
   "metadata": {},
   "outputs": [],
   "source": [
    "client = icdcl.CcxtHistoricalPqByTileClient(**config[\"data\"][\"im_client\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20771852",
   "metadata": {},
   "outputs": [],
   "source": [
    "universe = client.get_universe()\n",
    "universe"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2397cd03",
   "metadata": {},
   "source": [
    "## Binance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0d76a47",
   "metadata": {},
   "outputs": [],
   "source": [
    "binance_universe = [\n",
    "    full_symbol for full_symbol in universe if full_symbol.startswith(\"binance\")\n",
    "]\n",
    "binance_universe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40578a2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "binance_data = client.read_data(binance_universe, **config[\"data\"][\"read_data\"])\n",
    "binance_data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7006f20c",
   "metadata": {},
   "outputs": [],
   "source": [
    "vendor_name = \"CCXT\"\n",
    "binance_timestamp_stats = ramccqa.get_timestamp_stats(binance_data, vendor_name)\n",
    "binance_timestamp_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70249efd",
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_level_full_symbol = [\"full_symbol\"]\n",
    "binance_bad_data_stats = ramccqa.get_bad_data_stats(\n",
    "    binance_data, agg_level_full_symbol, vendor_name\n",
    ")\n",
    "binance_bad_data_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e3081d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "agg_level_full_symbol_year_month = [\"full_symbol\", \"year\", \"month\"]\n",
    "binance_bad_data_stats_by_year_month = ramccqa.get_bad_data_stats(\n",
    "    binance_data, agg_level_full_symbol_year_month, vendor_name\n",
    ")\n",
    "binance_bad_data_stats_by_year_month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f6acca4",
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = ramccqa.plot_bad_data_by_year_month_stats(\n",
    "    binance_bad_data_stats_by_year_month, config[\"stats\"][\"threshold\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "959cc76a",
   "metadata": {},
   "source": [
    "## FTX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38c8a019",
   "metadata": {},
   "outputs": [],
   "source": [
    "ftx_universe = [\n",
    "    full_symbol for full_symbol in universe if full_symbol.startswith(\"ftx\")\n",
    "]\n",
    "ftx_universe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d7bbb109",
   "metadata": {},
   "outputs": [],
   "source": [
    "ftx_data = client.read_data(ftx_universe, **config[\"data\"][\"read_data\"])\n",
    "ftx_data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7cab2b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "ftx_timestamp_stats = ramccqa.get_timestamp_stats(ftx_data, vendor_name)\n",
    "ftx_timestamp_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b82b54b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "ftx_bad_data_stats = ramccqa.get_bad_data_stats(\n",
    "    ftx_data, agg_level_full_symbol, vendor_name\n",
    ")\n",
    "ftx_bad_data_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91a738e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "ftx_bad_data_stats_by_year_month = ramccqa.get_bad_data_stats(\n",
    "    ftx_data, agg_level_full_symbol_year_month, vendor_name\n",
    ")\n",
    "ftx_bad_data_stats_by_year_month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "377f5c12",
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = ramccqa.plot_bad_data_by_year_month_stats(\n",
    "    ftx_bad_data_stats_by_year_month, config[\"stats\"][\"threshold\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "628a720a",
   "metadata": {},
   "source": [
    "## Gateio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d836ce13",
   "metadata": {},
   "outputs": [],
   "source": [
    "gateio_universe = [\n",
    "    full_symbol for full_symbol in universe if full_symbol.startswith(\"gateio\")\n",
    "]\n",
    "gateio_universe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8877f471",
   "metadata": {},
   "outputs": [],
   "source": [
    "gateio_data = client.read_data(gateio_universe, **config[\"data\"][\"read_data\"])\n",
    "gateio_data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ef42055",
   "metadata": {},
   "outputs": [],
   "source": [
    "gateio_timestamp_stats = ramccqa.get_timestamp_stats(gateio_data, vendor_name)\n",
    "gateio_timestamp_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a174a2bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "gateio_bad_data_stats = ramccqa.get_bad_data_stats(\n",
    "    gateio_data, agg_level_full_symbol, vendor_name\n",
    ")\n",
    "gateio_bad_data_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "824a08bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "gateio_bad_data_stats_by_year_month = ramccqa.get_bad_data_stats(\n",
    "    gateio_data, agg_level_full_symbol_year_month, vendor_name\n",
    ")\n",
    "gateio_bad_data_stats_by_year_month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d0febf1",
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = ramccqa.plot_bad_data_by_year_month_stats(\n",
    "    gateio_bad_data_stats_by_year_month, config[\"stats\"][\"threshold\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2da921c",
   "metadata": {},
   "source": [
    "## Kucoin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1146b097",
   "metadata": {},
   "outputs": [],
   "source": [
    "kucoin_universe = [\n",
    "    full_symbol for full_symbol in universe if full_symbol.startswith(\"kucoin\")\n",
    "]\n",
    "kucoin_universe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c119da9",
   "metadata": {},
   "outputs": [],
   "source": [
    "kucoin_data = client.read_data(kucoin_universe, **config[\"data\"][\"read_data\"])\n",
    "kucoin_data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8388cc28",
   "metadata": {},
   "outputs": [],
   "source": [
    "kucoin_timestamp_stats = ramccqa.get_timestamp_stats(kucoin_data, vendor_name)\n",
    "kucoin_timestamp_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "845a8a24",
   "metadata": {},
   "outputs": [],
   "source": [
    "kucoin_bad_data_stats = ramccqa.get_bad_data_stats(\n",
    "    kucoin_data, agg_level_full_symbol, vendor_name\n",
    ")\n",
    "kucoin_bad_data_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50855241",
   "metadata": {},
   "outputs": [],
   "source": [
    "kucoin_bad_data_stats_by_year_month = ramccqa.get_bad_data_stats(\n",
    "    kucoin_data, agg_level_full_symbol_year_month, vendor_name\n",
    ")\n",
    "kucoin_bad_data_stats_by_year_month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3335000",
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = ramccqa.plot_bad_data_by_year_month_stats(\n",
    "    kucoin_bad_data_stats_by_year_month, config[\"stats\"][\"threshold\"]\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}