docs/datapull/all.market_data.reference.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a0da8680",
   "metadata": {},
   "source": [
    "# Descriptions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "525119ff",
   "metadata": {},
   "source": [
    "The notebook demonstrates how to use `MarketData`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "875998a8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-19T16:57:59.663869Z",
     "start_time": "2024-02-19T16:57:57.503335Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0f7f1983",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-19T16:58:07.960487Z",
     "start_time": "2024-02-19T16:57:59.666984Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/app/amp/core/statistics/random_samples.py:12: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm\n",
      "\u001b[33mWARNING\u001b[0m: No module named 'gluonts'\n"
     ]
    }
   ],
   "source": [
    "import logging\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import core.config as cconfig\n",
    "import helpers.hdbg as hdbg\n",
    "import helpers.henv as henv\n",
    "import helpers.hprint as hprint\n",
    "import im_v2.ccxt.data.client as icdcl\n",
    "import im_v2.common.universe as ivcu\n",
    "import market_data.market_data_example as mdmadaex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b7cf3d04",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-19T16:58:09.171044Z",
     "start_time": "2024-02-19T16:58:07.965484Z"
    },
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[0m\u001b[36mINFO\u001b[0m: > cmd='/venv/lib/python3.9/site-packages/ipykernel_launcher.py -f /home/.local/share/jupyter/runtime/kernel-41bef88b-2880-47fe-95e0-535cc5e2b1ce.json'\n",
      "WARNING: Running in Jupyter\n",
      "INFO  # Git\n",
      "  branch_name='CmampTask6752_Merge_DAG_optimization_branches'\n",
      "  hash='4c0210707'\n",
      "  # Last commits:\n",
      "    *   4c0210707 Sameep Pote Merge branch 'master' into CmampTask6752_Merge_DAG_optimization_branches (    3 days ago) Fri Feb 16 14:47:59 2024  (HEAD -> CmampTask6752_Merge_DAG_optimization_branches, origin/CmampTask6752_Merge_DAG_optimization_branches)\n",
      "    |\\  \n",
      "    | * 54cee75d7 Vlad     CmampTask7097_currency_pair_has_incompatible_types_use_legacy_dataset (#7201) (    3 days ago) Fri Feb 16 14:24:22 2024           \n",
      "    | * b102c0dfd Juraj Smeriga CmampTask7137_Test_market_data_with_realtime_bidask_resampled_1_min_clients (#7171) (    3 days ago) Fri Feb 16 13:55:35 2024           \n",
      "# Machine info\n",
      "  system=Linux\n",
      "  node name=7edba6067cba\n",
      "  release=5.15.0-1052-aws\n",
      "  version=#57~20.04.1-Ubuntu SMP Mon Jan 15 17:04:56 UTC 2024\n",
      "  machine=x86_64\n",
      "  processor=x86_64\n",
      "  cpu count=8\n",
      "  cpu freq=scpufreq(current=2499.998, min=0.0, max=0.0)\n",
      "  memory=svmem(total=33280278528, available=8042139648, percent=75.8, used=24744648704, free=1999421440, active=1733988352, inactive=24592601088, buffers=1304649728, cached=5231558656, shared=11419648, slab=4564643840)\n",
      "  disk usage=sdiskusage(total=156052275200, used=99453730816, free=56581767168, percent=63.7)\n",
      "# Packages\n",
      "  python: 3.9.5\n",
      "  cvxopt: 1.3.2\n",
      "  cvxpy: 1.4.1\n",
      "  gluonnlp: ?\n",
      "  gluonts: ?\n",
      "  joblib: 1.3.2\n",
      "  mxnet: ?\n",
      "  numpy: 1.26.0\n",
      "  pandas: 2.1.1\n",
      "  pyarrow: 10.0.0\n",
      "  scipy: 1.11.3\n",
      "  seaborn: 0.13.0\n",
      "  sklearn: 1.3.1\n",
      "  statsmodels: 0.14.0\n"
     ]
    }
   ],
   "source": [
    "log_level = logging.INFO\n",
    "hdbg.init_logger(verbosity=log_level)\n",
    "\n",
    "_LOG = logging.getLogger(__name__)\n",
    "\n",
    "_LOG.info(\"%s\", henv.get_system_signature()[0])\n",
    "\n",
    "hprint.config_notebook()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d936296",
   "metadata": {},
   "source": [
    "# Get asset ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "09916612",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-19T16:58:09.228357Z",
     "start_time": "2024-02-19T16:58:09.174531Z"
    }
   },
   "outputs": [],
   "source": [
    "universe_version = \"v7.4\"\n",
    "universe_config = {\n",
    "    \"vendor\": \"CCXT\",\n",
    "    \"version\": universe_version,\n",
    "    \"mode\": \"trade\",\n",
    "    \"as_full_symbol\": True,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8700ec67",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-19T16:58:09.373535Z",
     "start_time": "2024-02-19T16:58:09.233314Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO  Full symbols=['binance::BNB_USDT', 'binance::BTC_USDT']\n"
     ]
    }
   ],
   "source": [
    "full_symbols = ivcu.get_vendor_universe(**universe_config)\n",
    "# Use only a subset for the demonstration.\n",
    "full_symbols = full_symbols[4:6]\n",
    "_LOG.info(\"Full symbols=%s\", full_symbols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5d3c8f6d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-19T16:58:09.424227Z",
     "start_time": "2024-02-19T16:58:09.376295Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO  Asset ids=[8968126878, 1467591036]\n"
     ]
    }
   ],
   "source": [
    "asset_ids = list(ivcu.build_numerical_to_string_id_mapping(full_symbols).keys())\n",
    "_LOG.info(\"Asset ids=%s\", asset_ids)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e778705",
   "metadata": {},
   "source": [
    "# `ImClientMarketData`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f8cec206",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-19T16:58:09.486657Z",
     "start_time": "2024-02-19T16:58:09.426764Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start_timestamp: 2023-09-11 00:00:00+00:00\n",
      "end_timestamp: 2023-09-11 04:00:00+00:00\n",
      "im_client: \n",
      "  universe_version: v7.4\n",
      "  root_dir: s3://cryptokaizen-data-test/v3\n",
      "  partition_mode: by_year_month\n",
      "  dataset: ohlcv\n",
      "  contract_type: futures\n",
      "  data_snapshot: \n",
      "  aws_profile: ck\n",
      "  resample_1min: False\n",
      "  version: v1_0_0\n",
      "  download_universe_version: v7_3\n",
      "  tag: downloaded_1min\n",
      "ts_col_name: timestamp\n",
      "columns: None\n",
      "column_remap: None\n",
      "filter_data_mode: assert\n",
      "wall_clock_time: 2100-01-01 00:00:00+00:00\n"
     ]
    }
   ],
   "source": [
    "im_client_market_data_config = {\n",
    "    \"start_timestamp\": pd.Timestamp(\"2023-09-11T00:00:00\", tz=\"UTC\"),\n",
    "    \"end_timestamp\": pd.Timestamp(\"2023-09-11T04:00:00\", tz=\"UTC\"),\n",
    "    \"im_client\": {\n",
    "        \"universe_version\": universe_version,\n",
    "        \"root_dir\": \"s3://cryptokaizen-data-test/v3\",\n",
    "        \"partition_mode\": \"by_year_month\",\n",
    "        \"dataset\": \"ohlcv\",\n",
    "        \"contract_type\": \"futures\",\n",
    "        \"data_snapshot\": \"\",\n",
    "        \"aws_profile\": \"ck\",\n",
    "        \"resample_1min\": False,\n",
    "        \"version\": \"v1_0_0\",\n",
    "        \"download_universe_version\": \"v7_3\",\n",
    "        \"tag\": \"downloaded_1min\",\n",
    "    },\n",
    "    \"ts_col_name\": \"timestamp\",\n",
    "    \"columns\": None,\n",
    "    \"column_remap\": None,\n",
    "    \"filter_data_mode\": \"assert\",\n",
    "    \"wall_clock_time\": pd.Timestamp(\"2100-01-01 00:00:00+00:00\"),\n",
    "}\n",
    "im_client_market_data_config = cconfig.Config().from_dict(\n",
    "    im_client_market_data_config\n",
    ")\n",
    "print(im_client_market_data_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0a8b6215",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-19T16:58:15.401864Z",
     "start_time": "2024-02-19T16:58:09.490905Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>asset_id</th>\n",
       "      <th>full_symbol</th>\n",
       "      <th>open</th>\n",
       "      <th>high</th>\n",
       "      <th>low</th>\n",
       "      <th>close</th>\n",
       "      <th>volume</th>\n",
       "      <th>knowledge_timestamp</th>\n",
       "      <th>start_ts</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>end_ts</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2023-09-10 20:00:00-04:00</th>\n",
       "      <td>1467591036</td>\n",
       "      <td>binance::BTC_USDT</td>\n",
       "      <td>25829.00</td>\n",
       "      <td>25830.50</td>\n",
       "      <td>25828.40</td>\n",
       "      <td>25828.40</td>\n",
       "      <td>35.063</td>\n",
       "      <td>2023-09-11 01:23:22.496982+00:00</td>\n",
       "      <td>2023-09-10 19:59:00-04:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-09-10 20:00:00-04:00</th>\n",
       "      <td>8968126878</td>\n",
       "      <td>binance::BNB_USDT</td>\n",
       "      <td>212.25</td>\n",
       "      <td>212.29</td>\n",
       "      <td>212.25</td>\n",
       "      <td>212.28</td>\n",
       "      <td>120.720</td>\n",
       "      <td>2023-09-11 01:23:28.294152+00:00</td>\n",
       "      <td>2023-09-10 19:59:00-04:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-09-10 20:01:00-04:00</th>\n",
       "      <td>1467591036</td>\n",
       "      <td>binance::BTC_USDT</td>\n",
       "      <td>25828.40</td>\n",
       "      <td>25835.80</td>\n",
       "      <td>25828.10</td>\n",
       "      <td>25835.70</td>\n",
       "      <td>152.568</td>\n",
       "      <td>2023-09-12 01:23:44.163254+00:00</td>\n",
       "      <td>2023-09-10 20:00:00-04:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             asset_id        full_symbol      open      high       low     close   volume              knowledge_timestamp                  start_ts\n",
       "end_ts                                                                                                                                                              \n",
       "2023-09-10 20:00:00-04:00  1467591036  binance::BTC_USDT  25829.00  25830.50  25828.40  25828.40   35.063 2023-09-11 01:23:22.496982+00:00 2023-09-10 19:59:00-04:00\n",
       "2023-09-10 20:00:00-04:00  8968126878  binance::BNB_USDT    212.25    212.29    212.25    212.28  120.720 2023-09-11 01:23:28.294152+00:00 2023-09-10 19:59:00-04:00\n",
       "2023-09-10 20:01:00-04:00  1467591036  binance::BTC_USDT  25828.40  25835.80  25828.10  25835.70  152.568 2023-09-12 01:23:44.163254+00:00 2023-09-10 20:00:00-04:00"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ohlcv_im_client = icdcl.CcxtHistoricalPqByTileClient(\n",
    "    **im_client_market_data_config[\"im_client\"]\n",
    ")\n",
    "ohlcv_market_data = mdmadaex.get_HistoricalImClientMarketData_example1(\n",
    "    ohlcv_im_client,\n",
    "    asset_ids,\n",
    "    im_client_market_data_config[\"columns\"],\n",
    "    im_client_market_data_config[\"column_remap\"],\n",
    "    wall_clock_time=im_client_market_data_config[\"wall_clock_time\"],\n",
    "    filter_data_mode=im_client_market_data_config[\"filter_data_mode\"],\n",
    ")\n",
    "ohlcv_data = ohlcv_market_data.get_data_for_interval(\n",
    "    im_client_market_data_config[\"start_timestamp\"],\n",
    "    im_client_market_data_config[\"end_timestamp\"],\n",
    "    im_client_market_data_config[\"ts_col_name\"],\n",
    "    asset_ids,\n",
    ")\n",
    "ohlcv_data.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a39bb6fa",
   "metadata": {},
   "source": [
    "# `StitchedMarketData`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9261279b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-19T17:07:48.667542Z",
     "start_time": "2024-02-19T17:07:48.552645Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start_timestamp: 2023-05-01 00:00:00+00:00\n",
      "end_timestamp: 2023-05-01 04:00:00+00:00\n",
      "ohlcv_market_data: \n",
      "  im_client: \n",
      "    universe_version: v7.4\n",
      "    root_dir: s3://cryptokaizen-data.preprod/v3\n",
      "    partition_mode: by_year_month\n",
      "    dataset: ohlcv\n",
      "    contract_type: futures\n",
      "    data_snapshot: \n",
      "    aws_profile: ck\n",
      "    resample_1min: False\n",
      "    version: v1_0_0\n",
      "    download_universe_version: v7_3\n",
      "    tag: downloaded_1min\n",
      "  ts_col_name: timestamp\n",
      "  columns: None\n",
      "  column_remap: None\n",
      "  filter_data_mode: assert\n",
      "bid_ask_market_data: \n",
      "  im_client: \n",
      "    universe_version: v7.4\n",
      "    dataset: bid_ask\n",
      "    contract_type: futures\n",
      "    data_snapshot: \n",
      "    root_dir: s3://cryptokaizen-data.preprod/v3\n",
      "    partition_mode: by_year_month\n",
      "    version: v2_0_0\n",
      "    download_universe_version: v7\n",
      "    tag: resampled_1min\n",
      "    aws_profile: ck\n",
      "  ts_col_name: timestamp\n",
      "  columns: None\n",
      "  column_remap: None\n",
      "  filter_data_mode: assert\n",
      "stitched_market_data: \n",
      "  ts_col_name: timestamp\n",
      "  columns: None\n",
      "  column_remap: None\n",
      "  filter_data_mode: warn_and_trim\n"
     ]
    }
   ],
   "source": [
    "stitched_market_data_config = {\n",
    "    \"start_timestamp\": pd.Timestamp(\"2023-05-01T00:00:00\", tz=\"UTC\"),\n",
    "    \"end_timestamp\": pd.Timestamp(\"2023-05-10T04:00:00\", tz=\"UTC\"),\n",
    "    \"ohlcv_market_data\": {\n",
    "        \"im_client\": {\n",
    "            \"universe_version\": universe_version,\n",
    "            \"root_dir\": \"s3://cryptokaizen-unit-test/outcomes/Test_run_all_market_data_reference_notebook/v3/\",\n",
    "            \"partition_mode\": \"by_year_month\",\n",
    "            \"dataset\": \"ohlcv\",\n",
    "            \"contract_type\": \"futures\",\n",
    "            \"data_snapshot\": \"\",\n",
    "            \"aws_profile\": \"ck\",\n",
    "            \"resample_1min\": False,\n",
    "            \"version\": \"v1_0_0\",\n",
    "            \"download_universe_version\": \"v7_3\",\n",
    "            \"tag\": \"downloaded_1min\",\n",
    "        },\n",
    "        \"ts_col_name\": \"timestamp\",\n",
    "        \"columns\": None,\n",
    "        \"column_remap\": None,\n",
    "        \"filter_data_mode\": \"assert\",\n",
    "    },\n",
    "    \"bid_ask_market_data\": {\n",
    "        \"im_client\": {\n",
    "            # Download universe version.\n",
    "            \"universe_version\": universe_version,\n",
    "            \"dataset\": \"bid_ask\",\n",
    "            \"contract_type\": \"futures\",\n",
    "            # Data snapshot is not applicable for data version = \"v3\".\n",
    "            \"data_snapshot\": \"\",\n",
    "            \"universe_version\": universe_version,\n",
    "            # Data currently residing in the test bucket\n",
    "            \"root_dir\": \"s3://cryptokaizen-unit-test/outcomes/Test_run_all_market_data_reference_notebook/v3/\",\n",
    "            \"partition_mode\": \"by_year_month\",\n",
    "            \"dataset\": \"bid_ask\",\n",
    "            \"contract_type\": \"futures\",\n",
    "            # v2_0_0 is used due to addition of new column in #CmTask7224.\n",
    "            \"version\": \"v2_0_0\",\n",
    "            \"download_universe_version\": \"v7\",\n",
    "            \"tag\": \"resampled_1min\",\n",
    "            \"aws_profile\": \"ck\",\n",
    "        },\n",
    "        \"ts_col_name\": \"timestamp\",\n",
    "        # TODO(Grisha): for some reason the current filtering mechanism filters out `asset_ids` which\n",
    "        # makes it impossible to stitch the 2 market data dfs. So adding the necessary columns manually.\n",
    "        # Note(Juraj): we currently resampled only top of the book so no need to filter the columns\n",
    "        # \"columns\": cfibiask.get_bid_ask_columns_by_level(1)\n",
    "        # + [\"asset_id\", \"full_symbol\", \"start_ts\", \"knowledge_timestamp\"],\n",
    "        \"columns\": None,\n",
    "        \"column_remap\": None,\n",
    "        \"filter_data_mode\": \"assert\",\n",
    "    },\n",
    "    \"stitched_market_data\": {\n",
    "        \"ts_col_name\": \"timestamp\",\n",
    "        \"columns\": None,\n",
    "        \"column_remap\": None,\n",
    "        # TODO(Grisha): check why it fails when the mode is `assert`.\n",
    "        \"filter_data_mode\": \"warn_and_trim\",\n",
    "    },\n",
    "}\n",
    "stitched_market_data_config = cconfig.Config().from_dict(\n",
    "    stitched_market_data_config\n",
    ")\n",
    "print(stitched_market_data_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "46a8fdbe",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-19T17:08:56.499978Z",
     "start_time": "2024-02-19T17:07:49.972422Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING Data that is downloaded before a bar ends accounts for=29 / 2116317 = 0.00%\n",
      "WARNING Data that is downloaded before a bar ends accounts for=27 / 2338920 = 0.00%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>asset_id</th>\n",
       "      <th>full_symbol</th>\n",
       "      <th>open</th>\n",
       "      <th>high</th>\n",
       "      <th>low</th>\n",
       "      <th>close</th>\n",
       "      <th>volume</th>\n",
       "      <th>knowledge_timestamp</th>\n",
       "      <th>start_ts</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>end_ts</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2023-04-30 20:00:00-04:00</th>\n",
       "      <td>1467591036</td>\n",
       "      <td>binance::BTC_USDT</td>\n",
       "      <td>29247.00</td>\n",
       "      <td>29253.20</td>\n",
       "      <td>29145.50</td>\n",
       "      <td>29223.00</td>\n",
       "      <td>2888.894</td>\n",
       "      <td>2023-05-01 01:22:07.571121+00:00</td>\n",
       "      <td>2023-04-30 19:59:00-04:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-04-30 20:00:00-04:00</th>\n",
       "      <td>8968126878</td>\n",
       "      <td>binance::BNB_USDT</td>\n",
       "      <td>336.96</td>\n",
       "      <td>337.15</td>\n",
       "      <td>336.41</td>\n",
       "      <td>336.84</td>\n",
       "      <td>6341.860</td>\n",
       "      <td>2023-05-01 01:22:16.403833+00:00</td>\n",
       "      <td>2023-04-30 19:59:00-04:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-04-30 20:01:00-04:00</th>\n",
       "      <td>1467591036</td>\n",
       "      <td>binance::BTC_USDT</td>\n",
       "      <td>29223.00</td>\n",
       "      <td>29245.80</td>\n",
       "      <td>29206.90</td>\n",
       "      <td>29240.70</td>\n",
       "      <td>1550.598</td>\n",
       "      <td>2023-05-02 01:22:09.764846+00:00</td>\n",
       "      <td>2023-04-30 20:00:00-04:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             asset_id        full_symbol      open      high       low     close    volume              knowledge_timestamp                  start_ts\n",
       "end_ts                                                                                                                                                               \n",
       "2023-04-30 20:00:00-04:00  1467591036  binance::BTC_USDT  29247.00  29253.20  29145.50  29223.00  2888.894 2023-05-01 01:22:07.571121+00:00 2023-04-30 19:59:00-04:00\n",
       "2023-04-30 20:00:00-04:00  8968126878  binance::BNB_USDT    336.96    337.15    336.41    336.84  6341.860 2023-05-01 01:22:16.403833+00:00 2023-04-30 19:59:00-04:00\n",
       "2023-04-30 20:01:00-04:00  1467591036  binance::BTC_USDT  29223.00  29245.80  29206.90  29240.70  1550.598 2023-05-02 01:22:09.764846+00:00 2023-04-30 20:00:00-04:00"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ohlcv_im_client = icdcl.CcxtHistoricalPqByTileClient(\n",
    "    **stitched_market_data_config[\"ohlcv_market_data\"][\"im_client\"]\n",
    ")\n",
    "ohlcv_market_data = mdmadaex.get_HistoricalImClientMarketData_example1(\n",
    "    ohlcv_im_client,\n",
    "    asset_ids,\n",
    "    stitched_market_data_config[\"ohlcv_market_data\"][\"columns\"],\n",
    "    stitched_market_data_config[\"ohlcv_market_data\"][\"column_remap\"],\n",
    "    filter_data_mode=stitched_market_data_config[\"ohlcv_market_data\"][\n",
    "        \"filter_data_mode\"\n",
    "    ],\n",
    ")\n",
    "ohlcv_data = ohlcv_market_data.get_data_for_interval(\n",
    "    stitched_market_data_config[\"start_timestamp\"],\n",
    "    stitched_market_data_config[\"end_timestamp\"],\n",
    "    stitched_market_data_config[\"ohlcv_market_data\"][\"ts_col_name\"],\n",
    "    asset_ids,\n",
    ")\n",
    "ohlcv_data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ea04d0ca",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-19T17:08:59.043908Z",
     "start_time": "2024-02-19T17:08:56.503994Z"
    }
   },
   "outputs": [],
   "source": [
    "bid_ask_im_client = icdcl.CcxtHistoricalPqByTileClient(\n",
    "    **stitched_market_data_config[\"bid_ask_market_data\"][\"im_client\"]\n",
    ")\n",
    "bid_ask_market_data = mdmadaex.get_HistoricalImClientMarketData_example1(\n",
    "    bid_ask_im_client,\n",
    "    asset_ids,\n",
    "    stitched_market_data_config[\"bid_ask_market_data\"][\"columns\"],\n",
    "    stitched_market_data_config[\"bid_ask_market_data\"][\"column_remap\"],\n",
    "    filter_data_mode=stitched_market_data_config[\"bid_ask_market_data\"][\n",
    "        \"filter_data_mode\"\n",
    "    ],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4e2699ff",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-19T17:09:06.651716Z",
     "start_time": "2024-02-19T17:08:59.048059Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>asset_id</th>\n",
       "      <th>full_symbol</th>\n",
       "      <th>level_1.bid_price.open</th>\n",
       "      <th>level_1.bid_size.open</th>\n",
       "      <th>level_1.ask_price.open</th>\n",
       "      <th>level_1.ask_size.open</th>\n",
       "      <th>level_1.bid_ask_midpoint.open</th>\n",
       "      <th>level_1.half_spread.open</th>\n",
       "      <th>level_1.log_size_imbalance.open</th>\n",
       "      <th>level_1.bid_price.close</th>\n",
       "      <th>level_1.bid_size.close</th>\n",
       "      <th>level_1.ask_price.close</th>\n",
       "      <th>level_1.ask_size.close</th>\n",
       "      <th>level_1.bid_ask_midpoint.close</th>\n",
       "      <th>level_1.half_spread.close</th>\n",
       "      <th>level_1.log_size_imbalance.close</th>\n",
       "      <th>level_1.bid_price.high</th>\n",
       "      <th>level_1.bid_size.max</th>\n",
       "      <th>level_1.ask_price.high</th>\n",
       "      <th>level_1.ask_size.max</th>\n",
       "      <th>level_1.bid_ask_midpoint.max</th>\n",
       "      <th>level_1.half_spread.max</th>\n",
       "      <th>level_1.log_size_imbalance.max</th>\n",
       "      <th>level_1.bid_price.low</th>\n",
       "      <th>level_1.bid_size.min</th>\n",
       "      <th>level_1.ask_price.low</th>\n",
       "      <th>level_1.ask_size.min</th>\n",
       "      <th>level_1.bid_ask_midpoint.min</th>\n",
       "      <th>level_1.half_spread.min</th>\n",
       "      <th>level_1.log_size_imbalance.min</th>\n",
       "      <th>level_1.bid_price.mean</th>\n",
       "      <th>level_1.bid_size.mean</th>\n",
       "      <th>level_1.ask_price.mean</th>\n",
       "      <th>level_1.ask_size.mean</th>\n",
       "      <th>level_1.bid_ask_midpoint.mean</th>\n",
       "      <th>level_1.half_spread.mean</th>\n",
       "      <th>level_1.log_size_imbalance.mean</th>\n",
       "      <th>level_1.bid_ask_midpoint_var.100ms</th>\n",
       "      <th>level_1.log_size_imbalance_var.100ms</th>\n",
       "      <th>start_ts</th>\n",
       "      <th>open</th>\n",
       "      <th>high</th>\n",
       "      <th>low</th>\n",
       "      <th>close</th>\n",
       "      <th>volume</th>\n",
       "      <th>knowledge_timestamp</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>end_ts</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2023-04-30 20:00:00-04:00</th>\n",
       "      <td>1467591036</td>\n",
       "      <td>binance::BTC_USDT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023-04-30 19:59:00-04:00</td>\n",
       "      <td>29247.00</td>\n",
       "      <td>29253.20</td>\n",
       "      <td>29145.50</td>\n",
       "      <td>29223.00</td>\n",
       "      <td>2888.894</td>\n",
       "      <td>2023-05-01 01:22:07.571121+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-04-30 20:00:00-04:00</th>\n",
       "      <td>8968126878</td>\n",
       "      <td>binance::BNB_USDT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2023-04-30 19:59:00-04:00</td>\n",
       "      <td>336.96</td>\n",
       "      <td>337.15</td>\n",
       "      <td>336.41</td>\n",
       "      <td>336.84</td>\n",
       "      <td>6341.860</td>\n",
       "      <td>2023-05-01 01:22:16.403833+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2023-04-30 20:01:00-04:00</th>\n",
       "      <td>1467591036</td>\n",
       "      <td>binance::BTC_USDT</td>\n",
       "      <td>29223.5</td>\n",
       "      <td>9.139</td>\n",
       "      <td>29223.6</td>\n",
       "      <td>0.421</td>\n",
       "      <td>29223.55</td>\n",
       "      <td>0.05</td>\n",
       "      <td>3.077673</td>\n",
       "      <td>29240.6</td>\n",
       "      <td>2.934</td>\n",
       "      <td>29240.7</td>\n",
       "      <td>10.842</td>\n",
       "      <td>29240.65</td>\n",
       "      <td>0.05</td>\n",
       "      <td>-1.307061</td>\n",
       "      <td>29245.7</td>\n",
       "      <td>28.248</td>\n",
       "      <td>29245.8</td>\n",
       "      <td>69.545</td>\n",
       "      <td>29245.75</td>\n",
       "      <td>3.4</td>\n",
       "      <td>9.682092</td>\n",
       "      <td>29210.0</td>\n",
       "      <td>0.001</td>\n",
       "      <td>29210.8</td>\n",
       "      <td>0.001</td>\n",
       "      <td>29210.4</td>\n",
       "      <td>0.05</td>\n",
       "      <td>-9.708142</td>\n",
       "      <td>29229.246311</td>\n",
       "      <td>3.819143</td>\n",
       "      <td>29229.440574</td>\n",
       "      <td>7.39173</td>\n",
       "      <td>29229.343443</td>\n",
       "      <td>0.097131</td>\n",
       "      <td>-0.40492</td>\n",
       "      <td>1752.21</td>\n",
       "      <td>1402.704628</td>\n",
       "      <td>2023-04-30 20:00:00-04:00</td>\n",
       "      <td>29223.00</td>\n",
       "      <td>29245.80</td>\n",
       "      <td>29206.90</td>\n",
       "      <td>29240.70</td>\n",
       "      <td>1550.598</td>\n",
       "      <td>2024-02-19 16:46:59.296616+00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             asset_id        full_symbol  level_1.bid_price.open  level_1.bid_size.open  level_1.ask_price.open  level_1.ask_size.open  level_1.bid_ask_midpoint.open  level_1.half_spread.open  level_1.log_size_imbalance.open  level_1.bid_price.close  level_1.bid_size.close  level_1.ask_price.close  level_1.ask_size.close  level_1.bid_ask_midpoint.close  level_1.half_spread.close  level_1.log_size_imbalance.close  level_1.bid_price.high  level_1.bid_size.max  level_1.ask_price.high  level_1.ask_size.max  level_1.bid_ask_midpoint.max  level_1.half_spread.max  level_1.log_size_imbalance.max  level_1.bid_price.low  level_1.bid_size.min  level_1.ask_price.low  level_1.ask_size.min  level_1.bid_ask_midpoint.min  level_1.half_spread.min  level_1.log_size_imbalance.min  level_1.bid_price.mean  level_1.bid_size.mean  level_1.ask_price.mean  level_1.ask_size.mean  level_1.bid_ask_midpoint.mean  level_1.half_spread.mean  level_1.log_size_imbalance.mean  \\\n",
       "end_ts                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        \n",
       "2023-04-30 20:00:00-04:00  1467591036  binance::BTC_USDT                     NaN                    NaN                     NaN                    NaN                            NaN                       NaN                              NaN                      NaN                     NaN                      NaN                     NaN                             NaN                        NaN                               NaN                     NaN                   NaN                     NaN                   NaN                           NaN                      NaN                             NaN                    NaN                   NaN                    NaN                   NaN                           NaN                      NaN                             NaN                     NaN                    NaN                     NaN                    NaN                            NaN                       NaN                              NaN   \n",
       "2023-04-30 20:00:00-04:00  8968126878  binance::BNB_USDT                     NaN                    NaN                     NaN                    NaN                            NaN                       NaN                              NaN                      NaN                     NaN                      NaN                     NaN                             NaN                        NaN                               NaN                     NaN                   NaN                     NaN                   NaN                           NaN                      NaN                             NaN                    NaN                   NaN                    NaN                   NaN                           NaN                      NaN                             NaN                     NaN                    NaN                     NaN                    NaN                            NaN                       NaN                              NaN   \n",
       "2023-04-30 20:01:00-04:00  1467591036  binance::BTC_USDT                 29223.5                  9.139                 29223.6                  0.421                       29223.55                      0.05                         3.077673                  29240.6                   2.934                  29240.7                  10.842                        29240.65                       0.05                         -1.307061                 29245.7                28.248                 29245.8                69.545                      29245.75                      3.4                        9.682092                29210.0                 0.001                29210.8                 0.001                       29210.4                     0.05                       -9.708142            29229.246311               3.819143            29229.440574                7.39173                   29229.343443                  0.097131                         -0.40492   \n",
       "\n",
       "                           level_1.bid_ask_midpoint_var.100ms  level_1.log_size_imbalance_var.100ms                  start_ts      open      high       low     close    volume              knowledge_timestamp  \n",
       "end_ts                                                                                                                                                                                                            \n",
       "2023-04-30 20:00:00-04:00                                 NaN                                   NaN 2023-04-30 19:59:00-04:00  29247.00  29253.20  29145.50  29223.00  2888.894 2023-05-01 01:22:07.571121+00:00  \n",
       "2023-04-30 20:00:00-04:00                                 NaN                                   NaN 2023-04-30 19:59:00-04:00    336.96    337.15    336.41    336.84  6341.860 2023-05-01 01:22:16.403833+00:00  \n",
       "2023-04-30 20:01:00-04:00                             1752.21                           1402.704628 2023-04-30 20:00:00-04:00  29223.00  29245.80  29206.90  29240.70  1550.598 2024-02-19 16:46:59.296616+00:00  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stitched_mdata = mdmadaex.get_HorizontalStitchedMarketData_example1(\n",
    "    bid_ask_market_data,\n",
    "    ohlcv_market_data,\n",
    "    asset_ids,\n",
    "    stitched_market_data_config[\"stitched_market_data\"][\"columns\"],\n",
    "    stitched_market_data_config[\"stitched_market_data\"][\"column_remap\"],\n",
    "    filter_data_mode=stitched_market_data_config[\"stitched_market_data\"][\n",
    "        \"filter_data_mode\"\n",
    "    ],\n",
    ")\n",
    "stitched_mdata_df = stitched_mdata.get_data_for_interval(\n",
    "    stitched_market_data_config[\"start_timestamp\"],\n",
    "    stitched_market_data_config[\"end_timestamp\"],\n",
    "    stitched_market_data_config[\"stitched_market_data\"][\"ts_col_name\"],\n",
    "    asset_ids,\n",
    ")\n",
    "stitched_mdata_df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f9a23ec",
   "metadata": {},
   "source": [
    "# `ReplayedMarketData`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8847a79",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-02-19T16:58:15.475142Z",
     "start_time": "2024-02-19T16:58:15.475121Z"
    }
   },
   "outputs": [],
   "source": [
    "# TODO(Dan): Add reference code for `ReplayedMarketData`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5b4d3166",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}