analyze_multiple.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c77eb3ae-a3a4-488b-8ea3-a33e85a359e1",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%matplotlib widget\n",
    "\n",
    "import math\n",
    "import multiprocessing\n",
    "import os\n",
    "from os import path\n",
    "import pickle\n",
    "\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "import analysis\n",
    "\n",
    "PARALLEL = False\n",
    "COLORS_FRIENDLY_HEX = [\n",
    "    \"#d73027\",  # red\n",
    "    \"#fc8d59\",  # orange\n",
    "    \"#a68f51\",  # gold\n",
    "    \"#91bfdb\",  # light blue\n",
    "    \"#4575b4\",  # dark blue\n",
    "    \"grey\",\n",
    "]\n",
    "FLOWSS = [50, 100, 150, 200, 500, 1000]\n",
    "DUR_MS = \"15ms\"\n",
    "DESIRED = {\n",
    "    \"incast_queue_across_bursts\",\n",
    "    \"inflight_metrics_across_bursts\",\n",
    "    \"sender_to_inflights_by_burst\",\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "845a4d27-3a3e-44bb-bbdf-d6e56d165349",
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def load_one(exp_dir):\n",
    "    if not path.exists(exp_dir):\n",
    "        return None\n",
    "\n",
    "    data_flp = path.join(exp_dir, \"data.pickle\")\n",
    "    if path.exists(data_flp):\n",
    "        # if False:\n",
    "        print(f\"Loading from file: {exp_dir}...\")\n",
    "        with open(data_flp, \"rb\") as fil:\n",
    "            try:\n",
    "                data = pickle.load(fil)\n",
    "                print(f\"Succeeded loading from file: {data_flp}\")\n",
    "                return data\n",
    "            except KeyboardInterrupt:\n",
    "                raise\n",
    "            except:\n",
    "                print(f\"Warning: Failed loading from file: {data_flp}\")\n",
    "\n",
    "    try:\n",
    "        data = analysis.get_all_metrics_for_exp(\n",
    "            exp_dir, interp_delta=1e5, desired=DESIRED\n",
    "        )\n",
    "    except KeyboardInterrupt:\n",
    "        raise\n",
    "    except:\n",
    "        print(f\"Error during: {exp_dir}\")\n",
    "        raise\n",
    "\n",
    "    with open(data_flp, \"wb\") as fil:\n",
    "        pickle.dump(data, fil)\n",
    "    return data\n",
    "\n",
    "\n",
    "def load_all(sweep_dir, filt=None):\n",
    "    exp_dirs = [\n",
    "        path.join(sweep_dir, dirn)\n",
    "        for dirn in os.listdir(sweep_dir)\n",
    "        if dirn != \"graphs\" and dirn != \"tmpfs\" and dirn[-7:] != \".pickle\"\n",
    "    ]\n",
    "    exp_dirs = [\n",
    "        exp_dir\n",
    "        for exp_dir in exp_dirs\n",
    "        if filt is None or filt(analysis.get_config_json(exp_dir))\n",
    "    ]\n",
    "\n",
    "    print(f\"Loading {len(exp_dirs)} experiments...\")\n",
    "    if PARALLEL:\n",
    "        with multiprocessing.Pool(processes=20) as pool:\n",
    "            exp_to_data = dict(zip(exp_dirs, pool.map(load_one, exp_dirs)))\n",
    "    else:\n",
    "        exp_to_data = {exp_dir: load_one(exp_dir) for exp_dir in exp_dirs}\n",
    "\n",
    "    exp_to_data = {exp: data for exp, data in exp_to_data.items() if data is not None}\n",
    "\n",
    "    print(f\"Loaded {len(exp_to_data)} experiments.\")\n",
    "    return exp_to_data\n",
    "\n",
    "\n",
    "def graph_simple(\n",
    "    lines, graph_dir, xlabel, ylabel, legend_title, prefix=None, suffix=None, width=10\n",
    "):\n",
    "    fig, axes = analysis.get_axes(width=width)\n",
    "    ax = axes[0]\n",
    "\n",
    "    max_x = 0\n",
    "    max_y = 0\n",
    "    for xs, ys, label in lines:\n",
    "        ax.plot(\n",
    "            xs,\n",
    "            ys,\n",
    "            # COLORS_FRIENDLY_HEX[i],\n",
    "            label=label,\n",
    "            linewidth=analysis.LINESIZE,\n",
    "            alpha=0.9,\n",
    "        )\n",
    "        max_x = max(max_x, xs[-1])\n",
    "        max_y = max(max_y, *ys)\n",
    "\n",
    "    ax.set_xlabel(xlabel, fontsize=analysis.FONTSIZE)\n",
    "    ax.set_ylabel(ylabel, fontsize=analysis.FONTSIZE)\n",
    "    ax.set_xlim(left=-0.01 * max_x, right=1.01 * max_x)\n",
    "    ax.set_ylim(bottom=-0.01 * max_y, top=1.1 * max_y)\n",
    "    ax.tick_params(axis=\"x\", labelsize=analysis.FONTSIZE)\n",
    "    ax.tick_params(axis=\"y\", labelsize=analysis.FONTSIZE)\n",
    "    ax.legend(\n",
    "        fontsize=analysis.FONTSIZE,\n",
    "        bbox_to_anchor=(1.02, 0.5),\n",
    "        loc=\"center left\",\n",
    "        title=legend_title,\n",
    "        title_fontsize=analysis.FONTSIZE,\n",
    "        ncols=math.ceil(len(lines) / 7),\n",
    "    )\n",
    "\n",
    "    analysis.show(fig)\n",
    "    analysis.save(graph_dir, prefix, suffix)\n",
    "\n",
    "\n",
    "def graph_avg_queue_length(exp_to_data, dur_ms, graph_dir):\n",
    "    flowss = sorted(\n",
    "        list({data[\"config\"][\"numBurstSenders\"] for data in exp_to_data.values()})\n",
    "    )\n",
    "    rwnds = [None] + sorted(\n",
    "        list(\n",
    "            {\n",
    "                data[\"config\"][\"staticRwndBytes\"]\n",
    "                for data in exp_to_data.values()\n",
    "                if data[\"config\"][\"rwndStrategy\"] == \"static\"\n",
    "            }\n",
    "        )\n",
    "    )\n",
    "\n",
    "    lines = []\n",
    "    for rwnd in rwnds:\n",
    "        xs = []\n",
    "        ys = []\n",
    "        for flows in flowss:\n",
    "            data = [\n",
    "                data\n",
    "                for data in exp_to_data.values()\n",
    "                if (\n",
    "                    (data[\"config\"][\"numBurstSenders\"] == flows)\n",
    "                    and (\n",
    "                        # No RWND\n",
    "                        (rwnd is None and data[\"config\"][\"rwndStrategy\"] == \"none\")\n",
    "                        or\n",
    "                        # RWND\n",
    "                        (\n",
    "                            rwnd is not None\n",
    "                            and data[\"config\"][\"rwndStrategy\"] == \"static\"\n",
    "                            and data[\"config\"][\"staticRwndBytes\"] == rwnd\n",
    "                        )\n",
    "                    )\n",
    "                )\n",
    "            ]\n",
    "            if len(data) == 0:\n",
    "                print(f\"looking for RWND {rwnd} and {flows} flows\")\n",
    "                print(\"\\n\".join(sorted(exp_to_data.keys())))\n",
    "            assert len(data) == 1, f\"Expected 1 but found: {len(data)}\"\n",
    "            data = data[0]\n",
    "            xs.append(data[\"config\"][\"numBurstSenders\"])\n",
    "\n",
    "            _, avg_lengths, _, _, _, _, _ = data[\"incast_queue_across_bursts\"]\n",
    "            ys.append(np.mean(avg_lengths))\n",
    "        lines.append((xs, ys, \"None\" if rwnd is None else round(rwnd / 1024)))\n",
    "\n",
    "    graph_simple(\n",
    "        lines,\n",
    "        graph_dir,\n",
    "        \"flows\",\n",
    "        \"average queue length\\n(packets)\",\n",
    "        \"RWND Clamp (KB)\",\n",
    "        f\"avg_queue_length_{dur_ms}ms\",\n",
    "    )\n",
    "\n",
    "\n",
    "def graph_avg_tput(exp_to_data, dur_ms, graph_dir):\n",
    "    flowss = sorted(\n",
    "        list({data[\"config\"][\"numBurstSenders\"] for data in exp_to_data.values()})\n",
    "    )\n",
    "    rwnds = [None] + sorted(\n",
    "        list(\n",
    "            {\n",
    "                data[\"config\"][\"staticRwndBytes\"]\n",
    "                for data in exp_to_data.values()\n",
    "                if data[\"config\"][\"rwndStrategy\"] == \"static\"\n",
    "            }\n",
    "        )\n",
    "    )\n",
    "\n",
    "    lines = []\n",
    "    for rwnd in rwnds:\n",
    "        xs = []\n",
    "        ys = []\n",
    "        for flows in flowss:\n",
    "            data = [\n",
    "                data\n",
    "                for data in exp_to_data.values()\n",
    "                if (\n",
    "                    (data[\"config\"][\"numBurstSenders\"] == flows)\n",
    "                    and (\n",
    "                        # No RWND\n",
    "                        (rwnd is None and data[\"config\"][\"rwndStrategy\"] == \"none\")\n",
    "                        or\n",
    "                        # RWND\n",
    "                        (\n",
    "                            rwnd is not None\n",
    "                            and data[\"config\"][\"rwndStrategy\"] == \"static\"\n",
    "                            and data[\"config\"][\"staticRwndBytes\"] == rwnd\n",
    "                        )\n",
    "                    )\n",
    "                )\n",
    "            ]\n",
    "            assert len(data) == 1\n",
    "            data = data[0]\n",
    "            xs.append(data[\"config\"][\"numBurstSenders\"])\n",
    "            ys.append(np.mean(data[\"avg_tput_by_burst_bps\"][1:]) / 1e9)\n",
    "        lines.append((xs, ys, \"None\" if rwnd is None else round(rwnd / 1024)))\n",
    "\n",
    "    graph_simple(\n",
    "        lines,\n",
    "        graph_dir,\n",
    "        \"flows\",\n",
    "        \"average throughput\\n(Gbps)\",\n",
    "        \"RWND Clamp (KB)\",\n",
    "        f\"avg_tput_{dur_ms}ms\",\n",
    "    )\n",
    "\n",
    "\n",
    "# Graph queue over time for various RWND thresholds.\n",
    "def graph_queue(\n",
    "    lines,\n",
    "    dur_ms,\n",
    "    marking_threshold_packets,\n",
    "    capacity_packets,\n",
    "    graph_dir,\n",
    "    legend_title=None,\n",
    "    prefix=None,\n",
    "    suffix=None,\n",
    "    ncols=3,\n",
    "):\n",
    "    fig, axes = analysis.get_axes()\n",
    "    ax = axes[0]\n",
    "\n",
    "    LINESTYLES: List[Any] = [\n",
    "        \"solid\",\n",
    "        \"dashdot\",\n",
    "        # (5, (10, 3)),\n",
    "        (0, (5, 7)),\n",
    "        (0, (3, 5, 1, 5)),\n",
    "    ]\n",
    "    \n",
    "    # Plot a line for each RWND clamp.\n",
    "    max_x = 0\n",
    "    max_y = 0\n",
    "    for idx, (xs, ys, label) in enumerate(lines):\n",
    "        # xs = xs - xs[0]\n",
    "        xs = xs * 1e3\n",
    "        ax.plot(\n",
    "            xs,\n",
    "            ys,\n",
    "            drawstyle=\"steps-post\",\n",
    "            label=label,\n",
    "            linewidth=analysis.LINESIZE,\n",
    "            linestyle=LINESTYLES[idx],\n",
    "            alpha=0.9,\n",
    "        )\n",
    "        max_x = max(max_x, xs[-1])\n",
    "        max_y = max(max_y, *ys)\n",
    "\n",
    "    # Draw a line at the marking threshold\n",
    "    ax.plot(\n",
    "        [0, max_x],\n",
    "        [marking_threshold_packets] * 2,\n",
    "        label=\"ECN\\nthreshold\" if dur_ms == 2 else \"ECN threshold\",\n",
    "        color=\"orange\",\n",
    "        linestyle=\"dashed\",\n",
    "        linewidth=analysis.LINESIZE,\n",
    "        alpha=0.9,\n",
    "    )\n",
    "    # Draw a line at the queue capacity\n",
    "    if max_y > capacity_packets / 2:\n",
    "        # Draw a line at the queue capacity\n",
    "        ax.plot(\n",
    "            [0, max_x],\n",
    "            [capacity_packets] * 2,\n",
    "            label=\"queue capacity\",\n",
    "            color=\"red\",\n",
    "            linestyle=\"dotted\",\n",
    "            linewidth=analysis.LINESIZE,\n",
    "            alpha=0.9,\n",
    "        )\n",
    "        max_y = capacity_packets\n",
    "    elif len(lines) == 1:\n",
    "        max_y = capacity_packets / 2\n",
    "\n",
    "    ax.set_xlabel(\"time (ms)\", fontsize=analysis.FONTSIZE)\n",
    "    ax.set_ylabel(\n",
    "        \"packets\" if len(lines) == 1 else \"queue length\\n(packets)\",\n",
    "        fontsize=analysis.FONTSIZE,\n",
    "    )\n",
    "    ax.tick_params(axis=\"x\", labelsize=analysis.FONTSIZE)\n",
    "    ax.tick_params(axis=\"y\", labelsize=analysis.FONTSIZE)\n",
    "    ax.set_xlim(left=-0.01 * max_x, right=1.01 * max_x)\n",
    "    ax.set_ylim(bottom=-0.01 * max_y, top=1.1 * max_y)\n",
    "    print(\"dur_ms\", dur_ms)\n",
    "    if dur_ms == 2:\n",
    "        print(\"putting legend outside\")\n",
    "        ax.legend(\n",
    "            fontsize=analysis.FONTSIZE,\n",
    "            ncols=ncols,\n",
    "            **(\n",
    "                {}\n",
    "                if legend_title is None\n",
    "                else {\"title\": legend_title, \"title_fontsize\": analysis.FONTSIZE}\n",
    "            ),\n",
    "            bbox_to_anchor=(1.02, 0.5),\n",
    "            loc=\"center left\",\n",
    "        )\n",
    "    else:\n",
    "        ax.legend(\n",
    "            fontsize=analysis.FONTSIZE,\n",
    "            loc=\"center right\" if max_y > capacity_packets / 2 else \"upper right\",\n",
    "            ncols=ncols,\n",
    "            **(\n",
    "                {}\n",
    "                if legend_title is None\n",
    "                else {\"title\": legend_title, \"title_fontsize\": analysis.FONTSIZE}\n",
    "            ),\n",
    "        )\n",
    "\n",
    "    plt.tight_layout()\n",
    "    analysis.show(fig)\n",
    "    analysis.save(graph_dir, prefix, suffix)\n",
    "\n",
    "\n",
    "# Graph FCT distribution for various RWND thresholds\n",
    "def graph_fct(lines, graph_dir, dur_ms, fln):\n",
    "    fig, axes = analysis.get_axes(width=6)  # if len(lines) > 1 else 3)\n",
    "    ax = axes[0]\n",
    "\n",
    "    max_x = 0\n",
    "    for fcts, label in lines:\n",
    "        max_x = max(max_x, *fcts)\n",
    "        count, bins_count = np.histogram(fcts, bins=len(fcts))\n",
    "        ax.plot(\n",
    "            bins_count[1:],\n",
    "            np.cumsum(count / sum(count)),\n",
    "            label=label,\n",
    "            linewidth=analysis.LINESIZE,\n",
    "            alpha=0.9,\n",
    "        )\n",
    "\n",
    "    ax.set_xlabel(\"flow duration (ms)\", fontsize=analysis.FONTSIZE)\n",
    "    ax.set_ylabel(\"CDF\", fontsize=analysis.FONTSIZE)\n",
    "    ax.tick_params(axis=\"x\", labelsize=analysis.FONTSIZE)\n",
    "    ax.tick_params(axis=\"y\", labelsize=analysis.FONTSIZE)\n",
    "    ax.set_xlim(left=-0.01 * max_x, right=1.01 * max_x)\n",
    "    ax.set_ylim(bottom=-0.01, top=1.01)\n",
    "    if len(lines) > 1:\n",
    "        ax.legend(\n",
    "            fontsize=analysis.FONTSIZE,\n",
    "            # bbox_to_anchor=(1.02, 0.5),\n",
    "            # loc=\"center left\",\n",
    "            loc=\"upper left\",\n",
    "            # ncols=ncols,\n",
    "            title=\"RWND clamp (KB)\",\n",
    "            title_fontsize=analysis.FONTSIZE,\n",
    "        )\n",
    "\n",
    "    analysis.show(fig)\n",
    "    analysis.save(graph_dir, f\"{fln}_{dur_ms}ms\")\n",
    "\n",
    "\n",
    "# Graph p95 in-flight data over time for various RWND thresholds\n",
    "def graph_p95_bytes_in_flight(exp_to_data, dur_ms, flows, graph_dir):\n",
    "    fig, axes = analysis.get_axes()\n",
    "    ax = axes[0]\n",
    "\n",
    "    # Plot a line for No RWND tuning\n",
    "    none = [\n",
    "        data\n",
    "        for data in exp_to_data.values()\n",
    "        if (\n",
    "            data[\"config\"][\"rwndStrategy\"] == \"none\"\n",
    "            and data[\"config\"][\"numBurstSenders\"] == flows\n",
    "        )\n",
    "    ]\n",
    "    assert len(none) == 1\n",
    "\n",
    "    # Plot a line for each RWND clamp.\n",
    "    max_x = 0\n",
    "    max_y = 0\n",
    "    for data in (\n",
    "        list(\n",
    "            reversed(\n",
    "                sorted(\n",
    "                    (\n",
    "                        data\n",
    "                        for data in exp_to_data.values()\n",
    "                        if (\n",
    "                            data[\"config\"][\"rwndStrategy\"] == \"static\"\n",
    "                            and data[\"config\"][\"staticRwndBytes\"] < 11000\n",
    "                            and data[\"config\"][\"numBurstSenders\"] == flows\n",
    "                        )\n",
    "                    ),\n",
    "                    key=lambda p: p[\"config\"][\"staticRwndBytes\"],\n",
    "                )\n",
    "            )\n",
    "        )\n",
    "        + none\n",
    "    ):\n",
    "        # Skip odd RWND clamps\n",
    "        if (\n",
    "            data[\"config\"][\"rwndStrategy\"] == \"static\"\n",
    "            and round(data[\"config\"][\"staticRwndBytes\"] / 1024) % 2 == 1\n",
    "        ):\n",
    "            continue\n",
    "\n",
    "        # Last element in the tuple is the total in-flight data\n",
    "        xs, _, _, _, _, percentiles, _ = data[\"inflight_metrics_across_bursts\"]\n",
    "        # xs = np.asarray(xs) - xs[0]\n",
    "        xs = xs * 1e3\n",
    "        # Extract p95\n",
    "        ys = percentiles[4]\n",
    "        ys = ys / 1e3\n",
    "        xs = xs[: len(ys)]\n",
    "        ys = ys[: len(xs)]\n",
    "        ax.plot(\n",
    "            xs,\n",
    "            ys,\n",
    "            # COLORS_FRIENDLY_HEX[i],\n",
    "            label=(\n",
    "                \"None\"\n",
    "                if data[\"config\"][\"rwndStrategy\"] == \"none\"\n",
    "                else round(data[\"config\"][\"staticRwndBytes\"] / 1024)\n",
    "            ),\n",
    "            linewidth=analysis.LINESIZE,\n",
    "            alpha=0.9,\n",
    "        )\n",
    "        max_x = max(max_x, xs[-1])\n",
    "        max_y = max(max_y, *ys)\n",
    "\n",
    "    ax.set_xlabel(\"time (ms)\", fontsize=analysis.FONTSIZE)\n",
    "    ax.set_ylabel(\"per-flow in-flight data\\n(p95, KB)\", fontsize=analysis.FONTSIZE)\n",
    "    ax.set_xlim(left=-0.01 * max_x, right=1.01 * max_x)\n",
    "    ax.set_ylim(bottom=-0.01 * max_y, top=1.1 * max_y)\n",
    "    ax.tick_params(axis=\"x\", labelsize=analysis.FONTSIZE)\n",
    "    ax.tick_params(axis=\"y\", labelsize=analysis.FONTSIZE)\n",
    "    ax.legend(\n",
    "        fontsize=analysis.FONTSIZE,\n",
    "        # bbox_to_anchor=(1.02, 0.5),\n",
    "        # loc=\"center left\",\n",
    "        loc=\"upper center\",\n",
    "        title=\"RWND clamp (KB)\",\n",
    "        title_fontsize=analysis.FONTSIZE,\n",
    "        ncols=3,\n",
    "    )\n",
    "\n",
    "    plt.tight_layout()\n",
    "    analysis.show(fig)\n",
    "    analysis.save(\n",
    "        graph_dir,\n",
    "        suffix=f\"staticRwnd_p95_bytes_in_flight_{dur_ms}ms_{none[0]['config']['numBurstSenders']}flows\",\n",
    "    )\n",
    "\n",
    "\n",
    "# Graph total in-flight data over time for various RWND thresholds\n",
    "def graph_total_inflight(exp_to_data, dur_ms, flows, graph_dir):\n",
    "    fig, axes = analysis.get_axes()\n",
    "    ax = axes[0]\n",
    "\n",
    "    # Plot a line for No RWND tuning\n",
    "    none = [\n",
    "        data\n",
    "        for data in exp_to_data.values()\n",
    "        if (\n",
    "            data[\"config\"][\"rwndStrategy\"] == \"none\"\n",
    "            and data[\"config\"][\"numBurstSenders\"] == flows\n",
    "        )\n",
    "    ]\n",
    "    assert len(none) == 1\n",
    "\n",
    "    # Plot a line for each RWND clamp.\n",
    "    max_x = 0\n",
    "    max_y = 0\n",
    "    for data in (\n",
    "        list(\n",
    "            reversed(\n",
    "                sorted(\n",
    "                    (\n",
    "                        data\n",
    "                        for data in exp_to_data.values()\n",
    "                        if (\n",
    "                            data[\"config\"][\"rwndStrategy\"] == \"static\"\n",
    "                            and data[\"config\"][\"staticRwndBytes\"] < 11000\n",
    "                            and data[\"config\"][\"numBurstSenders\"] == flows\n",
    "                        )\n",
    "                    ),\n",
    "                    key=lambda p: p[\"config\"][\"staticRwndBytes\"],\n",
    "                )\n",
    "            )\n",
    "        )\n",
    "        + none\n",
    "    ):\n",
    "        # Skip odd RWND clamps\n",
    "        if (\n",
    "            data[\"config\"][\"rwndStrategy\"] == \"static\"\n",
    "            and round(data[\"config\"][\"staticRwndBytes\"] / 1024) % 2 == 1\n",
    "        ):\n",
    "            continue\n",
    "\n",
    "        bdp_bytes = (\n",
    "            data[\"config\"][\"smallLinkBandwidthMbps\"]\n",
    "            * 1e6\n",
    "            / 8\n",
    "            * 6\n",
    "            * data[\"config\"][\"delayPerLinkUs\"]\n",
    "            / 1e6\n",
    "        )\n",
    "        # print(bdp_bytes)\n",
    "        # Last element in the tuple is the total inflight\n",
    "        xs, _, _, _, _, _, total_ys = data[\"inflight_metrics_across_bursts\"]\n",
    "        # print(total_ys[:10])\n",
    "        # xs = np.array(xs) - xs[0]\n",
    "        xs = xs * 1e3\n",
    "        total_ys = total_ys / bdp_bytes\n",
    "        ax.plot(\n",
    "            xs,\n",
    "            total_ys,\n",
    "            # COLORS_FRIENDLY_HEX[i],\n",
    "            label=(\n",
    "                \"None\"\n",
    "                if data[\"config\"][\"rwndStrategy\"] == \"none\"\n",
    "                else round(data[\"config\"][\"staticRwndBytes\"] / 1024)\n",
    "            ),\n",
    "            linewidth=analysis.LINESIZE,\n",
    "            alpha=0.9,\n",
    "        )\n",
    "        max_x = max(max_x, xs[-1])\n",
    "        max_y = max(max_y, *total_ys)\n",
    "\n",
    "    ax.set_xlabel(\"time (ms)\", fontsize=analysis.FONTSIZE)\n",
    "    ax.set_ylabel(\"total in-flight data\\n(x BDP)\", fontsize=analysis.FONTSIZE)\n",
    "    ax.set_xlim(left=-0.01 * max_x, right=1.01 * max_x)\n",
    "    ax.set_ylim(bottom=-0.01 * max_y, top=1.1 * max_y)\n",
    "    ax.tick_params(axis=\"x\", labelsize=analysis.FONTSIZE)\n",
    "    ax.tick_params(axis=\"y\", labelsize=analysis.FONTSIZE)\n",
    "    ax.legend(\n",
    "        fontsize=analysis.FONTSIZE,\n",
    "        # bbox_to_anchor=(1.02, 0.5),\n",
    "        # loc=\"center left\",\n",
    "        loc=\"upper right\",\n",
    "        title=\"RWND clamp (KB)\",\n",
    "        title_fontsize=analysis.FONTSIZE,\n",
    "        ncols=3,\n",
    "    )\n",
    "\n",
    "    plt.tight_layout()\n",
    "    analysis.show(fig)\n",
    "    analysis.save(\n",
    "        graph_dir,\n",
    "        suffix=f\"staticRwnd_total_in_flight_data_{dur_ms}ms_{none[0]['config']['numBurstSenders']}flows\",\n",
    "    )\n",
    "\n",
    "\n",
    "# Graph all flows in flight data over time for a specific RWND clamp\n",
    "def graph_sender_inflight(exp_to_data, dur_ms, graph_dir, clamp):\n",
    "    fig, axes = analysis.get_axes()\n",
    "    ax = axes[0]\n",
    "\n",
    "    options = [\n",
    "        data\n",
    "        for data in exp_to_data.values()\n",
    "        if data[\"config\"][\"rwndStrategy\"] == \"static\"\n",
    "        and data[\"config\"][\"staticRwndBytes\"] == clamp\n",
    "    ]\n",
    "    assert len(options) == 1\n",
    "    data = options[0]\n",
    "\n",
    "    max_x = 0\n",
    "    max_y = 0\n",
    "    for bursts in data[\"sender_to_inflights_by_burst\"].values():\n",
    "        burst = bursts[-1]\n",
    "        xs, ys = zip(*burst)\n",
    "        # xs = np.array(xs) - xs[0]\n",
    "        xs = xs * 1e3\n",
    "        ax.plot(\n",
    "            xs,\n",
    "            ys,\n",
    "            linewidth=analysis.LINESIZE,\n",
    "            alpha=0.9,\n",
    "        )\n",
    "        max_x = max(max_x, xs[-1])\n",
    "        max_y = max(max_y, *ys)\n",
    "\n",
    "    ax.set_xlabel(\"time (ms)\", fontsize=analysis.FONTSIZE)\n",
    "    ax.set_ylabel(\"in-flight data (bytes)\", fontsize=analysis.FONTSIZE)\n",
    "    # ax.set_xlim(left=0)\n",
    "    ax.set_ylim(bottom=-1, top=max_y * 1.1)\n",
    "    ax.tick_params(axis=\"x\", labelsize=analysis.FONTSIZE)\n",
    "    ax.tick_params(axis=\"y\", labelsize=analysis.FONTSIZE)\n",
    "\n",
    "    plt.tight_layout()\n",
    "    analysis.show(fig)\n",
    "    analysis.save(\n",
    "        graph_dir,\n",
    "        suffix=f\"in_flight_data_{dur_ms}ms_{data['config']['numBurstSenders']}flows_{clamp}KB\",\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8423aae9-a16d-467f-854b-430e831882f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_graphs_for_duration(exp_to_data, dur_ms, graph_dir):\n",
    "    ecn_threshs_packets = {\n",
    "        data[\"config\"][\"smallQueueMinThresholdPackets\"] for data in exp_to_data.values()\n",
    "    }\n",
    "    assert len(ecn_threshs_packets) == 1\n",
    "    ecn_thresh_packets = ecn_threshs_packets.pop()\n",
    "    queue_capacities_packets = {\n",
    "        data[\"config\"][\"smallQueueSizePackets\"] for data in exp_to_data.values()\n",
    "    }\n",
    "    assert len(queue_capacities_packets) == 1\n",
    "    queue_capacity_packets = queue_capacities_packets.pop()\n",
    "\n",
    "    # Average queue length vs. number of flows; Line for each RWND clamp ###########################\n",
    "\n",
    "    print(\n",
    "        f\"Average queue length vs. flow count for duration {dur_ms}ms, for various RWND clamps:\"\n",
    "    )\n",
    "    graph_avg_queue_length(exp_to_data, dur_ms, graph_dir)\n",
    "\n",
    "    # Average throughput vs. number of flows; Line for each RWND clamp ############################\n",
    "\n",
    "    print(\n",
    "        f\"Average throughput vs. flow count for duration {dur_ms}ms, for various RWND clamps:\"\n",
    "    )\n",
    "    graph_avg_tput(exp_to_data, dur_ms, graph_dir)\n",
    "\n",
    "    # Queue length - Special cases #################################################################\n",
    "\n",
    "    #### For 2ms, plot one graph with all flow counts\n",
    "    if dur_ms == 2:\n",
    "        datas = [\n",
    "            data\n",
    "            for data in exp_to_data.values()\n",
    "            if data[\"config\"][\"rwndStrategy\"] == \"none\"\n",
    "        ]\n",
    "\n",
    "        lines = []\n",
    "        for data in sorted(datas, key=lambda d: d[\"config\"][\"numBurstSenders\"]):\n",
    "            if data[\"config\"][\"numBurstSenders\"] not in {100, 150, 200, 500}:\n",
    "                continue\n",
    "            xs, avg_ys, _, _, _, _, _ = data[\"incast_queue_across_bursts\"]\n",
    "            lines.append((xs, avg_ys, data[\"config\"][\"numBurstSenders\"]))\n",
    "\n",
    "        print(f\"Queue length over time for duration {dur_ms}ms, no RWND:\")\n",
    "        graph_queue(\n",
    "            lines,\n",
    "            dur_ms,\n",
    "            ecn_thresh_packets,\n",
    "            queue_capacity_packets,\n",
    "            graph_dir,\n",
    "            prefix=f\"noRwnd_{dur_ms}ms_allFlowCounts\",\n",
    "            suffix=\"incast_queue\",\n",
    "            ncols=1,\n",
    "            legend_title=\"Flow count\",\n",
    "        )\n",
    "\n",
    "    for flows in FLOWSS:\n",
    "        print(flows)\n",
    "\n",
    "        # P95 per-flow in-flight data across bursts ###############################################\n",
    "\n",
    "        print(\n",
    "            f\"p95 (across senders) in-flight data for duration {dur_ms}ms and {flows} flows, various RWND clamps:\"\n",
    "        )\n",
    "        graph_p95_bytes_in_flight(exp_to_data, dur_ms, flows, graph_dir)\n",
    "\n",
    "        # Total in-flight data across bursts ######################################################\n",
    "\n",
    "        print(\n",
    "            f\"Total in-flight data for duration {dur_ms}ms and {flows} flows, various RWND clamps:\"\n",
    "        )\n",
    "        graph_total_inflight(exp_to_data, dur_ms, flows, graph_dir)\n",
    "\n",
    "        # FCT #####################################################################################\n",
    "\n",
    "        #### FCT - No RWND tuning\n",
    "\n",
    "        none = [\n",
    "            data\n",
    "            for data in exp_to_data.values()\n",
    "            if (\n",
    "                data[\"config\"][\"numBurstSenders\"] == flows\n",
    "                and data[\"config\"][\"rwndStrategy\"] == \"none\"\n",
    "            )\n",
    "        ]\n",
    "        assert len(none) == 1\n",
    "        none = none[0]\n",
    "\n",
    "        fcts = []\n",
    "        # Merge all samples across bursts\n",
    "        for burst_idx in range(1, none[\"config\"][\"numBursts\"]):\n",
    "            times = [\n",
    "                flow_times_by_burst[burst_idx]\n",
    "                for flow_times_by_burst in none[\n",
    "                    \"sender_to_flow_times_by_burst\"\n",
    "                ].values()\n",
    "            ]\n",
    "            fcts.extend([(end - start) * 1e3 for start, _, end, _ in times])\n",
    "        # (FCTs, label)\n",
    "        lines = [(fcts, None)]\n",
    "\n",
    "        print(f\"CDF of FCT for duration {dur_ms}ms and {flows} flows, no RWND:\")\n",
    "        graph_fct(\n",
    "            lines,\n",
    "            graph_dir,\n",
    "            dur_ms,\n",
    "            f\"noRwnd_{dur_ms}ms_{none['config']['numBurstSenders']}flows_fct\",\n",
    "        )\n",
    "\n",
    "        #### FCT - Line for each RWND clamp\n",
    "\n",
    "        datas = [\n",
    "            data\n",
    "            for data in exp_to_data.values()\n",
    "            if data[\"config\"][\"numBurstSenders\"] == flows\n",
    "        ]\n",
    "        # Add a line for No RWND tuning\n",
    "        none = [data for data in datas if data[\"config\"][\"rwndStrategy\"] == \"none\"]\n",
    "        assert len(none) == 1\n",
    "\n",
    "        # Add a line for each RWND clamp.\n",
    "        lines = []\n",
    "        for data in (\n",
    "            list(\n",
    "                reversed(\n",
    "                    sorted(\n",
    "                        (\n",
    "                            data\n",
    "                            for data in datas\n",
    "                            if (\n",
    "                                data[\"config\"][\"rwndStrategy\"] == \"static\"\n",
    "                                and data[\"config\"][\"staticRwndBytes\"] < 11000\n",
    "                            )\n",
    "                        ),\n",
    "                        key=lambda p: p[\"config\"][\"staticRwndBytes\"],\n",
    "                    )\n",
    "                )\n",
    "            )\n",
    "            + none\n",
    "        ):\n",
    "            # Skip odd RWND clamps\n",
    "            if (\n",
    "                data[\"config\"][\"rwndStrategy\"] == \"static\"\n",
    "                and round(data[\"config\"][\"staticRwndBytes\"] / 1024) % 2 == 1\n",
    "            ):\n",
    "                continue\n",
    "\n",
    "            fcts = []\n",
    "            for burst_idx in range(1, data[\"config\"][\"numBursts\"]):\n",
    "                times = [\n",
    "                    flow_times_by_burst[burst_idx]\n",
    "                    for flow_times_by_burst in data[\n",
    "                        \"sender_to_flow_times_by_burst\"\n",
    "                    ].values()\n",
    "                ]\n",
    "                fcts.extend([(end - start) * 1e3 for start, _, end, _ in times])\n",
    "            lines.append(\n",
    "                (\n",
    "                    fcts,\n",
    "                    # Label\n",
    "                    (\n",
    "                        \"None\"\n",
    "                        if data[\"config\"][\"rwndStrategy\"] == \"none\"\n",
    "                        else round(data[\"config\"][\"staticRwndBytes\"] / 1024)\n",
    "                    ),\n",
    "                )\n",
    "            )\n",
    "\n",
    "        print(\n",
    "            f\"CDF of FCT for duration {dur_ms}ms and {flows} flows, for various RWND clamps:\"\n",
    "        )\n",
    "        graph_fct(lines, graph_dir, dur_ms, f\"staticRwnd_{dur_ms}ms_{flows}flows_fct\")\n",
    "\n",
    "        # Queue length #############################################################################\n",
    "\n",
    "        #### Queue length - No RWND tuning\n",
    "\n",
    "        datas = [\n",
    "            data\n",
    "            for data in exp_to_data.values()\n",
    "            if data[\"config\"][\"numBurstSenders\"] == flows\n",
    "        ]\n",
    "        # Add a line for No RWND tuning\n",
    "        none = [data for data in datas if data[\"config\"][\"rwndStrategy\"] == \"none\"]\n",
    "        assert len(none) == 1\n",
    "        none = none[0]\n",
    "\n",
    "        xs, avg_ys, _, _, _, _, _ = none[\"incast_queue_across_bursts\"]\n",
    "        xs = np.asarray(xs)\n",
    "        lines = [(xs, avg_ys, \"queue length\")]\n",
    "\n",
    "        print(\n",
    "            f\"Queue length over time for duration {dur_ms}ms and {flows} flows, no RWND:\"\n",
    "        )\n",
    "        graph_queue(\n",
    "            lines,\n",
    "            dur_ms,\n",
    "            ecn_thresh_packets,\n",
    "            queue_capacity_packets,\n",
    "            graph_dir,\n",
    "            prefix=f\"noRwnd_{dur_ms}ms_{flows}flows\",\n",
    "            suffix=\"incast_queue\",\n",
    "            ncols=2,\n",
    "        )\n",
    "\n",
    "        #### Queue length - Line for each RWND clamp\n",
    "\n",
    "        datas = [\n",
    "            data\n",
    "            for data in exp_to_data.values()\n",
    "            if data[\"config\"][\"numBurstSenders\"] == flows\n",
    "        ]\n",
    "\n",
    "        # Add a line for No RWND tuning\n",
    "        none = [data for data in datas if data[\"config\"][\"rwndStrategy\"] == \"none\"]\n",
    "        assert len(none) == 1\n",
    "\n",
    "        # Add a line for each RWND clamp.\n",
    "        lines = []\n",
    "        for data in (\n",
    "            list(\n",
    "                reversed(\n",
    "                    sorted(\n",
    "                        (\n",
    "                            data\n",
    "                            for data in datas\n",
    "                            if (\n",
    "                                data[\"config\"][\"rwndStrategy\"] == \"static\"\n",
    "                                and data[\"config\"][\"staticRwndBytes\"] < 11000\n",
    "                            )\n",
    "                        ),\n",
    "                        key=lambda p: p[\"config\"][\"staticRwndBytes\"],\n",
    "                    )\n",
    "                )\n",
    "            )\n",
    "            + none\n",
    "        ):\n",
    "            # Skip odd RWND clamps\n",
    "            if (\n",
    "                data[\"config\"][\"rwndStrategy\"] == \"static\"\n",
    "                and round(data[\"config\"][\"staticRwndBytes\"] / 1024) % 2 == 1\n",
    "            ):\n",
    "                continue\n",
    "            xs, avg_ys, _, _, _, _, _ = data[\"incast_queue_across_bursts\"]\n",
    "            xs = np.asarray(xs)\n",
    "            lines.append(\n",
    "                (\n",
    "                    xs,\n",
    "                    avg_ys,\n",
    "                    \"None\"\n",
    "                    if data[\"config\"][\"rwndStrategy\"] == \"none\"\n",
    "                    else round(data[\"config\"][\"staticRwndBytes\"] / 1024),\n",
    "                )\n",
    "            )\n",
    "\n",
    "        print(\n",
    "            f\"Queue length over time for duration {dur_ms}ms and {flows} flows, for various RWND clamps:\"\n",
    "        )\n",
    "        graph_queue(\n",
    "            lines,\n",
    "            dur_ms,\n",
    "            ecn_thresh_packets,\n",
    "            queue_capacity_packets,\n",
    "            graph_dir,\n",
    "            legend_title=\"RWND clamp (KB)\",\n",
    "            prefix=f\"staticRwnd_{dur_ms}ms_{flows}flows\",\n",
    "            suffix=\"incast_queue\",\n",
    "            ncols=2 if dur_ms == 2 else 3,\n",
    "        )\n",
    "\n",
    "    # for flowS in FLOWSS:\n",
    "    #     print(flowS)\n",
    "    #     graph_sender_inflight(\n",
    "    #         {\n",
    "    #             exp: data\n",
    "    #             for exp, data in exp_to_data.items()\n",
    "    #             if data[\"config\"][\"numBurstSenders\"] == flowS\n",
    "    #         },\n",
    "    #         SWEEP_DIR,\n",
    "    #         2048\n",
    "    #     )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1b15f39-ebbf-4510-b665-5c381b5d731a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_duration(sweep_dir, dur_ms, reload):\n",
    "    save_flp = path.join(sweep_dir, f\"{dur_ms}.pickle\")\n",
    "    if reload or not path.exists(save_flp):\n",
    "        exp_to_data = load_all(\n",
    "            sweep_dir,\n",
    "            filt=lambda c: (\n",
    "                f\"{dur_ms}ms\" in c[\"outputDirectory\"]\n",
    "                # and c[\"numBurstSenders\"] <= 1000\n",
    "                # and c[\"numBurstSenders\"] < 1000\n",
    "                # and c[\"smallLinkBandwidthMbps\"] == 10000\n",
    "                # and c[\"smallQueueMinThresholdPackets\"] == 65\n",
    "                # and c[\"smallQueueSizePackets\"] == 667\n",
    "                # and c[\"smallQueueSizePackets\"] == 1334\n",
    "                # and c[\"rwndStrategy\"] in [\"none\", \"static\"]\n",
    "                # and c[\"numBurstSenders\"] in FLOWSS\n",
    "            ),\n",
    "        )\n",
    "        print(\"Saving:\", save_flp)\n",
    "        with open(save_flp, \"wb\") as fil:\n",
    "            pickle.dump(exp_to_data, fil)\n",
    "        print(\"Saved.\")\n",
    "    else:\n",
    "        print(\"Loading:\", save_flp)\n",
    "        with open(save_flp, \"rb\") as fil:\n",
    "            exp_to_data = pickle.load(fil)\n",
    "        print(\"Loaded.\")\n",
    "    return exp_to_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10e0a83d-3849-46a8-9a8f-eb94b4e91ac5",
   "metadata": {},
   "outputs": [],
   "source": [
    "SWEEP_DIR = \"/data_ssd/ccanel/data/imc2024/sweep/background-senders\"\n",
    "GRAPH_DIR = path.join(SWEEP_DIR, \"graphs\")\n",
    "if not path.isdir(GRAPH_DIR):\n",
    "    os.makedirs(GRAPH_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c625021-c30c-4790-8e83-565b4caa94ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "EXP_TO_DATA_2MS = load_duration(SWEEP_DIR, 2, False)\n",
    "generate_graphs_for_duration(EXP_TO_DATA_2MS, 2, GRAPH_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54a76669-dea9-45d8-9e38-1c66c45b2535",
   "metadata": {},
   "outputs": [],
   "source": [
    "EXP_TO_DATA_15MS = load_duration(SWEEP_DIR, 15, False)\n",
    "generate_graphs_for_duration(EXP_TO_DATA_15MS, 15, GRAPH_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6c22213-6bb3-4bd8-b58a-5b654d11496f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# delack_sweep_dir = \"/data_hdd/incast/out/delack_sweep_2ms\"\n",
    "# delack_sweep_exp_to_data = load_sweep(delack_sweep_dir)\n",
    "\n",
    "# def graph_delack_sweep(exp_to_data, sweep_dir):\n",
    "\n",
    "#     def get_x(data):\n",
    "#         return data[\"config\"][\"delAckCount\"]\n",
    "\n",
    "#     def get_y(data):\n",
    "#         return np.mean([end - start for start, end in data[\"burst_times\"][1:]])\n",
    "\n",
    "#     points = [(get_x(data), get_y(data)) for exp, data in exp_to_data.items()]\n",
    "#     points = sorted(points)\n",
    "#     xs, ys = zip(*points)\n",
    "#     ys = [y * 1e3 for y in ys]\n",
    "\n",
    "#     fig, axes = analysis.get_axes(width=5)\n",
    "#     ax = ax\n",
    "\n",
    "#     ax.plot(xs, ys, \"o-\", alpha=0.7)\n",
    "\n",
    "#     ax.set_title(f\"Average burst duration vs. DelAckCount\")\n",
    "#     ax.set_xlabel(\"DelAckCount\")\n",
    "#     ax.set_ylabel(\"Burst duration (ms)\")\n",
    "#     ax.set_xlim(left=0)\n",
    "#     ax.set_ylim(bottom=0)\n",
    "\n",
    "#     plt.tight_layout()\n",
    "#     analysis.show(fig)\n",
    "#     analysis.save(graph_dir, suffix=\"duration\")\n",
    "\n",
    "\n",
    "# def graph_delack_sweep_cdf(exp_to_data, sweep_dir):\n",
    "\n",
    "#     label_to_durations = {\n",
    "#         data[\"config\"][\"delAckCount\"]: [\n",
    "#             (end - start) * 1e3 for start, end in data[\"burst_times\"]\n",
    "#         ]\n",
    "#         for exp, data in exp_to_data.items()\n",
    "#     }\n",
    "\n",
    "#     fig, axes = analysis.get_axes(width=5)\n",
    "#     ax = ax\n",
    "\n",
    "#     for label, durations in sorted(label_to_durations.items()):\n",
    "#         plt.plot(\n",
    "#             np.sort(durations),\n",
    "#             np.arange(1, len(durations) + 1) / float(len(durations)),\n",
    "#             drawstyle=\"steps-post\",\n",
    "#             label=label,\n",
    "#             alpha=0.7,\n",
    "#         )\n",
    "\n",
    "#     ideal_ms = list(set(data[\"ideal_sec\"] * 1e3 for data in exp_to_data.values()))\n",
    "#     assert len(ideal_ms) == 1\n",
    "#     ideal_ms = ideal_ms[0]\n",
    "#     # plt.axvline(ideal_ms, color=\"b\", linestyle=\"dashed\", label=\"Ideal\")\n",
    "\n",
    "#     ax.set_title(f\"Burst duration CDF for various DelAckCount\")\n",
    "#     ax.set_xlabel(\"Burst duration (ms)\")\n",
    "#     ax.set_ylabel(\"CDF\")\n",
    "#     ax.set_xlim(left=0)\n",
    "#     ax.set_ylim(bottom=0, top=1.01)\n",
    "#     # ax.legend()\n",
    "\n",
    "#     plt.tight_layout()\n",
    "#     analysis.show(fig)\n",
    "#     analysis.save(graph_dir, suffix=\"duration_cdf\")\n",
    "\n",
    "# graph_delack_sweep(delack_sweep_exp_to_data, delack_sweep_dir)\n",
    "# graph_delack_sweep_cdf(delack_sweep_exp_to_data, delack_sweep_dir)\n",
    "\n",
    "# Interesting things to plot:\n",
    "# - RTT CDF: Randomly sample 1000 RTTs from each configuration, whenever there is actually traffic flowing\n",
    "# - Average non-empty queue length: Average length of the incast queue whenever it's not 0 or 1."
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "formats": "ipynb,py:percent"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}