T01-Saga roadmap.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "import sys\n",
    "sys.path.insert(0, \"/home/yymao/Dropbox/Academia/Collaborations/SAGA-local/saga\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import SAGA\n",
    "from easyquery import Query, QueryMaker\n",
    "from SAGA.database import FitsTable\n",
    "from SAGA import ObjectCuts as C\n",
    "from SAGA import HostCuts as H\n",
    "from astropy.table import unique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "saga = SAGA.QuickStart()\n",
    "saga.set_default_base_version(\"paper2\")\n",
    "\n",
    "saga_cur = SAGA.QuickStart()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "specs_1 = FitsTable(saga._shared_dir + \"/Paper1/saga_spectra_clean.fits.gz\").read()\n",
    "\n",
    "specs_1[\"HOSTID\"] = np.char.add(\"nsa\", specs_1[\"HOST_NSAID\"].astype(\"<7U\"))\n",
    "specs_1[\"r_mag\"] = specs_1[\"r\"] - specs_1[\"EXTINCTION_R\"]\n",
    "specs_1[\"Mr\"] = specs_1[\"r_mag\"] - np.log10(specs_1[\"HOST_DIST\"])*5 - 25\n",
    "\n",
    "assert len(np.unique(specs_1[\"HOSTID\"])) == 74\n",
    "\n",
    "specs_1 = Query(saga.host_catalog.construct_host_query(\"paper1_complete\"), C.is_galaxy).filter(specs_1)\n",
    "has_our_specs_only_p1 = (QueryMaker.contains(\"SPEC_REPEAT\", \"IMACS\") | QueryMaker.contains(\"SPEC_REPEAT\", \"WIYN\") | C.has_our_specs_only)\n",
    "print(has_our_specs_only_p1.count(specs_1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_duplicated(specs):\n",
    "    specs[\"SATS_sort\"] = np.where(specs[\"SATS\"] < 1, 999, specs[\"SATS\"])\n",
    "    specs.sort(\"SATS_sort\")\n",
    "    del specs[\"SATS_sort\"]\n",
    "    specs = unique(specs, \"OBJID\")\n",
    "    specs.sort(\"HOSTID\")\n",
    "    return specs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "specs_2 = saga.object_catalog.load_clean_specs()\n",
    "specs_2 = Query(C.is_clean2, C.is_galaxy2, saga.host_catalog.construct_host_query(\"paper2_complete\")).filter(specs_2)\n",
    "specs_2 = remove_duplicated(specs_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hostlist = saga_cur.host_catalog.load(\"good\", add_coord=False, include_stats=True)\n",
    "sorter = (hostlist[\"paper2_need_spec\"] / hostlist[\"paper2_total\"]).argsort()\n",
    "paper3_planned = QueryMaker.in1d(\"HOSTID\", hostlist[\"HOSTID\"][sorter[:100]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "specs_3 = saga_cur.object_catalog.load_clean_specs()\n",
    "specs_3 = Query(C.is_clean2, C.is_galaxy2, paper3_planned).filter(specs_3)\n",
    "specs_3 = remove_duplicated(specs_3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "q = Query(C.paper2_targeting_cut, C.basic_cut2)\n",
    "low_z = Query(\"SPEC_Z >= 0.003\", \"SPEC_Z < 0.03\")\n",
    "\n",
    "ratio_sf = Query(q, C.is_sat, ~C.sdss_limit).count(specs_2) / Query(q, ~C.sdss_limit).count(specs_2) \n",
    "ratio_sb = Query(q, C.is_sat, C.sdss_limit).count(specs_2) / Query(q, C.sdss_limit).count(specs_2)\n",
    "\n",
    "ratio_lf = Query(q, low_z, ~C.is_sat, ~C.sdss_limit).count(specs_2) / Query(q, ~C.sdss_limit).count(specs_2) \n",
    "ratio_lb = Query(q, low_z, ~C.is_sat,  C.sdss_limit).count(specs_2) / Query(q, C.sdss_limit).count(specs_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = []\n",
    "\n",
    "for i, specs in enumerate([\n",
    "    C.sat_rcut.filter(specs_1), \n",
    "    specs_1, \n",
    "    C.sat_rcut.filter(specs_2), \n",
    "    specs_2, \n",
    "    C.sat_rcut.filter(specs_3),\n",
    "    specs_3,\n",
    "]):\n",
    "    if i < 2:\n",
    "        has_our_specs_only = has_our_specs_only_p1\n",
    "    else:\n",
    "        has_our_specs_only = C.has_our_specs_only\n",
    "    \n",
    "    if i <= 1: \n",
    "        paper1_fix = -2\n",
    "    else:\n",
    "        paper1_fix = 0\n",
    "        \n",
    "    if i >= 4:\n",
    "        paper3_new = Query(paper3_planned, ~saga_cur.host_catalog.construct_host_query(\"paper2_complete\"))\n",
    "        spec_funge = paper3_new.filter(hostlist, \"paper2_need_spec\").sum() + (50 * paper3_new.count(hostlist))\n",
    "        nb = paper3_new.filter(hostlist, \"really_need_spec_bright\").sum()\n",
    "        nf = paper3_new.filter(hostlist, \"paper2_need_spec\").sum() - nb\n",
    "        sat_funge = int((nb * ratio_sb + nf * ratio_sf))\n",
    "        lowz_funge = int((nb * ratio_lb + nf * ratio_lf))\n",
    "        if i == 5:\n",
    "            spec_funge += int(85 * paper3_new.count(hostlist))\n",
    "            lowz_funge += int(3 * paper3_new.count(hostlist))\n",
    "            \n",
    "        def rounding(x):\n",
    "            if x >= 10000:\n",
    "                step = 1000\n",
    "            elif x >= 1000:\n",
    "                step = 100\n",
    "            else:\n",
    "                step = 50\n",
    "            return int(np.round(x / step)*step)\n",
    "    else:\n",
    "        spec_funge = 0\n",
    "        sat_funge = 0\n",
    "        lowz_funge = 0\n",
    "        rounding = lambda x: x\n",
    "        \n",
    "    X.append((\n",
    "        len(np.unique(specs[\"HOSTID\"])),\n",
    "        rounding(has_our_specs_only.count(specs) + spec_funge + paper1_fix),\n",
    "        rounding(len(specs) + spec_funge),\n",
    "        rounding(Query(C.is_sat, has_our_specs_only, C.r_abs_limit).count(specs) + sat_funge),\n",
    "        rounding(Query(C.is_sat, has_our_specs_only, ~C.r_abs_limit).count(specs)),\n",
    "        rounding(Query(C.is_sat, C.r_abs_limit).count(specs) + sat_funge),\n",
    "        rounding(Query(C.is_sat, ~C.r_abs_limit).count(specs)),\n",
    "        rounding(Query(low_z, has_our_specs_only).count(specs) + sat_funge + lowz_funge),\n",
    "        rounding(Query(low_z).count(specs) + sat_funge + lowz_funge),\n",
    "    ))\n",
    "    \n",
    "X = np.array(X).T\n",
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = [r\"\\# Complete Hosts\", r\"\\# Galaxy Redshifts SAGA Obtained\", r\"Total \\# Galaxy Redshifts\", \n",
    "          \"\", r\"\\# Satellites Discovered\",\n",
    "          \"\", r\"Total \\# Satellites\",\n",
    "          r\"\\# Low-$z$ Galaxies Discovered\", r\"Total \\# Low-$z$ Galaxies\"]\n",
    "\n",
    "sim = [\"\", \"\", r\"$\\sim$\\,\"]\n",
    "\n",
    "def print_row(label, out_iter, nline):\n",
    "    ending = \"\" if nline is None else (r\"\\\\ \" + (r\"\\hline\" * nline))\n",
    "    out = \" & \".join(out_iter)\n",
    "    out = out.replace(\" (0)\", \"\")\n",
    "    print(label, \"&\", out, ending)\n",
    "\n",
    "for i, (label, row) in enumerate(zip(labels, X)):\n",
    "    row = [\"{:,}\".format(r) for j, r in enumerate(row)]\n",
    "    \n",
    "    if i == 0:\n",
    "        print_row(label, (f\"\\multicolumn{{2}}{{c{bar}}}{{{item}}}\" for bar, item in zip([\"|\", \"|\", \"\"], row[::2])), 2)\n",
    "        print_row(\"Field Definition\", (r\"$\\mathbf{<300}$\\,\\textbf{kpc} & $<1^{\\circ}$\" for _ in range (3)), 1)\n",
    "    elif i in (3, 5):\n",
    "        hold = np.array(row)\n",
    "    elif i in (4, 6):\n",
    "        out_tier = (fr\"{s}\\textbf{{{item} ({i2})}} & \\ditto\" for item, i2, s in zip(hold[::2], row[::2], sim))\n",
    "        print_row(label, out_tier, 1)\n",
    "    else:\n",
    "        out_tier = (fr\"{s}\\textbf{{{i1}}} & {s}{i2}\" for i1, i2, s in zip(row[::2], row[1::2], sim))\n",
    "        print_row(label, out_tier, 1 if (i < len(X) -1) else None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p2_hostlist = saga.host_catalog.load(include_stats=True, query=\"paper2_complete\")\n",
    "\n",
    "numbers_dict = {\n",
    "    \"nhosts\": len(p2_hostlist),\n",
    "    \"nhostsnew\": (~H.paper1_complete).count(p2_hostlist), \n",
    "    \"nhoststotal\": len(saga.host_catalog.load(\"good\")), \n",
    "    \"nhostsinclnoimage\": len(saga.host_catalog.load(\"good_hosts\")), \n",
    "    \"nhostswsats\": Query(\"sats_total > 0\").count(p2_hostlist), \n",
    "    \"nhostswsatslimit\": Query(\"sats_Mr_limit > 0\").count(p2_hostlist), \n",
    "    \"nhostsnosats\": Query(\"sats_total == 0\").count(p2_hostlist), \n",
    "    \"nhostsnosatslimit\": Query(\"sats_Mr_limit == 0\").count(p2_hostlist), \n",
    "\n",
    "    \"nsats\": X[5, 2] + X[6, 2], \n",
    "    \"nsatslimit\": X[5, 2], \n",
    "    \"nsatsbelow\": X[6, 2], \n",
    "    \"nsatssaga\": X[3, 2] + X[4, 2], \n",
    "    \"nsatslimitsaga\": X[3, 2], \n",
    "    \"nsatsother\": X[5, 2] + X[6, 2] - X[3, 2] - X[4, 2], \n",
    "    \"nsatslimitother\": X[5, 2] - X[3, 2], \n",
    "\n",
    "    \"nzsaga\": X[1, 3], \n",
    "    \"nztotal\": X[2, 3], \n",
    "    \"nzsaganew\": X[1, 3] - X[1, 1],\n",
    "    \"nztotalnew\": X[2, 3] - X[2, 1], \n",
    "    \"nzother\": X[2, 3] - X[1, 3], \n",
    "    \"nzothernew\": (X[2, 3] - X[1, 3]) - (X[2, 1] - X[1, 1]), \n",
    "\n",
    "    \"nzsagarvir\": X[1, 2], \n",
    "    \"nztotalrvir\": X[2, 2],  \n",
    "    \"nzsaganewrvir\": X[1, 2] - X[1, 0],\n",
    "    \"nztotalnewrvir\": X[2, 2] - X[2, 0], \n",
    "    \"nzotherrvir\": X[2, 2] - X[1, 2], \n",
    "    \"nzothernewrvir\": (X[2, 2] - X[1, 2]) - (X[2, 0] - X[1, 0]),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k, n in numbers_dict.items():\n",
    "    print(\"\\\\newcommand*{{\\\\{}}}{{{:,}}}\".format(k, n))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}