experiments/5_nelisa/0_preprocess_profiles.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0. Preprocess nELISA and Cell Painting profiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from nelisa_utils import normalize_select, unify_pert_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "cp_feature_select_ops = [\n",
    "    \"variance_threshold\",\n",
    "    \"correlation_threshold\",\n",
    "    \"drop_na_columns\",\n",
    "    \"blocklist\",\n",
    "    \"drop_outliers\",\n",
    "]\n",
    "\n",
    "norm_confis = {\n",
    "    \"cellpainting\": {\n",
    "        \"path\": \"inputs/cellpainting_profiles.parquet\",\n",
    "        \"norm_group_col\": \"Metadata_Plate\",\n",
    "        \"select_features\": cp_feature_select_ops,\n",
    "    },\n",
    "    \"nelisa\": {\n",
    "        \"path\": \"inputs/nelisa_profiles.parquet\",\n",
    "        \"norm_group_col\": \"Metadata_nelisa_plate_id\",\n",
    "        \"select_features\": None,\n",
    "    },\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Normalizing cellpainting profiles of shape (1535, 919)...\n",
      "Normalized cellpainting profiles to shape (1535, 630)\n",
      "Normalizing nelisa profiles of shape (1525, 222)...\n",
      "Normalized nelisa profiles to shape (1525, 222)\n"
     ]
    }
   ],
   "source": [
    "normed_dfs = {}\n",
    "\n",
    "for norm_name, norm_conf in norm_confis.items():\n",
    "    df = pd.read_parquet(norm_conf.pop(\"path\"))\n",
    "    print(f\"Normalizing {norm_name} profiles of shape {df.shape}...\")\n",
    "    normed_df = normalize_select(df, **norm_conf)\n",
    "    print(f\"Normalized {norm_name} profiles to shape {normed_df.shape}\")\n",
    "    normed_dfs[norm_name] = normed_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unified cellpainting profiles to shape (1512, 630)\n",
      "Unified nelisa profiles to shape (1512, 222)\n"
     ]
    }
   ],
   "source": [
    "unified_dfs = unify_pert_ids(\n",
    "    *list(normed_dfs.values()), pert_col=\"Metadata_broad_sample\"\n",
    ")\n",
    "assert all(len(df) == len(unified_dfs[0]) for df in unified_dfs)\n",
    "\n",
    "for df_id, norm_name in enumerate(norm_confis.keys()):\n",
    "    print(f\"Unified {norm_name} profiles to shape {unified_dfs[df_id].shape}\")\n",
    "    normed_dfs[norm_name] = unified_dfs[df_id]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved normalized cellpainting profiles to outputs/cellpainting_profiles_normalized.parquet\n",
      "Saved normalized nelisa profiles to outputs/nelisa_profiles_normalized.parquet\n"
     ]
    }
   ],
   "source": [
    "for norm_name, normed_df in normed_dfs.items():\n",
    "    normed_df.to_parquet(f\"outputs/{norm_name}_profiles_normalized.parquet\")\n",
    "    print(\n",
    "        f\"Saved normalized {norm_name} profiles to outputs/{norm_name}_profiles_normalized.parquet\"\n",
    "    )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "map_benchmark",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}