docs/mnar_multivariate_examples.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MNAR mechanism\n",
    "\n",
    "This Jupyter Notebook provides various strategies to generate artificial missing data under MNAR mechanism in multivariate scenario. \n",
    "Our example is using the Breast Cancer Wiscosin dataset from Scikit-learn.\n",
    "\n",
    "The approaches covered in this Jupyter Notebook are:\n",
    "- Random\n",
    "- Correlated\n",
    "- Missigness Based on Own Values (MBOV) using a randomess to choose miss locations in each feature\n",
    "- Missingness Based on Intra-Relation (MBIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import the libraries\n",
    "import pandas as pd\n",
    "from sklearn.datasets import load_breast_cancer\n",
    "\n",
    "from mdatagen.multivariate.mMNAR import mMNAR\n",
    "\n",
    "# Load the data\n",
    "wiscosin = load_breast_cancer()\n",
    "wiscosin_df = pd.DataFrame(data=wiscosin.data, columns=wiscosin.feature_names)\n",
    "\n",
    "X = wiscosin_df.copy()   # Features\n",
    "y = wiscosin.target    # Label values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Global Missing rate = 0.19\n",
      "\n",
      "\n",
      "Columns that receive the missingness:\n",
      "mean radius\n",
      "mean compactness\n",
      "mean symmetry\n",
      "texture error\n",
      "worst texture\n",
      "worst smoothness\n",
      "worst concave points\n",
      "worst symmetry\n"
     ]
    }
   ],
   "source": [
    "# Create a instance for MNAR mechanism\n",
    "generator = mMNAR(X=X, \n",
    "                  y=y,\n",
    "                  n_xmiss = 8, # 8 feature that will receive the missingness randomly\n",
    "                  threshold = 1) # highest values\n",
    "\n",
    "# Generate the missing data under MNAR mechanism up to 20% missing rate\n",
    "generate_data = generator.random(missing_rate=20, \n",
    "                                 deterministic=True) # Missingness based on own values\n",
    "\n",
    "qtd_miss = sum(generate_data.isna().sum())\n",
    "print(f\"Global Missing rate = {round(qtd_miss/(generate_data.shape[0]*generate_data.shape[1]),2)}\")\n",
    "print(\"\\n\")\n",
    "print(\"Columns that receive the missingness:\")\n",
    "for col in generate_data.columns:\n",
    "    if generate_data[col].isna().sum() > 0:\n",
    "        print(col)\n",
    "    else:\n",
    "        pass\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Correlated "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Global Missing rate = 0.2\n",
      "\n",
      "\n",
      "Columns that receive the missingness:\n",
      "mean perimeter\n",
      "mean concave points\n",
      "radius error\n",
      "area error\n",
      "smoothness error\n",
      "compactness error\n",
      "concave points error\n",
      "worst texture\n",
      "worst perimeter\n",
      "worst area\n",
      "worst smoothness\n",
      "worst concavity\n",
      "worst concave points\n",
      "worst symmetry\n",
      "worst fractal dimension\n"
     ]
    }
   ],
   "source": [
    "# Create a instance for MNAR mechanism\n",
    "generator = mMNAR(X=X, \n",
    "                  y=y,\n",
    "                  threshold = 1) # highest values\n",
    "\n",
    "# Generate the missing data under MNAR mechanism up to 20% missing rate\n",
    "generate_data = generator.correlated(missing_rate=20, \n",
    "                                 deterministic=True) # Missingness based on own values\n",
    "\n",
    "qtd_miss = sum(generate_data.isna().sum())\n",
    "print(f\"Global Missing rate = {round(qtd_miss/(generate_data.shape[0]*generate_data.shape[1]),2)}\")\n",
    "print(\"\\n\")\n",
    "print(\"Columns that receive the missingness:\")\n",
    "for col in generate_data.columns:\n",
    "    if generate_data[col].isna().sum() > 0:\n",
    "        print(col)\n",
    "    else:\n",
    "        pass\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Missigness Based on Own Values (MBOV) using a randomess to choose miss locations in each feature -> MBOV_randomness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Global Missing rate = 0.19\n",
      "\n",
      "\n",
      "Columns that receive the missingness:\n",
      "mean radius\n",
      "texture error\n",
      "area error\n"
     ]
    }
   ],
   "source": [
    "# Create a instance for MNAR mechanism\n",
    "generator = mMNAR(X=X, \n",
    "                  y=y,\n",
    "                  ) \n",
    "\n",
    "list_to_gen_miss = [\"mean radius\",\"texture error\", \"area error\"]\n",
    "\n",
    "# Generate the missing data under MNAR mechanism up to 20% missing rate\n",
    "generate_data = generator.MBOV_randomness(missing_rate=20, \n",
    "                                          columns=list_to_gen_miss,\n",
    "                                          randomness=0.3)\n",
    "\n",
    "                                        \n",
    "qtd_miss = sum(generate_data.isna().sum())\n",
    "print(f\"Global Missing rate = {round(qtd_miss / (len(list_to_gen_miss) * generate_data.shape[0]),2)}\")\n",
    "print(\"\\n\")\n",
    "print(\"Columns that receive the missingness:\")\n",
    "for col in generate_data.columns:\n",
    "    if generate_data[col].isna().sum() > 0:\n",
    "        print(col)\n",
    "    else:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Missingness Based on Intra-Relation (MBIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Global Missing rate = 0.2\n",
      "\n",
      "\n",
      "Columns that receive the missingness:\n",
      "mean radius\n",
      "texture error\n",
      "area error\n"
     ]
    }
   ],
   "source": [
    "# Create a instance for MNAR mechanism\n",
    "generator = mMNAR(X=X, \n",
    "                  y=y,\n",
    "                  ) \n",
    "\n",
    "list_to_gen_miss = [\"mean radius\",\"texture error\", \"area error\"]\n",
    "\n",
    "# Generate the missing data under MNAR mechanism up to 20% missing rate\n",
    "generate_data = generator.MBIR(missing_rate=20, \n",
    "                               columns=list_to_gen_miss,\n",
    "                               statistical_method=\"Mann-Whitney\")\n",
    "                   \n",
    "qtd_miss = sum(generate_data.isna().sum())\n",
    "print(f\"Global Missing rate = {round(qtd_miss / (len(list_to_gen_miss) * generate_data.shape[0]),2)}\")\n",
    "print(\"\\n\")\n",
    "print(\"Columns that receive the missingness:\")\n",
    "for col in generate_data.columns:\n",
    "    if generate_data[col].isna().sum() > 0:\n",
    "        print(col)\n",
    "    else:\n",
    "        pass"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}