varying_model_architecture.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b025d5d-8d1b-4ed3-9733-ef66e4a82900",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1de80eda-bc27-43c0-b1d6-16015bb761eb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "datasets = ['ADULT', 'German', 'Lawschool', 'HealthHeritage']  # set to include only the datasets on which you have already obtained the data\n",
    "random_seed = 42 # set the random seed you ran your experiments on\n",
    "n_samples = 50\n",
    "\n",
    "networks = {'Linear': 'linear', 'FC NN': 'fc', 'FC NN Large': 'fc_large', 'CNN (BN)': 'cnn', 'ResNet (BN)': 'residual'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44e65421-36dc-4eed-ba99-f531dcb1d101",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataframes_per_dataset = {}\n",
    "\n",
    "for dataset_name in datasets:\n",
    "    \n",
    "    dataset_dataframe = pd.DataFrame()\n",
    "    dataset_dataframe['Attack'] = ['Inverting Gradients', 'GradInversion', 'TabLeak']\n",
    "    \n",
    "    data_per_network = np.zeros((3, 5), dtype='object')\n",
    "    \n",
    "    # GradInversion loading\n",
    "    for j, (network_name, network_load_name) in enumerate(networks.items()):\n",
    "        \n",
    "        n_iters = 1500 if network_load_name in ['linear', 'fc', 'fc_large'] else 7000\n",
    "        \n",
    "        load_path = f'experiment_data/attacks_with_priors/{dataset_name}/experiment_3/{network_load_name}/batch_size_32/inversion_data_all_3_{dataset_name}_{n_samples}_1_{n_iters}_32_0.319_{random_seed}.npy'\n",
    "        \n",
    "        if os.path.isfile(load_path):\n",
    "            experiment_data = np.load(load_path)[0, :2]\n",
    "            min_err_idx = np.argmin(experiment_data[0]).item()\n",
    "            err, std = experiment_data[:, min_err_idx].flatten()\n",
    "            data_per_network[1, j] = (np.around(100-100*err, 1), np.around(100*std, 1))\n",
    "        else:\n",
    "            print(f'Experiment not found, please run ./varying_model_architecture.sh {dataset_name} {n_samples} {random_seed} <max_n_cpus> <start_cpu>')\n",
    "            break\n",
    "            \n",
    "    # Inverting gradients and TabLeak\n",
    "    for idx, experiment_num in zip([0, 2], [0, 46]):\n",
    "        \n",
    "        post_selection = 1 if experiment_num == 0 else 30\n",
    "        \n",
    "        # load the linear model and the fully connected networks\n",
    "        load_path = f'experiment_data/network_size_variation/{dataset_name}/experiment_{experiment_num}/batch_size_32/inversion_data_all_{experiment_num}_{dataset_name}_{n_samples}_{post_selection}_1500_32_0.319_{random_seed}.npy'\n",
    "        \n",
    "        if os.path.isfile(load_path):\n",
    "            experiment_data = np.load(load_path)[0, [0, 3, 5], 0, :2]\n",
    "            tupled_data = [(np.around(100-100*err, 1), np.around(100*std, 1)) for err, std in experiment_data]\n",
    "        else:\n",
    "            print(f'Experiment not found, please run ./varying_model_architecture.sh {dataset_name} {n_samples} {random_seed} <max_n_cpus> <start_cpu>')\n",
    "            break\n",
    "        \n",
    "        # load cnn and resnet\n",
    "        load_path = f'experiment_data/network_layout_variation/{dataset_name}/experiment_{experiment_num}/batch_size_32/inversion_data_all_{experiment_num}_{dataset_name}_50_{post_selection}_7000_32_0.319_{random_seed}.npy'\n",
    "        \n",
    "        if os.path.isfile(load_path):\n",
    "            experiment_data = np.load(load_path)[0, :, 0, :2]\n",
    "            tupled_data += [(np.around(100-100*err, 1), np.around(100*std, 1)) for err, std in experiment_data]\n",
    "        else:\n",
    "            print(f'Experiment not found, please run ./varying_model_architecture.sh {dataset_name} {n_samples} {random_seed} <max_n_cpus> <start_cpu>')\n",
    "            break\n",
    "        \n",
    "        data_per_network[idx] = tupled_data\n",
    "    \n",
    "    for network_name, network_data in zip(networks.keys(), data_per_network.T):\n",
    "        dataset_dataframe[network_name] = network_data\n",
    "        \n",
    "    dataframes_per_dataset[dataset_name] = dataset_dataframe"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a41e614b-a302-48d1-8b5b-5782bbaea5b2",
   "metadata": {},
   "source": [
    "### Adult"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "894ffbe3-a1a5-44ff-a557-17b31fef3df9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataframes_per_dataset['ADULT']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d64894e0-6242-42e6-8955-9a5d2d6d6f1c",
   "metadata": {},
   "source": [
    "### German"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42fefc52-355e-4c27-849a-a4d6bbb43776",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataframes_per_dataset['German']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46342318-3aea-4174-8bb0-fced5fa4eba6",
   "metadata": {},
   "source": [
    "### Lawschool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2655042e-93ce-46f4-bc60-4429239780ea",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataframes_per_dataset['Lawschool']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ce72f80-c3d0-4d19-bce9-1b88b66f3aad",
   "metadata": {},
   "source": [
    "### HealthHeritage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f730b6fc-cbaa-41a2-bcfe-c25b25aaa9b4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataframes_per_dataset['HealthHeritage']"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}