Evaluate_Model_Generalization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import f1_score, jaccard_score, precision_score, recall_score, accuracy_score\n",
    "import os\n",
    "from datetime import datetime\n",
    "from tqdm import tqdm\n",
    "from PIL import Image\n",
    "from skimage.io import imread, imsave\n",
    "from ModelArchitecture.DiceLoss import dice_metric_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define function to save and load numpy arrays\n",
    "def save_numpy_arrays(images, masks, dataset_name):\n",
    "    np.save(f\"resized_data/{dataset_name}_images.npy\", images)\n",
    "    np.save(f\"resized_data/{dataset_name}_masks.npy\", masks)\n",
    "\n",
    "\n",
    "def load_numpy_arrays(dataset_name):\n",
    "    images = np.load(f\"resized_data/{dataset_name}_images.npy\")\n",
    "    masks = np.load(f\"resized_data/{dataset_name}_masks.npy\")\n",
    "    return images, masks\n",
    "\n",
    "\n",
    "# Define function to load data and resize images and masks\n",
    "def load_data(image_dir, mask_dir, img_height, img_width, images_to_be_loaded, dataset_name):\n",
    "\n",
    "    # Check if the resized data already exists\n",
    "    if os.path.exists(f\"resized_data/{dataset_name}_images.npy\") and os.path.exists(f\"resized_data/{dataset_name}_masks.npy\"):\n",
    "        print(f\"Loading preprocessed data for {dataset_name}\")\n",
    "        return load_numpy_arrays(dataset_name)\n",
    "\n",
    "    if dataset_name == \"Kvasir-SEG\":\n",
    "        test_ids = glob.glob(f\"{image_dir}/*.jpg\")\n",
    "    elif dataset_name == \"CVC-ClinicDB\":\n",
    "        test_ids = glob.glob(f\"{image_dir}/*.tif\")\n",
    "    elif dataset_name == \"CVC-ColonDB\" or dataset_name == \"ETIS-LaribpolypDB\":\n",
    "        test_ids = glob.glob(f\"{image_dir}/*.png\")\n",
    "\n",
    "    print(test_ids)\n",
    "\n",
    "    if images_to_be_loaded == -1:\n",
    "        images_to_be_loaded = len(test_ids)\n",
    "\n",
    "    images = np.zeros((images_to_be_loaded, img_height, img_width, 3), dtype=np.float32)\n",
    "    masks = np.zeros((images_to_be_loaded, img_height, img_width), dtype=np.uint8)\n",
    "\n",
    "    print(f\"Resizing training images and masks: {images_to_be_loaded}\")\n",
    "    for n, id_ in tqdm(enumerate(test_ids)):\n",
    "        if n == images_to_be_loaded:\n",
    "            break\n",
    "\n",
    "        image_path = id_\n",
    "\n",
    "        if \"images\" in image_path:\n",
    "            mask_path = image_path.replace(\"images\", \"masks\")\n",
    "        elif \"Original\" in image_path:\n",
    "            mask_path = image_path.replace(\"Original\", \"Ground Truth\")\n",
    "\n",
    "        resized_image_path = image_path.replace(\"original_data\", \"resized_data\")\n",
    "        resized_mask_path = mask_path.replace(\"original_data\", \"resized_data\")\n",
    "\n",
    "        # Check if the image has been resized\n",
    "        if os.path.exists(resized_image_path) and os.path.exists(resized_mask_path):\n",
    "            image = imread(resized_image_path)\n",
    "            mask_ = imread(resized_mask_path)\n",
    "        else:\n",
    "            image = imread(image_path)\n",
    "            mask_ = imread(mask_path)\n",
    "\n",
    "            pillow_image = Image.fromarray(image)\n",
    "            pillow_image = pillow_image.resize((img_height, img_width))\n",
    "            image = np.array(pillow_image)\n",
    "            imsave(resized_image_path, image)\n",
    "\n",
    "            pillow_mask = Image.fromarray(mask_)\n",
    "            pillow_mask = pillow_mask.resize((img_height, img_width), Image.Resampling.LANCZOS)\n",
    "            mask_ = np.array(pillow_mask)\n",
    "            imsave(resized_mask_path, mask_)\n",
    "\n",
    "        images[n] = image / 255.0\n",
    "\n",
    "        mask = np.zeros((img_height, img_width), dtype=np.bool_)\n",
    "        for i in range(img_height):\n",
    "            for j in range(img_width):\n",
    "                if np.any(mask_[i, j] >= 127):\n",
    "                    mask[i, j] = 1\n",
    "\n",
    "        masks[n] = mask\n",
    "\n",
    "    masks = np.expand_dims(masks, axis=-1)\n",
    "\n",
    "    # Save the preprocessed data\n",
    "    save_numpy_arrays(images, masks, dataset_name)\n",
    "\n",
    "    return images, masks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_size = 352\n",
    "\n",
    "\n",
    "# Define function to evaluate model\n",
    "def evaluate_model(model_path, dataset_name, image_dir, mask_dir):\n",
    "    # Load data\n",
    "    images, masks = load_data(image_dir, mask_dir, img_size, img_size, -1, dataset_name)\n",
    "\n",
    "    # Load model\n",
    "    model = tf.keras.models.load_model(model_path, custom_objects={\"dice_metric_loss\": dice_metric_loss})\n",
    "\n",
    "    # Make predictions\n",
    "    predictions = model.predict(images, batch_size=4)\n",
    "\n",
    "    # Calculate metrics\n",
    "    f1 = f1_score(np.ndarray.flatten(np.array(masks, dtype=bool)), np.ndarray.flatten(predictions > 0.5))\n",
    "    jaccard = jaccard_score(np.ndarray.flatten(np.array(masks, dtype=bool)), np.ndarray.flatten(predictions > 0.5))\n",
    "    precision = precision_score(np.ndarray.flatten(np.array(masks, dtype=bool)), np.ndarray.flatten(predictions > 0.5))\n",
    "    recall = recall_score(np.ndarray.flatten(np.array(masks, dtype=bool)), np.ndarray.flatten(predictions > 0.5))\n",
    "    accuracy = accuracy_score(np.ndarray.flatten(np.array(masks, dtype=bool)), np.ndarray.flatten(predictions > 0.5))\n",
    "\n",
    "    # Print results\n",
    "    print(f\"Results for {dataset_name} using model {model_path}:\")\n",
    "    print(f\"F1 Score: {f1}\")\n",
    "    print(f\"Jaccard Score: {jaccard}\")\n",
    "    print(f\"Precision: {precision}\")\n",
    "    print(f\"Recall: {recall}\")\n",
    "    print(f\"Accuracy: {accuracy}\")\n",
    "\n",
    "    # Save results to file\n",
    "    result_file = f\"results_{os.path.basename(model_path)}_{dataset_name}.txt\"\n",
    "    with open(result_file, \"w\") as f:\n",
    "        f.write(f\"F1 Score: {f1}\\n\")\n",
    "        f.write(f\"Jaccard Score: {jaccard}\\n\")\n",
    "        f.write(f\"Precision: {precision}\\n\")\n",
    "        f.write(f\"Recall: {recall}\\n\")\n",
    "        f.write(f\"Accuracy: {accuracy}\\n\")\n",
    "\n",
    "    return f1, jaccard, precision, recall, accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define function to plot metrics and save the plots\n",
    "def plot_metrics(metrics, dataset_names, model_name, save_dir):\n",
    "    if not os.path.exists(save_dir):\n",
    "        os.makedirs(save_dir)\n",
    "\n",
    "    plt.figure(figsize=(15, 5))\n",
    "\n",
    "    for metric_name, metric_values in metrics.items():\n",
    "        plt.plot(dataset_names, metric_values, label=metric_name)\n",
    "\n",
    "    plt.xlabel(\"Dataset\")\n",
    "    plt.ylabel(\"Score\")\n",
    "    plt.title(f\"Model {model_name} Metrics Across Different Datasets\")\n",
    "    plt.legend()\n",
    "\n",
    "    # Generate a filename with datetime to avoid overwriting\n",
    "    timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
    "    plot_filename = f\"{model_name}_metrics_{timestamp}.png\"\n",
    "    plot_filepath = os.path.join(save_dir, plot_filename)\n",
    "\n",
    "    plt.savefig(plot_filepath)\n",
    "    plt.show()\n",
    "\n",
    "    print(f\"Plot saved to {plot_filepath}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define datasets and models\n",
    "datasets = {\n",
    "    \"Kvasir-SEG\": {\n",
    "        \"image_dir\": \"original_data/Kvasir-SEG/images\",\n",
    "        \"mask_dir\": \"original_data/Kvasir-SEG/masks\",\n",
    "    },\n",
    "    \"CVC-ClinicDB\": {\n",
    "        \"image_dir\": \"original_data/CVC-ClinicDB/Original\",\n",
    "        \"mask_dir\": \"original_data/CVC-ClinicDB/Ground Truth\",\n",
    "    },\n",
    "    \"CVC-ColonDB\": {\n",
    "        \"image_dir\": \"original_data/CVC-ColonDB/images\",\n",
    "        \"mask_dir\": \"original_data/CVC-ColonDB/masks\",\n",
    "    },\n",
    "    \"ETIS-LaribpolypDB\": {\n",
    "        \"image_dir\": \"original_data/ETIS-LaribpolypDB/images\",\n",
    "        \"mask_dir\": \"original_data/ETIS-LaribpolypDB/masks\",\n",
    "    },\n",
    "}\n",
    "\n",
    "# Define datasets and models\n",
    "resized_datasets = {\n",
    "    \"Kvasir-SEG\": {\n",
    "        \"image_dir\": \"resized_data/Kvasir-SEG/images\",\n",
    "        \"mask_dir\": \"resized_data/Kvasir-SEG/masks\",\n",
    "    },\n",
    "    \"CVC-ClinicDB\": {\n",
    "        \"image_dir\": \"resized_data/CVC-ClinicDB/Original\",\n",
    "        \"mask_dir\": \"resized_data/CVC-ClinicDB/Ground Truth\",\n",
    "    },\n",
    "    \"CVC-ColonDB\": {\n",
    "        \"image_dir\": \"resized_data/CVC-ColonDB/images\",\n",
    "        \"mask_dir\": \"resized_data/CVC-ColonDB/masks\",\n",
    "    },\n",
    "    \"ETIS-LaribpolypDB\": {\n",
    "        \"image_dir\": \"resized_data/ETIS-LaribpolypDB/images\",\n",
    "        \"mask_dir\": \"resized_data/ETIS-LaribpolypDB/masks\",\n",
    "    },\n",
    "}\n",
    "\n",
    "for dataset_name, dataset_paths in resized_datasets.items():\n",
    "    if not os.path.exists(dataset_paths[\"image_dir\"]):\n",
    "        os.makedirs(dataset_paths[\"image_dir\"])\n",
    "    if not os.path.exists(dataset_paths[\"mask_dir\"]):\n",
    "        os.makedirs(dataset_paths[\"mask_dir\"])\n",
    "\n",
    "models = [\n",
    "    \"saved_model/kvasir-17\",\n",
    "    \"saved_model/kvasir-34\",\n",
    "    \"saved_model/cvc-clinicdb-17\",\n",
    "    \"saved_model/cvc-clinicdb-34\",\n",
    "    \"saved_model/cvc-colondb-17\",\n",
    "    \"saved_model/cvc-colondb-34\",\n",
    "    \"saved_model/etis-laribpolypdb-17\",\n",
    "    \"saved_model/etis-laribpolypdb-34\",\n",
    "]\n",
    "\n",
    "# Directory to save the plots\n",
    "plot_save_dir = \"plots\"\n",
    "\n",
    "# Evaluate models on different datasets\n",
    "for model_path in models:\n",
    "    metrics = {\n",
    "        \"F1 Score\": [],\n",
    "        \"Jaccard Score\": [],\n",
    "        \"Precision\": [],\n",
    "        \"Recall\": [],\n",
    "        \"Accuracy\": [],\n",
    "    }\n",
    "    dataset_names = []\n",
    "\n",
    "    for dataset_name, dataset_paths in datasets.items():\n",
    "        print(f\"Using model {os.path.basename(model_path)} to evaluate {dataset_name} dataset...\")\n",
    "\n",
    "        image_dir = dataset_paths[\"image_dir\"]\n",
    "        mask_dir = dataset_paths[\"mask_dir\"]\n",
    "\n",
    "        f1, jaccard, precision, recall, accuracy = evaluate_model(model_path, dataset_name, image_dir, mask_dir)\n",
    "\n",
    "        metrics[\"F1 Score\"].append(f1)\n",
    "        metrics[\"Jaccard Score\"].append(jaccard)\n",
    "        metrics[\"Precision\"].append(precision)\n",
    "        metrics[\"Recall\"].append(recall)\n",
    "        metrics[\"Accuracy\"].append(accuracy)\n",
    "\n",
    "        dataset_names.append(dataset_name)\n",
    "\n",
    "    # Plot metrics\n",
    "    plot_metrics(metrics, dataset_names, os.path.basename(model_path), plot_save_dir)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "duck",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}