example_cluster_n_outliers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %pip install umap-learn[plot]\n",
    "# %pip install sklearn --upgrade\n",
    "# %pip install ipywidgets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import umap\n",
    "import umap.plot\n",
    "import quick_torch as qt\n",
    "import matplotlib.pyplot as plt\n",
    "import sklearn.neighbors\n",
    "plt.rcParams[\"figure.figsize\"] = 15, 15\n",
    "%matplotlib inline\n",
    "\n",
    "# Dimension reduction and clustering libraries\n",
    "import umap\n",
    "import sklearn.cluster as cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_images(data, n_cols=10, n_rows=10, figsize=(10,10), n_plots=3):\n",
    "    n_data = len(data)\n",
    "    n_img = n_cols * n_rows\n",
    "    assert n_data >= n_img, f\"Number of images is less than {n_img}\"\n",
    "\n",
    "    n_plots = min(n_plots, n_data // n_img) if n_plots is not None else n_data // n_img\n",
    "    \n",
    "    init_range, final_range = 0, n_img\n",
    "    for i in range(n_plots):\n",
    "        _, axes = plt.subplots(n_rows, n_cols, figsize=figsize)\n",
    "        for i, ax in enumerate(axes.flatten()):\n",
    "            ax.imshow(data[init_range:final_range][i].reshape((28,28)))\n",
    "            plt.setp(ax, xticks=[], yticks=[])\n",
    "        plt.tight_layout()\n",
    "        plt.show()\n",
    "        init_range += n_img\n",
    "        final_range += n_img"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Smiley Faces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_smiley_face = qt.QuickDraw(\"dataset\", [qt.Category.SMILEY_FACE], download=True)\n",
    "data = dataset_smiley_face.data.reshape(-1, 784)\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapper = umap.UMAP(set_op_mix_ratio=0.25, verbose=True).fit(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "umap.plot.points(mapper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outlier_scores = sklearn.neighbors.LocalOutlierFactor().fit_predict(mapper.embedding_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(outlier_scores == -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_images(data[outlier_scores == -1], 10, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data =  data[outlier_scores != -1]\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del mapper, outlier_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapper = umap.UMAP(\n",
    "    n_neighbors=30,\n",
    "    min_dist=0.0,\n",
    "    set_op_mix_ratio=1, \n",
    "    verbose=True,\n",
    ").fit(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "umap.plot.points(mapper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clusterable_embedding = mapper.embedding_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = cluster.DBSCAN(min_samples=500).fit_predict(clusterable_embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "umap.plot.points(mapper, labels=labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.unique(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(labels == -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_images(data[labels == -1], 7, 4, n_plots=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_images(data[labels == 0], 10, 10, n_plots=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_images(data[labels == 1], 10, 10, n_plots=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[labels == 0]\n",
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del mapper, clusterable_embedding, labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Outliers, again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapper = umap.UMAP(\n",
    "    set_op_mix_ratio=0.25,\n",
    "    verbose=True,\n",
    ").fit(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "umap.plot.points(mapper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outlier_scores = sklearn.neighbors.LocalOutlierFactor().fit_predict(mapper.embedding_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.unique(outlier_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(outlier_scores == -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_images(data[outlier_scores == -1], 10, 7, n_plots=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[outlier_scores != -1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del mapper, outlier_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_smiley_face.data = data.reshape(-1, 28, 28)\n",
    "dataset_smiley_face.targets = dataset_smiley_face.targets[:len(data)]\n",
    "len(dataset_smiley_face)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Faces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_face = qt.QuickDraw(\"dataset\", [qt.Category.FACE], download=True)\n",
    "data = dataset_face.data.reshape(-1, 784)\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapper = umap.UMAP(set_op_mix_ratio=0.25, verbose=True).fit(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "umap.plot.points(mapper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outlier_scores = sklearn.neighbors.LocalOutlierFactor().fit_predict(mapper.embedding_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(outlier_scores == -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_images(data[outlier_scores == -1], 10, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[outlier_scores != -1]\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del mapper, outlier_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapper = umap.UMAP(\n",
    "    n_neighbors=30,\n",
    "    min_dist=0.0,\n",
    "    verbose=True,\n",
    "    n_epochs=500,\n",
    ").fit(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "umap.plot.points(mapper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clusterable_embedding = mapper.embedding_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = cluster.HDBSCAN(min_cluster_size=100).fit_predict(clusterable_embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.unique(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(labels == -1), sum(labels != -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in np.unique(labels):\n",
    "    print(f\"Cluster {i}: {sum(labels == i)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "umap.plot.points(mapper, labels=labels, theme=\"fire\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_images(data[labels == -1], 10, 10, n_plots=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_to_keep = labels == -1\n",
    "sum(labels_to_keep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = 0\n",
    "print(f\"Cluster {label}: {sum(labels == label)}\")\n",
    "plot_images(data[labels == label], 10, 10, n_plots=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_to_keep = np.logical_or(labels_to_keep, labels == label)\n",
    "sum(labels_to_keep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = 1\n",
    "print(f\"Cluster {label}: {sum(labels == label)}\")\n",
    "plot_images(data[labels == label], 10, 10, n_plots=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# labels_to_keep = np.logical_or(labels_to_keep, labels == label)\n",
    "# sum(labels_to_keep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = 2\n",
    "print(f\"Cluster {label}: {sum(labels == label)}\")\n",
    "plot_images(data[labels == label], 10, 10, n_plots=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# labels_to_keep = np.logical_or(labels_to_keep, labels == label)\n",
    "# sum(labels_to_keep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = 3\n",
    "print(f\"Cluster {label}: {sum(labels == label)}\")\n",
    "plot_images(data[labels == label], 10, 10, n_plots=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# labels_to_keep = np.logical_or(labels_to_keep, labels == label)\n",
    "# sum(labels_to_keep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = 4\n",
    "print(f\"Cluster {label}: {sum(labels == label)}\")\n",
    "plot_images(data[labels == label], 10, 10, n_plots=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# labels_to_keep = np.logical_or(labels_to_keep, labels == label)\n",
    "# sum(labels_to_keep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = 5\n",
    "print(f\"Cluster {label}: {sum(labels == label)}\")\n",
    "plot_images(data[labels == label], 10, 10, n_plots=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# labels_to_keep = np.logical_or(labels_to_keep, labels == label)\n",
    "# sum(labels_to_keep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = 6\n",
    "print(f\"Cluster {label}: {sum(labels == label)}\")\n",
    "plot_images(data[labels == label], 10, 10, n_plots=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = 7\n",
    "print(f\"Cluster {label}: {sum(labels == label)}\")\n",
    "plot_images(data[labels == label], 10, 10, n_plots=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = 8\n",
    "print(f\"Cluster {label}: {sum(labels == label)}\")\n",
    "plot_images(data[labels == label], 10, 10, n_plots=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(labels_to_keep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[labels_to_keep]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del mapper, clusterable_embedding, labels, labels_to_keep, label, i"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapper = umap.UMAP(\n",
    "    set_op_mix_ratio=0.25,\n",
    "    verbose=True,\n",
    ").fit(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "umap.plot.points(mapper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outlier_scores = sklearn.neighbors.LocalOutlierFactor().fit_predict(mapper.embedding_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(outlier_scores == -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_images(data[outlier_scores == -1], 10, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[outlier_scores != -1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del mapper, outlier_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clustering, again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapper = umap.UMAP(\n",
    "    n_neighbors=30,\n",
    "    min_dist=0.0,\n",
    "    verbose=True,\n",
    "    n_epochs=500,\n",
    ").fit(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "umap.plot.points(mapper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clusterable_embedding = mapper.embedding_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = cluster.HDBSCAN(min_cluster_size=100).fit_predict(clusterable_embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.unique(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(labels == -1), sum(labels != -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in np.unique(labels):\n",
    "    print(f\"Cluster {i}: {sum(labels == i)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "umap.plot.points(mapper, labels=labels, theme=\"fire\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_images(data[labels == -1], 10, 10, n_plots=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_to_keep = labels == -1\n",
    "sum(labels_to_keep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = 0\n",
    "print(f\"Cluster {label}: {sum(labels == label)}\")\n",
    "plot_images(data[labels == label], 10, 10, n_plots=1)\n",
    "# labels_to_keep = np.logical_or(labels_to_keep, labels == label)\n",
    "# sum(labels_to_keep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = 1\n",
    "print(f\"Cluster {label}: {sum(labels == label)}\")\n",
    "plot_images(data[labels == label], 10, 10, n_plots=1)\n",
    "labels_to_keep = np.logical_or(labels_to_keep, labels == label)\n",
    "sum(labels_to_keep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = 2\n",
    "print(f\"Cluster {label}: {sum(labels == label)}\")\n",
    "plot_images(data[labels == label], 10, 10, n_plots=1)\n",
    "# labels_to_keep = np.logical_or(labels_to_keep, labels == label)\n",
    "# sum(labels_to_keep)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[labels_to_keep]\n",
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_face.data = data.reshape(-1, 28, 28)\n",
    "dataset_face.targets = dataset_face.targets[:len(data)]\n",
    "len(dataset_face)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Juntar datasets y analizar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = dataset_smiley_face + dataset_face\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = dataset.data.reshape(-1, 784)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Outliers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapper = umap.UMAP(\n",
    "    set_op_mix_ratio=0.25,\n",
    "    verbose=True,\n",
    ").fit(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "umap.plot.points(mapper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outlier_scores = sklearn.neighbors.LocalOutlierFactor().fit_predict(mapper.embedding_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(outlier_scores == -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_images(data[outlier_scores == -1], 10, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[outlier_scores != -1]\n",
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del mapper, outlier_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapper = umap.UMAP(\n",
    "    n_neighbors=30,\n",
    "    min_dist=0.0,\n",
    "    verbose=True,\n",
    "    # n_epochs=500,\n",
    ").fit(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "umap.plot.points(mapper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clusterable_embedding = mapper.embedding_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = cluster.HDBSCAN().fit_predict(clusterable_embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.unique(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in np.unique(labels):\n",
    "    print(f\"Cluster {i}: {sum(labels == i)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "umap.plot.points(mapper, labels=labels, theme=\"fire\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_images(data[labels == 1], 10, 2, n_plots=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data[labels == 0]\n",
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "DATASET_PATH = Path(\"dataset\") / \"cleaned\"\n",
    "DATASET_PATH.mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(DATASET_PATH / \"data.npy\", data)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}