microbiome.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mc5xkI2hXrwh"
      },
      "source": [
        "# Microbial Community Disease Risk Prediction\n",
        "\n",
        "This case study involves developing and evaluating neural network models to predict disease risk from high-dimensional microbiome sequence data. Our dataset consists of 16S rRNA gene profiles from 16,344 samples, with approximately half of the samples coming from individuals diagnosed with type 1 diabetes (cases) and the other half from individuals without a diagnosis of type 1 diabetes (controls). By modeling patterns in these microbial community profiles, our goal is to accurately classify samples based on disease status. This could provide insight into whether an individual's gut microbiome composition is associated with their risk of developing type 1 diabetes. We implement and compare several neural network architectures, including convolutional and recurrent networks, to evaluate their ability to capture predictive signals from large-scale marker gene surveys of complex microbial communities."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        },
        "id": "alDR3GTbXqOw",
        "outputId": "ce196825-e15c-4f06-ebfb-b2ca4f477c23"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>DTA0</th>\n",
              "      <th>DTA1</th>\n",
              "      <th>DTA2</th>\n",
              "      <th>DTA3</th>\n",
              "      <th>DTA4</th>\n",
              "      <th>DTA5</th>\n",
              "      <th>DTA6</th>\n",
              "      <th>DTA7</th>\n",
              "      <th>DTA8</th>\n",
              "      <th>DTA9</th>\n",
              "      <th>...</th>\n",
              "      <th>DTA247</th>\n",
              "      <th>DTA248</th>\n",
              "      <th>DTA249</th>\n",
              "      <th>DTA250</th>\n",
              "      <th>DTA251</th>\n",
              "      <th>DTA252</th>\n",
              "      <th>DTA253</th>\n",
              "      <th>DTA254</th>\n",
              "      <th>DTA255</th>\n",
              "      <th>LBL0</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.92</td>\n",
              "      <td>1.80</td>\n",
              "      <td>1.44</td>\n",
              "      <td>1.79</td>\n",
              "      <td>1.68</td>\n",
              "      <td>1.42</td>\n",
              "      <td>1.52</td>\n",
              "      <td>1.58</td>\n",
              "      <td>1.43</td>\n",
              "      <td>1.45</td>\n",
              "      <td>...</td>\n",
              "      <td>-2.02</td>\n",
              "      <td>-2.32</td>\n",
              "      <td>-2.19</td>\n",
              "      <td>-2.25</td>\n",
              "      <td>-2.25</td>\n",
              "      <td>-2.29</td>\n",
              "      <td>-2.19</td>\n",
              "      <td>-2.63</td>\n",
              "      <td>-2.86</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1.97</td>\n",
              "      <td>1.98</td>\n",
              "      <td>2.16</td>\n",
              "      <td>2.12</td>\n",
              "      <td>1.78</td>\n",
              "      <td>1.71</td>\n",
              "      <td>1.69</td>\n",
              "      <td>1.60</td>\n",
              "      <td>1.74</td>\n",
              "      <td>1.64</td>\n",
              "      <td>...</td>\n",
              "      <td>-2.05</td>\n",
              "      <td>-1.97</td>\n",
              "      <td>-1.92</td>\n",
              "      <td>-2.12</td>\n",
              "      <td>-1.94</td>\n",
              "      <td>-2.18</td>\n",
              "      <td>-2.45</td>\n",
              "      <td>-2.63</td>\n",
              "      <td>-2.87</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2.25</td>\n",
              "      <td>2.11</td>\n",
              "      <td>2.05</td>\n",
              "      <td>1.92</td>\n",
              "      <td>2.08</td>\n",
              "      <td>1.93</td>\n",
              "      <td>1.87</td>\n",
              "      <td>1.57</td>\n",
              "      <td>1.81</td>\n",
              "      <td>1.61</td>\n",
              "      <td>...</td>\n",
              "      <td>-2.02</td>\n",
              "      <td>-1.87</td>\n",
              "      <td>-1.95</td>\n",
              "      <td>-2.09</td>\n",
              "      <td>-1.96</td>\n",
              "      <td>-1.99</td>\n",
              "      <td>-2.01</td>\n",
              "      <td>-2.57</td>\n",
              "      <td>-2.71</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2.25</td>\n",
              "      <td>2.07</td>\n",
              "      <td>1.92</td>\n",
              "      <td>1.84</td>\n",
              "      <td>1.83</td>\n",
              "      <td>1.80</td>\n",
              "      <td>1.88</td>\n",
              "      <td>1.48</td>\n",
              "      <td>1.70</td>\n",
              "      <td>1.46</td>\n",
              "      <td>...</td>\n",
              "      <td>-1.94</td>\n",
              "      <td>-2.11</td>\n",
              "      <td>-2.22</td>\n",
              "      <td>-1.98</td>\n",
              "      <td>-2.22</td>\n",
              "      <td>-2.00</td>\n",
              "      <td>-2.10</td>\n",
              "      <td>-2.59</td>\n",
              "      <td>-2.84</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2.28</td>\n",
              "      <td>2.27</td>\n",
              "      <td>2.26</td>\n",
              "      <td>2.20</td>\n",
              "      <td>2.01</td>\n",
              "      <td>2.00</td>\n",
              "      <td>1.99</td>\n",
              "      <td>1.92</td>\n",
              "      <td>1.68</td>\n",
              "      <td>1.79</td>\n",
              "      <td>...</td>\n",
              "      <td>-1.69</td>\n",
              "      <td>-1.66</td>\n",
              "      <td>-1.82</td>\n",
              "      <td>-1.88</td>\n",
              "      <td>-1.92</td>\n",
              "      <td>-1.89</td>\n",
              "      <td>-2.07</td>\n",
              "      <td>-2.50</td>\n",
              "      <td>-2.72</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 257 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "   DTA0  DTA1  DTA2  DTA3  DTA4  DTA5  DTA6  DTA7  DTA8  DTA9  ...  DTA247   \n",
              "0  1.92  1.80  1.44  1.79  1.68  1.42  1.52  1.58  1.43  1.45  ...   -2.02  \\\n",
              "1  1.97  1.98  2.16  2.12  1.78  1.71  1.69  1.60  1.74  1.64  ...   -2.05   \n",
              "2  2.25  2.11  2.05  1.92  2.08  1.93  1.87  1.57  1.81  1.61  ...   -2.02   \n",
              "3  2.25  2.07  1.92  1.84  1.83  1.80  1.88  1.48  1.70  1.46  ...   -1.94   \n",
              "4  2.28  2.27  2.26  2.20  2.01  2.00  1.99  1.92  1.68  1.79  ...   -1.69   \n",
              "\n",
              "   DTA248  DTA249  DTA250  DTA251  DTA252  DTA253  DTA254  DTA255  LBL0  \n",
              "0   -2.32   -2.19   -2.25   -2.25   -2.29   -2.19   -2.63   -2.86   1.0  \n",
              "1   -1.97   -1.92   -2.12   -1.94   -2.18   -2.45   -2.63   -2.87   0.0  \n",
              "2   -1.87   -1.95   -2.09   -1.96   -1.99   -2.01   -2.57   -2.71   1.0  \n",
              "3   -2.11   -2.22   -1.98   -2.22   -2.00   -2.10   -2.59   -2.84   0.0  \n",
              "4   -1.66   -1.82   -1.88   -1.92   -1.89   -2.07   -2.50   -2.72   0.0  \n",
              "\n",
              "[5 rows x 257 columns]"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import pandas\n",
        "dataframe = pandas.read_csv('https://raw.githubusercontent.com/bryankolaczkowski/ALS3200C/main/mbiome.data.csv')\n",
        "dataframe.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mlKLWh5rXq9g"
      },
      "source": [
        "The dataset contains 256 columns labeled DTA0 through DTA255, representing the relative abundances of 256 bacterial taxa across the 16,344 samples. Relative abundance is calculated as the proportion of sequencing reads assigned to each taxon in a given sample. To account for sequencing depth differences, the relative abundances have been normalized using a center log-ratio transformation - a standard technique for compositional microbial data. This transforms the data to a log scale centered at zero, where positive values indicate higher than average abundance, and negative values indicate lower than average abundance for a given taxon. After transformation, the relative abundance values typically range from +2.5 to -2.5.\n",
        "\n",
        "The target prediction variable is contained in the LBL0 column, with 0 indicating control samples and 1 indicating case samples positive for type 1 diabetes. Our modeling goal is to accurately predict this disease label based on the multivariate microbial community profiles in columns DTA0 through DTA255.\n",
        "\n",
        "We first split the data into training and validation subsets for model fitting and evaluation. We then extract the DTA columns as model inputs and the LBL0 column as labels. To leverage TensorFlow's sequence modeling capabilities, we expand the data to three dimensions, with the additional dimension representing the ordered sequence of taxa.\n",
        "\n",
        "In the following sections, we fit and compare several neural network architectures, including convolutional and recurrent networks, to evaluate their ability to capture predictive signals about disease status from the full set of microbial taxa abundances and their co-occurrence patterns."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XVTUJdVQYDkv",
        "outputId": "f43085fa-0e91-4750-ebba-d93aecf1bab5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(13075, 257) (3269, 257) (16344, 257)\n",
            "(13075, 256, 1) (3269, 256, 1)\n",
            "(13075,) (3269,)\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "\n",
        "# create train-validate split\n",
        "train_dataframe = dataframe.sample(frac=0.8, random_state=2100963)\n",
        "valid_dataframe = dataframe.drop(train_dataframe.index)\n",
        "print(train_dataframe.shape, valid_dataframe.shape, dataframe.shape)\n",
        "\n",
        "# extract explanatory variables\n",
        "dta_ids = [ x for x in dataframe.columns if x.find('DTA') == 0 ]\n",
        "train_x = np.expand_dims(train_dataframe[dta_ids].to_numpy(), axis=-1)\n",
        "valid_x = np.expand_dims(valid_dataframe[dta_ids].to_numpy(), axis=-1)\n",
        "print(train_x.shape, valid_x.shape)\n",
        "\n",
        "# extract labels\n",
        "train_y = train_dataframe['LBL0'].to_numpy()\n",
        "valid_y = valid_dataframe['LBL0'].to_numpy()\n",
        "print(train_y.shape, valid_y.shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a8CPzypUY4TL"
      },
      "source": [
        "We can see that there are 16,344 total samples in our dataframe. We've extracted 13,057 for training and 3,269 for validation.\n",
        "\n",
        "After expanding the data dimension, we have explanatory variables of shape (256,1), a one-dimensional sequence of 256 bacterial 'species'."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zYabR-E7fsni"
      },
      "source": [
        "## simple linear model\n",
        "\n",
        "Now that we have some training and validation data, we just need to build a classifier to predict disease risk from the data.\n",
        "\n",
        "First, we'll start with a simple linear model implemented using a single Dense neuron with sigmoid output, as this is a binary classification problem.\n",
        "\n",
        "In previous cases, we were able to send the data directly to the Dense layer. But in this case, because we have 'expanded' the last dimension of the data - in order to fit tensorflow's sequence models - we need to 'collapse' that dimension back down, so it can be properly analyzed by the Dense layer.\n",
        "\n",
        "It's pretty easy to do this; we just need to use a Flatten layer to 'flatten' the 'expanded' data back down to a simple vector. And, because the Flatten layer is just like any other tensorflow Layer object, we can use it as the *first* layer in our network, provided we set the input_shape option.\n",
        "\n",
        "The following code cell implements a simple linear classifier for our disease-risk prediction problem."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ldKzbLpyXIAW",
        "outputId": "bbf531eb-c53e-40ef-ee61-20854f4deff5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " flatten_1 (Flatten)         (None, 256)               0         \n",
            "                                                                 \n",
            " dense_1 (Dense)             (None, 1)                 257       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 257 (1.00 KB)\n",
            "Trainable params: 257 (1.00 KB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n",
            "Epoch 1/20\n",
            "409/409 [==============================] - 2s 4ms/step - loss: 0.6910 - accuracy: 0.5468 - val_loss: 0.6895 - val_accuracy: 0.5216\n",
            "Epoch 2/20\n",
            "409/409 [==============================] - 2s 6ms/step - loss: 0.6686 - accuracy: 0.6028 - val_loss: 0.6711 - val_accuracy: 0.5418\n",
            "Epoch 3/20\n",
            "409/409 [==============================] - 1s 3ms/step - loss: 0.6516 - accuracy: 0.6481 - val_loss: 0.6547 - val_accuracy: 0.5702\n",
            "Epoch 4/20\n",
            "409/409 [==============================] - 1s 4ms/step - loss: 0.6359 - accuracy: 0.6823 - val_loss: 0.6396 - val_accuracy: 0.6020\n",
            "Epoch 5/20\n",
            "409/409 [==============================] - 1s 4ms/step - loss: 0.6214 - accuracy: 0.7112 - val_loss: 0.6257 - val_accuracy: 0.6357\n",
            "Epoch 6/20\n",
            "409/409 [==============================] - 1s 4ms/step - loss: 0.6080 - accuracy: 0.7310 - val_loss: 0.6128 - val_accuracy: 0.6614\n",
            "Epoch 7/20\n",
            "409/409 [==============================] - 2s 4ms/step - loss: 0.5955 - accuracy: 0.7473 - val_loss: 0.6008 - val_accuracy: 0.6852\n",
            "Epoch 8/20\n",
            "409/409 [==============================] - 2s 4ms/step - loss: 0.5839 - accuracy: 0.7595 - val_loss: 0.5898 - val_accuracy: 0.7011\n",
            "Epoch 9/20\n",
            "409/409 [==============================] - 1s 4ms/step - loss: 0.5732 - accuracy: 0.7689 - val_loss: 0.5795 - val_accuracy: 0.7177\n",
            "Epoch 10/20\n",
            "409/409 [==============================] - 1s 4ms/step - loss: 0.5631 - accuracy: 0.7769 - val_loss: 0.5699 - val_accuracy: 0.7296\n",
            "Epoch 11/20\n",
            "409/409 [==============================] - 1s 3ms/step - loss: 0.5537 - accuracy: 0.7814 - val_loss: 0.5609 - val_accuracy: 0.7391\n",
            "Epoch 12/20\n",
            "409/409 [==============================] - 2s 4ms/step - loss: 0.5450 - accuracy: 0.7857 - val_loss: 0.5526 - val_accuracy: 0.7476\n",
            "Epoch 13/20\n",
            "409/409 [==============================] - 1s 3ms/step - loss: 0.5368 - accuracy: 0.7907 - val_loss: 0.5447 - val_accuracy: 0.7562\n",
            "Epoch 14/20\n",
            "409/409 [==============================] - 1s 4ms/step - loss: 0.5290 - accuracy: 0.7936 - val_loss: 0.5374 - val_accuracy: 0.7617\n",
            "Epoch 15/20\n",
            "409/409 [==============================] - 2s 5ms/step - loss: 0.5218 - accuracy: 0.7972 - val_loss: 0.5305 - val_accuracy: 0.7684\n",
            "Epoch 16/20\n",
            "409/409 [==============================] - 2s 4ms/step - loss: 0.5150 - accuracy: 0.8008 - val_loss: 0.5241 - val_accuracy: 0.7727\n",
            "Epoch 17/20\n",
            "409/409 [==============================] - 1s 3ms/step - loss: 0.5086 - accuracy: 0.8038 - val_loss: 0.5180 - val_accuracy: 0.7770\n",
            "Epoch 18/20\n",
            "409/409 [==============================] - 2s 4ms/step - loss: 0.5025 - accuracy: 0.8060 - val_loss: 0.5122 - val_accuracy: 0.7791\n",
            "Epoch 19/20\n",
            "409/409 [==============================] - 2s 5ms/step - loss: 0.4968 - accuracy: 0.8078 - val_loss: 0.5068 - val_accuracy: 0.7822\n",
            "Epoch 20/20\n",
            "409/409 [==============================] - 1s 4ms/step - loss: 0.4914 - accuracy: 0.8099 - val_loss: 0.5017 - val_accuracy: 0.7856\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "<keras.src.callbacks.History at 0x78333012c910>"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import pandas\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "\n",
        "# download data\n",
        "dataframe = pandas.read_csv('https://raw.githubusercontent.com/bryankolaczkowski/ALS3200C/main/mbiome.data.csv')\n",
        "\n",
        "# create train-validate split\n",
        "train_dataframe = dataframe.sample(frac=0.8, random_state=2100963)\n",
        "valid_dataframe = dataframe.drop(train_dataframe.index)\n",
        "\n",
        "# extract explanatory variables\n",
        "dta_ids = [ x for x in dataframe.columns if x.find('DTA') == 0 ]\n",
        "train_x = np.expand_dims(train_dataframe[dta_ids].to_numpy(), axis=-1)\n",
        "valid_x = np.expand_dims(valid_dataframe[dta_ids].to_numpy(), axis=-1)\n",
        "\n",
        "# extract labels\n",
        "train_y = train_dataframe['LBL0'].to_numpy()\n",
        "valid_y = valid_dataframe['LBL0'].to_numpy()\n",
        "\n",
        "# package data into tensorflow dataset\n",
        "train_data = tf.data.Dataset.from_tensor_slices((train_x, train_y)).batch(32)\n",
        "valid_data = tf.data.Dataset.from_tensor_slices((valid_x, valid_y)).batch(32)\n",
        "\n",
        "# build model\n",
        "model = tf.keras.models.Sequential()\n",
        "model.add(tf.keras.layers.Flatten(input_shape=(256,1)))\n",
        "model.add(tf.keras.layers.Dense(units=1, activation=tf.keras.activations.sigmoid))\n",
        "model.summary()\n",
        "\n",
        "# compile model\n",
        "model.compile(optimizer=tf.keras.optimizers.Adam(),\n",
        "              loss=tf.keras.losses.BinaryCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "# fit model\n",
        "model.fit(train_data, epochs=20, validation_data=valid_data)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WAKPoof9XyDd"
      },
      "source": [
        "Our initial baseline model is a simple linear classifier with only 257 trainable parameters - 256 input weights plus a single bias term. Despite its simplicity, it achieves solid performance, reaching approximately 80% training accuracy and 78% validation accuracy after 20 epochs. This indicates that a substantial amount of predictive signal for the disease labels is contained in a linear combination of the microbial taxa abundances. The linear model provides a reasonable baseline to benchmark more complex nonlinear models against. We now explore whether convolutional and recurrent neural network architectures can capture higher-order nonlinear relationships and covariation in the community structure to improve classification accuracy beyond this linear baseline."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gqMA5a8V_eMI"
      },
      "source": [
        "## convolution model\n",
        "\n",
        "In the next code section, we implement a basic one-layer convolutional neural network for disease classification. The convolution layer consists of 16 filters of length 3 applied with same padding, such that the output sequence length remains 256. We use a tanh activation, which is less common than ReLU for modern convnets but allows for more direct comparison to the default activation in recurrent networks. With 4,161 trainable weights, this nonlinear model has substantially more parameters than the linear baseline. To account for this increased model capacity, we train for 50 epochs which is longer than the linear model. This convolution network allows us to model localized nonlinear interactions among adjacent taxa in the community profiles. We can evaluate whether directly capturing these short-range taxon co-occurrence patterns improves predictive performance beyond the linear model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "93rSQ41XYduw",
        "outputId": "c8af18fb-0795-4f34-cb73-3cbfb5f97661"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"sequential_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " conv1d_4 (Conv1D)           (None, 256, 16)           64        \n",
            "                                                                 \n",
            " flatten_2 (Flatten)         (None, 4096)              0         \n",
            "                                                                 \n",
            " dense_4 (Dense)             (None, 1)                 4097      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 4161 (16.25 KB)\n",
            "Trainable params: 4161 (16.25 KB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n",
            "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/50\n",
            "409/409 [==============================] - 1s 1ms/step - loss: 0.6752 - accuracy: 0.5778 - val_loss: 0.6363 - val_accuracy: 0.7167\n",
            "Epoch 2/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.6129 - accuracy: 0.6747 - val_loss: 0.5996 - val_accuracy: 0.6455\n",
            "Epoch 3/50\n",
            "409/409 [==============================] - 1s 2ms/step - loss: 0.5505 - accuracy: 0.7339 - val_loss: 0.5484 - val_accuracy: 0.7222\n",
            "Epoch 4/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.5100 - accuracy: 0.7623 - val_loss: 0.5200 - val_accuracy: 0.7485\n",
            "Epoch 5/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.4852 - accuracy: 0.7798 - val_loss: 0.4999 - val_accuracy: 0.7589\n",
            "Epoch 6/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.4663 - accuracy: 0.7920 - val_loss: 0.4843 - val_accuracy: 0.7645\n",
            "Epoch 7/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.4516 - accuracy: 0.7998 - val_loss: 0.4722 - val_accuracy: 0.7687\n",
            "Epoch 8/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.4400 - accuracy: 0.8059 - val_loss: 0.4628 - val_accuracy: 0.7749\n",
            "Epoch 9/50\n",
            "409/409 [==============================] - 1s 2ms/step - loss: 0.4307 - accuracy: 0.8121 - val_loss: 0.4553 - val_accuracy: 0.7816\n",
            "Epoch 10/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.4230 - accuracy: 0.8149 - val_loss: 0.4491 - val_accuracy: 0.7822\n",
            "Epoch 11/50\n",
            "409/409 [==============================] - 1s 1ms/step - loss: 0.4165 - accuracy: 0.8180 - val_loss: 0.4439 - val_accuracy: 0.7862\n",
            "Epoch 12/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.4111 - accuracy: 0.8199 - val_loss: 0.4396 - val_accuracy: 0.7895\n",
            "Epoch 13/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.4063 - accuracy: 0.8220 - val_loss: 0.4359 - val_accuracy: 0.7920\n",
            "Epoch 14/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.4022 - accuracy: 0.8236 - val_loss: 0.4328 - val_accuracy: 0.7941\n",
            "Epoch 15/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3985 - accuracy: 0.8238 - val_loss: 0.4301 - val_accuracy: 0.7954\n",
            "Epoch 16/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3952 - accuracy: 0.8247 - val_loss: 0.4277 - val_accuracy: 0.7981\n",
            "Epoch 17/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3923 - accuracy: 0.8253 - val_loss: 0.4256 - val_accuracy: 0.7987\n",
            "Epoch 18/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3897 - accuracy: 0.8264 - val_loss: 0.4238 - val_accuracy: 0.8002\n",
            "Epoch 19/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3873 - accuracy: 0.8272 - val_loss: 0.4221 - val_accuracy: 0.8024\n",
            "Epoch 20/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3851 - accuracy: 0.8276 - val_loss: 0.4206 - val_accuracy: 0.8042\n",
            "Epoch 21/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3831 - accuracy: 0.8278 - val_loss: 0.4193 - val_accuracy: 0.8039\n",
            "Epoch 22/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3813 - accuracy: 0.8288 - val_loss: 0.4180 - val_accuracy: 0.8033\n",
            "Epoch 23/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3796 - accuracy: 0.8298 - val_loss: 0.4169 - val_accuracy: 0.8045\n",
            "Epoch 24/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3780 - accuracy: 0.8301 - val_loss: 0.4158 - val_accuracy: 0.8051\n",
            "Epoch 25/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3766 - accuracy: 0.8321 - val_loss: 0.4148 - val_accuracy: 0.8054\n",
            "Epoch 26/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3752 - accuracy: 0.8328 - val_loss: 0.4138 - val_accuracy: 0.8064\n",
            "Epoch 27/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3740 - accuracy: 0.8337 - val_loss: 0.4129 - val_accuracy: 0.8067\n",
            "Epoch 28/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3728 - accuracy: 0.8343 - val_loss: 0.4120 - val_accuracy: 0.8076\n",
            "Epoch 29/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3717 - accuracy: 0.8347 - val_loss: 0.4112 - val_accuracy: 0.8082\n",
            "Epoch 30/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3707 - accuracy: 0.8350 - val_loss: 0.4104 - val_accuracy: 0.8079\n",
            "Epoch 31/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3697 - accuracy: 0.8354 - val_loss: 0.4096 - val_accuracy: 0.8076\n",
            "Epoch 32/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3688 - accuracy: 0.8354 - val_loss: 0.4088 - val_accuracy: 0.8073\n",
            "Epoch 33/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3680 - accuracy: 0.8356 - val_loss: 0.4080 - val_accuracy: 0.8073\n",
            "Epoch 34/50\n",
            "409/409 [==============================] - 1s 1ms/step - loss: 0.3672 - accuracy: 0.8359 - val_loss: 0.4073 - val_accuracy: 0.8073\n",
            "Epoch 35/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3664 - accuracy: 0.8360 - val_loss: 0.4066 - val_accuracy: 0.8073\n",
            "Epoch 36/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3657 - accuracy: 0.8362 - val_loss: 0.4059 - val_accuracy: 0.8070\n",
            "Epoch 37/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3650 - accuracy: 0.8367 - val_loss: 0.4052 - val_accuracy: 0.8097\n",
            "Epoch 38/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3644 - accuracy: 0.8374 - val_loss: 0.4045 - val_accuracy: 0.8094\n",
            "Epoch 39/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3638 - accuracy: 0.8373 - val_loss: 0.4038 - val_accuracy: 0.8100\n",
            "Epoch 40/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3632 - accuracy: 0.8374 - val_loss: 0.4031 - val_accuracy: 0.8106\n",
            "Epoch 41/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3626 - accuracy: 0.8375 - val_loss: 0.4025 - val_accuracy: 0.8110\n",
            "Epoch 42/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3621 - accuracy: 0.8385 - val_loss: 0.4018 - val_accuracy: 0.8122\n",
            "Epoch 43/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3616 - accuracy: 0.8390 - val_loss: 0.4012 - val_accuracy: 0.8116\n",
            "Epoch 44/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3612 - accuracy: 0.8391 - val_loss: 0.4006 - val_accuracy: 0.8125\n",
            "Epoch 45/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3607 - accuracy: 0.8395 - val_loss: 0.3999 - val_accuracy: 0.8134\n",
            "Epoch 46/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3603 - accuracy: 0.8395 - val_loss: 0.3993 - val_accuracy: 0.8137\n",
            "Epoch 47/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3599 - accuracy: 0.8398 - val_loss: 0.3987 - val_accuracy: 0.8134\n",
            "Epoch 48/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3595 - accuracy: 0.8397 - val_loss: 0.3981 - val_accuracy: 0.8143\n",
            "Epoch 49/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3591 - accuracy: 0.8399 - val_loss: 0.3975 - val_accuracy: 0.8155\n",
            "Epoch 50/50\n",
            "409/409 [==============================] - 0s 1ms/step - loss: 0.3588 - accuracy: 0.8402 - val_loss: 0.3969 - val_accuracy: 0.8165\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "<keras.src.callbacks.History at 0x2c2333250>"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import pandas\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "\n",
        "# download data\n",
        "dataframe = pandas.read_csv('https://raw.githubusercontent.com/bryankolaczkowski/ALS3200C/main/mbiome.data.csv')\n",
        "\n",
        "# create train-validate split\n",
        "train_dataframe = dataframe.sample(frac=0.8, random_state=2100963)\n",
        "valid_dataframe = dataframe.drop(train_dataframe.index)\n",
        "\n",
        "# extract explanatory variables\n",
        "dta_ids = [ x for x in dataframe.columns if x.find('DTA') == 0 ]\n",
        "train_x = np.expand_dims(train_dataframe[dta_ids].to_numpy(), axis=-1)\n",
        "valid_x = np.expand_dims(valid_dataframe[dta_ids].to_numpy(), axis=-1)\n",
        "\n",
        "# extract labels\n",
        "train_y = train_dataframe['LBL0'].to_numpy()\n",
        "valid_y = valid_dataframe['LBL0'].to_numpy()\n",
        "\n",
        "# package data into tensorflow dataset\n",
        "train_data = tf.data.Dataset.from_tensor_slices((train_x, train_y)).batch(32)\n",
        "valid_data = tf.data.Dataset.from_tensor_slices((valid_x, valid_y)).batch(32)\n",
        "\n",
        "# build model\n",
        "model = tf.keras.models.Sequential()\n",
        "model.add(tf.keras.layers.Conv1D(filters=16,\n",
        "                                 kernel_size=(3,),\n",
        "                                 activation=tf.keras.activations.tanh,\n",
        "                                 padding='same',\n",
        "                                 input_shape=(256,1)))\n",
        "model.add(tf.keras.layers.Flatten())\n",
        "model.add(tf.keras.layers.Dense(units=1, activation=tf.keras.activations.sigmoid))\n",
        "model.summary()\n",
        "\n",
        "# compile model\n",
        "model.compile(optimizer=tf.keras.optimizers.Adam(),\n",
        "              loss=tf.keras.losses.BinaryCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "# fit model\n",
        "model.fit(train_data, epochs=50, validation_data=valid_data)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z4ZyqiR5BWOx"
      },
      "source": [
        "## LSTM model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ijrxhUsmZXtB",
        "outputId": "17e8c8ba-7174-42c2-d5f9-3d8e748d3194"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"sequential_3\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " lstm (LSTM)                 (None, 256, 16)           1152      \n",
            "                                                                 \n",
            " flatten_3 (Flatten)         (None, 4096)              0         \n",
            "                                                                 \n",
            " dense_5 (Dense)             (None, 1)                 4097      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 5249 (20.50 KB)\n",
            "Trainable params: 5249 (20.50 KB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "_________________________________________________________________\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n",
            "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/50\n",
            "409/409 [==============================] - 11s 25ms/step - loss: 0.6791 - accuracy: 0.5606 - val_loss: 0.6420 - val_accuracy: 0.7054\n",
            "Epoch 2/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.6165 - accuracy: 0.6766 - val_loss: 0.6013 - val_accuracy: 0.6601\n",
            "Epoch 3/50\n",
            "409/409 [==============================] - 11s 27ms/step - loss: 0.5451 - accuracy: 0.7407 - val_loss: 0.5271 - val_accuracy: 0.7443\n",
            "Epoch 4/50\n",
            "409/409 [==============================] - 11s 26ms/step - loss: 0.4920 - accuracy: 0.7725 - val_loss: 0.5017 - val_accuracy: 0.7489\n",
            "Epoch 5/50\n",
            "409/409 [==============================] - 10s 26ms/step - loss: 0.4624 - accuracy: 0.7846 - val_loss: 0.4892 - val_accuracy: 0.7525\n",
            "Epoch 6/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.4431 - accuracy: 0.7966 - val_loss: 0.4839 - val_accuracy: 0.7571\n",
            "Epoch 7/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.4292 - accuracy: 0.8054 - val_loss: 0.4808 - val_accuracy: 0.7620\n",
            "Epoch 8/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.4181 - accuracy: 0.8108 - val_loss: 0.4764 - val_accuracy: 0.7675\n",
            "Epoch 9/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.4092 - accuracy: 0.8163 - val_loss: 0.4726 - val_accuracy: 0.7715\n",
            "Epoch 10/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.4027 - accuracy: 0.8187 - val_loss: 0.4724 - val_accuracy: 0.7730\n",
            "Epoch 11/50\n",
            "409/409 [==============================] - 10s 24ms/step - loss: 0.3977 - accuracy: 0.8215 - val_loss: 0.4731 - val_accuracy: 0.7721\n",
            "Epoch 12/50\n",
            "409/409 [==============================] - 10s 24ms/step - loss: 0.3935 - accuracy: 0.8236 - val_loss: 0.4739 - val_accuracy: 0.7733\n",
            "Epoch 13/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3901 - accuracy: 0.8251 - val_loss: 0.4746 - val_accuracy: 0.7749\n",
            "Epoch 14/50\n",
            "409/409 [==============================] - 11s 26ms/step - loss: 0.3871 - accuracy: 0.8268 - val_loss: 0.4750 - val_accuracy: 0.7767\n",
            "Epoch 15/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3845 - accuracy: 0.8270 - val_loss: 0.4751 - val_accuracy: 0.7758\n",
            "Epoch 16/50\n",
            "409/409 [==============================] - 11s 26ms/step - loss: 0.3822 - accuracy: 0.8282 - val_loss: 0.4749 - val_accuracy: 0.7767\n",
            "Epoch 17/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3801 - accuracy: 0.8292 - val_loss: 0.4744 - val_accuracy: 0.7776\n",
            "Epoch 18/50\n",
            "409/409 [==============================] - 11s 26ms/step - loss: 0.3782 - accuracy: 0.8297 - val_loss: 0.4737 - val_accuracy: 0.7779\n",
            "Epoch 19/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3764 - accuracy: 0.8298 - val_loss: 0.4728 - val_accuracy: 0.7801\n",
            "Epoch 20/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3747 - accuracy: 0.8307 - val_loss: 0.4717 - val_accuracy: 0.7816\n",
            "Epoch 21/50\n",
            "409/409 [==============================] - 11s 26ms/step - loss: 0.3730 - accuracy: 0.8311 - val_loss: 0.4706 - val_accuracy: 0.7837\n",
            "Epoch 22/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3715 - accuracy: 0.8312 - val_loss: 0.4694 - val_accuracy: 0.7862\n",
            "Epoch 23/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3700 - accuracy: 0.8324 - val_loss: 0.4682 - val_accuracy: 0.7868\n",
            "Epoch 24/50\n",
            "409/409 [==============================] - 11s 26ms/step - loss: 0.3686 - accuracy: 0.8332 - val_loss: 0.4671 - val_accuracy: 0.7892\n",
            "Epoch 25/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3672 - accuracy: 0.8337 - val_loss: 0.4660 - val_accuracy: 0.7889\n",
            "Epoch 26/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3659 - accuracy: 0.8350 - val_loss: 0.4649 - val_accuracy: 0.7901\n",
            "Epoch 27/50\n",
            "409/409 [==============================] - 11s 26ms/step - loss: 0.3647 - accuracy: 0.8361 - val_loss: 0.4638 - val_accuracy: 0.7911\n",
            "Epoch 28/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3635 - accuracy: 0.8369 - val_loss: 0.4627 - val_accuracy: 0.7911\n",
            "Epoch 29/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3623 - accuracy: 0.8372 - val_loss: 0.4616 - val_accuracy: 0.7926\n",
            "Epoch 30/50\n",
            "409/409 [==============================] - 11s 28ms/step - loss: 0.3612 - accuracy: 0.8379 - val_loss: 0.4604 - val_accuracy: 0.7935\n",
            "Epoch 31/50\n",
            "409/409 [==============================] - 11s 26ms/step - loss: 0.3601 - accuracy: 0.8385 - val_loss: 0.4591 - val_accuracy: 0.7935\n",
            "Epoch 32/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3591 - accuracy: 0.8392 - val_loss: 0.4578 - val_accuracy: 0.7935\n",
            "Epoch 33/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3581 - accuracy: 0.8404 - val_loss: 0.4565 - val_accuracy: 0.7941\n",
            "Epoch 34/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3571 - accuracy: 0.8410 - val_loss: 0.4551 - val_accuracy: 0.7938\n",
            "Epoch 35/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3562 - accuracy: 0.8421 - val_loss: 0.4538 - val_accuracy: 0.7947\n",
            "Epoch 36/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3553 - accuracy: 0.8426 - val_loss: 0.4524 - val_accuracy: 0.7966\n",
            "Epoch 37/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3545 - accuracy: 0.8433 - val_loss: 0.4510 - val_accuracy: 0.7978\n",
            "Epoch 38/50\n",
            "409/409 [==============================] - 10s 24ms/step - loss: 0.3537 - accuracy: 0.8439 - val_loss: 0.4496 - val_accuracy: 0.7975\n",
            "Epoch 39/50\n",
            "409/409 [==============================] - 10s 24ms/step - loss: 0.3529 - accuracy: 0.8445 - val_loss: 0.4482 - val_accuracy: 0.7984\n",
            "Epoch 40/50\n",
            "409/409 [==============================] - 10s 24ms/step - loss: 0.3521 - accuracy: 0.8445 - val_loss: 0.4469 - val_accuracy: 0.8002\n",
            "Epoch 41/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3514 - accuracy: 0.8447 - val_loss: 0.4455 - val_accuracy: 0.8002\n",
            "Epoch 42/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3507 - accuracy: 0.8447 - val_loss: 0.4441 - val_accuracy: 0.8018\n",
            "Epoch 43/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3500 - accuracy: 0.8450 - val_loss: 0.4427 - val_accuracy: 0.8018\n",
            "Epoch 44/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3493 - accuracy: 0.8449 - val_loss: 0.4413 - val_accuracy: 0.8024\n",
            "Epoch 45/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3487 - accuracy: 0.8451 - val_loss: 0.4398 - val_accuracy: 0.8033\n",
            "Epoch 46/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3480 - accuracy: 0.8454 - val_loss: 0.4384 - val_accuracy: 0.8045\n",
            "Epoch 47/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3474 - accuracy: 0.8460 - val_loss: 0.4370 - val_accuracy: 0.8064\n",
            "Epoch 48/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3468 - accuracy: 0.8458 - val_loss: 0.4356 - val_accuracy: 0.8073\n",
            "Epoch 49/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3463 - accuracy: 0.8461 - val_loss: 0.4342 - val_accuracy: 0.8085\n",
            "Epoch 50/50\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3457 - accuracy: 0.8459 - val_loss: 0.4329 - val_accuracy: 0.8088\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "<keras.src.callbacks.History at 0x2c561ac10>"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import pandas\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "\n",
        "# download data\n",
        "dataframe = pandas.read_csv('https://raw.githubusercontent.com/bryankolaczkowski/ALS3200C/main/mbiome.data.csv')\n",
        "\n",
        "# create train-validate split\n",
        "train_dataframe = dataframe.sample(frac=0.8, random_state=2100963)\n",
        "valid_dataframe = dataframe.drop(train_dataframe.index)\n",
        "\n",
        "# extract explanatory variables\n",
        "dta_ids = [ x for x in dataframe.columns if x.find('DTA') == 0 ]\n",
        "train_x = np.expand_dims(train_dataframe[dta_ids].to_numpy(), axis=-1)\n",
        "valid_x = np.expand_dims(valid_dataframe[dta_ids].to_numpy(), axis=-1)\n",
        "\n",
        "# extract labels\n",
        "train_y = train_dataframe['LBL0'].to_numpy()\n",
        "valid_y = valid_dataframe['LBL0'].to_numpy()\n",
        "\n",
        "# package data into tensorflow dataset\n",
        "train_data = tf.data.Dataset.from_tensor_slices((train_x, train_y)).batch(32)\n",
        "valid_data = tf.data.Dataset.from_tensor_slices((valid_x, valid_y)).batch(32)\n",
        "\n",
        "# build model\n",
        "model = tf.keras.models.Sequential()\n",
        "model.add(tf.keras.layers.LSTM(units=16, return_sequences=True, input_shape=(256,1)))\n",
        "model.add(tf.keras.layers.Flatten())\n",
        "model.add(tf.keras.layers.Dense(units=1, activation=tf.keras.activations.sigmoid))\n",
        "model.summary()\n",
        "\n",
        "\n",
        "# compile model\n",
        "model.compile(optimizer=tf.keras.optimizers.Adam(),\n",
        "              loss=tf.keras.losses.BinaryCrossentropy(),\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "# fit model\n",
        "model.fit(train_data, epochs=50, validation_data=valid_data)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n6qAur4XVOGP"
      },
      "source": [
        "# Basic Transformer Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "H9KCqLwiVOGP",
        "outputId": "bdac22fd-dfd6-46eb-f3db-4d70950bcc1e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"model_5\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                Output Shape                 Param #   Connected to                  \n",
            "==================================================================================================\n",
            " input_6 (InputLayer)        [(None, 256, 2)]             0         []                            \n",
            "                                                                                                  \n",
            " dense_20 (Dense)            (None, 256, 4)               12        ['input_6[0][0]']             \n",
            "                                                                                                  \n",
            " multi_head_attention_5 (Mu  (None, 256, 4)               156       ['dense_20[0][0]',            \n",
            " ltiHeadAttention)                                                   'dense_20[0][0]',            \n",
            "                                                                     'dense_20[0][0]']            \n",
            "                                                                                                  \n",
            " add_10 (Add)                (None, 256, 4)               0         ['dense_20[0][0]',            \n",
            "                                                                     'multi_head_attention_5[0][0]\n",
            "                                                                    ']                            \n",
            "                                                                                                  \n",
            " layer_normalization_10 (La  (None, 256, 4)               8         ['add_10[0][0]']              \n",
            " yerNormalization)                                                                                \n",
            "                                                                                                  \n",
            " dense_21 (Dense)            (None, 256, 4)               20        ['layer_normalization_10[0][0]\n",
            "                                                                    ']                            \n",
            "                                                                                                  \n",
            " dense_22 (Dense)            (None, 256, 4)               20        ['dense_21[0][0]']            \n",
            "                                                                                                  \n",
            " add_11 (Add)                (None, 256, 4)               0         ['layer_normalization_10[0][0]\n",
            "                                                                    ',                            \n",
            "                                                                     'dense_22[0][0]']            \n",
            "                                                                                                  \n",
            " layer_normalization_11 (La  (None, 256, 4)               8         ['add_11[0][0]']              \n",
            " yerNormalization)                                                                                \n",
            "                                                                                                  \n",
            " flatten_5 (Flatten)         (None, 1024)                 0         ['layer_normalization_11[0][0]\n",
            "                                                                    ']                            \n",
            "                                                                                                  \n",
            " dense_23 (Dense)            (None, 1)                    1025      ['flatten_5[0][0]']           \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 1249 (4.88 KB)\n",
            "Trainable params: 1249 (4.88 KB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "__________________________________________________________________________________________________\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/50\n",
            "409/409 [==============================] - 6s 13ms/step - loss: 0.7063 - accuracy: 0.5259 - precision: 0.5197 - recall: 0.5022 - auc: 0.5345 - val_loss: 0.7007 - val_accuracy: 0.5222 - val_precision: 0.5180 - val_recall: 0.9546 - val_auc: 0.5785\n",
            "Epoch 2/50\n",
            "409/409 [==============================] - 6s 14ms/step - loss: 0.6784 - accuracy: 0.5660 - precision: 0.5617 - recall: 0.5437 - auc: 0.5976 - val_loss: 0.6570 - val_accuracy: 0.5990 - val_precision: 0.5691 - val_recall: 0.8906 - val_auc: 0.6941\n",
            "Epoch 3/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.6347 - accuracy: 0.6382 - precision: 0.6363 - recall: 0.6208 - auc: 0.6922 - val_loss: 0.6172 - val_accuracy: 0.6497 - val_precision: 0.7927 - val_recall: 0.4274 - val_auc: 0.7690\n",
            "Epoch 4/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.5874 - accuracy: 0.6892 - precision: 0.6901 - recall: 0.6706 - auc: 0.7561 - val_loss: 0.6510 - val_accuracy: 0.6133 - val_precision: 0.8895 - val_recall: 0.2791 - val_auc: 0.8104\n",
            "Epoch 5/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.5242 - accuracy: 0.7386 - precision: 0.7386 - recall: 0.7269 - auc: 0.8188 - val_loss: 0.5878 - val_accuracy: 0.6760 - val_precision: 0.8987 - val_recall: 0.4136 - val_auc: 0.8582\n",
            "Epoch 6/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.4462 - accuracy: 0.7901 - precision: 0.7896 - recall: 0.7828 - auc: 0.8747 - val_loss: 0.4048 - val_accuracy: 0.8195 - val_precision: 0.8044 - val_recall: 0.8553 - val_auc: 0.9031\n",
            "Epoch 7/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.4015 - accuracy: 0.8161 - precision: 0.8135 - recall: 0.8133 - auc: 0.9001 - val_loss: 0.3925 - val_accuracy: 0.8217 - val_precision: 0.7874 - val_recall: 0.8924 - val_auc: 0.9104\n",
            "Epoch 8/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3847 - accuracy: 0.8256 - precision: 0.8232 - recall: 0.8230 - auc: 0.9081 - val_loss: 0.3848 - val_accuracy: 0.8269 - val_precision: 0.7939 - val_recall: 0.8936 - val_auc: 0.9135\n",
            "Epoch 9/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3761 - accuracy: 0.8295 - precision: 0.8269 - recall: 0.8273 - auc: 0.9121 - val_loss: 0.3805 - val_accuracy: 0.8293 - val_precision: 0.7973 - val_recall: 0.8936 - val_auc: 0.9152\n",
            "Epoch 10/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3710 - accuracy: 0.8320 - precision: 0.8294 - recall: 0.8299 - auc: 0.9144 - val_loss: 0.3776 - val_accuracy: 0.8296 - val_precision: 0.7978 - val_recall: 0.8936 - val_auc: 0.9164\n",
            "Epoch 11/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3672 - accuracy: 0.8350 - precision: 0.8328 - recall: 0.8323 - auc: 0.9161 - val_loss: 0.3758 - val_accuracy: 0.8318 - val_precision: 0.7995 - val_recall: 0.8960 - val_auc: 0.9172\n",
            "Epoch 12/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3643 - accuracy: 0.8362 - precision: 0.8338 - recall: 0.8338 - auc: 0.9174 - val_loss: 0.3745 - val_accuracy: 0.8314 - val_precision: 0.8000 - val_recall: 0.8942 - val_auc: 0.9179\n",
            "Epoch 13/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3619 - accuracy: 0.8375 - precision: 0.8347 - recall: 0.8358 - auc: 0.9184 - val_loss: 0.3737 - val_accuracy: 0.8302 - val_precision: 0.7983 - val_recall: 0.8942 - val_auc: 0.9182\n",
            "Epoch 14/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3600 - accuracy: 0.8374 - precision: 0.8347 - recall: 0.8357 - auc: 0.9193 - val_loss: 0.3730 - val_accuracy: 0.8308 - val_precision: 0.7985 - val_recall: 0.8954 - val_auc: 0.9187\n",
            "Epoch 15/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3584 - accuracy: 0.8384 - precision: 0.8354 - recall: 0.8371 - auc: 0.9200 - val_loss: 0.3733 - val_accuracy: 0.8311 - val_precision: 0.7970 - val_recall: 0.8990 - val_auc: 0.9190\n",
            "Epoch 16/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3570 - accuracy: 0.8395 - precision: 0.8364 - recall: 0.8385 - auc: 0.9206 - val_loss: 0.3734 - val_accuracy: 0.8305 - val_precision: 0.7949 - val_recall: 0.9014 - val_auc: 0.9194\n",
            "Epoch 17/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3557 - accuracy: 0.8399 - precision: 0.8367 - recall: 0.8389 - auc: 0.9212 - val_loss: 0.3740 - val_accuracy: 0.8299 - val_precision: 0.7926 - val_recall: 0.9044 - val_auc: 0.9196\n",
            "Epoch 18/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3547 - accuracy: 0.8403 - precision: 0.8375 - recall: 0.8388 - auc: 0.9217 - val_loss: 0.3748 - val_accuracy: 0.8293 - val_precision: 0.7899 - val_recall: 0.9079 - val_auc: 0.9199\n",
            "Epoch 19/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3537 - accuracy: 0.8408 - precision: 0.8384 - recall: 0.8386 - auc: 0.9221 - val_loss: 0.3757 - val_accuracy: 0.8284 - val_precision: 0.7881 - val_recall: 0.9091 - val_auc: 0.9200\n",
            "Epoch 20/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3528 - accuracy: 0.8415 - precision: 0.8391 - recall: 0.8393 - auc: 0.9225 - val_loss: 0.3768 - val_accuracy: 0.8287 - val_precision: 0.7870 - val_recall: 0.9121 - val_auc: 0.9203\n",
            "Epoch 21/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3520 - accuracy: 0.8411 - precision: 0.8385 - recall: 0.8393 - auc: 0.9229 - val_loss: 0.3781 - val_accuracy: 0.8275 - val_precision: 0.7845 - val_recall: 0.9139 - val_auc: 0.9205\n",
            "Epoch 22/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3512 - accuracy: 0.8415 - precision: 0.8393 - recall: 0.8393 - auc: 0.9232 - val_loss: 0.3788 - val_accuracy: 0.8259 - val_precision: 0.7822 - val_recall: 0.9145 - val_auc: 0.9208\n",
            "Epoch 23/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3506 - accuracy: 0.8424 - precision: 0.8402 - recall: 0.8400 - auc: 0.9235 - val_loss: 0.3796 - val_accuracy: 0.8269 - val_precision: 0.7820 - val_recall: 0.9175 - val_auc: 0.9209\n",
            "Epoch 24/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3499 - accuracy: 0.8431 - precision: 0.8408 - recall: 0.8408 - auc: 0.9238 - val_loss: 0.3803 - val_accuracy: 0.8266 - val_precision: 0.7810 - val_recall: 0.9187 - val_auc: 0.9211\n",
            "Epoch 25/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3493 - accuracy: 0.8433 - precision: 0.8410 - recall: 0.8411 - auc: 0.9241 - val_loss: 0.3808 - val_accuracy: 0.8275 - val_precision: 0.7810 - val_recall: 0.9211 - val_auc: 0.9213\n",
            "Epoch 26/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3488 - accuracy: 0.8434 - precision: 0.8411 - recall: 0.8413 - auc: 0.9243 - val_loss: 0.3812 - val_accuracy: 0.8275 - val_precision: 0.7808 - val_recall: 0.9217 - val_auc: 0.9214\n",
            "Epoch 27/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3483 - accuracy: 0.8433 - precision: 0.8410 - recall: 0.8411 - auc: 0.9246 - val_loss: 0.3818 - val_accuracy: 0.8275 - val_precision: 0.7799 - val_recall: 0.9235 - val_auc: 0.9215\n",
            "Epoch 28/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3478 - accuracy: 0.8439 - precision: 0.8418 - recall: 0.8414 - auc: 0.9248 - val_loss: 0.3824 - val_accuracy: 0.8272 - val_precision: 0.7795 - val_recall: 0.9235 - val_auc: 0.9216\n",
            "Epoch 29/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3474 - accuracy: 0.8439 - precision: 0.8421 - recall: 0.8410 - auc: 0.9250 - val_loss: 0.3829 - val_accuracy: 0.8272 - val_precision: 0.7792 - val_recall: 0.9241 - val_auc: 0.9216\n",
            "Epoch 30/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3469 - accuracy: 0.8444 - precision: 0.8426 - recall: 0.8414 - auc: 0.9252 - val_loss: 0.3836 - val_accuracy: 0.8272 - val_precision: 0.7792 - val_recall: 0.9241 - val_auc: 0.9217\n",
            "Epoch 31/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3466 - accuracy: 0.8443 - precision: 0.8429 - recall: 0.8408 - auc: 0.9254 - val_loss: 0.3835 - val_accuracy: 0.8278 - val_precision: 0.7797 - val_recall: 0.9247 - val_auc: 0.9218\n",
            "Epoch 32/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3462 - accuracy: 0.8445 - precision: 0.8428 - recall: 0.8416 - auc: 0.9255 - val_loss: 0.3837 - val_accuracy: 0.8275 - val_precision: 0.7793 - val_recall: 0.9247 - val_auc: 0.9220\n",
            "Epoch 33/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3458 - accuracy: 0.8444 - precision: 0.8425 - recall: 0.8417 - auc: 0.9257 - val_loss: 0.3832 - val_accuracy: 0.8272 - val_precision: 0.7790 - val_recall: 0.9247 - val_auc: 0.9221\n",
            "Epoch 34/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3454 - accuracy: 0.8447 - precision: 0.8429 - recall: 0.8417 - auc: 0.9259 - val_loss: 0.3831 - val_accuracy: 0.8287 - val_precision: 0.7806 - val_recall: 0.9253 - val_auc: 0.9221\n",
            "Epoch 35/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3451 - accuracy: 0.8445 - precision: 0.8427 - recall: 0.8417 - auc: 0.9261 - val_loss: 0.3824 - val_accuracy: 0.8299 - val_precision: 0.7822 - val_recall: 0.9253 - val_auc: 0.9222\n",
            "Epoch 36/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3447 - accuracy: 0.8443 - precision: 0.8422 - recall: 0.8419 - auc: 0.9262 - val_loss: 0.3819 - val_accuracy: 0.8293 - val_precision: 0.7820 - val_recall: 0.9241 - val_auc: 0.9222\n",
            "Epoch 37/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3445 - accuracy: 0.8444 - precision: 0.8427 - recall: 0.8414 - auc: 0.9264 - val_loss: 0.3812 - val_accuracy: 0.8305 - val_precision: 0.7839 - val_recall: 0.9235 - val_auc: 0.9223\n",
            "Epoch 38/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3442 - accuracy: 0.8449 - precision: 0.8433 - recall: 0.8417 - auc: 0.9265 - val_loss: 0.3805 - val_accuracy: 0.8299 - val_precision: 0.7834 - val_recall: 0.9229 - val_auc: 0.9223\n",
            "Epoch 39/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3438 - accuracy: 0.8447 - precision: 0.8431 - recall: 0.8414 - auc: 0.9267 - val_loss: 0.3800 - val_accuracy: 0.8305 - val_precision: 0.7844 - val_recall: 0.9223 - val_auc: 0.9224\n",
            "Epoch 40/50\n",
            "409/409 [==============================] - 5s 13ms/step - loss: 0.3436 - accuracy: 0.8457 - precision: 0.8441 - recall: 0.8425 - auc: 0.9268 - val_loss: 0.3793 - val_accuracy: 0.8290 - val_precision: 0.7839 - val_recall: 0.9193 - val_auc: 0.9224\n",
            "Epoch 41/50\n",
            "409/409 [==============================] - 6s 14ms/step - loss: 0.3433 - accuracy: 0.8460 - precision: 0.8441 - recall: 0.8433 - auc: 0.9269 - val_loss: 0.3782 - val_accuracy: 0.8290 - val_precision: 0.7848 - val_recall: 0.9175 - val_auc: 0.9224\n",
            "Epoch 42/50\n",
            "409/409 [==============================] - 6s 14ms/step - loss: 0.3430 - accuracy: 0.8464 - precision: 0.8446 - recall: 0.8436 - auc: 0.9271 - val_loss: 0.3767 - val_accuracy: 0.8305 - val_precision: 0.7871 - val_recall: 0.9169 - val_auc: 0.9222\n",
            "Epoch 43/50\n",
            "409/409 [==============================] - 5s 12ms/step - loss: 0.3427 - accuracy: 0.8474 - precision: 0.8454 - recall: 0.8450 - auc: 0.9272 - val_loss: 0.3753 - val_accuracy: 0.8302 - val_precision: 0.7881 - val_recall: 0.9139 - val_auc: 0.9223\n",
            "Epoch 44/50\n",
            "409/409 [==============================] - 5s 12ms/step - loss: 0.3424 - accuracy: 0.8473 - precision: 0.8457 - recall: 0.8442 - auc: 0.9274 - val_loss: 0.3737 - val_accuracy: 0.8299 - val_precision: 0.7886 - val_recall: 0.9121 - val_auc: 0.9224\n",
            "Epoch 45/50\n",
            "409/409 [==============================] - 5s 12ms/step - loss: 0.3420 - accuracy: 0.8480 - precision: 0.8467 - recall: 0.8444 - auc: 0.9275 - val_loss: 0.3722 - val_accuracy: 0.8293 - val_precision: 0.7896 - val_recall: 0.9085 - val_auc: 0.9223\n",
            "Epoch 46/50\n",
            "409/409 [==============================] - 5s 12ms/step - loss: 0.3417 - accuracy: 0.8485 - precision: 0.8472 - recall: 0.8450 - auc: 0.9277 - val_loss: 0.3706 - val_accuracy: 0.8314 - val_precision: 0.7937 - val_recall: 0.9062 - val_auc: 0.9223\n",
            "Epoch 47/50\n",
            "409/409 [==============================] - 5s 12ms/step - loss: 0.3414 - accuracy: 0.8484 - precision: 0.8472 - recall: 0.8448 - auc: 0.9278 - val_loss: 0.3696 - val_accuracy: 0.8314 - val_precision: 0.7937 - val_recall: 0.9062 - val_auc: 0.9224\n",
            "Epoch 48/50\n",
            "409/409 [==============================] - 143s 350ms/step - loss: 0.3412 - accuracy: 0.8486 - precision: 0.8469 - recall: 0.8456 - auc: 0.9279 - val_loss: 0.3680 - val_accuracy: 0.8330 - val_precision: 0.7964 - val_recall: 0.9050 - val_auc: 0.9225\n",
            "Epoch 49/50\n",
            "409/409 [==============================] - 5s 12ms/step - loss: 0.3408 - accuracy: 0.8493 - precision: 0.8478 - recall: 0.8462 - auc: 0.9281 - val_loss: 0.3667 - val_accuracy: 0.8351 - val_precision: 0.8000 - val_recall: 0.9038 - val_auc: 0.9225\n",
            "Epoch 50/50\n",
            "409/409 [==============================] - 5s 12ms/step - loss: 0.3405 - accuracy: 0.8501 - precision: 0.8487 - recall: 0.8469 - auc: 0.9282 - val_loss: 0.3652 - val_accuracy: 0.8373 - val_precision: 0.8046 - val_recall: 0.9008 - val_auc: 0.9225\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "<keras.src.callbacks.History at 0x2f87cdc10>"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from sklearn.metrics import precision_score, recall_score, roc_auc_score\n",
        "\n",
        "# download data\n",
        "dataframe = pd.read_csv('https://raw.githubusercontent.com/bryankolaczkowski/ALS3200C/main/mbiome.data.csv')\n",
        "\n",
        "# create train-validate split\n",
        "train_dataframe = dataframe.sample(frac=0.8, random_state=827847)\n",
        "valid_dataframe = dataframe.drop(train_dataframe.index)\n",
        "\n",
        "# extract explanatory variables\n",
        "dta_ids = [ x for x in dataframe.columns if x.find('DTA') == 0 ]\n",
        "train_x = train_dataframe[dta_ids].to_numpy()\n",
        "valid_x = valid_dataframe[dta_ids].to_numpy()\n",
        "\n",
        "# add 'location' to sequence data\n",
        "loc = np.linspace(start=-2.5, stop=+2.5, num=train_x.shape[1])\n",
        "train_x = np.stack([ train_x, np.array([loc]*train_x.shape[0]) ], axis=-1)\n",
        "valid_x = np.stack([ valid_x, np.array([loc]*valid_x.shape[0]) ], axis=-1)\n",
        "\n",
        "# extract labels\n",
        "train_y = train_dataframe['LBL0'].to_numpy()\n",
        "valid_y = valid_dataframe['LBL0'].to_numpy()\n",
        "\n",
        "# package data into tensorflow dataset\n",
        "train_data = tf.data.Dataset.from_tensor_slices((train_x, train_y)).batch(32)\n",
        "valid_data = tf.data.Dataset.from_tensor_slices((valid_x, valid_y)).batch(32)\n",
        "\n",
        "# build model using functional api\n",
        "repdim = 4 # set internal data representation dimensionality\n",
        "\n",
        "# input and linear projection\n",
        "inlayer = tf.keras.Input(shape=(256, 2))\n",
        "proj = tf.keras.layers.Dense(units=repdim)(inlayer)\n",
        "\n",
        "# multi-head attention block\n",
        "mha1 = tf.keras.layers.MultiHeadAttention(num_heads=2, key_dim=repdim)(proj, proj, proj)\n",
        "res1 = tf.keras.layers.Add()([proj, mha1])\n",
        "nrm1 = tf.keras.layers.LayerNormalization()(res1)\n",
        "\n",
        "# feed-forward block\n",
        "ffa1 = tf.keras.layers.Dense(units=repdim, activation=tf.keras.activations.relu)(nrm1)\n",
        "ffb1 = tf.keras.layers.Dense(units=repdim)(ffa1)\n",
        "res2 = tf.keras.layers.Add()([nrm1, ffb1])\n",
        "nrm2 = tf.keras.layers.LayerNormalization()(res2)\n",
        "\n",
        "# classification block\n",
        "flt = tf.keras.layers.Flatten()(nrm2)\n",
        "outlayer = tf.keras.layers.Dense(units=1, activation=tf.keras.activations.sigmoid)(flt)\n",
        "\n",
        "model = tf.keras.Model(inputs=inlayer, outputs=outlayer)\n",
        "model.summary()\n",
        "\n",
        "# compile model\n",
        "model.compile(\n",
        "    optimizer=tf.keras.optimizers.Adam(),\n",
        "    loss=tf.keras.losses.BinaryCrossentropy(),\n",
        "    metrics=['accuracy', tf.keras.metrics.Precision(), tf.keras.metrics.Recall(), tf.keras.metrics.AUC()]\n",
        ")\n",
        "\n",
        "# fit model\n",
        "model.fit(train_data, epochs=50, validation_data=valid_data)\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zGkO8eQmVOGP"
      },
      "source": [
        "## Increased Complexity"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4OnwpiaoVOGQ",
        "outputId": "48008e5a-d9e5-4515-c97b-95a8ecd92e08"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/70\n",
            "409/409 [==============================] - 26s 27ms/step - loss: 0.8464 - accuracy: 0.6025 - precision: 0.5976 - recall: 0.5927 - auc: 0.6362 - val_loss: 0.7359 - val_accuracy: 0.6152 - val_precision: 0.8967 - val_recall: 0.2803 - val_auc: 0.8429\n",
            "Epoch 2/70\n",
            "409/409 [==============================] - 11s 27ms/step - loss: 0.6126 - accuracy: 0.7276 - precision: 0.7251 - recall: 0.7206 - auc: 0.8040 - val_loss: 0.6336 - val_accuracy: 0.6947 - val_precision: 0.9061 - val_recall: 0.4501 - val_auc: 0.8758\n",
            "Epoch 3/70\n",
            "409/409 [==============================] - 10s 23ms/step - loss: 0.5410 - accuracy: 0.7635 - precision: 0.7612 - recall: 0.7581 - auc: 0.8469 - val_loss: 0.6368 - val_accuracy: 0.7008 - val_precision: 0.9202 - val_recall: 0.4549 - val_auc: 0.8815\n",
            "Epoch 4/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.5100 - accuracy: 0.7759 - precision: 0.7735 - recall: 0.7711 - auc: 0.8615 - val_loss: 0.5196 - val_accuracy: 0.7611 - val_precision: 0.9145 - val_recall: 0.5882 - val_auc: 0.8995\n",
            "Epoch 5/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.4989 - accuracy: 0.7817 - precision: 0.7794 - recall: 0.7772 - auc: 0.8640 - val_loss: 0.4272 - val_accuracy: 0.8311 - val_precision: 0.8358 - val_recall: 0.8338 - val_auc: 0.9136\n",
            "Epoch 6/70\n",
            "409/409 [==============================] - 12s 30ms/step - loss: 0.4828 - accuracy: 0.7907 - precision: 0.7883 - recall: 0.7865 - auc: 0.8709 - val_loss: 0.4237 - val_accuracy: 0.8308 - val_precision: 0.8111 - val_recall: 0.8727 - val_auc: 0.9118\n",
            "Epoch 7/70\n",
            "409/409 [==============================] - 10s 24ms/step - loss: 0.4637 - accuracy: 0.7952 - precision: 0.7936 - recall: 0.7899 - auc: 0.8796 - val_loss: 0.4085 - val_accuracy: 0.8293 - val_precision: 0.8177 - val_recall: 0.8577 - val_auc: 0.9142\n",
            "Epoch 8/70\n",
            "409/409 [==============================] - 11s 26ms/step - loss: 0.4595 - accuracy: 0.7982 - precision: 0.7970 - recall: 0.7926 - auc: 0.8802 - val_loss: 0.4069 - val_accuracy: 0.8305 - val_precision: 0.8410 - val_recall: 0.8249 - val_auc: 0.9155\n",
            "Epoch 9/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.4503 - accuracy: 0.8008 - precision: 0.7983 - recall: 0.7975 - auc: 0.8843 - val_loss: 0.4091 - val_accuracy: 0.8342 - val_precision: 0.8637 - val_recall: 0.8027 - val_auc: 0.9158\n",
            "Epoch 10/70\n",
            "409/409 [==============================] - 11s 26ms/step - loss: 0.4417 - accuracy: 0.8011 - precision: 0.7976 - recall: 0.7994 - auc: 0.8879 - val_loss: 0.4207 - val_accuracy: 0.8192 - val_precision: 0.8909 - val_recall: 0.7370 - val_auc: 0.9187\n",
            "Epoch 11/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.4379 - accuracy: 0.8073 - precision: 0.8048 - recall: 0.8042 - auc: 0.8893 - val_loss: 0.3986 - val_accuracy: 0.8348 - val_precision: 0.7996 - val_recall: 0.9038 - val_auc: 0.9195\n",
            "Epoch 12/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.4339 - accuracy: 0.8059 - precision: 0.8027 - recall: 0.8037 - auc: 0.8911 - val_loss: 0.3909 - val_accuracy: 0.8345 - val_precision: 0.8155 - val_recall: 0.8745 - val_auc: 0.9196\n",
            "Epoch 13/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.4308 - accuracy: 0.8090 - precision: 0.8092 - recall: 0.8016 - auc: 0.8921 - val_loss: 0.3915 - val_accuracy: 0.8363 - val_precision: 0.8154 - val_recall: 0.8793 - val_auc: 0.9198\n",
            "Epoch 14/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.4272 - accuracy: 0.8126 - precision: 0.8105 - recall: 0.8090 - auc: 0.8937 - val_loss: 0.3910 - val_accuracy: 0.8363 - val_precision: 0.8023 - val_recall: 0.9026 - val_auc: 0.9210\n",
            "Epoch 15/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.4265 - accuracy: 0.8103 - precision: 0.8088 - recall: 0.8056 - auc: 0.8936 - val_loss: 0.3902 - val_accuracy: 0.8403 - val_precision: 0.8092 - val_recall: 0.9002 - val_auc: 0.9209\n",
            "Epoch 16/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.4189 - accuracy: 0.8100 - precision: 0.8066 - recall: 0.8084 - auc: 0.8974 - val_loss: 0.3865 - val_accuracy: 0.8397 - val_precision: 0.8311 - val_recall: 0.8619 - val_auc: 0.9205\n",
            "Epoch 17/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.4213 - accuracy: 0.8146 - precision: 0.8123 - recall: 0.8113 - auc: 0.8961 - val_loss: 0.3819 - val_accuracy: 0.8437 - val_precision: 0.8253 - val_recall: 0.8811 - val_auc: 0.9223\n",
            "Epoch 18/70\n",
            "409/409 [==============================] - 10s 24ms/step - loss: 0.4160 - accuracy: 0.8150 - precision: 0.8139 - recall: 0.8098 - auc: 0.8987 - val_loss: 0.3932 - val_accuracy: 0.8284 - val_precision: 0.7843 - val_recall: 0.9169 - val_auc: 0.9231\n",
            "Epoch 19/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.4131 - accuracy: 0.8193 - precision: 0.8165 - recall: 0.8169 - auc: 0.8999 - val_loss: 0.3797 - val_accuracy: 0.8385 - val_precision: 0.8207 - val_recall: 0.8757 - val_auc: 0.9226\n",
            "Epoch 20/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.4108 - accuracy: 0.8187 - precision: 0.8161 - recall: 0.8160 - auc: 0.9012 - val_loss: 0.3817 - val_accuracy: 0.8366 - val_precision: 0.8050 - val_recall: 0.8984 - val_auc: 0.9236\n",
            "Epoch 21/70\n",
            "409/409 [==============================] - 11s 26ms/step - loss: 0.4087 - accuracy: 0.8181 - precision: 0.8166 - recall: 0.8138 - auc: 0.9021 - val_loss: 0.3746 - val_accuracy: 0.8455 - val_precision: 0.8439 - val_recall: 0.8565 - val_auc: 0.9238\n",
            "Epoch 22/70\n",
            "409/409 [==============================] - 10s 24ms/step - loss: 0.4048 - accuracy: 0.8198 - precision: 0.8171 - recall: 0.8174 - auc: 0.9039 - val_loss: 0.3775 - val_accuracy: 0.8409 - val_precision: 0.8197 - val_recall: 0.8834 - val_auc: 0.9230\n",
            "Epoch 23/70\n",
            "409/409 [==============================] - 10s 26ms/step - loss: 0.4049 - accuracy: 0.8243 - precision: 0.8217 - recall: 0.8219 - auc: 0.9039 - val_loss: 0.3902 - val_accuracy: 0.8314 - val_precision: 0.7898 - val_recall: 0.9139 - val_auc: 0.9237\n",
            "Epoch 24/70\n",
            "409/409 [==============================] - 10s 26ms/step - loss: 0.4018 - accuracy: 0.8216 - precision: 0.8197 - recall: 0.8182 - auc: 0.9052 - val_loss: 0.3831 - val_accuracy: 0.8318 - val_precision: 0.7926 - val_recall: 0.9091 - val_auc: 0.9252\n",
            "Epoch 25/70\n",
            "409/409 [==============================] - 11s 28ms/step - loss: 0.4021 - accuracy: 0.8250 - precision: 0.8237 - recall: 0.8206 - auc: 0.9051 - val_loss: 0.3723 - val_accuracy: 0.8443 - val_precision: 0.8198 - val_recall: 0.8918 - val_auc: 0.9249\n",
            "Epoch 26/70\n",
            "409/409 [==============================] - 10s 24ms/step - loss: 0.3964 - accuracy: 0.8265 - precision: 0.8257 - recall: 0.8216 - auc: 0.9078 - val_loss: 0.3773 - val_accuracy: 0.8357 - val_precision: 0.8024 - val_recall: 0.9008 - val_auc: 0.9255\n",
            "Epoch 27/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3997 - accuracy: 0.8249 - precision: 0.8225 - recall: 0.8220 - auc: 0.9059 - val_loss: 0.3817 - val_accuracy: 0.8311 - val_precision: 0.7903 - val_recall: 0.9121 - val_auc: 0.9261\n",
            "Epoch 28/70\n",
            "409/409 [==============================] - 11s 26ms/step - loss: 0.3975 - accuracy: 0.8244 - precision: 0.8220 - recall: 0.8217 - auc: 0.9072 - val_loss: 0.3685 - val_accuracy: 0.8449 - val_precision: 0.8283 - val_recall: 0.8793 - val_auc: 0.9260\n",
            "Epoch 29/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3921 - accuracy: 0.8321 - precision: 0.8295 - recall: 0.8301 - auc: 0.9096 - val_loss: 0.3693 - val_accuracy: 0.8449 - val_precision: 0.8450 - val_recall: 0.8536 - val_auc: 0.9232\n",
            "Epoch 30/70\n",
            "409/409 [==============================] - 11s 27ms/step - loss: 0.3941 - accuracy: 0.8254 - precision: 0.8224 - recall: 0.8236 - auc: 0.9087 - val_loss: 0.3734 - val_accuracy: 0.8425 - val_precision: 0.8188 - val_recall: 0.8888 - val_auc: 0.9251\n",
            "Epoch 31/70\n",
            "409/409 [==============================] - 11s 28ms/step - loss: 0.3912 - accuracy: 0.8259 - precision: 0.8239 - recall: 0.8227 - auc: 0.9100 - val_loss: 0.3715 - val_accuracy: 0.8415 - val_precision: 0.8113 - val_recall: 0.8996 - val_auc: 0.9265\n",
            "Epoch 32/70\n",
            "409/409 [==============================] - 10s 24ms/step - loss: 0.3878 - accuracy: 0.8286 - precision: 0.8263 - recall: 0.8259 - auc: 0.9116 - val_loss: 0.3651 - val_accuracy: 0.8486 - val_precision: 0.8328 - val_recall: 0.8811 - val_auc: 0.9248\n",
            "Epoch 33/70\n",
            "409/409 [==============================] - 11s 26ms/step - loss: 0.3917 - accuracy: 0.8253 - precision: 0.8225 - recall: 0.8233 - auc: 0.9097 - val_loss: 0.3687 - val_accuracy: 0.8443 - val_precision: 0.8223 - val_recall: 0.8876 - val_auc: 0.9253\n",
            "Epoch 34/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3893 - accuracy: 0.8294 - precision: 0.8264 - recall: 0.8279 - auc: 0.9108 - val_loss: 0.3643 - val_accuracy: 0.8495 - val_precision: 0.8467 - val_recall: 0.8619 - val_auc: 0.9254\n",
            "Epoch 35/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3886 - accuracy: 0.8314 - precision: 0.8285 - recall: 0.8296 - auc: 0.9110 - val_loss: 0.3682 - val_accuracy: 0.8458 - val_precision: 0.8217 - val_recall: 0.8924 - val_auc: 0.9253\n",
            "Epoch 36/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3849 - accuracy: 0.8324 - precision: 0.8296 - recall: 0.8304 - auc: 0.9131 - val_loss: 0.3647 - val_accuracy: 0.8437 - val_precision: 0.8242 - val_recall: 0.8828 - val_auc: 0.9254\n",
            "Epoch 37/70\n",
            "409/409 [==============================] - 10s 24ms/step - loss: 0.3899 - accuracy: 0.8276 - precision: 0.8243 - recall: 0.8264 - auc: 0.9103 - val_loss: 0.3737 - val_accuracy: 0.8376 - val_precision: 0.7999 - val_recall: 0.9103 - val_auc: 0.9262\n",
            "Epoch 38/70\n",
            "409/409 [==============================] - 10s 24ms/step - loss: 0.3840 - accuracy: 0.8326 - precision: 0.8292 - recall: 0.8317 - auc: 0.9134 - val_loss: 0.3675 - val_accuracy: 0.8431 - val_precision: 0.8201 - val_recall: 0.8882 - val_auc: 0.9246\n",
            "Epoch 39/70\n",
            "409/409 [==============================] - 10s 24ms/step - loss: 0.3833 - accuracy: 0.8298 - precision: 0.8272 - recall: 0.8276 - auc: 0.9134 - val_loss: 0.3685 - val_accuracy: 0.8400 - val_precision: 0.8142 - val_recall: 0.8906 - val_auc: 0.9249\n",
            "Epoch 40/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3843 - accuracy: 0.8307 - precision: 0.8282 - recall: 0.8282 - auc: 0.9130 - val_loss: 0.3733 - val_accuracy: 0.8391 - val_precision: 0.8039 - val_recall: 0.9068 - val_auc: 0.9253\n",
            "Epoch 41/70\n",
            "409/409 [==============================] - 11s 27ms/step - loss: 0.3795 - accuracy: 0.8343 - precision: 0.8312 - recall: 0.8330 - auc: 0.9153 - val_loss: 0.3720 - val_accuracy: 0.8388 - val_precision: 0.8022 - val_recall: 0.9091 - val_auc: 0.9267\n",
            "Epoch 42/70\n",
            "409/409 [==============================] - 9s 23ms/step - loss: 0.3787 - accuracy: 0.8360 - precision: 0.8330 - recall: 0.8346 - auc: 0.9157 - val_loss: 0.3604 - val_accuracy: 0.8477 - val_precision: 0.8291 - val_recall: 0.8846 - val_auc: 0.9266\n",
            "Epoch 43/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3840 - accuracy: 0.8294 - precision: 0.8284 - recall: 0.8248 - auc: 0.9131 - val_loss: 0.3887 - val_accuracy: 0.8284 - val_precision: 0.7780 - val_recall: 0.9301 - val_auc: 0.9259\n",
            "Epoch 44/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3805 - accuracy: 0.8326 - precision: 0.8305 - recall: 0.8296 - auc: 0.9145 - val_loss: 0.3675 - val_accuracy: 0.8434 - val_precision: 0.8140 - val_recall: 0.8996 - val_auc: 0.9262\n",
            "Epoch 45/70\n",
            "409/409 [==============================] - 11s 28ms/step - loss: 0.3764 - accuracy: 0.8373 - precision: 0.8340 - recall: 0.8365 - auc: 0.9164 - val_loss: 0.3815 - val_accuracy: 0.8318 - val_precision: 0.7929 - val_recall: 0.9085 - val_auc: 0.9233\n",
            "Epoch 46/70\n",
            "409/409 [==============================] - 10s 24ms/step - loss: 0.3764 - accuracy: 0.8350 - precision: 0.8317 - recall: 0.8340 - auc: 0.9165 - val_loss: 0.3604 - val_accuracy: 0.8458 - val_precision: 0.8365 - val_recall: 0.8685 - val_auc: 0.9267\n",
            "Epoch 47/70\n",
            "409/409 [==============================] - 10s 24ms/step - loss: 0.3736 - accuracy: 0.8398 - precision: 0.8358 - recall: 0.8402 - auc: 0.9178 - val_loss: 0.3721 - val_accuracy: 0.8376 - val_precision: 0.8015 - val_recall: 0.9074 - val_auc: 0.9255\n",
            "Epoch 48/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3734 - accuracy: 0.8355 - precision: 0.8316 - recall: 0.8354 - auc: 0.9178 - val_loss: 0.3711 - val_accuracy: 0.8458 - val_precision: 0.8203 - val_recall: 0.8948 - val_auc: 0.9238\n",
            "Epoch 49/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3742 - accuracy: 0.8321 - precision: 0.8300 - recall: 0.8293 - auc: 0.9172 - val_loss: 0.3771 - val_accuracy: 0.8354 - val_precision: 0.7948 - val_recall: 0.9145 - val_auc: 0.9251\n",
            "Epoch 50/70\n",
            "409/409 [==============================] - 10s 24ms/step - loss: 0.3722 - accuracy: 0.8376 - precision: 0.8341 - recall: 0.8369 - auc: 0.9183 - val_loss: 0.3665 - val_accuracy: 0.8394 - val_precision: 0.8089 - val_recall: 0.8984 - val_auc: 0.9260\n",
            "Epoch 51/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3724 - accuracy: 0.8394 - precision: 0.8374 - recall: 0.8366 - auc: 0.9182 - val_loss: 0.3748 - val_accuracy: 0.8363 - val_precision: 0.8007 - val_recall: 0.9056 - val_auc: 0.9243\n",
            "Epoch 52/70\n",
            "409/409 [==============================] - 10s 25ms/step - loss: 0.3736 - accuracy: 0.8362 - precision: 0.8337 - recall: 0.8340 - auc: 0.9175 - val_loss: 0.3823 - val_accuracy: 0.8324 - val_precision: 0.7868 - val_recall: 0.9223 - val_auc: 0.9258\n",
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                Output Shape                 Param #   Connected to                  \n",
            "==================================================================================================\n",
            " input_1 (InputLayer)        [(None, 256, 2)]             0         []                            \n",
            "                                                                                                  \n",
            " dense (Dense)               (None, 256, 32)              96        ['input_1[0][0]']             \n",
            "                                                                                                  \n",
            " multi_head_attention (Mult  (None, 256, 32)              58720     ['dense[0][0]',               \n",
            " iHeadAttention)                                                     'dense[0][0]',               \n",
            "                                                                     'dense[0][0]']               \n",
            "                                                                                                  \n",
            " add (Add)                   (None, 256, 32)              0         ['dense[0][0]',               \n",
            "                                                                     'multi_head_attention[0][0]']\n",
            "                                                                                                  \n",
            " layer_normalization (Layer  (None, 256, 32)              64        ['add[0][0]']                 \n",
            " Normalization)                                                                                   \n",
            "                                                                                                  \n",
            " multi_head_attention_1 (Mu  (None, 256, 32)              58720     ['layer_normalization[0][0]', \n",
            " ltiHeadAttention)                                                   'layer_normalization[0][0]', \n",
            "                                                                     'layer_normalization[0][0]'] \n",
            "                                                                                                  \n",
            " add_1 (Add)                 (None, 256, 32)              0         ['layer_normalization[0][0]', \n",
            "                                                                     'multi_head_attention_1[0][0]\n",
            "                                                                    ']                            \n",
            "                                                                                                  \n",
            " layer_normalization_1 (Lay  (None, 256, 32)              64        ['add_1[0][0]']               \n",
            " erNormalization)                                                                                 \n",
            "                                                                                                  \n",
            " dense_1 (Dense)             (None, 256, 128)             4224      ['layer_normalization_1[0][0]'\n",
            "                                                                    ]                             \n",
            "                                                                                                  \n",
            " dense_2 (Dense)             (None, 256, 32)              4128      ['dense_1[0][0]']             \n",
            "                                                                                                  \n",
            " add_2 (Add)                 (None, 256, 32)              0         ['layer_normalization_1[0][0]'\n",
            "                                                                    , 'dense_2[0][0]']            \n",
            "                                                                                                  \n",
            " layer_normalization_2 (Lay  (None, 256, 32)              64        ['add_2[0][0]']               \n",
            " erNormalization)                                                                                 \n",
            "                                                                                                  \n",
            " dense_3 (Dense)             (None, 256, 128)             4224      ['layer_normalization_2[0][0]'\n",
            "                                                                    ]                             \n",
            "                                                                                                  \n",
            " dense_4 (Dense)             (None, 256, 32)              4128      ['dense_3[0][0]']             \n",
            "                                                                                                  \n",
            " add_3 (Add)                 (None, 256, 32)              0         ['layer_normalization_2[0][0]'\n",
            "                                                                    , 'dense_4[0][0]']            \n",
            "                                                                                                  \n",
            " layer_normalization_3 (Lay  (None, 256, 32)              64        ['add_3[0][0]']               \n",
            " erNormalization)                                                                                 \n",
            "                                                                                                  \n",
            " dropout (Dropout)           (None, 256, 32)              0         ['layer_normalization_3[0][0]'\n",
            "                                                                    ]                             \n",
            "                                                                                                  \n",
            " flatten (Flatten)           (None, 8192)                 0         ['dropout[0][0]']             \n",
            "                                                                                                  \n",
            " dense_5 (Dense)             (None, 1)                    8193      ['flatten[0][0]']             \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 142689 (557.38 KB)\n",
            "Trainable params: 142689 (557.38 KB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from sklearn.metrics import precision_score, recall_score, roc_auc_score\n",
        "from tensorflow.keras import regularizers\n",
        "tf.keras.optimizers.legacy.Adam\n",
        "\n",
        "# download data\n",
        "dataframe = pd.read_csv('https://raw.githubusercontent.com/bryankolaczkowski/ALS3200C/main/mbiome.data.csv')\n",
        "\n",
        "# create train-validate split\n",
        "train_dataframe = dataframe.sample(frac=0.8, random_state=827847)\n",
        "valid_dataframe = dataframe.drop(train_dataframe.index)\n",
        "\n",
        "# extract explanatory variables\n",
        "dta_ids = [ x for x in dataframe.columns if x.find('DTA') == 0 ]\n",
        "train_x = train_dataframe[dta_ids].to_numpy()\n",
        "valid_x = valid_dataframe[dta_ids].to_numpy()\n",
        "\n",
        "# add 'location' to sequence data\n",
        "loc = np.linspace(start=-2.5, stop=+2.5, num=train_x.shape[1])\n",
        "train_x = np.stack([ train_x, np.array([loc]*train_x.shape[0]) ], axis=-1)\n",
        "valid_x = np.stack([ valid_x, np.array([loc]*valid_x.shape[0]) ], axis=-1)\n",
        "\n",
        "# extract labels\n",
        "train_y = train_dataframe['LBL0'].to_numpy()\n",
        "valid_y = valid_dataframe['LBL0'].to_numpy()\n",
        "\n",
        "# package data into tensorflow dataset\n",
        "train_data = tf.data.Dataset.from_tensor_slices((train_x, train_y)).batch(32)\n",
        "valid_data = tf.data.Dataset.from_tensor_slices((valid_x, valid_y)).batch(32)\n",
        "\n",
        "# Build and compile the model\n",
        "repdim = 32  # Internal data representation dimensionality\n",
        "inlayer = tf.keras.Input(shape=(256, 2))\n",
        "proj = tf.keras.layers.Dense(units=repdim, kernel_regularizer=regularizers.l2(0.001))(inlayer)\n",
        "mha1 = tf.keras.layers.MultiHeadAttention(num_heads=14, key_dim=repdim)(proj, proj, proj)\n",
        "res1 = tf.keras.layers.Add()([proj, mha1])\n",
        "nrm1 = tf.keras.layers.LayerNormalization()(res1)\n",
        "mha2 = tf.keras.layers.MultiHeadAttention(num_heads=14, key_dim=repdim)(nrm1, nrm1, nrm1)\n",
        "res2 = tf.keras.layers.Add()([nrm1, mha2])\n",
        "nrm2 = tf.keras.layers.LayerNormalization()(res2)\n",
        "ffa1 = tf.keras.layers.Dense(units=repdim*4, activation='relu')(nrm2)\n",
        "ffb1 = tf.keras.layers.Dense(units=repdim, kernel_regularizer=regularizers.l2(0.001))(ffa1)\n",
        "res3 = tf.keras.layers.Add()([nrm2, ffb1])\n",
        "nrm3 = tf.keras.layers.LayerNormalization()(res3)\n",
        "ffa2 = tf.keras.layers.Dense(units=repdim*4, activation='relu')(nrm3)\n",
        "ffb2 = tf.keras.layers.Dense(units=repdim, kernel_regularizer=regularizers.l2(0.001))(ffa2)\n",
        "res5 = tf.keras.layers.Add()([nrm3, ffb2])\n",
        "nrm5 = tf.keras.layers.LayerNormalization()(res5)\n",
        "dropout_layer = tf.keras.layers.Dropout(0.4)(nrm5)\n",
        "flt = tf.keras.layers.Flatten()(dropout_layer)\n",
        "outlayer = tf.keras.layers.Dense(units=1, activation='sigmoid')(flt)\n",
        "\n",
        "model = tf.keras.Model(inputs=inlayer, outputs=outlayer)\n",
        "\n",
        "# compile model\n",
        "model.compile(\n",
        "    optimizer=tf.keras.optimizers.legacy.Adam(),\n",
        "    loss=tf.keras.losses.BinaryCrossentropy(),\n",
        "    metrics=['accuracy', tf.keras.metrics.Precision(), tf.keras.metrics.Recall(), tf.keras.metrics.AUC()]\n",
        ")\n",
        "\n",
        "# Early stopping callback\n",
        "early_stopping = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True)\n",
        "\n",
        "model.fit(train_data, epochs=70, validation_data=valid_data, callbacks=[early_stopping])\n",
        "\n",
        "model.summary()\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iwCqM599VOGQ"
      },
      "source": [
        "## Bi-Directional LSTM  + Transformer Block Integration"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "sbwAZf6EVOGQ",
        "outputId": "2b302b98-a428-48d2-e746-6f33e736ada3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/70\n",
            "409/409 [==============================] - 33s 45ms/step - loss: 0.7950 - accuracy: 0.5826 - precision: 0.5777 - recall: 0.5699 - auc: 0.6170 - val_loss: 0.7830 - val_accuracy: 0.6008 - val_precision: 0.9532 - val_recall: 0.2313 - val_auc: 0.8298\n",
            "Epoch 2/70\n",
            "409/409 [==============================] - 16s 39ms/step - loss: 0.5793 - accuracy: 0.7277 - precision: 0.7258 - recall: 0.7195 - auc: 0.8103 - val_loss: 0.5728 - val_accuracy: 0.7250 - val_precision: 0.9368 - val_recall: 0.4961 - val_auc: 0.8890\n",
            "Epoch 3/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.5273 - accuracy: 0.7664 - precision: 0.7663 - recall: 0.7570 - auc: 0.8473 - val_loss: 0.4714 - val_accuracy: 0.8033 - val_precision: 0.7702 - val_recall: 0.8775 - val_auc: 0.8962\n",
            "Epoch 4/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.4920 - accuracy: 0.7874 - precision: 0.7852 - recall: 0.7828 - auc: 0.8687 - val_loss: 0.5307 - val_accuracy: 0.7464 - val_precision: 0.6782 - val_recall: 0.9600 - val_auc: 0.8999\n",
            "Epoch 5/70\n",
            "409/409 [==============================] - 16s 39ms/step - loss: 0.4690 - accuracy: 0.7979 - precision: 0.7978 - recall: 0.7902 - auc: 0.8803 - val_loss: 0.4688 - val_accuracy: 0.7917 - val_precision: 0.7355 - val_recall: 0.9259 - val_auc: 0.9049\n",
            "Epoch 6/70\n",
            "409/409 [==============================] - 16s 39ms/step - loss: 0.4526 - accuracy: 0.8087 - precision: 0.8065 - recall: 0.8051 - auc: 0.8890 - val_loss: 0.4184 - val_accuracy: 0.8290 - val_precision: 0.8040 - val_recall: 0.8805 - val_auc: 0.9126\n",
            "Epoch 7/70\n",
            "409/409 [==============================] - 18s 44ms/step - loss: 0.4421 - accuracy: 0.8102 - precision: 0.8080 - recall: 0.8068 - auc: 0.8935 - val_loss: 0.4208 - val_accuracy: 0.8220 - val_precision: 0.7864 - val_recall: 0.8954 - val_auc: 0.9133\n",
            "Epoch 8/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.4347 - accuracy: 0.8141 - precision: 0.8138 - recall: 0.8078 - auc: 0.8961 - val_loss: 0.4254 - val_accuracy: 0.8186 - val_precision: 0.7719 - val_recall: 0.9163 - val_auc: 0.9142\n",
            "Epoch 9/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.4249 - accuracy: 0.8176 - precision: 0.8148 - recall: 0.8152 - auc: 0.9007 - val_loss: 0.4061 - val_accuracy: 0.8308 - val_precision: 0.8001 - val_recall: 0.8924 - val_auc: 0.9148\n",
            "Epoch 10/70\n",
            "409/409 [==============================] - 16s 39ms/step - loss: 0.4240 - accuracy: 0.8155 - precision: 0.8125 - recall: 0.8135 - auc: 0.8997 - val_loss: 0.4230 - val_accuracy: 0.8186 - val_precision: 0.7692 - val_recall: 0.9223 - val_auc: 0.9161\n",
            "Epoch 11/70\n",
            "409/409 [==============================] - 16s 40ms/step - loss: 0.4198 - accuracy: 0.8181 - precision: 0.8141 - recall: 0.8178 - auc: 0.9012 - val_loss: 0.4141 - val_accuracy: 0.8232 - val_precision: 0.7761 - val_recall: 0.9199 - val_auc: 0.9172\n",
            "Epoch 12/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.4145 - accuracy: 0.8223 - precision: 0.8200 - recall: 0.8192 - auc: 0.9032 - val_loss: 0.4125 - val_accuracy: 0.8223 - val_precision: 0.7820 - val_recall: 0.9050 - val_auc: 0.9143\n",
            "Epoch 13/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.4075 - accuracy: 0.8234 - precision: 0.8204 - recall: 0.8216 - auc: 0.9063 - val_loss: 0.4256 - val_accuracy: 0.8137 - val_precision: 0.7598 - val_recall: 0.9301 - val_auc: 0.9161\n",
            "Epoch 14/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.4048 - accuracy: 0.8230 - precision: 0.8221 - recall: 0.8180 - auc: 0.9068 - val_loss: 0.4099 - val_accuracy: 0.8226 - val_precision: 0.7756 - val_recall: 0.9193 - val_auc: 0.9182\n",
            "Epoch 15/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.4029 - accuracy: 0.8272 - precision: 0.8262 - recall: 0.8225 - auc: 0.9075 - val_loss: 0.4016 - val_accuracy: 0.8266 - val_precision: 0.7851 - val_recall: 0.9103 - val_auc: 0.9183\n",
            "Epoch 16/70\n",
            "409/409 [==============================] - 16s 40ms/step - loss: 0.3971 - accuracy: 0.8269 - precision: 0.8259 - recall: 0.8222 - auc: 0.9097 - val_loss: 0.4028 - val_accuracy: 0.8290 - val_precision: 0.7816 - val_recall: 0.9241 - val_auc: 0.9208\n",
            "Epoch 17/70\n",
            "409/409 [==============================] - 15s 38ms/step - loss: 0.3957 - accuracy: 0.8293 - precision: 0.8275 - recall: 0.8259 - auc: 0.9099 - val_loss: 0.3929 - val_accuracy: 0.8308 - val_precision: 0.7911 - val_recall: 0.9097 - val_auc: 0.9216\n",
            "Epoch 18/70\n",
            "409/409 [==============================] - 15s 38ms/step - loss: 0.3941 - accuracy: 0.8285 - precision: 0.8268 - recall: 0.8250 - auc: 0.9106 - val_loss: 0.4017 - val_accuracy: 0.8256 - val_precision: 0.7804 - val_recall: 0.9175 - val_auc: 0.9184\n",
            "Epoch 19/70\n",
            "409/409 [==============================] - 16s 40ms/step - loss: 0.3918 - accuracy: 0.8270 - precision: 0.8253 - recall: 0.8233 - auc: 0.9112 - val_loss: 0.4079 - val_accuracy: 0.8168 - val_precision: 0.7638 - val_recall: 0.9295 - val_auc: 0.9215\n",
            "Epoch 20/70\n",
            "409/409 [==============================] - 15s 38ms/step - loss: 0.3888 - accuracy: 0.8297 - precision: 0.8267 - recall: 0.8281 - auc: 0.9125 - val_loss: 0.3895 - val_accuracy: 0.8336 - val_precision: 0.7870 - val_recall: 0.9253 - val_auc: 0.9246\n",
            "Epoch 21/70\n",
            "409/409 [==============================] - 15s 38ms/step - loss: 0.3847 - accuracy: 0.8299 - precision: 0.8299 - recall: 0.8237 - auc: 0.9141 - val_loss: 0.3874 - val_accuracy: 0.8327 - val_precision: 0.7878 - val_recall: 0.9211 - val_auc: 0.9246\n",
            "Epoch 22/70\n",
            "409/409 [==============================] - 17s 40ms/step - loss: 0.3829 - accuracy: 0.8313 - precision: 0.8302 - recall: 0.8268 - auc: 0.9148 - val_loss: 0.3883 - val_accuracy: 0.8281 - val_precision: 0.7847 - val_recall: 0.9151 - val_auc: 0.9230\n",
            "Epoch 23/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3818 - accuracy: 0.8348 - precision: 0.8315 - recall: 0.8338 - auc: 0.9153 - val_loss: 0.3917 - val_accuracy: 0.8275 - val_precision: 0.7813 - val_recall: 0.9205 - val_auc: 0.9230\n",
            "Epoch 24/70\n",
            "409/409 [==============================] - 17s 40ms/step - loss: 0.3796 - accuracy: 0.8349 - precision: 0.8333 - recall: 0.8313 - auc: 0.9159 - val_loss: 0.3944 - val_accuracy: 0.8226 - val_precision: 0.7715 - val_recall: 0.9283 - val_auc: 0.9243\n",
            "Epoch 25/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3772 - accuracy: 0.8349 - precision: 0.8340 - recall: 0.8303 - auc: 0.9169 - val_loss: 0.3792 - val_accuracy: 0.8342 - val_precision: 0.7997 - val_recall: 0.9020 - val_auc: 0.9227\n",
            "Epoch 26/70\n",
            "409/409 [==============================] - 16s 39ms/step - loss: 0.3768 - accuracy: 0.8343 - precision: 0.8316 - recall: 0.8323 - auc: 0.9169 - val_loss: 0.3871 - val_accuracy: 0.8293 - val_precision: 0.7809 - val_recall: 0.9265 - val_auc: 0.9260\n",
            "Epoch 27/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3749 - accuracy: 0.8337 - precision: 0.8325 - recall: 0.8296 - auc: 0.9176 - val_loss: 0.3847 - val_accuracy: 0.8296 - val_precision: 0.7830 - val_recall: 0.9229 - val_auc: 0.9259\n",
            "Epoch 28/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3744 - accuracy: 0.8352 - precision: 0.8319 - recall: 0.8341 - auc: 0.9178 - val_loss: 0.3830 - val_accuracy: 0.8299 - val_precision: 0.7788 - val_recall: 0.9325 - val_auc: 0.9277\n",
            "Epoch 29/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3738 - accuracy: 0.8365 - precision: 0.8360 - recall: 0.8313 - auc: 0.9180 - val_loss: 0.3825 - val_accuracy: 0.8324 - val_precision: 0.7854 - val_recall: 0.9253 - val_auc: 0.9262\n",
            "Epoch 30/70\n",
            "409/409 [==============================] - 16s 40ms/step - loss: 0.3753 - accuracy: 0.8359 - precision: 0.8340 - recall: 0.8327 - auc: 0.9169 - val_loss: 0.3992 - val_accuracy: 0.8201 - val_precision: 0.7630 - val_recall: 0.9408 - val_auc: 0.9276\n",
            "Epoch 31/70\n",
            "409/409 [==============================] - 16s 40ms/step - loss: 0.3696 - accuracy: 0.8395 - precision: 0.8375 - recall: 0.8366 - auc: 0.9195 - val_loss: 0.3804 - val_accuracy: 0.8308 - val_precision: 0.7854 - val_recall: 0.9211 - val_auc: 0.9266\n",
            "Epoch 32/70\n",
            "409/409 [==============================] - 16s 39ms/step - loss: 0.3700 - accuracy: 0.8393 - precision: 0.8375 - recall: 0.8363 - auc: 0.9193 - val_loss: 0.3805 - val_accuracy: 0.8308 - val_precision: 0.7831 - val_recall: 0.9259 - val_auc: 0.9274\n",
            "Epoch 33/70\n",
            "409/409 [==============================] - 15s 38ms/step - loss: 0.3689 - accuracy: 0.8341 - precision: 0.8325 - recall: 0.8306 - auc: 0.9196 - val_loss: 0.3847 - val_accuracy: 0.8284 - val_precision: 0.7755 - val_recall: 0.9354 - val_auc: 0.9281\n",
            "Epoch 34/70\n",
            "409/409 [==============================] - 16s 39ms/step - loss: 0.3686 - accuracy: 0.8385 - precision: 0.8364 - recall: 0.8358 - auc: 0.9196 - val_loss: 0.3762 - val_accuracy: 0.8314 - val_precision: 0.7856 - val_recall: 0.9223 - val_auc: 0.9280\n",
            "Epoch 35/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3665 - accuracy: 0.8380 - precision: 0.8349 - recall: 0.8369 - auc: 0.9204 - val_loss: 0.3819 - val_accuracy: 0.8299 - val_precision: 0.7842 - val_recall: 0.9211 - val_auc: 0.9264\n",
            "Epoch 36/70\n",
            "409/409 [==============================] - 15s 38ms/step - loss: 0.3663 - accuracy: 0.8390 - precision: 0.8352 - recall: 0.8389 - auc: 0.9206 - val_loss: 0.3791 - val_accuracy: 0.8314 - val_precision: 0.7856 - val_recall: 0.9223 - val_auc: 0.9267\n",
            "Epoch 37/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3661 - accuracy: 0.8424 - precision: 0.8401 - recall: 0.8402 - auc: 0.9207 - val_loss: 0.3733 - val_accuracy: 0.8321 - val_precision: 0.7879 - val_recall: 0.9193 - val_auc: 0.9288\n",
            "Epoch 38/70\n",
            "409/409 [==============================] - 15s 38ms/step - loss: 0.3638 - accuracy: 0.8378 - precision: 0.8341 - recall: 0.8375 - auc: 0.9216 - val_loss: 0.3733 - val_accuracy: 0.8336 - val_precision: 0.7899 - val_recall: 0.9193 - val_auc: 0.9273\n",
            "Epoch 39/70\n",
            "409/409 [==============================] - 17s 40ms/step - loss: 0.3631 - accuracy: 0.8401 - precision: 0.8384 - recall: 0.8369 - auc: 0.9218 - val_loss: 0.3720 - val_accuracy: 0.8360 - val_precision: 0.7944 - val_recall: 0.9169 - val_auc: 0.9273\n",
            "Epoch 40/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3622 - accuracy: 0.8402 - precision: 0.8361 - recall: 0.8407 - auc: 0.9220 - val_loss: 0.3723 - val_accuracy: 0.8348 - val_precision: 0.7934 - val_recall: 0.9157 - val_auc: 0.9273\n",
            "Epoch 41/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3628 - accuracy: 0.8421 - precision: 0.8402 - recall: 0.8393 - auc: 0.9218 - val_loss: 0.3883 - val_accuracy: 0.8253 - val_precision: 0.7803 - val_recall: 0.9169 - val_auc: 0.9229\n",
            "Epoch 42/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3602 - accuracy: 0.8393 - precision: 0.8361 - recall: 0.8383 - auc: 0.9229 - val_loss: 0.3739 - val_accuracy: 0.8330 - val_precision: 0.7912 - val_recall: 0.9151 - val_auc: 0.9271\n",
            "Epoch 43/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3600 - accuracy: 0.8416 - precision: 0.8402 - recall: 0.8380 - auc: 0.9229 - val_loss: 0.3729 - val_accuracy: 0.8327 - val_precision: 0.7867 - val_recall: 0.9235 - val_auc: 0.9292\n",
            "Epoch 44/70\n",
            "409/409 [==============================] - 17s 41ms/step - loss: 0.3599 - accuracy: 0.8434 - precision: 0.8415 - recall: 0.8408 - auc: 0.9229 - val_loss: 0.3696 - val_accuracy: 0.8370 - val_precision: 0.7969 - val_recall: 0.9145 - val_auc: 0.9275\n",
            "Epoch 45/70\n",
            "409/409 [==============================] - 15s 38ms/step - loss: 0.3619 - accuracy: 0.8386 - precision: 0.8359 - recall: 0.8369 - auc: 0.9220 - val_loss: 0.3742 - val_accuracy: 0.8363 - val_precision: 0.7912 - val_recall: 0.9241 - val_auc: 0.9286\n",
            "Epoch 46/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3591 - accuracy: 0.8417 - precision: 0.8393 - recall: 0.8396 - auc: 0.9232 - val_loss: 0.3671 - val_accuracy: 0.8388 - val_precision: 0.8000 - val_recall: 0.9133 - val_auc: 0.9281\n",
            "Epoch 47/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3568 - accuracy: 0.8424 - precision: 0.8394 - recall: 0.8413 - auc: 0.9241 - val_loss: 0.3757 - val_accuracy: 0.8327 - val_precision: 0.7875 - val_recall: 0.9217 - val_auc: 0.9274\n",
            "Epoch 48/70\n",
            "409/409 [==============================] - 16s 39ms/step - loss: 0.3574 - accuracy: 0.8438 - precision: 0.8423 - recall: 0.8405 - auc: 0.9238 - val_loss: 0.3752 - val_accuracy: 0.8318 - val_precision: 0.7875 - val_recall: 0.9193 - val_auc: 0.9277\n",
            "Epoch 49/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3572 - accuracy: 0.8411 - precision: 0.8394 - recall: 0.8379 - auc: 0.9238 - val_loss: 0.3624 - val_accuracy: 0.8403 - val_precision: 0.8047 - val_recall: 0.9085 - val_auc: 0.9285\n",
            "Epoch 50/70\n",
            "409/409 [==============================] - 15s 38ms/step - loss: 0.3546 - accuracy: 0.8441 - precision: 0.8419 - recall: 0.8419 - auc: 0.9251 - val_loss: 0.3605 - val_accuracy: 0.8388 - val_precision: 0.8035 - val_recall: 0.9068 - val_auc: 0.9298\n",
            "Epoch 51/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3562 - accuracy: 0.8431 - precision: 0.8405 - recall: 0.8414 - auc: 0.9244 - val_loss: 0.3707 - val_accuracy: 0.8345 - val_precision: 0.7879 - val_recall: 0.9259 - val_auc: 0.9301\n",
            "Epoch 52/70\n",
            "409/409 [==============================] - 16s 39ms/step - loss: 0.3552 - accuracy: 0.8414 - precision: 0.8377 - recall: 0.8411 - auc: 0.9247 - val_loss: 0.3668 - val_accuracy: 0.8363 - val_precision: 0.7945 - val_recall: 0.9175 - val_auc: 0.9293\n",
            "Epoch 53/70\n",
            "409/409 [==============================] - 15s 38ms/step - loss: 0.3547 - accuracy: 0.8454 - precision: 0.8421 - recall: 0.8448 - auc: 0.9251 - val_loss: 0.3714 - val_accuracy: 0.8348 - val_precision: 0.7886 - val_recall: 0.9253 - val_auc: 0.9294\n",
            "Epoch 54/70\n",
            "409/409 [==============================] - 16s 39ms/step - loss: 0.3562 - accuracy: 0.8453 - precision: 0.8426 - recall: 0.8438 - auc: 0.9243 - val_loss: 0.3795 - val_accuracy: 0.8299 - val_precision: 0.7822 - val_recall: 0.9253 - val_auc: 0.9281\n",
            "Epoch 55/70\n",
            "409/409 [==============================] - 16s 40ms/step - loss: 0.3563 - accuracy: 0.8437 - precision: 0.8425 - recall: 0.8399 - auc: 0.9241 - val_loss: 0.3708 - val_accuracy: 0.8293 - val_precision: 0.7814 - val_recall: 0.9253 - val_auc: 0.9307\n",
            "Epoch 56/70\n",
            "409/409 [==============================] - 15s 37ms/step - loss: 0.3550 - accuracy: 0.8423 - precision: 0.8398 - recall: 0.8403 - auc: 0.9246 - val_loss: 0.3764 - val_accuracy: 0.8314 - val_precision: 0.7859 - val_recall: 0.9217 - val_auc: 0.9277\n",
            "Epoch 57/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3545 - accuracy: 0.8480 - precision: 0.8476 - recall: 0.8433 - auc: 0.9249 - val_loss: 0.3638 - val_accuracy: 0.8357 - val_precision: 0.7946 - val_recall: 0.9157 - val_auc: 0.9300\n",
            "Epoch 58/70\n",
            "409/409 [==============================] - 15s 37ms/step - loss: 0.3529 - accuracy: 0.8429 - precision: 0.8401 - recall: 0.8414 - auc: 0.9255 - val_loss: 0.3638 - val_accuracy: 0.8388 - val_precision: 0.7991 - val_recall: 0.9151 - val_auc: 0.9300\n",
            "Epoch 59/70\n",
            "409/409 [==============================] - 15s 37ms/step - loss: 0.3517 - accuracy: 0.8450 - precision: 0.8427 - recall: 0.8428 - auc: 0.9260 - val_loss: 0.3652 - val_accuracy: 0.8366 - val_precision: 0.7962 - val_recall: 0.9151 - val_auc: 0.9294\n",
            "Epoch 60/70\n",
            "409/409 [==============================] - 15s 37ms/step - loss: 0.3502 - accuracy: 0.8454 - precision: 0.8416 - recall: 0.8456 - auc: 0.9267 - val_loss: 0.3598 - val_accuracy: 0.8382 - val_precision: 0.8017 - val_recall: 0.9085 - val_auc: 0.9296\n",
            "Epoch 61/70\n",
            "409/409 [==============================] - 15s 37ms/step - loss: 0.3531 - accuracy: 0.8412 - precision: 0.8410 - recall: 0.8360 - auc: 0.9254 - val_loss: 0.3695 - val_accuracy: 0.8360 - val_precision: 0.7911 - val_recall: 0.9235 - val_auc: 0.9290\n",
            "Epoch 62/70\n",
            "409/409 [==============================] - 15s 37ms/step - loss: 0.3533 - accuracy: 0.8441 - precision: 0.8403 - recall: 0.8441 - auc: 0.9252 - val_loss: 0.3748 - val_accuracy: 0.8314 - val_precision: 0.7842 - val_recall: 0.9253 - val_auc: 0.9297\n",
            "Epoch 63/70\n",
            "409/409 [==============================] - 15s 38ms/step - loss: 0.3556 - accuracy: 0.8452 - precision: 0.8436 - recall: 0.8420 - auc: 0.9243 - val_loss: 0.3599 - val_accuracy: 0.8382 - val_precision: 0.8001 - val_recall: 0.9115 - val_auc: 0.9307\n",
            "Epoch 64/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3532 - accuracy: 0.8444 - precision: 0.8413 - recall: 0.8433 - auc: 0.9253 - val_loss: 0.3657 - val_accuracy: 0.8351 - val_precision: 0.7950 - val_recall: 0.9133 - val_auc: 0.9289\n",
            "Epoch 65/70\n",
            "409/409 [==============================] - 16s 38ms/step - loss: 0.3498 - accuracy: 0.8451 - precision: 0.8418 - recall: 0.8445 - auc: 0.9268 - val_loss: 0.3661 - val_accuracy: 0.8324 - val_precision: 0.7916 - val_recall: 0.9127 - val_auc: 0.9289\n",
            "Epoch 66/70\n",
            "409/409 [==============================] - 16s 39ms/step - loss: 0.3489 - accuracy: 0.8484 - precision: 0.8468 - recall: 0.8455 - auc: 0.9271 - val_loss: 0.3633 - val_accuracy: 0.8339 - val_precision: 0.7958 - val_recall: 0.9085 - val_auc: 0.9294\n",
            "Epoch 67/70\n",
            "409/409 [==============================] - 15s 37ms/step - loss: 0.3496 - accuracy: 0.8453 - precision: 0.8431 - recall: 0.8430 - auc: 0.9269 - val_loss: 0.3741 - val_accuracy: 0.8336 - val_precision: 0.7852 - val_recall: 0.9289 - val_auc: 0.9305\n",
            "Epoch 68/70\n",
            "409/409 [==============================] - 15s 37ms/step - loss: 0.3501 - accuracy: 0.8486 - precision: 0.8476 - recall: 0.8448 - auc: 0.9266 - val_loss: 0.3725 - val_accuracy: 0.8345 - val_precision: 0.7891 - val_recall: 0.9235 - val_auc: 0.9296\n",
            "Epoch 69/70\n",
            "409/409 [==============================] - 15s 37ms/step - loss: 0.3492 - accuracy: 0.8429 - precision: 0.8410 - recall: 0.8402 - auc: 0.9270 - val_loss: 0.3652 - val_accuracy: 0.8360 - val_precision: 0.7947 - val_recall: 0.9163 - val_auc: 0.9294\n",
            "Epoch 70/70\n",
            "409/409 [==============================] - 16s 39ms/step - loss: 0.3477 - accuracy: 0.8460 - precision: 0.8448 - recall: 0.8422 - auc: 0.9275 - val_loss: 0.3691 - val_accuracy: 0.8342 - val_precision: 0.7896 - val_recall: 0.9217 - val_auc: 0.9298\n",
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                Output Shape                 Param #   Connected to                  \n",
            "==================================================================================================\n",
            " input_1 (InputLayer)        [(None, 256, 2)]             0         []                            \n",
            "                                                                                                  \n",
            " bidirectional (Bidirection  (None, 256, 32)              2432      ['input_1[0][0]']             \n",
            " al)                                                                                              \n",
            "                                                                                                  \n",
            " dense (Dense)               (None, 256, 16)              528       ['bidirectional[0][0]']       \n",
            "                                                                                                  \n",
            " multi_head_attention (Mult  (None, 256, 16)              15024     ['dense[0][0]',               \n",
            " iHeadAttention)                                                     'dense[0][0]',               \n",
            "                                                                     'dense[0][0]']               \n",
            "                                                                                                  \n",
            " add (Add)                   (None, 256, 16)              0         ['dense[0][0]',               \n",
            "                                                                     'multi_head_attention[0][0]']\n",
            "                                                                                                  \n",
            " layer_normalization (Layer  (None, 256, 16)              32        ['add[0][0]']                 \n",
            " Normalization)                                                                                   \n",
            "                                                                                                  \n",
            " multi_head_attention_1 (Mu  (None, 256, 16)              15024     ['layer_normalization[0][0]', \n",
            " ltiHeadAttention)                                                   'layer_normalization[0][0]', \n",
            "                                                                     'layer_normalization[0][0]'] \n",
            "                                                                                                  \n",
            " add_1 (Add)                 (None, 256, 16)              0         ['layer_normalization[0][0]', \n",
            "                                                                     'multi_head_attention_1[0][0]\n",
            "                                                                    ']                            \n",
            "                                                                                                  \n",
            " layer_normalization_1 (Lay  (None, 256, 16)              32        ['add_1[0][0]']               \n",
            " erNormalization)                                                                                 \n",
            "                                                                                                  \n",
            " dense_1 (Dense)             (None, 256, 16)              272       ['layer_normalization_1[0][0]'\n",
            "                                                                    ]                             \n",
            "                                                                                                  \n",
            " dense_2 (Dense)             (None, 256, 16)              272       ['dense_1[0][0]']             \n",
            "                                                                                                  \n",
            " add_2 (Add)                 (None, 256, 16)              0         ['layer_normalization_1[0][0]'\n",
            "                                                                    , 'dense_2[0][0]']            \n",
            "                                                                                                  \n",
            " layer_normalization_2 (Lay  (None, 256, 16)              32        ['add_2[0][0]']               \n",
            " erNormalization)                                                                                 \n",
            "                                                                                                  \n",
            " dense_3 (Dense)             (None, 256, 16)              272       ['layer_normalization_2[0][0]'\n",
            "                                                                    ]                             \n",
            "                                                                                                  \n",
            " dense_4 (Dense)             (None, 256, 16)              272       ['dense_3[0][0]']             \n",
            "                                                                                                  \n",
            " add_3 (Add)                 (None, 256, 16)              0         ['layer_normalization_2[0][0]'\n",
            "                                                                    , 'dense_4[0][0]']            \n",
            "                                                                                                  \n",
            " layer_normalization_3 (Lay  (None, 256, 16)              32        ['add_3[0][0]']               \n",
            " erNormalization)                                                                                 \n",
            "                                                                                                  \n",
            " dropout (Dropout)           (None, 256, 16)              0         ['layer_normalization_3[0][0]'\n",
            "                                                                    ]                             \n",
            "                                                                                                  \n",
            " flatten (Flatten)           (None, 4096)                 0         ['dropout[0][0]']             \n",
            "                                                                                                  \n",
            " dense_5 (Dense)             (None, 1)                    4097      ['flatten[0][0]']             \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 38321 (149.69 KB)\n",
            "Trainable params: 38321 (149.69 KB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "\n",
        "from sklearn.metrics import precision_score, recall_score, roc_auc_score\n",
        "from tensorflow.keras import regularizers\n",
        "tf.keras.optimizers.legacy.Adam\n",
        "\n",
        "\n",
        "# download data\n",
        "dataframe = pd.read_csv('https://raw.githubusercontent.com/bryankolaczkowski/ALS3200C/main/mbiome.data.csv')\n",
        "\n",
        "# create train-validate split\n",
        "train_dataframe = dataframe.sample(frac=0.8, random_state=827847)\n",
        "valid_dataframe = dataframe.drop(train_dataframe.index)\n",
        "\n",
        "# extract explanatory variables\n",
        "dta_ids = [ x for x in dataframe.columns if x.find('DTA') == 0 ]\n",
        "train_x = train_dataframe[dta_ids].to_numpy()\n",
        "valid_x = valid_dataframe[dta_ids].to_numpy()\n",
        "\n",
        "# add 'location' to sequence data\n",
        "loc = np.linspace(start=-2.5, stop=+2.5, num=train_x.shape[1])\n",
        "train_x = np.stack([ train_x, np.array([loc]*train_x.shape[0]) ], axis=-1)\n",
        "valid_x = np.stack([ valid_x, np.array([loc]*valid_x.shape[0]) ], axis=-1)\n",
        "\n",
        "# extract labels\n",
        "train_y = train_dataframe['LBL0'].to_numpy()\n",
        "valid_y = valid_dataframe['LBL0'].to_numpy()\n",
        "\n",
        "# package data into tensorflow dataset\n",
        "train_data = tf.data.Dataset.from_tensor_slices((train_x, train_y)).batch(32)\n",
        "valid_data = tf.data.Dataset.from_tensor_slices((valid_x, valid_y)).batch(32)\n",
        "\n",
        "\n",
        "# Build model using functional API\n",
        "repdim = 16  # Set internal data representation dimensionality\n",
        "\n",
        "inlayer = tf.keras.Input(shape=(256, 2))\n",
        "\n",
        "# LSTM layer\n",
        "\n",
        "# Bidirectional LSTM layer\n",
        "lstm_layer = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(units=16, return_sequences=True))(inlayer)\n",
        "\n",
        "# Dense projection layer\n",
        "proj = tf.keras.layers.Dense(units=repdim, kernel_regularizer=regularizers.l2(0.001))(lstm_layer)\n",
        "\n",
        "# First multi-head attention block\n",
        "mha1 = tf.keras.layers.MultiHeadAttention(num_heads=14, key_dim=repdim)(proj, proj, proj)\n",
        "res1 = tf.keras.layers.Add()([proj, mha1])\n",
        "nrm1 = tf.keras.layers.LayerNormalization()(res1)\n",
        "\n",
        "# Second multi-head attention block\n",
        "mha2 = tf.keras.layers.MultiHeadAttention(num_heads=14, key_dim=repdim)(nrm1, nrm1, nrm1)\n",
        "res2 = tf.keras.layers.Add()([nrm1, mha2])\n",
        "nrm2 = tf.keras.layers.LayerNormalization()(res2)\n",
        "\n",
        "# Feed-forward block for the first two multi-heads\n",
        "ffa1 = tf.keras.layers.Dense(units=repdim, activation='relu')(nrm2)\n",
        "ffb1 = tf.keras.layers.Dense(units=repdim, kernel_regularizer=regularizers.l2(0.001))(ffa1)\n",
        "res3 = tf.keras.layers.Add()([nrm2, ffb1])\n",
        "nrm3 = tf.keras.layers.LayerNormalization()(res3)\n",
        "\n",
        "# Feed-forward block for the third multi-head\n",
        "ffa2 = tf.keras.layers.Dense(units=repdim, activation='relu')(nrm3)\n",
        "ffb2 = tf.keras.layers.Dense(units=repdim, kernel_regularizer=regularizers.l2(0.001))(ffa2)\n",
        "res5 = tf.keras.layers.Add()([nrm3, ffb2])\n",
        "nrm5 = tf.keras.layers.LayerNormalization()(res5)\n",
        "\n",
        "# Add dropout\n",
        "dropout_layer = tf.keras.layers.Dropout(0.4)(nrm5)\n",
        "\n",
        "# Flatten and classification block\n",
        "flt = tf.keras.layers.Flatten()(dropout_layer)\n",
        "outlayer = tf.keras.layers.Dense(units=1, activation='sigmoid')(flt)\n",
        "\n",
        "# Create the model\n",
        "model = tf.keras.Model(inputs=inlayer, outputs=outlayer)\n",
        "\n",
        "# Build and compile the model\n",
        "# compile model\n",
        "model.compile(\n",
        "    optimizer=tf.keras.optimizers.legacy.Adam(),\n",
        "    loss=tf.keras.losses.BinaryCrossentropy(),\n",
        "    metrics=['accuracy', tf.keras.metrics.Precision(), tf.keras.metrics.Recall(), tf.keras.metrics.AUC()]\n",
        ")\n",
        "\n",
        "# Early stopping callback\n",
        "early_stopping = tf.keras.callbacks.EarlyStopping(\n",
        "    monitor='val_loss',\n",
        "    patience=10,\n",
        "    restore_best_weights=True\n",
        ")\n",
        "\n",
        "# Fit model on training data\n",
        "model.fit(\n",
        "    train_data,\n",
        "    epochs=70,\n",
        "    validation_data=valid_data,\n",
        "    callbacks=[early_stopping]\n",
        ")\n",
        "\n",
        "# Model summary\n",
        "model.summary()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V5F7p1ErOUAo",
        "outputId": "b15c6d30-9013-429a-ce2a-4addac7afe7f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/70\n",
            "409/409 [==============================] - 24s 51ms/step - loss: 0.9013 - accuracy: 0.5639 - precision_1: 0.5606 - recall_1: 0.5587 - val_loss: 0.6014 - val_accuracy: 0.7583 - val_precision_1: 0.7933 - val_recall_1: 0.6981\n",
            "Epoch 2/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.6340 - accuracy: 0.7170 - precision_1: 0.7141 - recall_1: 0.7161 - val_loss: 0.5529 - val_accuracy: 0.7583 - val_precision_1: 0.8475 - val_recall_1: 0.6295\n",
            "Epoch 3/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.5630 - accuracy: 0.7521 - precision_1: 0.7524 - recall_1: 0.7456 - val_loss: 0.4894 - val_accuracy: 0.8027 - val_precision_1: 0.7702 - val_recall_1: 0.8622\n",
            "Epoch 4/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.5241 - accuracy: 0.7709 - precision_1: 0.7703 - recall_1: 0.7665 - val_loss: 0.4702 - val_accuracy: 0.8067 - val_precision_1: 0.7839 - val_recall_1: 0.8463\n",
            "Epoch 5/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.4985 - accuracy: 0.7831 - precision_1: 0.7841 - recall_1: 0.7766 - val_loss: 0.4523 - val_accuracy: 0.8162 - val_precision_1: 0.7585 - val_recall_1: 0.9271\n",
            "Epoch 6/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4795 - accuracy: 0.7881 - precision_1: 0.7861 - recall_1: 0.7867 - val_loss: 0.4281 - val_accuracy: 0.8287 - val_precision_1: 0.7804 - val_recall_1: 0.9143\n",
            "Epoch 7/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.4649 - accuracy: 0.7966 - precision_1: 0.7942 - recall_1: 0.7963 - val_loss: 0.4399 - val_accuracy: 0.8168 - val_precision_1: 0.7585 - val_recall_1: 0.9290\n",
            "Epoch 8/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4590 - accuracy: 0.7983 - precision_1: 0.7989 - recall_1: 0.7929 - val_loss: 0.4367 - val_accuracy: 0.8122 - val_precision_1: 0.7496 - val_recall_1: 0.9369\n",
            "Epoch 9/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.4456 - accuracy: 0.8010 - precision_1: 0.7998 - recall_1: 0.7988 - val_loss: 0.4294 - val_accuracy: 0.8140 - val_precision_1: 0.7491 - val_recall_1: 0.9437\n",
            "Epoch 10/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4442 - accuracy: 0.8028 - precision_1: 0.8015 - recall_1: 0.8006 - val_loss: 0.4472 - val_accuracy: 0.8100 - val_precision_1: 0.7423 - val_recall_1: 0.9492\n",
            "Epoch 11/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.4365 - accuracy: 0.8060 - precision_1: 0.8051 - recall_1: 0.8032 - val_loss: 0.3910 - val_accuracy: 0.8370 - val_precision_1: 0.7976 - val_recall_1: 0.9026\n",
            "Epoch 12/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4323 - accuracy: 0.8074 - precision_1: 0.8063 - recall_1: 0.8051 - val_loss: 0.4210 - val_accuracy: 0.8155 - val_precision_1: 0.7510 - val_recall_1: 0.9437\n",
            "Epoch 13/70\n",
            "409/409 [==============================] - 21s 52ms/step - loss: 0.4264 - accuracy: 0.8115 - precision_1: 0.8104 - recall_1: 0.8093 - val_loss: 0.3984 - val_accuracy: 0.8296 - val_precision_1: 0.7765 - val_recall_1: 0.9253\n",
            "Epoch 14/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4243 - accuracy: 0.8090 - precision_1: 0.8080 - recall_1: 0.8066 - val_loss: 0.3924 - val_accuracy: 0.8348 - val_precision_1: 0.7836 - val_recall_1: 0.9247\n",
            "Epoch 15/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.4198 - accuracy: 0.8106 - precision_1: 0.8059 - recall_1: 0.8142 - val_loss: 0.4212 - val_accuracy: 0.8186 - val_precision_1: 0.7522 - val_recall_1: 0.9498\n",
            "Epoch 16/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4137 - accuracy: 0.8154 - precision_1: 0.8145 - recall_1: 0.8131 - val_loss: 0.4125 - val_accuracy: 0.8232 - val_precision_1: 0.7582 - val_recall_1: 0.9486\n",
            "Epoch 17/70\n",
            "409/409 [==============================] - 21s 52ms/step - loss: 0.4145 - accuracy: 0.8139 - precision_1: 0.8120 - recall_1: 0.8131 - val_loss: 0.3978 - val_accuracy: 0.8272 - val_precision_1: 0.7694 - val_recall_1: 0.9339\n",
            "Epoch 18/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4129 - accuracy: 0.8161 - precision_1: 0.8143 - recall_1: 0.8150 - val_loss: 0.3937 - val_accuracy: 0.8308 - val_precision_1: 0.7741 - val_recall_1: 0.9339\n",
            "Epoch 19/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.4093 - accuracy: 0.8173 - precision_1: 0.8149 - recall_1: 0.8173 - val_loss: 0.4046 - val_accuracy: 0.8217 - val_precision_1: 0.7574 - val_recall_1: 0.9461\n",
            "Epoch 20/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4037 - accuracy: 0.8196 - precision_1: 0.8177 - recall_1: 0.8188 - val_loss: 0.3824 - val_accuracy: 0.8333 - val_precision_1: 0.7845 - val_recall_1: 0.9186\n",
            "Epoch 21/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.4024 - accuracy: 0.8215 - precision_1: 0.8199 - recall_1: 0.8202 - val_loss: 0.3688 - val_accuracy: 0.8397 - val_precision_1: 0.7964 - val_recall_1: 0.9124\n",
            "Epoch 22/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4031 - accuracy: 0.8214 - precision_1: 0.8215 - recall_1: 0.8176 - val_loss: 0.3752 - val_accuracy: 0.8363 - val_precision_1: 0.7862 - val_recall_1: 0.9235\n",
            "Epoch 23/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.3937 - accuracy: 0.8268 - precision_1: 0.8250 - recall_1: 0.8261 - val_loss: 0.3671 - val_accuracy: 0.8415 - val_precision_1: 0.7964 - val_recall_1: 0.9173\n",
            "Epoch 24/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3966 - accuracy: 0.8229 - precision_1: 0.8212 - recall_1: 0.8221 - val_loss: 0.3786 - val_accuracy: 0.8357 - val_precision_1: 0.7819 - val_recall_1: 0.9308\n",
            "Epoch 25/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.3961 - accuracy: 0.8286 - precision_1: 0.8286 - recall_1: 0.8251 - val_loss: 0.3971 - val_accuracy: 0.8235 - val_precision_1: 0.7596 - val_recall_1: 0.9461\n",
            "Epoch 26/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3900 - accuracy: 0.8274 - precision_1: 0.8248 - recall_1: 0.8278 - val_loss: 0.3579 - val_accuracy: 0.8449 - val_precision_1: 0.8104 - val_recall_1: 0.9002\n",
            "Epoch 27/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3896 - accuracy: 0.8279 - precision_1: 0.8274 - recall_1: 0.8251 - val_loss: 0.3774 - val_accuracy: 0.8333 - val_precision_1: 0.7790 - val_recall_1: 0.9302\n",
            "Epoch 28/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3869 - accuracy: 0.8314 - precision_1: 0.8287 - recall_1: 0.8322 - val_loss: 0.3813 - val_accuracy: 0.8345 - val_precision_1: 0.7774 - val_recall_1: 0.9369\n",
            "Epoch 29/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.3867 - accuracy: 0.8269 - precision_1: 0.8254 - recall_1: 0.8258 - val_loss: 0.3757 - val_accuracy: 0.8397 - val_precision_1: 0.7881 - val_recall_1: 0.9290\n",
            "Epoch 30/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3857 - accuracy: 0.8316 - precision_1: 0.8311 - recall_1: 0.8288 - val_loss: 0.3497 - val_accuracy: 0.8526 - val_precision_1: 0.8329 - val_recall_1: 0.8818\n",
            "Epoch 31/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.3853 - accuracy: 0.8277 - precision_1: 0.8265 - recall_1: 0.8259 - val_loss: 0.3794 - val_accuracy: 0.8357 - val_precision_1: 0.7793 - val_recall_1: 0.9363\n",
            "Epoch 32/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3800 - accuracy: 0.8294 - precision_1: 0.8261 - recall_1: 0.8308 - val_loss: 0.3530 - val_accuracy: 0.8467 - val_precision_1: 0.8131 - val_recall_1: 0.9002\n",
            "Epoch 33/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.3801 - accuracy: 0.8330 - precision_1: 0.8313 - recall_1: 0.8321 - val_loss: 0.3740 - val_accuracy: 0.8388 - val_precision_1: 0.7862 - val_recall_1: 0.9302\n",
            "Epoch 34/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3816 - accuracy: 0.8319 - precision_1: 0.8303 - recall_1: 0.8308 - val_loss: 0.3485 - val_accuracy: 0.8535 - val_precision_1: 0.8297 - val_recall_1: 0.8892\n",
            "Epoch 35/70\n",
            "409/409 [==============================] - 21s 52ms/step - loss: 0.3794 - accuracy: 0.8350 - precision_1: 0.8321 - recall_1: 0.8359 - val_loss: 0.3558 - val_accuracy: 0.8477 - val_precision_1: 0.8123 - val_recall_1: 0.9039\n",
            "Epoch 36/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.3769 - accuracy: 0.8349 - precision_1: 0.8317 - recall_1: 0.8362 - val_loss: 0.3560 - val_accuracy: 0.8498 - val_precision_1: 0.8124 - val_recall_1: 0.9094\n",
            "Epoch 37/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3765 - accuracy: 0.8336 - precision_1: 0.8318 - recall_1: 0.8328 - val_loss: 0.3498 - val_accuracy: 0.8486 - val_precision_1: 0.8248 - val_recall_1: 0.8849\n",
            "Epoch 38/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3770 - accuracy: 0.8350 - precision_1: 0.8327 - recall_1: 0.8350 - val_loss: 0.3673 - val_accuracy: 0.8449 - val_precision_1: 0.7998 - val_recall_1: 0.9198\n",
            "Epoch 39/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3757 - accuracy: 0.8366 - precision_1: 0.8346 - recall_1: 0.8362 - val_loss: 0.3521 - val_accuracy: 0.8483 - val_precision_1: 0.8143 - val_recall_1: 0.9020\n",
            "Epoch 40/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.3719 - accuracy: 0.8384 - precision_1: 0.8347 - recall_1: 0.8407 - val_loss: 0.3447 - val_accuracy: 0.8501 - val_precision_1: 0.8286 - val_recall_1: 0.8824\n",
            "Epoch 41/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3711 - accuracy: 0.8362 - precision_1: 0.8347 - recall_1: 0.8350 - val_loss: 0.3453 - val_accuracy: 0.8544 - val_precision_1: 0.8327 - val_recall_1: 0.8867\n",
            "Epoch 42/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.3713 - accuracy: 0.8373 - precision_1: 0.8346 - recall_1: 0.8381 - val_loss: 0.3448 - val_accuracy: 0.8526 - val_precision_1: 0.8352 - val_recall_1: 0.8781\n",
            "Epoch 43/70\n",
            "409/409 [==============================] - 21s 52ms/step - loss: 0.3702 - accuracy: 0.8389 - precision_1: 0.8370 - recall_1: 0.8386 - val_loss: 0.3460 - val_accuracy: 0.8553 - val_precision_1: 0.8333 - val_recall_1: 0.8879\n",
            "Epoch 44/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3677 - accuracy: 0.8384 - precision_1: 0.8349 - recall_1: 0.8404 - val_loss: 0.3397 - val_accuracy: 0.8556 - val_precision_1: 0.8437 - val_recall_1: 0.8726\n",
            "Epoch 45/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.3680 - accuracy: 0.8385 - precision_1: 0.8355 - recall_1: 0.8396 - val_loss: 0.3444 - val_accuracy: 0.8544 - val_precision_1: 0.8373 - val_recall_1: 0.8794\n",
            "Epoch 46/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3688 - accuracy: 0.8365 - precision_1: 0.8351 - recall_1: 0.8353 - val_loss: 0.3422 - val_accuracy: 0.8522 - val_precision_1: 0.8359 - val_recall_1: 0.8763\n",
            "Epoch 47/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3643 - accuracy: 0.8418 - precision_1: 0.8390 - recall_1: 0.8429 - val_loss: 0.3391 - val_accuracy: 0.8538 - val_precision_1: 0.8423 - val_recall_1: 0.8702\n",
            "Epoch 48/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.3638 - accuracy: 0.8411 - precision_1: 0.8381 - recall_1: 0.8423 - val_loss: 0.3482 - val_accuracy: 0.8513 - val_precision_1: 0.8242 - val_recall_1: 0.8928\n",
            "Epoch 49/70\n",
            "409/409 [==============================] - 22s 53ms/step - loss: 0.3647 - accuracy: 0.8387 - precision_1: 0.8364 - recall_1: 0.8389 - val_loss: 0.3384 - val_accuracy: 0.8535 - val_precision_1: 0.8575 - val_recall_1: 0.8475\n",
            "Epoch 50/70\n",
            "409/409 [==============================] - 21s 52ms/step - loss: 0.3609 - accuracy: 0.8411 - precision_1: 0.8413 - recall_1: 0.8376 - val_loss: 0.3493 - val_accuracy: 0.8480 - val_precision_1: 0.8156 - val_recall_1: 0.8990\n",
            "Epoch 51/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3650 - accuracy: 0.8389 - precision_1: 0.8364 - recall_1: 0.8395 - val_loss: 0.3407 - val_accuracy: 0.8529 - val_precision_1: 0.8551 - val_recall_1: 0.8494\n",
            "Epoch 52/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.3648 - accuracy: 0.8368 - precision_1: 0.8347 - recall_1: 0.8365 - val_loss: 0.3390 - val_accuracy: 0.8501 - val_precision_1: 0.8474 - val_recall_1: 0.8536\n",
            "Epoch 53/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3611 - accuracy: 0.8437 - precision_1: 0.8422 - recall_1: 0.8427 - val_loss: 0.3390 - val_accuracy: 0.8529 - val_precision_1: 0.8466 - val_recall_1: 0.8616\n",
            "Epoch 54/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3609 - accuracy: 0.8447 - precision_1: 0.8439 - recall_1: 0.8427 - val_loss: 0.3406 - val_accuracy: 0.8535 - val_precision_1: 0.8366 - val_recall_1: 0.8781\n",
            "Epoch 55/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3606 - accuracy: 0.8466 - precision_1: 0.8445 - recall_1: 0.8466 - val_loss: 0.3415 - val_accuracy: 0.8544 - val_precision_1: 0.8369 - val_recall_1: 0.8800\n",
            "Epoch 56/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.3610 - accuracy: 0.8417 - precision_1: 0.8405 - recall_1: 0.8402 - val_loss: 0.3423 - val_accuracy: 0.8513 - val_precision_1: 0.8290 - val_recall_1: 0.8849\n",
            "Epoch 57/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3593 - accuracy: 0.8476 - precision_1: 0.8457 - recall_1: 0.8475 - val_loss: 0.3387 - val_accuracy: 0.8559 - val_precision_1: 0.8414 - val_recall_1: 0.8769\n",
            "Epoch 58/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3606 - accuracy: 0.8408 - precision_1: 0.8376 - recall_1: 0.8423 - val_loss: 0.3443 - val_accuracy: 0.8519 - val_precision_1: 0.8288 - val_recall_1: 0.8867\n",
            "Epoch 59/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.3527 - accuracy: 0.8451 - precision_1: 0.8455 - recall_1: 0.8415 - val_loss: 0.3394 - val_accuracy: 0.8504 - val_precision_1: 0.8438 - val_recall_1: 0.8598\n",
            "Model: \"model_2\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                Output Shape                 Param #   Connected to                  \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)        [(None, 256, 2)]             0         []                            \n",
            "                                                                                                  \n",
            " dense_6 (Dense)             (None, 256, 32)              96        ['input_2[0][0]']             \n",
            "                                                                                                  \n",
            " multi_head_attention_2 (Mu  (None, 256, 32)              58720     ['dense_6[0][0]',             \n",
            " ltiHeadAttention)                                                   'dense_6[0][0]',             \n",
            "                                                                     'dense_6[0][0]']             \n",
            "                                                                                                  \n",
            " add_4 (Add)                 (None, 256, 32)              0         ['dense_6[0][0]',             \n",
            "                                                                     'multi_head_attention_2[0][0]\n",
            "                                                                    ']                            \n",
            "                                                                                                  \n",
            " layer_normalization_4 (Lay  (None, 256, 32)              64        ['add_4[0][0]']               \n",
            " erNormalization)                                                                                 \n",
            "                                                                                                  \n",
            " multi_head_attention_3 (Mu  (None, 256, 32)              58720     ['layer_normalization_4[0][0]'\n",
            " ltiHeadAttention)                                                  , 'layer_normalization_4[0][0]\n",
            "                                                                    ',                            \n",
            "                                                                     'layer_normalization_4[0][0]'\n",
            "                                                                    ]                             \n",
            "                                                                                                  \n",
            " add_5 (Add)                 (None, 256, 32)              0         ['layer_normalization_4[0][0]'\n",
            "                                                                    , 'multi_head_attention_3[0][0\n",
            "                                                                    ]']                           \n",
            "                                                                                                  \n",
            " layer_normalization_5 (Lay  (None, 256, 32)              64        ['add_5[0][0]']               \n",
            " erNormalization)                                                                                 \n",
            "                                                                                                  \n",
            " dense_7 (Dense)             (None, 256, 128)             4224      ['layer_normalization_5[0][0]'\n",
            "                                                                    ]                             \n",
            "                                                                                                  \n",
            " dense_8 (Dense)             (None, 256, 32)              4128      ['dense_7[0][0]']             \n",
            "                                                                                                  \n",
            " add_6 (Add)                 (None, 256, 32)              0         ['layer_normalization_5[0][0]'\n",
            "                                                                    , 'dense_8[0][0]']            \n",
            "                                                                                                  \n",
            " layer_normalization_6 (Lay  (None, 256, 32)              64        ['add_6[0][0]']               \n",
            " erNormalization)                                                                                 \n",
            "                                                                                                  \n",
            " dense_9 (Dense)             (None, 256, 128)             4224      ['layer_normalization_6[0][0]'\n",
            "                                                                    ]                             \n",
            "                                                                                                  \n",
            " dense_10 (Dense)            (None, 256, 32)              4128      ['dense_9[0][0]']             \n",
            "                                                                                                  \n",
            " add_7 (Add)                 (None, 256, 32)              0         ['layer_normalization_6[0][0]'\n",
            "                                                                    , 'dense_10[0][0]']           \n",
            "                                                                                                  \n",
            " layer_normalization_7 (Lay  (None, 256, 32)              64        ['add_7[0][0]']               \n",
            " erNormalization)                                                                                 \n",
            "                                                                                                  \n",
            " dropout_1 (Dropout)         (None, 256, 32)              0         ['layer_normalization_7[0][0]'\n",
            "                                                                    ]                             \n",
            "                                                                                                  \n",
            " flatten_1 (Flatten)         (None, 8192)                 0         ['dropout_1[0][0]']           \n",
            "                                                                                                  \n",
            " dense_11 (Dense)            (None, 1)                    8193      ['flatten_1[0][0]']           \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 142689 (557.38 KB)\n",
            "Trainable params: 142689 (557.38 KB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras import regularizers\n",
        "from sklearn.model_selection import train_test_split\n",
        "from tensorflow.keras.metrics import Precision, Recall\n",
        "from tensorflow.keras.optimizers.legacy import Adam\n",
        "from sklearn.metrics import roc_auc_score\n",
        "\n",
        "# Load your data\n",
        "dataframe = pd.read_csv('https://raw.githubusercontent.com/bryankolaczkowski/ALS3200C/main/mbiome.data.csv')\n",
        "\n",
        "# Extract explanatory variables and label\n",
        "dta_ids = [x for x in dataframe.columns if x.startswith('DTA')]\n",
        "X = dataframe[dta_ids].to_numpy()\n",
        "y = dataframe['LBL0'].to_numpy()\n",
        "\n",
        "# Add 'location' to sequence data\n",
        "loc = np.linspace(start=-2.5, stop=+2.5, num=X.shape[1])\n",
        "X = np.stack([X, np.array([loc] * X.shape[0])], axis=-1)\n",
        "\n",
        "# Train-validation split\n",
        "train_x, valid_x, train_y, valid_y = train_test_split(X, y, test_size=0.2, random_state=42)\n",
        "\n",
        "# Package data into TensorFlow datasets\n",
        "train_data = tf.data.Dataset.from_tensor_slices((train_x, train_y)).batch(32)\n",
        "valid_data = tf.data.Dataset.from_tensor_slices((valid_x, valid_y)).batch(32)\n",
        "# Define the model building function\n",
        "def build_model():\n",
        "    repdim = 32  # Set internal data representation dimensionality\n",
        "    inlayer = tf.keras.Input(shape=(256, 2))\n",
        "    proj = tf.keras.layers.Dense(units=repdim, kernel_regularizer=regularizers.l2(0.001))(inlayer)\n",
        "    mha1 = tf.keras.layers.MultiHeadAttention(num_heads=14, key_dim=repdim)(proj, proj, proj)\n",
        "    res1 = tf.keras.layers.Add()([proj, mha1])\n",
        "    nrm1 = tf.keras.layers.LayerNormalization()(res1)\n",
        "    mha2 = tf.keras.layers.MultiHeadAttention(num_heads=14, key_dim=repdim)(nrm1, nrm1, nrm1)\n",
        "    res2 = tf.keras.layers.Add()([nrm1, mha2])\n",
        "    nrm2 = tf.keras.layers.LayerNormalization()(res2)\n",
        "    ffa1 = tf.keras.layers.Dense(units=repdim*4, activation='relu')(nrm2)\n",
        "    ffb1 = tf.keras.layers.Dense(units=repdim, kernel_regularizer=regularizers.l2(0.001))(ffa1)\n",
        "    res3 = tf.keras.layers.Add()([nrm2, ffb1])\n",
        "    nrm3 = tf.keras.layers.LayerNormalization()(res3)\n",
        "    ffa2 = tf.keras.layers.Dense(units=repdim*4, activation='relu')(nrm3)\n",
        "    ffb2 = tf.keras.layers.Dense(units=repdim, kernel_regularizer=regularizers.l2(0.001))(ffa2)\n",
        "    res5 = tf.keras.layers.Add()([nrm3, ffb2])\n",
        "    nrm5 = tf.keras.layers.LayerNormalization()(res5)\n",
        "    dropout_layer = tf.keras.layers.Dropout(0.4)(nrm5)\n",
        "    flt = tf.keras.layers.Flatten()(dropout_layer)\n",
        "    outlayer = tf.keras.layers.Dense(units=1, activation='sigmoid')(flt)\n",
        "    model = tf.keras.Model(inputs=inlayer, outputs=outlayer)\n",
        "    return model\n",
        "\n",
        "# Build and compile the model\n",
        "model = build_model()\n",
        "model.compile(optimizer=Adam(learning_rate=0.001),\n",
        "              loss='binary_crossentropy',\n",
        "              metrics=['accuracy', Precision(), Recall()])\n",
        "\n",
        "# Early stopping callback\n",
        "early_stopping = tf.keras.callbacks.EarlyStopping(\n",
        "    monitor='val_loss',\n",
        "    patience=10,\n",
        "    restore_best_weights=True\n",
        ")\n",
        "\n",
        "# Fit model on training data\n",
        "history = model.fit(\n",
        "    train_data,\n",
        "    epochs=70,\n",
        "    validation_data=valid_data,\n",
        "    callbacks=[early_stopping]\n",
        ")\n",
        "\n",
        "# Model summary\n",
        "model.summary()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ehBm11hDFNh4",
        "outputId": "4cdde5de-5b24-4bda-e01f-70c03c7fc617"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/70\n",
            "409/409 [==============================] - 34s 53ms/step - loss: 0.8777 - accuracy: 0.5420 - precision: 0.5386 - recall: 0.5345 - val_loss: 0.6727 - val_accuracy: 0.7385 - val_precision: 0.7020 - val_recall: 0.8279\n",
            "Epoch 2/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.7385 - accuracy: 0.6494 - precision: 0.6474 - recall: 0.6436 - val_loss: 0.5603 - val_accuracy: 0.7880 - val_precision: 0.7652 - val_recall: 0.8304\n",
            "Epoch 3/70\n",
            "409/409 [==============================] - 20s 49ms/step - loss: 0.6472 - accuracy: 0.7150 - precision: 0.7141 - recall: 0.7095 - val_loss: 0.5146 - val_accuracy: 0.8100 - val_precision: 0.7741 - val_recall: 0.8751\n",
            "Epoch 4/70\n",
            "409/409 [==============================] - 21s 52ms/step - loss: 0.6047 - accuracy: 0.7459 - precision: 0.7441 - recall: 0.7431 - val_loss: 0.4886 - val_accuracy: 0.8275 - val_precision: 0.7974 - val_recall: 0.8775\n",
            "Epoch 5/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.5838 - accuracy: 0.7607 - precision: 0.7590 - recall: 0.7584 - val_loss: 0.4705 - val_accuracy: 0.8305 - val_precision: 0.8288 - val_recall: 0.8328\n",
            "Epoch 6/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.5566 - accuracy: 0.7748 - precision: 0.7723 - recall: 0.7744 - val_loss: 0.4630 - val_accuracy: 0.8370 - val_precision: 0.8544 - val_recall: 0.8120\n",
            "Epoch 7/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.5465 - accuracy: 0.7807 - precision: 0.7783 - recall: 0.7801 - val_loss: 0.4542 - val_accuracy: 0.8385 - val_precision: 0.8460 - val_recall: 0.8273\n",
            "Epoch 8/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.5260 - accuracy: 0.7946 - precision: 0.7937 - recall: 0.7918 - val_loss: 0.4481 - val_accuracy: 0.8446 - val_precision: 0.8432 - val_recall: 0.8463\n",
            "Epoch 9/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.5188 - accuracy: 0.7969 - precision: 0.7936 - recall: 0.7980 - val_loss: 0.4431 - val_accuracy: 0.8425 - val_precision: 0.8363 - val_recall: 0.8512\n",
            "Epoch 10/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.5042 - accuracy: 0.8050 - precision: 0.8047 - recall: 0.8012 - val_loss: 0.4391 - val_accuracy: 0.8467 - val_precision: 0.8333 - val_recall: 0.8665\n",
            "Epoch 11/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.5035 - accuracy: 0.8078 - precision: 0.8074 - recall: 0.8043 - val_loss: 0.4351 - val_accuracy: 0.8477 - val_precision: 0.8380 - val_recall: 0.8616\n",
            "Epoch 12/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4952 - accuracy: 0.8080 - precision: 0.8054 - recall: 0.8080 - val_loss: 0.4328 - val_accuracy: 0.8461 - val_precision: 0.8324 - val_recall: 0.8665\n",
            "Epoch 13/70\n",
            "409/409 [==============================] - 21s 52ms/step - loss: 0.4909 - accuracy: 0.8102 - precision: 0.8098 - recall: 0.8068 - val_loss: 0.4319 - val_accuracy: 0.8467 - val_precision: 0.8272 - val_recall: 0.8763\n",
            "Epoch 14/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.4903 - accuracy: 0.8099 - precision: 0.8091 - recall: 0.8072 - val_loss: 0.4285 - val_accuracy: 0.8532 - val_precision: 0.8556 - val_recall: 0.8494\n",
            "Epoch 15/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4818 - accuracy: 0.8162 - precision: 0.8138 - recall: 0.8162 - val_loss: 0.4248 - val_accuracy: 0.8522 - val_precision: 0.8528 - val_recall: 0.8512\n",
            "Epoch 16/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.4802 - accuracy: 0.8158 - precision: 0.8156 - recall: 0.8122 - val_loss: 0.4282 - val_accuracy: 0.8455 - val_precision: 0.8179 - val_recall: 0.8885\n",
            "Epoch 17/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4776 - accuracy: 0.8149 - precision: 0.8127 - recall: 0.8145 - val_loss: 0.4234 - val_accuracy: 0.8486 - val_precision: 0.8339 - val_recall: 0.8702\n",
            "Epoch 18/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.4652 - accuracy: 0.8260 - precision: 0.8240 - recall: 0.8254 - val_loss: 0.4227 - val_accuracy: 0.8532 - val_precision: 0.8592 - val_recall: 0.8445\n",
            "Epoch 19/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4679 - accuracy: 0.8198 - precision: 0.8169 - recall: 0.8207 - val_loss: 0.4190 - val_accuracy: 0.8538 - val_precision: 0.8515 - val_recall: 0.8567\n",
            "Epoch 20/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.4608 - accuracy: 0.8285 - precision: 0.8276 - recall: 0.8264 - val_loss: 0.4208 - val_accuracy: 0.8541 - val_precision: 0.8672 - val_recall: 0.8359\n",
            "Epoch 21/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4629 - accuracy: 0.8250 - precision: 0.8259 - recall: 0.8200 - val_loss: 0.4169 - val_accuracy: 0.8504 - val_precision: 0.8496 - val_recall: 0.8512\n",
            "Epoch 22/70\n",
            "409/409 [==============================] - 20s 49ms/step - loss: 0.4599 - accuracy: 0.8234 - precision: 0.8220 - recall: 0.8219 - val_loss: 0.4170 - val_accuracy: 0.8541 - val_precision: 0.8457 - val_recall: 0.8659\n",
            "Epoch 23/70\n",
            "409/409 [==============================] - 22s 53ms/step - loss: 0.4572 - accuracy: 0.8252 - precision: 0.8240 - recall: 0.8233 - val_loss: 0.4127 - val_accuracy: 0.8535 - val_precision: 0.8468 - val_recall: 0.8628\n",
            "Epoch 24/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4513 - accuracy: 0.8311 - precision: 0.8306 - recall: 0.8285 - val_loss: 0.4144 - val_accuracy: 0.8535 - val_precision: 0.8347 - val_recall: 0.8812\n",
            "Epoch 25/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.4524 - accuracy: 0.8285 - precision: 0.8273 - recall: 0.8267 - val_loss: 0.4123 - val_accuracy: 0.8538 - val_precision: 0.8399 - val_recall: 0.8739\n",
            "Epoch 26/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4489 - accuracy: 0.8293 - precision: 0.8262 - recall: 0.8305 - val_loss: 0.4118 - val_accuracy: 0.8526 - val_precision: 0.8407 - val_recall: 0.8696\n",
            "Epoch 27/70\n",
            "409/409 [==============================] - 20s 49ms/step - loss: 0.4458 - accuracy: 0.8296 - precision: 0.8282 - recall: 0.8282 - val_loss: 0.4095 - val_accuracy: 0.8550 - val_precision: 0.8593 - val_recall: 0.8487\n",
            "Epoch 28/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4475 - accuracy: 0.8300 - precision: 0.8284 - recall: 0.8290 - val_loss: 0.4071 - val_accuracy: 0.8547 - val_precision: 0.8488 - val_recall: 0.8628\n",
            "Epoch 29/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4437 - accuracy: 0.8306 - precision: 0.8303 - recall: 0.8276 - val_loss: 0.4070 - val_accuracy: 0.8562 - val_precision: 0.8570 - val_recall: 0.8549\n",
            "Epoch 30/70\n",
            "409/409 [==============================] - 20s 49ms/step - loss: 0.4418 - accuracy: 0.8330 - precision: 0.8319 - recall: 0.8313 - val_loss: 0.4061 - val_accuracy: 0.8562 - val_precision: 0.8501 - val_recall: 0.8647\n",
            "Epoch 31/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4418 - accuracy: 0.8318 - precision: 0.8297 - recall: 0.8316 - val_loss: 0.4048 - val_accuracy: 0.8574 - val_precision: 0.8691 - val_recall: 0.8414\n",
            "Epoch 32/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.4442 - accuracy: 0.8309 - precision: 0.8293 - recall: 0.8299 - val_loss: 0.4035 - val_accuracy: 0.8584 - val_precision: 0.8607 - val_recall: 0.8549\n",
            "Epoch 33/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4368 - accuracy: 0.8345 - precision: 0.8319 - recall: 0.8350 - val_loss: 0.4011 - val_accuracy: 0.8590 - val_precision: 0.8622 - val_recall: 0.8543\n",
            "Epoch 34/70\n",
            "409/409 [==============================] - 20s 49ms/step - loss: 0.4425 - accuracy: 0.8311 - precision: 0.8287 - recall: 0.8311 - val_loss: 0.4010 - val_accuracy: 0.8584 - val_precision: 0.8528 - val_recall: 0.8659\n",
            "Epoch 35/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4361 - accuracy: 0.8323 - precision: 0.8311 - recall: 0.8307 - val_loss: 0.4014 - val_accuracy: 0.8541 - val_precision: 0.8380 - val_recall: 0.8775\n",
            "Epoch 36/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.4355 - accuracy: 0.8379 - precision: 0.8375 - recall: 0.8353 - val_loss: 0.3994 - val_accuracy: 0.8571 - val_precision: 0.8521 - val_recall: 0.8641\n",
            "Epoch 37/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4301 - accuracy: 0.8386 - precision: 0.8397 - recall: 0.8338 - val_loss: 0.3989 - val_accuracy: 0.8599 - val_precision: 0.8660 - val_recall: 0.8512\n",
            "Epoch 38/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.4315 - accuracy: 0.8369 - precision: 0.8362 - recall: 0.8347 - val_loss: 0.3976 - val_accuracy: 0.8553 - val_precision: 0.8465 - val_recall: 0.8677\n",
            "Epoch 39/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4306 - accuracy: 0.8369 - precision: 0.8363 - recall: 0.8344 - val_loss: 0.3978 - val_accuracy: 0.8547 - val_precision: 0.8426 - val_recall: 0.8720\n",
            "Epoch 40/70\n",
            "409/409 [==============================] - 20s 49ms/step - loss: 0.4303 - accuracy: 0.8364 - precision: 0.8363 - recall: 0.8333 - val_loss: 0.3968 - val_accuracy: 0.8550 - val_precision: 0.8447 - val_recall: 0.8696\n",
            "Epoch 41/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.4299 - accuracy: 0.8356 - precision: 0.8353 - recall: 0.8328 - val_loss: 0.3940 - val_accuracy: 0.8587 - val_precision: 0.8500 - val_recall: 0.8708\n",
            "Epoch 42/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4293 - accuracy: 0.8340 - precision: 0.8319 - recall: 0.8339 - val_loss: 0.3947 - val_accuracy: 0.8550 - val_precision: 0.8531 - val_recall: 0.8573\n",
            "Epoch 43/70\n",
            "409/409 [==============================] - 20s 49ms/step - loss: 0.4247 - accuracy: 0.8386 - precision: 0.8370 - recall: 0.8378 - val_loss: 0.3987 - val_accuracy: 0.8535 - val_precision: 0.8260 - val_recall: 0.8953\n",
            "Epoch 44/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4265 - accuracy: 0.8403 - precision: 0.8403 - recall: 0.8372 - val_loss: 0.3919 - val_accuracy: 0.8574 - val_precision: 0.8475 - val_recall: 0.8714\n",
            "Epoch 45/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4274 - accuracy: 0.8383 - precision: 0.8383 - recall: 0.8352 - val_loss: 0.3918 - val_accuracy: 0.8590 - val_precision: 0.8488 - val_recall: 0.8732\n",
            "Epoch 46/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4263 - accuracy: 0.8400 - precision: 0.8381 - recall: 0.8396 - val_loss: 0.3923 - val_accuracy: 0.8568 - val_precision: 0.8482 - val_recall: 0.8690\n",
            "Epoch 47/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4207 - accuracy: 0.8405 - precision: 0.8413 - recall: 0.8362 - val_loss: 0.3910 - val_accuracy: 0.8587 - val_precision: 0.8504 - val_recall: 0.8702\n",
            "Epoch 48/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4183 - accuracy: 0.8430 - precision: 0.8436 - recall: 0.8390 - val_loss: 0.3922 - val_accuracy: 0.8565 - val_precision: 0.8360 - val_recall: 0.8867\n",
            "Epoch 49/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4229 - accuracy: 0.8390 - precision: 0.8385 - recall: 0.8365 - val_loss: 0.3905 - val_accuracy: 0.8587 - val_precision: 0.8462 - val_recall: 0.8763\n",
            "Epoch 50/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4203 - accuracy: 0.8413 - precision: 0.8423 - recall: 0.8367 - val_loss: 0.3886 - val_accuracy: 0.8584 - val_precision: 0.8602 - val_recall: 0.8555\n",
            "Epoch 51/70\n",
            "409/409 [==============================] - 21s 52ms/step - loss: 0.4182 - accuracy: 0.8393 - precision: 0.8409 - recall: 0.8338 - val_loss: 0.3886 - val_accuracy: 0.8593 - val_precision: 0.8452 - val_recall: 0.8794\n",
            "Epoch 52/70\n",
            "409/409 [==============================] - 21s 50ms/step - loss: 0.4181 - accuracy: 0.8392 - precision: 0.8391 - recall: 0.8361 - val_loss: 0.3885 - val_accuracy: 0.8593 - val_precision: 0.8592 - val_recall: 0.8592\n",
            "Epoch 53/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4189 - accuracy: 0.8395 - precision: 0.8403 - recall: 0.8350 - val_loss: 0.3872 - val_accuracy: 0.8590 - val_precision: 0.8443 - val_recall: 0.8800\n",
            "Epoch 54/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.4167 - accuracy: 0.8415 - precision: 0.8397 - recall: 0.8410 - val_loss: 0.3870 - val_accuracy: 0.8590 - val_precision: 0.8419 - val_recall: 0.8836\n",
            "Epoch 55/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4154 - accuracy: 0.8416 - precision: 0.8428 - recall: 0.8367 - val_loss: 0.3858 - val_accuracy: 0.8587 - val_precision: 0.8454 - val_recall: 0.8775\n",
            "Epoch 56/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.4128 - accuracy: 0.8426 - precision: 0.8424 - recall: 0.8398 - val_loss: 0.3856 - val_accuracy: 0.8608 - val_precision: 0.8557 - val_recall: 0.8677\n",
            "Epoch 57/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4143 - accuracy: 0.8424 - precision: 0.8403 - recall: 0.8423 - val_loss: 0.3839 - val_accuracy: 0.8593 - val_precision: 0.8497 - val_recall: 0.8726\n",
            "Epoch 58/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4133 - accuracy: 0.8463 - precision: 0.8456 - recall: 0.8443 - val_loss: 0.3837 - val_accuracy: 0.8617 - val_precision: 0.8504 - val_recall: 0.8775\n",
            "Epoch 59/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.4144 - accuracy: 0.8418 - precision: 0.8413 - recall: 0.8393 - val_loss: 0.3839 - val_accuracy: 0.8578 - val_precision: 0.8489 - val_recall: 0.8702\n",
            "Epoch 60/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4146 - accuracy: 0.8398 - precision: 0.8393 - recall: 0.8375 - val_loss: 0.3826 - val_accuracy: 0.8562 - val_precision: 0.8463 - val_recall: 0.8702\n",
            "Epoch 61/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.4088 - accuracy: 0.8428 - precision: 0.8425 - recall: 0.8402 - val_loss: 0.3820 - val_accuracy: 0.8626 - val_precision: 0.8575 - val_recall: 0.8696\n",
            "Epoch 62/70\n",
            "409/409 [==============================] - 22s 53ms/step - loss: 0.4112 - accuracy: 0.8414 - precision: 0.8404 - recall: 0.8396 - val_loss: 0.3807 - val_accuracy: 0.8648 - val_precision: 0.8616 - val_recall: 0.8690\n",
            "Epoch 63/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.4142 - accuracy: 0.8383 - precision: 0.8386 - recall: 0.8347 - val_loss: 0.3799 - val_accuracy: 0.8630 - val_precision: 0.8589 - val_recall: 0.8683\n",
            "Epoch 64/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4120 - accuracy: 0.8407 - precision: 0.8426 - recall: 0.8347 - val_loss: 0.3818 - val_accuracy: 0.8614 - val_precision: 0.8491 - val_recall: 0.8788\n",
            "Epoch 65/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.4100 - accuracy: 0.8411 - precision: 0.8406 - recall: 0.8386 - val_loss: 0.3795 - val_accuracy: 0.8620 - val_precision: 0.8569 - val_recall: 0.8690\n",
            "Epoch 66/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4113 - accuracy: 0.8422 - precision: 0.8403 - recall: 0.8419 - val_loss: 0.3800 - val_accuracy: 0.8620 - val_precision: 0.8657 - val_recall: 0.8567\n",
            "Epoch 67/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4080 - accuracy: 0.8440 - precision: 0.8423 - recall: 0.8433 - val_loss: 0.3813 - val_accuracy: 0.8574 - val_precision: 0.8383 - val_recall: 0.8855\n",
            "Epoch 68/70\n",
            "409/409 [==============================] - 20s 50ms/step - loss: 0.4081 - accuracy: 0.8410 - precision: 0.8400 - recall: 0.8393 - val_loss: 0.3798 - val_accuracy: 0.8596 - val_precision: 0.8528 - val_recall: 0.8690\n",
            "Epoch 69/70\n",
            "409/409 [==============================] - 21s 52ms/step - loss: 0.4057 - accuracy: 0.8442 - precision: 0.8435 - recall: 0.8421 - val_loss: 0.3779 - val_accuracy: 0.8636 - val_precision: 0.8556 - val_recall: 0.8745\n",
            "Epoch 70/70\n",
            "409/409 [==============================] - 21s 51ms/step - loss: 0.4062 - accuracy: 0.8434 - precision: 0.8442 - recall: 0.8392 - val_loss: 0.3773 - val_accuracy: 0.8642 - val_precision: 0.8537 - val_recall: 0.8788\n",
            "103/103 [==============================] - 3s 22ms/step\n",
            "Validation AUROC: 0.9365418619936907\n",
            "Model: \"model_1\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                Output Shape                 Param #   Connected to                  \n",
            "==================================================================================================\n",
            " input_1 (InputLayer)        [(None, 256, 2)]             0         []                            \n",
            "                                                                                                  \n",
            " bidirectional (Bidirection  (None, 256, 24)              1440      ['input_1[0][0]']             \n",
            " al)                                                                                              \n",
            "                                                                                                  \n",
            " dense (Dense)               (None, 256, 32)              800       ['bidirectional[0][0]']       \n",
            "                                                                                                  \n",
            " multi_head_attention (Mult  (None, 256, 32)              33568     ['dense[0][0]',               \n",
            " iHeadAttention)                                                     'dense[0][0]',               \n",
            "                                                                     'dense[0][0]']               \n",
            "                                                                                                  \n",
            " add (Add)                   (None, 256, 32)              0         ['dense[0][0]',               \n",
            "                                                                     'multi_head_attention[0][0]']\n",
            "                                                                                                  \n",
            " layer_normalization (Layer  (None, 256, 32)              64        ['add[0][0]']                 \n",
            " Normalization)                                                                                   \n",
            "                                                                                                  \n",
            " multi_head_attention_1 (Mu  (None, 256, 32)              33568     ['layer_normalization[0][0]', \n",
            " ltiHeadAttention)                                                   'layer_normalization[0][0]', \n",
            "                                                                     'layer_normalization[0][0]'] \n",
            "                                                                                                  \n",
            " add_1 (Add)                 (None, 256, 32)              0         ['layer_normalization[0][0]', \n",
            "                                                                     'multi_head_attention_1[0][0]\n",
            "                                                                    ']                            \n",
            "                                                                                                  \n",
            " layer_normalization_1 (Lay  (None, 256, 32)              64        ['add_1[0][0]']               \n",
            " erNormalization)                                                                                 \n",
            "                                                                                                  \n",
            " dense_1 (Dense)             (None, 256, 32)              1056      ['layer_normalization_1[0][0]'\n",
            "                                                                    ]                             \n",
            "                                                                                                  \n",
            " dense_2 (Dense)             (None, 256, 32)              1056      ['dense_1[0][0]']             \n",
            "                                                                                                  \n",
            " add_2 (Add)                 (None, 256, 32)              0         ['layer_normalization_1[0][0]'\n",
            "                                                                    , 'dense_2[0][0]']            \n",
            "                                                                                                  \n",
            " layer_normalization_2 (Lay  (None, 256, 32)              64        ['add_2[0][0]']               \n",
            " erNormalization)                                                                                 \n",
            "                                                                                                  \n",
            " dense_3 (Dense)             (None, 256, 32)              1056      ['layer_normalization_2[0][0]'\n",
            "                                                                    ]                             \n",
            "                                                                                                  \n",
            " dense_4 (Dense)             (None, 256, 32)              1056      ['dense_3[0][0]']             \n",
            "                                                                                                  \n",
            " add_3 (Add)                 (None, 256, 32)              0         ['layer_normalization_2[0][0]'\n",
            "                                                                    , 'dense_4[0][0]']            \n",
            "                                                                                                  \n",
            " layer_normalization_3 (Lay  (None, 256, 32)              64        ['add_3[0][0]']               \n",
            " erNormalization)                                                                                 \n",
            "                                                                                                  \n",
            " dropout (Dropout)           (None, 256, 32)              0         ['layer_normalization_3[0][0]'\n",
            "                                                                    ]                             \n",
            "                                                                                                  \n",
            " flatten (Flatten)           (None, 8192)                 0         ['dropout[0][0]']             \n",
            "                                                                                                  \n",
            " dense_5 (Dense)             (None, 1)                    8193      ['flatten[0][0]']             \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 82049 (320.50 KB)\n",
            "Trainable params: 82049 (320.50 KB)\n",
            "Non-trainable params: 0 (0.00 Byte)\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras import regularizers\n",
        "from sklearn.model_selection import train_test_split\n",
        "from tensorflow.keras.metrics import Precision, Recall\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from sklearn.metrics import roc_auc_score\n",
        "\n",
        "# Load your data\n",
        "dataframe = pd.read_csv('https://raw.githubusercontent.com/bryankolaczkowski/ALS3200C/main/mbiome.data.csv')\n",
        "\n",
        "# Extract explanatory variables and label\n",
        "dta_ids = [x for x in dataframe.columns if x.startswith('DTA')]\n",
        "X = dataframe[dta_ids].to_numpy()\n",
        "y = dataframe['LBL0'].to_numpy()\n",
        "\n",
        "# Add 'location' to sequence data\n",
        "loc = np.linspace(start=-2.5, stop=+2.5, num=X.shape[1])\n",
        "X = np.stack([X, np.array([loc] * X.shape[0])], axis=-1)\n",
        "\n",
        "# Train-validation split\n",
        "train_x, valid_x, train_y, valid_y = train_test_split(X, y, test_size=0.2, random_state=42)\n",
        "\n",
        "# Package data into TensorFlow datasets\n",
        "train_data = tf.data.Dataset.from_tensor_slices((train_x, train_y)).batch(32)\n",
        "valid_data = tf.data.Dataset.from_tensor_slices((valid_x, valid_y)).batch(32)\n",
        "\n",
        "\n",
        "# Define the model building function\n",
        "def build_model():\n",
        "\n",
        "  # Build model using functional API\n",
        "  repdim = 32  # Set internal data representation dimensionality\n",
        "\n",
        "  inlayer = tf.keras.Input(shape=(256, 2))\n",
        "\n",
        "  # LSTM layer\n",
        "\n",
        "  # Bidirectional LSTM layer\n",
        "  lstm_layer = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(units=12, return_sequences=True))(inlayer)\n",
        "\n",
        "  # Dense projection layer (if needed to adjust dimensions)\n",
        "  proj = tf.keras.layers.Dense(units=repdim, kernel_regularizer=regularizers.l2(0.001))(lstm_layer)\n",
        "\n",
        "  # First multi-head attention block\n",
        "  mha1 = tf.keras.layers.MultiHeadAttention(num_heads=8, key_dim=repdim)(proj, proj, proj)\n",
        "  res1 = tf.keras.layers.Add()([proj, mha1])\n",
        "  nrm1 = tf.keras.layers.LayerNormalization()(res1)\n",
        "\n",
        "  # Second multi-head attention block\n",
        "  mha2 = tf.keras.layers.MultiHeadAttention(num_heads=8, key_dim=repdim)(nrm1, nrm1, nrm1)\n",
        "  res2 = tf.keras.layers.Add()([nrm1, mha2])\n",
        "  nrm2 = tf.keras.layers.LayerNormalization()(res2)\n",
        "\n",
        "  # Feed-forward block for the first two multi-heads\n",
        "  ffa1 = tf.keras.layers.Dense(units=repdim, activation='relu')(nrm2)\n",
        "  ffb1 = tf.keras.layers.Dense(units=repdim, kernel_regularizer=regularizers.l2(0.001))(ffa1)\n",
        "  res3 = tf.keras.layers.Add()([nrm2, ffb1])\n",
        "  nrm3 = tf.keras.layers.LayerNormalization()(res3)\n",
        "\n",
        "  # Feed-forward block for the third multi-head\n",
        "  ffa2 = tf.keras.layers.Dense(units=repdim, activation='relu')(nrm3)\n",
        "  ffb2 = tf.keras.layers.Dense(units=repdim, kernel_regularizer=regularizers.l2(0.001))(ffa2)\n",
        "  res5 = tf.keras.layers.Add()([nrm3, ffb2])\n",
        "  nrm5 = tf.keras.layers.LayerNormalization()(res5)\n",
        "\n",
        "  # Add dropout\n",
        "  dropout_layer = tf.keras.layers.Dropout(0.4)(nrm5)\n",
        "\n",
        "  # Flatten and classification block\n",
        "  flt = tf.keras.layers.Flatten()(dropout_layer)\n",
        "  outlayer = tf.keras.layers.Dense(units=1, activation='sigmoid')(flt)\n",
        "\n",
        "  # Create the model\n",
        "  model = tf.keras.Model(inputs=inlayer, outputs=outlayer)\n",
        "  model = tf.keras.Model(inputs=inlayer, outputs=outlayer)\n",
        "  return model\n",
        "# Build and compile the model\n",
        "model = build_model()\n",
        "adam_optimizer = Adam(learning_rate=0.0001)\n",
        "model.compile(optimizer=adam_optimizer,\n",
        "              loss='binary_crossentropy',\n",
        "              metrics=['accuracy', Precision(), Recall()])\n",
        "\n",
        "# Early stopping callback\n",
        "early_stopping = tf.keras.callbacks.EarlyStopping(\n",
        "    monitor='val_loss',\n",
        "    patience=10,\n",
        "    restore_best_weights=True\n",
        ")\n",
        "\n",
        "# Fit model on training data\n",
        "history = model.fit(\n",
        "    train_data,\n",
        "    epochs=70,\n",
        "    validation_data=valid_data,\n",
        "    callbacks=[early_stopping]\n",
        ")\n",
        "\n",
        "# Predictions and evaluation on validation data\n",
        "valid_predictions = model.predict(valid_x)\n",
        "auroc_score = roc_auc_score(valid_y, valid_predictions)\n",
        "print(f'Validation AUROC: {auroc_score}')\n",
        "\n",
        "# Model summary\n",
        "model.summary()\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.18"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}