feedforward neural network from scratch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Activation function\n",
    "def sigmoid(z):\n",
    "    return 1/(1+np.exp(-z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to initialize parameters\n",
    "def initialize_parameters(inputUnits, layerUnits):\n",
    "    w = np.random.randn(layerUnits, inputUnits)*0.01\n",
    "    b = np.zeros((layerUnits, 1))\n",
    "    \n",
    "    return w, b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to add layers to the neural network\n",
    "def add_layer(model, inputUnits, layerUnits):\n",
    "    model.append({\"w\":initialize_parameters(inputUnits, layerUnits)[0],        \n",
    "                \"b\":initialize_parameters(inputUnits, layerUnits)[1]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to propagate forward and backword through the neural network\n",
    "def propagate(model, X, Y):\n",
    "    Z = []\n",
    "    A = []\n",
    "    # forward propagation\n",
    "    for i in range(len(model)):\n",
    "        W = model[i][\"w\"]\n",
    "        b = model[i][\"b\"]\n",
    "\n",
    "        if i == 0: \n",
    "            Z.append(sigmoid(np.dot(W, X) + b))\n",
    "            A.append(np.dot(W, X) + b)\n",
    "        else:\n",
    "            Z.append(sigmoid(np.dot(W, Z[i-1]) + b))\n",
    "            A.append(np.dot(W, Z[i-1]) + b)\n",
    "\n",
    "    m = X.shape[1]\n",
    "    cost = -1/m*(np.sum(Y*np.log(Z[len(model)-1])+(1-Y)*np.log(1-Z[len(model)-1])))\n",
    "\n",
    "    # backward propagation\n",
    "    dA = []\n",
    "    dZ = []\n",
    "    dW = []\n",
    "    db = []\n",
    "\n",
    "    for i in range(len(model)-1, -1, -1):\n",
    "\n",
    "        if i == len(model) - 1:\n",
    "            dA.insert(0, Z[i] - Y)\n",
    "        else:\n",
    "            W = model[i+1][\"w\"]\n",
    "            dA.insert(0, np.dot(W.T, dA[0])*(sigmoid(A[i])*(1-sigmoid(A[i]))))\n",
    "            \n",
    "        if i != 0:\n",
    "            dW.insert(0, 1/m*np.dot(dA[0], Z[i-1].T))\n",
    "        else:\n",
    "            dW.insert(0, 1/m*np.dot(dA[0], X.T))\n",
    "        \n",
    "        db.insert(0, 1/m*np.sum(dA[0], axis=1, keepdims=True))\n",
    "    \n",
    "        grads = {\"dW\": dW, \"db\": db}\n",
    "    return grads, cost "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to find optimal value of parameters (weights and bias)\n",
    "def optimize(model, X, Y, iterations, learning_rate, print_cost=False):\n",
    "    costs = []\n",
    "\n",
    "    for i in range(iterations):\n",
    "        grads, cost = propagate(model, X, Y)\n",
    "\n",
    "        dW = grads[\"dW\"]\n",
    "        db = grads[\"db\"]\n",
    "\n",
    "        for j in range(len(model)):\n",
    "            model[j][\"w\"] -= learning_rate*dW[j]\n",
    "            model[j][\"b\"] -= learning_rate*db[j] \n",
    "\n",
    "        if i%100 == 0:\n",
    "            costs.append(cost)\n",
    "    \n",
    "        if print_cost and i%100 == 0:\n",
    "            print(\"Cost at the end of %i th iteration is: %.4f\" %(i, cost))\n",
    "\n",
    "    return model, grads, costs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predicting output for the give value of model parameters\n",
    "def predict(model, X):\n",
    "\n",
    "    for i in range(len(model)):\n",
    "        W = model[i][\"w\"]\n",
    "        b = model[i][\"b\"]\n",
    "\n",
    "        if i == 0:\n",
    "            Z = sigmoid(np.dot(W, X) + b)\n",
    "        else:\n",
    "            Z = sigmoid(np.dot(W, Z) + b)\n",
    "    \n",
    "    Y_prediction = (Z > .5)*1\n",
    "    \n",
    "    return Y_prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A function to define model\n",
    "def model(X_train, Y_train, X_test, Y_test, iterations = 2000, learning_rate = 0.5, print_cost=False):\n",
    "    NN = []\n",
    "    # Define the layers here \n",
    "    add_layer(NN, X_train.shape[0], 32)\n",
    "    add_layer(NN, 32, 1)\n",
    "    \n",
    "    NN, grads, costs = optimize(NN, X_train, Y_train, iterations, learning_rate, print_cost)\n",
    "\n",
    "    test_predictions = predict(NN, X_test)\n",
    "    train_predictions = predict(NN, X_train)\n",
    "\n",
    "    print(\"train accuracy is: {} %\".format(100 - np.mean(np.abs(train_predictions - Y_train)) * 100))\n",
    "    print(\"test accuracy is: {} %\".format(100 - np.mean(np.abs(test_predictions - Y_test)) * 100))\n",
    "    \n",
    "    results = {\"costs\": costs,\n",
    "               \"test_predictions\": test_predictions, \n",
    "               \"train_predictions\" : train_predictions, \n",
    "               \"NN\" : NN,\n",
    "               \"learning_rate\" : learning_rate,\n",
    "               \"iterations\": iterations}\n",
    "    \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}