.gitignore

    def gradient_checking(self, epsilon=1e-7):
        grad_approx = []
        grad = np.array([])
        count = 0
        # Preparing necessary analytic gradient values (only require dW and dB)
        for i in self.grads.keys():  # Adding every weight and bias (activations and outputs excluded)
            if i.startswith("dW") or i.startswith("dB"):
                new_vector = np.reshape(self.grads[i], (-1, 1))
                if count == 0:
                    grad = new_vector
                else:
                    grad = np.concatenate((grad, new_vector), axis=0)
                count = count + 1
        grad = np.array(grad)  # Array of gradients to compare to approximated gradients

        # Building the numerical gradient approximations
        for i in self.params.keys():
            for idx in np.ndindex(self.params[i].shape):
                thetaplus = self.params[i][idx] + epsilon  # calculating theta plus for each parameter
                modified_params = self.params.copy()
                modified_params[i][idx] = thetaplus
                # testing network based on modified params
                output = self.forward_propagation(self.X, modified_params)
                J_Plus = losses.cross_entropy(output, self.Y)

                thetaminus = self.params[i][idx] - epsilon
                modified_params = self.params.copy()
                modified_params[i][idx] = thetaminus
                output = self.forward_propagation(self.X, modified_params)
                J_Minus = losses.cross_entropy(output, self.Y)
                # Adding the approximation to a list
                grad_approx.append((J_Plus - J_Minus) / (2 * epsilon))
        grad_approx = np.array(grad_approx).reshape(-1, 1)
        # Comparing values for debugging
        # for i in range(0, grad.shape[0]):
        #    print("Value: {}, Real value: {}".format(grad[i], grad_approx[i]))

        # Calculating relative error
        numerator = np.linalg.norm(grad - grad_approx)  # Step 1'
        denominator = np.linalg.norm(grad) + np.linalg.norm(grad_approx)  # Step 2'
        difference = numerator / denominator  # Step 3'

        if difference > 2e-7:
            print("\033[93m" + "There is a mistake in the backward propagation! difference = " + str(
                difference) + "\033[0m")
        else:
            print("\033[92m" + "Backward propagation works perfectly fine! difference = " + str(
                difference) + "\033[0m")