Logistic Regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learned weights (w): [ 4.1477231  -2.13970423]\n",
      "Learned bias (b): -3.8604997547629063\n",
      "Predictions on X_test: [0 1]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "class LogisticRegressionScratch:\n",
    "    def __init__(self, learning_rate=0.01, num_iterations=1000):\n",
    "        \"\"\"\n",
    "        :param learning_rate: 学习率\n",
    "        :param num_iterations: 梯度下降的最大迭代次数\n",
    "        \"\"\"\n",
    "        self.learning_rate = learning_rate\n",
    "        self.num_iterations = num_iterations\n",
    "        self.w = None  # 模型参数 w\n",
    "        self.b = 0.0   # 模型偏置 b\n",
    "\n",
    "    def sigmoid(self, z):\n",
    "        \"\"\"\n",
    "        sigmoid 函数，将 z 映射到 (0,1)\n",
    "        \"\"\"\n",
    "        return 1 / (1 + np.exp(-z))\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        \"\"\"\n",
    "        训练逻辑回归模型\n",
    "        :param X: 训练数据特征，形状为 (m, n)\n",
    "        :param y: 标签向量，形状为 (m,)\n",
    "        \"\"\"\n",
    "        m, n = X.shape\n",
    "        # 初始化 w, b\n",
    "        self.w = np.zeros(n)\n",
    "        self.b = 0.0\n",
    "\n",
    "        for i in range(self.num_iterations):\n",
    "            # 1) 计算线性部分 z = w^T x + b\n",
    "            z = np.dot(X, self.w) + self.b\n",
    "            # 2) 计算预测值 (概率) A = sigmoid(z)\n",
    "            A = self.sigmoid(z)\n",
    "\n",
    "            # 3) 计算损失 (可选：如果需要查看损失曲线，可在此打印或存储)\n",
    "            # 交叉熵损失\n",
    "            # loss = -(1/m) * np.sum(y*np.log(A) + (1-y)*np.log(1-A))\n",
    "\n",
    "            # 4) 计算梯度\n",
    "            # dw = (1/m) * X^T (A - y)\n",
    "            dw = (1/m) * np.dot(X.T, (A - y))\n",
    "            # db = (1/m) * sum(A - y)\n",
    "            db = (1/m) * np.sum(A - y)\n",
    "\n",
    "            # 5) 参数更新\n",
    "            self.w -= self.learning_rate * dw\n",
    "            self.b -= self.learning_rate * db\n",
    "\n",
    "        return self\n",
    "\n",
    "    def predict_proba(self, X):\n",
    "        \"\"\"\n",
    "        返回预测的概率值，形状为 (m,)\n",
    "        :param X: 测试/预测数据，形状为 (m, n)\n",
    "        \"\"\"\n",
    "        z = np.dot(X, self.w) + self.b\n",
    "        return self.sigmoid(z)\n",
    "\n",
    "    def predict(self, X, threshold=0.5):\n",
    "        \"\"\"\n",
    "        返回二分类预测结果（0 or 1）\n",
    "        :param X: 测试/预测数据，形状为 (m, n)\n",
    "        :param threshold: 判定阈值，默认 0.5\n",
    "        \"\"\"\n",
    "        proba = self.predict_proba(X)\n",
    "        return (proba >= threshold).astype(int)\n",
    "\n",
    "\n",
    "# ====================\n",
    "# 以下是一个简单的使用示例\n",
    "# ====================\n",
    "if __name__ == \"__main__\":\n",
    "    # 构造一份小规模的虚拟数据集\n",
    "    # 假设有两维特征 (x1, x2)\n",
    "    X = np.array([\n",
    "        [0.50, 1.0],\n",
    "        [1.50, 2.0],\n",
    "        [3.00, 6.0],\n",
    "        [4.00, 6.5],\n",
    "        [4.50, 6.0],\n",
    "        [5.00, 7.0],\n",
    "        [6.00, 8.0],\n",
    "        [7.00, 8.5]\n",
    "    ])\n",
    "    # 对应标签\n",
    "    y = np.array([0, 0, 0, 0, 1, 1, 1, 1])\n",
    "\n",
    "    # 初始化并训练模型\n",
    "    model = LogisticRegressionScratch(learning_rate=0.1, num_iterations=1000)\n",
    "    model.fit(X, y)\n",
    "\n",
    "    # 打印训练得到的模型参数\n",
    "    print(\"Learned weights (w):\", model.w)\n",
    "    print(\"Learned bias (b):\", model.b)\n",
    "\n",
    "    # 测试一下预测\n",
    "    X_test = np.array([\n",
    "        [2.0, 4.0],  # 可能是类 0\n",
    "        [6.5, 8.0],  # 可能是类 1\n",
    "    ])\n",
    "    preds = model.predict(X_test)\n",
    "    print(\"Predictions on X_test:\", preds)  # 预测结果\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=== 使用交叉熵损失训练 ===\n",
      "迭代 0: loss = 0.69315\n",
      "迭代 100: loss = 0.29066\n",
      "迭代 200: loss = 0.21870\n",
      "迭代 300: loss = 0.18497\n",
      "迭代 400: loss = 0.16433\n",
      "迭代 500: loss = 0.15001\n",
      "迭代 600: loss = 0.13931\n",
      "迭代 700: loss = 0.13091\n",
      "迭代 800: loss = 0.12408\n",
      "迭代 900: loss = 0.11838\n",
      "训练结束。最终参数 w = [ 4.76808035 -2.67622331]\n",
      "在训练集上的准确率: 0.990\n",
      "\n",
      "=== 使用均方误差损失训练 ===\n",
      "迭代 0: loss = 0.25000\n",
      "迭代 100: loss = 0.11400\n",
      "迭代 200: loss = 0.08494\n",
      "迭代 300: loss = 0.07149\n",
      "迭代 400: loss = 0.06332\n",
      "迭代 500: loss = 0.05766\n",
      "迭代 600: loss = 0.05345\n",
      "迭代 700: loss = 0.05015\n",
      "迭代 800: loss = 0.04748\n",
      "迭代 900: loss = 0.04525\n",
      "训练结束。最终参数 w = [ 2.93018933 -1.67307736]\n",
      "在训练集上的准确率: 0.990\n",
      "\n",
      "=== 使用hinge loss训练 ===\n",
      "迭代 0: loss = 1.00000\n",
      "迭代 100: loss = 0.16382\n",
      "迭代 200: loss = 0.12763\n",
      "迭代 300: loss = 0.11155\n",
      "迭代 400: loss = 0.10343\n",
      "迭代 500: loss = 0.09729\n",
      "迭代 600: loss = 0.09221\n",
      "迭代 700: loss = 0.08812\n",
      "迭代 800: loss = 0.08452\n",
      "迭代 900: loss = 0.08156\n",
      "训练结束。最终参数 w = [ 4.05079356 -2.37794326]\n",
      "在训练集上的准确率: 0.990\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "def sigmoid(z):\n",
    "    \"\"\"\n",
    "    Sigmoid 函数，将实数映射到 (0, 1)，常用于二分类问题中输出概率值。\n",
    "    \"\"\"\n",
    "    return 1 / (1 + np.exp(-z))\n",
    "\n",
    "def cross_entropy_loss(y_true, y_pred, eps=1e-15):\n",
    "    \"\"\"\n",
    "    交叉熵损失（Cross Entropy Loss）\n",
    "    y_true: 真实标签, shape = (n_samples, )\n",
    "    y_pred: 预测概率, shape = (n_samples, )\n",
    "    eps: 避免log(0)的平滑项\n",
    "    \"\"\"\n",
    "    # clip 操作是为了防止 log(0) 导致的数值不稳定\n",
    "    y_pred = np.clip(y_pred, eps, 1 - eps)\n",
    "    # 交叉熵损失的公式\n",
    "    # L = -1/N * Σ[ y_true*log(y_pred) + (1 - y_true)*log(1 - y_pred) ]\n",
    "    n = len(y_true)\n",
    "    loss = -np.sum(y_true * np.log(y_pred) + (1 - y_true)*np.log(1 - y_pred)) / n\n",
    "    return loss\n",
    "\n",
    "def cross_entropy_gradient(X, y_true, y_pred):\n",
    "    \"\"\"\n",
    "    交叉熵损失的梯度（相对于模型参数 w 的偏导）。\n",
    "    X: 输入特征矩阵, shape = (n_samples, n_features)\n",
    "    y_true: 真实标签, shape = (n_samples, )\n",
    "    y_pred: 预测概率, shape = (n_samples, )\n",
    "    return: 形状为 (n_features, ) 的梯度\n",
    "    \"\"\"\n",
    "    # gradient = (1/N) * Σ( (y_pred - y_true) * X )\n",
    "    # 其中每个样本对 w 的偏导是 (y_pred[i] - y_true[i]) * X[i]\n",
    "    n = len(y_true)\n",
    "    grad = np.dot(X.T, (y_pred - y_true)) / n\n",
    "    return grad\n",
    "\n",
    "def mse_loss(y_true, y_pred):\n",
    "    \"\"\"\n",
    "    均方误差（Mean Squared Error, MSE）损失\n",
    "    \"\"\"\n",
    "    n = len(y_true)\n",
    "    loss = np.mean((y_pred - y_true) ** 2)\n",
    "    return loss\n",
    "\n",
    "def mse_gradient(X, y_true, y_pred):\n",
    "    \"\"\"\n",
    "    MSE 对应的梯度\n",
    "    注意：这里的 y_pred 是概率值，但在计算 MSE 时假设其为模型输出，\n",
    "    因此计算梯度时会跟交叉熵略有不同。\n",
    "    \"\"\"\n",
    "    n = len(y_true)\n",
    "    # 对 w 的偏导: (2/N) * Σ( (y_pred - y_true) * y_pred*(1-y_pred) * X )\n",
    "    # 其中 y_pred*(1-y_pred) 是 sigmoid 函数本身对 z 的偏导数\n",
    "    # 因为 logistic 回归最终对 z 的导数 = 对 y_pred 的导数 * 对 w 的导数\n",
    "    grad = np.dot(X.T, (y_pred - y_true) * y_pred * (1 - y_pred)) * (2 / n)\n",
    "    return grad\n",
    "\n",
    "def hinge_loss(y_true, y_pred, eps=1e-15):\n",
    "    \"\"\"\n",
    "    Hinge Loss 常用于支持向量机，也可以用于逻辑回归的参考。\n",
    "    要求 y_true ∈ { -1, 1 }。\n",
    "    这里我们需要将 {0,1} 标签转换为 { -1,1 } 后再计算，或者直接根据实际情况做映射。\n",
    "    公式: L = 1/N * Σ max(0, 1 - y_true * z)\n",
    "    其中 z = Xw, 为线性输出, 若我们使用 y_pred = sigmoid(z) 则需要单独处理。\n",
    "    这里为了演示，简单直接使用 z = logit^-1(y_pred) = log(y_pred / (1 - y_pred)) 逆过来做示意。\n",
    "    \"\"\"\n",
    "    # 先把标签从0/1转换成-1/1\n",
    "    y_true_transformed = np.where(y_true > 0.5, 1, -1)\n",
    "    \n",
    "    # 这里假设 y_pred 是概率，先将概率转成 logit (z)\n",
    "    # z = log( p / (1-p) ), 注意要clip防止出现除 0\n",
    "    y_pred_clip = np.clip(y_pred, eps, 1 - eps)\n",
    "    z = np.log(y_pred_clip / (1 - y_pred_clip))\n",
    "    \n",
    "    # hinge loss\n",
    "    loss = np.mean(np.maximum(0, 1 - y_true_transformed * z))\n",
    "    return loss\n",
    "\n",
    "def hinge_gradient(X, y_true, y_pred, eps=1e-15):\n",
    "    \"\"\"\n",
    "    计算 hinge loss 对 w 的梯度。\n",
    "    hinge loss 的公式: max(0, 1 - y_true*z), z = Xw\n",
    "    在 y_true*z < 1 的情况下, 对 w 的梯度 = -y_true * X\n",
    "    注意 y_true ∈ { -1, 1 }，这里先要转换一下标签，然后再根据是否满足 y_true*z < 1 来更新。\n",
    "    \"\"\"\n",
    "    y_true_transformed = np.where(y_true > 0.5, 1, -1)\n",
    "    \n",
    "    # 将 y_pred 从概率空间映射回 z = Xw\n",
    "    y_pred_clip = np.clip(y_pred, eps, 1 - eps)\n",
    "    z = np.log(y_pred_clip / (1 - y_pred_clip))\n",
    "    \n",
    "    # 判断哪些样本满足 y_true * z < 1\n",
    "    mask = (y_true_transformed * z) < 1\n",
    "    # 如果满足 y_true*z < 1, 梯度= -y_true*X；否则梯度= 0\n",
    "    # 这里是对所有样本的和再除以 n\n",
    "    grad = np.zeros(X.shape[1])\n",
    "    for i in range(X.shape[0]):\n",
    "        if mask[i]:\n",
    "            grad += -y_true_transformed[i] * X[i]\n",
    "    grad /= X.shape[0]\n",
    "    \n",
    "    return grad\n",
    "\n",
    "\n",
    "class LogisticRegression:\n",
    "    def __init__(self, learning_rate=0.1, max_iter=1000, loss_type='cross_entropy'):\n",
    "        \"\"\"\n",
    "        Logistic Regression 模型\n",
    "        learning_rate: 学习率\n",
    "        max_iter: 最大迭代次数\n",
    "        loss_type: 损失函数类型，可选：'cross_entropy', 'mse', 'hinge'\n",
    "        \"\"\"\n",
    "        self.learning_rate = learning_rate\n",
    "        self.max_iter = max_iter\n",
    "        self.loss_type = loss_type\n",
    "        self.w = None  # 模型参数\n",
    "        \n",
    "    def fit(self, X, y):\n",
    "        \"\"\"\n",
    "        拟合模型参数 w。\n",
    "        X: 训练集特征, shape = (n_samples, n_features)\n",
    "        y: 训练集标签, shape = (n_samples, ), 取值{0,1}\n",
    "        \"\"\"\n",
    "        n_samples, n_features = X.shape\n",
    "        # 初始化参数 w\n",
    "        self.w = np.zeros(n_features)\n",
    "        \n",
    "        for i in range(self.max_iter):\n",
    "            # 计算线性输出 z = Xw\n",
    "            z = np.dot(X, self.w)\n",
    "            # 计算预测概率 y_pred = sigmoid(z)\n",
    "            y_pred = sigmoid(z)\n",
    "            \n",
    "            # 根据不同的损失函数计算 loss 和 gradient\n",
    "            if self.loss_type == 'cross_entropy':\n",
    "                loss = cross_entropy_loss(y, y_pred)\n",
    "                grad = cross_entropy_gradient(X, y, y_pred)\n",
    "            elif self.loss_type == 'mse':\n",
    "                loss = mse_loss(y, y_pred)\n",
    "                grad = mse_gradient(X, y, y_pred)\n",
    "            elif self.loss_type == 'hinge':\n",
    "                loss = hinge_loss(y, y_pred)\n",
    "                grad = hinge_gradient(X, y, y_pred)\n",
    "            else:\n",
    "                raise ValueError(f\"未知的损失函数类型：{self.loss_type}\")\n",
    "            \n",
    "            # 参数更新\n",
    "            self.w -= self.learning_rate * grad\n",
    "            \n",
    "            if i % 100 == 0:\n",
    "                print(f\"迭代 {i}: loss = {loss:.5f}\")\n",
    "\n",
    "    def predict_proba(self, X):\n",
    "        \"\"\"\n",
    "        预测属于正类（1）的概率\n",
    "        \"\"\"\n",
    "        z = np.dot(X, self.w)\n",
    "        return sigmoid(z)\n",
    "    \n",
    "    def predict(self, X, threshold=0.5):\n",
    "        \"\"\"\n",
    "        预测标签，返回0或1\n",
    "        \"\"\"\n",
    "        return (self.predict_proba(X) >= threshold).astype(int)\n",
    "\n",
    "# ------------------ 测试示例 ------------------\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    # 构造一个简单的二分类任务\n",
    "    np.random.seed(42)\n",
    "    # 生成 100 个样本, 2 个特征\n",
    "    X = np.random.randn(100, 2)\n",
    "    # 人为构造标签: w_true = [2, -1], b=0\n",
    "    w_true = np.array([2.0, -1.0])\n",
    "    z_true = np.dot(X, w_true)\n",
    "    y = (z_true > 0).astype(int)  # 大于0则为正类1，否则为负类0\n",
    "    \n",
    "    print(\"=== 使用交叉熵损失训练 ===\")\n",
    "    model_ce = LogisticRegression(learning_rate=0.1, max_iter=1000, loss_type='cross_entropy')\n",
    "    model_ce.fit(X, y)\n",
    "    print(\"训练结束。最终参数 w =\", model_ce.w)\n",
    "    acc_ce = np.mean(model_ce.predict(X) == y)\n",
    "    print(f\"在训练集上的准确率: {acc_ce:.3f}\\n\")\n",
    "\n",
    "    print(\"=== 使用均方误差损失训练 ===\")\n",
    "    model_mse = LogisticRegression(learning_rate=0.1, max_iter=1000, loss_type='mse')\n",
    "    model_mse.fit(X, y)\n",
    "    print(\"训练结束。最终参数 w =\", model_mse.w)\n",
    "    acc_mse = np.mean(model_mse.predict(X) == y)\n",
    "    print(f\"在训练集上的准确率: {acc_mse:.3f}\\n\")\n",
    "\n",
    "    print(\"=== 使用hinge loss训练 ===\")\n",
    "    model_hinge = LogisticRegression(learning_rate=0.1, max_iter=1000, loss_type='hinge')\n",
    "    model_hinge.fit(X, y)\n",
    "    print(\"训练结束。最终参数 w =\", model_hinge.w)\n",
    "    acc_hinge = np.mean(model_hinge.predict(X) == y)\n",
    "    print(f\"在训练集上的准确率: {acc_hinge:.3f}\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}