From 1f23b7fa18e37bf579c80f167f8370bdc34775f0 Mon Sep 17 00:00:00 2001 From: Eljan Mahammadli Date: Mon, 6 Nov 2023 20:17:14 -0500 Subject: [PATCH] added stable version of the softmax link --- gradipy/tensor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gradipy/tensor.py b/gradipy/tensor.py index 3e091a5..0c2f5e8 100644 --- a/gradipy/tensor.py +++ b/gradipy/tensor.py @@ -97,6 +97,8 @@ def _backward() -> None: def matmul(self, other: "Tensor") -> "Tensor": return self @ other + # there is more stable way to compute softmax :D + # https://ogunlao.github.io/2020/04/26/you_dont_really_know_softmax.html def softmax(self) -> "Tensor": exps = np.exp(self.data - np.max(self.data, axis=1, keepdims=True)) probs = exps / np.sum(exps, axis=1, keepdims=True)