From c7f0a1a8671a171818e4692361f62dead1dcc514 Mon Sep 17 00:00:00 2001 From: Kazuki Adachi Date: Thu, 14 Mar 2024 19:44:25 +0900 Subject: [PATCH 01/21] add cosine similarity --- ignite/contrib/metrics/__init__.py | 1 + ignite/contrib/metrics/cosine_similarity.py | 97 +++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 ignite/contrib/metrics/cosine_similarity.py diff --git a/ignite/contrib/metrics/__init__.py b/ignite/contrib/metrics/__init__.py index a9163ea6a73..1d16545797d 100644 --- a/ignite/contrib/metrics/__init__.py +++ b/ignite/contrib/metrics/__init__.py @@ -4,3 +4,4 @@ from ignite.contrib.metrics.gpu_info import GpuInfo from ignite.contrib.metrics.precision_recall_curve import PrecisionRecallCurve from ignite.contrib.metrics.roc_auc import ROC_AUC, RocCurve +from ignite.contrib.metrics.cosine_similarity import CosineSimilarity diff --git a/ignite/contrib/metrics/cosine_similarity.py b/ignite/contrib/metrics/cosine_similarity.py new file mode 100644 index 00000000000..1ad77d47f90 --- /dev/null +++ b/ignite/contrib/metrics/cosine_similarity.py @@ -0,0 +1,97 @@ +from typing import Sequence, Union, Callable + +import torch + +from ignite.exceptions import NotComputableError +from ignite.metrics.metric import Metric, reinit__is_reduced, sync_all_reduce + +__all__ = ["CosineSimilarity"] + + +class CosineSimilarity(Metric): + r"""Calculates the mean of the `cosine similarity `_. + + .. math:: \text{cosine\_similarity} = \frac{1}{N} \sum_{i=1}^N \frac{x_i \cdot y_i}{\max ( \| x_i \|_2 \| y_i \|_2 , \epsilon)} + + where :math:`y_{i}` is the prediction tensor and :math:`x_{i}` is ground true tensor. + + - ``update`` must receive output of the form ``(y_pred, y)``. + + Args: + eps: a small value to avoid division by zero. Default: 1e-8 + output_transform: a callable that is used to transform the + :class:`~ignite.engine.engine.Engine`'s ``process_function``'s output into the + form expected by the metric. This can be useful if, for example, you have a multi-output model and + you want to compute the metric with respect to one of the outputs. + By default, metrics require the output as ``(y_pred, y)`` or ``{'y_pred': y_pred, 'y': y}``. + device: specifies which device updates are accumulated on. Setting the + metric's device to be the same as your ``update`` arguments ensures the ``update`` method is + non-blocking. By default, CPU. + + Examples: + To use with ``Engine`` and ``process_function``, simply attach the metric instance to the engine. + The output of the engine's ``process_function`` needs to be in the format of + ``(y_pred, y)`` or ``{'y_pred': y_pred, 'y': y, ...}``. If not, ``output_tranform`` can be added + to the metric to transform the output into the form expected by the metric. + + ``y_pred`` and ``y`` should have the same shape. + + For more information on how metric works with :class:`~ignite.engine.engine.Engine`, visit :ref:`attach-engine`. + + .. include:: defaults.rst + :start-after: :orphan: + + .. testcode:: + + metric = CosineSimilarity() + metric.attach(default_evaluator, 'cosine_similarity') + preds = torch.tensor([ + [1, 2, 4, 1], + [2, 3, 1, 5], + [1, 3, 5, 1], + [1, 5, 1 ,11] + ]).float() + target = torch.tensor([ + [1, 5, 1 ,11], + [1, 3, 5, 1], + [2, 3, 1, 5], + [1, 2, 4, 1] + ]).float() + state = default_evaluator.run([[preds, target]]) + print(state.metrics['cosine_similarity']) + + .. testoutput:: + + 0.5080491304397583 + """ + + def __init__( + self, + eps: float = 1e-8, + output_transform: Callable = lambda x: x, + device: Union[str, torch.device] = torch.device("cpu") + ): + super().__init__(output_transform, device) + + self.eps = eps + + _state_dict_all_req_keys = ("_sum_of_cos_similarities", "_num_examples") + + @reinit__is_reduced + def reset(self) -> None: + self._sum_of_cos_similarities = torch.tensor(0.0, device=self._device) + self._num_examples = 0 + + @reinit__is_reduced + def update(self, output: Sequence[torch.Tensor]) -> None: + y_pred = output[0].flatten(start_dim=1).detach() + y = output[1].flatten(start_dim=1).detach() + cos_similarities = torch.nn.functional.cosine_similarity(y_pred, y, dim=1, eps=self.eps) + self._sum_of_cos_similarities += torch.sum(cos_similarities).to(self._device) + self._num_examples += y.shape[0] + + @sync_all_reduce("_sum_of_cos_similarities", "_num_examples") + def compute(self) -> Union[float, torch.Tensor]: + if self._num_examples == 0: + raise NotComputableError("CosineSimilarity must have at least one example before it can be computed.") + return self._sum_of_cos_similarities.item() / self._num_examples From 9c87580d58a10831a5037c69c563c168ff0e4725 Mon Sep 17 00:00:00 2001 From: Kazuki Adachi Date: Thu, 14 Mar 2024 19:56:39 +0900 Subject: [PATCH 02/21] update doc for cosine similarity metric --- docs/source/contrib/metrics.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/contrib/metrics.rst b/docs/source/contrib/metrics.rst index eccaf9e7808..dfced004b49 100644 --- a/docs/source/contrib/metrics.rst +++ b/docs/source/contrib/metrics.rst @@ -40,6 +40,7 @@ Complete list of metrics: :toctree: ../generated CanberraMetric + CosineSimilarity FractionalAbsoluteError FractionalBias GeometricMeanAbsoluteError From 12a4d56fa571c34800c352be10e12d0878508c92 Mon Sep 17 00:00:00 2001 From: Kazuki Adachi Date: Thu, 14 Mar 2024 20:42:59 +0900 Subject: [PATCH 03/21] fix the position of the CosineSimilarity --- docs/source/contrib/metrics.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/contrib/metrics.rst b/docs/source/contrib/metrics.rst index dfced004b49..a2227e25411 100644 --- a/docs/source/contrib/metrics.rst +++ b/docs/source/contrib/metrics.rst @@ -12,6 +12,7 @@ Contrib module metrics AveragePrecision CohenKappa + CosineSimilarity GpuInfo PrecisionRecallCurve ROC_AUC @@ -40,7 +41,6 @@ Complete list of metrics: :toctree: ../generated CanberraMetric - CosineSimilarity FractionalAbsoluteError FractionalBias GeometricMeanAbsoluteError From e09b86fded74707b48ef3ab2a657701dcffc357b Mon Sep 17 00:00:00 2001 From: Kazuki Adachi Date: Mon, 18 Mar 2024 15:19:33 +0900 Subject: [PATCH 04/21] Update ignite/contrib/metrics/cosine_similarity.py Co-authored-by: vfdev --- ignite/contrib/metrics/cosine_similarity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ignite/contrib/metrics/cosine_similarity.py b/ignite/contrib/metrics/cosine_similarity.py index 1ad77d47f90..ed543dac2a8 100644 --- a/ignite/contrib/metrics/cosine_similarity.py +++ b/ignite/contrib/metrics/cosine_similarity.py @@ -91,7 +91,7 @@ def update(self, output: Sequence[torch.Tensor]) -> None: self._num_examples += y.shape[0] @sync_all_reduce("_sum_of_cos_similarities", "_num_examples") - def compute(self) -> Union[float, torch.Tensor]: + def compute(self) -> float: if self._num_examples == 0: raise NotComputableError("CosineSimilarity must have at least one example before it can be computed.") return self._sum_of_cos_similarities.item() / self._num_examples From 99852d50f2ef02641656a516b7ab16c96f4a455d Mon Sep 17 00:00:00 2001 From: kzkadc Date: Mon, 18 Mar 2024 06:20:06 +0000 Subject: [PATCH 05/21] autopep8 fix --- ignite/contrib/metrics/__init__.py | 2 +- ignite/contrib/metrics/cosine_similarity.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ignite/contrib/metrics/__init__.py b/ignite/contrib/metrics/__init__.py index 1d16545797d..c3038ab5978 100644 --- a/ignite/contrib/metrics/__init__.py +++ b/ignite/contrib/metrics/__init__.py @@ -1,7 +1,7 @@ import ignite.contrib.metrics.regression from ignite.contrib.metrics.average_precision import AveragePrecision from ignite.contrib.metrics.cohen_kappa import CohenKappa +from ignite.contrib.metrics.cosine_similarity import CosineSimilarity from ignite.contrib.metrics.gpu_info import GpuInfo from ignite.contrib.metrics.precision_recall_curve import PrecisionRecallCurve from ignite.contrib.metrics.roc_auc import ROC_AUC, RocCurve -from ignite.contrib.metrics.cosine_similarity import CosineSimilarity diff --git a/ignite/contrib/metrics/cosine_similarity.py b/ignite/contrib/metrics/cosine_similarity.py index ed543dac2a8..71c8ce2cb7a 100644 --- a/ignite/contrib/metrics/cosine_similarity.py +++ b/ignite/contrib/metrics/cosine_similarity.py @@ -1,4 +1,4 @@ -from typing import Sequence, Union, Callable +from typing import Callable, Sequence, Union import torch @@ -69,7 +69,7 @@ def __init__( self, eps: float = 1e-8, output_transform: Callable = lambda x: x, - device: Union[str, torch.device] = torch.device("cpu") + device: Union[str, torch.device] = torch.device("cpu"), ): super().__init__(output_transform, device) From 54c16c11525629a85e32f2921fe8092dccceca8d Mon Sep 17 00:00:00 2001 From: Kazuki Adachi Date: Mon, 18 Mar 2024 20:51:13 +0900 Subject: [PATCH 06/21] move CosineSimilarity from contrib.metrics to metrics --- docs/source/contrib/metrics.rst | 1 - docs/source/metrics.rst | 1 + ignite/contrib/metrics/__init__.py | 1 - ignite/metrics/__init__.py | 2 ++ ignite/{contrib => }/metrics/cosine_similarity.py | 0 5 files changed, 3 insertions(+), 2 deletions(-) rename ignite/{contrib => }/metrics/cosine_similarity.py (100%) diff --git a/docs/source/contrib/metrics.rst b/docs/source/contrib/metrics.rst index a2227e25411..eccaf9e7808 100644 --- a/docs/source/contrib/metrics.rst +++ b/docs/source/contrib/metrics.rst @@ -12,7 +12,6 @@ Contrib module metrics AveragePrecision CohenKappa - CosineSimilarity GpuInfo PrecisionRecallCurve ROC_AUC diff --git a/docs/source/metrics.rst b/docs/source/metrics.rst index bd5038f0814..ca0f41661a1 100644 --- a/docs/source/metrics.rst +++ b/docs/source/metrics.rst @@ -350,6 +350,7 @@ Complete list of metrics RougeN InceptionScore FID + CosineSimilarity Helpers for customizing metrics ------------------------------- diff --git a/ignite/contrib/metrics/__init__.py b/ignite/contrib/metrics/__init__.py index c3038ab5978..a9163ea6a73 100644 --- a/ignite/contrib/metrics/__init__.py +++ b/ignite/contrib/metrics/__init__.py @@ -1,7 +1,6 @@ import ignite.contrib.metrics.regression from ignite.contrib.metrics.average_precision import AveragePrecision from ignite.contrib.metrics.cohen_kappa import CohenKappa -from ignite.contrib.metrics.cosine_similarity import CosineSimilarity from ignite.contrib.metrics.gpu_info import GpuInfo from ignite.contrib.metrics.precision_recall_curve import PrecisionRecallCurve from ignite.contrib.metrics.roc_auc import ROC_AUC, RocCurve diff --git a/ignite/metrics/__init__.py b/ignite/metrics/__init__.py index d001436a3ad..5303bdded2a 100644 --- a/ignite/metrics/__init__.py +++ b/ignite/metrics/__init__.py @@ -2,6 +2,7 @@ from ignite.metrics.accuracy import Accuracy from ignite.metrics.classification_report import ClassificationReport from ignite.metrics.confusion_matrix import ConfusionMatrix, DiceCoefficient, IoU, JaccardIndex, mIoU +from ignite.metrics.cosine_similarity import CosineSimilarity from ignite.metrics.epoch_metric import EpochMetric from ignite.metrics.fbeta import Fbeta from ignite.metrics.frequency import Frequency @@ -33,6 +34,7 @@ "MeanPairwiseDistance", "MeanSquaredError", "ConfusionMatrix", + "CosineSimilarity" "ClassificationReport", "TopKCategoricalAccuracy", "Average", diff --git a/ignite/contrib/metrics/cosine_similarity.py b/ignite/metrics/cosine_similarity.py similarity index 100% rename from ignite/contrib/metrics/cosine_similarity.py rename to ignite/metrics/cosine_similarity.py From 9524d20827bb2c4fcb29fe505d060522eb69288f Mon Sep 17 00:00:00 2001 From: kzkadc Date: Mon, 18 Mar 2024 11:52:35 +0000 Subject: [PATCH 07/21] autopep8 fix --- ignite/metrics/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ignite/metrics/__init__.py b/ignite/metrics/__init__.py index 5303bdded2a..78c9c93e79d 100644 --- a/ignite/metrics/__init__.py +++ b/ignite/metrics/__init__.py @@ -34,8 +34,7 @@ "MeanPairwiseDistance", "MeanSquaredError", "ConfusionMatrix", - "CosineSimilarity" - "ClassificationReport", + "CosineSimilarity" "ClassificationReport", "TopKCategoricalAccuracy", "Average", "DiceCoefficient", From b7dd7ea8f22659e08095f53088f723644884427a Mon Sep 17 00:00:00 2001 From: Kazuki Adachi Date: Mon, 18 Mar 2024 20:55:24 +0900 Subject: [PATCH 08/21] fix typo --- ignite/metrics/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ignite/metrics/__init__.py b/ignite/metrics/__init__.py index 5303bdded2a..9d63cfdc4ac 100644 --- a/ignite/metrics/__init__.py +++ b/ignite/metrics/__init__.py @@ -34,7 +34,7 @@ "MeanPairwiseDistance", "MeanSquaredError", "ConfusionMatrix", - "CosineSimilarity" + "CosineSimilarity", "ClassificationReport", "TopKCategoricalAccuracy", "Average", From 52aeaeefc6fae0abbf209aa3244778d3c91c1669 Mon Sep 17 00:00:00 2001 From: Kazuki Adachi Date: Mon, 18 Mar 2024 20:56:59 +0900 Subject: [PATCH 09/21] fix typo --- ignite/metrics/__init__.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ignite/metrics/__init__.py b/ignite/metrics/__init__.py index 7f91880dfe7..9d63cfdc4ac 100644 --- a/ignite/metrics/__init__.py +++ b/ignite/metrics/__init__.py @@ -34,12 +34,8 @@ "MeanPairwiseDistance", "MeanSquaredError", "ConfusionMatrix", -<<<<<<< HEAD "CosineSimilarity", "ClassificationReport", -======= - "CosineSimilarity" "ClassificationReport", ->>>>>>> 9524d20827bb2c4fcb29fe505d060522eb69288f "TopKCategoricalAccuracy", "Average", "DiceCoefficient", From 78b4029f1e49ed01bdfe035c5157e23c921036cf Mon Sep 17 00:00:00 2001 From: Kazuki Adachi Date: Mon, 18 Mar 2024 23:32:59 +0900 Subject: [PATCH 10/21] Update ignite/metrics/cosine_similarity.py Co-authored-by: vfdev --- ignite/metrics/cosine_similarity.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ignite/metrics/cosine_similarity.py b/ignite/metrics/cosine_similarity.py index 71c8ce2cb7a..017d0f66a56 100644 --- a/ignite/metrics/cosine_similarity.py +++ b/ignite/metrics/cosine_similarity.py @@ -86,7 +86,9 @@ def reset(self) -> None: def update(self, output: Sequence[torch.Tensor]) -> None: y_pred = output[0].flatten(start_dim=1).detach() y = output[1].flatten(start_dim=1).detach() - cos_similarities = torch.nn.functional.cosine_similarity(y_pred, y, dim=1, eps=self.eps) + cos_similarities = torch.nn.functional.cosine_similarity( + y_pred, y, dim=1, eps=self.eps + ) self._sum_of_cos_similarities += torch.sum(cos_similarities).to(self._device) self._num_examples += y.shape[0] From 6bfbe8860ed7ce72b882af6faf4eabe310c09b89 Mon Sep 17 00:00:00 2001 From: kzkadc Date: Mon, 18 Mar 2024 14:33:26 +0000 Subject: [PATCH 11/21] autopep8 fix --- ignite/metrics/cosine_similarity.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ignite/metrics/cosine_similarity.py b/ignite/metrics/cosine_similarity.py index 017d0f66a56..71c8ce2cb7a 100644 --- a/ignite/metrics/cosine_similarity.py +++ b/ignite/metrics/cosine_similarity.py @@ -86,9 +86,7 @@ def reset(self) -> None: def update(self, output: Sequence[torch.Tensor]) -> None: y_pred = output[0].flatten(start_dim=1).detach() y = output[1].flatten(start_dim=1).detach() - cos_similarities = torch.nn.functional.cosine_similarity( - y_pred, y, dim=1, eps=self.eps - ) + cos_similarities = torch.nn.functional.cosine_similarity(y_pred, y, dim=1, eps=self.eps) self._sum_of_cos_similarities += torch.sum(cos_similarities).to(self._device) self._num_examples += y.shape[0] From a83e02ce9db21bf6a7546871c97534a5e71979ff Mon Sep 17 00:00:00 2001 From: Kazuki Adachi Date: Mon, 18 Mar 2024 23:43:12 +0900 Subject: [PATCH 12/21] Update ignite/metrics/cosine_similarity.py Co-authored-by: vfdev --- ignite/metrics/cosine_similarity.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ignite/metrics/cosine_similarity.py b/ignite/metrics/cosine_similarity.py index 71c8ce2cb7a..017d0f66a56 100644 --- a/ignite/metrics/cosine_similarity.py +++ b/ignite/metrics/cosine_similarity.py @@ -86,7 +86,9 @@ def reset(self) -> None: def update(self, output: Sequence[torch.Tensor]) -> None: y_pred = output[0].flatten(start_dim=1).detach() y = output[1].flatten(start_dim=1).detach() - cos_similarities = torch.nn.functional.cosine_similarity(y_pred, y, dim=1, eps=self.eps) + cos_similarities = torch.nn.functional.cosine_similarity( + y_pred, y, dim=1, eps=self.eps + ) self._sum_of_cos_similarities += torch.sum(cos_similarities).to(self._device) self._num_examples += y.shape[0] From 9e4fd311d21db570fb4497b74e73357aeac54812 Mon Sep 17 00:00:00 2001 From: Kazuki Adachi Date: Mon, 18 Mar 2024 23:43:32 +0900 Subject: [PATCH 13/21] fix formatting --- ignite/metrics/cosine_similarity.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ignite/metrics/cosine_similarity.py b/ignite/metrics/cosine_similarity.py index 017d0f66a56..91edc00e28a 100644 --- a/ignite/metrics/cosine_similarity.py +++ b/ignite/metrics/cosine_similarity.py @@ -11,7 +11,9 @@ class CosineSimilarity(Metric): r"""Calculates the mean of the `cosine similarity `_. - .. math:: \text{cosine\_similarity} = \frac{1}{N} \sum_{i=1}^N \frac{x_i \cdot y_i}{\max ( \| x_i \|_2 \| y_i \|_2 , \epsilon)} + .. math:: + \text{cosine\_similarity} = \frac{1}{N} \sum_{i=1}^N + \frac{x_i \cdot y_i}{\max ( \| x_i \|_2 \| y_i \|_2 , \epsilon)} where :math:`y_{i}` is the prediction tensor and :math:`x_{i}` is ground true tensor. From 377baea8ed5cc297a2d2c24029a6becc30a35f4f Mon Sep 17 00:00:00 2001 From: kzkadc Date: Mon, 18 Mar 2024 14:43:42 +0000 Subject: [PATCH 14/21] autopep8 fix --- ignite/metrics/cosine_similarity.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ignite/metrics/cosine_similarity.py b/ignite/metrics/cosine_similarity.py index 017d0f66a56..71c8ce2cb7a 100644 --- a/ignite/metrics/cosine_similarity.py +++ b/ignite/metrics/cosine_similarity.py @@ -86,9 +86,7 @@ def reset(self) -> None: def update(self, output: Sequence[torch.Tensor]) -> None: y_pred = output[0].flatten(start_dim=1).detach() y = output[1].flatten(start_dim=1).detach() - cos_similarities = torch.nn.functional.cosine_similarity( - y_pred, y, dim=1, eps=self.eps - ) + cos_similarities = torch.nn.functional.cosine_similarity(y_pred, y, dim=1, eps=self.eps) self._sum_of_cos_similarities += torch.sum(cos_similarities).to(self._device) self._num_examples += y.shape[0] From ad16ce77f334d616d43a66271d0a6906e6dd79a6 Mon Sep 17 00:00:00 2001 From: Kazuki Adachi Date: Mon, 18 Mar 2024 23:45:02 +0900 Subject: [PATCH 15/21] fix formatting --- ignite/metrics/cosine_similarity.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ignite/metrics/cosine_similarity.py b/ignite/metrics/cosine_similarity.py index a9760530ea7..91edc00e28a 100644 --- a/ignite/metrics/cosine_similarity.py +++ b/ignite/metrics/cosine_similarity.py @@ -88,7 +88,9 @@ def reset(self) -> None: def update(self, output: Sequence[torch.Tensor]) -> None: y_pred = output[0].flatten(start_dim=1).detach() y = output[1].flatten(start_dim=1).detach() - cos_similarities = torch.nn.functional.cosine_similarity(y_pred, y, dim=1, eps=self.eps) + cos_similarities = torch.nn.functional.cosine_similarity( + y_pred, y, dim=1, eps=self.eps + ) self._sum_of_cos_similarities += torch.sum(cos_similarities).to(self._device) self._num_examples += y.shape[0] From 6d3193eef9a0a197638ea9d8df3b5b1fcf9018c0 Mon Sep 17 00:00:00 2001 From: kzkadc Date: Mon, 18 Mar 2024 14:46:06 +0000 Subject: [PATCH 16/21] autopep8 fix --- ignite/metrics/cosine_similarity.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ignite/metrics/cosine_similarity.py b/ignite/metrics/cosine_similarity.py index 91edc00e28a..a9760530ea7 100644 --- a/ignite/metrics/cosine_similarity.py +++ b/ignite/metrics/cosine_similarity.py @@ -88,9 +88,7 @@ def reset(self) -> None: def update(self, output: Sequence[torch.Tensor]) -> None: y_pred = output[0].flatten(start_dim=1).detach() y = output[1].flatten(start_dim=1).detach() - cos_similarities = torch.nn.functional.cosine_similarity( - y_pred, y, dim=1, eps=self.eps - ) + cos_similarities = torch.nn.functional.cosine_similarity(y_pred, y, dim=1, eps=self.eps) self._sum_of_cos_similarities += torch.sum(cos_similarities).to(self._device) self._num_examples += y.shape[0] From b2b8de4ee981c3eaf20334cb5dfc37d27da7149d Mon Sep 17 00:00:00 2001 From: Kazuki Adachi Date: Mon, 18 Mar 2024 23:50:53 +0900 Subject: [PATCH 17/21] add test for CosineSimilarity metric --- .../ignite/metrics/test_cosine_similarity.py | 201 ++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 tests/ignite/metrics/test_cosine_similarity.py diff --git a/tests/ignite/metrics/test_cosine_similarity.py b/tests/ignite/metrics/test_cosine_similarity.py new file mode 100644 index 00000000000..072cb36f773 --- /dev/null +++ b/tests/ignite/metrics/test_cosine_similarity.py @@ -0,0 +1,201 @@ +import os + +import numpy as np +import pytest +import torch + +import ignite.distributed as idist +from ignite.exceptions import NotComputableError +from ignite.metrics import CosineSimilarity + + +def test_zero_sample(): + cos_sim = CosineSimilarity() + with pytest.raises( + NotComputableError, match=r"CosineSimilarity must have at least one example before it can be computed" + ): + cos_sim.compute() + + +@pytest.fixture(params=[item for item in range(4)]) +def test_case(request): + return [ + (torch.randn((100, 50)), torch.randn((100, 50)), 10**np.random.uniform(-8, 0), 1), + (torch.normal(1.0, 2.0, size=(100, 10)), torch.normal(3.0, 4.0, size=(100, 10)), 10**np.random.uniform(-8, 0), 1), + # updated batches + (torch.rand((100, 128)), torch.rand((100, 128)), 10**np.random.uniform(-8, 0), 16), + (torch.normal(0.0, 5.0, size=(100, 30)), torch.normal(5.0, 1.0, size=(100, 30)), 10**np.random.uniform(-8, 0), 16), + ][request.param] + + +@pytest.mark.parametrize("n_times", range(5)) +def test_compute(n_times, test_case): + y_pred, y, eps, batch_size = test_case + + cos = CosineSimilarity(eps=eps) + + cos.reset() + if batch_size > 1: + n_iters = y.shape[0] // batch_size + 1 + for i in range(n_iters): + idx = i * batch_size + cos.update((y_pred[idx: idx + batch_size], y[idx: idx + batch_size])) + else: + cos.update((y_pred, y)) + + np_y = y.numpy() + np_y_pred = y_pred.numpy() + + np_y_norm = np.clip(np.linalg.norm(np_y, axis=1, keepdims=True), eps, None) + np_y_pred_norm = np.clip(np.linalg.norm(np_y_pred, axis=1, keepdims=True), eps, None) + np_res = np.sum((np_y / np_y_norm) * (np_y_pred / np_y_pred_norm), axis=1) + np_res = np.mean(np_res) + + assert isinstance(cos.compute(), float) + assert pytest.approx(np_res, rel=2e-5) == cos.compute() + + +def _test_distrib_integration(device, tol=2e-5): + from ignite.engine import Engine + + rank = idist.get_rank() + torch.manual_seed(12 + rank) + + def _test(metric_device): + n_iters = 100 + batch_size = 10 + n_dims = 100 + + y_true = torch.randn((n_iters * batch_size, n_dims), dtype=torch.float).to(device) + y_preds = torch.normal(2.0, 3.0, size=(n_iters * batch_size, n_dims), dtype=torch.float).to(device) + + def update(engine, i): + return ( + y_preds[i * batch_size: (i + 1) * batch_size], + y_true[i * batch_size: (i + 1) * batch_size], + ) + + engine = Engine(update) + + m = CosineSimilarity(device=metric_device) + m.attach(engine, "cosine_similarity") + + data = list(range(n_iters)) + engine.run(data=data, max_epochs=1) + + y_preds = idist.all_gather(y_preds) + y_true = idist.all_gather(y_true) + + assert "cosine_similarity" in engine.state.metrics + res = engine.state.metrics["cosine_similarity"] + + y_true_np = y_true.cpu().numpy() + y_preds_np = y_preds.cpu().numpy() + y_true_norm = np.clip(np.linalg.norm(y_true_np, axis=1, keepdims=True), 1e-8, None) + y_preds_norm = np.clip(np.linalg.norm(y_preds, axis=1, keepdims=True), 1e-8, None) + true_res = np.sum((y_true_np / y_true_norm) * (y_preds_np / y_preds_norm), axis=1) + true_res = np.mean(true_res) + + assert pytest.approx(res, rel=tol) == true_res + + _test("cpu") + if device.type != "xla": + _test(idist.device()) + + +def _test_distrib_accumulator_device(device): + metric_devices = [torch.device("cpu")] + if device.type != "xla": + metric_devices.append(idist.device()) + for metric_device in metric_devices: + device = torch.device(device) + cos = CosineSimilarity(device=metric_device) + + for dev in [cos._device, cos._sum_of_cos_similarities.device]: + assert dev == metric_device, f"{type(dev)}:{dev} vs {type(metric_device)}:{metric_device}" + + y_pred = torch.tensor([[2.0, 3.0], [-2.0, 1.0]], dtype=torch.float) + y = torch.ones(2, 2, dtype=torch.float) + cos.update((y_pred, y)) + + for dev in [cos._device, cos._sum_of_cos_similarities.device]: + assert dev == metric_device, f"{type(dev)}:{dev} vs {type(metric_device)}:{metric_device}" + + +def test_accumulator_detached(): + cos = CosineSimilarity() + + y_pred = torch.tensor([[2.0, 3.0], [-2.0, 1.0]], dtype=torch.float) + y = torch.ones(2, 2, dtype=torch.float) + cos.update((y_pred, y)) + + assert not cos._sum_of_cos_similarities.requires_grad + + +@pytest.mark.distributed +@pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support") +@pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU") +def test_distrib_nccl_gpu(distributed_context_single_node_nccl): + device = idist.device() + _test_distrib_integration(device) + _test_distrib_accumulator_device(device) + + +@pytest.mark.distributed +@pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support") +def test_distrib_gloo_cpu_or_gpu(distributed_context_single_node_gloo): + device = idist.device() + _test_distrib_integration(device) + _test_distrib_accumulator_device(device) + + +@pytest.mark.distributed +@pytest.mark.skipif(not idist.has_hvd_support, reason="Skip if no Horovod dist support") +@pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc") +def test_distrib_hvd(gloo_hvd_executor): + device = torch.device("cpu" if not torch.cuda.is_available() else "cuda") + nproc = 4 if not torch.cuda.is_available() else torch.cuda.device_count() + + gloo_hvd_executor(_test_distrib_integration, (device,), np=nproc, do_init=True) + gloo_hvd_executor(_test_distrib_accumulator_device, (device,), np=nproc, do_init=True) + + +@pytest.mark.multinode_distributed +@pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support") +@pytest.mark.skipif("MULTINODE_DISTRIB" not in os.environ, reason="Skip if not multi-node distributed") +def test_multinode_distrib_gloo_cpu_or_gpu(distributed_context_multi_node_gloo): + device = idist.device() + _test_distrib_integration(device) + _test_distrib_accumulator_device(device) + + +@pytest.mark.multinode_distributed +@pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support") +@pytest.mark.skipif("GPU_MULTINODE_DISTRIB" not in os.environ, reason="Skip if not multi-node distributed") +def test_multinode_distrib_nccl_gpu(distributed_context_multi_node_nccl): + device = idist.device() + _test_distrib_integration(device) + _test_distrib_accumulator_device(device) + + +@pytest.mark.tpu +@pytest.mark.skipif("NUM_TPU_WORKERS" in os.environ, reason="Skip if NUM_TPU_WORKERS is in env vars") +@pytest.mark.skipif(not idist.has_xla_support, reason="Skip if no PyTorch XLA package") +def test_distrib_single_device_xla(): + device = idist.device() + _test_distrib_integration(device, tol=1e-4) + _test_distrib_accumulator_device(device) + + +def _test_distrib_xla_nprocs(index): + device = idist.device() + _test_distrib_integration(device, tol=1e-4) + _test_distrib_accumulator_device(device) + + +@pytest.mark.tpu +@pytest.mark.skipif("NUM_TPU_WORKERS" not in os.environ, reason="Skip if no NUM_TPU_WORKERS in env vars") +@pytest.mark.skipif(not idist.has_xla_support, reason="Skip if no PyTorch XLA package") +def test_distrib_xla_nprocs(xmp_executor): + n = int(os.environ["NUM_TPU_WORKERS"]) + xmp_executor(_test_distrib_xla_nprocs, args=(), nprocs=n) From 8156dc675ef5ccdb74907b631e116089dc281bde Mon Sep 17 00:00:00 2001 From: kzkadc Date: Mon, 18 Mar 2024 14:52:01 +0000 Subject: [PATCH 18/21] autopep8 fix --- .../ignite/metrics/test_cosine_similarity.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/ignite/metrics/test_cosine_similarity.py b/tests/ignite/metrics/test_cosine_similarity.py index 072cb36f773..3e0c99a5eee 100644 --- a/tests/ignite/metrics/test_cosine_similarity.py +++ b/tests/ignite/metrics/test_cosine_similarity.py @@ -20,11 +20,21 @@ def test_zero_sample(): @pytest.fixture(params=[item for item in range(4)]) def test_case(request): return [ - (torch.randn((100, 50)), torch.randn((100, 50)), 10**np.random.uniform(-8, 0), 1), - (torch.normal(1.0, 2.0, size=(100, 10)), torch.normal(3.0, 4.0, size=(100, 10)), 10**np.random.uniform(-8, 0), 1), + (torch.randn((100, 50)), torch.randn((100, 50)), 10 ** np.random.uniform(-8, 0), 1), + ( + torch.normal(1.0, 2.0, size=(100, 10)), + torch.normal(3.0, 4.0, size=(100, 10)), + 10 ** np.random.uniform(-8, 0), + 1, + ), # updated batches - (torch.rand((100, 128)), torch.rand((100, 128)), 10**np.random.uniform(-8, 0), 16), - (torch.normal(0.0, 5.0, size=(100, 30)), torch.normal(5.0, 1.0, size=(100, 30)), 10**np.random.uniform(-8, 0), 16), + (torch.rand((100, 128)), torch.rand((100, 128)), 10 ** np.random.uniform(-8, 0), 16), + ( + torch.normal(0.0, 5.0, size=(100, 30)), + torch.normal(5.0, 1.0, size=(100, 30)), + 10 ** np.random.uniform(-8, 0), + 16, + ), ][request.param] @@ -39,7 +49,7 @@ def test_compute(n_times, test_case): n_iters = y.shape[0] // batch_size + 1 for i in range(n_iters): idx = i * batch_size - cos.update((y_pred[idx: idx + batch_size], y[idx: idx + batch_size])) + cos.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size])) else: cos.update((y_pred, y)) @@ -71,8 +81,8 @@ def _test(metric_device): def update(engine, i): return ( - y_preds[i * batch_size: (i + 1) * batch_size], - y_true[i * batch_size: (i + 1) * batch_size], + y_preds[i * batch_size : (i + 1) * batch_size], + y_true[i * batch_size : (i + 1) * batch_size], ) engine = Engine(update) From d05f8548dd6d89cff3ce55235a5ede84b290e0ac Mon Sep 17 00:00:00 2001 From: Kazuki Adachi Date: Wed, 20 Mar 2024 20:41:49 +0900 Subject: [PATCH 19/21] fix error when testing in xla --- tests/ignite/metrics/test_cosine_similarity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ignite/metrics/test_cosine_similarity.py b/tests/ignite/metrics/test_cosine_similarity.py index 3e0c99a5eee..db7a5d5d9f1 100644 --- a/tests/ignite/metrics/test_cosine_similarity.py +++ b/tests/ignite/metrics/test_cosine_similarity.py @@ -102,7 +102,7 @@ def update(engine, i): y_true_np = y_true.cpu().numpy() y_preds_np = y_preds.cpu().numpy() y_true_norm = np.clip(np.linalg.norm(y_true_np, axis=1, keepdims=True), 1e-8, None) - y_preds_norm = np.clip(np.linalg.norm(y_preds, axis=1, keepdims=True), 1e-8, None) + y_preds_norm = np.clip(np.linalg.norm(y_preds_np, axis=1, keepdims=True), 1e-8, None) true_res = np.sum((y_true_np / y_true_norm) * (y_preds_np / y_preds_norm), axis=1) true_res = np.mean(true_res) From 4a883bab91cc2d8ed6a96190e12e4c0649b78c92 Mon Sep 17 00:00:00 2001 From: Kazuki Adachi Date: Sun, 24 Mar 2024 02:01:16 +0900 Subject: [PATCH 20/21] update test for CosineSimilarity metric --- .../ignite/metrics/test_cosine_similarity.py | 179 ++++++------------ 1 file changed, 57 insertions(+), 122 deletions(-) diff --git a/tests/ignite/metrics/test_cosine_similarity.py b/tests/ignite/metrics/test_cosine_similarity.py index db7a5d5d9f1..c9ac1b0ea9d 100644 --- a/tests/ignite/metrics/test_cosine_similarity.py +++ b/tests/ignite/metrics/test_cosine_similarity.py @@ -1,10 +1,12 @@ -import os +from typing import Tuple import numpy as np import pytest import torch +from torch import Tensor import ignite.distributed as idist +from ignite.engine import Engine from ignite.exceptions import NotComputableError from ignite.metrics import CosineSimilarity @@ -17,7 +19,7 @@ def test_zero_sample(): cos_sim.compute() -@pytest.fixture(params=[item for item in range(4)]) +@pytest.fixture(params=list(range(4))) def test_case(request): return [ (torch.randn((100, 50)), torch.randn((100, 50)), 10 ** np.random.uniform(-8, 0), 1), @@ -39,7 +41,7 @@ def test_case(request): @pytest.mark.parametrize("n_times", range(5)) -def test_compute(n_times, test_case): +def test_compute(n_times, test_case: Tuple[Tensor, Tensor, float, int]): y_pred, y, eps, batch_size = test_case cos = CosineSimilarity(eps=eps) @@ -65,73 +67,6 @@ def test_compute(n_times, test_case): assert pytest.approx(np_res, rel=2e-5) == cos.compute() -def _test_distrib_integration(device, tol=2e-5): - from ignite.engine import Engine - - rank = idist.get_rank() - torch.manual_seed(12 + rank) - - def _test(metric_device): - n_iters = 100 - batch_size = 10 - n_dims = 100 - - y_true = torch.randn((n_iters * batch_size, n_dims), dtype=torch.float).to(device) - y_preds = torch.normal(2.0, 3.0, size=(n_iters * batch_size, n_dims), dtype=torch.float).to(device) - - def update(engine, i): - return ( - y_preds[i * batch_size : (i + 1) * batch_size], - y_true[i * batch_size : (i + 1) * batch_size], - ) - - engine = Engine(update) - - m = CosineSimilarity(device=metric_device) - m.attach(engine, "cosine_similarity") - - data = list(range(n_iters)) - engine.run(data=data, max_epochs=1) - - y_preds = idist.all_gather(y_preds) - y_true = idist.all_gather(y_true) - - assert "cosine_similarity" in engine.state.metrics - res = engine.state.metrics["cosine_similarity"] - - y_true_np = y_true.cpu().numpy() - y_preds_np = y_preds.cpu().numpy() - y_true_norm = np.clip(np.linalg.norm(y_true_np, axis=1, keepdims=True), 1e-8, None) - y_preds_norm = np.clip(np.linalg.norm(y_preds_np, axis=1, keepdims=True), 1e-8, None) - true_res = np.sum((y_true_np / y_true_norm) * (y_preds_np / y_preds_norm), axis=1) - true_res = np.mean(true_res) - - assert pytest.approx(res, rel=tol) == true_res - - _test("cpu") - if device.type != "xla": - _test(idist.device()) - - -def _test_distrib_accumulator_device(device): - metric_devices = [torch.device("cpu")] - if device.type != "xla": - metric_devices.append(idist.device()) - for metric_device in metric_devices: - device = torch.device(device) - cos = CosineSimilarity(device=metric_device) - - for dev in [cos._device, cos._sum_of_cos_similarities.device]: - assert dev == metric_device, f"{type(dev)}:{dev} vs {type(metric_device)}:{metric_device}" - - y_pred = torch.tensor([[2.0, 3.0], [-2.0, 1.0]], dtype=torch.float) - y = torch.ones(2, 2, dtype=torch.float) - cos.update((y_pred, y)) - - for dev in [cos._device, cos._sum_of_cos_similarities.device]: - assert dev == metric_device, f"{type(dev)}:{dev} vs {type(metric_device)}:{metric_device}" - - def test_accumulator_detached(): cos = CosineSimilarity() @@ -142,70 +77,70 @@ def test_accumulator_detached(): assert not cos._sum_of_cos_similarities.requires_grad -@pytest.mark.distributed -@pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support") -@pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU") -def test_distrib_nccl_gpu(distributed_context_single_node_nccl): - device = idist.device() - _test_distrib_integration(device) - _test_distrib_accumulator_device(device) +@pytest.mark.usefixtures("distributed") +class TestDistributed: + def test_integration(self): + tol = 2e-5 + rank = idist.get_rank() + torch.manual_seed(12 + rank) -@pytest.mark.distributed -@pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support") -def test_distrib_gloo_cpu_or_gpu(distributed_context_single_node_gloo): - device = idist.device() - _test_distrib_integration(device) - _test_distrib_accumulator_device(device) + device = idist.device() + metric_devices = [torch.device("cpu")] + if device.type != "xla": + metric_devices.append(device) + n_iters = 100 + batch_size = 10 + n_dims = 100 -@pytest.mark.distributed -@pytest.mark.skipif(not idist.has_hvd_support, reason="Skip if no Horovod dist support") -@pytest.mark.skipif("WORLD_SIZE" in os.environ, reason="Skip if launched as multiproc") -def test_distrib_hvd(gloo_hvd_executor): - device = torch.device("cpu" if not torch.cuda.is_available() else "cuda") - nproc = 4 if not torch.cuda.is_available() else torch.cuda.device_count() + for metric_device in metric_devices: + y_true = torch.randn((n_iters * batch_size, n_dims)).float().to(device) + y_preds = torch.normal(2.0, 3.0, size=(n_iters * batch_size, n_dims)).float().to(device) - gloo_hvd_executor(_test_distrib_integration, (device,), np=nproc, do_init=True) - gloo_hvd_executor(_test_distrib_accumulator_device, (device,), np=nproc, do_init=True) + engine = Engine( + lambda e, i: ( + y_preds[i * batch_size : (i + 1) * batch_size], + y_true[i * batch_size : (i + 1) * batch_size], + ) + ) + m = CosineSimilarity(device=metric_device) + m.attach(engine, "cosine_similarity") -@pytest.mark.multinode_distributed -@pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support") -@pytest.mark.skipif("MULTINODE_DISTRIB" not in os.environ, reason="Skip if not multi-node distributed") -def test_multinode_distrib_gloo_cpu_or_gpu(distributed_context_multi_node_gloo): - device = idist.device() - _test_distrib_integration(device) - _test_distrib_accumulator_device(device) + data = list(range(n_iters)) + engine.run(data=data, max_epochs=1) + y_preds = idist.all_gather(y_preds) + y_true = idist.all_gather(y_true) -@pytest.mark.multinode_distributed -@pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support") -@pytest.mark.skipif("GPU_MULTINODE_DISTRIB" not in os.environ, reason="Skip if not multi-node distributed") -def test_multinode_distrib_nccl_gpu(distributed_context_multi_node_nccl): - device = idist.device() - _test_distrib_integration(device) - _test_distrib_accumulator_device(device) + assert "cosine_similarity" in engine.state.metrics + res = engine.state.metrics["cosine_similarity"] + y_true_np = y_true.cpu().numpy() + y_preds_np = y_preds.cpu().numpy() + y_true_norm = np.clip(np.linalg.norm(y_true_np, axis=1, keepdims=True), 1e-8, None) + y_preds_norm = np.clip(np.linalg.norm(y_preds_np, axis=1, keepdims=True), 1e-8, None) + true_res = np.sum((y_true_np / y_true_norm) * (y_preds_np / y_preds_norm), axis=1) + true_res = np.mean(true_res) -@pytest.mark.tpu -@pytest.mark.skipif("NUM_TPU_WORKERS" in os.environ, reason="Skip if NUM_TPU_WORKERS is in env vars") -@pytest.mark.skipif(not idist.has_xla_support, reason="Skip if no PyTorch XLA package") -def test_distrib_single_device_xla(): - device = idist.device() - _test_distrib_integration(device, tol=1e-4) - _test_distrib_accumulator_device(device) + assert pytest.approx(res, rel=tol) == true_res + def test_accumulator_device(self): + device = idist.device() + metric_devices = [torch.device("cpu")] + if device.type != "xla": + metric_devices.append(idist.device()) + for metric_device in metric_devices: + device = torch.device(device) + cos = CosineSimilarity(device=metric_device) -def _test_distrib_xla_nprocs(index): - device = idist.device() - _test_distrib_integration(device, tol=1e-4) - _test_distrib_accumulator_device(device) + for dev in (cos._device, cos._sum_of_cos_similarities.device): + assert dev == metric_device, f"{type(dev)}:{dev} vs {type(metric_device)}:{metric_device}" + y_pred = torch.tensor([[2.0, 3.0], [-2.0, 1.0]]).float() + y = torch.ones(2, 2).float() + cos.update((y_pred, y)) -@pytest.mark.tpu -@pytest.mark.skipif("NUM_TPU_WORKERS" not in os.environ, reason="Skip if no NUM_TPU_WORKERS in env vars") -@pytest.mark.skipif(not idist.has_xla_support, reason="Skip if no PyTorch XLA package") -def test_distrib_xla_nprocs(xmp_executor): - n = int(os.environ["NUM_TPU_WORKERS"]) - xmp_executor(_test_distrib_xla_nprocs, args=(), nprocs=n) + for dev in (cos._device, cos._sum_of_cos_similarities.device): + assert dev == metric_device, f"{type(dev)}:{dev} vs {type(metric_device)}:{metric_device}" From 3c73a22e3565b02fd64ed95051268b6f3bbdeb6c Mon Sep 17 00:00:00 2001 From: Kazuki Adachi Date: Sun, 24 Mar 2024 11:18:07 +0900 Subject: [PATCH 21/21] update test for CosineSimilarity metric --- tests/ignite/metrics/test_cosine_similarity.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/ignite/metrics/test_cosine_similarity.py b/tests/ignite/metrics/test_cosine_similarity.py index c9ac1b0ea9d..1ee7be7b073 100644 --- a/tests/ignite/metrics/test_cosine_similarity.py +++ b/tests/ignite/metrics/test_cosine_similarity.py @@ -81,6 +81,9 @@ def test_accumulator_detached(): class TestDistributed: def test_integration(self): tol = 2e-5 + n_iters = 100 + batch_size = 10 + n_dims = 100 rank = idist.get_rank() torch.manual_seed(12 + rank) @@ -90,10 +93,6 @@ def test_integration(self): if device.type != "xla": metric_devices.append(device) - n_iters = 100 - batch_size = 10 - n_dims = 100 - for metric_device in metric_devices: y_true = torch.randn((n_iters * batch_size, n_dims)).float().to(device) y_preds = torch.normal(2.0, 3.0, size=(n_iters * batch_size, n_dims)).float().to(device) @@ -130,9 +129,8 @@ def test_accumulator_device(self): device = idist.device() metric_devices = [torch.device("cpu")] if device.type != "xla": - metric_devices.append(idist.device()) + metric_devices.append(device) for metric_device in metric_devices: - device = torch.device(device) cos = CosineSimilarity(device=metric_device) for dev in (cos._device, cos._sum_of_cos_similarities.device):