Merge pull request #4 from zbw/zero-recall

Set predicted recall to zero for docs without predicted labels
zbw · Oct 5, 2022 · 34248b7 · 34248b7
2 parents f558a75 + 86c7690
commit 34248b7
Show file tree

Hide file tree

Showing 57 changed files with 150 additions and 66 deletions.
diff --git a/qualle/__init__.py b/qualle/__init__.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/evaluate.py b/qualle/evaluate.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/features/__init__.py b/qualle/features/__init__.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/features/base.py b/qualle/features/base.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/features/combined.py b/qualle/features/combined.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/features/confidence.py b/qualle/features/confidence.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/features/label_calibration/__init__.py b/qualle/features/label_calibration/__init__.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/features/label_calibration/base.py b/qualle/features/label_calibration/base.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/features/label_calibration/simple_label_calibration.py b/qualle/features/label_calibration/simple_label_calibration.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/features/label_calibration/thesauri_label_calibration.py b/qualle/features/label_calibration/thesauri_label_calibration.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/features/text.py b/qualle/features/text.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/interface/__init__.py b/qualle/interface/__init__.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/interface/cli.py b/qualle/interface/cli.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/interface/config.py b/qualle/interface/config.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/interface/internal.py b/qualle/interface/internal.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/interface/rest.py b/qualle/interface/rest.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/label_calibration/__init__.py b/qualle/label_calibration/__init__.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/label_calibration/category.py b/qualle/label_calibration/category.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/label_calibration/simple.py b/qualle/label_calibration/simple.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/main.py b/qualle/main.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/models.py b/qualle/models.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/pipeline.py b/qualle/pipeline.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 from contextlib import contextmanager
-from typing import List, Callable, Any
+from typing import List, Callable, Any, Collection
 
 from sklearn.model_selection import cross_val_predict
 
@@ -65,15 +65,67 @@ def train(self, data: TrainData):
             self._recall_predictor.fit(features_data, true_recall)
 
     def predict(self, data: PredictData) -> List[float]:
-        predicted_no_of_labels = self._label_calibrator.predict(data.docs)
-        label_calibration_data = LabelCalibrationData(
-            predicted_labels=data.predicted_labels,
-            predicted_no_of_labels=predicted_no_of_labels
-        )
-        features_data = self._features_data_mapper(
-            data, label_calibration_data
+        zero_idxs = self._get_pdata_idxs_with_zero_labels(data)
+        data_with_labels = self._get_pdata_with_labels(data, zero_idxs)
+        if data_with_labels.docs:
+            predicted_no_of_labels = self._label_calibrator.predict(
+                data_with_labels.docs
+            )
+            label_calibration_data = LabelCalibrationData(
+                predicted_labels=data_with_labels.predicted_labels,
+                predicted_no_of_labels=predicted_no_of_labels,
+            )
+            features_data = self._features_data_mapper(
+                data_with_labels, label_calibration_data
+            )
+            predicted_recall = self._recall_predictor.predict(
+                features_data
+            )
+            recall_scores = self._merge_zero_recall_with_predicted_recall(
+                predicted_recall=predicted_recall,
+                zero_labels_idx=zero_idxs,
+            )
+        else:
+            recall_scores = [0] * len(data.predicted_labels)
+        return recall_scores
+
+    @staticmethod
+    def _get_pdata_idxs_with_zero_labels(data: PredictData) -> Collection[int]:
+        return [
+            i for i in range(len(data.predicted_labels))
+            if not data.predicted_labels[i]
+        ]
+
+    @staticmethod
+    def _get_pdata_with_labels(
+        data: PredictData, zero_labels_idxs: Collection[int]
+    ) -> PredictData:
+        non_zero_idxs = [
+            i for i in range(len(data.predicted_labels))
+            if i not in zero_labels_idxs
+        ]
+        return PredictData(
+            docs=[data.docs[i] for i in non_zero_idxs],
+            predicted_labels=[data.predicted_labels[i] for i in non_zero_idxs],
+            scores=[data.scores[i] for i in non_zero_idxs],
         )
-        return self._recall_predictor.predict(features_data)
+
+    @staticmethod
+    def _merge_zero_recall_with_predicted_recall(
+        predicted_recall: List[float],
+        zero_labels_idx: Collection[int],
+    ):
+        recall_scores = []
+        j = 0
+        for i in range(
+                len(zero_labels_idx) +
+                len(predicted_recall)):
+            if i in zero_labels_idx:
+                recall_scores.append(0)
+            else:
+                recall_scores.append(predicted_recall[j])
+                j += 1
+        return recall_scores
 
     @contextmanager
     def _debug(self, method_name):

diff --git a/qualle/quality_estimation.py b/qualle/quality_estimation.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/train.py b/qualle/train.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/qualle/utils.py b/qualle/utils.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/__init__.py b/tests/__init__.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/common.py b/tests/common.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/features/__init__.py b/tests/features/__init__.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/features/label_calibration/__init__.py b/tests/features/label_calibration/__init__.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/features/label_calibration/test_simple_label_calibration/__init__.py b/tests/features/label_calibration/test_simple_label_calibration/__init__.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/features/label_calibration/test_simple_label_calibration/test_calibrator.py b/tests/features/label_calibration/test_simple_label_calibration/test_calibrator.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/features/label_calibration/test_simple_label_calibration/test_features.py b/tests/features/label_calibration/test_simple_label_calibration/test_features.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/features/label_calibration/test_thesauri_label_calibration/__init__.py b/tests/features/label_calibration/test_thesauri_label_calibration/__init__.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/features/label_calibration/test_thesauri_label_calibration/common.py b/tests/features/label_calibration/test_thesauri_label_calibration/common.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/features/label_calibration/test_thesauri_label_calibration/conftest.py b/tests/features/label_calibration/test_thesauri_label_calibration/conftest.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/features/label_calibration/test_thesauri_label_calibration/test_calibrator.py b/tests/features/label_calibration/test_thesauri_label_calibration/test_calibrator.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/features/label_calibration/test_thesauri_label_calibration/test_features.py b/tests/features/label_calibration/test_thesauri_label_calibration/test_features.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/features/label_calibration/test_thesauri_label_calibration/test_transformer.py b/tests/features/label_calibration/test_thesauri_label_calibration/test_transformer.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/features/test_combined.py b/tests/features/test_combined.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/features/test_confidence.py b/tests/features/test_confidence.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/features/test_text.py b/tests/features/test_text.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/interface/__init__.py b/tests/interface/__init__.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/interface/common.py b/tests/interface/common.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/interface/test_cli.py b/tests/interface/test_cli.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/interface/test_internal.py b/tests/interface/test_internal.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/interface/test_rest.py b/tests/interface/test_rest.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/label_calibration/__init__.py b/tests/label_calibration/__init__.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/label_calibration/conftest.py b/tests/label_calibration/conftest.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/label_calibration/test_category.py b/tests/label_calibration/test_category.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/label_calibration/test_simple.py b/tests/label_calibration/test_simple.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/tests/test_eval.py b/tests/test_eval.py
@@ -1,4 +1,4 @@
-#  Copyright 2021 ZBW – Leibniz Information Centre for Economics
+#  Copyright 2021-2022 ZBW – Leibniz Information Centre for Economics
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.