From 1a1b91a384f21fe9034d7c9acf9d3ac50b9a476f Mon Sep 17 00:00:00 2001 From: Adam Narozniak Date: Mon, 22 Jul 2024 11:00:03 +0200 Subject: [PATCH 01/10] Update pylint version --- datasets/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasets/pyproject.toml b/datasets/pyproject.toml index e3afd8b87075..c4144703dbf2 100644 --- a/datasets/pyproject.toml +++ b/datasets/pyproject.toml @@ -68,7 +68,7 @@ isort = "==5.13.2" black = { version = "==24.2.0", extras = ["jupyter"] } docformatter = "==1.7.5" mypy = "==1.4.0" -pylint = "==2.13.9" +pylint = "==3.0.3" flake8 = "==3.9.2" parameterized = "==0.9.0" pytest = "==7.1.2" From e2207a3e0a674777377328ce459bb32b0c5fd174 Mon Sep 17 00:00:00 2001 From: Adam Narozniak Date: Mon, 22 Jul 2024 11:07:54 +0200 Subject: [PATCH 02/10] Fix R0022 --- datasets/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasets/pyproject.toml b/datasets/pyproject.toml index c4144703dbf2..0a3b14990699 100644 --- a/datasets/pyproject.toml +++ b/datasets/pyproject.toml @@ -95,7 +95,7 @@ line-length = 88 target-version = ["py38", "py39", "py310", "py311"] [tool.pylint."MESSAGES CONTROL"] -disable = "bad-continuation,duplicate-code,too-few-public-methods,useless-import-alias" +disable = "duplicate-code,too-few-public-methods,useless-import-alias" [tool.pytest.ini_options] minversion = "6.2" From 1e288f3f34a90a6c41aa4f49414ef50774ba503c Mon Sep 17 00:00:00 2001 From: Adam Narozniak Date: Mon, 22 Jul 2024 11:09:41 +0200 Subject: [PATCH 03/10] Fix R0201 no self use --- datasets/flwr_datasets/federated_dataset_test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/datasets/flwr_datasets/federated_dataset_test.py b/datasets/flwr_datasets/federated_dataset_test.py index fe1ee5f1f14e..53bc8286ab91 100644 --- a/datasets/flwr_datasets/federated_dataset_test.py +++ b/datasets/flwr_datasets/federated_dataset_test.py @@ -245,7 +245,6 @@ class ShufflingResplittingOnArtificialDatasetTest(unittest.TestCase): The load_dataset method is mocked and the artificial dataset is returned. """ - # pylint: disable=no-self-use def _dummy_setup(self, train_rows: int = 10, test_rows: int = 5) -> DatasetDict: """Create a dummy DatasetDict with train, test splits.""" data_train = { @@ -436,14 +435,14 @@ def test_if_the_partitions_have_unique_values(self) -> None: class IncorrectUsageFederatedDatasets(unittest.TestCase): """Test incorrect usages in FederatedDatasets.""" - def test_no_partitioner_for_split(self) -> None: # pylint: disable=R0201 + def test_no_partitioner_for_split(self) -> None: """Test using load_partition with missing partitioner.""" dataset_fds = FederatedDataset(dataset="mnist", partitioners={"train": 100}) with pytest.raises(ValueError): dataset_fds.load_partition(0, "test") - def test_no_split_in_the_dataset(self) -> None: # pylint: disable=R0201 + def test_no_split_in_the_dataset(self) -> None: """Test using load_partition with non-existent split name.""" dataset_fds = FederatedDataset( dataset="mnist", partitioners={"non-existent-split": 100} @@ -452,7 +451,7 @@ def test_no_split_in_the_dataset(self) -> None: # pylint: disable=R0201 with pytest.raises(ValueError): dataset_fds.load_partition(0, "non-existent-split") - def test_unsupported_dataset(self) -> None: # pylint: disable=R0201 + def test_unsupported_dataset(self) -> None: """Test creating FederatedDataset for unsupported dataset.""" with pytest.warns(UserWarning): FederatedDataset(dataset="food101", partitioners={"train": 100}) From f9539a9dabf8663e54d0a33f21d91dd0a339b2a2 Mon Sep 17 00:00:00 2001 From: Adam Narozniak Date: Mon, 22 Jul 2024 11:13:52 +0200 Subject: [PATCH 04/10] Fix R1728 (consider-using-generator) --- datasets/flwr_datasets/federated_dataset_test.py | 5 ++--- datasets/flwr_datasets/visualization/bar_plot.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/datasets/flwr_datasets/federated_dataset_test.py b/datasets/flwr_datasets/federated_dataset_test.py index 53bc8286ab91..024790eeaf26 100644 --- a/datasets/flwr_datasets/federated_dataset_test.py +++ b/datasets/flwr_datasets/federated_dataset_test.py @@ -169,7 +169,7 @@ def test_resplit_dataset_into_one(self) -> None: if self.test_split is None: return dataset = datasets.load_dataset(self.dataset_name) - dataset_length = sum([len(ds) for ds in dataset.values()]) + dataset_length = sum(len(ds) for ds in dataset.values()) fds = FederatedDataset( dataset=self.dataset_name, partitioners={"train": 100}, @@ -216,7 +216,7 @@ def resplit(dataset: DatasetDict) -> DatasetDict: ) full = fds.load_split("full") dataset = datasets.load_dataset(self.dataset_name) - dataset_length = sum([len(ds) for ds in dataset.values()]) + dataset_length = sum(len(ds) for ds in dataset.values()) self.assertEqual(len(full), dataset_length) def test_use_load_dataset_kwargs(self) -> None: @@ -459,7 +459,6 @@ def test_unsupported_dataset(self) -> None: def test_cannot_use_the_old_split_names(self) -> None: """Test if the initial split names can not be used.""" dataset = datasets.load_dataset("mnist") - sum([len(ds) for ds in dataset.values()]) fds = FederatedDataset( dataset="mnist", partitioners={"train": 100}, diff --git a/datasets/flwr_datasets/visualization/bar_plot.py b/datasets/flwr_datasets/visualization/bar_plot.py index 339ff0967906..352c99a572f5 100644 --- a/datasets/flwr_datasets/visualization/bar_plot.py +++ b/datasets/flwr_datasets/visualization/bar_plot.py @@ -100,7 +100,7 @@ def _plot_bar( legend_kwargs["loc"] = "outside center right" if "bbox_to_anchor" not in legend_kwargs: - max_len_label_str = max([len(str(column)) for column in dataframe.columns]) + max_len_label_str = max(len(str(column)) for column in dataframe.columns) shift = min(0.05 + max_len_label_str / 100, 0.15) legend_kwargs["bbox_to_anchor"] = (1.0 + shift, 0.5) From a69ccbb0fc892c6ac255715ec87bdda07c4ddbcb Mon Sep 17 00:00:00 2001 From: Adam Narozniak Date: Mon, 22 Jul 2024 11:14:21 +0200 Subject: [PATCH 05/10] Format --- datasets/flwr_datasets/federated_dataset_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasets/flwr_datasets/federated_dataset_test.py b/datasets/flwr_datasets/federated_dataset_test.py index 024790eeaf26..38f032a4ccfe 100644 --- a/datasets/flwr_datasets/federated_dataset_test.py +++ b/datasets/flwr_datasets/federated_dataset_test.py @@ -458,7 +458,7 @@ def test_unsupported_dataset(self) -> None: def test_cannot_use_the_old_split_names(self) -> None: """Test if the initial split names can not be used.""" - dataset = datasets.load_dataset("mnist") + datasets.load_dataset("mnist") fds = FederatedDataset( dataset="mnist", partitioners={"train": 100}, From 0fec787968332c9e3117736109a30e325f1b6d15 Mon Sep 17 00:00:00 2001 From: Adam Narozniak Date: Mon, 22 Jul 2024 11:15:41 +0200 Subject: [PATCH 06/10] Fix R0913 (too-many-arguments) --- datasets/flwr_datasets/federated_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasets/flwr_datasets/federated_dataset.py b/datasets/flwr_datasets/federated_dataset.py index 23d98e9c7bb0..7514516e09e7 100644 --- a/datasets/flwr_datasets/federated_dataset.py +++ b/datasets/flwr_datasets/federated_dataset.py @@ -103,7 +103,7 @@ class FederatedDataset: >>> ) """ - # pylint: disable=too-many-instance-attributes + # pylint: disable=too-many-instance-attributes, too-many-arguments def __init__( self, *, From 37279df2a243b1cac0155ce293752c436d4eb810 Mon Sep 17 00:00:00 2001 From: Adam Narozniak Date: Mon, 22 Jul 2024 11:18:56 +0200 Subject: [PATCH 07/10] Fix R0022 (useless-option-value) --- datasets/flwr_datasets/metrics/utils_test.py | 1 - datasets/flwr_datasets/partitioner/natural_id_partitioner.py | 1 - datasets/flwr_datasets/preprocessor/merger_test.py | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/datasets/flwr_datasets/metrics/utils_test.py b/datasets/flwr_datasets/metrics/utils_test.py index 9e0f3acdf805..9fab9994c430 100644 --- a/datasets/flwr_datasets/metrics/utils_test.py +++ b/datasets/flwr_datasets/metrics/utils_test.py @@ -13,7 +13,6 @@ # limitations under the License. # ============================================================================== """Tests for metrics utils.""" -# pylint: disable=no-self-use import unittest diff --git a/datasets/flwr_datasets/partitioner/natural_id_partitioner.py b/datasets/flwr_datasets/partitioner/natural_id_partitioner.py index 85f1b3af43c2..f71d7004b42b 100644 --- a/datasets/flwr_datasets/partitioner/natural_id_partitioner.py +++ b/datasets/flwr_datasets/partitioner/natural_id_partitioner.py @@ -143,7 +143,6 @@ def partition_id_to_natural_id(self) -> Dict[int, str]: """ return self._partition_id_to_natural_id - # pylint: disable=R0201 @partition_id_to_natural_id.setter def partition_id_to_natural_id(self, value: Dict[int, str]) -> None: raise AttributeError( diff --git a/datasets/flwr_datasets/preprocessor/merger_test.py b/datasets/flwr_datasets/preprocessor/merger_test.py index d5c69387e53d..137b0dd1a660 100644 --- a/datasets/flwr_datasets/preprocessor/merger_test.py +++ b/datasets/flwr_datasets/preprocessor/merger_test.py @@ -107,7 +107,7 @@ def test_nonexistent_split_in_strategy(self) -> None: ): merger(self.dataset_dict) - def test_duplicate_merge_split_name(self) -> None: # pylint: disable=R0201 + def test_duplicate_merge_split_name(self) -> None: """Test that the new split names are not the same.""" strategy: Dict[str, Tuple[str, ...]] = { "new_train": ("train", "valid"), From 17d42b5df51b7704ac01dc7a6f46b74d75e0857d Mon Sep 17 00:00:00 2001 From: Adam Narozniak Date: Mon, 22 Jul 2024 11:21:49 +0200 Subject: [PATCH 08/10] Fix W0719 (broad-exception-raised) --- datasets/flwr_datasets/partitioner/partitioner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasets/flwr_datasets/partitioner/partitioner.py b/datasets/flwr_datasets/partitioner/partitioner.py index 10ade52640e8..24ca22bbebcb 100644 --- a/datasets/flwr_datasets/partitioner/partitioner.py +++ b/datasets/flwr_datasets/partitioner/partitioner.py @@ -44,7 +44,7 @@ def dataset(self) -> Dataset: @dataset.setter def dataset(self, value: Dataset) -> None: if self._dataset is not None: - raise Exception( + raise ValueError( "The dataset should be assigned only once to the partitioner." "This operation might also wipe out the saved references to the " "created partitions (in case the partitioning scheme needs to create " From 807028cfb911bf7b7f05b2b0587fd80e5c4fad18 Mon Sep 17 00:00:00 2001 From: Adam Narozniak Date: Mon, 22 Jul 2024 11:22:37 +0200 Subject: [PATCH 09/10] Fix R1728 (consider-using-generator) --- .../flwr_datasets/partitioner/natural_id_partitioner_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasets/flwr_datasets/partitioner/natural_id_partitioner_test.py b/datasets/flwr_datasets/partitioner/natural_id_partitioner_test.py index f447634ad9ed..b74a044967ef 100644 --- a/datasets/flwr_datasets/partitioner/natural_id_partitioner_test.py +++ b/datasets/flwr_datasets/partitioner/natural_id_partitioner_test.py @@ -86,7 +86,7 @@ def test_load_partition_max_partition_size( print(num_unique_natural_ids) _, partitioner = _dummy_setup(num_rows, num_unique_natural_ids) max_size = max( - [len(partitioner.load_partition(i)) for i in range(num_unique_natural_ids)] + len(partitioner.load_partition(i)) for i in range(num_unique_natural_ids) ) self.assertEqual(max_size, math.ceil(num_rows / num_unique_natural_ids)) From 978f7b7bb0eed42fb1f7261a31cd5e2dd1fb527d Mon Sep 17 00:00:00 2001 From: Adam Narozniak Date: Mon, 22 Jul 2024 11:25:15 +0200 Subject: [PATCH 10/10] Fix W0133 (pointless-exception-statement) --- datasets/flwr_datasets/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasets/flwr_datasets/utils.py b/datasets/flwr_datasets/utils.py index fa3f7a54f198..578270c3735a 100644 --- a/datasets/flwr_datasets/utils.py +++ b/datasets/flwr_datasets/utils.py @@ -183,7 +183,7 @@ def _create_division_indices_ranges( ranges.append(range(start_idx, end_idx)) start_idx = end_idx else: - TypeError( + raise TypeError( f"The type of the `division` should be dict, " f"tuple or list but is {type(division)} instead. " )