Merge branch 'main' into fds-add-how-to-use-local-data

.github/workflows/docker-client.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       flwr-version:
-        description: "Version of Flower e.g. (1.7.0)."
+        description: "Version of Flower"
         required: true
         type: string
 

.github/workflows/docker-server.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       flwr-version:
-        description: "Version of Flower e.g. (1.7.0)."
+        description: "Version of Flower"
         required: true
         type: string
       base-image-tag:

.github/workflows/framework-release.yml

@@ -42,4 +42,4 @@ jobs:
         curl $wheel_url --output dist/$wheel_name
         curl $tar_url --output dist/$tar_name
 
-        python -m poetry publish -u __token__ -p ${{ secrets.PYPI_TOKEN }}
+        python -m poetry publish -u __token__ -p ${{ secrets.PYPI_TOKEN_RELEASE_FLWR }}

.pre-commit-config.yaml

@@ -0,0 +1,18 @@
+repos:
+  - repo: local
+    hooks:
+      - id: format-code
+        name: Format Code
+        entry: ./dev/format.sh
+        language: script
+        # Ensures the script runs from the repository root:
+        pass_filenames: false
+        stages: [commit]
+
+      - id: run-tests
+        name: Run Tests
+        entry: ./dev/test.sh
+        language: script
+        # Ensures the script runs from the repository root:
+        pass_filenames: false
+        stages: [commit]

baselines/doc/source/conf.py

@@ -37,7 +37,7 @@
 author = "The Flower Authors"
 
 # The full version, including alpha/beta/rc tags
-release = "1.7.0"
+release = "1.8.0"
 
 
 # -- General configuration ---------------------------------------------------

baselines/flwr_baselines/pyproject.toml

@@ -51,6 +51,7 @@ wget = "^3.2"
 virtualenv = "^20.24.6"
 pandas = "^1.5.3"
 pyhamcrest = "^2.0.4"
+pillow = "==10.2.0"
 
 [tool.poetry.dev-dependencies]
 isort = "==5.13.2"

datasets/doc/source/how-to-use-with-pytorch.rst

@@ -63,7 +63,7 @@ expected by a model with a convolutional layer.
 
 If you want to divide the dataset, you can use (at any point before passing the dataset to the DataLoader)::
 
-  partition_train_test = partition.train_test_split(test_size=0.2)
+  partition_train_test = partition.train_test_split(test_size=0.2, seed=42)
   partition_train = partition_train_test["train"]
   partition_test = partition_train_test["test"]
 

datasets/e2e/pytorch/pytorch_test.py

@@ -65,7 +65,7 @@ def _create_trainloader(self, batch_size: int) -> DataLoader:
         partition_id = 0
         fds = FederatedDataset(dataset=self.dataset_name, partitioners={"train": 100})
         partition = fds.load_partition(partition_id, "train")
-        partition_train_test = partition.train_test_split(test_size=0.2)
+        partition_train_test = partition.train_test_split(test_size=0.2, seed=42)
         partition_train_test = partition_train_test.map(
             lambda img: {"img": self.transforms(img)}, input_columns="img"
         )

datasets/e2e/scikit-learn/sklearn_test.py

@@ -29,7 +29,7 @@ def _get_partition_data(self):
         fds = FederatedDataset(dataset=self.dataset_name, partitioners={"train": 10})
         partition = fds.load_partition(partition_id, "train")
         partition.set_format("numpy")
-        partition_train_test = partition.train_test_split(test_size=0.2)
+        partition_train_test = partition.train_test_split(test_size=0.2, seed=42)
         X_train, y_train = partition_train_test["train"]["image"], partition_train_test[
             "train"]["label"]
         X_test, y_test = partition_train_test["test"]["image"], partition_train_test[

datasets/flwr_datasets/utils.py

@@ -133,6 +133,7 @@ def divide_dataset(
     >>> train_test = divide_dataset(dataset=partition, division=division)
     >>> train, test = train_test["train"], train_test["test"]
     """
+    _check_division_config_correctness(division)
     dataset_length = len(dataset)
     ranges = _create_division_indices_ranges(dataset_length, division)
     if isinstance(division, (list, tuple)):
@@ -162,15 +163,15 @@ def _create_division_indices_ranges(
         for fraction in division:
             end_idx += int(dataset_length * fraction)
             ranges.append(range(start_idx, end_idx))
-            start_idx += end_idx
+            start_idx = end_idx
     elif isinstance(division, dict):
         ranges = []
         start_idx = 0
         end_idx = 0
         for fraction in division.values():
             end_idx += int(dataset_length * fraction)
             ranges.append(range(start_idx, end_idx))
-            start_idx += end_idx
+            start_idx = end_idx
     else:
         TypeError(
             f"The type of the `division` should be dict, "
@@ -274,6 +275,7 @@ def concatenate_divisions(
     concatenated_divisions : Dataset
         A dataset created as concatenation of the divisions from all partitions.
     """
+    _check_division_config_correctness(partition_division)
     divisions = []
     zero_len_divisions = 0
     for partition_id in range(partitioner.num_partitions):

datasets/flwr_datasets/utils_test.py

@@ -31,13 +31,32 @@
         "expected_concatenation_size",
     ),
     [
+        # Create 1 division
+        ((1.0,), [40], 0, 40),
+        ({"train": 1.0}, [40], "train", 40),
+        # Create 2 divisions
         ((0.8, 0.2), [32, 8], 1, 8),
-        ([0.8, 0.2], [32, 8], 1, 8),
         ({"train": 0.8, "test": 0.2}, [32, 8], "test", 8),
+        # Create 3 divisions
+        ([0.6, 0.2, 0.2], [24, 8, 8], 1, 8),
+        ({"train": 0.6, "valid": 0.2, "test": 0.2}, [24, 8, 8], "test", 8),
+        # Create 4 divisions
+        ([0.4, 0.2, 0.2, 0.2], [16, 8, 8, 8], 1, 8),
+        ({"0": 0.4, "1": 0.2, "2": 0.2, "3": 0.2}, [16, 8, 8, 8], "1", 8),
         # Not full dataset
+        # Create 1 division
+        ([0.8], [32], 0, 32),
+        ({"train": 0.8}, [32], "train", 32),
+        # Create 2 divisions
         ([0.2, 0.1], [8, 4], 1, 4),
         ((0.2, 0.1), [8, 4], 0, 8),
         ({"train": 0.2, "test": 0.1}, [8, 4], "test", 4),
+        # Create 3 divisions
+        ([0.6, 0.2, 0.1], [24, 8, 4], 2, 4),
+        ({"train": 0.6, "valid": 0.2, "test": 0.1}, [24, 8, 4], "test", 4),
+        # Create 4 divisions
+        ([0.4, 0.2, 0.1, 0.2], [16, 8, 4, 8], 2, 4),
+        ({"0": 0.4, "1": 0.2, "2": 0.1, "3": 0.2}, [16, 8, 4, 8], "2", 4),
     ],
 )
 class UtilsTests(unittest.TestCase):
@@ -60,7 +79,7 @@ def test_correct_sizes(self) -> None:
         else:
             lengths = [len(split) for split in divided_dataset.values()]
 
-        self.assertEqual(lengths, self.sizes)
+        self.assertEqual(self.sizes, lengths)
 
     def test_correct_return_types(self) -> None:
         """Test correct types of the divided dataset based on the config."""

datasets/pyproject.toml

@@ -54,7 +54,7 @@ exclude = [
 [tool.poetry.dependencies]
 python = "^3.8"
 numpy = "^1.21.0"
-datasets = "^2.14.3"
+datasets = "^2.14.6"
 pillow = { version = ">=6.2.1", optional = true }
 soundfile = { version = ">=0.12.1", optional = true }
 librosa = { version = ">=0.10.0.post2", optional = true }

dev/update-examples.sh

@@ -16,20 +16,47 @@ echo "---" >> $INDEX
 echo "maxdepth: 1" >> $INDEX
 echo "---" >> $INDEX
 
-rm -f "examples/doc/source/*.md"
+rm -f examples/doc/source/*.md
 
 cd examples/
 for d in $(printf '%s\n' */ | sort -V); do
   example=${d%/}
-  # For each example, copy the README into the source of the Example docs
-  [[ $example != doc ]] && cp $example/README.md $ROOT/examples/doc/source/$example.md 2>&1 >/dev/null
-  # For each example, copy all images of the _static folder into the examples
-  # docs static folder
-  [[ $example != doc ]] && [ -d "$example/_static" ] && {
-    cp $example/_static/**.{jpg,png,jpeg} $ROOT/examples/doc/source/_static/ 2>/dev/null || true
-  }
-  # For each example, insert the name of the example into the index file
-  [[ $example != doc ]] && (echo $INSERT_LINE; echo a; echo $example; echo .; echo wq) | ed $INDEX 2>&1 >/dev/null
+
+  if [[ $example != doc ]]; then
+
+    for file in $example/*.md; do
+      # For each example, copy the README into the source of the Example docs
+      if [[ $(basename "$file") = "README.md" ]]; then
+        cp $file $ROOT/examples/doc/source/$example.md 2>&1 >/dev/null
+      else
+        # If the example contains other markdown files, copy them to the source of the Example docs
+        cp $file $ROOT/examples/doc/source/$(basename "$file") 2>&1 >/dev/null
+      fi
+    done
+
+    gh_text="[<img src=\"_static/view-gh.png\" alt=\"View on GitHub\" width=\"200\"/>](https://github.com/adap/flower/blob/main/examples/$example)"
+    readme_file="$ROOT/examples/doc/source/$example.md"
+
+    if ! grep -Fq "$gh_text" "$readme_file"; then
+      awk -v text="$gh_text" '
+      /^# / && !found {
+        print $0 "\n" text;
+        found=1;
+        next;
+      }
+      { print }
+      ' "$readme_file" > tmpfile && mv tmpfile "$readme_file"
+    fi
+
+    # For each example, copy all images of the _static folder into the examples
+    # docs static folder
+    [ -d "$example/_static" ] && {
+      cp $example/_static/**.{jpg,png,jpeg} $ROOT/examples/doc/source/_static/ 2>/dev/null || true
+    }
+    # For each example, insert the name of the example into the index file
+    (echo $INSERT_LINE; echo a; echo $example; echo .; echo wq) | ed $INDEX 2>&1 >/dev/null
+
+  fi
 done
 
 echo "\`\`\`" >> $INDEX

doc/source/conf.py

@@ -86,7 +86,7 @@
 author = "The Flower Authors"
 
 # The full version, including alpha/beta/rc tags
-release = "1.8.0"
+release = "1.9.0"
 
 # -- General configuration ---------------------------------------------------
 

doc/source/contributor-how-to-install-development-versions.rst

@@ -48,13 +48,13 @@ Install ``flwr`` from a specific GitHub branch (``branch-name``):
 Open Jupyter Notebooks on Google Colab
 --------------------------------------
 
-Open the notebook ``doc/source/tutorial-get-started-with-flower-pytorch.ipynb``:
+Open the notebook ``doc/source/tutorial-series-get-started-with-flower-pytorch.ipynb``:
 
-- https://colab.research.google.com/github/adap/flower/blob/main/doc/source/tutorial-get-started-with-flower-pytorch.ipynb
+- https://colab.research.google.com/github/adap/flower/blob/main/doc/source/tutorial-series-get-started-with-flower-pytorch.ipynb
 
 Open a development version of the same notebook from branch `branch-name` by changing ``main`` to ``branch-name`` (right after ``blob``):
 
-- https://colab.research.google.com/github/adap/flower/blob/branch-name/doc/source/tutorial-get-started-with-flower-pytorch.ipynb
+- https://colab.research.google.com/github/adap/flower/blob/branch-name/doc/source/tutorial-series-get-started-with-flower-pytorch.ipynb
 
 Install a `whl` on Google Colab:
 

doc/source/contributor-tutorial-get-started-as-a-contributor.rst

@@ -102,6 +102,33 @@ Run Linters and Tests
 
   $ ./dev/test.sh
 
+Add a pre-commit hook
+~~~~~~~~~~~~~~~~~~~~~
+
+Developers may integrate a pre-commit hook into their workflow utilizing the `pre-commit <https://pre-commit.com/#install>`_ library. The pre-commit hook is configured to execute two primary operations: ``./dev/format.sh`` and ``./dev/test.sh`` scripts.
+
+There are multiple ways developers can use this:
+
+1. Install the pre-commit hook to your local git directory by simply running:
+
+   ::
+      
+      $ pre-commit install
+
+   - Each ``git commit`` will trigger the execution of formatting and linting/test scripts.
+   - If in a hurry, bypass the hook using ``--no-verify`` with the ``git commit`` command.
+     ::
+          
+       $ git commit --no-verify -m "Add new feature"
+    
+2. For developers who prefer not to install the hook permanently, it is possible to execute a one-time check prior to committing changes by using the following command:
+
+   ::
+
+      $ pre-commit run --all-files
+   
+   This executes the formatting and linting checks/tests on all the files without modifying the default behavior of ``git commit``.
+
 Run Github Actions (CI) locally
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~