Merge branch 'main' into fds-add-typing-module

.github/workflows/framework.yml

@@ -25,7 +25,7 @@ jobs:
         # In case of a mismatch, the job has to download Python to install it.
         # Note: Due to a bug in actions/setup-python we have to put 3.10 in
         # qoutes as it will otherwise will assume 3.1
-        python: [3.8, 3.9, '3.10']
+        python: [3.8, 3.9, '3.10', '3.11']
 
     name: Python ${{ matrix.python }}
 

datasets/e2e/tensorflow/pyproject.toml

@@ -12,4 +12,5 @@ authors = ["The Flower Authors <[email protected]>"]
 python = ">=3.8,<3.11"
 flwr-datasets = { path = "./../../", extras = ["vision"] }
 tensorflow-cpu = "^2.9.1, !=2.11.1"
+tensorflow-io-gcs-filesystem = "<0.35.0"
 parameterized = "==0.9.0"

doc/source/contributor-how-to-release-flower.rst

@@ -3,23 +3,15 @@ Release Flower
 
 This document describes the current release process. It may or may not change in the future.
 
-Before the release
-------------------
-
-Update the changelog (``changelog.md``) with all relevant changes that happened after the last release. If the last release was tagged ``v1.2.0``, you can use the following URL to see all commits that got merged into ``main`` since then:
-
-`GitHub: Compare v1.2.0...main <https://github.com/adap/flower/compare/v1.2.0...main>`_
-
-Thank the authors who contributed since the last release. This can be done by running the ``./dev/add-shortlog.sh <new version>`` convenience script (it can be ran multiple times and will update the names in the list if new contributors were added in the meantime).
-
 During the release
 ------------------
 
 The version number of a release is stated in ``pyproject.toml``. To release a new version of Flower, the following things need to happen (in that order):
 
-1. Update the ``changelog.md`` section header ``Unreleased`` to contain the version number and date for the release you are building. Create a pull request with the change.
-2. Tag the release commit with the version number as soon as the PR is merged: ``git tag v0.12.3``, then ``git push --tags``. This will create a draft release on GitHub containing the correct artifacts and the relevant part of the changelog.
-3. Check the draft release on GitHub, and if everything is good, publish it.
+1. Run ``python3 src/py/flwr_tool/update_changelog.py <YOUR_GH_TOKEN>`` in order to add every new change to the changelog (feel free to make manual changes to the changelog afterwards until it looks good).
+2. Once the changelog has been updated with all the changes, run ``./dev/prepare-release-changelog.sh v<NEW_VERSION>``, where ``<NEW_VERSION>`` is the version stated in ``pyproject.toml`` (notice the ``v`` added before it). This will replace the ``Unreleased`` header of the changelog by the version and current date, and it will add a thanking message for the contributors. Open a pull request with those changes.
+3. Once the pull request is merged, tag the release commit with the version number as soon as the PR is merged: ``git tag v<NEW_VERSION>`` (notice the ``v`` added before the version number), then ``git push --tags``. This will create a draft release on GitHub containing the correct artifacts and the relevant part of the changelog.
+4. Check the draft release on GitHub, and if everything is good, publish it.
 
 After the release
 -----------------

pyproject.toml

@@ -61,8 +61,8 @@ flower-client = "flwr.client:run_client"
 python = "^3.8"
 # Mandatory dependencies
 numpy = "^1.21.0"
-grpcio = "^1.48.2,!=1.52.0"
-protobuf = "^3.19.0"
+grpcio = "^1.60.0"
+protobuf = "^4.25.2"
 cryptography = "^41.0.2"
 pycryptodome = "^3.18.0"
 iterators = "^0.0.2"
@@ -93,7 +93,7 @@ flake8 = "==5.0.4"
 pytest = "==7.4.3"
 pytest-cov = "==4.1.0"
 pytest-watch = "==4.2.0"
-grpcio-tools = "==1.48.2"
+grpcio-tools = "==1.60.0"
 mypy-protobuf = "==3.2.0"
 jupyterlab = "==4.0.9"
 rope = "==1.11.0"

src/py/flwr/client/app.py

@@ -35,7 +35,7 @@
     TRANSPORT_TYPES,
 )
 from flwr.common.logger import log, warn_experimental_feature
-from flwr.proto.task_pb2 import TaskIns, TaskRes
+from flwr.proto.task_pb2 import TaskIns, TaskRes  # pylint: disable=E0611
 
 from .flower import load_flower_callable
 from .grpc_client.connection import grpc_connection

src/py/flwr/client/grpc_client/connection.py

@@ -25,10 +25,13 @@
 from flwr.common import GRPC_MAX_MESSAGE_LENGTH
 from flwr.common.grpc import create_channel
 from flwr.common.logger import log
-from flwr.proto.node_pb2 import Node
-from flwr.proto.task_pb2 import Task, TaskIns, TaskRes
-from flwr.proto.transport_pb2 import ClientMessage, ServerMessage
-from flwr.proto.transport_pb2_grpc import FlowerServiceStub
+from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
+from flwr.proto.task_pb2 import Task, TaskIns, TaskRes  # pylint: disable=E0611
+from flwr.proto.transport_pb2 import (  # pylint: disable=E0611
+    ClientMessage,
+    ServerMessage,
+)
+from flwr.proto.transport_pb2_grpc import FlowerServiceStub  # pylint: disable=E0611
 
 # The following flags can be uncommented for debugging. Other possible values:
 # https://github.com/grpc/grpc/blob/master/doc/environment_variables.md

src/py/flwr/client/grpc_client/connection_test.py

@@ -23,8 +23,11 @@
 
 import grpc
 
-from flwr.proto.task_pb2 import Task, TaskRes
-from flwr.proto.transport_pb2 import ClientMessage, ServerMessage
+from flwr.proto.task_pb2 import Task, TaskRes  # pylint: disable=E0611
+from flwr.proto.transport_pb2 import (  # pylint: disable=E0611
+    ClientMessage,
+    ServerMessage,
+)
 from flwr.server.client_manager import SimpleClientManager
 from flwr.server.fleet.grpc_bidi.grpc_server import start_grpc_server
 

src/py/flwr/client/grpc_rere_client/connection.py

@@ -29,15 +29,15 @@
 from flwr.common import GRPC_MAX_MESSAGE_LENGTH
 from flwr.common.grpc import create_channel
 from flwr.common.logger import log, warn_experimental_feature
-from flwr.proto.fleet_pb2 import (
+from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     DeleteNodeRequest,
     PullTaskInsRequest,
     PushTaskResRequest,
 )
-from flwr.proto.fleet_pb2_grpc import FleetStub
-from flwr.proto.node_pb2 import Node
-from flwr.proto.task_pb2 import TaskIns, TaskRes
+from flwr.proto.fleet_pb2_grpc import FleetStub  # pylint: disable=E0611
+from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
+from flwr.proto.task_pb2 import TaskIns, TaskRes  # pylint: disable=E0611
 
 KEY_NODE = "node"
 KEY_TASK_INS = "current_task_ins"

src/py/flwr/client/message_handler/message_handler.py

@@ -32,8 +32,17 @@
 from flwr.client.secure_aggregation import SecureAggregationHandler
 from flwr.client.typing import ClientFn
 from flwr.common import serde
-from flwr.proto.task_pb2 import SecureAggregation, Task, TaskIns, TaskRes
-from flwr.proto.transport_pb2 import ClientMessage, Reason, ServerMessage
+from flwr.proto.task_pb2 import (  # pylint: disable=E0611
+    SecureAggregation,
+    Task,
+    TaskIns,
+    TaskRes,
+)
+from flwr.proto.transport_pb2 import (  # pylint: disable=E0611
+    ClientMessage,
+    Reason,
+    ServerMessage,
+)
 
 
 class UnexpectedServerMessage(Exception):

src/py/flwr/client/message_handler/message_handler_test.py

@@ -33,9 +33,14 @@
     serde,
     typing,
 )
-from flwr.proto.node_pb2 import Node
-from flwr.proto.task_pb2 import Task, TaskIns, TaskRes
-from flwr.proto.transport_pb2 import ClientMessage, Code, ServerMessage, Status
+from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
+from flwr.proto.task_pb2 import Task, TaskIns, TaskRes  # pylint: disable=E0611
+from flwr.proto.transport_pb2 import (  # pylint: disable=E0611
+    ClientMessage,
+    Code,
+    ServerMessage,
+    Status,
+)
 
 from .message_handler import handle, handle_control_message
 

src/py/flwr/client/message_handler/task_handler.py

@@ -17,10 +17,13 @@
 
 from typing import Optional
 
-from flwr.proto.fleet_pb2 import PullTaskInsResponse
-from flwr.proto.node_pb2 import Node
-from flwr.proto.task_pb2 import Task, TaskIns, TaskRes
-from flwr.proto.transport_pb2 import ClientMessage, ServerMessage
+from flwr.proto.fleet_pb2 import PullTaskInsResponse  # pylint: disable=E0611
+from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
+from flwr.proto.task_pb2 import Task, TaskIns, TaskRes  # pylint: disable=E0611
+from flwr.proto.transport_pb2 import (  # pylint: disable=E0611
+    ClientMessage,
+    ServerMessage,
+)
 
 
 def validate_task_ins(task_ins: TaskIns, discard_reconnect_ins: bool) -> bool:

src/py/flwr/client/message_handler/task_handler_test.py

@@ -22,9 +22,17 @@
     validate_task_res,
     wrap_client_message_in_task_res,
 )
-from flwr.proto.fleet_pb2 import PullTaskInsResponse
-from flwr.proto.task_pb2 import SecureAggregation, Task, TaskIns, TaskRes
-from flwr.proto.transport_pb2 import ClientMessage, ServerMessage
+from flwr.proto.fleet_pb2 import PullTaskInsResponse  # pylint: disable=E0611
+from flwr.proto.task_pb2 import (  # pylint: disable=E0611
+    SecureAggregation,
+    Task,
+    TaskIns,
+    TaskRes,
+)
+from flwr.proto.transport_pb2 import (  # pylint: disable=E0611
+    ClientMessage,
+    ServerMessage,
+)
 
 
 def test_validate_task_ins_no_task() -> None:

src/py/flwr/client/middleware/utils_test.py

@@ -20,7 +20,7 @@
 
 from flwr.client.run_state import RunState
 from flwr.client.typing import Bwd, FlowerCallable, Fwd, Layer
-from flwr.proto.task_pb2 import TaskIns, TaskRes
+from flwr.proto.task_pb2 import TaskIns, TaskRes  # pylint: disable=E0611
 
 from .utils import make_ffn
 

src/py/flwr/client/node_state_tests.py

@@ -17,7 +17,7 @@
 
 from flwr.client.node_state import NodeState
 from flwr.client.run_state import RunState
-from flwr.proto.task_pb2 import TaskIns
+from flwr.proto.task_pb2 import TaskIns  # pylint: disable=E0611
 
 
 def _run_dummy_task(state: RunState) -> RunState:

src/py/flwr/client/rest_client/connection.py

@@ -29,7 +29,7 @@
 from flwr.common import GRPC_MAX_MESSAGE_LENGTH
 from flwr.common.constant import MISSING_EXTRA_REST
 from flwr.common.logger import log
-from flwr.proto.fleet_pb2 import (
+from flwr.proto.fleet_pb2 import (  # pylint: disable=E0611
     CreateNodeRequest,
     CreateNodeResponse,
     DeleteNodeRequest,
@@ -38,8 +38,8 @@
     PushTaskResRequest,
     PushTaskResResponse,
 )
-from flwr.proto.node_pb2 import Node
-from flwr.proto.task_pb2 import TaskIns, TaskRes
+from flwr.proto.node_pb2 import Node  # pylint: disable=E0611
+from flwr.proto.task_pb2 import TaskIns, TaskRes  # pylint: disable=E0611
 
 try:
     import requests

src/py/flwr/client/typing.py

@@ -18,7 +18,7 @@
 from typing import Callable
 
 from flwr.client.run_state import RunState
-from flwr.proto.task_pb2 import TaskIns, TaskRes
+from flwr.proto.task_pb2 import TaskIns, TaskRes  # pylint: disable=E0611
 
 from .client import Client as Client
 

src/py/flwr/common/configsrecord.py

@@ -0,0 +1,98 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""ConfigsRecord."""
+
+
+from dataclasses import dataclass, field
+from typing import Dict, Optional, get_args
+
+from .typing import ConfigsRecordValues, ConfigsScalar
+
+
+@dataclass
+class ConfigsRecord:
+    """Configs record."""
+
+    keep_input: bool
+    data: Dict[str, ConfigsRecordValues] = field(default_factory=dict)
+
+    def __init__(
+        self,
+        configs_dict: Optional[Dict[str, ConfigsRecordValues]] = None,
+        keep_input: bool = True,
+    ):
+        """Construct a ConfigsRecord object.
+
+        Parameters
+        ----------
+        configs_dict : Optional[Dict[str, ConfigsRecordValues]]
+            A dictionary that stores basic types (i.e. `str`, `int`, `float`, `bytes` as
+            defined in `ConfigsScalar`) and lists of such types (see
+            `ConfigsScalarList`).
+        keep_input : bool (default: True)
+            A boolean indicating whether config passed should be deleted from the input
+            dictionary immediately after adding them to the record. When set
+            to True, the data is duplicated in memory. If memory is a concern, set
+            it to False.
+        """
+        self.keep_input = keep_input
+        self.data = {}
+        if configs_dict:
+            self.set_configs(configs_dict)
+
+    def set_configs(self, configs_dict: Dict[str, ConfigsRecordValues]) -> None:
+        """Add configs to the record.
+
+        Parameters
+        ----------
+        configs_dict : Dict[str, ConfigsRecordValues]
+            A dictionary that stores basic types (i.e. `str`,`int`, `float`, `bytes` as
+            defined in `ConfigsRecordValues`) and list of such types (see
+            `ConfigsScalarList`).
+        """
+        if any(not isinstance(k, str) for k in configs_dict.keys()):
+            raise TypeError(f"Not all keys are of valid type. Expected {str}")
+
+        def is_valid(value: ConfigsScalar) -> None:
+            """Check if value is of expected type."""
+            if not isinstance(value, get_args(ConfigsScalar)):
+                raise TypeError(
+                    "Not all values are of valid type."
+                    f" Expected {ConfigsRecordValues} but you passed {type(value)}."
+                )
+
+        # Check types of values
+        # Split between those values that are list and those that aren't
+        # then process in the same way
+        for value in configs_dict.values():
+            if isinstance(value, list):
+                # If your lists are large (e.g. 1M+ elements) this will be slow
+                # 1s to check 10M element list on a M2 Pro
+                # In such settings, you'd be better of treating such config as
+                # an array and pass it to a ParametersRecord.
+                for list_value in value:
+                    is_valid(list_value)
+            else:
+                is_valid(value)
+
+        # Add configs to record
+        if self.keep_input:
+            # Copy
+            self.data = configs_dict.copy()
+        else:
+            # Add entries to dataclass without duplicating memory
+            for key in list(configs_dict.keys()):
+                self.data[key] = configs_dict[key]
+                del configs_dict[key]