Merge branch 'main' into add-flwr-log-functions

doc/source/docker/index.rst

@@ -44,3 +44,4 @@ Run Flower using Docker Compose
    :maxdepth: 1
 
    tutorial-quickstart-docker-compose
+   run-quickstart-examples-docker-compose

doc/source/docker/run-quickstart-examples-docker-compose.rst

@@ -0,0 +1,122 @@
+Run Flower Quickstart Examples with Docker Compose
+==================================================
+
+Flower provides a set of `quickstart examples <https://github.com/adap/flower/tree/main/examples>`_
+to help you get started with the framework. These examples are designed to demonstrate the
+capabilities of Flower and by default run using the Simulation Engine. This guide demonstrates
+how to run them using Flower's Deployment Engine via Docker Compose.
+
+.. important::
+
+   Some quickstart examples may have limitations or requirements that prevent them from running
+   on every environment. For more information, please see `Limitations`_.
+
+Prerequisites
+-------------
+
+Before you start, make sure that:
+
+- The ``flwr`` CLI is :doc:`installed <../how-to-install-flower>` locally.
+- The Docker daemon is running.
+- Docker Compose is `installed <https://docs.docker.com/compose/install/>`_.
+
+Run the Quickstart Example
+--------------------------
+
+#. Clone the quickstart example you like to run. For example, ``quickstart-pytorch``:
+
+   .. code-block:: bash
+
+      $ git clone --depth=1 https://github.com/adap/flower.git \
+           && mv flower/examples/quickstart-pytorch . \
+           && rm -rf flower && cd quickstart-pytorch
+
+#. Download the `compose.yml <https://github.com/adap/flower/blob/main/src/docker/complete/compose.yml>`_ file into the example directory:
+
+   .. code-block:: bash
+
+      $ curl https://raw.githubusercontent.com/adap/flower/refs/heads/main/src/docker/complete/compose.yml \
+          -o compose.yml
+
+#. Build and start the services using the following command:
+
+   .. code-block:: bash
+
+      $ docker compose up --build -d
+
+#. Append the following lines to the end of the ``pyproject.toml`` file and save it:
+
+   .. code-block:: toml
+      :caption: pyproject.toml
+
+      [tool.flwr.federations.local-deployment]
+      address = "127.0.0.1:9093"
+      insecure = true
+
+   .. note::
+
+      You can customize the string that follows ``tool.flwr.federations.`` to fit your needs.
+      However, please note that the string cannot contain a dot (``.``).
+
+      In this example, ``local-deployment`` has been used. Just remember to replace
+      ``local-deployment`` with your chosen name in both the ``tool.flwr.federations.`` string
+      and the corresponding ``flwr run .`` command.
+
+#. Run the example:
+
+   .. code-block:: bash
+
+      $ flwr run . local-deployment
+
+#. Follow the logs of the SuperExec service:
+
+   .. code-block:: bash
+
+      $ docker compose logs superexec -f
+
+That is all it takes! You can monitor the progress of the run through the logs of the SuperExec.
+
+Run a Different Quickstart Example
+----------------------------------
+
+To run a different quickstart example, such as ``quickstart-tensorflow``, first, shut down the Docker
+Compose services of the current example:
+
+.. code-block:: bash
+
+   $ docker compose down
+
+After that, you can repeat the steps above.
+
+Limitations
+-----------
+
+.. list-table::
+   :header-rows: 1
+
+   * - Quickstart Example
+     - Limitations
+   * - quickstart-fastai
+     - None
+   * - examples/quickstart-huggingface
+     - For CPU-only environments, it requires at least 32GB of memory.
+   * - quickstart-jax
+     - The example has not yet been updated to work with the latest ``flwr`` version.
+   * - quickstart-mlcube
+     - The example has not yet been updated to work with the latest ``flwr`` version.
+   * - quickstart-mlx
+     - `Requires to run on macOS with Apple Silicon <https://ml-explore.github.io/mlx/build/html/install.html#python-installation>`_.
+   * - quickstart-monai
+     - None
+   * - quickstart-pandas
+     - The example has not yet been updated to work with the latest ``flwr`` version.
+   * - quickstart-pytorch-lightning
+     - Requires an older pip version that is not supported by the Flower Docker images.
+   * - quickstart-pytorch
+     - None
+   * - quickstart-sklearn-tabular
+     - None
+   * - quickstart-tabnet
+     - The example has not yet been updated to work with the latest ``flwr`` version.
+   * - quickstart-tensorflow
+     - Only runs on AMD64.

doc/source/docker/tutorial-quickstart-docker-compose.rst

@@ -396,3 +396,8 @@ Remove all services and volumes:
 
    $ docker compose down -v
    $ docker compose -f certs.yml down -v
+
+Where to Go Next
+----------------
+
+* :doc:`run-quickstart-examples-docker-compose`

examples/quickstart-mlx/README.md

@@ -67,4 +67,4 @@ flwr run . --run-config "num-server-rounds=5 learning-rate=0.05"
 ### Run with the Deployment Engine
 
 > \[!NOTE\]
-> An update to this example will show how to run this Flower project with the Deployment Engine and TLS certificates, or with Docker.
+> An update to this example will show how to run this Flower project with the Deployment Engine and TLS certificates.

src/proto/flwr/proto/control.proto

@@ -0,0 +1,25 @@
+// Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ==============================================================================
+
+syntax = "proto3";
+
+package flwr.proto;
+
+import "flwr/proto/run.proto";
+
+service Control {
+  // Request to create a new run
+  rpc CreateRun(CreateRunRequest) returns (CreateRunResponse) {}
+}

src/proto/flwr/proto/driver.proto

@@ -21,7 +21,6 @@ import "flwr/proto/node.proto";
 import "flwr/proto/task.proto";
 import "flwr/proto/run.proto";
 import "flwr/proto/fab.proto";
-import "flwr/proto/transport.proto";
 
 service Driver {
   // Request run_id
@@ -43,15 +42,6 @@ service Driver {
   rpc GetFab(GetFabRequest) returns (GetFabResponse) {}
 }
 
-// CreateRun
-message CreateRunRequest {
-  string fab_id = 1;
-  string fab_version = 2;
-  map<string, Scalar> override_config = 3;
-  Fab fab = 4;
-}
-message CreateRunResponse { uint64 run_id = 1; }
-
 // GetNodes messages
 message GetNodesRequest { uint64 run_id = 1; }
 message GetNodesResponse { repeated Node nodes = 1; }

src/proto/flwr/proto/run.proto

@@ -17,6 +17,7 @@ syntax = "proto3";
 
 package flwr.proto;
 
+import "flwr/proto/fab.proto";
 import "flwr/proto/transport.proto";
 
 message Run {
@@ -26,5 +27,16 @@ message Run {
   map<string, Scalar> override_config = 4;
   string fab_hash = 5;
 }
+
+// CreateRun
+message CreateRunRequest {
+  string fab_id = 1;
+  string fab_version = 2;
+  map<string, Scalar> override_config = 3;
+  Fab fab = 4;
+}
+message CreateRunResponse { uint64 run_id = 1; }
+
+// GetRun
 message GetRunRequest { uint64 run_id = 1; }
 message GetRunResponse { Run run = 1; }

src/py/flwr/cli/new/new.py

@@ -275,7 +275,7 @@ def new(
         )
     )
 
-    _add = "	huggingface-cli login\n" if framework_str == "flowertune" else ""
+    _add = "	huggingface-cli login\n" if llm_challenge_str else ""
     print(
         typer.style(
             f"	cd {package_name}\n" + "	pip install -e .\n" + _add + "	flwr run\n",

src/py/flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl

@@ -17,6 +17,7 @@ dependencies = [
     "transformers==4.39.3",
     "sentencepiece==0.2.0",
     "omegaconf==2.3.0",
+    "hf_transfer==0.1.8",
 ]
 
 [tool.hatch.build.targets.wheel]

src/py/flwr/proto/control_pb2.pyi

@@ -0,0 +1,7 @@
+"""
+@generated by mypy-protobuf.  Do not edit manually!
+isort:skip_file
+"""
+import google.protobuf.descriptor
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor

src/py/flwr/proto/control_pb2_grpc.py

@@ -0,0 +1,67 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+from flwr.proto import run_pb2 as flwr_dot_proto_dot_run__pb2
+
+
+class ControlStub(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def __init__(self, channel):
+        """Constructor.
+
+        Args:
+            channel: A grpc.Channel.
+        """
+        self.CreateRun = channel.unary_unary(
+                '/flwr.proto.Control/CreateRun',
+                request_serializer=flwr_dot_proto_dot_run__pb2.CreateRunRequest.SerializeToString,
+                response_deserializer=flwr_dot_proto_dot_run__pb2.CreateRunResponse.FromString,
+                )
+
+
+class ControlServicer(object):
+    """Missing associated documentation comment in .proto file."""
+
+    def CreateRun(self, request, context):
+        """Request to create a new run
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+
+def add_ControlServicer_to_server(servicer, server):
+    rpc_method_handlers = {
+            'CreateRun': grpc.unary_unary_rpc_method_handler(
+                    servicer.CreateRun,
+                    request_deserializer=flwr_dot_proto_dot_run__pb2.CreateRunRequest.FromString,
+                    response_serializer=flwr_dot_proto_dot_run__pb2.CreateRunResponse.SerializeToString,
+            ),
+    }
+    generic_handler = grpc.method_handlers_generic_handler(
+            'flwr.proto.Control', rpc_method_handlers)
+    server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class Control(object):
+    """Missing associated documentation comment in .proto file."""
+
+    @staticmethod
+    def CreateRun(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/flwr.proto.Control/CreateRun',
+            flwr_dot_proto_dot_run__pb2.CreateRunRequest.SerializeToString,
+            flwr_dot_proto_dot_run__pb2.CreateRunResponse.FromString,
+            options, channel_credentials,
+            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

src/py/flwr/proto/control_pb2_grpc.pyi

@@ -0,0 +1,27 @@
+"""
+@generated by mypy-protobuf.  Do not edit manually!
+isort:skip_file
+"""
+import abc
+import flwr.proto.run_pb2
+import grpc
+
+class ControlStub:
+    def __init__(self, channel: grpc.Channel) -> None: ...
+    CreateRun: grpc.UnaryUnaryMultiCallable[
+        flwr.proto.run_pb2.CreateRunRequest,
+        flwr.proto.run_pb2.CreateRunResponse]
+    """Request to create a new run"""
+
+
+class ControlServicer(metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def CreateRun(self,
+        request: flwr.proto.run_pb2.CreateRunRequest,
+        context: grpc.ServicerContext,
+    ) -> flwr.proto.run_pb2.CreateRunResponse:
+        """Request to create a new run"""
+        pass
+
+
+def add_ControlServicer_to_server(servicer: ControlServicer, server: grpc.Server) -> None: ...