Merge branch 'main' into add-flwr-run-logstream

doc/source/docker/index.rst

@@ -44,3 +44,4 @@ Run Flower using Docker Compose
    :maxdepth: 1
 
    tutorial-quickstart-docker-compose
+   run-quickstart-examples-docker-compose

doc/source/docker/run-quickstart-examples-docker-compose.rst

@@ -0,0 +1,122 @@
+Run Flower Quickstart Examples with Docker Compose
+==================================================
+
+Flower provides a set of `quickstart examples <https://github.com/adap/flower/tree/main/examples>`_
+to help you get started with the framework. These examples are designed to demonstrate the
+capabilities of Flower and by default run using the Simulation Engine. This guide demonstrates
+how to run them using Flower's Deployment Engine via Docker Compose.
+
+.. important::
+
+   Some quickstart examples may have limitations or requirements that prevent them from running
+   on every environment. For more information, please see `Limitations`_.
+
+Prerequisites
+-------------
+
+Before you start, make sure that:
+
+- The ``flwr`` CLI is :doc:`installed <../how-to-install-flower>` locally.
+- The Docker daemon is running.
+- Docker Compose is `installed <https://docs.docker.com/compose/install/>`_.
+
+Run the Quickstart Example
+--------------------------
+
+#. Clone the quickstart example you like to run. For example, ``quickstart-pytorch``:
+
+   .. code-block:: bash
+
+      $ git clone --depth=1 https://github.com/adap/flower.git \
+           && mv flower/examples/quickstart-pytorch . \
+           && rm -rf flower && cd quickstart-pytorch
+
+#. Download the `compose.yml <https://github.com/adap/flower/blob/main/src/docker/complete/compose.yml>`_ file into the example directory:
+
+   .. code-block:: bash
+
+      $ curl https://raw.githubusercontent.com/adap/flower/refs/heads/main/src/docker/complete/compose.yml \
+          -o compose.yml
+
+#. Build and start the services using the following command:
+
+   .. code-block:: bash
+
+      $ docker compose up --build -d
+
+#. Append the following lines to the end of the ``pyproject.toml`` file and save it:
+
+   .. code-block:: toml
+      :caption: pyproject.toml
+
+      [tool.flwr.federations.local-deployment]
+      address = "127.0.0.1:9093"
+      insecure = true
+
+   .. note::
+
+      You can customize the string that follows ``tool.flwr.federations.`` to fit your needs.
+      However, please note that the string cannot contain a dot (``.``).
+
+      In this example, ``local-deployment`` has been used. Just remember to replace
+      ``local-deployment`` with your chosen name in both the ``tool.flwr.federations.`` string
+      and the corresponding ``flwr run .`` command.
+
+#. Run the example:
+
+   .. code-block:: bash
+
+      $ flwr run . local-deployment
+
+#. Follow the logs of the SuperExec service:
+
+   .. code-block:: bash
+
+      $ docker compose logs superexec -f
+
+That is all it takes! You can monitor the progress of the run through the logs of the SuperExec.
+
+Run a Different Quickstart Example
+----------------------------------
+
+To run a different quickstart example, such as ``quickstart-tensorflow``, first, shut down the Docker
+Compose services of the current example:
+
+.. code-block:: bash
+
+   $ docker compose down
+
+After that, you can repeat the steps above.
+
+Limitations
+-----------
+
+.. list-table::
+   :header-rows: 1
+
+   * - Quickstart Example
+     - Limitations
+   * - quickstart-fastai
+     - None
+   * - examples/quickstart-huggingface
+     - For CPU-only environments, it requires at least 32GB of memory.
+   * - quickstart-jax
+     - The example has not yet been updated to work with the latest ``flwr`` version.
+   * - quickstart-mlcube
+     - The example has not yet been updated to work with the latest ``flwr`` version.
+   * - quickstart-mlx
+     - `Requires to run on macOS with Apple Silicon <https://ml-explore.github.io/mlx/build/html/install.html#python-installation>`_.
+   * - quickstart-monai
+     - None
+   * - quickstart-pandas
+     - The example has not yet been updated to work with the latest ``flwr`` version.
+   * - quickstart-pytorch-lightning
+     - Requires an older pip version that is not supported by the Flower Docker images.
+   * - quickstart-pytorch
+     - None
+   * - quickstart-sklearn-tabular
+     - None
+   * - quickstart-tabnet
+     - The example has not yet been updated to work with the latest ``flwr`` version.
+   * - quickstart-tensorflow
+     - Only runs on AMD64.

doc/source/docker/tutorial-quickstart-docker-compose.rst

@@ -396,3 +396,8 @@ Remove all services and volumes:
 
    $ docker compose down -v
    $ docker compose -f certs.yml down -v
+
+Where to Go Next
+----------------
+
+* :doc:`run-quickstart-examples-docker-compose`

examples/flower-secure-aggregation/secaggexample/server_app.py

@@ -40,6 +40,7 @@ def main(driver: Driver, context: Context) -> None:
     strategy = FedAvg(
         # Select all available clients
         fraction_fit=1.0,
+        min_fit_clients=5,
         # Disable evaluation in demo
         fraction_evaluate=(0.0 if is_demo else context.run_config["fraction-evaluate"]),
         min_available_clients=5,

examples/quickstart-mlx/README.md

@@ -67,4 +67,4 @@ flwr run . --run-config "num-server-rounds=5 learning-rate=0.05"
 ### Run with the Deployment Engine
 
 > \[!NOTE\]
-> An update to this example will show how to run this Flower project with the Deployment Engine and TLS certificates, or with Docker.
+> An update to this example will show how to run this Flower project with the Deployment Engine and TLS certificates.

src/proto/flwr/proto/control.proto

@@ -0,0 +1,32 @@
+// Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ==============================================================================
+
+syntax = "proto3";
+
+package flwr.proto;
+
+import "flwr/proto/run.proto";
+
+service Control {
+  // Request to create a new run
+  rpc CreateRun(CreateRunRequest) returns (CreateRunResponse) {}
+
+  // Get the status of a given run
+  rpc GetRunStatus(GetRunStatusRequest) returns (GetRunStatusResponse) {}
+
+  // Update the status of a given run
+  rpc UpdateRunStatus(UpdateRunStatusRequest)
+      returns (UpdateRunStatusResponse) {}
+}

src/proto/flwr/proto/driver.proto

@@ -21,7 +21,6 @@ import "flwr/proto/node.proto";
 import "flwr/proto/task.proto";
 import "flwr/proto/run.proto";
 import "flwr/proto/fab.proto";
-import "flwr/proto/transport.proto";
 
 service Driver {
   // Request run_id
@@ -43,15 +42,6 @@ service Driver {
   rpc GetFab(GetFabRequest) returns (GetFabResponse) {}
 }
 
-// CreateRun
-message CreateRunRequest {
-  string fab_id = 1;
-  string fab_version = 2;
-  map<string, Scalar> override_config = 3;
-  Fab fab = 4;
-}
-message CreateRunResponse { uint64 run_id = 1; }
-
 // GetNodes messages
 message GetNodesRequest { uint64 run_id = 1; }
 message GetNodesResponse { repeated Node nodes = 1; }

src/proto/flwr/proto/run.proto

@@ -17,6 +17,7 @@ syntax = "proto3";
 
 package flwr.proto;
 
+import "flwr/proto/fab.proto";
 import "flwr/proto/transport.proto";
 
 message Run {
@@ -26,5 +27,36 @@ message Run {
   map<string, Scalar> override_config = 4;
   string fab_hash = 5;
 }
+
+message RunStatus {
+  // "starting", "running", "finished"
+  string status = 1;
+  // "completed", "failed", "stopped" or "" (non-finished)
+  string sub_status = 2;
+  // failure details
+  string details = 3;
+}
+
+// CreateRun
+message CreateRunRequest {
+  string fab_id = 1;
+  string fab_version = 2;
+  map<string, Scalar> override_config = 3;
+  Fab fab = 4;
+}
+message CreateRunResponse { uint64 run_id = 1; }
+
+// GetRun
 message GetRunRequest { uint64 run_id = 1; }
 message GetRunResponse { Run run = 1; }
+
+// UpdateRunStatus
+message UpdateRunStatusRequest {
+  uint64 run_id = 1;
+  RunStatus run_status = 2;
+}
+message UpdateRunStatusResponse {}
+
+// GetRunStatus
+message GetRunStatusRequest { repeated uint64 run_ids = 1; }
+message GetRunStatusResponse { map<uint64, RunStatus> run_status_dict = 1; }

src/py/flwr/cli/app.py

@@ -19,6 +19,7 @@
 
 from .build import build
 from .install import install
+from .log import log
 from .new import new
 from .run import run
 
@@ -35,6 +36,7 @@
 app.command()(run)
 app.command()(build)
 app.command()(install)
+app.command()(log)
 
 typer_click_object = get_command(app)