Merge branch 'main' into introduce-flwr-simulation-process

.github/workflows/docker-build-main.yml

@@ -56,7 +56,6 @@ jobs:
           { repository: "flwr/superlink", file_dir: "src/docker/superlink" },
           { repository: "flwr/supernode", file_dir: "src/docker/supernode" },
           { repository: "flwr/serverapp", file_dir: "src/docker/serverapp" },
-          { repository: "flwr/superexec", file_dir: "src/docker/superexec" },
           { repository: "flwr/clientapp", file_dir: "src/docker/clientapp" }
         ]
     with:

.github/workflows/release-nightly.yml

@@ -70,7 +70,6 @@ jobs:
           { repository: "flwr/superlink", file_dir: "src/docker/superlink" },
           { repository: "flwr/supernode", file_dir: "src/docker/supernode" },
           { repository: "flwr/serverapp", file_dir: "src/docker/serverapp" },
-          { repository: "flwr/superexec", file_dir: "src/docker/superexec" },
           { repository: "flwr/clientapp", file_dir: "src/docker/clientapp" }
         ]
     with:

benchmarks/flowertune-llm/evaluation/finance/benchmarks.py

@@ -122,7 +122,10 @@ def inference(dataset, model, tokenizer, batch_size):
                 **tokens, max_length=512, eos_token_id=tokenizer.eos_token_id
             )
             res_sentences = [tokenizer.decode(i, skip_special_tokens=True) for i in res]
-            out_text = [o.split("Answer: ")[1] for o in res_sentences]
+            out_text = [
+                o.split("Answer: ")[1] if len(o.split("Answer: ")) > 1 else "None"
+                for o in res_sentences
+            ]
             out_text_list += out_text
             torch.cuda.empty_cache()
 

dev/build-docker-image-matrix.py

@@ -172,13 +172,6 @@ def tag_latest_ubuntu_with_flwr_version(image: BaseImage) -> List[str]:
             lambda image: image.distro.name == DistroName.UBUNTU,
         )
         # ubuntu images for each supported python version
-        + generate_binary_images(
-            "superexec",
-            base_images,
-            tag_latest_ubuntu_with_flwr_version,
-            lambda image: image.distro.name == DistroName.UBUNTU,
-        )
-        # ubuntu images for each supported python version
         + generate_binary_images(
             "clientapp",
             base_images,

doc/source/docker/run-quickstart-examples-docker-compose.rst

@@ -39,13 +39,16 @@ Run the Quickstart Example
    .. code-block:: bash
        :substitutions:
 
-       $ curl https://raw.githubusercontent.com/adap/flower/refs/tags/v|stable_flwr_version|/src/docker/complete/compose.yml \
+       $ curl https://raw.githubusercontent.com/adap/flower/24b2861465431a5ab234a8c4f76faea7a742b1fd/src/docker/complete/compose.yml \
            -o compose.yml
 
-3. Build and start the services using the following command:
+3. Export the version of Flower that your environment uses. Then, build and start the
+   services using the following command:
 
    .. code-block:: bash
+       :substitutions:
 
+       $ export FLWR_VERSION="|stable_flwr_version|" # update with your version
        $ docker compose up --build -d
 
 4. Append the following lines to the end of the ``pyproject.toml`` file and save it:

src/docker/base/ubuntu-cuda/Dockerfile

@@ -0,0 +1,116 @@
+# Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# hadolint global ignore=DL3008
+ARG CUDA_VERSION=12.4.1
+ARG DISTRO=ubuntu
+ARG DISTRO_VERSION=24.04
+FROM nvidia/cuda:${CUDA_VERSION}-base-${DISTRO}${DISTRO_VERSION} AS python
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install system dependencies
+RUN apt-get update \
+    && apt-get -y --no-install-recommends install \
+    clang-format git unzip ca-certificates openssh-client liblzma-dev \
+    build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev wget\
+    libsqlite3-dev curl llvm libncursesw5-dev xz-utils tk-dev libxml2-dev \
+    libxmlsec1-dev libffi-dev liblzma-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install PyEnv and Python
+ARG PYTHON_VERSION=3.11
+ENV PYENV_ROOT=/root/.pyenv
+ENV PATH=$PYENV_ROOT/bin:$PATH
+# https://github.com/hadolint/hadolint/wiki/DL4006
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
+
+# hadolint ignore=DL3003
+RUN git clone https://github.com/pyenv/pyenv.git \
+    && cd pyenv/plugins/python-build || exit \
+    && ./install.sh
+
+# Issue: python-build only accepts the exact Python version e.g. 3.11.1 but
+# we want to allow more general versions like 3.11
+# Solution: first use pyenv to get the exact version and then pass it to python-build
+RUN LATEST=$(pyenv latest -k ${PYTHON_VERSION}) \
+    && python-build "${LATEST}" /usr/local/bin/python
+
+ENV PATH=/usr/local/bin/python/bin:$PATH
+
+ARG PIP_VERSION
+ARG SETUPTOOLS_VERSION
+# Keep the version of system Python pip and setuptools in sync with those installed in the
+# virtualenv.
+RUN pip install -U --no-cache-dir pip==${PIP_VERSION} setuptools==${SETUPTOOLS_VERSION} \
+    # Use a virtual environment to ensure that Python packages are installed in the same location
+    # regardless of whether the subsequent image build is run with the app or the root user
+    && python -m venv /python/venv
+ENV PATH=/python/venv/bin:$PATH
+
+RUN pip install -U --no-cache-dir \
+    pip==${PIP_VERSION} \
+    setuptools==${SETUPTOOLS_VERSION}
+
+ARG FLWR_VERSION
+ARG FLWR_VERSION_REF
+ARG FLWR_PACKAGE=flwr
+# hadolint ignore=DL3013
+RUN if [ -z "${FLWR_VERSION_REF}" ]; then \
+    pip install -U --no-cache-dir ${FLWR_PACKAGE}==${FLWR_VERSION}; \
+    else \
+    pip install -U --no-cache-dir ${FLWR_PACKAGE}@${FLWR_VERSION_REF}; \
+    fi
+
+FROM nvidia/cuda:${CUDA_VERSION}-base-${DISTRO}${DISTRO_VERSION} AS base
+
+COPY --from=python /usr/local/bin/python /usr/local/bin/python
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    PATH=/usr/local/bin/python/bin:$PATH
+
+RUN  apt-get update \
+    && apt-get -y --no-install-recommends install \
+    libsqlite3-0 \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/* \
+    # add non-root user
+    && useradd \
+    --no-create-home \
+    --home-dir /app \
+    -c "" \
+    --uid 49999 app \
+    && mkdir -p /app \
+    && chown -R app:app /app
+
+COPY --from=python --chown=app:app /python/venv /python/venv
+
+ENV PATH=/python/venv/bin:$PATH \
+    # Send stdout and stderr stream directly to the terminal. Ensures that no
+    # output is retained in a buffer if the application crashes.
+    PYTHONUNBUFFERED=1 \
+    # Typically, bytecode is created on the first invocation to speed up following invocation.
+    # However, in Docker we only make a single invocation (when we start the container).
+    # Therefore, we can disable bytecode writing.
+    PYTHONDONTWRITEBYTECODE=1 \
+    # Ensure that python encoding is always UTF-8.
+    PYTHONIOENCODING=UTF-8 \
+    LANG=C.UTF-8 \
+    LC_ALL=C.UTF-8
+
+WORKDIR /app
+USER app
+ENV HOME=/app

src/proto/flwr/proto/message.proto

@@ -28,10 +28,11 @@ message Message {
 }
 
 message Context {
-  uint64 node_id = 1;
-  map<string, Scalar> node_config = 2;
-  RecordSet state = 3;
-  map<string, Scalar> run_config = 4;
+  uint64 run_id = 1;
+  uint64 node_id = 2;
+  map<string, Scalar> node_config = 3;
+  RecordSet state = 4;
+  map<string, Scalar> run_config = 5;
 }
 
 message Metadata {

src/py/flwr/client/clientapp/clientappio_servicer_test.py

@@ -66,6 +66,7 @@ def test_set_inputs(self) -> None:
             content=self.maker.recordset(2, 2, 1),
         )
         context = Context(
+            run_id=1,
             node_id=1,
             node_config={"nodeconfig1": 4.2},
             state=self.maker.recordset(2, 2, 1),
@@ -122,6 +123,7 @@ def test_get_outputs(self) -> None:
             content=self.maker.recordset(2, 2, 1),
         )
         context = Context(
+            run_id=1,
             node_id=1,
             node_config={"nodeconfig1": 4.2},
             state=self.maker.recordset(2, 2, 1),
@@ -186,6 +188,7 @@ def test_push_clientapp_outputs(self) -> None:
             content=self.maker.recordset(2, 2, 1),
         )
         context = Context(
+            run_id=1,
             node_id=1,
             node_config={"nodeconfig1": 4.2},
             state=self.maker.recordset(2, 2, 1),

src/py/flwr/client/message_handler/message_handler_test.py

@@ -142,7 +142,9 @@ def test_client_without_get_properties() -> None:
     actual_msg = handle_legacy_message_from_msgtype(
         client_fn=_get_client_fn(client),
         message=message,
-        context=Context(node_id=1123, node_config={}, state=RecordSet(), run_config={}),
+        context=Context(
+            run_id=2234, node_id=1123, node_config={}, state=RecordSet(), run_config={}
+        ),
     )
 
     # Assert
@@ -206,7 +208,9 @@ def test_client_with_get_properties() -> None:
     actual_msg = handle_legacy_message_from_msgtype(
         client_fn=_get_client_fn(client),
         message=message,
-        context=Context(node_id=1123, node_config={}, state=RecordSet(), run_config={}),
+        context=Context(
+            run_id=2234, node_id=1123, node_config={}, state=RecordSet(), run_config={}
+        ),
     )
 
     # Assert

src/py/flwr/client/mod/secure_aggregation/secaggplus_mod_test.py

@@ -74,6 +74,7 @@ def func(configs: dict[str, ConfigsRecordValues]) -> ConfigsRecord:
 def _make_ctxt() -> Context:
     cfg = ConfigsRecord(SecAggPlusState().to_dict())
     return Context(
+        run_id=234,
         node_id=123,
         node_config={},
         state=RecordSet(configs_records={RECORD_KEY_STATE: cfg}),

src/py/flwr/client/mod/utils_test.py

@@ -104,7 +104,9 @@ def test_multiple_mods(self) -> None:
 
         state = RecordSet()
         state.metrics_records[METRIC] = MetricsRecord({COUNTER: 0.0})
-        context = Context(node_id=0, node_config={}, state=state, run_config={})
+        context = Context(
+            run_id=1, node_id=0, node_config={}, state=state, run_config={}
+        )
         message = _get_dummy_flower_message()
 
         # Execute
@@ -129,7 +131,9 @@ def test_filter(self) -> None:
         # Prepare
         footprint: list[str] = []
         mock_app = make_mock_app("app", footprint)
-        context = Context(node_id=0, node_config={}, state=RecordSet(), run_config={})
+        context = Context(
+            run_id=1, node_id=0, node_config={}, state=RecordSet(), run_config={}
+        )
         message = _get_dummy_flower_message()
 
         def filter_mod(

src/py/flwr/client/run_info_store.py

@@ -83,6 +83,7 @@ def register_context(
             self.run_infos[run_id] = RunInfo(
                 initial_run_config=initial_run_config,
                 context=Context(
+                    run_id=run_id,
                     node_id=self.node_id,
                     node_config=self.node_config,
                     state=RecordSet(),

src/py/flwr/common/context.py

@@ -27,36 +27,41 @@ class Context:
 
     Parameters
     ----------
+    run_id : int
+        The ID that identifies the run.
     node_id : int
         The ID that identifies the node.
     node_config : UserConfig
         A config (key/value mapping) unique to the node and independent of the
         `run_config`. This config persists across all runs this node participates in.
     state : RecordSet
-        Holds records added by the entity in a given run and that will stay local.
+        Holds records added by the entity in a given `run_id` and that will stay local.
         This means that the data it holds will never leave the system it's running from.
         This can be used as an intermediate storage or scratchpad when
         executing mods. It can also be used as a memory to access
         at different points during the lifecycle of this entity (e.g. across
         multiple rounds)
     run_config : UserConfig
-        A config (key/value mapping) held by the entity in a given run and that will
-        stay local. It can be used at any point during the lifecycle of this entity
+        A config (key/value mapping) held by the entity in a given `run_id` and that
+        will stay local. It can be used at any point during the lifecycle of this entity
         (e.g. across multiple rounds)
     """
 
+    run_id: int
     node_id: int
     node_config: UserConfig
     state: RecordSet
     run_config: UserConfig
 
-    def __init__(  # pylint: disable=too-many-arguments
+    def __init__(  # pylint: disable=too-many-arguments, too-many-positional-arguments
         self,
+        run_id: int,
         node_id: int,
         node_config: UserConfig,
         state: RecordSet,
         run_config: UserConfig,
     ) -> None:
+        self.run_id = run_id
         self.node_id = node_id
         self.node_config = node_config
         self.state = state

src/py/flwr/common/serde.py

@@ -840,6 +840,7 @@ def message_from_proto(message_proto: ProtoMessage) -> Message:
 def context_to_proto(context: Context) -> ProtoContext:
     """Serialize `Context` to ProtoBuf."""
     proto = ProtoContext(
+        run_id=context.run_id,
         node_id=context.node_id,
         node_config=user_config_to_proto(context.node_config),
         state=recordset_to_proto(context.state),
@@ -851,6 +852,7 @@ def context_to_proto(context: Context) -> ProtoContext:
 def context_from_proto(context_proto: ProtoContext) -> Context:
     """Deserialize `Context` from ProtoBuf."""
     context = Context(
+        run_id=context_proto.run_id,
         node_id=context_proto.node_id,
         node_config=user_config_from_proto(context_proto.node_config),
         state=recordset_from_proto(context_proto.state),

src/py/flwr/common/serde_test.py

@@ -503,6 +503,7 @@ def test_context_serialization_deserialization() -> None:
     # Prepare
     maker = RecordMaker()
     original = Context(
+        run_id=0,
         node_id=1,
         node_config=maker.user_config(),
         state=maker.recordset(1, 1, 1),