Merge branch 'main' into update-pylint

examples/llm-flowertune/requirements.txt

@@ -7,3 +7,4 @@ scipy==1.11.2
 peft==0.4.0
 fschat[model_worker,webui]==0.2.35
 transformers==4.38.1
+hf_transfer==0.1.8

pyproject.toml

@@ -59,6 +59,7 @@ flower-supernode = "flwr.client:run_supernode"
 flower-client-app = "flwr.client:run_client_app"
 flower-server-app = "flwr.server:run_server_app"
 flower-simulation = "flwr.simulation.run_simulation:run_simulation_from_cli"
+flwr-clientapp = "flwr.client.supernode:flwr_clientapp"
 
 [tool.poetry.dependencies]
 python = "^3.8"

src/proto/flwr/proto/clientappio.proto

@@ -0,0 +1,40 @@
+syntax = "proto3";
+
+package flwr.proto;
+
+import "flwr/proto/fab.proto";
+import "flwr/proto/run.proto";
+import "flwr/proto/message.proto";
+
+service ClientAppIo {
+  // Get Message, Context, and Run
+  rpc PullClientAppInputs(PullClientAppInputsRequest)
+      returns (PullClientAppInputsResponse) {}
+
+  // Send updated Message and Context
+  rpc PushClientAppOutputs(PushClientAppOutputsRequest)
+      returns (PushClientAppOutputsResponse) {}
+}
+
+enum ClientAppOutputCode {
+  SUCCESS = 0;
+  DEADLINE_EXCEEDED = 1;
+  UNKNOWN_ERROR = 2;
+}
+message ClientAppOutputStatus {
+  ClientAppOutputCode code = 1;
+  string message = 2;
+}
+
+message PullClientAppInputsRequest { sint64 token = 1; }
+message PullClientAppInputsResponse {
+  Message message = 1;
+  Context context = 2;
+  Run run = 3;
+}
+message PushClientAppOutputsRequest {
+  sint64 token = 1;
+  Message message = 2;
+  Context context = 3;
+}
+message PushClientAppOutputsResponse { ClientAppOutputStatus status = 1; }

src/proto/flwr/proto/message.proto

@@ -0,0 +1,46 @@
+// Copyright 2024 Flower Labs GmbH. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ==============================================================================
+
+syntax = "proto3";
+
+package flwr.proto;
+
+import "flwr/proto/error.proto";
+import "flwr/proto/recordset.proto";
+import "flwr/proto/transport.proto";
+
+message Message {
+  Metadata metadata = 1;
+  RecordSet content = 2;
+  Error error = 3;
+}
+
+message Context {
+  sint64 node_id = 1;
+  map<string, Scalar> node_config = 2;
+  RecordSet state = 3;
+  map<string, Scalar> run_config = 4;
+}
+
+message Metadata {
+  sint64 run_id = 1;
+  string message_id = 2;
+  sint64 src_node_id = 3;
+  sint64 dst_node_id = 4;
+  string reply_to_message = 5;
+  string group_id = 6;
+  double ttl = 7;
+  string message_type = 8;
+}

src/py/flwr/client/supernode/__init__.py

@@ -15,10 +15,12 @@
 """Flower SuperNode."""
 
 
+from .app import flwr_clientapp as flwr_clientapp
 from .app import run_client_app as run_client_app
 from .app import run_supernode as run_supernode
 
 __all__ = [
+    "flwr_clientapp",
     "run_client_app",
     "run_supernode",
 ]

src/py/flwr/client/supernode/app.py

@@ -120,6 +120,31 @@ def run_client_app() -> None:
     register_exit_handlers(event_type=EventType.RUN_CLIENT_APP_LEAVE)
 
 
+def flwr_clientapp() -> None:
+    """Run process-isolated Flower ClientApp."""
+    log(INFO, "Starting Flower ClientApp")
+
+    parser = argparse.ArgumentParser(
+        description="Run a Flower ClientApp",
+    )
+    parser.add_argument(
+        "--address",
+        help="Address of SuperNode ClientAppIo gRPC servicer",
+    )
+    parser.add_argument(
+        "--token",
+        help="Unique token generated by SuperNode for each ClientApp execution",
+    )
+    args = parser.parse_args()
+    log(
+        DEBUG,
+        "Staring isolated `ClientApp` connected to SuperNode ClientAppIo at %s "
+        "with the token %s",
+        args.address,
+        args.token,
+    )
+
+
 def _warn_deprecated_server_arg(args: argparse.Namespace) -> None:
     """Warn about the deprecated argument `--server`."""
     if args.server != ADDRESS_FLEET_API_GRPC_RERE:

src/py/flwr/proto/clientappio_pb2.pyi

@@ -0,0 +1,110 @@
+"""
+@generated by mypy-protobuf.  Do not edit manually!
+isort:skip_file
+"""
+import builtins
+import flwr.proto.message_pb2
+import flwr.proto.run_pb2
+import google.protobuf.descriptor
+import google.protobuf.internal.enum_type_wrapper
+import google.protobuf.message
+import typing
+import typing_extensions
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+class _ClientAppOutputCode:
+    ValueType = typing.NewType('ValueType', builtins.int)
+    V: typing_extensions.TypeAlias = ValueType
+class _ClientAppOutputCodeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_ClientAppOutputCode.ValueType], builtins.type):
+    DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+    SUCCESS: _ClientAppOutputCode.ValueType  # 0
+    DEADLINE_EXCEEDED: _ClientAppOutputCode.ValueType  # 1
+    UNKNOWN_ERROR: _ClientAppOutputCode.ValueType  # 2
+class ClientAppOutputCode(_ClientAppOutputCode, metaclass=_ClientAppOutputCodeEnumTypeWrapper):
+    pass
+
+SUCCESS: ClientAppOutputCode.ValueType  # 0
+DEADLINE_EXCEEDED: ClientAppOutputCode.ValueType  # 1
+UNKNOWN_ERROR: ClientAppOutputCode.ValueType  # 2
+global___ClientAppOutputCode = ClientAppOutputCode
+
+
+class ClientAppOutputStatus(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+    CODE_FIELD_NUMBER: builtins.int
+    MESSAGE_FIELD_NUMBER: builtins.int
+    code: global___ClientAppOutputCode.ValueType
+    message: typing.Text
+    def __init__(self,
+        *,
+        code: global___ClientAppOutputCode.ValueType = ...,
+        message: typing.Text = ...,
+        ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["code",b"code","message",b"message"]) -> None: ...
+global___ClientAppOutputStatus = ClientAppOutputStatus
+
+class PullClientAppInputsRequest(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+    TOKEN_FIELD_NUMBER: builtins.int
+    token: builtins.int
+    def __init__(self,
+        *,
+        token: builtins.int = ...,
+        ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["token",b"token"]) -> None: ...
+global___PullClientAppInputsRequest = PullClientAppInputsRequest
+
+class PullClientAppInputsResponse(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+    MESSAGE_FIELD_NUMBER: builtins.int
+    CONTEXT_FIELD_NUMBER: builtins.int
+    RUN_FIELD_NUMBER: builtins.int
+    @property
+    def message(self) -> flwr.proto.message_pb2.Message: ...
+    @property
+    def context(self) -> flwr.proto.message_pb2.Context: ...
+    @property
+    def run(self) -> flwr.proto.run_pb2.Run: ...
+    def __init__(self,
+        *,
+        message: typing.Optional[flwr.proto.message_pb2.Message] = ...,
+        context: typing.Optional[flwr.proto.message_pb2.Context] = ...,
+        run: typing.Optional[flwr.proto.run_pb2.Run] = ...,
+        ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["context",b"context","message",b"message","run",b"run"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["context",b"context","message",b"message","run",b"run"]) -> None: ...
+global___PullClientAppInputsResponse = PullClientAppInputsResponse
+
+class PushClientAppOutputsRequest(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+    TOKEN_FIELD_NUMBER: builtins.int
+    MESSAGE_FIELD_NUMBER: builtins.int
+    CONTEXT_FIELD_NUMBER: builtins.int
+    token: builtins.int
+    @property
+    def message(self) -> flwr.proto.message_pb2.Message: ...
+    @property
+    def context(self) -> flwr.proto.message_pb2.Context: ...
+    def __init__(self,
+        *,
+        token: builtins.int = ...,
+        message: typing.Optional[flwr.proto.message_pb2.Message] = ...,
+        context: typing.Optional[flwr.proto.message_pb2.Context] = ...,
+        ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["context",b"context","message",b"message"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["context",b"context","message",b"message","token",b"token"]) -> None: ...
+global___PushClientAppOutputsRequest = PushClientAppOutputsRequest
+
+class PushClientAppOutputsResponse(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+    STATUS_FIELD_NUMBER: builtins.int
+    @property
+    def status(self) -> global___ClientAppOutputStatus: ...
+    def __init__(self,
+        *,
+        status: typing.Optional[global___ClientAppOutputStatus] = ...,
+        ) -> None: ...
+    def HasField(self, field_name: typing_extensions.Literal["status",b"status"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing_extensions.Literal["status",b"status"]) -> None: ...
+global___PushClientAppOutputsResponse = PushClientAppOutputsResponse