1. FedLearner Framework and Core Dependency RA-TLS Configuration

To ensure code consistency and implement hash signature-based remote attestation for the FedLearner framework and its core dependencies, an environment variable has been introduced in Gramine that prevents the generation of .pyc files. 2. Gramine Template Configuration for FedLearner The Gramine template configuration has been updated to include the code locations for the FedLearner framework and some essential dependencies. This addition facilitates bidirectional remote attestation between parties. 3. Meituan HDFS File Path Management Optimization The code responsible for handling file paths in Meituan's Hadoop Distributed File System (HDFS) has been migrated from the main entry point to be processed by the master node. This change aims to prevent file read and write conflicts that could arise from multiple workers operating simultaneously.
bytedance · Apr 11, 2024 · 5cc16a8 · 5cc16a8
1 parent c2a5523
commit 5cc16a8
Show file tree

Hide file tree

Showing 3 changed files with 117 additions and 44 deletions.
diff --git a/fedlearner-sgx-dev.dockerfile b/fedlearner-sgx-dev.dockerfile
@@ -6,6 +6,8 @@ ENV DEBIAN_FRONTEND=noninteractive
 ENV INSTALL_PREFIX=/usr/local
 ENV LD_LIBRARY_PATH=${INSTALL_PREFIX}/lib:${INSTALL_PREFIX}/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH}
 ENV PATH=${INSTALL_PREFIX}/bin:${LD_LIBRARY_PATH}:${PATH}
+# For Gramine RA-TLS
+ENV PYTHONDONTWRITEBYTECODE=1
 
 # Add steps here to set up common dependencies
 RUN apt-get update \
@@ -60,8 +62,8 @@ RUN apt-get install -y libcurl4-openssl-dev libprotobuf-c-dev python3-protobuf p
 RUN apt-get install -y libgmp-dev libmpfr-dev libmpc-dev libisl-dev nasm protobuf-compiler
 
 RUN ln -s /usr/bin/python3 /usr/bin/python \
-    && pip3 install --upgrade pip \
-    && pip3 install toml meson pyelftools
+    && pip3 install --no-compile --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ \
+    && pip3 install --no-compile toml meson pyelftools -i https://mirrors.aliyun.com/pypi/simple/
 
 RUN git clone https://github.com/analytics-zoo/gramine ${GRAMINEDIR} \
     && cd ${GRAMINEDIR} \
@@ -101,17 +103,17 @@ ENV GRPC_VERSION=v1.38.1
 
 RUN git clone --recurse-submodules -b ${GRPC_VERSION} https://github.com/grpc/grpc ${GRPC_PATH}
 
-RUN pip3 install --upgrade pip \
-    && pip3 install -r ${GRPC_PATH}/requirements.txt
+RUN pip3 install --no-compile --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ \
+    && pip3 install --no-compile -r ${GRPC_PATH}/requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
 
 # Tensorflow dependencies
 ENV BAZEL_VERSION=3.1.0
 ENV TF_VERSION=v2.4.2
 ENV TF_BUILD_PATH=/tf/src
 ENV TF_BUILD_OUTPUT=/tf/output
 
-RUN pip3 install --upgrade pip \
-    && pip3 install numpy keras_preprocessing
+RUN pip3 install --no-compile --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ \
+    && pip3 install --no-compile numpy keras_preprocessing -i https://mirrors.aliyun.com/pypi/simple/
 
 RUN wget "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel_${BAZEL_VERSION}-linux-x86_64.deb" \
     && dpkg -i bazel_*.deb
@@ -127,7 +129,7 @@ RUN apt-get install -y libmysqlclient-dev
 COPY sgx/grpc/common ${GRPC_PATH}
 COPY sgx/grpc/v1.38.1 ${GRPC_PATH}
 
-RUN pip3 install 'cython==0.29.36'
+RUN pip3 install --no-compile 'cython==0.29.36' -i https://mirrors.aliyun.com/pypi/simple/
 RUN ${GRPC_PATH}/build_python.sh
 
 # Build tensorflow
@@ -150,17 +152,17 @@ RUN if [ -f ${FEDLEARNER_PATH}/docker/hadoop-mt-2.7.0.tar.gz ]; then mkdir -p /o
 # For meituan hadoop auth
 RUN apt-get install -y libkrb5-dev openjdk-8-jdk
 
-RUN pip3 install --upgrade pip \
-    && pip3 install -r ${FEDLEARNER_PATH}/requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
+RUN pip3 install --no-compile --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ \
+    && pip3 install --no-compile  -r ${FEDLEARNER_PATH}/requirements.txt -i https://mirrors.aliyun.com/pypi/simple/
 
 RUN cd ${FEDLEARNER_PATH} \
     && make protobuf \
     && python3 setup.py bdist_wheel \
-    && pip3 install ./dist/*.whl
+    && pip3 install --no-compile ./dist/*.whl
 
 # Re-install tensorflow, uninstall tensorflow_io, mock it
 RUN pip3 uninstall -y tensorflow tensorflow-io \
-    && pip3 install ${TF_BUILD_OUTPUT}/*.whl
+    && pip3 install --no-compile ${TF_BUILD_OUTPUT}/*.whl
 
 # Re-install fedlearner plugin
 RUN cd ${FEDLEARNER_PATH} \
@@ -170,7 +172,7 @@ RUN cd ${FEDLEARNER_PATH} \
 
 # Re-install grpcio
 RUN pip3 uninstall -y grpcio \
-    && pip3 install ${GRPC_PATH}/dist/grpcio*.whl
+    && pip3 install --no-compile ${GRPC_PATH}/dist/grpcio*.whl
 
 # For debug
 RUN apt-get install -y strace gdb ctags vim
@@ -182,6 +184,11 @@ COPY sgx/configs /
 RUN echo "enabled=0" > /etc/default/apport
 RUN echo "exit 0" > /usr/sbin/policy-rc.d
 
+# For gramine ra-tls
+RUN dpkg --remove --force-depends libgtk2.0-0 \
+    && pip3 uninstall -y numpy keras_preprocessing protobuf \
+    && pip3 install --no-compile numpy keras_preprocessing protobuf -i https://mirrors.aliyun.com/pypi/simple/
+
 # Clean tmp files
 RUN apt-get clean all \
     && rm -rf /var/lib/apt/lists/* \

diff --git a/fedlearner/trainer/trainer_worker.py b/fedlearner/trainer/trainer_worker.py
@@ -232,6 +232,33 @@ def _run_master(role,
         cluster_server = ClusterServer(cluster_spec, "master",
                                        server_port=args.server_port)
 
+    # use Meituan hadoop
+    # first：convert Meituan HDFS path to local storage path, if local exit psi result file, user local file
+    # second：if local not exit psi result file，from Meituan HDFS download to local
+    if args.using_mt_hadoop:
+        data_path = args.data_path
+        if data_path:
+            local_data_path = get_local_temp_path(data_path)
+            if not exists(local_data_path):
+                data_path = mt_hadoop_download(data_path)
+            else:
+                data_path = local_data_path
+            args.data_path = data_path
+
+        checkpoint_path = args.checkpoint_path
+        if checkpoint_path:
+            args.checkpoint_path = get_local_temp_path(checkpoint_path)
+
+        load_checkpoint_path = args.load_checkpoint_path
+        if load_checkpoint_path:
+            args.load_checkpoint_path = get_local_temp_path(load_checkpoint_path)
+            if not exists(args.load_checkpoint_path):
+                mt_hadoop_download(load_checkpoint_path)
+
+        export_path = args.export_path
+        if export_path:
+            args.export_path = get_local_temp_path(export_path)
+
     checkpoint_filename_with_path = _get_checkpoint_filename_with_path(args)
     data_visitor = _create_data_visitor(args)
     master_factory = LeaderTrainerMaster \
@@ -479,33 +506,6 @@ def train(role,
     if not isinstance(role, str) or role.lower() not in (LEADER, FOLLOER):
         raise ValueError("--role must set one of %s or %s"%(LEADER, FOLLOER))
 
-    # use Meituan hadoop
-    # first：convert Meituan HDFS path to local storage path, if local exit psi result file, user local file
-    # second：if local not exit psi result file，from Meituan HDFS download to local
-    if args.using_mt_hadoop:
-        data_path = args.data_path
-        if data_path:
-            local_data_path = get_local_temp_path(data_path)
-            if not exists(local_data_path):
-                data_path = mt_hadoop_download(data_path)
-            else:
-                data_path = local_data_path
-            args.data_path = data_path
-
-        checkpoint_path = args.checkpoint_path
-        if checkpoint_path:
-            args.checkpoint_path = get_local_temp_path(checkpoint_path)
-
-        load_checkpoint_path = args.load_checkpoint_path
-        if load_checkpoint_path:
-            args.load_checkpoint_path = get_local_temp_path(load_checkpoint_path)
-            if not exists(args.load_checkpoint_path):
-                mt_hadoop_download(load_checkpoint_path)
-
-        export_path = args.export_path
-        if export_path:
-            args.export_path = get_local_temp_path(export_path)
-
     if args.loglevel:
         fl_logging.set_level(args.loglevel)
 

diff --git a/sgx/gramine/CI-Examples/generate-token/python.manifest.template b/sgx/gramine/CI-Examples/generate-token/python.manifest.template
@@ -67,12 +67,75 @@ sgx.trusted_files = [
   "file:/usr/{{ arch_libdir }}/",
   "file:/etc/ssl/certs/ca-certificates.crt",
   "file:/etc/default/apport",
-  "file:/usr/local/lib/",
-  "file:{{ python.stdlib }}/",
-  "file:{{ python.distlib }}/", 
   "file:/etc/mime.types",
   "file:/gramine/leader/",
-  "file:/gramine/follower/"
+  "file:/gramine/follower/",
+  "file:/usr/local/lib/x86_64-linux-gnu/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/libtensorflow_framework.so.2",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/__init__.py",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/__internal__/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/__operators__/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/audio/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/autodiff/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/autograph/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/bitwise/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/compat/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/config/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/data/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/debugging/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/distribute/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/dtypes/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/errors/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/experimental/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/feature_column/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/graph_util/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/image/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/io/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/linalg/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/lite/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/lookup/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/math/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/mixed_precision/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/mlir/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/nest/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/nn/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/profiler/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/quantization/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/queue/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/ragged/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/random/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/raw_ops/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/saved_model/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/sets/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/signal/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/sparse/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/strings/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/summary/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/sysconfig/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/test/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/tpu/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/train/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/types/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/version/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/_api/v2/xla/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/compiler/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/core/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/include/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/keras/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/python/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/tools/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/lite/experimental/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/lite/toco/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow/xla_aot_runtime_src/",
+  "file:/usr/local/lib/python3.6/dist-packages/pyspark/",
+  "file:/usr/local/lib/python3.6/dist-packages/Cython/",
+  "file:/usr/local/lib/python3.6/dist-packages/numpy/",
+  "file:/usr/local/lib/python3.6/dist-packages/grpc/",
+  "file:/usr/local/lib/python3.6/dist-packages/google/",
+  "file:/usr/local/lib/python3.6/dist-packages/tensorflow_estimator/",
+  "file:/usr/local/lib/python3.6/dist-packages/sklearn/",
+  "file:/usr/local/lib/python3.6/dist-packages/fedlearner/",
+  "file:/usr/local/lib/python3.6/dist-packages/pandas/"
 ]
 
 sgx.allowed_files = [
@@ -81,6 +144,9 @@ sgx.allowed_files = [
   "file:/opt/meituan/",
   "file:/usr/lib/ssl/openssl.cnf",
   "file:/usr/lib/gcc",
+  "file:/usr/local/lib/",
+  "file:{{ python.stdlib }}/",
+  "file:{{ python.distlib }}/",
   "file:/etc/ethers",
   "file:/etc/hosts",
   "file:/etc/group",
@@ -103,4 +169,4 @@ sgx.allowed_files = [
   "file:/lib/",
   "file:/bin/",
   "file:/data/",
-]
+]