OSPP: Implementation of Heterogeneous Multi-Edge Collaborative Neural Network Inference for High Mobility Scenarios

.gitignore

@@ -26,6 +26,8 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+dataset/
+initial_model/
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

core/testcasecontroller/algorithm/paradigm/base.py

@@ -124,6 +124,6 @@ def build_paradigm_job(self, paradigm_type):
             )
         # pylint: disable=E1101
         if paradigm_type == ParadigmType.MULTIEDGE_INFERENCE.value:
-            return self.modules_funcs.get(ModuleType.BASEMODEL.value)()
+            return self.module_instances.get(ModuleType.BASEMODEL.value)
 
         return None

core/testcasecontroller/algorithm/paradigm/multiedge_inference/multiedge_inference.py

@@ -16,6 +16,10 @@
 
 import os
 
+# pylint: disable=E0401
+import onnx
+
+from core.common.log import LOGGER
 from core.common.constant import ParadigmType
 from core.testcasecontroller.algorithm.paradigm.base import ParadigmBase
 
@@ -63,8 +67,15 @@ def run(self):
         """
 
         job = self.build_paradigm_job(ParadigmType.MULTIEDGE_INFERENCE.value)
-
-        inference_result = self._inference(job, self.initial_model)
+        if not job.__dict__.get('model_parallel'):
+            inference_result = self._inference(job, self.initial_model)
+        else:
+            if 'partition' in dir(job):
+                models_dir, map_info = job.partition(self.initial_model)
+            else:
+                models_dir, map_info = self._partition(job.__dict__.get('partition_point_list'),
+                                    self.initial_model, os.path.dirname(self.initial_model))
+            inference_result = self._inference_mp(job, models_dir, map_info)
 
         return inference_result, self.system_metric_info
 
@@ -77,3 +88,26 @@ def _inference(self, job, trained_model):
         job.load(trained_model)
         infer_res = job.predict(inference_dataset.x, train_dataset=train_dataset)
         return infer_res
+
+    def _inference_mp(self, job, models_dir, map_info):
+        inference_dataset = self.dataset.load_data(self.dataset.test_url, "inference")
+        inference_output_dir = os.path.join(self.workspace, "output/inference/")
+        os.environ["RESULT_SAVED_URL"] = inference_output_dir
+        job.load(models_dir, map_info)
+        infer_res = job.predict(inference_dataset.x)
+        return infer_res
+
+    # pylint: disable=W0718, C0103
+    def _partition(self, partition_point_list, initial_model_path, sub_model_dir):
+        map_info = dict({})
+        for idx, point in enumerate(partition_point_list):
+            input_names = point['input_names']
+            output_names = point['output_names']
+            sub_model_path = sub_model_dir + '/' + 'sub_model_' + str(idx+1) + '.onnx'
+            try:
+                onnx.utils.extract_model(initial_model_path,
+                                         sub_model_path, input_names, output_names)
+            except Exception as e:
+                LOGGER.info(str(e))
+            map_info[sub_model_path.split('/')[-1]] = point['device_name']
+        return sub_model_dir, map_info

examples/imagenet/multiedge_inference_bench/README.md

@@ -0,0 +1,104 @@
+# Benchmarking of Image Clasification for High Mobility Scenarios
+
+In high-mobility scenarios such as highways and high-speed railways, the connection between personal terminal devices and cloud servers is significantly weakened. However, in recent years, artificial intelligence technology has permeated every aspect of our lives, and we also need to use artificial intelligence technologies with high computational and storage demands and sensitive to latency in high-mobility scenarios. For example, even when driving through a tunnel with a weak network environment, we may still need to use AI capabilities such as image classification. Therefore, in the event that edge devices lose connection with the cloud, offloading AI computing tasks to adjacent edge devices and achieving computational aggregation based on the mutual collaboration between devices, to complete computing tasks that traditionally require cloud-edge collaboration, has become an issue worth addressing. This benchmarking job aims to simulate such scenario: using multiple heterogeneous computing units on the edge (such as personal mobile phones, tablets, bracelets, laptops, and other computing devices) for collaborative ViT inference, enabling image classification to be completed with lower latency using devices that are closer to the edge, thereby enhancing the user experience.After running benchmarking jobs, a report will be generated.
+
+With Ianvs installed and related environment prepared, users is then able to run the benchmarking process using the following steps. If you haven't installed Ianvs, please refer to [how-to-install-ianvs](../../../docs/guides/how-to-install-ianvs.md).
+
+## Prerequisites
+
+To setup the environment, run the following commands:
+```shell
+cd <Ianvs_HOME>
+pip install ./examples/resources/third_party/*
+pip install -r requirements.txt
+cd ./examples/imagenet/multiedge_inference_bench/
+pip install -r requirements.txt
+cd <Ianvs_HOME>
+mkdir dataset initial_model
+```
+Please refer to [this link](https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html) and ensure that the versions of CUDA and cuDNN are compatible with the version of ONNX Runtime.
+
+Note that it is advisable to avoid using lower versions of the ONNX library, as they are very time-consuming when performing computational graph partitioning. The version of onnx-cuda-cudnn we used in our tests is as follows:
+![onnx_version](images/onnx_version.png)
+
+## Step 1. Prepare Dataset
+Download [ImageNet 2012 dataset](https://image-net.org/download.php) and put it under <Ianvs_HOME>/dataset in the following structure:
+
+```
+dataset
+   |------ILSVRC2012_devkit_t12.tar.gz
+   |------ILSVRC2012_img_val.tar
+```
+Then, you need to process the dataset and generate the _train.txt_ and _val.txt_:
+
+```shell
+cd <Ianvs_HOME>
+python ./examples/imagenet/multiedge_inference_bench/testalgorithms/manual/dataset.py
+```
+
+## Step 2. Prepare Model
+
+Next, download pretrained model via [[huggingface]](https://huggingface.co/optimum/vit-base-patch16-224/tree/main), rename it to vit-base-patch16-224.onnx and put it under <Ianvs_HOME>/initial_model/
+
+## Step 3. Run Benchmarking Job - Manual
+We are now ready to run the ianvs for benchmarking image classification for high mobility scenarios on the ImageNet dataset.
+
+```python
+ianvs -f ./examples/imagenet/multiedge_inference_bench/classification_job_manual.yaml
+```
+
+The benchmarking process takes a few minutes and varies depending on devices.
+
+## Step 4. Check the Result
+
+Finally, the user can check the result of benchmarking on the console and also in the output path (/ianvs/multiedge_inference_bench/workspace) defined in the benchmarking config file (classification_job.yaml).
+
+The final output might look like this:
+![result](images/result.png)
+
+You can view the graphical representation of relevant metrics in /ianvs/multiedge_inference_bench/workspace/images/, such as the following:
+![plot](images/plot.png)
+
+To compare the running conditions of the model with and without parallelism in the multiedge inference scenario, you can modify the value of --devices_info in base_model.py to devices_one.yaml to view the relevant metrics when the model runs on a single device.
+
+## Step 5. Run Benchmarking Job - Automatic
+We offer a profiling-based and memory matching partition algorithm to compare with the method of manually specifying partitioning points. This method prioritizes the memory matching between the computational subgraph and the device. First, we profile the initial model on the CPU to collect memory usage, the number of parameters, computational cost, and the input and output data shapes for each layer, as well as the total number of layers and their names in the entire model. To facilitate subsequent integration, we have implemented profiling for three types of transformer models: vit, bert, and deit. Secondly, based on the results of the profiling and the device information provided in devices.yaml, we can identify the partitioning point that matches the device memory through a single traversal and perform model partitioning.
+
+You should first run the following command to generate a profiling result:
+```shell
+cd <Ianvs_HOME>
+python ./examples/imagenet/multiedge_inference_bench/testalgorithms/automatic/profiler.py
+```
+
+Then you will find a profiler_results.yml file in the <Ianvs_HOME>/examples/imagenet/multiedge_inference_bench/testalgorithms/automatic directory, just like this:
+![profiler_result](images/profiler_results.png)
+
+Then you can run the following command to perform benchmarking:
+```shell
+ianvs -f ./examples/imagenet/multiedge_inference_bench/classification_job_auto.yaml
+```
+
+After running, you will see the profit from the automatic method compared with the manual method.
+![result](images/auto_result.png)
+
+## Explanation for devices.yaml
+
+This file defines the specific information of edge-side multi-devices and the model's partition points. The devices section includes the computing resource type, memory, frequency, and bandwidth for each device. The partition_points section defines the input and output names of each computational subgraph and their mapping relationships with devices. This benchmarking job achieves the partitioning of the computational graph and model parallelism by manually defining partition points. You can implement custom partitioning algorithms based on the rich device information in devices.yaml.
+
+## Custom Partitioning Algorithms
+
+How to partition an ONNX model based on device information is an interesting question. You can solve this issue using greedy algorithms, dynamic programming algorithms, or other innovative graph algorithms to achieve optimal resource utilization and the lowest inference latency.
+
+More partitioning algorithms will be added in the future and you can customize their own partition methods in basemodel.py, they only need to comply with the input and output specifications defined by the interface as follows:
+
+```
+def partiton(self, initial_model):
+    ## 1. parsing
+    ## 2. modeling
+    ## 3. partition
+    return models_dir, map_info
+```
+
+Hope you have a perfect journey in solving this problem!
+
+

examples/imagenet/multiedge_inference_bench/classification_job_automatic.yaml

@@ -0,0 +1,72 @@
+benchmarkingjob:
+  # job name of benchmarking; string type;
+  name: "classification_job"
+  # the url address of job workspace that will reserve the output of tests; string type;
+  workspace: "./multiedge_inference_bench/workspace"
+
+  # the url address of test environment configuration file; string type;
+  # the file format supports yaml/yml;
+  testenv: "./examples/imagenet/multiedge_inference_bench/testenv/testenv.yaml"
+
+  # the configuration of test object
+  test_object:
+    # test type; string type;
+    # currently the option of value is "algorithms",the others will be added in succession.
+    type: "algorithms"
+    # test algorithm configuration files; list type;
+    algorithms:
+      # algorithm name; string type;
+      - name: "classification"
+      #   # the url address of test algorithm configuration file; string type;
+      #   # the file format supports yaml/yml;
+        url: "./examples/imagenet/multiedge_inference_bench/testalgorithms/automatic/classification_algorithm.yaml"
+
+  # the configuration of ranking leaderboard
+  rank:
+    # rank leaderboard with metric of test case's evaluation and order ; list type;
+    # the sorting priority is based on the sequence of metrics in the list from front to back;
+    sort_by: [ { "mota": "descend" } ]
+
+    # visualization configuration
+    visualization:
+      # mode of visualization in the leaderboard; string type;
+      # There are quite a few possible dataitems in the leaderboard. Not all of them can be shown simultaneously on the screen.
+      # In the leaderboard, we provide the "selected_only" mode for the user to configure what is shown or is not shown.
+      mode: "selected_only"
+      # method of visualization for selected dataitems; string type;
+      # currently the options of value are as follows:
+      #  1> "print_table": print selected dataitems;
+      method: "print_table"
+
+    # selected dataitem configuration
+    # The user can add his/her interested dataitems in terms of "paradigms", "modules", "hyperparameters" and "metrics",
+    # so that the selected columns will be shown.
+    selected_dataitem:
+      # currently the options of value are as follows:
+      #   1> "all": select all paradigms in the leaderboard;
+      #   2> paradigms in the leaderboard, e.g., "singletasklearning"
+      paradigms: [ "all" ]
+      # currently the options of value are as follows:
+      #   1> "all": select all modules in the leaderboard;
+      #   2> modules in the leaderboard, e.g., "basemodel"
+      modules: [ "all" ]
+      # currently the options of value are as follows:
+      #   1> "all": select all hyperparameters in the leaderboard;
+      #   2> hyperparameters in the leaderboard, e.g., "momentum"
+      hyperparameters: [ "all" ]
+      # currently the options of value are as follows:
+      #   1> "all": select all metrics in the leaderboard;
+      #   2> metrics in the leaderboard, e.g., "f1_score"
+      metrics: [ "all" ]
+
+    # model of save selected and all dataitems in workspace; string type;
+    # currently the options of value are as follows:
+    #  1> "selected_and_all": save selected and all dataitems;
+    #  2> "selected_only": save selected dataitems;
+    save_mode: "selected_and_all"
+
+
+
+
+
+

examples/imagenet/multiedge_inference_bench/classification_job_manual.yaml

@@ -0,0 +1,72 @@
+benchmarkingjob:
+  # job name of benchmarking; string type;
+  name: "classification_job"
+  # the url address of job workspace that will reserve the output of tests; string type;
+  workspace: "./multiedge_inference_bench/workspace"
+
+  # the url address of test environment configuration file; string type;
+  # the file format supports yaml/yml;
+  testenv: "./examples/imagenet/multiedge_inference_bench/testenv/testenv.yaml"
+
+  # the configuration of test object
+  test_object:
+    # test type; string type;
+    # currently the option of value is "algorithms",the others will be added in succession.
+    type: "algorithms"
+    # test algorithm configuration files; list type;
+    algorithms:
+      # algorithm name; string type;
+      - name: "classification"
+      #   # the url address of test algorithm configuration file; string type;
+      #   # the file format supports yaml/yml;
+        url: "./examples/imagenet/multiedge_inference_bench/testalgorithms/manual/classification_algorithm.yaml"
+
+  # the configuration of ranking leaderboard
+  rank:
+    # rank leaderboard with metric of test case's evaluation and order ; list type;
+    # the sorting priority is based on the sequence of metrics in the list from front to back;
+    sort_by: [ { "mota": "descend" } ]
+
+    # visualization configuration
+    visualization:
+      # mode of visualization in the leaderboard; string type;
+      # There are quite a few possible dataitems in the leaderboard. Not all of them can be shown simultaneously on the screen.
+      # In the leaderboard, we provide the "selected_only" mode for the user to configure what is shown or is not shown.
+      mode: "selected_only"
+      # method of visualization for selected dataitems; string type;
+      # currently the options of value are as follows:
+      #  1> "print_table": print selected dataitems;
+      method: "print_table"
+
+    # selected dataitem configuration
+    # The user can add his/her interested dataitems in terms of "paradigms", "modules", "hyperparameters" and "metrics",
+    # so that the selected columns will be shown.
+    selected_dataitem:
+      # currently the options of value are as follows:
+      #   1> "all": select all paradigms in the leaderboard;
+      #   2> paradigms in the leaderboard, e.g., "singletasklearning"
+      paradigms: [ "all" ]
+      # currently the options of value are as follows:
+      #   1> "all": select all modules in the leaderboard;
+      #   2> modules in the leaderboard, e.g., "basemodel"
+      modules: [ "all" ]
+      # currently the options of value are as follows:
+      #   1> "all": select all hyperparameters in the leaderboard;
+      #   2> hyperparameters in the leaderboard, e.g., "momentum"
+      hyperparameters: [ "all" ]
+      # currently the options of value are as follows:
+      #   1> "all": select all metrics in the leaderboard;
+      #   2> metrics in the leaderboard, e.g., "f1_score"
+      metrics: [ "all" ]
+
+    # model of save selected and all dataitems in workspace; string type;
+    # currently the options of value are as follows:
+    #  1> "selected_and_all": save selected and all dataitems;
+    #  2> "selected_only": save selected dataitems;
+    save_mode: "selected_and_all"
+
+
+
+
+
+