Add more vision models from mlagility

.github/workflows/mlagility_validation.yml

@@ -25,15 +25,13 @@ jobs:
       - name: Install dependencies and mlagility
         run: |
           python -m pip install --upgrade pip
-          python -m pip install onnx onnxruntime requests py-cpuinfo
-          # Print CPU info for debugging ONNX Runtime inference difference
-          python -m cpuinfo
           git clone https://github.com/groq/mlagility.git
           cd mlagility
           pip install -r models/requirements.txt
           pip install -e .
 
       - name: Validate created ONNX model from mlagility
         run: |
+          # TODO: remove the following after mlagility has resovled version contradict issue
           pip install -r models/mlagility/requirements.txt
           python workflow_scripts/run_mlagility.py

workflow_scripts/mlagility_config.py

@@ -0,0 +1,18 @@
+models_info = [
+    "torch_hub/alexnet.py",
+    "torch_hub/resnet50.py",
+    "torchvision/fasterrcnn_resnet50_fpn_v2.py",
+    "torchvision/maskrcnn_resnet50_fpn.py",
+    "torchvision/ssd_mobilenet_v2.py",
+    "torch_hub/mobilenet_v2.py",
+    "torch_hub/resnet18.py",
+    "torch_hub/resnet34.py",
+    "torch_hub/resnet101.py",
+    "torch_hub/vgg16.py",
+    "torch_hub/vgg16_bn.py",
+    "torch_hub/vgg19.py",
+    "torch_hub/vgg19_bn.py",
+    "torch_hub/densenet121.py",
+    "torch_hub/inception_v3.py",
+    "torch_hub/googlenet.py",
+]

workflow_scripts/run_mlagility.py

@@ -1,11 +1,24 @@
-import config
+from mlagility_config import models_info
 import os.path as osp
+from os import listdir
 from pathlib import Path
 import shutil
 import subprocess
 import sys
 
 
+def get_immediate_subdirectories_count(dir_name):
+    return len([name for name in listdir(dir_name)
+            if osp.isdir(osp.join(dir_name, name))])
+
+
+def find_model_hash_name(dir_name, cache_dir_prefix):
+    for dir in listdir(dir_name):
+        if dir.startswith(cache_dir_prefix):
+            return dir
+    raise Exception(f"Cannot find model hash name: {cache_dir_prefix} in cache directory.")
+
+
 ZOO_OPSET_VERSION = "18"
 base_name = f"-op{ZOO_OPSET_VERSION}-base.onnx"
 cwd_path = Path.cwd()
@@ -15,15 +28,19 @@
 
 errors = 0
 
-for script_path, model_name, model_zoo_dir in config.models_info:
+for model_info in models_info:
+    directory_name, model_name = model_info.split("/")
+    model_name = model_name.replace(".py", "")
+    model_zoo_dir = model_name
     try:
         print(f"----------------Checking {model_zoo_dir}----------------")
         final_model_path = osp.join(mlagility_models_dir, model_zoo_dir, f"{model_zoo_dir}-{ZOO_OPSET_VERSION}.onnx")
-        subprocess.run(["benchit", osp.join(mlagility_root, script_path), "--cache-dir", cache_converted_dir,
+        subprocess.run(["benchit", osp.join(mlagility_root, model_info), "--cache-dir", cache_converted_dir,
                         "--onnx-opset", ZOO_OPSET_VERSION, "--export-only"],
                         cwd=cwd_path, stdout=sys.stdout,
                         stderr=sys.stderr)
-        shutil.copy(osp.join(cache_converted_dir, model_name, "onnx", model_name + base_name), final_model_path)
+        model_hash_name = find_model_hash_name(".cache", model_name + "_" + directory_name + "_")
+        shutil.copy(osp.join(cache_converted_dir, model_hash_name, "onnx", model_hash_name + base_name), final_model_path)
         subprocess.run(["git", "diff", "--exit-code", "--", final_model_path],
                         cwd=cwd_path, stdout=sys.stdout,
                         stderr=sys.stderr)
@@ -33,7 +50,14 @@
         print(f"Failed to check {model_zoo_dir} because of {e}.")
 
 if errors > 0:
-    print(f"All {len(config.models_info)} model(s) have been checked, but {errors} model(s) failed.")
+    print(f"All {len(models_info)} model(s) have been checked, but {errors} model(s) failed.")
     sys.exit(1)
 else:
-    print(f"All {len(config.models_info)} model(s) have been checked.")
+    print(f"All {len(models_info)} model(s) have been checked.")
+
+mlagility_subdir_count = get_immediate_subdirectories_count(mlagility_models_dir)
+if mlagility_subdir_count != len(models_info):
+    print(f"Expected {len(models_info)} model(s) in {mlagility_models_dir}, but got {mlagility_subdir_count} model(s) under models/mlagility."
+          f"Please check if you have added new model(s) to models_info in mlagility_config.py.")
+    sys.exit(1)
+