diff --git a/tools/scripts/generate_binary_build_matrix.py b/tools/scripts/generate_binary_build_matrix.py
index 25f15ab414..757d06b469 100755
--- a/tools/scripts/generate_binary_build_matrix.py
+++ b/tools/scripts/generate_binary_build_matrix.py
@@ -353,7 +353,10 @@ def generate_libtorch_matrix(
         if os == WINDOWS:
             abi_versions = [RELEASE, DEBUG]
         elif os == LINUX:
-            abi_versions = [PRE_CXX11_ABI, CXX11_ABI]
+            if with_rocm == ENABLE:
+               abi_versions = [CXX11_ABI]
+            else:
+               abi_versions = [PRE_CXX11_ABI, CXX11_ABI]
         elif os in [MACOS_ARM64]:
             abi_versions = [CXX11_ABI]
         else: