Merge branch 'main' into sdpa_decomp

pytorch · Jan 30, 2025 · 5b4757a · 5b4757a
2 parents 0562381 + 002bacf
commit 5b4757a
Show file tree

Hide file tree

Showing 257 changed files with 1,296 additions and 1,270 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -30,11 +30,12 @@ build:windows --cxxopt="/GS-" --cxxopt="/std:c++17" --cxxopt="/permissive-"
 build:windows --cxxopt="/wd4244" --cxxopt="/wd4267" --cxxopt="/wd4819"
 build:windows --features=windows_export_all_symbols
 
-build:python --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0"
-build:python --linkopt="-D_GLIBCXX_USE_CXX11_ABI=0"
-build:python --define=abi=pre_cxx11_abi
 build:python --define=target_lang=python
 
+build:cxx11_abi --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=1"
+build:cxx11_abi --linkopt="-D_GLIBCXX_USE_CXX11_ABI=1"
+build:cxx11_abi --define=abi=cxx11_abi
+
 build:pre_cxx11_abi --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0"
 build:pre_cxx11_abi --linkopt="-D_GLIBCXX_USE_CXX11_ABI=0"
 build:pre_cxx11_abi --define=abi=pre_cxx11_abi

diff --git a/.github/scripts/generate_binary_build_matrix.py b/.github/scripts/generate_binary_build_matrix.py
@@ -469,7 +469,6 @@ def generate_wheels_matrix(
     ret: List[Dict[str, Any]] = []
     for python_version in python_versions:
         for arch_version in arches:
-
             # TODO: Enable Python 3.13 support for ROCM
             if arch_version in ROCM_ARCHES and python_version == "3.13":
                 continue

diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml
@@ -42,7 +42,7 @@ jobs:
               run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
             - name: Build Python Package
               env:
-                  USE_CXX11_ABI: 1
+                  USE_PRE_CXX11_ABI: 0
               run: |
                   python3 -m pip install --pre . --extra-index-url https://download.pytorch.org/whl/nightly/cu126
             - name: Generate New Docs

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -75,7 +75,7 @@ Environment variables supported by nox
 ```
 PYT_PATH          - To use different PYTHONPATH than system installed Python packages
 TOP_DIR           - To set the root directory of the noxfile
-USE_CXX11         - To use cxx11_abi (Defaults to 0)
+USE_PRE_CXX11     - To use pre_cxx11_abi (Defaults to 0)
 USE_HOST_DEPS     - To use host dependencies for tests (Defaults to 0)
 ```
 

diff --git a/core/conversion/converters/impl/batch_norm.cpp b/core/conversion/converters/impl/batch_norm.cpp
@@ -134,9 +134,14 @@ auto batch_norm_registrations TORCHTRT_UNUSED =
 
               auto eps = static_cast<float>(args[7].unwrapToDouble(1e-5f));
 
-              auto scales = args[1].unwrapToTensor(at::ones(shape[1], options)).cpu().contiguous();
-              auto bias = args[2].unwrapToTensor(at::zeros(shape[1], options)).cpu().contiguous();
-
+              auto scales = at::ones(shape[1], options);
+              if (!args[1].IValue()->isNone()) {
+                scales = args[1].unwrapToTensor(at::ones(shape[1], options)).cpu().contiguous();
+              }
+              auto bias = at::zeros(shape[1], options);
+              if (!args[2].IValue()->isNone()) {
+                bias = args[2].unwrapToTensor(at::zeros(shape[1], options)).cpu().contiguous();
+              }
               // track_running_stats=True
               if (!args[3].IValue()->isNone() || !args[4].IValue()->isNone()) {
                 auto running_mean = args[3].unwrapToTensor();
@@ -154,6 +159,8 @@ auto batch_norm_registrations TORCHTRT_UNUSED =
                 return true;
               }
 
+              // Not sure this actually does something since the cudnn_enabled is from the PyTorch context.
+              // We need cuDNN either way to run this converter
               auto cudnn_enabled = static_cast<bool>(args[8].unwrapToBool(false));
               if (!cudnn_enabled) {
                 LOG_DEBUG(
@@ -162,7 +169,7 @@ auto batch_norm_registrations TORCHTRT_UNUSED =
                     so for some functionalities, users need to install correct \
                     cuDNN version by themselves. Please see our support matrix \
                     here: https://docs.nvidia.com/deeplearning/tensorrt/support-matrix/index.html.");
-                return false;
+                // return false;
               }
 
               const int relu = 0;

diff --git a/core/util/prelude.h b/core/util/prelude.h
@@ -2,6 +2,7 @@
 
 // A collection of headers from util that will typically get included in most
 // files
+#include <cstdint>
 #include "core/util/Exception.h"
 #include "core/util/build_info.h"
 #include "core/util/jit_util.h"

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -12,8 +12,8 @@ RUN test -n "$TENSORRT_VERSION" || (echo "No tensorrt version specified, please
 ARG PYTHON_VERSION=3.10
 ENV PYTHON_VERSION=${PYTHON_VERSION}
 
-ARG USE_CXX11_ABI
-ENV USE_CXX11=${USE_CXX11_ABI}
+ARG USE_PRE_CXX11_ABI
+ENV USE_PRE_CXX11=${USE_PRE_CXX11_ABI}
 ENV DEBIAN_FRONTEND=noninteractive
 
 # Install basic dependencies

diff --git a/docker/README.md b/docker/README.md
@@ -1,30 +1,30 @@
 # Building a Torch-TensorRT container
 
-* Use `Dockerfile` to build a container which provides the exact development environment that our master branch is usually tested against.
+* Use `Dockerfile` to build a container which provides the exact development environment that our main branch is usually tested against.
 
 * The `Dockerfile` currently uses <a href="https://github.com/bazelbuild/bazelisk">Bazelisk</a> to select the Bazel version, and uses the exact library versions of Torch and CUDA listed in <a href="https://github.com/pytorch/TensorRT#dependencies">dependencies</a>.
   * The desired versions of TensorRT must be specified as build-args, with major and minor versions as in: `--build-arg TENSORRT_VERSION=a.b`
-  * [**Optional**] The desired base image be changed by explicitly setting a base image, as in `--build-arg BASE_IMG=nvidia/cuda:11.8.0-devel-ubuntu22.04`, though this is optional
+  * [**Optional**] The desired base image be changed by explicitly setting a base image, as in `--build-arg BASE_IMG=nvidia/cuda:11.8.0-devel-ubuntu22.04`, though this is optional.
   * [**Optional**] Additionally, the desired Python version can be changed by explicitly setting a version, as in `--build-arg PYTHON_VERSION=3.10`, though this is optional as well.
 
-* This `Dockerfile` installs `pre-cxx11-abi` versions of Pytorch and builds Torch-TRT using `pre-cxx11-abi` libtorch as well.
+* This `Dockerfile` installs `cxx11-abi` versions of Pytorch and builds Torch-TRT using `cxx11-abi` libtorch as well. As of torch 2.7, torch requires `cxx11-abi` for all CUDA 11.8, 12.4, and 12.6.
 
-Note: By default the container uses the `pre-cxx11-abi` version of Torch + Torch-TRT. If you are using a workflow that requires a build of PyTorch on the CXX11 ABI (e.g. using the PyTorch NGC containers as a base image), add the Docker build argument: `--build-arg USE_CXX11_ABI=1`
+Note: By default the container uses the `cxx11-abi` version of Torch + Torch-TRT. If you are using a workflow that requires a build of PyTorch on the PRE CXX11 ABI, please add the Docker build argument: `--build-arg USE_PRE_CXX11_ABI=1`
 
 ### Dependencies
 
 * Install nvidia-docker by following https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker
 
 ### Instructions
 
-- The example below uses TensorRT 10.6.0.26
+- The example below uses TensorRT 10.7.0.23
 - See <a href="https://github.com/pytorch/TensorRT#dependencies">dependencies</a> for a list of current default dependencies.
 
 > From root of Torch-TensorRT repo
 
 Build:
 ```
-DOCKER_BUILDKIT=1 docker build --build-arg TENSORRT_VERSION=10.6.0 -f docker/Dockerfile -t torch_tensorrt:latest .
+DOCKER_BUILDKIT=1 docker build --build-arg TENSORRT_VERSION=10.7.0 -f docker/Dockerfile -t torch_tensorrt:latest .
 ```
 
 Run:

diff --git a/docker/dist-build.sh b/docker/dist-build.sh
@@ -4,10 +4,10 @@ set -x
 
 TOP_DIR=$(cd $(dirname $0); pwd)/..
 
-if [[ -z "${USE_CXX11}" ]]; then
+if [[ -z "${USE_PRE_CXX11}" ]]; then
     BUILD_CMD="python -m pip wheel .  --extra-index-url https://download.pytorch.org/whl/nightly/cu124 -w dist"
 else
-    BUILD_CMD="python -m pip wheel . --config-setting="--build-option=--use-cxx11-abi" --extra-index-url https://download.pytorch.org/whl/nightly/cu124 -w dist"
+    BUILD_CMD="python -m pip wheel . --config-setting="--build-option=--use-pre-cxx11-abi" --extra-index-url https://download.pytorch.org/whl/nightly/cu124 -w dist"
 fi
 
 # TensorRT restricts our pip version

diff --git a/docs/_cpp_api/classtorch__tensorrt_1_1DataType.html b/docs/_cpp_api/classtorch__tensorrt_1_1DataType.html
@@ -10,7 +10,7 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Class DataType &mdash; Torch-TensorRT v2.7.0.dev0+d6be4ba documentation</title>
+  <title>Class DataType &mdash; Torch-TensorRT v2.7.0.dev0+f2a38f5 documentation</title>
 
 
 
@@ -293,7 +293,7 @@
 
 
                 <div class="version">
-                  v2.7.0.dev0+d6be4ba
+                  v2.7.0.dev0+f2a38f5
                 </div>
 
 

diff --git a/docs/_cpp_api/classtorch__tensorrt_1_1Device_1_1DeviceType.html b/docs/_cpp_api/classtorch__tensorrt_1_1Device_1_1DeviceType.html
@@ -10,7 +10,7 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Class Device::DeviceType &mdash; Torch-TensorRT v2.7.0.dev0+d6be4ba documentation</title>
+  <title>Class Device::DeviceType &mdash; Torch-TensorRT v2.7.0.dev0+f2a38f5 documentation</title>
 
 
 
@@ -293,7 +293,7 @@
 
 
                 <div class="version">
-                  v2.7.0.dev0+d6be4ba
+                  v2.7.0.dev0+f2a38f5
                 </div>
 
 

diff --git a/docs/_cpp_api/classtorch__tensorrt_1_1TensorFormat.html b/docs/_cpp_api/classtorch__tensorrt_1_1TensorFormat.html
@@ -10,7 +10,7 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Class TensorFormat &mdash; Torch-TensorRT v2.7.0.dev0+d6be4ba documentation</title>
+  <title>Class TensorFormat &mdash; Torch-TensorRT v2.7.0.dev0+f2a38f5 documentation</title>
 
 
 
@@ -293,7 +293,7 @@
 
 
                 <div class="version">
-                  v2.7.0.dev0+d6be4ba
+                  v2.7.0.dev0+f2a38f5
                 </div>
 
 

diff --git a/docs/_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8CacheCalibrator.html b/docs/_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8CacheCalibrator.html
@@ -10,7 +10,7 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Template Class Int8CacheCalibrator &mdash; Torch-TensorRT v2.7.0.dev0+d6be4ba documentation</title>
+  <title>Template Class Int8CacheCalibrator &mdash; Torch-TensorRT v2.7.0.dev0+f2a38f5 documentation</title>
 
 
 
@@ -293,7 +293,7 @@
 
 
                 <div class="version">
-                  v2.7.0.dev0+d6be4ba
+                  v2.7.0.dev0+f2a38f5
                 </div>
 
 

diff --git a/docs/_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8Calibrator.html b/docs/_cpp_api/classtorch__tensorrt_1_1ptq_1_1Int8Calibrator.html
@@ -10,7 +10,7 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Template Class Int8Calibrator &mdash; Torch-TensorRT v2.7.0.dev0+d6be4ba documentation</title>
+  <title>Template Class Int8Calibrator &mdash; Torch-TensorRT v2.7.0.dev0+f2a38f5 documentation</title>
 
 
 
@@ -293,7 +293,7 @@
 
 
                 <div class="version">
-                  v2.7.0.dev0+d6be4ba
+                  v2.7.0.dev0+f2a38f5
                 </div>
 
 

diff --git a/docs/_cpp_api/define_macros_8h_1a18d295a837ac71add5578860b55e5502.html b/docs/_cpp_api/define_macros_8h_1a18d295a837ac71add5578860b55e5502.html
@@ -10,7 +10,7 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Define STR &mdash; Torch-TensorRT v2.7.0.dev0+d6be4ba documentation</title>
+  <title>Define STR &mdash; Torch-TensorRT v2.7.0.dev0+f2a38f5 documentation</title>
 
 
 
@@ -293,7 +293,7 @@
 
 
                 <div class="version">
-                  v2.7.0.dev0+d6be4ba
+                  v2.7.0.dev0+f2a38f5
                 </div>
 
 

diff --git a/docs/_cpp_api/define_macros_8h_1a282fd3c0b1c3a215148ae372070e1268.html b/docs/_cpp_api/define_macros_8h_1a282fd3c0b1c3a215148ae372070e1268.html
@@ -10,7 +10,7 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Define TORCH_TENSORRT_PATCH_VERSION &mdash; Torch-TensorRT v2.7.0.dev0+d6be4ba documentation</title>
+  <title>Define TORCH_TENSORRT_PATCH_VERSION &mdash; Torch-TensorRT v2.7.0.dev0+f2a38f5 documentation</title>
 
 
 
@@ -293,7 +293,7 @@
 
 
                 <div class="version">
-                  v2.7.0.dev0+d6be4ba
+                  v2.7.0.dev0+f2a38f5
                 </div>
 
 

diff --git a/docs/_cpp_api/define_macros_8h_1a31398a6d4d27e28817afb0f0139e909e.html b/docs/_cpp_api/define_macros_8h_1a31398a6d4d27e28817afb0f0139e909e.html
@@ -10,7 +10,7 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Define TORCH_TENSORRT_MAJOR_VERSION &mdash; Torch-TensorRT v2.7.0.dev0+d6be4ba documentation</title>
+  <title>Define TORCH_TENSORRT_MAJOR_VERSION &mdash; Torch-TensorRT v2.7.0.dev0+f2a38f5 documentation</title>
 
 
 
@@ -293,7 +293,7 @@
 
 
                 <div class="version">
-                  v2.7.0.dev0+d6be4ba
+                  v2.7.0.dev0+f2a38f5
                 </div>
 
 

diff --git a/docs/_cpp_api/define_macros_8h_1a35703561b26b1a9d2738ad7d58b27827.html b/docs/_cpp_api/define_macros_8h_1a35703561b26b1a9d2738ad7d58b27827.html
@@ -10,7 +10,7 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Define TORCH_TENSORRT_MINOR_VERSION &mdash; Torch-TensorRT v2.7.0.dev0+d6be4ba documentation</title>
+  <title>Define TORCH_TENSORRT_MINOR_VERSION &mdash; Torch-TensorRT v2.7.0.dev0+f2a38f5 documentation</title>
 
 
 
@@ -293,7 +293,7 @@
 
 
                 <div class="version">
-                  v2.7.0.dev0+d6be4ba
+                  v2.7.0.dev0+f2a38f5
                 </div>
 
 

diff --git a/docs/_cpp_api/define_macros_8h_1abd1465eb38256d3f22cc1426b23d516b.html b/docs/_cpp_api/define_macros_8h_1abd1465eb38256d3f22cc1426b23d516b.html
@@ -10,7 +10,7 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Define TORCHTRT_API &mdash; Torch-TensorRT v2.7.0.dev0+d6be4ba documentation</title>
+  <title>Define TORCHTRT_API &mdash; Torch-TensorRT v2.7.0.dev0+f2a38f5 documentation</title>
 
 
 
@@ -293,7 +293,7 @@
 
 
                 <div class="version">
-                  v2.7.0.dev0+d6be4ba
+                  v2.7.0.dev0+f2a38f5
                 </div>
 
 

diff --git a/docs/_cpp_api/define_macros_8h_1abe87b341f562fd1cf40b7672e4d759da.html b/docs/_cpp_api/define_macros_8h_1abe87b341f562fd1cf40b7672e4d759da.html
@@ -10,7 +10,7 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Define XSTR &mdash; Torch-TensorRT v2.7.0.dev0+d6be4ba documentation</title>
+  <title>Define XSTR &mdash; Torch-TensorRT v2.7.0.dev0+f2a38f5 documentation</title>
 
 
 
@@ -293,7 +293,7 @@
 
 
                 <div class="version">
-                  v2.7.0.dev0+d6be4ba
+                  v2.7.0.dev0+f2a38f5
                 </div>
 
 

diff --git a/docs/_cpp_api/define_macros_8h_1ad19939408f7be171a74a89928b36eb59.html b/docs/_cpp_api/define_macros_8h_1ad19939408f7be171a74a89928b36eb59.html
@@ -10,7 +10,7 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Define TORCHTRT_HIDDEN &mdash; Torch-TensorRT v2.7.0.dev0+d6be4ba documentation</title>
+  <title>Define TORCHTRT_HIDDEN &mdash; Torch-TensorRT v2.7.0.dev0+f2a38f5 documentation</title>
 
 
 
@@ -293,7 +293,7 @@
 
 
                 <div class="version">
-                  v2.7.0.dev0+d6be4ba
+                  v2.7.0.dev0+f2a38f5
                 </div>
 
 

diff --git a/docs/_cpp_api/define_macros_8h_1adad592a7b1b7eed529cdf6acd584c883.html b/docs/_cpp_api/define_macros_8h_1adad592a7b1b7eed529cdf6acd584c883.html
@@ -10,7 +10,7 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Define TORCH_TENSORRT_VERSION &mdash; Torch-TensorRT v2.7.0.dev0+d6be4ba documentation</title>
+  <title>Define TORCH_TENSORRT_VERSION &mdash; Torch-TensorRT v2.7.0.dev0+f2a38f5 documentation</title>
 
 
 
@@ -293,7 +293,7 @@
 
 
                 <div class="version">
-                  v2.7.0.dev0+d6be4ba
+                  v2.7.0.dev0+f2a38f5
                 </div>
 
 

diff --git a/docs/_cpp_api/dir_cpp.html b/docs/_cpp_api/dir_cpp.html
@@ -10,7 +10,7 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-  <title>Directory cpp &mdash; Torch-TensorRT v2.7.0.dev0+d6be4ba documentation</title>
+  <title>Directory cpp &mdash; Torch-TensorRT v2.7.0.dev0+f2a38f5 documentation</title>
 
 
 
@@ -291,7 +291,7 @@
 
 
                 <div class="version">
-                  v2.7.0.dev0+d6be4ba
+                  v2.7.0.dev0+f2a38f5
                 </div>