diff --git a/.github/workflows/linux-test.yml b/.github/workflows/linux-test.yml
index c1c267af58..b801c91878 100644
--- a/.github/workflows/linux-test.yml
+++ b/.github/workflows/linux-test.yml
@@ -25,7 +25,7 @@ on:
         type: string
       test-infra-ref:
         description: "Test infra reference to use"
-        default: ""
+        default: "release/2.3"
         type: string
       build-matrix:
         description: "Build matrix to utilize"
diff --git a/WORKSPACE b/WORKSPACE
index bbc1803296..edc5c9a050 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -54,14 +54,14 @@ http_archive(
     name = "libtorch",
     build_file = "@//third_party/libtorch:BUILD",
     strip_prefix = "libtorch",
-    urls = ["https://download.pytorch.org/libtorch/nightly/cu121/libtorch-cxx11-abi-shared-with-deps-latest.zip"],
+    urls = ["https://download.pytorch.org/libtorch/test/cu121/libtorch-cxx11-abi-shared-with-deps-2.3.0%2Bcu121.zip"],
 )
 
 http_archive(
     name = "libtorch_pre_cxx11_abi",
     build_file = "@//third_party/libtorch:BUILD",
     strip_prefix = "libtorch",
-    urls = ["https://download.pytorch.org/libtorch/nightly/cu121/libtorch-shared-with-deps-latest.zip"],
+    urls = ["https://download.pytorch.org/libtorch/test/cu121/libtorch-shared-with-deps-2.3.0%2Bcu121.zip"],
 )
 
 # Download these tarballs manually from the NVIDIA website
diff --git a/docker/dist-build.sh b/docker/dist-build.sh
index 0bc2fc09f9..ba7dd83713 100755
--- a/docker/dist-build.sh
+++ b/docker/dist-build.sh
@@ -3,9 +3,9 @@
 TOP_DIR=$(cd $(dirname $0); pwd)/..
 
 if [[ -z "${USE_CXX11}" ]]; then
-    BUILD_CMD="python -m pip wheel .  --extra-index-url https://download.pytorch.org/whl/nightly/cu121 -w dist"
+    BUILD_CMD="python -m pip wheel .  --extra-index-url https://download.pytorch.org/whl/test/cu121 -w dist"
 else
-    BUILD_CMD="python -m pip wheel . --config-setting="--build-option=--use-cxx11-abi" --extra-index-url https://download.pytorch.org/whl/nightly/cu121 -w dist"
+    BUILD_CMD="python -m pip wheel . --config-setting="--build-option=--use-cxx11-abi" --extra-index-url https://download.pytorch.org/whl/test/cu121 -w dist"
 fi
 
 # TensorRT restricts our pip version
diff --git a/py/requirements.txt b/py/requirements.txt
index 621297d46c..d402fd501e 100644
--- a/py/requirements.txt
+++ b/py/requirements.txt
@@ -1,9 +1,9 @@
 numpy
 packaging
 pybind11==2.6.2
---extra-index-url https://download.pytorch.org/whl/nightly/cu121
-torch>=2.4.0.dev,<2.5.0
-torchvision>=0.19.0.dev,<0.20.0
+--extra-index-url https://download.pytorch.org/whl/test/cu121
+torch==2.3.0
+torchvision==0.18.0
 --extra-index-url https://pypi.ngc.nvidia.com
 pyyaml
 tensorrt
\ No newline at end of file
diff --git a/toolchains/ci_workspaces/WORKSPACE.x86_64.release.rhel.tmpl b/toolchains/ci_workspaces/WORKSPACE.x86_64.release.rhel.tmpl
index 7a13798ee3..cad54b1707 100644
--- a/toolchains/ci_workspaces/WORKSPACE.x86_64.release.rhel.tmpl
+++ b/toolchains/ci_workspaces/WORKSPACE.x86_64.release.rhel.tmpl
@@ -58,14 +58,14 @@ http_archive(
     name = "libtorch",
     build_file = "@//third_party/libtorch:BUILD",
     strip_prefix = "libtorch",
-    urls = ["https://download.pytorch.org/libtorch/nightly/cu121/libtorch-cxx11-abi-shared-with-deps-latest.zip"],
+    urls = ["https://download.pytorch.org/libtorch/test/cu121/libtorch-cxx11-abi-shared-with-deps-2.3.0%2Bcu121.zip"],
 )
 
 http_archive(
     name = "libtorch_pre_cxx11_abi",
     build_file = "@//third_party/libtorch:BUILD",
     strip_prefix = "libtorch",
-    urls = ["https://download.pytorch.org/libtorch/nightly/cu121/libtorch-shared-with-deps-latest.zip"],
+    urls = ["https://download.pytorch.org/libtorch/test/cu121/libtorch-shared-with-deps-2.3.0%2Bcu121.zip"],
 )
 
 http_archive(