diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml
index 9b499d5165..12f634c524 100644
--- a/.github/actions/setup-binary-builds/action.yml
+++ b/.github/actions/setup-binary-builds/action.yml
@@ -73,6 +73,8 @@ runs:
         with:
           miniconda-version: "py39_4.12.0"
           python-version: 3.9
+          activate-environment: ""
+          auto-activate-base: true
       - name: Clean conda environment
         shell: bash -l {0}
         run: |
@@ -109,38 +111,17 @@ runs:
           set -euxo pipefail
           CONDA_ENV="${RUNNER_TEMP}/conda_environment_${GITHUB_RUN_ID}"
 
-          if [[ "${PYTHON_VERSION}" = "3.11" ]]; then
+          conda create \
+            --yes --quiet \
+            --prefix "${CONDA_ENV}" \
+            "python=${PYTHON_VERSION}" \
+            cmake=3.22 \
+            conda-build=3.21 \
+            ninja=1.10 \
+            pkg-config=0.29 \
+            wheel=0.37
 
-            if [[ "${PACKAGE_TYPE:-}" == "conda" ]]; then
-              conda create \
-                --yes --quiet \
-                --prefix "${CONDA_ENV}" \
-                "python=3.8" \
-                cmake=3.22 \
-                conda-build=3.21 \
-                ninja=1.10 \
-                pkg-config=0.29 \
-                wheel=0.37
-            else
-              conda create \
-                --yes --quiet \
-                --prefix "${CONDA_ENV}" \
-                "python=${PYTHON_VERSION}" \
-                cmake=3.22 \
-                ninja=1.10 \
-                pkg-config=0.29 \
-                wheel=0.37
-            fi
-          else
-            conda create \
-              --yes --quiet \
-              --prefix "${CONDA_ENV}" \
-              "python=${PYTHON_VERSION}" \
-              cmake=3.22 \
-              conda-build=3.21 \
-              ninja=1.10 \
-              pkg-config=0.29 \
-              wheel=0.37
+          if [[ "${PACKAGE_TYPE:-}" == "conda" ]]; then
           fi
           echo "CONDA_ENV=${CONDA_ENV}" >> "${GITHUB_ENV}"
           echo "CONDA_RUN=conda run -p ${CONDA_ENV}" >> "${GITHUB_ENV}"
diff --git a/.github/workflows/build_conda_linux.yml b/.github/workflows/build_conda_linux.yml
index 96a3963cc4..78cf6b57d3 100644
--- a/.github/workflows/build_conda_linux.yml
+++ b/.github/workflows/build_conda_linux.yml
@@ -96,9 +96,6 @@ jobs:
           ref: ${{ inputs.test-infra-ref }}
           path: test-infra
       - uses: ./test-infra/.github/actions/set-channel
-      - uses: ./test-infra/.github/actions/setup-ssh
-        with:
-          github-secret: ${{ github.token }}
       - uses: ./test-infra/.github/actions/setup-binary-builds
         with:
           repository: ${{ inputs.repository }}
@@ -132,17 +129,16 @@ jobs:
           # shellcheck disable=SC1090
           source "${BUILD_ENV_FILE}"
 
-          export PATH=$CONDA_ENV/bin:$CONDA_ENV:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH
-          export Python3_ROOT_DIR=${CONDA_ENV}
-          export Python3_EXECUTABLE="${CONDA_ENV}"
-
+          # Please note these are required changes in
+          # export PATH=$CONDA_ENV/bin:$CONDA_ENV:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH
+          # export Python3_ROOT_DIR=${CONDA_ENV}
+          # export Python3_EXECUTABLE="${CONDA_ENV}"
 
           ${CONDA_RUN} conda build \
             -c defaults \
             -c "${CUDATOOLKIT_CHANNEL}" \
             -c "pytorch-${CHANNEL}" \
             --no-anaconda-upload \
-            --debug \
             --python "${PYTHON_VERSION}" \
             --output-folder distr/ \
             "${CONDA_PACKAGE_DIRECTORY}"