diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml
index 2c7b242a71..1660d596f0 100644
--- a/.github/actions/setup-binary-builds/action.yml
+++ b/.github/actions/setup-binary-builds/action.yml
@@ -108,16 +108,23 @@ runs:
         run: |
           set -euxo pipefail
           CONDA_ENV="${RUNNER_TEMP}/conda_environment_${GITHUB_RUN_ID}"
+          if [[ "${PACKAGE_TYPE:-}" == "conda" ]]; then
+            # For conda package host python version is irrelevant
+            export PYTHON_VERSION=3.9
+            export CONDA_BUILD_EXTRA="conda=23.10.0 conda-build=3.27.0"
+          else
+            # For wheel builds we don't need neither conda nor conda-build
+            export CONDA_BUILD_EXTRA=""
+          fi
 
           conda create \
             --yes --quiet \
             --prefix "${CONDA_ENV}" \
             "python=${PYTHON_VERSION}" \
             cmake=3.22 \
-            conda-build=3.27.0 \
-            conda=23.10.0 \
             ninja=1.10 \
             pkg-config=0.29 \
+            ${CONDA_BUILD_EXTRA} \
             wheel=0.37
 
           echo "CONDA_ENV=${CONDA_ENV}" >> "${GITHUB_ENV}"