diff --git a/.github/actions/setup-binary-builds/action.yml b/.github/actions/setup-binary-builds/action.yml
index e0e6202885..8376a5b14b 100644
--- a/.github/actions/setup-binary-builds/action.yml
+++ b/.github/actions/setup-binary-builds/action.yml
@@ -109,10 +109,6 @@ runs:
           set -euxo pipefail
           CONDA_ENV="${RUNNER_TEMP}/conda_environment_${GITHUB_RUN_ID}"
 
-          # Both conda-build and conda need to be advanced togeather.
-          # This will make system conda use required value.
-          conda install --yes conda=23.7.3
-
           if [[ "${PYTHON_VERSION}" = "3.11" ]]; then
 
             if [[ "${PACKAGE_TYPE:-}" == "conda" ]]; then
diff --git a/.github/workflows/build_conda_macos.yml b/.github/workflows/build_conda_macos.yml
index bd0b58de1c..ef1c0fb815 100644
--- a/.github/workflows/build_conda_macos.yml
+++ b/.github/workflows/build_conda_macos.yml
@@ -141,6 +141,10 @@ jobs:
             export USE_OPENMP="0"
           fi
 
+          # conda 23.10 is installed by default on macos M1
+          # and this creates an issue. Hence downgrading it here
+          conda install --yes conda=23.7.3
+
           ${CONDA_RUN} conda build \
             -c defaults \
             -c "pytorch-${CHANNEL}" \