try this

.github/workflows/sphinx.yaml

@@ -9,7 +9,23 @@ jobs:
   docs:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v1
-    - uses: ammaraskar/sphinx-action@master
-      with:
-        docs-folder: "docs/"
+      - uses: actions/checkout@v2
+      - name: Cache conda
+        uses: actions/cache@v2
+        env:
+          # Increase this value to reset cache if ci/environment.yaml has not changed
+          CACHE_NUMBER: 0
+        with:
+          path: ~/conda_pkgs_dir
+          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/environment.yaml') }}
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: xesn # Defined in ci/environment*.yml
+          auto-update-conda: false
+          python-version: ${{ matrix.python-version }}
+          environment-file: docs/environment.yaml
+          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+      - uses: actions/checkout@v1
+      - uses: ammaraskar/sphinx-action@master
+        with:
+          docs-folder: "docs/"