From a93a09d8287eef8e2cb7b968d2d384007f8d0697 Mon Sep 17 00:00:00 2001 From: Elvis Pranskevichus Date: Sun, 18 Feb 2024 13:07:00 -0800 Subject: [PATCH] Remove use of `syphar/restore-virtualenv` (#6875) Use regular `setup-python`'s caching capabilities in tandem with `pip-compile`. --- .github/workflows.src/tests.inc.yml | 85 ++++++----- .github/workflows.src/tests.tpl.yml | 28 +--- .github/workflows/.gitattributes | 1 - .github/workflows/tests-ha.yml | 77 +++++----- .github/workflows/tests-managed-pg.yml | 193 ++++++++++++++---------- .github/workflows/tests-patches.yml | 77 +++++----- .github/workflows/tests-pg-versions.yml | 77 +++++----- .github/workflows/tests.yml | 134 ++++++++-------- pyproject.toml | 6 +- setup.py | 23 +-- 10 files changed, 362 insertions(+), 339 deletions(-) diff --git a/.github/workflows.src/tests.inc.yml b/.github/workflows.src/tests.inc.yml index 6c7b9fc2b38..e8192147202 100644 --- a/.github/workflows.src/tests.inc.yml +++ b/.github/workflows.src/tests.inc.yml @@ -1,4 +1,4 @@ -<% macro _init() -%> +<% macro init() -%> - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -13,38 +13,31 @@ uses: actions/setup-python@v5 with: python-version: '3.12.2' -<%- endmacro %> - -<% macro _init_venv() -%> - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1 - id: venv-cache - with: - requirement_files: | - setup.py + cache: 'pip' + cache-dependency-path: | pyproject.toml - custom_cache_key_element: v2 -<%- endmacro -%> -<% macro build() %> - << _init() >> +<%- endmacro %> - # Build virtualenv +<% macro build() %> + << init() >> - << _init_venv() >> + - name: Cached requirements.txt + uses: actions/cache@v4 + id: requirements-cache + with: + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} - - name: Install Python dependencies - if: steps.venv-cache.outputs.cache-hit != 'true' - shell: bash + - name: Compute requirements.txt + if: steps.requirements-cache.outputs.cache-hit != 'true' run: | - set -e - python -m pip install -U pip setuptools wheel build - bdeps_script="import build; print('\n'.join(build.ProjectBuilder('.').build_system_requires))" - readarray -t build_deps < <(python -c "${bdeps_script}") - python -m pip download --dest=$VIRTUAL_ENV/deps "${build_deps[@]}" .[test] - python -m pip install -U --no-index --find-links=$VIRTUAL_ENV/deps $VIRTUAL_ENV/deps/* + python -m pip install pip-tools + pip-compile --no-strip-extras --all-build-deps --extra test \ + --output-file requirements.txt pyproject.toml - # Prepare environment variables and shared artifacts + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt << caller() >> @@ -259,19 +252,13 @@ # Install edgedb-server and populate egg-info - - name: Install edgedb-server and populate egg-info + - name: Install edgedb-server env: - CACHE_HIT: ${{ steps.venv-cache.outputs.cache-hit }} BUILD_EXT_MODE: skip run: | - if [[ "$CACHE_HIT" == "true" ]]; then - rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/ - else - # --no-build-isolation because we have explicitly installed all deps - # and don't want them to be reinstalled in an "isolated env". - pip install --no-build-isolation --no-deps -e .[test,docs] - rsync -av ./edgedb_server.egg-info/ $VIRTUAL_ENV/edgedb_server.egg-info/ - fi + # --no-build-isolation because we have explicitly installed all deps + # and don't want them to be reinstalled in an "isolated env". + pip install --no-build-isolation --no-deps -e .[test,docs] # Refresh the bootstrap cache @@ -306,10 +293,20 @@ echo BUILD_TEMP=$(python setup.py -q ci_helper --type build_temp) >> $GITHUB_ENV <%- endmacro %> -<% macro restore_cache() %> - << _init() >> +<% macro install_python_requirements() %> + - name: Download requirements.txt + uses: actions/cache@v4 + with: + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} + + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt +<%- endmacro %> - << _init_venv() >> +<% macro restore_cache() %> + << init() >> + << install_python_requirements() >> # Restore the artifacts and environment variables @@ -382,7 +379,6 @@ - name: Stop if we cannot retrieve the cache if: | - steps.venv-cache.outputs.cache-hit != 'true' || steps.cli-cache.outputs.cache-hit != 'true' || steps.rust-cache.outputs.cache-hit != 'true' || steps.ext-cache.outputs.cache-hit != 'true' || @@ -401,7 +397,14 @@ rsync -av ./build/extensions/edb/ ./edb/ rsync -av ./build/lib/edb/ ./edb/ cp build/postgres/install/stamp build/postgres/ - rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/ + + - name: Install edgedb-server + env: + BUILD_EXT_MODE: skip + run: | + # --no-build-isolation because we have explicitly installed all deps + # and don't want them to be reinstalled in an "isolated env". + pip install --no-build-isolation --no-deps -e .[test,docs] <%- endmacro %> <% macro setup_terraform() -%> diff --git a/.github/workflows.src/tests.tpl.yml b/.github/workflows.src/tests.tpl.yml index 9ea4fc733e6..4a463097220 100644 --- a/.github/workflows.src/tests.tpl.yml +++ b/.github/workflows.src/tests.tpl.yml @@ -1,4 +1,4 @@ -<% from "tests.inc.yml" import build, calc_cache_key, restore_cache -%> +<% from "tests.inc.yml" import init, build, calc_cache_key, install_python_requirements, restore_cache -%> name: Tests on: @@ -38,30 +38,8 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: false - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12.2' - - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1.2 - id: venv-cache - with: - requirement_files: | - setup.py - pyproject.toml - custom_cache_key_element: v2 - - - name: Stop if we cannot retrieve the cache - if: steps.venv-cache.outputs.cache-hit != 'true' - run: | - echo ::error::Cannot retrieve venv cache. - exit 1 + << init() >> + << install_python_requirements() >> - name: Download cache key uses: actions/download-artifact@v4 diff --git a/.github/workflows/.gitattributes b/.github/workflows/.gitattributes index c819ee46242..f05ab75e04c 100644 --- a/.github/workflows/.gitattributes +++ b/.github/workflows/.gitattributes @@ -1,2 +1 @@ -*.yml -diff -merge *.yml linguist-generated=true diff --git a/.github/workflows/tests-ha.yml b/.github/workflows/tests-ha.yml index fbac77c3c1f..7c35885d8b0 100644 --- a/.github/workflows/tests-ha.yml +++ b/.github/workflows/tests-ha.yml @@ -28,30 +28,26 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.12.2' + cache: 'pip' + cache-dependency-path: | + pyproject.toml - # Build virtualenv - - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1 - id: venv-cache + - name: Cached requirements.txt + uses: actions/cache@v4 + id: requirements-cache with: - requirement_files: | - setup.py - pyproject.toml - custom_cache_key_element: v2 + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} - - name: Install Python dependencies - if: steps.venv-cache.outputs.cache-hit != 'true' - shell: bash + - name: Compute requirements.txt + if: steps.requirements-cache.outputs.cache-hit != 'true' run: | - set -e - python -m pip install -U pip setuptools wheel build - bdeps_script="import build; print('\n'.join(build.ProjectBuilder('.').build_system_requires))" - readarray -t build_deps < <(python -c "${bdeps_script}") - python -m pip download --dest=$VIRTUAL_ENV/deps "${build_deps[@]}" .[test] - python -m pip install -U --no-index --find-links=$VIRTUAL_ENV/deps $VIRTUAL_ENV/deps/* + python -m pip install pip-tools + pip-compile --no-strip-extras --all-build-deps --extra test \ + --output-file requirements.txt pyproject.toml - # Prepare environment variables and shared artifacts + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt # Our HA tests currently only work on Postgres 14 (see #6332), # so check it out before we compute our build cache keys. @@ -295,19 +291,13 @@ jobs: # Install edgedb-server and populate egg-info - - name: Install edgedb-server and populate egg-info + - name: Install edgedb-server env: - CACHE_HIT: ${{ steps.venv-cache.outputs.cache-hit }} BUILD_EXT_MODE: skip run: | - if [[ "$CACHE_HIT" == "true" ]]; then - rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/ - else - # --no-build-isolation because we have explicitly installed all deps - # and don't want them to be reinstalled in an "isolated env". - pip install --no-build-isolation --no-deps -e .[test,docs] - rsync -av ./edgedb_server.egg-info/ $VIRTUAL_ENV/edgedb_server.egg-info/ - fi + # --no-build-isolation because we have explicitly installed all deps + # and don't want them to be reinstalled in an "isolated env". + pip install --no-build-isolation --no-deps -e .[test,docs] # Refresh the bootstrap cache @@ -344,15 +334,18 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.12.2' - - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1 - id: venv-cache - with: - requirement_files: | - setup.py + cache: 'pip' + cache-dependency-path: | pyproject.toml - custom_cache_key_element: v2 + + - name: Download requirements.txt + uses: actions/cache@v4 + with: + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} + + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt # Restore the artifacts and environment variables @@ -425,7 +418,6 @@ jobs: - name: Stop if we cannot retrieve the cache if: | - steps.venv-cache.outputs.cache-hit != 'true' || steps.cli-cache.outputs.cache-hit != 'true' || steps.rust-cache.outputs.cache-hit != 'true' || steps.ext-cache.outputs.cache-hit != 'true' || @@ -444,7 +436,14 @@ jobs: rsync -av ./build/extensions/edb/ ./edb/ rsync -av ./build/lib/edb/ ./edb/ cp build/postgres/install/stamp build/postgres/ - rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/ + + - name: Install edgedb-server + env: + BUILD_EXT_MODE: skip + run: | + # --no-build-isolation because we have explicitly installed all deps + # and don't want them to be reinstalled in an "isolated env". + pip install --no-build-isolation --no-deps -e .[test,docs] # Run the test diff --git a/.github/workflows/tests-managed-pg.yml b/.github/workflows/tests-managed-pg.yml index 6fa4a31a909..0dab4494e70 100644 --- a/.github/workflows/tests-managed-pg.yml +++ b/.github/workflows/tests-managed-pg.yml @@ -28,30 +28,26 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.12.2' + cache: 'pip' + cache-dependency-path: | + pyproject.toml - # Build virtualenv - - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1 - id: venv-cache + - name: Cached requirements.txt + uses: actions/cache@v4 + id: requirements-cache with: - requirement_files: | - setup.py - pyproject.toml - custom_cache_key_element: v2 + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} - - name: Install Python dependencies - if: steps.venv-cache.outputs.cache-hit != 'true' - shell: bash + - name: Compute requirements.txt + if: steps.requirements-cache.outputs.cache-hit != 'true' run: | - set -e - python -m pip install -U pip setuptools wheel build - bdeps_script="import build; print('\n'.join(build.ProjectBuilder('.').build_system_requires))" - readarray -t build_deps < <(python -c "${bdeps_script}") - python -m pip download --dest=$VIRTUAL_ENV/deps "${build_deps[@]}" .[test] - python -m pip install -U --no-index --find-links=$VIRTUAL_ENV/deps $VIRTUAL_ENV/deps/* + python -m pip install pip-tools + pip-compile --no-strip-extras --all-build-deps --extra test \ + --output-file requirements.txt pyproject.toml - # Prepare environment variables and shared artifacts + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt - name: Compute cache keys run: | @@ -280,19 +276,13 @@ jobs: # Install edgedb-server and populate egg-info - - name: Install edgedb-server and populate egg-info + - name: Install edgedb-server env: - CACHE_HIT: ${{ steps.venv-cache.outputs.cache-hit }} BUILD_EXT_MODE: skip run: | - if [[ "$CACHE_HIT" == "true" ]]; then - rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/ - else - # --no-build-isolation because we have explicitly installed all deps - # and don't want them to be reinstalled in an "isolated env". - pip install --no-build-isolation --no-deps -e .[test,docs] - rsync -av ./edgedb_server.egg-info/ $VIRTUAL_ENV/edgedb_server.egg-info/ - fi + # --no-build-isolation because we have explicitly installed all deps + # and don't want them to be reinstalled in an "isolated env". + pip install --no-build-isolation --no-deps -e .[test,docs] # Refresh the bootstrap cache @@ -375,15 +365,18 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.12.2' - - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1 - id: venv-cache - with: - requirement_files: | - setup.py + cache: 'pip' + cache-dependency-path: | pyproject.toml - custom_cache_key_element: v2 + + - name: Download requirements.txt + uses: actions/cache@v4 + with: + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} + + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt # Restore the artifacts and environment variables @@ -456,7 +449,6 @@ jobs: - name: Stop if we cannot retrieve the cache if: | - steps.venv-cache.outputs.cache-hit != 'true' || steps.cli-cache.outputs.cache-hit != 'true' || steps.rust-cache.outputs.cache-hit != 'true' || steps.ext-cache.outputs.cache-hit != 'true' || @@ -475,7 +467,14 @@ jobs: rsync -av ./build/extensions/edb/ ./edb/ rsync -av ./build/lib/edb/ ./edb/ cp build/postgres/install/stamp build/postgres/ - rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/ + + - name: Install edgedb-server + env: + BUILD_EXT_MODE: skip + run: | + # --no-build-isolation because we have explicitly installed all deps + # and don't want them to be reinstalled in an "isolated env". + pip install --no-build-isolation --no-deps -e .[test,docs] # Run the test @@ -581,15 +580,18 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.12.2' - - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1 - id: venv-cache - with: - requirement_files: | - setup.py + cache: 'pip' + cache-dependency-path: | pyproject.toml - custom_cache_key_element: v2 + + - name: Download requirements.txt + uses: actions/cache@v4 + with: + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} + + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt # Restore the artifacts and environment variables @@ -662,7 +664,6 @@ jobs: - name: Stop if we cannot retrieve the cache if: | - steps.venv-cache.outputs.cache-hit != 'true' || steps.cli-cache.outputs.cache-hit != 'true' || steps.rust-cache.outputs.cache-hit != 'true' || steps.ext-cache.outputs.cache-hit != 'true' || @@ -681,7 +682,14 @@ jobs: rsync -av ./build/extensions/edb/ ./edb/ rsync -av ./build/lib/edb/ ./edb/ cp build/postgres/install/stamp build/postgres/ - rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/ + + - name: Install edgedb-server + env: + BUILD_EXT_MODE: skip + run: | + # --no-build-isolation because we have explicitly installed all deps + # and don't want them to be reinstalled in an "isolated env". + pip install --no-build-isolation --no-deps -e .[test,docs] - name: Setup Terraform uses: hashicorp/setup-terraform@v1 @@ -835,15 +843,18 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.12.2' - - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1 - id: venv-cache - with: - requirement_files: | - setup.py + cache: 'pip' + cache-dependency-path: | pyproject.toml - custom_cache_key_element: v2 + + - name: Download requirements.txt + uses: actions/cache@v4 + with: + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} + + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt # Restore the artifacts and environment variables @@ -916,7 +927,6 @@ jobs: - name: Stop if we cannot retrieve the cache if: | - steps.venv-cache.outputs.cache-hit != 'true' || steps.cli-cache.outputs.cache-hit != 'true' || steps.rust-cache.outputs.cache-hit != 'true' || steps.ext-cache.outputs.cache-hit != 'true' || @@ -935,7 +945,14 @@ jobs: rsync -av ./build/extensions/edb/ ./edb/ rsync -av ./build/lib/edb/ ./edb/ cp build/postgres/install/stamp build/postgres/ - rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/ + + - name: Install edgedb-server + env: + BUILD_EXT_MODE: skip + run: | + # --no-build-isolation because we have explicitly installed all deps + # and don't want them to be reinstalled in an "isolated env". + pip install --no-build-isolation --no-deps -e .[test,docs] # Run the test @@ -1055,15 +1072,18 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.12.2' - - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1 - id: venv-cache - with: - requirement_files: | - setup.py + cache: 'pip' + cache-dependency-path: | pyproject.toml - custom_cache_key_element: v2 + + - name: Download requirements.txt + uses: actions/cache@v4 + with: + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} + + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt # Restore the artifacts and environment variables @@ -1136,7 +1156,6 @@ jobs: - name: Stop if we cannot retrieve the cache if: | - steps.venv-cache.outputs.cache-hit != 'true' || steps.cli-cache.outputs.cache-hit != 'true' || steps.rust-cache.outputs.cache-hit != 'true' || steps.ext-cache.outputs.cache-hit != 'true' || @@ -1155,7 +1174,14 @@ jobs: rsync -av ./build/extensions/edb/ ./edb/ rsync -av ./build/lib/edb/ ./edb/ cp build/postgres/install/stamp build/postgres/ - rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/ + + - name: Install edgedb-server + env: + BUILD_EXT_MODE: skip + run: | + # --no-build-isolation because we have explicitly installed all deps + # and don't want them to be reinstalled in an "isolated env". + pip install --no-build-isolation --no-deps -e .[test,docs] # Run the test @@ -1263,15 +1289,18 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.12.2' - - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1 - id: venv-cache - with: - requirement_files: | - setup.py + cache: 'pip' + cache-dependency-path: | pyproject.toml - custom_cache_key_element: v2 + + - name: Download requirements.txt + uses: actions/cache@v4 + with: + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} + + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt # Restore the artifacts and environment variables @@ -1344,7 +1373,6 @@ jobs: - name: Stop if we cannot retrieve the cache if: | - steps.venv-cache.outputs.cache-hit != 'true' || steps.cli-cache.outputs.cache-hit != 'true' || steps.rust-cache.outputs.cache-hit != 'true' || steps.ext-cache.outputs.cache-hit != 'true' || @@ -1363,7 +1391,14 @@ jobs: rsync -av ./build/extensions/edb/ ./edb/ rsync -av ./build/lib/edb/ ./edb/ cp build/postgres/install/stamp build/postgres/ - rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/ + + - name: Install edgedb-server + env: + BUILD_EXT_MODE: skip + run: | + # --no-build-isolation because we have explicitly installed all deps + # and don't want them to be reinstalled in an "isolated env". + pip install --no-build-isolation --no-deps -e .[test,docs] - name: Setup Terraform uses: hashicorp/setup-terraform@v1 diff --git a/.github/workflows/tests-patches.yml b/.github/workflows/tests-patches.yml index f137d9e4cfb..8b6bf171840 100644 --- a/.github/workflows/tests-patches.yml +++ b/.github/workflows/tests-patches.yml @@ -30,30 +30,26 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.12.2' + cache: 'pip' + cache-dependency-path: | + pyproject.toml - # Build virtualenv - - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1 - id: venv-cache + - name: Cached requirements.txt + uses: actions/cache@v4 + id: requirements-cache with: - requirement_files: | - setup.py - pyproject.toml - custom_cache_key_element: v2 + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} - - name: Install Python dependencies - if: steps.venv-cache.outputs.cache-hit != 'true' - shell: bash + - name: Compute requirements.txt + if: steps.requirements-cache.outputs.cache-hit != 'true' run: | - set -e - python -m pip install -U pip setuptools wheel build - bdeps_script="import build; print('\n'.join(build.ProjectBuilder('.').build_system_requires))" - readarray -t build_deps < <(python -c "${bdeps_script}") - python -m pip download --dest=$VIRTUAL_ENV/deps "${build_deps[@]}" .[test] - python -m pip install -U --no-index --find-links=$VIRTUAL_ENV/deps $VIRTUAL_ENV/deps/* + python -m pip install pip-tools + pip-compile --no-strip-extras --all-build-deps --extra test \ + --output-file requirements.txt pyproject.toml - # Prepare environment variables and shared artifacts + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt - name: Compute cache keys run: | @@ -282,19 +278,13 @@ jobs: # Install edgedb-server and populate egg-info - - name: Install edgedb-server and populate egg-info + - name: Install edgedb-server env: - CACHE_HIT: ${{ steps.venv-cache.outputs.cache-hit }} BUILD_EXT_MODE: skip run: | - if [[ "$CACHE_HIT" == "true" ]]; then - rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/ - else - # --no-build-isolation because we have explicitly installed all deps - # and don't want them to be reinstalled in an "isolated env". - pip install --no-build-isolation --no-deps -e .[test,docs] - rsync -av ./edgedb_server.egg-info/ $VIRTUAL_ENV/edgedb_server.egg-info/ - fi + # --no-build-isolation because we have explicitly installed all deps + # and don't want them to be reinstalled in an "isolated env". + pip install --no-build-isolation --no-deps -e .[test,docs] # Refresh the bootstrap cache @@ -344,15 +334,18 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.12.2' - - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1 - id: venv-cache - with: - requirement_files: | - setup.py + cache: 'pip' + cache-dependency-path: | pyproject.toml - custom_cache_key_element: v2 + + - name: Download requirements.txt + uses: actions/cache@v4 + with: + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} + + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt # Restore the artifacts and environment variables @@ -425,7 +418,6 @@ jobs: - name: Stop if we cannot retrieve the cache if: | - steps.venv-cache.outputs.cache-hit != 'true' || steps.cli-cache.outputs.cache-hit != 'true' || steps.rust-cache.outputs.cache-hit != 'true' || steps.ext-cache.outputs.cache-hit != 'true' || @@ -444,7 +436,14 @@ jobs: rsync -av ./build/extensions/edb/ ./edb/ rsync -av ./build/lib/edb/ ./edb/ cp build/postgres/install/stamp build/postgres/ - rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/ + + - name: Install edgedb-server + env: + BUILD_EXT_MODE: skip + run: | + # --no-build-isolation because we have explicitly installed all deps + # and don't want them to be reinstalled in an "isolated env". + pip install --no-build-isolation --no-deps -e .[test,docs] # Run the test diff --git a/.github/workflows/tests-pg-versions.yml b/.github/workflows/tests-pg-versions.yml index abb0c86db76..fd83d586ef2 100644 --- a/.github/workflows/tests-pg-versions.yml +++ b/.github/workflows/tests-pg-versions.yml @@ -28,30 +28,26 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.12.2' + cache: 'pip' + cache-dependency-path: | + pyproject.toml - # Build virtualenv - - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1 - id: venv-cache + - name: Cached requirements.txt + uses: actions/cache@v4 + id: requirements-cache with: - requirement_files: | - setup.py - pyproject.toml - custom_cache_key_element: v2 + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} - - name: Install Python dependencies - if: steps.venv-cache.outputs.cache-hit != 'true' - shell: bash + - name: Compute requirements.txt + if: steps.requirements-cache.outputs.cache-hit != 'true' run: | - set -e - python -m pip install -U pip setuptools wheel build - bdeps_script="import build; print('\n'.join(build.ProjectBuilder('.').build_system_requires))" - readarray -t build_deps < <(python -c "${bdeps_script}") - python -m pip download --dest=$VIRTUAL_ENV/deps "${build_deps[@]}" .[test] - python -m pip install -U --no-index --find-links=$VIRTUAL_ENV/deps $VIRTUAL_ENV/deps/* + python -m pip install pip-tools + pip-compile --no-strip-extras --all-build-deps --extra test \ + --output-file requirements.txt pyproject.toml - # Prepare environment variables and shared artifacts + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt - name: Compute cache keys run: | @@ -280,19 +276,13 @@ jobs: # Install edgedb-server and populate egg-info - - name: Install edgedb-server and populate egg-info + - name: Install edgedb-server env: - CACHE_HIT: ${{ steps.venv-cache.outputs.cache-hit }} BUILD_EXT_MODE: skip run: | - if [[ "$CACHE_HIT" == "true" ]]; then - rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/ - else - # --no-build-isolation because we have explicitly installed all deps - # and don't want them to be reinstalled in an "isolated env". - pip install --no-build-isolation --no-deps -e .[test,docs] - rsync -av ./edgedb_server.egg-info/ $VIRTUAL_ENV/edgedb_server.egg-info/ - fi + # --no-build-isolation because we have explicitly installed all deps + # and don't want them to be reinstalled in an "isolated env". + pip install --no-build-isolation --no-deps -e .[test,docs] # Refresh the bootstrap cache @@ -363,15 +353,18 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.12.2' - - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1 - id: venv-cache - with: - requirement_files: | - setup.py + cache: 'pip' + cache-dependency-path: | pyproject.toml - custom_cache_key_element: v2 + + - name: Download requirements.txt + uses: actions/cache@v4 + with: + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} + + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt # Restore the artifacts and environment variables @@ -444,7 +437,6 @@ jobs: - name: Stop if we cannot retrieve the cache if: | - steps.venv-cache.outputs.cache-hit != 'true' || steps.cli-cache.outputs.cache-hit != 'true' || steps.rust-cache.outputs.cache-hit != 'true' || steps.ext-cache.outputs.cache-hit != 'true' || @@ -463,7 +455,14 @@ jobs: rsync -av ./build/extensions/edb/ ./edb/ rsync -av ./build/lib/edb/ ./edb/ cp build/postgres/install/stamp build/postgres/ - rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/ + + - name: Install edgedb-server + env: + BUILD_EXT_MODE: skip + run: | + # --no-build-isolation because we have explicitly installed all deps + # and don't want them to be reinstalled in an "isolated env". + pip install --no-build-isolation --no-deps -e .[test,docs] # Run the test diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2fad8bce7da..1955062d64d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,30 +31,26 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.12.2' + cache: 'pip' + cache-dependency-path: | + pyproject.toml - # Build virtualenv - - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1 - id: venv-cache + - name: Cached requirements.txt + uses: actions/cache@v4 + id: requirements-cache with: - requirement_files: | - setup.py - pyproject.toml - custom_cache_key_element: v2 + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} - - name: Install Python dependencies - if: steps.venv-cache.outputs.cache-hit != 'true' - shell: bash + - name: Compute requirements.txt + if: steps.requirements-cache.outputs.cache-hit != 'true' run: | - set -e - python -m pip install -U pip setuptools wheel build - bdeps_script="import build; print('\n'.join(build.ProjectBuilder('.').build_system_requires))" - readarray -t build_deps < <(python -c "${bdeps_script}") - python -m pip download --dest=$VIRTUAL_ENV/deps "${build_deps[@]}" .[test] - python -m pip install -U --no-index --find-links=$VIRTUAL_ENV/deps $VIRTUAL_ENV/deps/* + python -m pip install pip-tools + pip-compile --no-strip-extras --all-build-deps --extra test \ + --output-file requirements.txt pyproject.toml - # Prepare environment variables and shared artifacts + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt - name: Compute cache keys and download the running times log env: @@ -292,19 +288,13 @@ jobs: # Install edgedb-server and populate egg-info - - name: Install edgedb-server and populate egg-info + - name: Install edgedb-server env: - CACHE_HIT: ${{ steps.venv-cache.outputs.cache-hit }} BUILD_EXT_MODE: skip run: | - if [[ "$CACHE_HIT" == "true" ]]; then - rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/ - else - # --no-build-isolation because we have explicitly installed all deps - # and don't want them to be reinstalled in an "isolated env". - pip install --no-build-isolation --no-deps -e .[test,docs] - rsync -av ./edgedb_server.egg-info/ $VIRTUAL_ENV/edgedb_server.egg-info/ - fi + # --no-build-isolation because we have explicitly installed all deps + # and don't want them to be reinstalled in an "isolated env". + pip install --no-build-isolation --no-deps -e .[test,docs] # Refresh the bootstrap cache @@ -332,25 +322,27 @@ jobs: fetch-depth: 0 submodules: false + - uses: actions/checkout@v4 + with: + fetch-depth: 50 + submodules: true + - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.12.2' - - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1.2 - id: venv-cache - with: - requirement_files: | - setup.py + cache: 'pip' + cache-dependency-path: | pyproject.toml - custom_cache_key_element: v2 + + - name: Download requirements.txt + uses: actions/cache@v4 + with: + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} - - name: Stop if we cannot retrieve the cache - if: steps.venv-cache.outputs.cache-hit != 'true' - run: | - echo ::error::Cannot retrieve venv cache. - exit 1 + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt - name: Download cache key uses: actions/download-artifact@v4 @@ -410,15 +402,18 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.12.2' - - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1 - id: venv-cache - with: - requirement_files: | - setup.py + cache: 'pip' + cache-dependency-path: | pyproject.toml - custom_cache_key_element: v2 + + - name: Download requirements.txt + uses: actions/cache@v4 + with: + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} + + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt # Restore the artifacts and environment variables @@ -491,7 +486,6 @@ jobs: - name: Stop if we cannot retrieve the cache if: | - steps.venv-cache.outputs.cache-hit != 'true' || steps.cli-cache.outputs.cache-hit != 'true' || steps.rust-cache.outputs.cache-hit != 'true' || steps.ext-cache.outputs.cache-hit != 'true' || @@ -510,7 +504,14 @@ jobs: rsync -av ./build/extensions/edb/ ./edb/ rsync -av ./build/lib/edb/ ./edb/ cp build/postgres/install/stamp build/postgres/ - rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/ + + - name: Install edgedb-server + env: + BUILD_EXT_MODE: skip + run: | + # --no-build-isolation because we have explicitly installed all deps + # and don't want them to be reinstalled in an "isolated env". + pip install --no-build-isolation --no-deps -e .[test,docs] # Run the test @@ -547,15 +548,18 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.12.2' - - - name: Handle virtualenv - uses: syphar/restore-virtualenv@v1 - id: venv-cache - with: - requirement_files: | - setup.py + cache: 'pip' + cache-dependency-path: | pyproject.toml - custom_cache_key_element: v2 + + - name: Download requirements.txt + uses: actions/cache@v4 + with: + path: requirements.txt + key: edb-requirements-${{ hashFiles('pyproject.toml') }} + + - name: Install Python dependencies + run: python -m pip install -U -r requirements.txt # Restore the artifacts and environment variables @@ -628,7 +632,6 @@ jobs: - name: Stop if we cannot retrieve the cache if: | - steps.venv-cache.outputs.cache-hit != 'true' || steps.cli-cache.outputs.cache-hit != 'true' || steps.rust-cache.outputs.cache-hit != 'true' || steps.ext-cache.outputs.cache-hit != 'true' || @@ -647,7 +650,14 @@ jobs: rsync -av ./build/extensions/edb/ ./edb/ rsync -av ./build/lib/edb/ ./edb/ cp build/postgres/install/stamp build/postgres/ - rsync -av $VIRTUAL_ENV/edgedb_server.egg-info/ ./edgedb_server.egg-info/ + + - name: Install edgedb-server + env: + BUILD_EXT_MODE: skip + run: | + # --no-build-isolation because we have explicitly installed all deps + # and don't want them to be reinstalled in an "isolated env". + pip install --no-build-isolation --no-deps -e .[test,docs] # Collect tests and upload diff --git a/pyproject.toml b/pyproject.toml index f4138f7219d..f8c50185b3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dependencies = [ 'setproctitle~=1.2', 'httpx~=0.24.1', - 'hishel==0.0.21', + 'hishel==0.0.24', 'webauthn~=2.0.0', 'argon2-cffi~=23.1.0', 'aiosmtplib~=2.0', @@ -51,7 +51,7 @@ test = [ 'mypy~=1.7.1', # mypy stub packages; when updating, you can use mypy --install-types # to install stub packages and then pip freeze to read out the specifier - 'types-docutils~=0.17.0,<0.17.6', # incomplete nodes.document.__init__ + 'types-docutils~=0.17.0,<0.17.6', # incomplete nodes.document.__init__ 'types-Jinja2~=2.11', 'types-MarkupSafe~=1.1', 'types-pkg-resources~=0.1.3', @@ -100,7 +100,7 @@ build-backend = "build_backend" backend-path = ["."] [tool.setuptools] -packages = {find = { include = ["edb", "edb.*"] }} +packages = { find = { include = ["edb", "edb.*"] } } zip-safe = false diff --git a/setup.py b/setup.py index 4aae35ffb8c..7b2e16bfdad 100644 --- a/setup.py +++ b/setup.py @@ -433,21 +433,22 @@ def _compile_cli(build_base, build_temp): ) +_PYTHON_ONLY = os.environ.get("BUILD_EXT_MODE", "both") == "skip" + + class build(setuptools_build.build): user_options = setuptools_build.build.user_options - sub_commands = ( - [ - ("build_libpg_query", lambda self: True), - *setuptools_build.build.sub_commands, - ("build_metadata", lambda self: True), - ("build_parsers", lambda self: True), - ("build_postgres", lambda self: True), - ("build_cli", lambda self: True), - ("build_ui", lambda self: True), - ] - ) + sub_commands = setuptools_build.build.sub_commands if _PYTHON_ONLY else [ + ("build_libpg_query", lambda self: True), + *setuptools_build.build.sub_commands, + ("build_metadata", lambda self: True), + ("build_parsers", lambda self: True), + ("build_postgres", lambda self: True), + ("build_cli", lambda self: True), + ("build_ui", lambda self: True), + ] class build_metadata(setuptools.Command):