Replace threading.Lock with asyncio.Lock when batching to avoid deadlocks
#3562
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Main | |
| on: | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - '**' | |
| paths-ignore: | |
| - docs/** | |
| - README.rst | |
| - LICENSE.md | |
| - publishing.md | |
| pull_request: | |
| env: | |
| WEAVIATE_123: 1.23.16 | |
| WEAVIATE_124: 1.24.21 | |
| WEAVIATE_125: 1.25.8 | |
| WEAVIATE_126: preview-increase-version-number-8b44fe6 | |
| jobs: | |
| lint-and-format: | |
| name: Run Linter and Formatter | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: 'pip' # caching pip dependencies | |
| - run: pip install -r requirements-devel.txt | |
| - name: "Black" | |
| run: black --check weaviate test mock_tests integration | |
| - name: "Flake 8" | |
| run: flake8 weaviate test mock_tests integration | |
| - name: "Check release for pypi" | |
| run: | | |
| python -m build | |
| python -m twine check dist/* | |
| type-checking: | |
| name: Run Type Checking | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| folder: ["weaviate"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.version }} | |
| cache: 'pip' # caching pip dependencies | |
| - run: pip install -r requirements-devel.txt | |
| - name: Run mypy | |
| run: mypy --config-file ./pyproject.toml --warn-unused-ignores --python-version ${{matrix.version}} ${{ matrix.folder }} | |
| - uses: jakebailey/pyright-action@v2 | |
| with: | |
| version: 1.1.347 | |
| unit-tests: | |
| name: Run Unit Tests | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| folder: ["test", "mock_tests"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.version }} | |
| cache: 'pip' # caching pip dependencies | |
| - run: pip install -r requirements-devel.txt | |
| - name: Run unittests | |
| run: pytest --cov -v --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-${{ matrix.folder }}.xml ${{ matrix.folder }} | |
| - name: Archive code coverage results | |
| if: matrix.version == '3.10' && (github.ref_name != 'main') | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-report-${{ matrix.folder }} | |
| path: coverage-${{ matrix.folder }}.xml | |
| integration-tests-embedded: | |
| name: Run Integration Tests Embedded | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| version: ["3.8", "3.9", "3.10", "3.11", "3.12"] | |
| optional_dependencies: [false] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.version }} | |
| cache: 'pip' # caching pip dependencies | |
| - run: | | |
| pip install -r requirements-devel.txt | |
| pip install . | |
| - name: Run integration tests | |
| if: ${{ !github.event.pull_request.head.repo.fork }} | |
| run: pytest -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration-embedded.xml integration_embedded | |
| - name: Archive code coverage results | |
| if: matrix.version == '3.10' && (github.ref_name != 'main') | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-report-integration-embedded | |
| path: coverage-integration-embedded.xml | |
| integration-tests-v3: | |
| name: Run Integration Tests v3 | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| versions: [ | |
| { py: "3.8", weaviate: $WEAVIATE_126}, | |
| { py: "3.9", weaviate: $WEAVIATE_126}, | |
| { py: "3.10", weaviate: $WEAVIATE_126}, | |
| { py: "3.11", weaviate: $WEAVIATE_123}, | |
| { py: "3.11", weaviate: $WEAVIATE_124}, | |
| { py: "3.11", weaviate: $WEAVIATE_125}, | |
| { py: "3.11", weaviate: $WEAVIATE_126}, | |
| { py: "3.12", weaviate: $WEAVIATE_126} | |
| ] | |
| optional_dependencies: [false] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.versions.py }} | |
| cache: 'pip' # caching pip dependencies | |
| - run: | | |
| pip install -r requirements-devel.txt | |
| pip install . | |
| - name: free space | |
| run: sudo rm -rf /usr/local/lib/android | |
| - name: start weaviate | |
| run: /bin/bash ci/start_weaviate.sh ${{ matrix.versions.weaviate }} | |
| - name: Run integration tests with auth secrets | |
| if: ${{ !github.event.pull_request.head.repo.fork }} | |
| env: | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} | |
| OKTA_CLIENT_SECRET: ${{ secrets.OKTA_CLIENT_SECRET }} | |
| WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW }} | |
| OKTA_DUMMY_CI_PW: ${{ secrets.OKTA_DUMMY_CI_PW }} | |
| # OPENAI_APIKEY: ${{ secrets.OPENAI_APIKEY }} disabled until we have a working key | |
| run: pytest -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration-v3.xml integration_v3 | |
| - name: Run integration tests without auth secrets (for forks) | |
| if: ${{ github.event.pull_request.head.repo.fork }} | |
| run: pytest -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration-v3.xml integration_v3 | |
| - name: Archive code coverage results | |
| if: matrix.versions.py == '3.10' && (github.ref_name != 'main') | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-report-integration-v3 | |
| path: coverage-integration-v3.xml | |
| integration-tests: | |
| name: Run Integration Tests | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| versions: [ | |
| { py: "3.8", weaviate: $WEAVIATE_126}, | |
| { py: "3.9", weaviate: $WEAVIATE_126}, | |
| { py: "3.10", weaviate: $WEAVIATE_126}, | |
| { py: "3.11", weaviate: $WEAVIATE_123}, | |
| { py: "3.11", weaviate: $WEAVIATE_124}, | |
| { py: "3.11", weaviate: $WEAVIATE_125}, | |
| { py: "3.11", weaviate: $WEAVIATE_126}, | |
| { py: "3.12", weaviate: $WEAVIATE_126} | |
| ] | |
| optional_dependencies: [false] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| fetch-tags: true | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.versions.py }} | |
| cache: 'pip' # caching pip dependencies | |
| - run: | | |
| pip install -r requirements-devel.txt | |
| pip install . | |
| - name: free space | |
| run: sudo rm -rf /usr/local/lib/android | |
| - name: start weaviate | |
| run: /bin/bash ci/start_weaviate.sh ${{ matrix.versions.weaviate }} | |
| - name: Run integration tests with auth secrets | |
| if: ${{ !github.event.pull_request.head.repo.fork }} | |
| env: | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} | |
| OKTA_CLIENT_SECRET: ${{ secrets.OKTA_CLIENT_SECRET }} | |
| WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW }} | |
| OKTA_DUMMY_CI_PW: ${{ secrets.OKTA_DUMMY_CI_PW }} | |
| # OPENAI_APIKEY: ${{ secrets.OPENAI_APIKEY }} disabled until we have a working key | |
| run: pytest -n auto -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration.xml integration | |
| - name: Run integration tests without auth secrets (for forks) | |
| if: ${{ github.event.pull_request.head.repo.fork }} | |
| run: pytest -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration.xml integration | |
| - name: Archive code coverage results | |
| if: matrix.versions.py == '3.10' && (github.ref_name != 'main') | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-report-integration | |
| path: coverage-integration.xml | |
| journey-tests: | |
| name: Run Journey Tests | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| versions: [ | |
| { py: "3.8", weaviate: $WEAVIATE_125}, | |
| { py: "3.9", weaviate: $WEAVIATE_125}, | |
| { py: "3.10", weaviate: $WEAVIATE_125}, | |
| { py: "3.11", weaviate: $WEAVIATE_125}, | |
| { py: "3.12", weaviate: $WEAVIATE_125} | |
| ] | |
| optional_dependencies: [false] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: 'pip' # caching pip dependencies | |
| - run: pip install -r requirements-devel.txt | |
| - run: pip install . | |
| - name: free space | |
| run: sudo rm -rf /usr/local/lib/android | |
| - name: start weaviate | |
| run: /bin/bash ci/start_weaviate.sh ${{ matrix.versions.weaviate }} | |
| - name: Run journey tests | |
| run: pytest journey_tests | |
| Codecov: | |
| needs: [Unit-Tests, Integration-Tests, Integration-Tests-v3] | |
| runs-on: ubuntu-latest | |
| if: github.ref_name != 'main' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download coverage artifacts mock | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: coverage-report-mock_tests | |
| - name: Download coverage artifacts unit | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: coverage-report-test | |
| - name: Download coverage integration | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: coverage-report-integration | |
| - name: Download coverage integration v3 | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: coverage-report-integration-v3 | |
| - name: Download coverage integration embedded | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: coverage-report-integration-embedded | |
| - name: Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| fail_ci_if_error: true | |
| files: ./coverage-integration.xml, ./coverage-integration-v3.xml, ./coverage-integration-embedded.xml, ./coverage-test.xml, ./coverage-mock_tests.xml | |
| verbose: true | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| build-package: | |
| name: Build package | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: 'pip' # caching pip dependencies | |
| - name: Install dependencies | |
| run: pip install -r requirements-devel.txt | |
| - name: Build a binary wheel | |
| run: python -m build | |
| - name: Create Wheel Artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: "dist/*.whl" | |
| name: weaviate-python-client-wheel | |
| retention-days: 30 | |
| test-package: | |
| needs: [build-package] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| server: [ | |
| $WEAVIATE_123, | |
| $WEAVIATE_124, | |
| $WEAVIATE_125, | |
| $WEAVIATE_126 | |
| ] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Download build artifact to append to release | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: weaviate-python-client-wheel | |
| - run: | | |
| pip install weaviate_client-*.whl | |
| pip install -r requirements-devel.txt # install test dependencies | |
| - name: free space | |
| run: sudo rm -rf /usr/local/lib/android | |
| - run: rm -r weaviate | |
| - name: start weaviate | |
| run: /bin/bash ci/start_weaviate.sh ${{ matrix.server }} | |
| - name: Run integration tests with auth secrets | |
| if: ${{ !github.event.pull_request.head.repo.fork }} | |
| env: | |
| AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }} | |
| OKTA_CLIENT_SECRET: ${{ secrets.OKTA_CLIENT_SECRET }} | |
| WCS_DUMMY_CI_PW: ${{ secrets.WCS_DUMMY_CI_PW }} | |
| OKTA_DUMMY_CI_PW: ${{ secrets.OKTA_DUMMY_CI_PW }} | |
| run: pytest -v -n auto integration | |
| - name: Run integration tests without auth secrets (for forks) | |
| if: ${{ github.event.pull_request.head.repo.fork }} | |
| run: pytest -v -n auto integration | |
| build-and-publish: | |
| name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI | |
| needs: [integration-tests, unit-tests, lint-and-format, type-checking, test-package] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| cache: 'pip' # caching pip dependencies | |
| - name: Install dependencies | |
| run: pip install -r requirements-devel.txt | |
| - name: Build a binary wheel | |
| run: python -m build | |
| - name: Publish distribution 📦 to PyPI on new tags | |
| if: startsWith(github.ref, 'refs/tags') | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| verbose: true | |
| password: ${{ secrets.PYPI_API_TOKEN }} | |
| gh-release: | |
| name: Create a GitHub Release on new tags | |
| if: startsWith(github.ref, 'refs/tags') | |
| runs-on: ubuntu-latest | |
| needs: [build-and-publish] | |
| steps: | |
| - name: Download build artifact to append to release | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: weaviate-python-client-wheel | |
| path: dist | |
| - name: Release | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| generate_release_notes: true | |
| draft: true | |
| files: dist/*.whl |