Merge branch 'main' into revert-4479-bld-single-rust-dockerfile

Author: codetheweb

.github/actions/docker/action.yaml

@@ -20,10 +20,6 @@ runs:
     # https://github.com/docker/setup-qemu-action - for multiplatform builds
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v2
-    # https://github.com/docker/setup-buildx-action - for multiplatform builds
-    - name: Set up Docker Buildx
-      id: buildx
-      uses: docker/setup-buildx-action@v2
     - name: Log in to the Github Container registry
       uses: docker/[email protected]
       with:

.github/actions/python/action.yaml

@@ -8,27 +8,26 @@ inputs:
 runs:
   using: "composite"
   steps:
-
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
+      with:
+        enable-cache: true
     - name: Set up Python 3.9 for protos
       uses: useblacksmith/setup-python@v6
       # NOTE(hammadb) Blacksmith does not support windows, so we use the official action for windows and
       # skip it on other platforms
       if: runner.os != 'Windows'
       with:
         python-version: "3.9"
-        cache: "pip"
-        cache-dependency-path: "requirements*.txt"
     - name: Set up Python 3.9 for protos (Windows)
       if: runner.os == 'Windows'
       uses: actions/setup-python@v5
       with:
         python-version: "3.9"
-        cache: "pip"
-        cache-dependency-path: "requirements*.txt"
 
     - name: Install proto dependencies
       run: |
-        python -m pip install grpcio==1.58.0 grpcio-tools==1.58.0
+        uv pip install --system grpcio==1.58.0 grpcio-tools==1.58.0
       shell: bash
     - name: Generate Proto Files
       if: runner.os != 'Windows'
@@ -40,30 +39,24 @@ runs:
       shell: cmd
     - name: Uninstall proto dependencies
       run: |
-        python -m pip uninstall -y grpcio grpcio-tools
+        uv pip uninstall --system grpcio grpcio-tools
       shell: bash
     - name: Set up Python ${{ inputs.python-version }}
       uses: useblacksmith/setup-python@v6
       with:
         python-version: ${{ inputs.python-version }}
-        cache: "pip"
-        cache-dependency-path: "requirements*.txt"
     - name: Set up Python ${{ inputs.python-version }} (Windows)
       if: runner.os == 'Windows'
       uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python-version }}
-        cache: "pip"
-        cache-dependency-path: "requirements*.txt"
     - name: Install dependencies
       run: |
-        python -m pip install -r requirements.txt && python -m pip install -r requirements_dev.txt
+        uv pip install --system -r requirements.txt -r requirements_dev.txt
       shell: bash
     - name: Install protobuf compiler (protoc) - Linux
       if: runner.os != 'Windows'
       run: |
-        sudo apt-get update
-        sudo apt-get install -y wget unzip
         wget https://github.com/protocolbuffers/protobuf/releases/download/v28.2/protoc-28.2-linux-x86_64.zip
         sudo unzip protoc-28.2-linux-x86_64.zip -d /usr/local/
         sudo rm protoc-28.2-linux-x86_64.zip

.github/actions/rust/action.yaml

@@ -7,40 +7,31 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Setup Rust
+    - name: Cache Rust toolchain
+      id: cache-rustup
+      uses: useblacksmith/cache@v5
+      if: ${{ runner.os != 'windows' }}
+      with:
+        path: ~/.rustup
+        key: toolchain-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('rust-toolchain.toml') }}
+    - name: Install Rust toolchain
+      if: ${{ steps.cache-rustup.outputs.cache-hit != 'true' }}
+      uses: actions-rust-lang/setup-rust-toolchain@v1
+      with:
+        cache: false # we use Blacksmith's cache package below
+    - name: Set channel in rust-toolchain.toml as default
       shell: bash
-      # (reads from rust-toolchain.toml)
-      run: |
-        rustup --version
-        rustup toolchain install 1.81.0
-        rustup default 1.81.0
-    # Needed for sccache to work on Windows
-    - name: Set default toolchain to rust-toolchain.toml on Windows
-      if: runner.os == 'Windows'
-      shell: pwsh
       run: |
-        # Read the file content as a single string
-        $toolchainToml = Get-Content .\rust-toolchain.toml -Raw
-
-        # Use regex to match the line 'channel = "<something>"'
-        if ($toolchainToml -match 'channel\s*=\s*"([^"]+)"') {
-          $channel = $matches[1]
-          Write-Host "Setting Rust default channel to: $channel"
-          rustup default $channel
-        } else {
-          Write-Error "Could not parse 'channel' from rust-toolchain.toml"
-          exit 1
-        }
+        rustup default $(grep -m1 '^channel' rust-toolchain.toml | cut -d'"' -f2)
     - name: Install Protoc
       uses: arduino/setup-protoc@v3
       with:
         repo-token: ${{ inputs.github-token }}
-    - name: Use sccache
-      uses: mozilla-actions/[email protected]
-    - name: Enable sccache
-      shell: bash
-      run: |
-        echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
-        echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+    - name: Set up cache
+      uses: useblacksmith/rust-cache@v3
+      if: ${{ runner.os != 'windows' }}
+    - name: Set up cache (Windows)
+      uses: Swatinem/rust-cache@v2
+      if: ${{ runner.os == 'windows' }}
     - name: Setup Nextest
       uses: taiki-e/install-action@nextest

.github/actions/tilt/action.yaml

@@ -6,18 +6,34 @@ runs:
     - name: Install Tilt
       shell: bash
       run: |
-        TILT_VERSION="0.33.3"
+        TILT_VERSION="0.34.2"
         curl -fsSL https://github.com/tilt-dev/tilt/releases/download/v$TILT_VERSION/tilt.$TILT_VERSION.linux.x86_64.tar.gz | \
           tar -xzv -C /usr/local/bin tilt
-    - name: Install ctlptlc
-      shell: bash
-      run: |
-        CTLPTL_VERSION="0.8.20"
-        curl -fsSL https://github.com/tilt-dev/ctlptl/releases/download/v$CTLPTL_VERSION/ctlptl.$CTLPTL_VERSION.linux.x86_64.tar.gz | \
-          tar -xzv -C /usr/local/bin ctlptl
     - name: Set up kind
+      uses: Wandalen/[email protected]
+      with:
+        action: helm/kind-action@v1
+        attempt_limit: 3
+        attempt_delay: 2000
+        with: |
+          registry: true
+          registry_name: my-registry
+          registry_port: 5001
+          registry_enable_delete: true
+    - name: Add blacksmith registry mirror
       shell: bash
-      run: ctlptl create cluster kind --registry=ctlptl-registry
+      run: |
+        REGISTRY_DIR="/etc/containerd/certs.d/docker.io"
+        for node in $(kind get nodes -n chart-testing); do
+          docker exec "${node}" mkdir -p "${REGISTRY_DIR}"
+          cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
+        server = "http://192.168.127.1:5000"
+        [host."http://192.168.127.1:5000"]
+          capabilities = ["pull", "resolve"]
+        EOF
+          echo "Added registry mirror to ${node}:"
+          docker exec "${node}" cat "${REGISTRY_DIR}/hosts.toml"
+        done
     - name: Start Tilt
       shell: bash
       run: tilt ci

.github/workflows/_build_release_pypi.yml

@@ -117,7 +117,6 @@ jobs:
         with:
           target: ${{ matrix.platform.target }}
           args: ${{ matrix.platform.os == 'linux' && '--zig' || '' }} --release --out dist
-          sccache: "true"
           container: "off"
 
       - name: Upload wheels

.github/workflows/_python-tests.yml

@@ -45,18 +45,9 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           command: build
-          sccache: true
       - name: Install built wheel
         shell: bash
         run: pip install --no-index --find-links target/wheels/ chromadb
-      - name: Configure pytest to upload results to Datadog
-        uses: datadog/test-visibility-github-action@v2
-        # This action currently fails on Windows (https://github.com/DataDog/test-visibility-github-action/issues/36)
-        if: ${{ !contains(matrix.platform, 'windows') }}
-        with:
-          languages: python
-          api_key: ${{ secrets.DD_API_KEY }}
-          site: ${{ vars.DD_SITE }}
       - name: Test
         run: python -m pytest ${{ matrix.test-globs }} ${{ matrix.parallelized && '-n auto --dist worksteal' || '' }} -v --color=yes --durations 10
         shell: bash
@@ -93,14 +84,6 @@ jobs:
       uses: ./.github/actions/rust
       with:
           github-token: ${{ github.token }}
-    - name: Configure pytest to upload results to Datadog
-      uses: datadog/test-visibility-github-action@v2
-      # This action currently fails on Windows (https://github.com/DataDog/test-visibility-github-action/issues/36)
-      if: ${{ !contains(matrix.platform, 'windows') }}
-      with:
-        languages: python
-        api_key: ${{ secrets.DD_API_KEY }}
-        site: ${{ vars.DD_SITE }}
     - name: Rust Integration Test
       run: bin/rust-integration-test.sh ${{ matrix.test-globs }}
       shell: bash
@@ -129,14 +112,6 @@ jobs:
         uses: ./.github/actions/rust
         with:
           github-token: ${{ github.token }}
-      - name: Configure pytest to upload results to Datadog
-        uses: datadog/test-visibility-github-action@v2
-        # This action currently fails on Windows (https://github.com/DataDog/test-visibility-github-action/issues/36)
-        if: ${{ !contains(matrix.platform, 'windows') }}
-        with:
-          languages: python
-          api_key: ${{ secrets.DD_API_KEY }}
-          site: ${{ vars.DD_SITE }}
       - name: Test
         run: bin/rust-integration-test.sh ${{ matrix.test-globs }}
         shell: bash
@@ -161,7 +136,8 @@ jobs:
                     "chromadb/test/property/test_embeddings.py",
                     "chromadb/test/property/test_collections_with_database_tenant.py",
                     "chromadb/test/property/test_collections_with_database_tenant_overwrite.py",
-                    "chromadb/test/distributed/test_sanity.py"]
+                    "chromadb/test/distributed/test_sanity.py",
+                    "chromadb/test/distributed/test_log_failover.py"]
     runs-on: ${{ matrix.platform }}
     # OIDC token auth for AWS
     permissions:
@@ -172,16 +148,8 @@ jobs:
       - uses: ./.github/actions/python
         with:
           python-version: ${{ matrix.python }}
-      # TODO(adityamaru): Add Datadog test visibility when running in Chroma's repo.
-      # - name: Configure pytest to upload results to Datadog
-      #   uses: datadog/test-visibility-github-action@v2
-      #   with:
-      #     languages: python
-      #     api_key: ${{ secrets.DD_API_KEY }}
-      #     site: ${{ vars.DD_SITE }}
-      - uses: useblacksmith/[email protected]
-        with:
-          setup-only: true
+      - name: Set up Docker
+        uses: ./.github/actions/docker
       - uses: ./.github/actions/tilt
       - name: Test
         run: bin/cluster-test.sh bash -c 'python -m pytest "${{ matrix.test-globs }}"' --durations 10
@@ -233,18 +201,9 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           command: build
-          sccache: true
       - name: Install built wheel
         shell: bash
         run: pip install --no-index --find-links target/wheels/ chromadb
-      - name: Configure pytest to upload results to Datadog
-        uses: datadog/test-visibility-github-action@v2
-        # This action currently fails on Windows (https://github.com/DataDog/test-visibility-github-action/issues/36)
-        if: ${{ !contains(matrix.platform, 'windows') }}
-        with:
-          languages: python
-          api_key: ${{ secrets.DD_API_KEY }}
-          site: ${{ vars.DD_SITE }}
       - name: Test
         run: python -m pytest ${{ matrix.test-globs }} --durations 10
         shell: bash
@@ -278,18 +237,9 @@ jobs:
       uses: PyO3/maturin-action@v1
       with:
         command: build
-        sccache: true
     - name: Install built wheel
       shell: bash
       run: pip install --no-index --find-links target/wheels/ chromadb
-    - name: Configure pytest to upload results to Datadog
-      uses: datadog/test-visibility-github-action@v2
-      # This action currently fails on Windows (https://github.com/DataDog/test-visibility-github-action/issues/36)
-      if: ${{ !contains(matrix.platform, 'windows') }}
-      with:
-        languages: python
-        api_key: ${{ secrets.DD_API_KEY }}
-        site: ${{ vars.DD_SITE }}
     - name: Integration Test
       run: python -m pytest ${{ matrix.test-globs }}
       shell: bash

.github/workflows/_rust-tests.yml

@@ -23,15 +23,6 @@ jobs:
         run: cargo build --bin chroma
       - name: Test
         run: cargo nextest run --no-capture --profile ci
-      # TODO(adityamaru): Add Datadog test visibility when running in Chroma's repo.
-      # - name: Upload test results
-      #   uses: datadog/junit-upload-github-action@v1
-      #   with:
-      #       api_key: ${{ secrets.DD_API_KEY }}
-      #       site: ${{ vars.DD_SITE }}
-      #       service: chroma
-      #       files: target/nextest/ci/junit.xml
-      #       logs: true
   test-long:
     runs-on: blacksmith-4vcpu-ubuntu-2204
     env:
@@ -48,15 +39,6 @@ jobs:
         run: cargo build --bin chroma
       - name: Test
         run: cargo nextest run --no-capture --profile ci_long_running
-      # TODO(adityamaru): Add Datadog test visibility when running in Chroma's repo.
-      # - name: Upload test results
-      #   uses: datadog/junit-upload-github-action@v1
-      #   with:
-      #       api_key: ${{ secrets.DD_API_KEY }}
-      #       site: ${{ vars.DD_SITE }}
-      #       service: chroma
-      #       files: target/nextest/ci/junit.xml
-      #       logs: true
 
   test-integration:
     strategy:
@@ -85,15 +67,6 @@ jobs:
         run: cargo build --bin chroma
       - name: Run tests
         run: cargo nextest run --profile ci_k8s_integration
-      # TODO(adityamaru): Add Datadog test visibility when running in Chroma's repo.
-      # - name: Upload test results
-      #   uses: datadog/junit-upload-github-action@v1
-      #   with:
-      #       api_key: ${{ secrets.DD_API_KEY }}
-      #       site: ${{ vars.DD_SITE }}
-      #       service: chroma
-      #       files: target/nextest/ci/junit.xml
-      #       logs: true
       - name: Save service logs to artifact
         if: always()
         uses: ./.github/actions/export-tilt-logs

.github/workflows/release-chromadb.yml

@@ -220,7 +220,7 @@ jobs:
                 // - This workflow finishes for A and deploys A to Chroma Cloud
                 // Chroma Cloud is now running A, but the last commit was B.
                 "environment": "staging",
-                "planes": "data,control"
+                "planes": "control,data"
               }
             })
 

.github/workflows/trigger-deploy.yaml

@@ -5,8 +5,9 @@ on:
   push:
     branches:
       # main is not included here because it is handled by the release-chromadb.yaml workflow
+      # production deploys (release branches) are also not here because they are currently
+      # meant to be handled manually.
       - rc/**-**-**
-      - release/**-**-**
 
 jobs:
   deploy:
@@ -25,7 +26,7 @@ jobs:
               ref: 'main',
               inputs: {
                 'planes': 'control,data',
-                 environment: '${{ contains(github.ref, 'release/') && 'production' || 'staging' }}',
+                 environment: 'staging',
                 'ignore-lock': true,
                 'oss-ref': '${{ github.ref_name }}',
                 'hosted-ref': '${{ github.ref_name }}'