feat: expose storage options in LanceDB (lancedb#1204)

Exposes `storage_options` in LanceDB. This is provided for Python async,
Node `lancedb`, and Node `vectordb` (and Rust of course). Python
synchronous is omitted because it's not compatible with the PyArrow
filesystems we use there currently. In the future, we will move the sync
API to wrap the async one, and then it will get support for
`storage_options`.

1. Fixes lancedb#1168
2. Closes lancedb#1165
3. Closes lancedb#1082
4. Closes lancedb#439
5. Closes lancedb#897
6. Closes lancedb#642
7. Closes lancedb#281
8. Closes lancedb#114
9. Closes lancedb#990
10. Deprecating `awsCredentials` and `awsRegion`. Users are encouraged
to use `storageOptions` instead.

Author: wjones127

.github/workflows/node.yml

@@ -107,6 +107,7 @@ jobs:
       AWS_ENDPOINT: http://localhost:4566
       # this one is for dynamodb
       DYNAMODB_ENDPOINT: http://localhost:4566
+      ALLOW_HTTP: true
     steps:
     - uses: actions/checkout@v4
       with:

.github/workflows/nodejs.yml

@@ -85,7 +85,12 @@ jobs:
       run: |
         npm ci
         npm run build
+    - name: Setup localstack
+      working-directory: .
+      run: docker compose up --detach --wait
     - name: Test
+      env:
+        S3_TEST: "1"
       run: npm run test
   macos:
     timeout-minutes: 30

.github/workflows/python.yml

@@ -99,6 +99,8 @@ jobs:
           workspaces: python
       - uses: ./.github/workflows/build_linux_wheel
       - uses: ./.github/workflows/run_tests
+        with:
+          integration: true
       # Make sure wheels are not included in the Rust cache
       - name: Delete wheels
         run: rm -rf target/wheels
@@ -190,4 +192,4 @@ jobs:
           pip install -e .[tests]
           pip install tantivy
       - name: Run tests
-        run: pytest -m "not slow" -x -v --durations=30 python/tests
+        run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests

.github/workflows/run_tests/action.yml

@@ -5,13 +5,27 @@ inputs:
   python-minor-version:
     required: true
     description: "8 9 10 11 12"
+  integration:
+    required: false
+    description: "Run integration tests"
+    default: "false"
 runs:
   using: "composite"
   steps:
     - name: Install lancedb
       shell: bash
       run: |
         pip3 install $(ls target/wheels/lancedb-*.whl)[tests,dev]
-    - name: pytest
+    - name: Setup localstack for integration tests
+      if: ${{ inputs.integration == 'true' }}
       shell: bash
+      working-directory: .
+      run: docker compose up --detach --wait
+    - name: pytest (with integration)
+      shell: bash
+      if: ${{ inputs.integration == 'true' }}
       run: pytest -m "not slow" -x -v --durations=30 python/python/tests
+    - name: pytest (no integration tests)
+      shell: bash
+      if: ${{ inputs.integration != 'true' }}
+      run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/python/tests

.github/workflows/rust.yml

@@ -76,6 +76,9 @@ jobs:
           sudo apt install -y protobuf-compiler libssl-dev
     - name: Build
       run: cargo build --all-features
+    - name: Start S3 integration test environment
+      working-directory: .
+      run: docker compose up --detach --wait
     - name: Run tests
       run: cargo test --all-features
     - name: Run examples
@@ -105,7 +108,8 @@ jobs:
       - name: Build
         run: cargo build --all-features
       - name: Run tests
-        run: cargo test --all-features
+        # Run with everything except the integration tests.
+        run: cargo test --features remote,fp16kernels
   windows:
     runs-on: windows-2022
     steps:

Cargo.toml

@@ -14,10 +14,10 @@ keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
 
 [workspace.dependencies]
-lance = { "version" = "=0.10.9", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.10.9" }
-lance-linalg = { "version" = "=0.10.9" }
-lance-testing = { "version" = "=0.10.9" }
+lance = { "version" = "=0.10.10", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.10.10" }
+lance-linalg = { "version" = "=0.10.10" }
+lance-testing = { "version" = "=0.10.10" }
 # Note that this one does not include pyarrow
 arrow = { version = "50.0", optional = false }
 arrow-array = "50.0"

docker-compose.yml

@@ -1,18 +1,18 @@
 version: "3.9"
 services:
   localstack:
-    image: localstack/localstack:0.14
+    image: localstack/localstack:3.3
     ports:
       - 4566:4566
     environment:
-      - SERVICES=s3,dynamodb
+      - SERVICES=s3,dynamodb,kms
       - DEBUG=1
       - LS_LOG=trace
       - DOCKER_HOST=unix:///var/run/docker.sock
       - AWS_ACCESS_KEY_ID=ACCESSKEY
       - AWS_SECRET_ACCESS_KEY=SECRETKEY
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:4566/health" ]
+      test: [ "CMD", "curl", "-s", "http://localhost:4566/_localstack/health" ]
       interval: 5s
       retries: 3
       start_period: 10s