feat(ibis): Support pyspark sql data source #502
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ibis CI | |
| on: | |
| pull_request: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.number }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| working-directory: ibis-server | |
| jobs: | |
| ci: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Ruff check | |
| uses: chartboost/ruff-action@v1 | |
| with: | |
| src: './ibis-server' | |
| args: 'format --check' | |
| - uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '21' | |
| cache: 'maven' | |
| - name: Start Wren JAVA engine | |
| working-directory: . | |
| run: | | |
| mkdir etc | |
| echo "node.environment=production" >> etc/config.properties | |
| echo "wren.directory=./etc/mdl" >> etc/config.properties | |
| echo "wren.experimental-enable-dynamic-fields=true" >> etc/config.properties | |
| ./mvnw clean install -B -DskipTests -P exec-jar | |
| java -Dconfig=etc/config.properties \ | |
| --add-opens=java.base/java.nio=ALL-UNNAMED \ | |
| -jar ./wren-server/target/wren-server-*-executable.jar & | |
| - name: Install poetry | |
| run: pipx install poetry | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version-file: ./ibis-server/pyproject.toml | |
| cache: 'poetry' | |
| - uses: extractions/setup-just@v2 | |
| - name: Cache Cargo | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| wren-core-py/target/ | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('wren-core-py/Cargo.lock') }} | |
| - name: Install MS ODBC SQL driver | |
| run: | | |
| sudo apt-get update | |
| curl https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc | |
| curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list | |
| sudo apt-get update | |
| sudo ACCEPT_EULA=Y apt-get -y install unixodbc-dev msodbcsql18 | |
| - name: Install dependencies | |
| run: | | |
| just install | |
| just install-core | |
| just update-core | |
| - name: Run tests | |
| env: | |
| WREN_ENGINE_ENDPOINT: http://localhost:8080 | |
| run: poetry run pytest -m "not bigquery and not snowflake and not canner" | |
| - name: Test bigquery if need | |
| if: contains(github.event.pull_request.labels.*.name, 'bigquery') | |
| env: | |
| WREN_ENGINE_ENDPOINT: http://localhost:8080 | |
| TEST_BIG_QUERY_PROJECT_ID: ${{ secrets.TEST_BIG_QUERY_PROJECT_ID }} | |
| TEST_BIG_QUERY_CREDENTIALS_BASE64_JSON: ${{ secrets.TEST_BIG_QUERY_CREDENTIALS_BASE64_JSON }} | |
| run: just test bigquery | |
| - name: Test snowflake if need | |
| if: contains(github.event.pull_request.labels.*.name, 'snowflake') | |
| env: | |
| WREN_ENGINE_ENDPOINT: http://localhost:8080 | |
| SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} | |
| SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} | |
| SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} | |
| run: just test snowflake |