Skip to content

Commit

Permalink
Merge pull request #615 from sfu-db/py312
Browse files Browse the repository at this point in the history 
support python 3.12

A pandas destination dispatcher is added to release the GIL before parallel data fetching. It may need some refractor to be able to reuse the code in original connectorx::dispatcher
  • Loading branch information
@wangxiaoying
wangxiaoying authored May 9, 2024
2 parents e75197a + d06bd52 commit eeb8042
Show file tree
Hide file tree
Showing 51 changed files with 2,877 additions and 2,218 deletions.
247 changes: 106 additions & 141 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,131 +10,109 @@ on:
- main

jobs:
# rust:
# runs-on: ubuntu-latest
# container: ubuntu:20.04
# services:
# # Label used to access the service container
# postgres:
# # Docker Hub image
# image: postgres
# env:
# POSTGRES_PASSWORD: postgres
# # Set health checks to wait until postgres has started
# options: >-
# --health-cmd pg_isready
# --health-interval 10s
# --health-timeout 5s
# --health-retries 5
# # mysql
# mysql:
# image: ghcr.io/wangxiaoying/mysql:latest
# env:
# MYSQL_DATABASE: mysql
# MYSQL_ROOT_PASSWORD: mysql
# LANG: C.UTF-8
# ports:
# - 3306:3306
# options: >-
# --health-cmd "mysqladmin ping"
# --health-interval 10s
# --health-timeout 10s
# --health-retries 5
# mssql:
# image: mcr.microsoft.com/mssql/server:2019-latest
# env:
# ACCEPT_EULA: y
# SA_PASSWORD: mssql!Password
# ports:
# - 1433:1433
# options: >-
# --health-cmd "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P \"$SA_PASSWORD\" -Q 'SELECT 1' || exit 1"
# --health-interval 10s
# --health-timeout 5s
# --health-retries 20
# steps:
# - uses: actions/checkout@v4
rust:
runs-on: ubuntu-latest
container: ubuntu:22.04
services:
postgres:
image: postgres
env:
POSTGRES_PASSWORD: postgres
# Set health checks to wait until postgres has started
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
mysql:
image: ghcr.io/wangxiaoying/mysql:latest
env:
MYSQL_DATABASE: mysql
MYSQL_ROOT_PASSWORD: mysql
LANG: C.UTF-8
ports:
- 3306:3306
options: >-
--health-cmd "mysqladmin ping"
--health-interval 10s
--health-timeout 10s
--health-retries 5
# - name: Install tools
# run: |
# apt-get update
# apt-get install -y curl postgresql-client build-essential pkg-config libssl-dev git sqlite3 libsqlite3-dev mysql-client python3 python3-pip libicu66 libkrb5-dev libclang-dev
# pip3 install mssql-cli
# pip3 install cli-helpers==2.2.0
# ln -s /usr/bin/python3 /usr/bin/python
# echo "Cache Version ${{ secrets.CACHE_VERSION }}"
# env:
# DEBIAN_FRONTEND: noninteractive
steps:
- uses: actions/checkout@v4

# - name: Install Rust
# uses: actions-rs/toolchain@v1
# with:
# toolchain: stable
# components: rustfmt
# default: true
- name: Install tools
run: |
apt-get update
apt-get install -y curl postgresql-client build-essential pkg-config libssl-dev git sqlite3 libsqlite3-dev mysql-client python3 python3-pip libkrb5-dev libclang-dev
echo "Cache Version ${{ secrets.CACHE_VERSION }}"
env:
DEBIAN_FRONTEND: noninteractive

# - uses: actions/cache@v4
# with:
# path: |
# ~/.cargo/bin/
# ~/.cargo/registry/index/
# ~/.cargo/registry/cache/
# ~/.cargo/git/db/
# target/
# key: ${{ runner.os }}-cargo-${{ secrets.CACHE_VERSION }}-${{ hashFiles('**/Cargo.lock') }}
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: stable
components: rustfmt
default: true

# - name: Install other dependent tools
# run: |
# if [ ! -f "$HOME/.cargo/bin/just" ]; then curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to ~/.cargo/bin; fi
- uses: actions/cache@v4
with:
path: |
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: ${{ runner.os }}-cargo-${{ secrets.CACHE_VERSION }}-${{ hashFiles('**/Cargo.lock') }}

# - name: Seed the database
# run: just seed-db
# env:
# POSTGRES_URL: "postgresql://postgres:postgres@postgres:5432/postgres"
# SQLITE_URL: "sqlite:///tmp/test.db"
# MYSQL_HOST: mysql
# MYSQL_PORT: 3306
# MYSQL_DB: mysql
# MYSQL_USER: root
# MYSQL_PASSWORD: mysql
# MSSQL_HOST: mssql
# MSSQL_PORT: 1433
# MSSQL_DB: tempdb
# MSSQL_USER: sa
# MSSQL_PASSWORD: mssql!Password
- uses: extractions/setup-just@v2

- name: Seed the database
run: just seed-db
env:
POSTGRES_URL: "postgresql://postgres:postgres@postgres:5432/postgres"
SQLITE_URL: "sqlite:///tmp/test.db"
MYSQL_HOST: mysql
MYSQL_PORT: 3306
MYSQL_DB: mysql
MYSQL_USER: root
MYSQL_PASSWORD: mysql

# - name: Is the code formatted?
# uses: actions-rs/cargo@v1
# with:
# command: fmt
# args: --all -- --check -q
- name: Is the code formatted?
uses: actions-rs/cargo@v1
with:
command: fmt
args: --all -- --check -q

# - name: Clippy linting
# uses: actions-rs/cargo@v1
# with:
# command: clippy
# args: --features all
- name: Clippy linting
uses: actions-rs/cargo@v1
with:
command: clippy
args: --features all

# - name: Check each feature gate
# run: just test-feature-gate
# - name: Check each feature gate
# run: just test-feature-gate

# - name: Run tests
# run: cargo clean && just test
# env:
# POSTGRES_URL: "postgresql://postgres:postgres@postgres:5432/postgres"
# SQLITE_URL: "sqlite:///tmp/test.db"
# MYSQL_URL: "mysql://root:mysql@mysql:3306/mysql"
# MSSQL_URL: "mssql://sa:mssql!Password@mssql:1433/tempdb"
- name: Run tests
run: cargo clean && just test-ci
env:
POSTGRES_URL: "postgresql://postgres:postgres@postgres:5432/postgres"
SQLITE_URL: "sqlite:///tmp/test.db"
MYSQL_URL: "mysql://root:mysql@mysql:3306/mysql"

# - name: Test build docs
# uses: actions-rs/cargo@v1
# with:
# command: doc
# args: --no-deps --features all
- name: Test build docs
uses: actions-rs/cargo@v1
with:
command: doc
args: --no-deps --features all

python:
runs-on: ubuntu-latest
container: ubuntu:20.04
container: ubuntu:22.04
strategy:
matrix:
python-version: ["3.12"]
services:
# Label used to access the service container
postgres:
Expand Down Expand Up @@ -162,32 +140,30 @@ jobs:
--health-interval 10s
--health-timeout 10s
--health-retries 5
mssql:
image: mcr.microsoft.com/mssql/server:2019-latest
env:
ACCEPT_EULA: y
SA_PASSWORD: mssql!Password
ports:
- 1433:1433
options: >-
--health-cmd "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P \"$SA_PASSWORD\" -Q 'SELECT 1' || exit 1"
--health-interval 10s
--health-timeout 5s
--health-retries 20
steps:
- uses: actions/checkout@v4

- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}

- name: Setup python env
run: |
echo "$Python3_ROOT_DIR/bin" >> $GITHUB_PATH
echo $GITHUB_PATH
- name: Install tools
run: |
apt-get update
apt-get install -y curl postgresql-client build-essential python3-dev python3-pip pkg-config libssl-dev git sqlite3 libsqlite3-dev mysql-client libmysqlclient-dev python3 python3-pip libicu66 libkrb5-dev libclang-dev
pip3 install mssql-cli
pip3 install cli-helpers==2.2.0
ln -s /usr/bin/python3 /usr/bin/python
apt-get install -y wget curl postgresql-client build-essential pkg-config libssl-dev git sqlite3 libsqlite3-dev mysql-client libmysqlclient-dev libkrb5-dev libclang-dev
env:
DEBIAN_FRONTEND: noninteractive

- uses: Gr1N/setup-poetry@v9

- uses: extractions/setup-just@v2

- name: Install Rust
uses: actions-rs/toolchain@v1
with:
Expand All @@ -208,12 +184,7 @@ jobs:
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: ${{ runner.os }}-cargo-python-${{ secrets.CACHE_VERSION }}-${{ hashFiles('**/Cargo.lock') }}

- name: Install other dependent tools
run: |
pip3 install poetry
if [ ! -f "$HOME/.cargo/bin/just" ]; then curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to ~/.cargo/bin; fi
key: ${{ runner.os }}-cargo-python-${{ matrix.python-version }}-${{ secrets.CACHE_VERSION }}-${{ hashFiles('**/Cargo.lock') }}

- name: Seed the database
run: just seed-db
Expand All @@ -225,11 +196,6 @@ jobs:
MYSQL_DB: mysql
MYSQL_USER: root
MYSQL_PASSWORD: mysql
MSSQL_HOST: mssql
MSSQL_PORT: 1433
MSSQL_DB: tempdb
MSSQL_USER: sa
MSSQL_PASSWORD: mssql!Password

- name: Clippy linting
uses: actions-rs/cargo@v1
Expand All @@ -241,7 +207,7 @@ jobs:
uses: actions/cache@v4
with:
path: ~/.cache/pypoetry/virtualenvs
key: ${{ runner.os }}-venv-${{ secrets.CACHE_VERSION }}-${{ hashFiles('connectorx-python/poetry.lock') }}
key: ${{ runner.os }}-${{ matrix.python-version }}-venv-${{ secrets.CACHE_VERSION }}-${{ hashFiles('connectorx-python/poetry.lock') }}

- name: Install python dependencies
run: just bootstrap-python
Expand All @@ -252,7 +218,6 @@ jobs:
POSTGRES_URL: "postgresql://postgres:postgres@postgres:5432/postgres"
SQLITE_URL: "sqlite:///tmp/test.db"
MYSQL_URL: "mysql://root:mysql@mysql:3306/mysql"
MSSQL_URL: "mssql://sa:mssql!Password@mssql:1433/tempdb"
DB1: "postgresql://postgres:postgres@postgres:5432/postgres"
DB2: "postgresql://postgres:postgres@postgres:5432/postgres"
FED_CONFIG_PATH: ${{ github.workspace }}/.github/config
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/release.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ jobs:
container: quay.io/pypa/manylinux_2_28_x86_64
strategy:
matrix:
python-version: [[38, "3.8"], [39, "3.9"], [310, "3.10"], [311, "3.11"]]
python-version: [[39, "3.9"], [310, "3.10"], [311, "3.11"], [312, "3.12"]]
steps:
- uses: actions/checkout@v4

Expand Down Expand Up @@ -162,7 +162,7 @@ jobs:
strategy:
matrix:
os: ["windows-latest", "macos-latest"]
python-version: ["3.8", "3.9", "3.10", "3.11"]
python-version: ["3.9", "3.10", "3.11", "3.12"]
include:
- os: "macos-latest"
features: "--features integrated-auth-gssapi"
Expand Down Expand Up @@ -226,7 +226,7 @@ jobs:
runs-on: macos-latest
strategy:
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11"]
python-version: ["3.9", "3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v4

Expand Down Expand Up @@ -290,7 +290,7 @@ jobs:
needs: [win-and-mac, linux, apple-arm]
strategy:
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11"]
python-version: ["3.9", "3.10", "3.11", "3.12"]
os: [macos-latest, ubuntu-latest, windows-latest]
steps:
- uses: actions/checkout@v4
Expand Down
5 changes: 4 additions & 1 deletion Justfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ build-cpp-release +ARGS="":
test +ARGS="":
cargo test --features all {{ARGS}} -- --nocapture

test-ci:
cargo test --features all --test test_postgres --test test_polars --test test_arrow --test test_mysql

test-feature-gate:
cargo c --features src_postgres
cargo c --features src_mysql
Expand Down Expand Up @@ -54,10 +57,10 @@ seed-db:
psql $POSTGRES_URL -f scripts/postgres.sql
sqlite3 ${SQLITE_URL#sqlite://} < scripts/sqlite.sql
mysql --protocol tcp -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DB < scripts/mysql.sql
mssql-cli -S$MSSQL_HOST -U$MSSQL_USER -P$MSSQL_PASSWORD -d$MSSQL_DB -i scripts/mssql.sql

# dbs not included in ci
seed-db-more:
mssql-cli -S$MSSQL_HOST -U$MSSQL_USER -P$MSSQL_PASSWORD -d$MSSQL_DB -i scripts/mssql.sql
mysql --protocol tcp -h$CLICKHOUSE_HOST -P$CLICKHOUSE_PORT -u$CLICKHOUSE_USER -p$CLICKHOUSE_PASSWORD $CLICKHOUSE_DB < scripts/clickhouse.sql
psql $REDSHIFT_URL -f scripts/redshift.sql
ORACLE_URL_SCRIPT=`echo ${ORACLE_URL#oracle://} | sed "s/:/\//"`
Expand Down
Loading

0 comments on commit eeb8042

Please sign in to comment.