diff --git a/.github/workflows/pgrx.yml b/.github/workflows/pgrx.yml index 212de57..9bd8489 100644 --- a/.github/workflows/pgrx.yml +++ b/.github/workflows/pgrx.yml @@ -25,7 +25,7 @@ permissions: jobs: build: - runs-on: ubicloud-standard-8 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - name: Set up QEMU diff --git a/.github/workflows/psql.yml b/.github/workflows/psql.yml index a631189..42c108c 100644 --- a/.github/workflows/psql.yml +++ b/.github/workflows/psql.yml @@ -41,14 +41,22 @@ jobs: strategy: matrix: version: ["14", "15", "16", "17"] - runner: ["ubicloud-standard-4", "ubicloud-standard-4-arm"] + runner: ["ubuntu-22.04", "ubuntu-22.04-arm"] env: PGRX_IMAGE: "ghcr.io/tensorchord/vectorchord-pgrx:0.12.9-nightly-2024-12-25" SQLLOGICTEST: "0.25.0" - ARCH: ${{ matrix.runner == 'ubicloud-standard-4' && 'x86_64' || 'aarch64' }} - PLATFORM: ${{ matrix.runner == 'ubicloud-standard-4' && 'amd64' || 'arm64' }} + ARCH: ${{ matrix.runner == 'ubuntu-22.04' && 'x86_64' || 'aarch64' }} + PLATFORM: ${{ matrix.runner == 'ubuntu-22.04' && 'amd64' || 'arm64' }} steps: + - name: Set up Environment + run: | + sudo apt-get remove -y '^postgres.*' '^libpq.*' '^clang.*' '^llvm.*' '^libclang.*' '^libllvm.*' '^mono-llvm.*' + sudo apt-get purge -y '^postgres.*' '^libpq.*' '^clang.*' '^llvm.*' '^libclang.*' '^libllvm.*' '^mono-llvm.*' + + sudo apt-get install -y postgresql-common + sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + sudo apt-get install -y postgresql-client-17 - uses: actions/checkout@v4 - name: Configure sccache uses: actions/github-script@v7 @@ -67,14 +75,14 @@ jobs: sudo chmod -R 777 . - name: Build - env: - SEMVER: "0.0.0" - VERSION: ${{ matrix.version }} run: | - docker run --rm -v .:/workspace $CACHE_ENVS $PGRX_IMAGE cargo build --lib --features pg${{ matrix.version }} --release - docker run --rm -v .:/workspace $CACHE_ENVS $PGRX_IMAGE ./tools/schema.sh --features pg${{ matrix.version }} --release - ./tools/package.sh - docker build -t vchord:pg${{ matrix.version }} --build-arg PG_VERSION=${{ matrix.version }} -f ./docker/Dockerfile . + docker run --rm -v .:/workspace $CACHE_ENVS \ + -e SEMVER=0.0.0 \ + -e VERSION=${{ matrix.version }} \ + -e ARCH=$ARCH \ + -e PLATFORM=$PLATFORM \ + $PGRX_IMAGE ./tools/package.sh + docker build -t vchord:pg${{ matrix.version }} --build-arg PG_VERSION=${{ matrix.version }} --build-arg SEMVER=0.0.0 -f ./docker/Dockerfile . - name: Setup SQL Logic Test run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9608824..1662bbd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,14 +38,22 @@ jobs: strategy: matrix: version: ["14", "15", "16", "17"] - runner: ["ubicloud-standard-4", "ubicloud-standard-4-arm"] + runner: ["ubuntu-22.04", "ubuntu-22.04-arm"] env: PGRX_IMAGE: "ghcr.io/tensorchord/vectorchord-pgrx:0.12.9-nightly-2024-12-25" SEMVER: ${{ needs.semver.outputs.SEMVER }} - ARCH: ${{ matrix.runner == 'ubicloud-standard-4' && 'x86_64' || 'aarch64' }} - PLATFORM: ${{ matrix.runner == 'ubicloud-standard-4' && 'amd64' || 'arm64' }} + ARCH: ${{ matrix.runner == 'ubuntu-22.04' && 'x86_64' || 'aarch64' }} + PLATFORM: ${{ matrix.runner == 'ubuntu-22.04' && 'amd64' || 'arm64' }} steps: + - name: Set up Environment + run: | + sudo apt-get remove -y '^postgres.*' '^libpq.*' '^clang.*' '^llvm.*' '^libclang.*' '^libllvm.*' '^mono-llvm.*' + sudo apt-get purge -y '^postgres.*' '^libpq.*' '^clang.*' '^llvm.*' '^libclang.*' '^libllvm.*' '^mono-llvm.*' + + sudo apt-get install -y postgresql-common + sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + sudo apt-get install -y postgresql-client-17 - uses: actions/checkout@v4 - name: Configure sccache uses: actions/github-script@v7 @@ -65,15 +73,17 @@ jobs: - name: Build env: - VERSION: ${{ matrix.version }} GH_TOKEN: ${{ github.token }} run: | - docker run --rm -v .:/workspace $CACHE_ENVS $PGRX_IMAGE cargo build --lib --features pg$VERSION --release - docker run --rm -v .:/workspace $CACHE_ENVS -e SEMVER=${SEMVER} $PGRX_IMAGE ./tools/schema.sh --features pg$VERSION --release - ./tools/package.sh + docker run --rm -v .:/workspace $CACHE_ENVS \ + -e SEMVER=$SEMVER \ + -e VERSION=${{ matrix.version }} \ + -e ARCH=$ARCH \ + -e PLATFORM=$PLATFORM \ + $PGRX_IMAGE ./tools/package.sh ls ./build - gh release upload --clobber $SEMVER ./build/vchord-pg${VERSION}_${SEMVER}_${PLATFORM}.deb - gh release upload --clobber $SEMVER ./build/vchord-pg${VERSION}_${ARCH}-unknown-linux-gnu_${SEMVER}.zip + gh release upload --clobber $SEMVER ./build/postgresql-${{ matrix.version }}-vchord_${SEMVER}-1_${PLATFORM}.deb + gh release upload --clobber $SEMVER ./build/postgresql-${{ matrix.version }}-vchord_${SEMVER}_${ARCH}-linux-gnu.zip docker: runs-on: ubuntu-latest @@ -92,7 +102,7 @@ jobs: run: | mkdir -p build for arch in amd64 arm64; do - gh release download $SEMVER --pattern "vchord-pg${{ matrix.version }}_${SEMVER}_${arch}.deb" --output ./build/vchord-pg${{ matrix.version }}_${SEMVER}_${arch}.deb + gh release download $SEMVER --pattern "postgresql-${{ matrix.version }}-vchord_${SEMVER}-1_${arch}.deb" --output ./build/postgresql-${{ matrix.version }}-vchord_${SEMVER}-1_${arch}.deb done - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index df5547a..8b1b4e2 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -31,15 +31,23 @@ jobs: strategy: matrix: include: - - runner: "ubicloud-standard-4" + - runner: "ubuntu-22.04" arch: "x86_64" - - runner: "ubicloud-standard-4-arm" + - runner: "ubuntu-22.04-arm" arch: "aarch64" runs-on: ${{ matrix.runner }} env: PGRX_IMAGE: "ghcr.io/tensorchord/vectorchord-pgrx:0.12.9-nightly-2024-12-25" steps: + - name: Set up Environment + run: | + sudo apt-get remove -y '^postgres.*' '^libpq.*' '^clang.*' '^llvm.*' '^libclang.*' '^libllvm.*' '^mono-llvm.*' + sudo apt-get purge -y '^postgres.*' '^libpq.*' '^clang.*' '^llvm.*' '^libclang.*' '^libllvm.*' '^mono-llvm.*' + + sudo apt-get install -y postgresql-common + sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y + sudo apt-get install -y postgresql-client-17 - uses: actions/checkout@v4 - name: Configure sccache uses: actions/github-script@v7 diff --git a/docker/Dockerfile b/docker/Dockerfile index ac5b1aa..48576ee 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,11 +4,11 @@ ARG PGVECTOR=0.8.0 FROM pgvector/pgvector:${PGVECTOR}-pg${PG_VERSION} ARG PG_VERSION -ARG SEMVER=0.0.0 +ARG SEMVER ARG TARGETARCH RUN echo ${PG_VERSION} -COPY ./build/vchord-pg${PG_VERSION}_${SEMVER}_${TARGETARCH}.deb /tmp/vchord.deb +COPY ./build/postgresql-${PG_VERSION}-vchord_${SEMVER}-1_${TARGETARCH}.deb /tmp/vchord.deb RUN apt-get install -y /tmp/vchord.deb && rm -f /tmp/vchord.deb CMD ["postgres", "-c" ,"shared_preload_libraries=vchord.so"] diff --git a/docker/binary.Dockerfile b/docker/binary.Dockerfile index 001bcc9..438031d 100644 --- a/docker/binary.Dockerfile +++ b/docker/binary.Dockerfile @@ -5,4 +5,4 @@ ARG PG_VERSION ARG TARGETARCH WORKDIR /workspace -COPY ./build/vchord-pg${PG_VERSION}_${SEMVER}_${TARGETARCH}.deb /workspace/ +COPY ./build/postgresql-${PG_VERSION}-vchord_${SEMVER}-1_${TARGETARCH}.deb /workspace/ diff --git a/docker/pg-cnpg/Dockerfile b/docker/pg-cnpg/Dockerfile index 7f7c621..4871e1d 100644 --- a/docker/pg-cnpg/Dockerfile +++ b/docker/pg-cnpg/Dockerfile @@ -19,7 +19,7 @@ ARG ALTDIR=/var/lib/postgresql/data/tensorchord USER root -COPY --from=binary /workspace/vchord-pg${PG_MAJOR}_${SEMVER}_${TARGETARCH}.deb /tmp/vchord.deb +COPY --from=binary /workspace/postgresql-${PG_MAJOR}-vchord_${SEMVER}-1_${TARGETARCH}.deb /tmp/vchord.deb RUN apt-get install -y /tmp/vchord.deb && rm -f /tmp/vchord.deb # PGDATA is set in pg-slim and used by dependents on this image. diff --git a/tools/package.sh b/tools/package.sh index 1b4b052..e8c92bf 100755 --- a/tools/package.sh +++ b/tools/package.sh @@ -6,40 +6,40 @@ printf "VERSION = ${VERSION}\n" printf "ARCH = ${ARCH}\n" printf "PLATFORM = ${PLATFORM}\n" -rm -rf ./build/dir_zip -rm -rf ./build/vchord-pg${VERSION}_${ARCH}-unknown-linux-gnu_${SEMVER}.zip -rm -rf ./build/dir_deb -rm -rf ./build/vchord-pg${VERSION}_${SEMVER}_${PLATFORM}.deb +cargo build --lib --features pg$VERSION --release +cargo pgrx schema --features pg$VERSION --out ./target/schema.sql -mkdir -p ./build/dir_zip -cp -a ./sql/upgrade/. ./build/dir_zip/ -cp ./target/release/schema.sql ./build/dir_zip/vchord--$SEMVER.sql -sed -e "s/@CARGO_VERSION@/$SEMVER/g" < ./vchord.control > ./build/dir_zip/vchord.control -cp ./target/release/libvchord.so ./build/dir_zip/vchord.so -zip ./build/vchord-pg${VERSION}_${ARCH}-unknown-linux-gnu_${SEMVER}.zip -j ./build/dir_zip/* +rm -rf ./build -mkdir -p ./build/dir_deb -mkdir -p ./build/dir_deb/DEBIAN -mkdir -p ./build/dir_deb/usr/share/postgresql/$VERSION/extension/ -mkdir -p ./build/dir_deb/usr/lib/postgresql/$VERSION/lib/ -for file in $(ls ./build/dir_zip/*.sql | xargs -n 1 basename); do - cp ./build/dir_zip/$file ./build/dir_deb/usr/share/postgresql/$VERSION/extension/$file +mkdir -p ./build/zip +cp -a ./sql/upgrade/. ./build/zip/ +cp ./target/schema.sql ./build/zip/vchord--$SEMVER.sql +sed -e "s/@CARGO_VERSION@/$SEMVER/g" < ./vchord.control > ./build/zip/vchord.control +cp ./target/release/libvchord.so ./build/zip/vchord.so +zip ./build/postgresql-${VERSION}-vchord_${SEMVER}_${ARCH}-linux-gnu.zip -j ./build/zip/* + +mkdir -p ./build/deb +mkdir -p ./build/deb/DEBIAN +mkdir -p ./build/deb/usr/share/postgresql/$VERSION/extension/ +mkdir -p ./build/deb/usr/lib/postgresql/$VERSION/lib/ +for file in $(ls ./build/zip/*.sql | xargs -n 1 basename); do + cp ./build/zip/$file ./build/deb/usr/share/postgresql/$VERSION/extension/$file done -for file in $(ls ./build/dir_zip/*.control | xargs -n 1 basename); do - cp ./build/dir_zip/$file ./build/dir_deb/usr/share/postgresql/$VERSION/extension/$file +for file in $(ls ./build/zip/*.control | xargs -n 1 basename); do + cp ./build/zip/$file ./build/deb/usr/share/postgresql/$VERSION/extension/$file done -for file in $(ls ./build/dir_zip/*.so | xargs -n 1 basename); do - cp ./build/dir_zip/$file ./build/dir_deb/usr/lib/postgresql/$VERSION/lib/$file +for file in $(ls ./build/zip/*.so | xargs -n 1 basename); do + cp ./build/zip/$file ./build/deb/usr/lib/postgresql/$VERSION/lib/$file done -echo "Package: vchord-pg${VERSION} -Version: ${SEMVER} +echo "Package: postgresql-${VERSION}-vchord +Version: ${SEMVER}-1 Section: database Priority: optional Architecture: ${PLATFORM} Maintainer: Tensorchord Description: Vector database plugin for Postgres, written in Rust, specifically designed for LLM -Homepage: https://pgvecto.rs/ -License: apache2" \ -> ./build/dir_deb/DEBIAN/control -(cd ./build/dir_deb && md5sum usr/share/postgresql/$VERSION/extension/* usr/lib/postgresql/$VERSION/lib/*) > ./build/dir_deb/DEBIAN/md5sums -dpkg-deb -Zxz --build ./build/dir_deb/ ./build/vchord-pg${VERSION}_${SEMVER}_${PLATFORM}.deb +Homepage: https://vectorchord.ai/ +License: AGPL-3 or Elastic-2" \ +> ./build/deb/DEBIAN/control +(cd ./build/deb && md5sum usr/share/postgresql/$VERSION/extension/* usr/lib/postgresql/$VERSION/lib/*) > ./build/deb/DEBIAN/md5sums +dpkg-deb --root-owner-group -Zxz --build ./build/deb/ ./build/postgresql-${VERSION}-vchord_${SEMVER}-1_${PLATFORM}.deb diff --git a/tools/schema-codegen.sh b/tools/schema-codegen.sh deleted file mode 100755 index 12f8289..0000000 --- a/tools/schema-codegen.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -e - -# CONTROL_FILEPATH -# SO_FILEPATH - -printf "fn main() {\n" - -cat << EOF - vchord::__pgrx_marker(); - - let mut entities = Vec::new(); - let control_file_path = std::path::PathBuf::from("$CONTROL_FILEPATH"); - let control_file = ::pgrx::pgrx_sql_entity_graph::ControlFile::try_from(control_file_path).expect(".control file should properly formatted"); - let control_file_entity = ::pgrx::pgrx_sql_entity_graph::SqlGraphEntity::ExtensionRoot(control_file); - - entities.push(control_file_entity); -EOF - -while read -r name_ident; do -cat << EOF - extern "Rust" { - fn $name_ident() -> ::pgrx::pgrx_sql_entity_graph::SqlGraphEntity; - } - let entity = unsafe { $name_ident() }; - entities.push(entity); -EOF -done <<< $(nm -D -g $SO_FILEPATH | grep "T __pgrx_internals_" | awk '{print $3}') - -cat << EOF - let pgrx_sql = ::pgrx::pgrx_sql_entity_graph::PgrxSql::build( - entities.into_iter(), - "vchord".to_string(), - false, - ) - .expect("SQL generation error"); - eprintln!("SQL entities to {}", "/dev/stdout",); - pgrx_sql - .write(&mut std::io::stdout()) - .expect("Could not write SQL to stdout"); -EOF - -printf "}\n" \ No newline at end of file diff --git a/tools/schema.sh b/tools/schema.sh deleted file mode 100755 index c43f7f8..0000000 --- a/tools/schema.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -e -if [[ " $@ " =~ --target' '([^ ]+) ]]; then - TARGET="${BASH_REMATCH[1]}" - if [[ " $@ " =~ " --release " ]]; then - DIR="./target/$TARGET/release" - else - DIR="./target/$TARGET/debug" - fi -else - if [[ " $@ " =~ " --release " ]]; then - DIR="./target/release" - else - DIR="./target/debug" - fi -fi - -if [ "$TARGET" = "" ]; then - printf "Target: [not specified]\n" 1>&2 - RUNNER=() -elif [ "$TARGET" = $(rustc -vV | awk '/^host/ { print $2 }') ]; then - printf "Target: [host]\n" 1>&2 - RUNNER=() -elif [ "$TARGET" = "aarch64-unknown-linux-gnu" ]; then - printf "Target: $TARGET\n" 1>&2 - QEMU_LD_PREFIX="/usr/aarch64-linux-gnu" - RUNNER=("qemu-aarch64-static") -elif [ "$TARGET" = "riscv64gc-unknown-linux-gnu" ]; then - printf "Target: $TARGET\n" 1>&2 - QEMU_LD_PREFIX="/usr/riscv64-linux-gnu" - RUNNER=("qemu-riscv64-static") -else - printf "Unknown target: $TARGET\n" 1>&2 - exit 1 -fi - -code=$(mktemp) -chmod 700 $code -CONTROL_FILEPATH="./vchord.control" SO_FILEPATH="$DIR/libvchord.so" $(dirname "$0")/schema-codegen.sh >> $code - -PGRX_EMBED=$code cargo rustc --package vchord --bin pgrx_embed_vchord "$@" -- --cfg pgrx_embed - -CARGO_PKG_VERSION=${SEMVER} QEMU_LD_PREFIX=$QEMU_LD_PREFIX "${RUNNER[@]}" "$DIR/pgrx_embed_vchord" | expand -t 4 > $DIR/schema.sql