Test build fix for linux

ArroyoSystems · Sep 24, 2024 · d8d2b4f · d8d2b4f
1 parent 55dd2e3
commit d8d2b4f
Show file tree

Hide file tree

Showing 4 changed files with 61 additions and 42 deletions.
diff --git a/.cargo/config.toml b/.cargo/config.toml
@@ -0,0 +1,11 @@
+[target.x86_64-apple-darwin]
+rustflags = [
+    "-C", "link-arg=-undefined",
+    "-C", "link-arg=dynamic_lookup",
+]
+
+[target.aarch64-apple-darwin]
+rustflags = [
+    "-C", "link-arg=-undefined",
+    "-C", "link-arg=dynamic_lookup",
+]
diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml
@@ -17,8 +17,8 @@ jobs:
       matrix:
         # see https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
         config:
-          - { runner: buildjet-32vcpu-ubuntu-2204, protoc: linux-x86_64, artifact: linux-x86_64 }
-          - { runner: buildjet-32vcpu-ubuntu-2204-arm, protoc: linux-aarch_64, artifact: linux-arm64 }
+          - { runner: buildjet-32vcpu-ubuntu-2204, protoc: linux-x86_64, pyarch: x86_64, artifact: linux-x86_64 }
+          - { runner: buildjet-32vcpu-ubuntu-2204-arm, protoc: linux-aarch_64, pyarch: aarch64, artifact: linux-arm64 }
     runs-on: ${{ matrix.config.runner }}
     services:
       postgres:
@@ -47,6 +47,15 @@ jobs:
       - name: Update rust
         run: |
           rustup update
+      - name: Install Python 3.12
+        run: |
+          curl -OL https://github.com/indygreg/python-build-standalone/releases/download/20240814/cpython-3.12.5+20240814-${{ matrix.config.pyarch }}-unknown-linux-gnu-install_only.tar.gz
+          tar xvfz cpython*.tar.gz
+          sudo cp -r python/bin/* /usr/local/bin/
+          sudo cp -r python/include/* /usr/local/include/
+          sudo cp -r python/lib/* /usr/local/lib/
+          sudo cp -r python/share/* /usr/local/share/
+          sudo ldconfig          
 
       - name: Run DB migrations
         run: |          
@@ -62,41 +71,41 @@ jobs:
           path: target/release/arroyo
           if-no-files-found: error
 
-  macos:
-    strategy:
-      fail-fast: true
-      matrix:
-        # see https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
-        config:
-          - { runner: macos-13, protoc: osx-x86_64, artifact: macos-x86_64 }
-          - { runner: macos-14, protoc: osx-aarch_64, artifact: macos-m1 }
-    runs-on: ${{ matrix.config.runner }}
-    steps:
-      - name: Check out
-        uses: actions/checkout@v4
-      - name: Setup pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 9.7.1
-      - name: Install protoc compiler
-        run: |
-          wget https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/protoc-$PROTOC_VERSION-${{ matrix.config.protoc }}.zip
-          unzip protoc*.zip && sudo mv bin/protoc /usr/local/bin
-      - name: Install Postgres and prepare DB
-        run: |
-          brew install postgresql@14 && brew services start postgresql && sleep 10
-          psql postgres -c "CREATE USER arroyo WITH PASSWORD 'arroyo' SUPERUSER;"
-          createdb arroyo
-      - name: Run DB migrations
-        run: |
-          cargo install --debug refinery_cli --version $REFINERY_VERSION
-          refinery migrate -e REFINERY_CONFIG -p crates/arroyo-api/migrations
-      - name: Run frontend build
-        run: cd webui && pnpm install && pnpm build
-      - name: Build Arroyo
-        run: cargo build --release --package arroyo && strip target/release/arroyo
-      - uses: actions/upload-artifact@v4
-        with:
-          name: arroyo-${{ matrix.config.artifact }}
-          path: target/release/arroyo
-          if-no-files-found: error
+#  macos:
+#    strategy:
+#      fail-fast: true
+#      matrix:
+#        # see https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories
+#        config:
+#          - { runner: macos-13, protoc: osx-x86_64, artifact: macos-x86_64 }
+#          - { runner: macos-14, protoc: osx-aarch_64, artifact: macos-m1 }
+#    runs-on: ${{ matrix.config.runner }}
+#    steps:
+#      - name: Check out
+#        uses: actions/checkout@v4
+#      - name: Setup pnpm
+#        uses: pnpm/action-setup@v4
+#        with:
+#          version: 9.7.1
+#      - name: Install protoc compiler
+#        run: |
+#          wget https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/protoc-$PROTOC_VERSION-${{ matrix.config.protoc }}.zip
+#          unzip protoc*.zip && sudo mv bin/protoc /usr/local/bin
+#      - name: Install Postgres and prepare DB
+#        run: |
+#          brew install postgresql@14 && brew services start postgresql && sleep 10
+#          psql postgres -c "CREATE USER arroyo WITH PASSWORD 'arroyo' SUPERUSER;"
+#          createdb arroyo
+#      - name: Run DB migrations
+#        run: |
+#          cargo install --debug refinery_cli --version $REFINERY_VERSION
+#          refinery migrate -e REFINERY_CONFIG -p crates/arroyo-api/migrations
+#      - name: Run frontend build
+#        run: cd webui && pnpm install && pnpm build
+#      - name: Build Arroyo
+#        run: cargo build --release --package arroyo && strip target/release/arroyo
+#      - uses: actions/upload-artifact@v4
+#        with:
+#          name: arroyo-${{ matrix.config.artifact }}
+#          path: target/release/arroyo
+#          if-no-files-found: error
diff --git a/Cargo.toml b/Cargo.toml
@@ -65,7 +65,6 @@ debug = 1
 [profile.dev]
 split-debuginfo = "unpacked"
 
-
 [patch.crates-io]
 deltalake = { git = 'https://github.com/delta-io/delta-rs', rev = 'e75a0b49b40f35ed361444bbea0e5720f359d732' }
 typify = { git = 'https://github.com/ArroyoSystems/typify.git', branch = 'arroyo' }

diff --git a/crates/arroyo-udf/arroyo-udf-python/Cargo.toml b/crates/arroyo-udf/arroyo-udf-python/Cargo.toml
@@ -7,7 +7,7 @@ edition = "2021"
 arroyo-udf-common = { path = "../arroyo-udf-common" }
 arrow = { workspace = true, features = ["ffi"] }
 datafusion = { workspace = true }
-pyo3 = { version = "0.21"}
+pyo3 = { version = "0.21", features = ["auto-initialize"]}
 anyhow = "1"
 tokio = { version = "1", features = ["full"] }
 itertools = "0.13.0"