Merge branch 'main' into ufuncs

pola-rs · Feb 13, 2024 · db28fec · db28fec
2 parents b5ff692 + 921ddea
commit db28fec
Show file tree

Hide file tree

Showing 1,097 changed files with 44,450 additions and 23,721 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,5 +1,6 @@
-*  @ritchie46 @stinodego @orlp @c-peters
+*  @ritchie46 @stinodego @c-peters
 
+/crates/  @ritchie46 @stinodego @orlp @orlp @c-peters
 /crates/polars-sql/  @ritchie46 @stinodego @orlp @c-peters @universalmind303
 /crates/polars-time/  @ritchie46 @stinodego @orlp @c-peters @MarcoGorelli
-/py-polars/  @ritchie46 @stinodego @orlp @c-peters @alexander-beedie @MarcoGorelli
+/py-polars/  @ritchie46 @stinodego @c-peters @alexander-beedie @MarcoGorelli
diff --git a/.github/ISSUE_TEMPLATE/bug_report_python.yml b/.github/ISSUE_TEMPLATE/bug_report_python.yml
@@ -1,19 +1,16 @@
 name: '🐞 Bug report - Python'
 description: Report an issue with Python Polars.
-labels: [bug, python]
+labels: [bug, needs triage, python]
 
 body:
   - type: checkboxes
     id: checks
     attributes:
       label: Checks
       options:
-        - label: >
-            I have checked that this issue has not already been reported.
+        - label: I have checked that this issue has not already been reported.
           required: true
-        - label: >
-            I have confirmed this bug exists on the
-            [latest version](https://pypi.org/project/polars/) of Polars.
+        - label: I have confirmed this bug exists on the [latest version](https://pypi.org/project/polars/) of Polars.
           required: true
 
   - type: textarea

diff --git a/.github/ISSUE_TEMPLATE/bug_report_rust.yml b/.github/ISSUE_TEMPLATE/bug_report_rust.yml
@@ -1,19 +1,16 @@
 name: '🐞 Bug report - Rust'
 description: Report an issue with Rust Polars.
-labels: [bug, rust]
+labels: [bug, needs triage, rust]
 
 body:
   - type: checkboxes
     id: checks
     attributes:
       label: Checks
       options:
-        - label: >
-            I have checked that this issue has not already been reported.
+        - label: I have checked that this issue has not already been reported.
           required: true
-        - label: >
-            I have confirmed this bug exists on the
-            [latest version](https://crates.io/crates/polars) of Polars.
+        - label: I have confirmed this bug exists on the [latest version](https://crates.io/crates/polars) of Polars.
           required: true
 
   - type: textarea

diff --git a/.github/release-drafter-python.yml b/.github/release-drafter-python.yml
@@ -29,8 +29,9 @@ categories:
     labels: enhancement
   - title: 🐞 Bug fixes
     labels: fix
+  - title: 📖 Documentation
+    labels: documentation
+  - title: 📦 Build system
+    labels: build
   - title: 🛠️ Other improvements
-    labels:
-      - build
-      - documentation
-      - internal
+    labels: internal
diff --git a/.github/release-drafter-rust.yml b/.github/release-drafter-rust.yml
@@ -27,9 +27,11 @@ categories:
     labels: enhancement
   - title: 🐞 Bug fixes
     labels: fix
+  - title: 📖 Documentation
+    labels: documentation
+  - title: 📦 Build system
+    labels: build
   - title: 🛠️ Other improvements
     labels:
-      - build
       - deprecation
-      - documentation
       - internal
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -46,7 +46,7 @@ jobs:
 
       - name: Load benchmark data from cache
         id: cache-data
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: py-polars/tests/benchmark/G1_1e7_1e2_5_0.csv
           key: benchmark-data
@@ -66,7 +66,7 @@ jobs:
 
       - name: Save benchmark data in cache
         if: github.ref_name == 'main'
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         with:
           path: py-polars/tests/benchmark/G1_1e7_1e2_5_0.csv
           key: ${{ steps.cache-data.outputs.cache-primary-key }}

diff --git a/.github/workflows/docs-global.yml b/.github/workflows/docs-global.yml
@@ -87,7 +87,7 @@ jobs:
           maturin develop
 
       - name: Set up Graphviz
-        uses: ts-graphviz/setup-graphviz@v1
+        uses: ts-graphviz/setup-graphviz@v2
 
       - name: Build documentation
         env:

diff --git a/.github/workflows/lint-global.yml b/.github/workflows/lint-global.yml
@@ -15,4 +15,4 @@ jobs:
       - name: Lint Markdown and TOML
         uses: dprint/[email protected]
       - name: Spell Check with Typos
-        uses: crate-ci/typos@v1.16.21
+        uses: crate-ci/typos@v1.17.2
diff --git a/.github/workflows/lint-rust.yml b/.github/workflows/lint-rust.yml
@@ -44,7 +44,7 @@ jobs:
           save-if: ${{ github.ref_name == 'main' }}
 
       - name: Run cargo clippy with all features enabled
-        run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
+        run: cargo clippy --workspace --all-targets --all-features --locked -- -D warnings -D clippy::dbg_macro
 
   # Default feature set should compile on the stable toolchain
   clippy-stable:
@@ -64,7 +64,7 @@ jobs:
           save-if: ${{ github.ref_name == 'main' }}
 
       - name: Run cargo clippy
-        run: cargo clippy --all-targets --locked -- -D warnings
+        run: cargo clippy --all-targets --locked -- -D warnings -D clippy::dbg_macro
 
   rustfmt:
     if: github.ref_name != 'main'

diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml
@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Label pull request
-        uses: release-drafter/release-drafter@v5
+        uses: release-drafter/release-drafter@v6
         with:
           disable-releaser: true
         env:

diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml
@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Draft Rust release
-        uses: release-drafter/release-drafter@v5
+        uses: release-drafter/release-drafter@v6
         with:
           config-name: release-drafter-rust.yml
           commitish: ${{ inputs.sha || github.sha }}
@@ -29,7 +29,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Draft Python release
-        uses: release-drafter/release-drafter@v5
+        uses: release-drafter/release-drafter@v6
         with:
           config-name: release-drafter-python.yml
           commitish: ${{ inputs.sha || github.sha }}

diff --git a/.github/workflows/release-python.yml b/.github/workflows/release-python.yml
@@ -94,6 +94,7 @@ jobs:
 
     env:
       SED_INPLACE: ${{ matrix.os == 'macos-latest' && '-i ''''' || '-i'}}
+      CPU_CHECK_MODULE: py-polars/polars/_cpu_check.py
 
     steps:
       - uses: actions/checkout@v4
@@ -122,31 +123,37 @@ jobs:
         if: matrix.package == 'polars-u64-idx'
         run: tomlq -i -t '.dependencies.polars.features += ["bigidx"]' py-polars/Cargo.toml
 
-      - name: Set _POLARS_ARCH variable
-        run: sed $SED_INPLACE 's/^_POLARS_ARCH = \"unknown\"$/_POLARS_ARCH = \"${{ matrix.architecture }}\"/g' py-polars/polars/_cpu_check.py
-
-      - name: Set _POLARS_LTS_CPU variable
-        if: matrix.package == 'polars-lts-cpu'
-        run: sed $SED_INPLACE 's/^_LTS_CPU = False$/_LTS_CPU = True/g' py-polars/polars/_cpu_check.py
+      - name: Determine CPU features for x86-64
+        id: features
+        if: matrix.architecture == 'x86-64'
+        env:
+          IS_LTS_CPU: ${{ matrix.package == 'polars-lts-cpu' }}
+          IS_MACOS: ${{ matrix.os == 'macos-latest' }}
+        run: |
+          if [[ "$IS_LTS_CPU" = true ]]; then
+            FEATURES=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt
+          elif [[ "$IS_MACOS" = true ]]; then
+            FEATURES=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+fma
+          else
+            FEATURES=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+avx2,+fma,+bmi1,+bmi2,+lzcnt
+          fi
+          echo "features=$FEATURES" >> $GITHUB_OUTPUT
 
       - name: Set RUSTFLAGS for x86-64
-        if: matrix.architecture == 'x86-64' && matrix.package != 'polars-lts-cpu' && matrix.os != 'macos-latest'
-        run: |
-          FEATURES=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+avx2,+fma,+bmi1,+bmi2,+lzcnt
-          echo "RUSTFLAGS=-C target-feature=$FEATURES" >> $GITHUB_ENV
-          sed $SED_INPLACE "s/^_POLARS_FEATURE_FLAGS = \"\"\$/_POLARS_FEATURE_FLAGS = \"$FEATURES\"/g" py-polars/polars/_cpu_check.py
-      - name: Set RUSTFLAGS for x86-64 MacOS
-        if: matrix.architecture == 'x86-64' && matrix.package != 'polars-lts-cpu' && matrix.os == 'macos-latest'
+        if: matrix.architecture == 'x86-64'
+        env:
+          FEATURES: ${{ steps.features.outputs.features }}
+          CFG: ${{ matrix.package == 'polars-lts-cpu' && '--cfg use_mimalloc' || '' }}
+        run: echo "RUSTFLAGS=-C target-feature=${{ steps.features.outputs.features }} $CFG" >> $GITHUB_ENV
+
+      - name: Set variables in CPU check module
         run: |
-          FEATURES=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+fma
-          echo "RUSTFLAGS=-C target-feature=$FEATURES" >> $GITHUB_ENV
-          sed $SED_INPLACE "s/^_POLARS_FEATURE_FLAGS = \"\"\$/_POLARS_FEATURE_FLAGS = \"$FEATURES\"/g" py-polars/polars/_cpu_check.py
-      - name: Set RUSTFLAGS for x86-64 LTS CPU
-        if: matrix.architecture == 'x86-64' && matrix.package == 'polars-lts-cpu'
+          sed $SED_INPLACE 's/^_POLARS_ARCH = \"unknown\"$/_POLARS_ARCH = \"${{ matrix.architecture }}\"/g' $CPU_CHECK_MODULE
+          sed $SED_INPLACE 's/^_POLARS_FEATURE_FLAGS = \"\"$/_POLARS_FEATURE_FLAGS = \"${{ steps.features.outputs.features }}\"/g' $CPU_CHECK_MODULE
+      - name: Set variables in CPU check module - LTS_CPU
+        if: matrix.package == 'polars-lts-cpu'
         run: |
-          FEATURES=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt
-          echo "RUSTFLAGS=-C target-feature=$FEATURES --cfg use_mimalloc" >> $GITHUB_ENV
-          sed $SED_INPLACE "s/^_POLARS_FEATURE_FLAGS = \"\"\$/_POLARS_FEATURE_FLAGS = \"$FEATURES\"/g" py-polars/polars/_cpu_check.py
+          sed $SED_INPLACE 's/^_LTS_CPU = False$/_LTS_CPU = True/g' $CPU_CHECK_MODULE
 
       - name: Set Rust target for aarch64
         if: matrix.architecture == 'aarch64'
@@ -228,7 +235,7 @@ jobs:
 
       - name: Create GitHub release
         id: github-release
-        uses: release-drafter/release-drafter@v5
+        uses: release-drafter/release-drafter@v6
         with:
           config-name: release-drafter-python.yml
           name: Python Polars ${{ steps.version.outputs.version }}
@@ -256,7 +263,7 @@ jobs:
 
       - name: Trigger other workflows related to the release
         if: inputs.dry-run == false && steps.version.outputs.is_prerelease == 'false'
-        uses: peter-evans/repository-dispatch@v2
+        uses: peter-evans/repository-dispatch@v3
         with:
           event-type: python-release
           client-payload: >

diff --git a/.github/workflows/test-bytecode-parser.yml b/.github/workflows/test-bytecode-parser.yml
@@ -16,7 +16,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        # Only the versions that are not already run as part of the regular test suite
+        python-version: ['3.9', '3.10']
 
     steps:
       - uses: actions/checkout@v4

diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml
@@ -3,6 +3,7 @@ name: Test Python
 on:
   pull_request:
     paths:
+      - Cargo.lock
       - py-polars/**
       - docs/src/python/**
       - crates/**
@@ -11,6 +12,7 @@ on:
     branches:
       - main
     paths:
+      - Cargo.lock
       - crates/**
       - docs/src/python/**
       - py-polars/**
@@ -49,7 +51,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Set up Graphviz
-        uses: ts-graphviz/setup-graphviz@v1
+        uses: ts-graphviz/setup-graphviz@v2
 
       - name: Create virtual environment
         env:
@@ -80,7 +82,7 @@ jobs:
         run: |
           python tests/docs/run_doctest.py
           pytest tests/docs/test_user_guide.py -m docs
-  
+
       - name: Run tests and report coverage
         if: github.ref_name != 'main'
         env:

diff --git a/.gitignore b/.gitignore
@@ -29,6 +29,7 @@ __pycache__/
 .coverage
 
 # Rust
+.cargo/
 target/
 
 # Project
-Original file line number
+Diff line change
@@ Expand Up / @@ -29,6 +29,7 @@ __pycache__/ @@
     .coverage
     # Rust
+    .cargo/
     target/
     # Project
@@ Expand Down @@