Nix build including polars causes ModuleNotFoundError: No module named 'maturin'

[gschwim]

Describe the issue

In testing poetry2nix as a potential upgrade to my current workflow, I developed a simply application in attempt to get it to build with nix. The build works fine in poetry, however when building with poetry2nix it fails with the error ModuleNotFoundError: No module named 'maturin'

Additional context

Found information about preferWheels in the edgecases.md file. Implementing this in my default.nix does allow the build to complete. I'm reporting this as the edgecases file suggests you'd like to be informed about this issue.

shell.nix:

{ pkgs ? import <nixpkgs> {}}:

pkgs.mkShell {
  buildInputs = [ pkgs.python310 pkgs.poetry ];
}

default.nix

{ pkgs ? import <nixpkgs> {} }:

let
  inherit (pkgs) fetchFromGitHub callPackage;

  poetry2nix-src = fetchFromGitHub {
      owner = "nix-community";
      repo = "poetry2nix";
      rev = "0a592572706db14e49202892318d3812061340a0";
      sha256 = "0m048vxj6zlvk56dm7m3wdgfqw49p68yvx1wyjqjvw01248dpa71";
  };

  poetry2nix = callPackage poetry2nix-src { };

in
  poetry2nix.mkPoetryEnv {
#    preferWheels = true;
    projectDir = ./.;
  }

pyproject.toml:

[tool.poetry]
name = "blah2"
version = "0.1.0"
description = ""
authors = ["Greg Schwimer <[email protected]>"]
readme = "README.md"

[tool.poetry.dependencies]
python = "^3.10"
polars = "^0.20.30"


[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

[tool.poetry.scripts]
blah = "blah2.main:main"

poetry.lock:

[[package]]
name = "polars"
version = "0.20.30"
description = "Blazingly fast DataFrame library"
optional = false
python-versions = ">=3.8"
files = [
    {file = "polars-0.20.30-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:7802cb8a6d3fa0fadf446bf19b61ef8450b81229766f1817980bc6e0b2488c03"},
    {file = "polars-0.20.30-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:0d7278f6e12e6623c8ee6e637d866b96b69f960b358125c955ac76abeb942192"},
    {file = "polars-0.20.30-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:510f708332a0e0d563a5bac266c11ccd12932606c7331bf28f7173cf669d0c69"},
    {file = "polars-0.20.30-cp38-abi3-manylinux_2_24_aarch64.whl", hash = "sha256:70d58b89cf9902fb5c3506afef76c5c2b247581d0991b082bcdc6d73d983aa2b"},
    {file = "polars-0.20.30-cp38-abi3-win_amd64.whl", hash = "sha256:039c84f7a294aa71438048e14899d59ce005f213872e27fe6bf6ce3ae8730d7d"},
    {file = "polars-0.20.30.tar.gz", hash = "sha256:b9fd2c6b5100caa7c26496ce17ec289cae23c2db6c41713bc52c54c76d4451ae"},
]

[package.extras]
adbc = ["adbc-driver-manager", "adbc-driver-sqlite"]
all = ["polars[adbc,async,cloudpickle,connectorx,deltalake,fastexcel,fsspec,gevent,iceberg,numpy,pandas,plot,pyarrow,pydantic,sqlalchemy,timezone,xlsx2csv,xlsxwriter]"]
async = ["nest-asyncio"]
cloudpickle = ["cloudpickle"]
connectorx = ["connectorx (>=0.3.2)"]
deltalake = ["deltalake (>=0.15.0)"]
fastexcel = ["fastexcel (>=0.9)"]
fsspec = ["fsspec"]
gevent = ["gevent"]
iceberg = ["pyiceberg (>=0.5.0)"]
matplotlib = ["matplotlib"]
numpy = ["numpy (>=1.16.0)"]
openpyxl = ["openpyxl (>=3.0.0)"]
pandas = ["pandas", "pyarrow (>=7.0.0)"]
plot = ["hvplot (>=0.9.1)"]
pyarrow = ["pyarrow (>=7.0.0)"]
pydantic = ["pydantic"]
pyxlsb = ["pyxlsb (>=1.0)"]
sqlalchemy = ["pandas", "sqlalchemy"]
timezone = ["backports-zoneinfo", "tzdata"]
xlsx2csv = ["xlsx2csv (>=0.8.0)"]
xlsxwriter = ["xlsxwriter"]

[metadata]
lock-version = "2.0"
python-versions = "^3.9"
content-hash = "c20dd89c7288db4e98967313b268587bc9c4ded72a6085da645a6e8fc57c71cc"

Full output of the build attempt:

building '/nix/store/csy31ndxrcanpdpwpkff61mz8fkpj2df-source.drv'...

trying https://github.com/nix-community/poetry2nix/archive/0a592572706db14e49202892318d3812061340a0.tar.gz
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
100 1477k    0 1477k    0     0  1312k      0 --:--:--  0:00:01 --:--:-- 1312k
unpacking source archive /private/tmp/nix-build-source.drv-0/0a592572706db14e49202892318d3812061340a0.tar.gz
these 2 derivations will be built:
  /nix/store/64gkjayh7lxpih2b5chy1s21aq8lr76i-python3.11-polars-0.20.30.drv
  /nix/store/rl9q5szn1p48g2qk4qbi5ixr274idnbj-python3-3.11.9-env.drv
building '/nix/store/64gkjayh7lxpih2b5chy1s21aq8lr76i-python3.11-polars-0.20.30.drv'...
Sourcing python-remove-tests-dir-hook
Sourcing python-catch-conflicts-hook.sh
Sourcing python-remove-bin-bytecode-hook.sh
Sourcing pypa-install-hook
Using pypaInstallPhase
Sourcing python-imports-check-hook.sh
Using pythonImportsCheckPhase
Sourcing python-namespaces-hook
Sourcing pip-build-hook
Using pipBuildPhase
Using pipShellHook
Running phase: unpackPhase
unpacking source archive /nix/store/djnvhsbrm4hwgl0xchv4ch6xx6jgdbg2-polars-0.20.30.tar.gz
source root is polars-0.20.30
setting SOURCE_DATE_EPOCH to timestamp 1716744372 of file polars-0.20.30/rust-toolchain.toml
Running phase: patchPhase
Removing path dependencies
Finished removing path dependencies
Removing git dependencies
Finished removing git dependencies
Removing wheel-url dependencies
Finished removing wheel-url dependencies
Running phase: configurePhase
no configure script, doing nothing
Running phase: buildPhase
Executing pipBuildPhase
Creating a wheel...
WARNING: The directory '/homeless-shelter/Library/Caches/pip' or its parent directory is not owned or is not writable by the current user. The cache has been disabled. Check the permissions and owner of that directory. If executing pip with sudo, you should use sudo's -H flag.
Processing /private/tmp/nix-build-python3.11-polars-0.20.30.drv-0/polars-0.20.30
  Running command Preparing metadata (pyproject.toml)
  Preparing metadata (pyproject.toml) ... done
ERROR: Exception:
Traceback (most recent call last):
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/cli/base_command.py", line 180, in exc_logging_wrapper
    status = run_func(*args)
             ^^^^^^^^^^^^^^^
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/cli/req_command.py", line 245, in wrapper
    return func(self, options, args)
           ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/commands/wheel.py", line 147, in run
    requirement_set = resolver.resolve(reqs, check_supported_wheels=True)
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/resolution/resolvelib/resolver.py", line 76, in resolve
    collected = self.factory.collect_root_requirements(root_reqs)
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/resolution/resolvelib/factory.py", line 534, in collect_root_requirements
    reqs = list(
           ^^^^^
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/resolution/resolvelib/factory.py", line 490, in _make_requirements_from_install_req
    cand = self._make_base_candidate_from_link(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/resolution/resolvelib/factory.py", line 228, in _make_base_candidate_from_link
    self._link_candidate_cache[link] = LinkCandidate(
                                       ^^^^^^^^^^^^^^
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 290, in __init__
    super().__init__(
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 156, in __init__
    self.dist = self._prepare()
                ^^^^^^^^^^^^^^^
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 222, in _prepare
    dist = self._prepare_distribution()
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/resolution/resolvelib/candidates.py", line 301, in _prepare_distribution
    return preparer.prepare_linked_requirement(self._ireq, parallel_builds=True)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/operations/prepare.py", line 525, in prepare_linked_requirement
    return self._prepare_linked_requirement(req, parallel_builds)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/operations/prepare.py", line 640, in _prepare_linked_requirement
    dist = _get_prepared_distribution(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/operations/prepare.py", line 71, in _get_prepared_distribution
    abstract_dist.prepare_distribution_metadata(
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/distributions/sdist.py", line 67, in prepare_distribution_metadata
    self.req.prepare_metadata()
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/req/req_install.py", line 579, in prepare_metadata
    self.metadata_directory = generate_metadata(
                              ^^^^^^^^^^^^^^^^^^
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/operations/build/metadata.py", line 35, in generate_metadata
    distinfo_dir = backend.prepare_metadata_for_build_wheel(metadata_dir)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/utils/misc.py", line 766, in prepare_metadata_for_build_wheel
    return super().prepare_metadata_for_build_wheel(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_vendor/pyproject_hooks/_impl.py", line 186, in prepare_metadata_for_build_wheel
    return self._call_hook('prepare_metadata_for_build_wheel', {
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_vendor/pyproject_hooks/_impl.py", line 321, in _call_hook
    raise BackendUnavailable(data.get('traceback', ''))
pip._vendor.pyproject_hooks._impl.BackendUnavailable: Traceback (most recent call last):
  File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_vendor/pyproject_hooks/_in_process/_in_process.py", line 77, in _build_backend
    obj = import_module(mod_path)
          ^^^^^^^^^^^^^^^^^^^^^^^
  File "/nix/store/6jy28lfjr42adzi10p158aq6m1q9qgs0-python3-3.11.9/lib/python3.11/importlib/__init__.py", line 126, in import_module
    return _bootstrap._gcd_import(name[level:], package, level)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<frozen importlib._bootstrap>", line 1204, in _gcd_import
  File "<frozen importlib._bootstrap>", line 1176, in _find_and_load
  File "<frozen importlib._bootstrap>", line 1140, in _find_and_load_unlocked
ModuleNotFoundError: No module named 'maturin'


error: builder for '/nix/store/64gkjayh7lxpih2b5chy1s21aq8lr76i-python3.11-polars-0.20.30.drv' failed with exit code 2;
       last 25 log lines:
       >                               ^^^^^^^^^^^^^^^^^^
       >   File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/operations/build/metadata.py", line 35, in generate_metadata
       >     distinfo_dir = backend.prepare_metadata_for_build_wheel(metadata_dir)
       >                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
       >   File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_internal/utils/misc.py", line 766, in prepare_metadata_for_build_wheel
       >     return super().prepare_metadata_for_build_wheel(
       >            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
       >   File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_vendor/pyproject_hooks/_impl.py", line 186, in prepare_metadata_for_build_wheel
       >     return self._call_hook('prepare_metadata_for_build_wheel', {
       >            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
       >   File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_vendor/pyproject_hooks/_impl.py", line 321, in _call_hook
       >     raise BackendUnavailable(data.get('traceback', ''))
       > pip._vendor.pyproject_hooks._impl.BackendUnavailable: Traceback (most recent call last):
       >   File "/nix/store/5qmy00anjahlh2s2wd0lgs9k46b8dq7r-python3.11-pip-24.0/lib/python3.11/site-packages/pip/_vendor/pyproject_hooks/_in_process/_in_process.py", line 77, in _build_backend
       >     obj = import_module(mod_path)
       >           ^^^^^^^^^^^^^^^^^^^^^^^
       >   File "/nix/store/6jy28lfjr42adzi10p158aq6m1q9qgs0-python3-3.11.9/lib/python3.11/importlib/__init__.py", line 126, in import_module
       >     return _bootstrap._gcd_import(name[level:], package, level)
       >            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
       >   File "<frozen importlib._bootstrap>", line 1204, in _gcd_import
       >   File "<frozen importlib._bootstrap>", line 1176, in _find_and_load
       >   File "<frozen importlib._bootstrap>", line 1140, in _find_and_load_unlocked
       > ModuleNotFoundError: No module named 'maturin'
       >
       >
       For full logs, run 'nix log /nix/store/64gkjayh7lxpih2b5chy1s21aq8lr76i-python3.11-polars-0.20.30.drv'.
error: 1 dependencies of derivation '/nix/store/rl9q5szn1p48g2qk4qbi5ixr274idnbj-python3-3.11.9-env.drv' failed to build

Thanks!
GS

[TyberiusPrime]

So the quick fix is to add preferWheel (or preferWheels).

As for building it with poetry2nix, we need an override.
I have gotten a half working one - but now it seems that the argminmax crate used by polars needs nightly rust,
and well my time budget here is exceed, so I'm dropping this non working override here.

      polars = prev.polars.overridePythonAttrs (
        old: {
           cargoDeps = let
             unpacked_src = pkgs.runCommand "polars-cargo-lock" {} ''
               mkdir -p $out
               cd $out && tar xf ${old.src} ${old.pname}-${old.version}/Cargo.lock --strip-components=1
               '';
           in
             pkgs.rustPlatform.importCargoLock {
               lockFile = "${unpacked_src}/Cargo.lock";
           };
           buildAndTestSubdir = "py-polars";
           nativeBuildInputs = old.nativeBuildInputs or [ ] ++ [
             pkgs.rustPlatform.cargoSetupHook
             pkgs.rustPlatform.maturinBuildHook ];
        }
      );

edit: Though I don't know how useful the wheels for polars are.. they only seem to be available for python 3.8?
edit2: never mind, they are tagged 'cp38', but also 'abi3' which means they work on any python3 version.

[MMesch]

Same problem here. Didn't manage to get it running without preferWheels = true. @TyberiusPrime got me a bit futher but I hit some hash problem afterwards:

error:
       … while calling the 'derivationStrict' builtin
         at <nix/derivation-internal.nix>:34:12:
           33|
           34|   strict = derivationStrict drvAttrs;
             |            ^
           35|

       … while evaluating derivation 'python3-3.12.5-env'
         whose name attribute is located at /nix/store/qpg5mwsind2hy35b9vpk6mx4jimnypw0-source/pkgs/stdenv/generic/make-derivation.nix:336:7

       … while evaluating attribute 'passAsFile' of derivation 'python3-3.12.5-env'
         at /nix/store/qpg5mwsind2hy35b9vpk6mx4jimnypw0-source/pkgs/build-support/trivial-builders/default.nix:60:9:
           59|         inherit buildCommand name;
           60|         passAsFile = [ "buildCommand" ]
             |         ^
           61|           ++ (derivationArgs.passAsFile or [ ]);

       (stack trace truncated; use '--show-trace' to show the full, detailed trace)

       error: No hash was found while vendoring the git dependency numpy-0.21.0. You can add
       a hash through the `outputHashes` argument of `importCargoLock`:

       outputHashes = {
         "numpy-0.21.0" = "<hash>";
       };

       If you use `buildRustPackage`, you can add this attribute to the `cargoLock`
       attribute set.

[TyberiusPrime]

that reads as if it's not using the wheel.

Here's what I had for 1.1.0 in my somewhat stalled 'many overrides' PR:

 (final: prev: (
                {
                    "polars"  = prev."polars".overridePythonAttrs (old: 
                    (offlineMaturin {
                          furtherArgs = {
                            cargoRoot = "";
                            
                    cargoDeps = (pkgs.rustPlatform.importCargoLock {
                        lockFile = ./cargo.locks/polars/1.1.0.lock;
                        outputHashes = {"numpy-0.21.0" = "sha256-u0Z+6L8pXSPaA3cE1sUpY6sCoaU1clXUcj/avnNzmsw=";
};
                    });
                    
                          };
                     }
                    old)
                    );
                }
            ))

but that fails with
"#![feature]` may not be used on the stable release channel"