Core: Fix pytest rebuilder and improve editable installation detection

src/faebryk/core/cpp/CMakeLists.txt

@@ -19,14 +19,18 @@ project(${PROJECT_NAME} LANGUAGES CXX)
 # Currently, Scikit-build does not support FindPython, so we convert the
 # provided hints ourselves.
 if(SKBUILD)
-  set(Python_EXECUTABLE "${PYTHON_EXECUTABLE}")
-  set(Python_INCLUDE_DIR "${PYTHON_INCLUDE_DIR}")
-  set(Python_LIBRARY "${PYTHON_LIBRARY}")
+    set(Python_EXECUTABLE "${PYTHON_EXECUTABLE}")
+    set(Python_INCLUDE_DIR "${PYTHON_INCLUDE_DIR}")
+    set(Python_LIBRARY "${PYTHON_LIBRARY}")
 endif()
 set(PYBIND11_FINDPYTHON OFF)
 find_package(Python COMPONENTS Interpreter Development.Module)
 find_package(pybind11 CONFIG REQUIRED)
 
+message(STATUS "Python_EXECUTABLE: ${Python_EXECUTABLE}")
+message(STATUS "Python_INCLUDE_DIR: ${Python_INCLUDE_DIR}")
+message(STATUS "Python_LIBRARY: ${Python_LIBRARY}")
+
 # configure ------------------------------------------------------------
 # c++ standard
 set(CMAKE_CXX_STANDARD 20)

src/faebryk/core/cpp/__init__.py

@@ -1,19 +1,24 @@
 # This file is part of the faebryk project
 # SPDX-License-Identifier: MIT
 
+import json
 import logging
 import pathlib
 import shutil
-import site
+from importlib.metadata import Distribution
 from typing import Callable
 
 logger = logging.getLogger(__name__)
 
 
 # Check if installed as editable
 def is_editable_install():
-    site_packages = site.getsitepackages()
-    return not any((pathlib.Path(sp) / "faebryk").exists() for sp in site_packages)
+    distro = Distribution.from_name("faebryk")
+    return (
+        json.loads(distro.read_text("direct_url.json"))
+        .get("dir_info", {})
+        .get("editable", False)
+    )
 
 
 def compile_and_load():
@@ -22,17 +27,9 @@ def compile_and_load():
     into _cpp.
     """
     import platform
-    import subprocess
     import sys
 
-    def _do(*args, **kwargs):
-        try:
-            return subprocess.check_output(
-                *args, stderr=subprocess.PIPE, text=True, **kwargs
-            )
-        except subprocess.CalledProcessError as e:
-            logger.error(f"Subprocess error: {e.stderr}")
-            raise
+    from faebryk.libs.util import run_live
 
     cpp_dir = pathlib.Path(__file__).parent
     build_dir = cpp_dir / "build"
@@ -43,7 +40,7 @@ def _do(*args, **kwargs):
             "cmake not found, needed for compiling c++ code in editable mode"
         )
 
-    pybind11_dir = _do(["python", "-m", "pybind11", "--cmakedir"]).strip()
+    pybind11_dir = run_live([sys.executable, "-m", "pybind11", "--cmakedir"]).strip()
 
     # Force recompile
     # subprocess.run(["rm", "-rf", str(build_dir)], check=True)
@@ -56,7 +53,7 @@ def _do(*args, **kwargs):
         if arch in ["arm64", "x86_64"]:
             other_flags += [f"-DCMAKE_OSX_ARCHITECTURES={arch}"]
 
-    _do(
+    run_live(
         [
             "cmake",
             "-S",
@@ -65,10 +62,11 @@ def _do(*args, **kwargs):
             str(build_dir),
             "-DEDITABLE=1",
             f"-DCMAKE_PREFIX_PATH={pybind11_dir}",
+            "-DPython_EXECUTABLE=" + sys.executable,
         ]
         + other_flags,
     )
-    _do(
+    run_live(
         [
             "cmake",
             "--build",

src/faebryk/libs/util.py

@@ -6,6 +6,8 @@
 import inspect
 import logging
 import os
+import select
+import subprocess
 import sys
 from abc import abstractmethod
 from collections import defaultdict
@@ -1082,3 +1084,54 @@ def setdefault(self, key: T, default: U) -> U:
         except KeyError:
             self[key] = default
         return default
+
+
+def run_live(
+    *args,
+    logger: logging.Logger = logger,
+    stdout_level: int | None = logging.DEBUG,
+    stderr_level: int | None = logging.ERROR,
+    **kwargs,
+) -> str:
+    """Runs a process and logs the output live."""
+
+    process = subprocess.Popen(
+        *args,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+        bufsize=1,  # Line buffered
+        **kwargs,
+    )
+
+    # Set up file descriptors to monitor
+    reads = [process.stdout, process.stderr]
+    stdout = []
+    while reads and process.poll() is None:
+        # Wait for output on either stream
+        readable, _, _ = select.select(reads, [], [])
+
+        for stream in readable:
+            line = stream.readline()
+            if not line:  # EOF
+                reads.remove(stream)
+                continue
+
+            if stream == process.stdout:
+                stdout.append(line)
+                if stdout_level is not None:
+                    logger.log(stdout_level, line.rstrip())
+            else:
+                if stderr_level is not None:
+                    logger.log(stderr_level, line.rstrip())
+
+    # Ensure the process has finished
+    process.wait()
+
+    # Get return code and check for errors
+    if process.returncode != 0:
+        raise subprocess.CalledProcessError(
+            process.returncode, args[0], "".join(stdout)
+        )
+
+    return "\n".join(stdout)

test/core/cpp/test_importcpp.py

@@ -1,8 +1,8 @@
 # This file is part of the faebryk project
 # SPDX-License-Identifier: MIT
 
-from faebryk.core.cpp import add
-
 
 def test_add():
+    from faebryk.core.cpp import add
+
     assert add(1, 2) == 3