Core: Lazy MIFs (#129)

Second iteration algorithm improvements of the c++ graph backend for another enormous speed up.

This will lazily check for `mif` connections rather than cross-connecting up and down the specialisation chain at connection-time. Performance is vastly improved over the previous implementation, and there's an opportunity to 10x it again.

examples/minimal_led_orderable.py

@@ -19,7 +19,7 @@
 from faebryk.exporters.pcb.layout.typehierarchy import LayoutTypeHierarchy
 from faebryk.libs.app.checks import run_checks
 from faebryk.libs.app.manufacturing import export_pcba_artifacts
-from faebryk.libs.app.parameters import replace_tbd_with_any
+from faebryk.libs.app.parameters import replace_tbd_with_any, resolve_dynamic_parameters
 from faebryk.libs.brightness import TypicalLuminousIntensity
 from faebryk.libs.examples.buildutil import BUILD_DIR, PCB_FILE, apply_design_to_pcb
 from faebryk.libs.examples.pickers import add_example_pickers
@@ -130,6 +130,7 @@ def main():
     logger.info("Building app")
     app = App()
     G = app.get_graph()
+    resolve_dynamic_parameters(G)
 
     # picking ----------------------------------------------------------------
     replace_tbd_with_any(app, recursive=True)

scripts/find_duplicate_test_files.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+parent_dir=$(dirname "$0")/..
+test_dir="$parent_dir/test"
+
+# find all files in test_dir ending in .py
+# make sure the filenames are unique
+find "$test_dir" -type f -name "*.py" -exec basename {} \; | sort | uniq -d

src/faebryk/core/cpp/CMakeLists.txt

@@ -56,6 +56,11 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2")
 if(${EDITABLE} AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always")
 endif()
+
+if(GLOBAL_PRINTF_DEBUG)
+    add_definitions(-DGLOBAL_PRINTF_DEBUG=1)
+endif()
+
 # source files ---------------------------------------------------------
 include_directories(${CMAKE_SOURCE_DIR}/include)
 file(GLOB_RECURSE SOURCE_FILES 

src/faebryk/core/cpp/__init__.py

@@ -12,6 +12,7 @@
 
 LEAK_WARNINGS = ConfigFlag("CPP_LEAK_WARNINGS", default=False)
 DEBUG_BUILD = ConfigFlag("CPP_DEBUG_BUILD", default=False)
+PRINTF_DEBUG = ConfigFlag("CPP_PRINTF_DEBUG", default=False)
 
 
 # Check if installed as editable
@@ -66,6 +67,7 @@ def compile_and_load():
 
     if DEBUG_BUILD:
         other_flags += ["-DCMAKE_BUILD_TYPE=Debug"]
+    other_flags += [f"-DGLOBAL_PRINTF_DEBUG={int(bool(PRINTF_DEBUG))}"]
 
     with global_lock(_build_dir / "lock", timeout_s=60):
         run_live(
@@ -115,6 +117,7 @@ def compile_and_load():
                 is_pyi=True,
             )
         )
+        run_live(["ruff", "check", "--fix", pyi_out], logger=logger)
 
 
 # Re-export c++ with type hints provided by __init__.pyi

src/faebryk/core/cpp/__init__.pyi

@@ -8,9 +8,39 @@ import enum
 from collections.abc import Callable, Sequence, Set
 from typing import overload
 
+class Counter:
+    @property
+    def in_cnt(self) -> int: ...
+    @property
+    def weak_in_cnt(self) -> int: ...
+    @property
+    def out_weaker(self) -> int: ...
+    @property
+    def out_stronger(self) -> int: ...
+    @property
+    def out_cnt(self) -> int: ...
+    @property
+    def time_spent_s(self) -> float: ...
+    @property
+    def hide(self) -> bool: ...
+    @property
+    def name(self) -> str: ...
+    @property
+    def multi(self) -> bool: ...
+    @property
+    def total_counter(self) -> bool: ...
+
+class Edge:
+    def __repr__(self) -> str: ...
+    @property
+    def to(self) -> GraphInterface: ...
+
 class Graph:
     def __init__(self) -> None: ...
     def get_edges(self, arg: GraphInterface, /) -> dict[GraphInterface, Link]: ...
+    @property
+    def edges(self) -> list[tuple[GraphInterface, GraphInterface, Link]]: ...
+    def get_gifs(self) -> set[GraphInterface]: ...
     def invalidate(self) -> None: ...
     @property
     def node_count(self) -> int: ...
@@ -50,6 +80,8 @@ class GraphInterface:
     def connect(self, others: Sequence[GraphInterface]) -> None: ...
     @overload
     def connect(self, other: GraphInterface, link: Link) -> None: ...
+    @overload
+    def connect(self, others: Sequence[GraphInterface], link: Link) -> None: ...
 
 class GraphInterfaceHierarchical(GraphInterface):
     def __init__(self, is_parent: bool) -> None: ...
@@ -77,18 +109,17 @@ class GraphInterfaceSelf(GraphInterface):
     def __init__(self) -> None: ...
 
 class Link:
-    pass
+    def __eq__(self, arg: Link, /) -> bool: ...
+    def is_cloneable(self) -> bool: ...
 
 class LinkDirect(Link):
     def __init__(self) -> None: ...
 
 class LinkDirectConditional(LinkDirect):
     def __init__(
         self,
-        arg: Callable[
-            [GraphInterface, GraphInterface], LinkDirectConditionalFilterResult
-        ],
-        /,
+        filter: Callable[[Path], LinkDirectConditionalFilterResult],
+        needs_only_first_in_path: bool = False,
     ) -> None: ...
 
 class LinkDirectConditionalFilterResult(enum.Enum):
@@ -98,6 +129,12 @@ class LinkDirectConditionalFilterResult(enum.Enum):
 
     FILTER_FAIL_UNRECOVERABLE = 2
 
+class LinkDirectDerived(LinkDirectConditional):
+    def __init__(self, arg: Path, /) -> None: ...
+
+class LinkExists(Exception):
+    pass
+
 class LinkFilteredException(Exception):
     pass
 
@@ -145,9 +182,22 @@ class NodeException(Exception):
 class NodeNoParent(Exception):
     pass
 
+class Path:
+    def __repr__(self) -> str: ...
+    def __len__(self) -> int: ...
+    def contains(self, arg: GraphInterface, /) -> bool: ...
+    def last(self) -> GraphInterface: ...
+    def first(self) -> GraphInterface: ...
+    def get_link(self, arg: Edge, /) -> Link: ...
+    def iterate_edges(self, arg: Callable[[Edge], bool], /) -> None: ...
+    def __getitem__(self, arg: int, /) -> GraphInterface: ...
+
 def add(i: int, j: int = 1) -> int:
     """A function that adds two numbers"""
 
 def call_python_function(func: Callable[[], int]) -> int: ...
+def find_paths(src: Node, dst: Sequence[Node]) -> tuple[list[Path], list[Counter]]: ...
 def print_obj(obj: object) -> None: ...
+def set_indiv_measure(value: bool) -> None: ...
 def set_leak_warnings(value: bool) -> None: ...
+def set_max_paths(arg0: int, arg1: int, arg2: int, /) -> None: ...

src/faebryk/core/cpp/include/graph/graph.hpp

@@ -10,6 +10,7 @@
 #include <nanobind/stl/pair.h>
 #include <nanobind/stl/shared_ptr.h>
 #include <nanobind/stl/string.h>
+#include <nanobind/stl/tuple.h>
 #include <nanobind/stl/unordered_map.h>
 #include <nanobind/stl/unordered_set.h>
 #include <nanobind/stl/vector.h>
@@ -35,11 +36,22 @@ using Node_ref = std::shared_ptr<Node>;
 using Graph_ref = std::shared_ptr<Graph>;
 using GI_refs_weak = std::vector<GI_ref_weak>;
 using HierarchicalNodeRef = std::pair<Node_ref, std::string>;
+using Link_weak_ref = Link *;
 
-class Node {
+class LinkExists : public std::runtime_error {
   private:
-    std::optional<nb::object> py_handle{};
+    Link_ref existing_link;
+    Link_ref new_link;
+    std::string make_msg(Link_ref existing_link, Link_ref new_link,
+                         const std::string &msg);
+
+  public:
+    LinkExists(Link_ref existing_link, Link_ref new_link, const std::string &msg);
+    Link_ref get_existing_link();
+    Link_ref get_new_link();
+};
 
+class Node {
   public:
     struct NodeException : public std::runtime_error {
         NodeException(Node &node, const std::string &msg)
@@ -53,8 +65,27 @@ class Node {
         }
     };
 
+    class Type {
+      private:
+        nb::handle type;
+        bool hack_cache_is_moduleinterface;
+
+      public:
+        Type(nb::handle type);
+        bool operator==(const Type &other) const;
+        std::string get_name();
+        // TODO these are weird
+        bool is_moduleinterface();
+        static nb::type_object get_moduleinterface_type();
+    };
+
+  private:
+    std::optional<nb::object> py_handle{};
+    std::optional<Type> type{};
+
   private:
     std::shared_ptr<GraphInterfaceSelf> self;
+
     std::shared_ptr<GraphInterfaceHierarchical> children;
     std::shared_ptr<GraphInterfaceHierarchical> parent;
 
@@ -80,6 +111,7 @@ class Node {
     std::string get_full_name(bool types = false);
     std::string repr();
 
+    Type get_type();
     std::string get_type_name();
     // TODO replace with constructor
     void set_py_handle(nb::object handle);
@@ -119,6 +151,7 @@ class GraphInterface {
     void connect(GI_ref_weak other);
     void connect(GI_refs_weak others);
     void connect(GI_ref_weak other, Link_ref link);
+    void connect(GI_refs_weak others, Link_ref link);
     // TODO replace with set_node(Node_ref node, std::string name)
     void set_node(Node_ref node);
     Node_ref get_node();
@@ -128,6 +161,9 @@ class GraphInterface {
     std::string repr();
     // force vtable, for typename
     virtual void do_stuff() {};
+
+    /** Index in Graph::v */
+    size_t v_i = 0;
 };
 
 class Link {
@@ -138,14 +174,52 @@ class Link {
   protected:
     Link();
     Link(GI_ref_weak from, GI_ref_weak to);
+    Link(const Link &other);
 
   public:
     std::pair<GI_ref_weak, GI_ref_weak> get_connections();
     virtual void set_connections(GI_ref_weak from, GI_ref_weak to);
     bool is_setup();
+    virtual Link_ref clone() const = 0;
+    virtual bool is_cloneable() const = 0;
+    virtual bool operator==(const Link &other) const;
+    virtual std::string str() const;
+};
+
+struct Edge {
+    /*const*/ GI_ref_weak from;
+    /*const*/ GI_ref_weak to;
+
+    std::string str() const;
 };
 
-using Path = std::vector<GI_ref_weak>;
+using TriEdge = std::tuple</*const*/ GI_ref_weak, /*const*/ GI_ref_weak,
+                           /*const*/ GI_ref_weak>;
+
+class Path {
+  public:
+    Path(/*const*/ GI_ref_weak path_head);
+    Path(std::vector<GI_ref_weak> path);
+    Path(const Path &other);
+    Path(Path &&other);
+    ~Path();
+
+    std::vector</*const*/ GI_ref_weak> path;
+
+    /*const*/ Link_weak_ref get_link(Edge edge) /*const*/;
+    std::optional<Edge> last_edge() /*const*/;
+    std::optional<TriEdge> last_tri_edge() /*const*/;
+    /*const*/ GI_ref_weak last() /*const*/;
+    /*const*/ GI_ref_weak first() /*const*/;
+    /*const*/ GI_ref_weak operator[](int idx) /*const*/;
+    size_t size() /*const*/;
+    bool contains(/*const*/ GI_ref_weak gif) /*const*/;
+    void iterate_edges(std::function<bool(Edge &)> visitor) /*const*/;
+    /*const*/ std::vector</*const*/ GI_ref_weak> &get_path() /*const*/;
+    size_t index(/*const*/ GI_ref_weak gif) /*const*/;
+
+    std::string str() const;
+};
 
 class Graph {
     Set<GI_ref> v;
@@ -176,6 +250,9 @@ class Graph {
 
     std::string repr();
 
+    Set<GI_ref> get_gifs();
+    std::vector<std::tuple<GI_ref_weak, GI_ref_weak, Link_ref>> all_edges();
+
     // Algorithms
     std::unordered_set<Node_ref> node_projection();
     std::vector<std::pair<Node_ref, std::string>>