Merge branch 'develop' of github.com:opsmill/infrahub into ple-permis…

…sions-ui-graphql-IFC-657

backend/infrahub/core/validators/uniqueness/model.py

@@ -44,6 +44,20 @@ def __bool__(self) -> bool:
             return True
         return False
 
+    def __str__(self) -> str:
+        return (
+            "ATTRS: "
+            + "; ".join(
+                q.attribute_name + " " + str(q.property_name) + " " + (str(q.value) if q.value is not None else "")
+                for q in self.unique_attribute_paths
+            )
+            + " RELS: "
+            + "; ".join(
+                q.identifier + " " + str(q.attribute_name) + " " + (str(q.value) if q.value is not None else "")
+                for q in self.relationship_attribute_paths
+            )
+        )
+
 
 class NonUniqueRelatedAttribute(BaseModel):
     relationship: RelationshipSchema

backend/tests/conftest.py

@@ -48,6 +48,7 @@
     NEO4J_IMAGE,
     PORT_BOLT_NEO4J,
     PORT_CLIENT_RABBITMQ,
+    PORT_HTTP_NEO4J,
     PORT_HTTP_RABBITMQ,
     PORT_MEMGRAPH,
     PORT_NATS,
@@ -183,7 +184,10 @@ def neo4j(request: pytest.FixtureRequest, load_settings_before_session) -> Optio
     container = start_neo4j_container(NEO4J_IMAGE)
     request.addfinalizer(container.stop)
 
-    return {PORT_BOLT_NEO4J: get_exposed_port(container, PORT_BOLT_NEO4J)}
+    return {
+        PORT_BOLT_NEO4J: get_exposed_port(container, PORT_BOLT_NEO4J),
+        PORT_HTTP_NEO4J: get_exposed_port(container, PORT_HTTP_NEO4J),
+    }
 
 
 @pytest.fixture(scope="session")

backend/tests/helpers/constants.py

@@ -7,6 +7,7 @@
 PORT_REDIS = 6379
 PORT_CLIENT_RABBITMQ = 5672
 PORT_HTTP_RABBITMQ = 15672
+PORT_HTTP_NEO4J = 7474
 PORT_BOLT_NEO4J = 7687
 PORT_MEMGRAPH = 7687
 PORT_PREFECT = 4200

backend/tests/helpers/query_benchmark/car_person_generators.py

@@ -1,6 +1,6 @@
 import random
 import uuid
-from typing import Optional
+from typing import Optional, Tuple
 
 from infrahub.core import registry
 from infrahub.core.node import Node
@@ -44,7 +44,11 @@ class PersonGenerator(DataGenerator):
     async def load_data(self, nb_elements: int) -> None:
         await self.load_persons(nb_persons=nb_elements)
 
-    async def load_persons(self, nb_persons: int, cars: Optional[dict[str, Node]] = None) -> dict[str, Node]:
+    async def load_persons(
+        self,
+        nb_persons: int,
+        cars: Optional[dict[str, Node]] = None,
+    ) -> dict[str, Node]:
         """
         Load persons and return a mapping person_name -> person_node.
         If 'cars' is specified, each person created is linked to a few random cars.
@@ -58,6 +62,7 @@ async def load_persons(self, nb_persons: int, cars: Optional[dict[str, Node]] =
             short_id = str(uuid.uuid4())[:8]
             person_name = f"person-{short_id}"
             person_node = await Node.init(db=self.db, schema=person_schema, branch=default_branch)
+
             if cars is not None:
                 random_cars = [cars[car_name] for car_name in random.choices(list(cars.keys()), k=5)]
                 await person_node.new(db=self.db, name=person_name, cars=random_cars)
@@ -90,6 +95,67 @@ async def load_data(self, nb_elements: int) -> None:
         await self.load_persons(nb_persons=nb_elements, cars=self.cars)
 
 
+class CarFromExistingPersonGenerator(CarGenerator):
+    persons: Optional[dict[str, Node]]  # mapping of existing cars names -> node
+    nb_persons: int
+
+    def __init__(self, db: InfrahubDatabaseProfiler, nb_persons: int) -> None:
+        super().__init__(db)
+        self.nb_persons = nb_persons
+        self.persons = None
+
+    async def init(self) -> None:
+        """Load persons, that will be later connected to generated cars."""
+        self.persons = await PersonGenerator(self.db).load_persons(nb_persons=self.nb_persons)
+
+    async def load_data(self, nb_elements: int) -> None:
+        assert self.persons is not None, "'init' method should be called before 'load_data'"
+        await self.load_cars(nb_cars=nb_elements, persons=self.persons)
+
+
+class CarGeneratorWithOwnerHavingUniqueCar(CarGenerator):
+    persons: list[Tuple[str, Node]]  # mapping of existing cars names -> node
+    nb_persons: int
+    nb_cars_loaded: int
+
+    def __init__(self, db: InfrahubDatabaseProfiler, nb_persons: int) -> None:
+        super().__init__(db)
+        self.nb_persons = nb_persons
+        self.persons = []
+        self.nb_cars_loaded = 0
+
+    async def init(self) -> None:
+        """Load persons, that will be later connected to generated cars."""
+        persons = await PersonGenerator(self.db).load_persons(nb_persons=self.nb_persons)
+        self.persons = list(persons.items())
+
+    async def load_data(self, nb_elements: int) -> None:
+        """
+        Generate cars with an owner, in a way that an owner can't have multiple cars.
+        Also generate distinct nb_seats per car.
+        """
+
+        default_branch = await registry.get_branch(db=self.db)
+        car_schema = registry.schema.get_node_schema(name="TestCar", branch=default_branch)
+
+        for i in range(nb_elements):
+            short_id = str(uuid.uuid4())[:8]
+            car_name = f"car-{short_id}"
+            car_node = await Node.init(db=self.db, schema=car_schema, branch=default_branch)
+
+            await car_node.new(
+                db=self.db,
+                name=car_name,
+                nbr_seats=self.nb_cars_loaded + i,
+                owner=self.persons[self.nb_cars_loaded + i][1],
+            )
+
+            async with self.db.start_session():
+                await car_node.save(db=self.db)
+
+        self.nb_cars_loaded += nb_elements
+
+
 class CarAndPersonIsolatedGenerator(DataGenerator):
     def __init__(self, db: InfrahubDatabaseProfiler) -> None:
         super().__init__(db)

backend/tests/helpers/query_benchmark/data_generator.py

@@ -5,7 +5,10 @@
 from rich.console import Console
 from rich.progress import Progress
 
-from tests.helpers.query_benchmark.db_query_profiler import InfrahubDatabaseProfiler, ProfilerEnabler, QueryAnalyzer
+from tests.helpers.query_benchmark.db_query_profiler import (
+    InfrahubDatabaseProfiler,
+    ProfilerEnabler,
+)
 
 
 class DataGenerator:
@@ -33,7 +36,6 @@ async def load_data_and_profile(
     profile_frequency: int,
     graphs_output_location: Path,
     test_label: str,
-    query_analyzer: QueryAnalyzer,
     memory_profiling_rate: int = 25,
 ) -> None:
     """
@@ -54,11 +56,11 @@ async def load_data_and_profile(
 
     await data_generator.init()
 
-    query_analyzer.reset()
-
     q, r = divmod(nb_elements, profile_frequency)
     nb_elem_per_batch = [profile_frequency] * q + ([r] if r else [])
 
+    query_analyzer = data_generator.db.query_analyzer
+
     with Progress(console=Console(force_terminal=True)) as progress:  # Need force_terminal to display with pytest
         task = progress.add_task(
             f"Loading elements from {data_generator.__class__.__name__}", total=len(nb_elem_per_batch)

backend/tests/helpers/query_benchmark/db_query_profiler.py

@@ -6,17 +6,29 @@
 
 import matplotlib.pyplot as plt
 import pandas as pd
-from infrahub_sdk import Timestamp
 from neo4j import Record
 
+from infrahub.config import SETTINGS
+
 # pylint: skip-file
 from infrahub.database import InfrahubDatabase
 from infrahub.database.constants import Neo4jRuntime
 from infrahub.log import get_logger
+from tests.helpers.constants import NEO4J_ENTERPRISE_IMAGE
 
 log = get_logger()
 
 
+@dataclass
+class BenchmarkConfig:
+    neo4j_image: str = NEO4J_ENTERPRISE_IMAGE
+    neo4j_runtime: Neo4jRuntime = Neo4jRuntime.DEFAULT
+    load_db_indexes: bool = False
+
+    def __str__(self) -> str:
+        return f"{self.neo4j_image=} ; runtime: {self.neo4j_runtime} ; indexes: {self.load_db_indexes}"
+
+
 @dataclass
 class QueryMeasurement:
     duration: float
@@ -27,56 +39,27 @@ class QueryMeasurement:
 
 
 class QueryAnalyzer:
-    _start_time: Optional[Timestamp]
     name: Optional[str]
-    count: int
     measurements: list[QueryMeasurement]
-    count_per_query: dict[str, int]
-    _df: Optional[pd.DataFrame]
-    measure_memory_usage: bool
-    sampling_memory_usage: int
     output_location: Path
-    neo4j_runtime: Neo4jRuntime
     nb_elements_loaded: int
-    query_to_nb_elts_loaded_to_measurements: dict[str, dict[int, QueryMeasurement]]
     profile_memory: bool
     profile_duration: bool
 
     def __init__(self) -> None:
         self.reset()
 
     def reset(self) -> None:
-        self._start_time = Timestamp()
         self.name = None
-        self.count = 0
         self.measurements = []
-        self.query_to_nb_elts_loaded_to_measurements = {}
-        self._df = None
         self.output_location = Path.cwd()
-        self.neo4j_runtime = Neo4jRuntime.DEFAULT
         self.nb_elements_loaded = 0
         self.profile_duration = False
         self.profile_memory = False
 
     def increase_nb_elements_loaded(self, increment: int) -> None:
         self.nb_elements_loaded += increment
 
-    @property
-    def start_time(self) -> Timestamp:
-        if self._start_time:
-            return self._start_time
-        raise ValueError("start_time hasnt't been initialized yet")
-
-    def create_directory(self, prefix: str, output_location: Path) -> Path:
-        time_str = self.start_time.to_string()
-        for char in [":", "-", "."]:
-            time_str = time_str.replace(char, "_")
-        directory_name = f"{time_str}_{prefix}"
-        full_directory = output_location / directory_name
-        if not full_directory.exists():
-            full_directory.mkdir(parents=True)
-        return full_directory
-
     def get_df(self) -> pd.DataFrame:
         data = {}
         for item in QueryMeasurement.__dataclass_fields__.keys():
@@ -97,7 +80,7 @@ def create_graphs(self, output_location: Path, label: str) -> None:
 
         for query_name in query_names:
             self.create_duration_graph(query_name=query_name, label=label, output_dir=output_location)
-            self.create_memory_graph(query_name=query_name, label=label, output_dir=output_location)
+            # self.create_memory_graph(query_name=query_name, label=label, output_dir=output_location)
 
     def create_duration_graph(self, query_name: str, label: str, output_dir: Path) -> None:
         metric = "duration"
@@ -113,14 +96,14 @@ def create_duration_graph(self, query_name: str, label: str, output_dir: Path) -
         y = df_query[metric].values * 1000
         plt.plot(x, y, label=label)
 
-        plt.legend()
+        plt.legend(bbox_to_anchor=(1.04, 1), borderaxespad=0)
 
         plt.ylabel("msec", fontsize=15)
         plt.title(f"Query - {query_name} | {metric}", fontsize=20)
         plt.grid()
 
         file_name = f"{name}.png"
-        plt.savefig(str(output_dir / file_name))
+        plt.savefig(str(output_dir / file_name), bbox_inches="tight")
 
     def create_memory_graph(self, query_name: str, label: str, output_dir: Path) -> None:
         metric = "memory"
@@ -168,9 +151,9 @@ def __exit__(
 
 
 class InfrahubDatabaseProfiler(InfrahubDatabase):
-    def __init__(self, query_analyzer: QueryAnalyzer, **kwargs: Any) -> None:
+    def __init__(self, **kwargs: Any) -> None:
         super().__init__(**kwargs)
-        self.query_analyzer = query_analyzer
+        self.query_analyzer = QueryAnalyzer()
         # Note that any attribute added here should be added to get_context method.
 
     def get_context(self) -> dict[str, Any]:
@@ -193,11 +176,15 @@ async def execute_query_with_metadata(
         else:
             profile_memory = False
 
+        assert profile_memory is False, "Do not profile memory for now"
+
         # Do the query and measure duration
         time_start = time.time()
         response, metadata = await super().execute_query_with_metadata(query, params, name)
         duration_time = time.time() - time_start
 
+        assert len(response) < SETTINGS.database.query_size_limit // 2, "make sure data return is small"
+
         measurement = QueryMeasurement(
             duration=duration_time,
             memory=metadata["profile"]["args"]["GlobalMemory"] if profile_memory else None,

backend/tests/helpers/utils.py

@@ -1,7 +1,7 @@
 from testcontainers.core.container import DockerContainer
 from testcontainers.core.waiting_utils import wait_for_logs
 
-from tests.helpers.constants import PORT_BOLT_NEO4J
+from tests.helpers.constants import PORT_BOLT_NEO4J, PORT_HTTP_NEO4J
 
 
 def get_exposed_port(container: DockerContainer, port: int) -> int:
@@ -23,6 +23,7 @@ def start_neo4j_container(neo4j_image: str) -> DockerContainer:
         .with_env("NEO4J_dbms_security_procedures_unrestricted", "apoc.*")
         .with_env("NEO4J_dbms_security_auth__minimum__password__length", "4")
         .with_exposed_ports(PORT_BOLT_NEO4J)
+        .with_exposed_ports(PORT_HTTP_NEO4J)
     )
 
     container.start()