log of cycles, clustering of isolated graphs

OsgiliathEnterprise · Oct 12, 2024 · 180c6fa · 180c6fa
1 parent e81fbc1
commit 180c6fa
Show file tree

Hide file tree

Showing 14 changed files with 4,265 additions and 44 deletions.
diff --git a/core/src/main/java/net/osgiliath/migrator/core/TransformationSequencer.java b/core/src/main/java/net/osgiliath/migrator/core/TransformationSequencer.java
@@ -80,7 +80,7 @@ public void run(String... args) {
         Graph<MetamodelVertex, FieldEdge<MetamodelVertex>> fullEntityMetamodelGraph = graphBuilder.metamodelGraphFromRawElementClasses(metamodelClasses);
         graphRequester.displayGraphWithGraphiz(fullEntityMetamodelGraph);
         Collection<Graph<MetamodelVertex, FieldEdge<MetamodelVertex>>> clusteredEntityMetamodelGraph = graphBuilder.clusterGraphs(fullEntityMetamodelGraph);
-        clusteredEntityMetamodelGraph.stream().parallel().forEach(
+        clusteredEntityMetamodelGraph.forEach(
                 metamodelGraph -> {
                     if (metamodelGraph.edgeSet().isEmpty() && dataMigratorConfiguration.getSequence().isEmpty()) {
                         streamingInjector.injectVerticesInTargetDb(metamodelGraph.vertexSet());// Just a simple migration,

diff --git a/core/src/main/java/net/osgiliath/migrator/core/db/inject/SinkEntityInjector.java b/core/src/main/java/net/osgiliath/migrator/core/db/inject/SinkEntityInjector.java
@@ -70,18 +70,28 @@ public void persist(GraphTraversalSource modelGraph, Graph<MetamodelVertex, Fiel
     }
 
     private void processEntitiesWithoutCycles(GraphTraversalSource modelGraph, Graph<MetamodelVertex, FieldEdge<MetamodelVertex>> entityMetamodelGraph, Collection<Vertex> processedVertices) {
-        GraphTraversal leafElements = modelGraph.V()
+        GraphTraversal leafElements = modelGraph
+                .V()
                 .repeat(out())
                 .until(
-                        out().filter(not(is(P.within(processedVertices)))).count().is(0)// .or().loops().is(CYCLE_DETECTION_DEPTH)
-                )
-                .filter(not(is(P.within(processedVertices))));
-        if (!leafElements.hasNext()) {
-            vertexPersister.persistVertices(modelGraph.V().filter(not(is(P.within(processedVertices)))).toStream()
-                    .map(modelVertex -> vertexResolver.getModelElement(modelVertex)));
-            return;
+                        out().filter(not(is(P.within(processedVertices)))).count().is(0)
+                ).filter(not(is(P.within(processedVertices))));// .or().loops().is(CYCLE_DETECTION_DEPTH)
+        if (leafElements.hasNext()) {
+            persistTraversal(entityMetamodelGraph, processedVertices, leafElements);
+            processEntitiesWithoutCycles(modelGraph, entityMetamodelGraph, processedVertices);
+        } else {
+            GraphTraversal orphansElements = modelGraph.V().filter(not(is(P.within(processedVertices)))).filter(out().count().is(0));
+            if (orphansElements.hasNext()) {
+                persistTraversal(entityMetamodelGraph, processedVertices, orphansElements);
+                processEntitiesWithoutCycles(modelGraph, entityMetamodelGraph, processedVertices);
+            } else {
+                persistTraversal(entityMetamodelGraph, processedVertices, modelGraph.V().filter(not(is(P.within(processedVertices)))));
+            }
         }
-        Stream<Vertex> res = leafElements.toStream()
+    }
+
+    private void persistTraversal(Graph<MetamodelVertex, FieldEdge<MetamodelVertex>> entityMetamodelGraph, Collection<Vertex> processedVertices, GraphTraversal traversal) {
+        Stream<ModelElement> res = traversal.toStream()
                 .map(e -> {
                     Vertex modelVertex = (Vertex) e;
                     updateRawElementRelationshipsAccordingToGraphEdges(modelVertex, entityMetamodelGraph);
@@ -90,14 +100,14 @@ private void processEntitiesWithoutCycles(GraphTraversalSource modelGraph, Graph
                 .peek(mv -> {
                     Vertex tv = (Vertex) mv;
                     log.info("Persisting vertex of type {} with id {}", tv.label(), vertexResolver.getVertexModelElementId(tv));
-                });
-        vertexPersister.persistVertices(res
-                .map(modelVertex -> {
-                    processedVertices.add(modelVertex);
-                    return modelVertex;
                 })
-                .map(modelVertex -> vertexResolver.getModelElement(modelVertex)));
-        processEntitiesWithoutCycles(modelGraph, entityMetamodelGraph, processedVertices);
+                .map(m -> {
+                    Vertex v = (Vertex) m;
+                    processedVertices.add(v);
+                    return v;
+                })
+                .map(me -> vertexResolver.getModelElement((Vertex) me));
+        vertexPersister.persistVertices(res);
     }
 
     void updateRawElementRelationshipsAccordingToGraphEdges(Vertex sourceVertex, Graph<MetamodelVertex, FieldEdge<MetamodelVertex>> entityMetamodelGraph) {
@@ -125,7 +135,25 @@ private void removeCyclicElements(GraphTraversalSource modelGraph) {
                 .repeat(out())
                 .until(where(eq("a"))
                         .or().loops().is(CYCLE_DETECTION_DEPTH))
-                .filter(where(eq("a"))).drop().iterate();
+                .filter(where(eq("a")))
+                .filter(t -> {
+                    Vertex v = t.get();
+                    log.warn("Cyclic element of type {} with id {} found in the graph", v.label(), vertexResolver.getVertexModelElementId(v));
+                    return true;
+                })
+                .inE()
+                .drop().iterate();
+        modelGraph.V().as("a")
+                .repeat(out())
+                .until(where(eq("a"))
+                        .or().loops().is(CYCLE_DETECTION_DEPTH))
+                .filter(where(eq("a")))
+                .filter(t -> {
+                    Vertex v = t.get();
+                    log.warn("Cyclic element of type {} with id {} found in the graph", v.label(), vertexResolver.getVertexModelElementId(v));
+                    return true;
+                })
+                .drop().iterate();
 /*        cyclicElements.toStream()
                 .peek(v -> {
                     Vertex ve = (Vertex) v;

diff --git a/core/src/main/java/net/osgiliath/migrator/core/processing/SequenceProcessor.java b/core/src/main/java/net/osgiliath/migrator/core/processing/SequenceProcessor.java
@@ -72,7 +72,6 @@ public void process(GraphTraversalSource modelGraph, Graph<MetamodelVertex, Fiel
         TransactionTemplate tpl = new TransactionTemplate(sourcePlatformTxManager);
         tpl.setReadOnly(true);
         tpl.executeWithoutResult(status -> { // TODO refine
-
             dataMigratorConfiguration.getSequence().stream()
                     .flatMap(sequenceName -> dataMigratorConfiguration.getSequencers().stream().filter(seq -> seq.getName().equals(sequenceName)))
                     .map(seq -> {

diff --git a/core/src/main/java/net/osgiliath/migrator/core/rawelement/jpa/JpaEntityProcessor.java b/core/src/main/java/net/osgiliath/migrator/core/rawelement/jpa/JpaEntityProcessor.java
@@ -146,11 +146,13 @@ public Optional<Object> getId(MetamodelVertex metamodelVertex, Object entity) {
     private Optional<Object> getRawId(Class entityClass, Object entity) {
         // Cannot use getRawFieldValue due to cycle and the @Transactional aspect
         // log.debug("getting id for entity of class {} and entity {}", entityClass.getSimpleName(), entity);
-        return getPrimaryKeyGetterMethod(entityClass).map(
+        return getPrimaryKeyGetterMethod(entity.getClass()).map(// TODO investigate why entityClass doesn't work as expected
                 primaryKeyGetterMethod -> {
                     try {
                         if (null != entity) {
                             return primaryKeyGetterMethod.invoke(entity);
+                        /*} else if (null != entity && entityClass != entity.getClass()) {
+                            return getRawId(entity.getClass(), entityClass);*/
                         } else {
                             log.error("Cannot get the id because entity is null for entityclass {}", entityClass);
                             return Optional.empty();
@@ -475,11 +477,11 @@ public boolean isFkIgnored(Class<?> entityClass, String attributeName) {
 
     public static boolean joinTableIgnoresFk(JoinTable a1) {
         boolean ignores = a1.foreignKey().value().equals(ConstraintMode.NO_CONSTRAINT);
-        return a1.inverseForeignKey().value().equals(ConstraintMode.NO_CONSTRAINT) || Stream.of(a1.joinColumns()).anyMatch(jc -> joinColumnIgnoresFk(jc));
+        return ignores || a1.foreignKey().value().equals(ConstraintMode.NO_CONSTRAINT) || Stream.of(a1.joinColumns()).anyMatch(JpaEntityProcessor::joinColumnIgnoresFk);
     }
 
     public static boolean joinColumnsIgnoresFk(JoinColumns a1) {
-        return Stream.of(a1.value()).anyMatch(v -> v.foreignKey().value().equals(ConstraintMode.NO_CONSTRAINT));
+        return Stream.of(a1.value()).anyMatch(JpaEntityProcessor::joinColumnIgnoresFk);
     }
 
     public static boolean joinColumnIgnoresFk(JoinColumn a1) {

diff --git a/...rator-archetype/src/main/resources/archetype-resources/src/main/resources/application.yml b/...rator-archetype/src/main/resources/archetype-resources/src/main/resources/application.yml
@@ -29,12 +29,16 @@ data-migrator:
     #    port: 8182
     #    host: 127.0.0.1
 spring:
+  #  threads:
+  #    virtual:
+  #      enabled: true
   main:
     lazy-initialization: true
   jpa:
     source:
       properties:
         hibernate:
+          enable_lazy_load_no_trans: true
           dialect: @source.jdbc.dialect@
           #        cache:
           #          use_second_level_cache: true
@@ -52,6 +56,7 @@ spring:
     sink:
       properties:
         hibernate:
+          enable_lazy_load_no_trans: true
           dialect: @target.jdbc.dialect@
           id.new_generator_mappings: true
           #        cache:

diff --git a/report-aggregate/pom.xml b/report-aggregate/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>net.osgiliath.datamigrator</groupId>
         <artifactId>data-migrator</artifactId>
-        <version>1.81-SNAPSHOT</version>
+        <version>1.82-SNAPSHOT</version>
     </parent>
     <artifactId>report-aggregate</artifactId>
     <properties>

diff --git a/sample-mono/compose.yml b/sample-mono/compose.yml
@@ -2,7 +2,7 @@ services:
   mysql:
     image: "mysql:latest"
     ports:
-      - "3307:3306"
+      - "3306:3306"
     environment:
       - MYSQL_ROOT_PASSWORD=test
       - MYSQL_DATABASE=test # name of the source db