hop graph model improvements. apache#3032

bamaer · Nov 17, 2023 · 4c9596d · 4c9596d
1 parent ba86833
commit 4c9596d
Show file tree

Hide file tree

Showing 17 changed files with 98 additions and 34 deletions.
diff --git a/core/src/main/java/org/apache/hop/core/graph/BaseBoltGraphDatabaseMeta.java b/core/src/main/java/org/apache/hop/core/graph/BaseBoltGraphDatabaseMeta.java
@@ -22,6 +22,8 @@
 import org.apache.hop.core.encryption.Encr;
 import org.apache.hop.core.exception.HopConfigException;
 import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.graph.model.GraphNode;
+import org.apache.hop.core.graph.model.GraphRelationship;
 import org.apache.hop.core.gui.plugin.GuiElementType;
 import org.apache.hop.core.gui.plugin.GuiWidgetElement;
 import org.apache.hop.core.logging.ILogChannel;
@@ -41,7 +43,6 @@
 import org.neo4j.driver.Session;
 import org.neo4j.driver.SessionConfig;
 import org.neo4j.driver.Value;
-
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URLEncoder;
@@ -813,6 +814,30 @@ public void writeUnwindData(
             String[] toPropNames,
             boolean[] toPropPrimary
     ){
+    }
+
+    @Override
+    public void writeToGraph(
+            List<GraphNode> fromNodes,
+            List<GraphNode> toNodes,
+            List<Object[]> fromNodeProperties,
+            List<Object[]> toNodeProperties,
+            List<GraphRelationship> graphRelationships,
+            List<Object[]> relationshipProperties
+    ){
+        String dummy = "";
+    }
+
+    @Override
+    public void writeNode(
+            List<GraphNode> fromNodes,
+            List<Object[]> fromNodeProperties
+    ){
+        StringBuilder cypher = new StringBuilder();
+
+        cypher.append("CREATE(");
+
+
 
     }
 

diff --git a/core/src/main/java/org/apache/hop/core/graph/BaseGraphDatabaseMeta.java b/core/src/main/java/org/apache/hop/core/graph/BaseGraphDatabaseMeta.java
@@ -17,6 +17,8 @@
 
 package org.apache.hop.core.graph;
 
+import org.apache.hop.core.graph.model.GraphNode;
+import org.apache.hop.core.graph.model.GraphRelationship;
 import org.apache.hop.core.gui.plugin.GuiElementType;
 import org.apache.hop.core.gui.plugin.GuiWidgetElement;
 import org.apache.hop.core.logging.ILogChannel;
@@ -244,4 +246,21 @@ public List<String> getSupportedProtocols(ILogChannel log, IHopMetadataProvider
         return protocols;
     }
 
+    @Override
+    public void writeToGraph(
+            List<GraphNode> fromNodes,
+            List<GraphNode> toNodes,
+            List<Object[]> fromNodeProperties,
+            List<Object[]> toNodeProperties,
+            List<GraphRelationship> graphRelationships,
+            List<Object[]> relationshipProperties
+    ){}
+
+    @Override
+    public void writeNode(
+            List<GraphNode> nodes,
+            List<Object[]> nodeProperties
+    ){
+    }
+
 }
diff --git a/core/src/main/java/org/apache/hop/core/graph/IGraphDatabase.java b/core/src/main/java/org/apache/hop/core/graph/IGraphDatabase.java
@@ -19,6 +19,8 @@
 
 import org.apache.hop.core.exception.HopConfigException;
 import org.apache.hop.core.exception.HopException;
+import org.apache.hop.core.graph.model.GraphNode;
+import org.apache.hop.core.graph.model.GraphRelationship;
 import org.apache.hop.core.variables.IVariables;
 import org.apache.hop.metadata.api.HopMetadataObject;
 
@@ -129,4 +131,17 @@ public interface IGraphDatabase extends Cloneable{
 
     void writeData(IVariables variables, String query, Map<String, Object> properties);
 
+    void writeToGraph(
+            List<GraphNode> fromNodes,
+            List<GraphNode> toNodes,
+            List<Object[]> fromNodeProperties,
+            List<Object[]> toNodeProperties,
+            List<GraphRelationship> graphRelationships,
+            List<Object[]> relationshipProperties
+    );
+
+    void writeNode(
+            List<GraphNode> nodes,
+            List<Object[]> nodeProperties
+    );
 }
diff --git a/.../hop/graphdatabases/core/GraphEntity.java → ...che/hop/core/graph/model/GraphEntity.java b/.../hop/graphdatabases/core/GraphEntity.java → ...che/hop/core/graph/model/GraphEntity.java
@@ -1,12 +1,7 @@
-package org.apache.hop.graphdatabases.core;
+package org.apache.hop.core.graph.model;
 
 
-import org.apache.hop.core.row.IValueMeta;
-
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 public abstract class GraphEntity implements IGraphEntity{
 

diff --git a/...he/hop/graphdatabases/core/GraphNode.java → ...pache/hop/core/graph/model/GraphNode.java b/...he/hop/graphdatabases/core/GraphNode.java → ...pache/hop/core/graph/model/GraphNode.java
@@ -1,10 +1,7 @@
-package org.apache.hop.graphdatabases.core;
-
-import org.apache.hop.core.row.IValueMeta;
+package org.apache.hop.core.graph.model;
 
 import java.util.Collection;
 import java.util.List;
-import java.util.Map;
 
 public class GraphNode extends GraphEntity implements IGraphNode{
 

diff --git a/...tabases/core/GraphNodePropertiesMeta.java → .../graph/model/GraphNodePropertiesMeta.java b/...tabases/core/GraphNodePropertiesMeta.java → .../graph/model/GraphNodePropertiesMeta.java
@@ -1,7 +1,6 @@
-package org.apache.hop.graphdatabases.core;
+package org.apache.hop.core.graph.model;
 
 import org.apache.hop.core.row.IValueMeta;
-import org.apache.hop.core.variables.IVariables;
 
 import java.util.List;
 

diff --git a/...phdatabases/core/GraphPropertiesMeta.java → ...core/graph/model/GraphPropertiesMeta.java b/...phdatabases/core/GraphPropertiesMeta.java → ...core/graph/model/GraphPropertiesMeta.java
@@ -1,7 +1,6 @@
-package org.apache.hop.graphdatabases.core;
+package org.apache.hop.core.graph.model;
 
 import org.apache.hop.core.row.IValueMeta;
-import org.apache.hop.core.variables.IVariables;
 
 public abstract class GraphPropertiesMeta implements IGraphProperties{
 

diff --git a/...raphdatabases/core/GraphRelationship.java → ...p/core/graph/model/GraphRelationship.java b/...raphdatabases/core/GraphRelationship.java → ...p/core/graph/model/GraphRelationship.java
@@ -1,9 +1,6 @@
-package org.apache.hop.graphdatabases.core;
-
-import org.apache.hop.core.row.IValueMeta;
+package org.apache.hop.core.graph.model;
 
 import java.util.List;
-import java.util.Map;
 
 public class GraphRelationship extends GraphEntity implements IGraphRelationship {
 

diff --git a/...core/GraphRelationshipPropertiesMeta.java → ...odel/GraphRelationshipPropertiesMeta.java b/...core/GraphRelationshipPropertiesMeta.java → ...odel/GraphRelationshipPropertiesMeta.java
@@ -1,4 +1,4 @@
-package org.apache.hop.graphdatabases.core;
+package org.apache.hop.core.graph.model;
 
 import org.apache.hop.core.row.IValueMeta;
 

diff --git a/...e/hop/graphdatabases/core/GraphUsage.java → ...ache/hop/core/graph/model/GraphUsage.java b/...e/hop/graphdatabases/core/GraphUsage.java → ...ache/hop/core/graph/model/GraphUsage.java
@@ -16,7 +16,7 @@
  *
  */
 
-package org.apache.hop.graphdatabases.core;
+package org.apache.hop.core.graph.model;
 
 public enum GraphUsage {
   NODE_CREATE,

diff --git a/...hop/graphdatabases/core/IGraphEntity.java → ...he/hop/core/graph/model/IGraphEntity.java b/...hop/graphdatabases/core/IGraphEntity.java → ...he/hop/core/graph/model/IGraphEntity.java
@@ -1,4 +1,4 @@
-package org.apache.hop.graphdatabases.core;
+package org.apache.hop.core.graph.model;
 
 import org.apache.hop.core.row.IValueMeta;
 

diff --git a/...e/hop/graphdatabases/core/IGraphNode.java → ...ache/hop/core/graph/model/IGraphNode.java b/...e/hop/graphdatabases/core/IGraphNode.java → ...ache/hop/core/graph/model/IGraphNode.java
@@ -1,4 +1,4 @@
-package org.apache.hop.graphdatabases.core;
+package org.apache.hop.core.graph.model;
 
 public interface IGraphNode extends IGraphEntity{
 

diff --git a/...p/graphdatabases/core/IGraphNodeMeta.java → .../hop/core/graph/model/IGraphNodeMeta.java b/...p/graphdatabases/core/IGraphNodeMeta.java → .../hop/core/graph/model/IGraphNodeMeta.java
@@ -1,4 +1,4 @@
-package org.apache.hop.graphdatabases.core;
+package org.apache.hop.core.graph.model;
 
 import java.util.List;
 

diff --git a/...graphdatabases/core/IGraphProperties.java → ...op/core/graph/model/IGraphProperties.java b/...graphdatabases/core/IGraphProperties.java → ...op/core/graph/model/IGraphProperties.java
@@ -1,4 +1,4 @@
-package org.apache.hop.graphdatabases.core;
+package org.apache.hop.core.graph.model;
 
 import org.apache.hop.core.row.IValueMeta;
 

diff --git a/...aphdatabases/core/IGraphRelationship.java → .../core/graph/model/IGraphRelationship.java b/...aphdatabases/core/IGraphRelationship.java → .../core/graph/model/IGraphRelationship.java
@@ -1,4 +1,4 @@
-package org.apache.hop.graphdatabases.core;
+package org.apache.hop.core.graph.model;
 
 public interface IGraphRelationship extends IGraphEntity{
 

diff --git a/...atabases/core/IGraphRelationshipMeta.java → ...e/graph/model/IGraphRelationshipMeta.java b/...atabases/core/IGraphRelationshipMeta.java → ...e/graph/model/IGraphRelationshipMeta.java
@@ -1,4 +1,4 @@
-package org.apache.hop.graphdatabases.core;
+package org.apache.hop.core.graph.model;
 
 public interface IGraphRelationshipMeta extends IGraphProperties{
 

diff --git a/...hdatabases/src/main/java/org/apache/hop/graphdatabases/transforms/output/GraphOutput.java b/...hdatabases/src/main/java/org/apache/hop/graphdatabases/transforms/output/GraphOutput.java
@@ -25,16 +25,14 @@
 import org.apache.hop.core.graph.IGraphDatabase;
 import org.apache.hop.core.row.IRowMeta;
 import org.apache.hop.core.row.IValueMeta;
-import org.apache.hop.core.row.RowDataUtil;
 import org.apache.hop.core.row.value.ValueMetaDate;
 import org.apache.hop.core.row.value.ValueMetaInteger;
 import org.apache.hop.core.row.value.ValueMetaString;
-import org.apache.hop.core.util.StringUtil;
-import org.apache.hop.graphdatabases.core.GraphNodePropertiesMeta;
-import org.apache.hop.graphdatabases.core.GraphRelationship;
-import org.apache.hop.graphdatabases.core.GraphRelationshipPropertiesMeta;
-import org.apache.hop.graphdatabases.core.GraphUsage;
-import org.apache.hop.graphdatabases.core.GraphNode;
+import org.apache.hop.core.graph.model.GraphNodePropertiesMeta;
+import org.apache.hop.core.graph.model.GraphRelationship;
+import org.apache.hop.core.graph.model.GraphRelationshipPropertiesMeta;
+import org.apache.hop.core.graph.model.GraphUsage;
+import org.apache.hop.core.graph.model.GraphNode;
 import org.apache.hop.graphdatabases.model.GraphPropertyType;
 import org.apache.hop.graphdatabases.shared.GraphConnectionUtils;
 import org.apache.hop.graphdatabases.transforms.BaseGraphTransform;
@@ -58,6 +56,7 @@ public class GraphOutput extends BaseGraphTransform<GraphOutputMeta, GraphOutput
   private GraphRelationshipPropertiesMeta relationshipPropertiesMeta;
   private List<GraphNode> fromNodes, toNodes;
   private List<GraphRelationship> relationships;
+  private GraphOutputMeta meta;
   private String[] fromNodePropertyNames, toNodePropertyNames;
   private Object[] fromNodePropertyValues, toNodePropertyValues;
 //  private Map<String, IValueMeta> fromNodeProps;
@@ -86,6 +85,8 @@ public GraphOutput(
     super(s, meta, data, c, t, dis);
     fromNodes = new ArrayList<>();
     toNodes = new ArrayList<>();
+    relationships = new ArrayList<>();
+    this.meta = meta;
 //    fromNodeProps = new HashMap<>();
 //    toNodeProps = new HashMap<>();
 //
@@ -422,8 +423,10 @@ public boolean processRow() throws HopException {
 
     } else {
 
+
       boolean changedLabel = calculateLabelsAndDetectChanges(row);
       if (changedLabel || data.unwindList.size() >= data.batchSize) {
+        writeToGraph();
         emptyUnwindList();
       }
 
@@ -547,6 +550,20 @@ private void addPropertiesToMap(
     }
   }
 
+  private void writeToGraph() throws HopException {
+
+    IGraphDatabase graphDatabase = data.graphDatabaseMeta.getIGraphDatabase();
+    graphDatabase.writeNode(fromNodes, fromNodePropertyObjects);
+//    graphDatabase.writeToGraph(
+//            fromNodes,
+//            toNodes,
+//            fromNodePropertyObjects,
+//            toNodePropertyObjects,
+//            relationships,
+//            relationshipPropertyObjects
+//    );
+
+  }
   private void emptyUnwindList() throws HopException {
 
     try {
@@ -1240,6 +1257,7 @@ private void wrapUpTransaction() throws HopException {
 
     if (!isStopped()) {
       if (data.unwindList != null && data.unwindList.size() > 0) {
+        writeToGraph();
         emptyUnwindList(); // force write!
       }
     }