diff --git a/.idea/compiler.xml b/.idea/compiler.xml index 7333b10b..15cd31f0 100644 --- a/.idea/compiler.xml +++ b/.idea/compiler.xml @@ -1,11 +1,12 @@ + - + @@ -81,26 +82,12 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/csv-editor.xml b/.idea/csv-editor.xml index e04f0cb8..24ceae36 100644 --- a/.idea/csv-editor.xml +++ b/.idea/csv-editor.xml @@ -3,27 +3,6 @@ diff --git a/core/src/main/java/net/osgiliath/migrator/core/configuration/ColumnTransformationDefinition.java b/core/src/main/java/net/osgiliath/migrator/core/configuration/ColumnTransformationDefinition.java new file mode 100644 index 00000000..cb31245d --- /dev/null +++ b/core/src/main/java/net/osgiliath/migrator/core/configuration/ColumnTransformationDefinition.java @@ -0,0 +1,51 @@ +package net.osgiliath.migrator.core.configuration; + +/*- + * #%L + * data-migrator-core + * %% + * Copyright (C) 2024 Osgiliath Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import java.util.HashMap; +import java.util.Map; + +public class ColumnTransformationDefinition { + private String columnName; + + private Map options = new HashMap<>(); + + public String getColumnName() { + return columnName; + } + + public void setColumnName(String columnName) { + this.columnName = columnName; + } + + public Map getOptions() { + Map ret = new HashMap<>(); + for (Map.Entry entry: options.entrySet()) { + String key = entry.getKey(); + ret.put(key.replaceFirst("\\d+\\.", ""), entry.getValue()); + } + return ret; + } + + public void setOptions(Map options) { + this.options = options; + } +} diff --git a/core/src/main/java/net/osgiliath/migrator/core/configuration/DataMigratorConfiguration.java b/core/src/main/java/net/osgiliath/migrator/core/configuration/DataMigratorConfiguration.java index aead4aa2..f9e1777a 100644 --- a/core/src/main/java/net/osgiliath/migrator/core/configuration/DataMigratorConfiguration.java +++ b/core/src/main/java/net/osgiliath/migrator/core/configuration/DataMigratorConfiguration.java @@ -49,7 +49,7 @@ public class DataMigratorConfiguration { /** * List of sequencers being able to handle the sequence. */ - private List sequencers; + private List sequencers; /** * Graph datasource configuration. @@ -103,7 +103,7 @@ public void setSequence(List sequence) { * List of sequencers being able to handle the sequence. * @return the list of sequencers. */ - public List getSequencers() { + public List getSequencers() { return sequencers; } @@ -111,7 +111,7 @@ public List getSequencers() { * List of sequencers being able to handle the sequence. * @param sequencers the list of sequencers. */ - public void setSequencers(List sequencers) { + public void setSequencers(List sequencers) { this.sequencers = sequencers; } } diff --git a/core/src/main/java/net/osgiliath/migrator/core/configuration/TransformationConfigurationDefinition.java b/core/src/main/java/net/osgiliath/migrator/core/configuration/SequencerDefinition.java similarity index 84% rename from core/src/main/java/net/osgiliath/migrator/core/configuration/TransformationConfigurationDefinition.java rename to core/src/main/java/net/osgiliath/migrator/core/configuration/SequencerDefinition.java index 81890adf..51874f8c 100644 --- a/core/src/main/java/net/osgiliath/migrator/core/configuration/TransformationConfigurationDefinition.java +++ b/core/src/main/java/net/osgiliath/migrator/core/configuration/SequencerDefinition.java @@ -26,7 +26,7 @@ /** * Configuration of the transformation sequences. */ -public class TransformationConfigurationDefinition { +public class SequencerDefinition { /** * Name of the transformation sequencer. */ @@ -50,7 +50,7 @@ public class TransformationConfigurationDefinition { /** * Columns to be handled by the transformer. */ - private Collection columns = new HashSet<>(); + private Collection columnTransformationDefinitions = new HashSet<>(); /** * Get name of the sequencer to be referenced by the sequence. @@ -118,15 +118,15 @@ public void setEntityClass(String entityClass) { * Get the columns to be handled by the transformer. * @return */ - public Collection getColumns() { - return columns; + public Collection getColumnTransformationDefinitions() { + return columnTransformationDefinitions; } /** * Set the columns to be handled by the transformer. - * @param columns the columns to be handled by the transformer. + * @param columnTransformationDefinitions the columns to be handled by the transformer. */ - public void setColumns(Collection columns) { - this.columns = columns; + public void setColumnTransformationDefinitions(Collection columnTransformationDefinitions) { + this.columnTransformationDefinitions = columnTransformationDefinitions; } } diff --git a/core/src/main/java/net/osgiliath/migrator/core/metamodel/helper/JpaEntityHelper.java b/core/src/main/java/net/osgiliath/migrator/core/metamodel/helper/JpaEntityHelper.java index 7fba1671..c62722b7 100644 --- a/core/src/main/java/net/osgiliath/migrator/core/metamodel/helper/JpaEntityHelper.java +++ b/core/src/main/java/net/osgiliath/migrator/core/metamodel/helper/JpaEntityHelper.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,11 +20,11 @@ * #L% */ -import net.osgiliath.migrator.core.api.metamodel.RelationshipType; import jakarta.persistence.ManyToMany; import jakarta.persistence.ManyToOne; import jakarta.persistence.OneToMany; import jakarta.persistence.OneToOne; +import net.osgiliath.migrator.core.api.metamodel.RelationshipType; import org.springframework.stereotype.Component; import java.lang.annotation.Annotation; @@ -47,19 +47,20 @@ public class JpaEntityHelper { /** * Assess if the class relationship is derived (not the owner side). - * @param entityClass the entity class. + * + * @param entityClass the entity class. * @param attributeName the attribute name. * @return the entity class name. */ public boolean isDerived(Class entityClass, String attributeName) { try { Method m = entityClass.getDeclaredMethod(fieldToGetter(attributeName)); - for (Annotation a: m.getDeclaredAnnotations()) { + for (Annotation a : m.getDeclaredAnnotations()) { if (a instanceof OneToMany) { return !((OneToMany) a).mappedBy().isEmpty(); } else if (a instanceof ManyToMany) { addEntityClassAsOwningSideIfMappedByIsNotDefinedOnBothSides(entityClass, m); - return !((ManyToMany) a).mappedBy().isEmpty() || randomManyToManyOwningSide.contains(((ParameterizedType)m.getGenericReturnType()).getActualTypeArguments()[0]); + return !((ManyToMany) a).mappedBy().isEmpty() || randomManyToManyOwningSide.contains(((ParameterizedType) m.getGenericReturnType()).getActualTypeArguments()[0]); } else if (a instanceof OneToOne) { return !((OneToOne) a).mappedBy().isEmpty(); } @@ -72,7 +73,8 @@ public boolean isDerived(Class entityClass, String attributeName) { /** * selects the owning side of a many to many relationship. - * @param entityClass the entity class. + * + * @param entityClass the entity class. * @param manyToManyMethod the many to many method. */ private void addEntityClassAsOwningSideIfMappedByIsNotDefinedOnBothSides(Class entityClass, Method manyToManyMethod) { @@ -80,16 +82,16 @@ private void addEntityClassAsOwningSideIfMappedByIsNotDefinedOnBothSides(Class a instanceof ManyToMany) - .anyMatch(a -> !((ManyToMany) a).mappedBy().isEmpty()); + .filter(a -> a instanceof ManyToMany) + .anyMatch(a -> !((ManyToMany) a).mappedBy().isEmpty()); if (isMappedBy) { return; } - Class targetEntityClass = (Class) ((ParameterizedType)manyToManyMethod.getGenericReturnType()).getActualTypeArguments()[0]; - for (Method targetEntityClassMethod: targetEntityClass.getDeclaredMethods()) { + Class targetEntityClass = (Class) ((ParameterizedType) manyToManyMethod.getGenericReturnType()).getActualTypeArguments()[0]; + for (Method targetEntityClassMethod : targetEntityClass.getDeclaredMethods()) { for (Annotation a : targetEntityClassMethod.getDeclaredAnnotations()) { if (a instanceof ManyToMany && ((ManyToMany) a).mappedBy().isEmpty()) { - Class targetEntityClassManyToManyTargetEntity = (Class) ((ParameterizedType)targetEntityClassMethod.getGenericReturnType()).getActualTypeArguments()[0]; + Class targetEntityClassManyToManyTargetEntity = (Class) ((ParameterizedType) targetEntityClassMethod.getGenericReturnType()).getActualTypeArguments()[0]; if (targetEntityClassManyToManyTargetEntity.equals(entityClass) && !randomManyToManyOwningSide.contains(targetEntityClass)) { randomManyToManyOwningSide.add(entityClass); break; @@ -101,34 +103,40 @@ private void addEntityClassAsOwningSideIfMappedByIsNotDefinedOnBothSides(Class entityClass) { + public Optional getPrimaryKeyGetterMethod(Class entityClass) { return Arrays.stream(entityClass.getDeclaredMethods()).filter( m -> Arrays.stream(m.getDeclaredAnnotations()).anyMatch(a -> a instanceof jakarta.persistence.Id) - ).findAny().orElseThrow(() -> new RuntimeException("No getter for primary key in class" + entityClass)); + ).findAny(); } /** * Gets the primary key value. + * * @param entityClass the entity class. - * @param entity the entity. + * @param entity the entity. * @return the primary key value. */ public Object getId(Class entityClass, Object entity) { - Method primaryKeyGetterMethod = getPrimaryKeyGetterMethod(entityClass); - try { - return primaryKeyGetterMethod.invoke(entity); - } catch (Exception e) { - throw new RuntimeException("The primary key getter method couldn't be invoked", e); - } + return getPrimaryKeyGetterMethod(entityClass).map( + primaryKeyGetterMethod -> { + try { + return primaryKeyGetterMethod.invoke(entity); + } catch (Exception e) { + throw new RuntimeException("The primary key getter method couldn't be invoked", e); + } + } + ); } /** * Gets the getter method for a field. + * * @param entityClass the entity class. - * @param attribute the attribute. + * @param attribute the attribute. * @return the getter method. */ public Method getterMethod(Class entityClass, Field attribute) { @@ -138,6 +146,7 @@ public Method getterMethod(Class entityClass, Field attribute) { /** * Gets the getter method name for a field. + * * @param attributeName the attribute name to get the getter name. * @return the getter name. */ @@ -147,6 +156,7 @@ private static String fieldToGetter(String attributeName) { /** * Gets the setter method name for a field. + * * @param attributeName the attribute name to get the setter name. * @return the setter name. */ @@ -156,17 +166,17 @@ private String fieldToSetter(String attributeName) { /** * Gets the primary key field name. + * * @param entityClass the entity class. * @return the primary key field name. */ public String getPrimaryKeyFieldName(Class entityClass) { - String primaryKeyGetterName = getPrimaryKeyGetterMethod(entityClass).getName(); - String primaryKeyFieldName = getterToFieldName(primaryKeyGetterName); - return primaryKeyFieldName; + return getPrimaryKeyGetterMethod(entityClass).map(primaryKeyGetter -> getterToFieldName(primaryKeyGetter.getName())).get(); } /** * Gets the field name from a getter method name. + * * @param getterName the getter name. * @return the field name. */ @@ -176,6 +186,7 @@ private static String getterToFieldName(String getterName) { /** * Gets Relationship type of a relationship between two entities. + * * @param getterMethod the getter method of the relationship. * @return the type of the relationship (one to one, one to many, many to many). */ @@ -195,8 +206,9 @@ public RelationshipType relationshipType(Method getterMethod) { /** * Gets the Setter method for a field + * * @param entityClass the entity class. - * @param field the field. + * @param field the field. * @return the setter method. */ public Optional setterMethod(Class entityClass, Field field) { @@ -206,7 +218,8 @@ public Optional setterMethod(Class entityClass, Field field) { /** * Gets the inverse relationship field. - * @param getterMethod the getter method that targets an entity (relationship). + * + * @param getterMethod the getter method that targets an entity (relationship). * @param targetEntityClass the target entity class. * @return the inverse relationship field. */ @@ -223,34 +236,36 @@ public Optional inverseRelationshipField(Method getterMethod, Class ta /** * Finds the inverse relationship field without mappedBy information. + * * @param targetEntityClass the target entity class. - * @param getterMethod the getter method that targets the target entity. - * @param relationshipType the relationship type. + * @param getterMethod the getter method that targets the target entity. + * @param relationshipType the relationship type. * @return the inverse relationship field. */ private Optional findInverseRelationshipFieldWithoutMappedByInformation(Class targetEntityClass, Method getterMethod, RelationshipType relationshipType) { Class sourceClass = getterMethod.getDeclaringClass(); return Arrays.stream(targetEntityClass.getDeclaredFields()) - .filter((Field field) -> { - if (field.getGenericType().equals(sourceClass)) { - return true; - } else if(Collection.class.isAssignableFrom(field.getType())) { - Class typeOfCollection = (Class) ((ParameterizedType) field.getGenericType()).getActualTypeArguments()[0]; - if (typeOfCollection.equals(sourceClass)) { + .filter((Field field) -> { + if (field.getGenericType().equals(sourceClass)) { return true; + } else if (Collection.class.isAssignableFrom(field.getType())) { + Class typeOfCollection = (Class) ((ParameterizedType) field.getGenericType()).getActualTypeArguments()[0]; + if (typeOfCollection.equals(sourceClass)) { + return true; + } } - } - return false; - }).map((Field field) -> { - Method getterMethodOfField = getterMethod(targetEntityClass, field); - RelationshipType inverseRelationshipType = relationshipType(getterMethodOfField); - return new AbstractMap.SimpleEntry<>(field, inverseRelationshipType); - }).filter(entry -> isInverseRelationshipType(relationshipType, entry.getValue())) - .map(entry -> entry.getKey()).findAny(); + return false; + }).map((Field field) -> { + Method getterMethodOfField = getterMethod(targetEntityClass, field); + RelationshipType inverseRelationshipType = relationshipType(getterMethodOfField); + return new AbstractMap.SimpleEntry<>(field, inverseRelationshipType); + }).filter(entry -> isInverseRelationshipType(relationshipType, entry.getValue())) + .map(entry -> entry.getKey()).findAny(); } /** * Gets the mappedBy value. + * * @param getterMethod the getter method to get the information from (the annotation value). * @return the mappedBy value. */ @@ -270,8 +285,9 @@ private static Optional getMappedByValue(Method getterMethod) { /** * Checks if the inverse relationship type is the inverse type of a the relationship type. - * @param relationshipType the relationship type. - * (one to one, one to many, many to many, many to one). + * + * @param relationshipType the relationship type. + * (one to one, one to many, many to many, many to one). * @param inverseRelationshipType the inverse relationship type. * @return the inverse relationship field. */ @@ -291,27 +307,29 @@ private boolean isInverseRelationshipType(RelationshipType relationshipType, Rel /** * Gets the field value. - * @param entityClass the entity class. - * @param entity the entity. + * + * @param entityClass the entity class. + * @param entity the entity. * @param attributeName the attribute name to get value from. * @return the field value. */ public Object getFieldValue(Class entityClass, Object entity, String attributeName) { Optional field = attributeToField(entityClass, attributeName); return field.map(f -> getterMethod(entityClass, f)).map(getterMethod -> { - try { - return getterMethod.invoke(entity); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } catch (InvocationTargetException e) { - throw new RuntimeException(e); - } - }) - .orElseThrow(() -> new RuntimeException("No field named " + attributeName + " in " + entityClass.getName())); + try { + return getterMethod.invoke(entity); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } + }) + .orElseThrow(() -> new RuntimeException("No field named " + attributeName + " in " + entityClass.getName())); } /** * Gets the field value regarding an attribute name. + * * @param entityClass the entity class. * @return the field. */ @@ -321,22 +339,26 @@ private Optional attributeToField(Class entityClass, String attributeN /** * Sets the field value. - * @param entityClass the entity class. - * @param entity the entity to set value. + * + * @param entityClass the entity class. + * @param entity the entity to set value. * @param attributeName the attribute name to set value. - * @param value the value to set. + * @param value the value to set. */ public void setFieldValue(Class entityClass, Object entity, String attributeName, Object value) { Optional field = attributeToField(entityClass, attributeName); - field.ifPresentOrElse(f -> setFieldValue(entityClass, entity, f, value), () -> {throw new RuntimeException("No field with name "+ attributeName +" in class " + entityClass.getSimpleName());}); + field.ifPresentOrElse(f -> setFieldValue(entityClass, entity, f, value), () -> { + throw new RuntimeException("No field with name " + attributeName + " in class " + entityClass.getSimpleName()); + }); } /** * Sets the field value. + * * @param entityClass the entity class. - * @param entity the entity to set value. - * @param field the field to set value. - * @param value the value to set. + * @param entity the entity to set value. + * @param field the field to set value. + * @param value the value to set. */ public void setFieldValue(Class entityClass, Object entity, Field field, Object value) { setterMethod(entityClass, field).ifPresentOrElse(setterMethod -> { @@ -348,12 +370,13 @@ public void setFieldValue(Class entityClass, Object entity, Field field, Obje throw new RuntimeException(e); } }, () -> { - throw new RuntimeException("No setter with name "+ fieldToSetter(field.getName()) +" in class " + entityClass.getSimpleName()); + throw new RuntimeException("No setter with name " + fieldToSetter(field.getName()) + " in class " + entityClass.getSimpleName()); }); } /** * Gets the primary key value of an entity. + * * @param entity the entity. * @return the primary key value. */ diff --git a/core/src/main/java/net/osgiliath/migrator/core/modelgraph/ModelGraphBuilder.java b/core/src/main/java/net/osgiliath/migrator/core/modelgraph/ModelGraphBuilder.java index 57d3b4b2..e91daceb 100644 --- a/core/src/main/java/net/osgiliath/migrator/core/modelgraph/ModelGraphBuilder.java +++ b/core/src/main/java/net/osgiliath/migrator/core/modelgraph/ModelGraphBuilder.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,11 +20,11 @@ * #L% */ +import net.osgiliath.migrator.core.api.metamodel.model.FieldEdge; +import net.osgiliath.migrator.core.api.metamodel.model.MetamodelVertex; import net.osgiliath.migrator.core.api.model.ModelElement; import net.osgiliath.migrator.core.api.sourcedb.EntityImporter; import net.osgiliath.migrator.core.configuration.beans.GraphTraversalSourceProvider; -import net.osgiliath.migrator.core.api.metamodel.model.MetamodelVertex; -import net.osgiliath.migrator.core.api.metamodel.model.FieldEdge; import net.osgiliath.migrator.core.modelgraph.model.*; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; @@ -39,8 +39,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.apache.tinkerpop.gremlin.process.traversal.AnonymousTraversalSource.traversal; - @Component public class ModelGraphBuilder { @@ -70,28 +68,30 @@ public GraphTraversalSource modelGraphFromMetamodelGraph(org.jgrapht.Graph entityMetamodelGraph) { GraphTraversal entities = modelGraph.V(); - entities.toList().stream().flatMap(v -> { - TinkerVertex modelVertex = (TinkerVertex) v; - MetamodelVertex metamodelVertex = (MetamodelVertex) modelVertex.values(MODEL_GRAPH_VERTEX_METAMODEL_VERTEX).next(); - log.info("looking for edges for vertex of type {} with id {}", metamodelVertex.getTypeName(), v.values(MODEL_GRAPH_VERTEX_ENTITY_ID).next()); - Collection edges = metamodelVertex.getOutboundFieldEdges(entityMetamodelGraph).stream().collect(Collectors.toList()); - return edges.stream().map(edge -> - new FieldEdgeTargetVertices(edge, relatedVerticesOfOutgoingEdgeFromModelElementRelationship(modelVertex, edge, modelGraph)) - ).map(edgeAndTargetVertex -> - new SourceVertexFieldEdgeAndTargetVertices(modelVertex, edgeAndTargetVertex)); - }).flatMap(edgeAndTargetVertex -> edgeAndTargetVertex.getTargetVertices().stream().map(targetVertex -> new SourceVertexEdgeAndTargetVertex(edgeAndTargetVertex, targetVertex))) - .forEach(sourceVertexEdgeAndTargetVertex -> - sourceVertexEdgeAndTargetVertex.getSourceVertex().addEdge(sourceVertexEdgeAndTargetVertex.getEdge().getFieldName(), sourceVertexEdgeAndTargetVertex.getTargetVertex()).property(MODEL_GRAPH_EDGE_METAMODEL_FIELD, sourceVertexEdgeAndTargetVertex.getEdge().getMetamodelField()) - ); + List list = entities.toList(); + list.stream().flatMap(v -> { + TinkerVertex modelVertex = (TinkerVertex) v; + MetamodelVertex metamodelVertex = v.value(MODEL_GRAPH_VERTEX_METAMODEL_VERTEX); + log.info("looking for edges for vertex of type {} with id {}", metamodelVertex.getTypeName(), v.value(MODEL_GRAPH_VERTEX_ENTITY_ID)); + Collection edges = metamodelVertex.getOutboundFieldEdges(entityMetamodelGraph).stream().collect(Collectors.toList()); + return edges.stream().map(edge -> + new FieldEdgeTargetVertices(edge, relatedVerticesOfOutgoingEdgeFromModelElementRelationship(modelVertex, edge, modelGraph)) + ).map(edgeAndTargetVertex -> + new SourceVertexFieldEdgeAndTargetVertices(modelVertex, edgeAndTargetVertex)); + }) + .flatMap(edgeAndTargetVertex -> edgeAndTargetVertex.getTargetVertices().stream().map(targetVertex -> new SourceVertexEdgeAndTargetVertex(edgeAndTargetVertex, targetVertex))) + .forEach(sourceVertexEdgeAndTargetVertex -> + sourceVertexEdgeAndTargetVertex.getSourceVertex().addEdge(sourceVertexEdgeAndTargetVertex.getEdge().getFieldName(), sourceVertexEdgeAndTargetVertex.getTargetVertex()).property(MODEL_GRAPH_EDGE_METAMODEL_FIELD, sourceVertexEdgeAndTargetVertex.getEdge().getMetamodelField()) + ); } private Collection relatedVerticesOfOutgoingEdgeFromModelElementRelationship(TinkerVertex modelVertex, FieldEdge edge, GraphTraversalSource modelGraph) { log.debug("looking for related vertices for edge {}", edge); - ModelElement modelElement = (ModelElement) modelVertex.values(MODEL_GRAPH_VERTEX_ENTITY).next(); + ModelElement modelElement = modelVertex.value(MODEL_GRAPH_VERTEX_ENTITY); Object targetModelElements = modelElement.getEdgeValueFromModelElementRelationShip(edge, modelGraph); if (targetModelElements instanceof Collection) { - return ((Collection) targetModelElements).stream().map(targetModelElement -> - targetEdgeVertexOrEmpty(edge, getTargetEntityId(edge, targetModelElement), modelGraph) + return ((Collection) targetModelElements).stream().map(targetModelElement -> + targetEdgeVertexOrEmpty(edge, getTargetEntityId(edge, targetModelElement), modelGraph) ).filter(Optional::isPresent).map(Optional::get).toList(); } else { if (null != targetModelElements) { @@ -117,27 +117,27 @@ private Vertex targetEdgeVertex(FieldEdge edge, Object relatedEntityId, GraphTra String targetVertexType = edge.getTarget().getTypeName(); log.debug("looking for related target vertex type {} with id value {}", targetVertexType, relatedEntityId); return modelGraph.V() - .has(targetVertexType, - MODEL_GRAPH_VERTEX_ENTITY_ID, - relatedEntityId) - .property(MODEL_GRAPH_EDGE_METAMODEL_FIELD, edge.getFieldName()) + .has(targetVertexType, + MODEL_GRAPH_VERTEX_ENTITY_ID, + relatedEntityId) + .property(MODEL_GRAPH_EDGE_METAMODEL_FIELD, edge.getFieldName()) .next(); } private void createVertices(Set metamodelVertices, GraphTraversalSource modelGraph) { metamodelVertices.stream() - .map(mv -> new MetamodelVertexAndModelElements(mv, entityImporter.importEntities(mv, new ArrayList<>()))) - .flatMap(mvae -> mvae.getEntities().stream().map(entity -> new MetamodelVertexAndModelElement(mvae.getMetamodelVertex(), entity))) - .forEach( - mvae -> { - Object entityId = mvae.getModelElement().getId(mvae.getMetamodelVertex()); - GraphTraversal traversal = modelGraph - .addV(mvae.getMetamodelVertex().getTypeName()) - .property(MODEL_GRAPH_VERTEX_ENTITY_ID, entityId) - .property(MODEL_GRAPH_VERTEX_METAMODEL_VERTEX, mvae.getMetamodelVertex()) - .property(MODEL_GRAPH_VERTEX_ENTITY, mvae.getModelElement()); - mvae.getMetamodelVertex().getAdditionalModelVertexProperties(mvae.getModelElement()).forEach((k, v) -> traversal.property(k, v)); - traversal.next(); - }); + .map(mv -> new MetamodelVertexAndModelElements(mv, entityImporter.importEntities(mv, new ArrayList<>()))) + .flatMap(mvae -> mvae.getEntities().stream().map(entity -> new MetamodelVertexAndModelElement(mvae.getMetamodelVertex(), entity))) + .forEach( + mvae -> { + Object entityId = mvae.getModelElement().getId(mvae.getMetamodelVertex()); + GraphTraversal traversal = modelGraph + .addV(mvae.getMetamodelVertex().getTypeName()) + .property(MODEL_GRAPH_VERTEX_ENTITY_ID, entityId) + .property(MODEL_GRAPH_VERTEX_METAMODEL_VERTEX, mvae.getMetamodelVertex()) + .property(MODEL_GRAPH_VERTEX_ENTITY, mvae.getModelElement()); + mvae.getMetamodelVertex().getAdditionalModelVertexProperties(mvae.getModelElement()).forEach((k, v) -> traversal.property(k, v)); + traversal.next(); + }); } } diff --git a/core/src/main/java/net/osgiliath/migrator/core/processing/FactorySequencer.java b/core/src/main/java/net/osgiliath/migrator/core/processing/FactorySequencer.java index 6a9da911..832c9e0d 100644 --- a/core/src/main/java/net/osgiliath/migrator/core/processing/FactorySequencer.java +++ b/core/src/main/java/net/osgiliath/migrator/core/processing/FactorySequencer.java @@ -22,7 +22,8 @@ import net.osgiliath.migrator.core.api.metamodel.model.FieldEdge; import net.osgiliath.migrator.core.api.metamodel.model.MetamodelVertex; -import net.osgiliath.migrator.core.configuration.TransformationConfigurationDefinition; +import net.osgiliath.migrator.core.configuration.ColumnTransformationDefinition; +import net.osgiliath.migrator.core.configuration.SequencerDefinition; import net.osgiliath.migrator.core.api.model.ModelElement; import org.jgrapht.Graph; @@ -44,8 +45,8 @@ public interface FactorySequencer { * @param graph the metamodel graph. * @param metamodelVertex the metamodel vertex representing the entity definition . * @param entity the entity to be handled by the sequencer. - * @param columnName the column name to be handled by the sequencer. + * @param columnTransformationDefinition the column name and options to be handled by the sequencer. * @return the resulting configured sequencer bean. */ - Object createSequencerBean(Class beanClass, TransformationConfigurationDefinition definition, Graph graph, MetamodelVertex metamodelVertex, ModelElement entity, String columnName); + Object createSequencerBean(Class beanClass, SequencerDefinition definition, Graph graph, MetamodelVertex metamodelVertex, ModelElement entity, ColumnTransformationDefinition columnTransformationDefinition); } diff --git a/core/src/main/java/net/osgiliath/migrator/core/processing/SequenceProcessor.java b/core/src/main/java/net/osgiliath/migrator/core/processing/SequenceProcessor.java index d5f43633..6160305d 100644 --- a/core/src/main/java/net/osgiliath/migrator/core/processing/SequenceProcessor.java +++ b/core/src/main/java/net/osgiliath/migrator/core/processing/SequenceProcessor.java @@ -24,6 +24,7 @@ import net.osgiliath.migrator.core.api.metamodel.model.MetamodelVertex; import net.osgiliath.migrator.core.api.transformers.JpaEntityColumnTransformer; import net.osgiliath.migrator.core.api.transformers.MetamodelColumnCellTransformer; +import net.osgiliath.migrator.core.configuration.ColumnTransformationDefinition; import net.osgiliath.migrator.core.configuration.DataMigratorConfiguration; import net.osgiliath.migrator.core.configuration.TRANSFORMER_TYPE; import net.osgiliath.migrator.core.modelgraph.ModelGraphBuilder; @@ -73,10 +74,10 @@ public void process(GraphTraversalSource modelGraph, Graph factorySequencers) { } - public Object createSequencerBean(Class beanClass, TransformationConfigurationDefinition definition, Graph graph, MetamodelVertex metamodelVertex, ModelElement entity, String columnName) { - return factorySequencers.stream().filter(factorySequencer -> factorySequencer.canHandle(beanClass)).findFirst().orElseThrow(() -> new RuntimeException("No factory sequencer found for " + beanClass)).createSequencerBean(beanClass, definition, graph, metamodelVertex, entity, columnName); + public Object createSequencerBean(Class beanClass, SequencerDefinition definition, Graph graph, MetamodelVertex metamodelVertex, ModelElement entity, ColumnTransformationDefinition columnTransformationDefinition) { + return factorySequencers.stream().filter(factorySequencer -> factorySequencer.canHandle(beanClass)).findFirst().orElseThrow(() -> new RuntimeException("No factory sequencer found for " + beanClass)).createSequencerBean(beanClass, definition, graph, metamodelVertex, entity, columnTransformationDefinition); } } diff --git a/core/src/main/java/net/osgiliath/migrator/core/processing/model/SequencerDefinitionAndBean.java b/core/src/main/java/net/osgiliath/migrator/core/processing/model/SequencerDefinitionAndBean.java index 79ee208d..d7e1fcef 100644 --- a/core/src/main/java/net/osgiliath/migrator/core/processing/model/SequencerDefinitionAndBean.java +++ b/core/src/main/java/net/osgiliath/migrator/core/processing/model/SequencerDefinitionAndBean.java @@ -20,18 +20,18 @@ * #L% */ -import net.osgiliath.migrator.core.configuration.TransformationConfigurationDefinition; +import net.osgiliath.migrator.core.configuration.SequencerDefinition; public class SequencerDefinitionAndBean { - private final TransformationConfigurationDefinition sequencerConfiguration; + private final SequencerDefinition sequencerConfiguration; private final Class transformerClass; - public SequencerDefinitionAndBean(TransformationConfigurationDefinition sequencerConfiguration, Class transformerClass) { + public SequencerDefinitionAndBean(SequencerDefinition sequencerConfiguration, Class transformerClass) { this.sequencerConfiguration = sequencerConfiguration; this.transformerClass = transformerClass; } - public TransformationConfigurationDefinition getSequencerConfiguration() { + public SequencerDefinition getSequencerConfiguration() { return sequencerConfiguration; } diff --git a/core/src/main/java/net/osgiliath/migrator/core/processing/model/VertexAndSequencerBeanClass.java b/core/src/main/java/net/osgiliath/migrator/core/processing/model/VertexAndSequencerBeanClass.java index 547a4656..c6295d20 100644 --- a/core/src/main/java/net/osgiliath/migrator/core/processing/model/VertexAndSequencerBeanClass.java +++ b/core/src/main/java/net/osgiliath/migrator/core/processing/model/VertexAndSequencerBeanClass.java @@ -20,7 +20,7 @@ * #L% */ -import net.osgiliath.migrator.core.configuration.TransformationConfigurationDefinition; +import net.osgiliath.migrator.core.configuration.SequencerDefinition; import org.apache.tinkerpop.gremlin.structure.Vertex; public class VertexAndSequencerBeanClass { @@ -40,7 +40,7 @@ public Class getBeanClass() { return definitionAndBeanClass.getBeanClass(); } - public TransformationConfigurationDefinition getDefinition() { + public SequencerDefinition getDefinition() { return definitionAndBeanClass.getSequencerConfiguration(); } } diff --git a/datamigrator-archetype/src/main/resources/archetype-resources/pom.xml b/datamigrator-archetype/src/main/resources/archetype-resources/pom.xml index 9ef937ec..d3d50955 100644 --- a/datamigrator-archetype/src/main/resources/archetype-resources/pom.xml +++ b/datamigrator-archetype/src/main/resources/archetype-resources/pom.xml @@ -1,5 +1,6 @@ - + 4.0.0 net.osgiliath.datamigrator @@ -122,27 +123,12 @@ ${java.version} ${java.version} - - -Aquerydsl.entityAccessors=true - -Aquerydsl.useFields=false - org.springframework.boot spring-boot-configuration-processor ${spring-boot.version} - - org.hibernate - hibernate-jpamodelgen - ${hibernate.version} - - - com.querydsl - querydsl-apt - ${querydsl.version} - jakarta - @@ -150,19 +136,19 @@ org.codehaus.mojo properties-maven-plugin - - initialize - - read-project-properties - - - - ${project.basedir}/database.properties - - - + + initialize + + read-project-properties + + + + ${project.basedir}/database.properties + + + - + org.apache.maven.plugins maven-surefire-plugin @@ -201,6 +187,38 @@ entities-from-source-schema + + org.apache.maven.plugins + maven-compiler-plugin + + ${java.version} + ${java.version} + + + -Aquerydsl.entityAccessors=true + -Aquerydsl.useFields=false + + + + org.springframework.boot + spring-boot-configuration-processor + ${spring-boot.version} + + + org.hibernate + hibernate-jpamodelgen + ${hibernate.version} + + + com.querydsl + querydsl-apt + ${querydsl.version} + jakarta + + + + + org.hibernate.tool hibernate-tools-maven @@ -300,9 +318,10 @@ generate-sources - + - + @@ -339,8 +358,8 @@ - org.liquibase - liquibase-maven-plugin + org.liquibase + liquibase-maven-plugin generate-changelog @@ -351,7 +370,8 @@ - tables,views,columns,indexes,primarykeys,foreignkeys,uniqueconstraints + tables,views,columns,indexes,primarykeys,foreignkeys,uniqueconstraints + @@ -362,8 +382,8 @@ - org.liquibase - liquibase-maven-plugin + org.liquibase + liquibase-maven-plugin generate-changelog @@ -385,6 +405,37 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + ${java.version} + ${java.version} + + + -Aquerydsl.entityAccessors=true + -Aquerydsl.useFields=false + + + + org.springframework.boot + spring-boot-configuration-processor + ${spring-boot.version} + + + org.hibernate + hibernate-jpamodelgen + ${hibernate.version} + + + com.querydsl + querydsl-apt + ${querydsl.version} + jakarta + + + + io.fabric8 docker-maven-plugin @@ -403,31 +454,33 @@ - database - ${source.database.docker.image} - - - - - - - ${source.datasource.password} - ${source.datasource.defaultSchemaName} - ${source.datasource.username} - ${source.datasource.password} - - - ${source.database.port}:${source.database.port} - - - + database + ${source.database.docker.image} + + + + + + + ${source.datasource.password} + + ${source.datasource.defaultSchemaName} + + ${source.datasource.username} + ${source.datasource.password} + + + ${source.database.port}:${source.database.port} + + + @@ -590,9 +643,10 @@ generate-sources - + - + @@ -629,8 +683,8 @@ - org.liquibase - liquibase-maven-plugin + org.liquibase + liquibase-maven-plugin generate-changelog @@ -640,30 +694,39 @@ - - + + WARN - ${project.build.directory}/classes/config/liquibase-sink/master.xml - ${project.build.directory}/classes/config/liquibase-sink/changelog/${maven.build.timestamp}_diff.xml - ${project.build.directory}/classes/config/liquibase-sink/changelog/${maven.build.timestamp}_changelog.xml - ${project.build.directory}/classes/config/liquibase-sink/data - ${target.database.driver} - ${target.database.url} - ${target.datasource.defaultSchemaName} - ${target.datasource.username} - ${target.datasource.password} - - - - hibernate:spring:${jpa.domain.package}?dialect=${target.jdbc.dialect}&hibernate.physical_naming_strategy=org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy&hibernate.implicit_naming_strategy=org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy - true - - tables,views,columns,indexes,primarykeys,foreignkeys,uniqueconstraints,data - oauth_access_token, oauth_approvals, oauth_client_details, oauth_client_token, oauth_code, oauth_refresh_token - true - ${project.build.directory}/classes/config/liquibase/changelog-sink - + ${project.build.directory}/classes/config/liquibase-sink/master.xml + + + ${project.build.directory}/classes/config/liquibase-sink/changelog/${maven.build.timestamp}_diff.xml + + + ${project.build.directory}/classes/config/liquibase-sink/changelog/${maven.build.timestamp}_changelog.xml + + ${project.build.directory}/classes/config/liquibase-sink/data + ${target.database.driver} + ${target.database.url} + ${target.datasource.defaultSchemaName} + ${target.datasource.username} + ${target.datasource.password} + + + + hibernate:spring:${jpa.domain.package}?dialect=${target.jdbc.dialect}&hibernate.physical_naming_strategy=org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy&hibernate.implicit_naming_strategy=org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy + true + + tables,views,columns,indexes,primarykeys,foreignkeys,uniqueconstraints,data + + oauth_access_token, oauth_approvals, oauth_client_details, + oauth_client_token, oauth_code, oauth_refresh_token + + true + ${project.build.directory}/classes/config/liquibase/changelog-sink + + org.springframework.boot @@ -680,8 +743,8 @@ - org.liquibase - liquibase-maven-plugin + org.liquibase + liquibase-maven-plugin generate-changelog @@ -691,29 +754,37 @@ - - - /config/liquibase-sink/master.xml - ${project.build.directory}/classes/config/liquibase-sink/changelog/${maven.build.timestamp}_diff.xml - ${project.build.directory}/classes/config/liquibase-sink/changelog/${maven.build.timestamp}_changelog.xml - ${project.build.directory}/classes/config/liquibase-sink/data - ${project.build.directory}/classes/config/liquibase/changelog-sink - ${target.database.driver} - ${target.database.url} - ${target.datasource.defaultSchemaName} - ${target.datasource.username} - ${target.datasource.password} - - - - hibernate:spring:${jpa.domain.package}?dialect=${target.jdbc.dialect}&hibernate.physical_naming_strategy=org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy&hibernate.implicit_naming_strategy=org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy - true - - tables,views,columns,indexes,primarykeys,foreignkeys,uniqueconstraints,data - oauth_access_token, oauth_approvals, oauth_client_details, oauth_client_token, oauth_code, oauth_refresh_token - true - + + + /config/liquibase-sink/master.xml + + ${project.build.directory}/classes/config/liquibase-sink/changelog/${maven.build.timestamp}_diff.xml + + + ${project.build.directory}/classes/config/liquibase-sink/changelog/${maven.build.timestamp}_changelog.xml + + ${project.build.directory}/classes/config/liquibase-sink/data + ${project.build.directory}/classes/config/liquibase/changelog-sink + + ${target.database.driver} + ${target.database.url} + ${target.datasource.defaultSchemaName} + ${target.datasource.username} + ${target.datasource.password} + + + + hibernate:spring:${jpa.domain.package}?dialect=${target.jdbc.dialect}&hibernate.physical_naming_strategy=org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy&hibernate.implicit_naming_strategy=org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy + true + + tables,views,columns,indexes,primarykeys,foreignkeys,uniqueconstraints,data + + oauth_access_token, oauth_approvals, oauth_client_details, + oauth_client_token, oauth_code, oauth_refresh_token + + true + org.springframework.boot diff --git a/datamigrator-archetype/src/main/resources/archetype-resources/src/main/resources/application.yml b/datamigrator-archetype/src/main/resources/archetype-resources/src/main/resources/application.yml index a49a0b6e..11e874d2 100644 --- a/datamigrator-archetype/src/main/resources/archetype-resources/src/main/resources/application.yml +++ b/datamigrator-archetype/src/main/resources/archetype-resources/src/main/resources/application.yml @@ -1,29 +1,28 @@ data-migrator: model-base-package: @jpa.domain.package@ sequence: -# - rows-minimize-1 -# - column-anonymize-1 -# - column-anonymize-2 + # - rows-minimize-1 + # - column-anonymize-1 + # - column-anonymize-2 - anonymizer-custom-1 sequencers: -# - name: rows-minimize-1 -# type: rows-minimize -# rows-to-keep: 10 -# - name: column-anonymize-1 -# type: column-anonymizers -# table: virtuals -# column: password -# faker: RockBand -# - name: column-anonymize-2 -# type: column-anonymizers -# table: virtuals -# column: biosspass -# faker: Disease + # TODO - name: rows-minimize-1 + # type: rows-minimize + # rows-to-keep: 10 + # Included - name: column-anonymize-1 + # type: factory + # transformer-class: net.osgiliath.migrator.modules.faker.ColumnFaker + # entity-class: Employee + # column-transformation-definitions: + # - column-name: firstName + # options: + # - faker-class: net.datafaker.providers.entertainment.Dragonball + # faker-function: character # https://www.datafaker.net/documentation/getting-started/ for options, will default to a dragonball character if not set -# - name: anonymizer-custom-1 -# type: bean -# transformer-class: net.osgiliath.migrator.sample.transformers.CountryTransformer -# entity-class: Country + # - name: anonymizer-custom-1 + # type: bean + # transformer-class: net.osgiliath.migrator.sample.transformers.CountryTransformer + # entity-class: Country graph-datasource: type: embedded spring: @@ -48,10 +47,10 @@ spring: properties: hibernate: id.new_generator_mappings: true -# cache: -# use_second_level_cache: true -# use_query_cache: false -# region.factory_class: jcache + # cache: + # use_second_level_cache: true + # use_query_cache: false + # region.factory_class: jcache generate_statistics: false # modify batch size as necessary jdbc: diff --git a/modules/faker/src/main/java/net/osgiliath/migrator/modules/faker/AbstractFaker.java b/modules/faker/src/main/java/net/osgiliath/migrator/modules/faker/AbstractFaker.java index 974cab84..a6911c9d 100644 --- a/modules/faker/src/main/java/net/osgiliath/migrator/modules/faker/AbstractFaker.java +++ b/modules/faker/src/main/java/net/osgiliath/migrator/modules/faker/AbstractFaker.java @@ -9,9 +9,9 @@ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,95 +21,116 @@ */ -import jakarta.persistence.metamodel.Attribute; import net.datafaker.Faker; -import net.osgiliath.migrator.core.api.metamodel.model.FieldEdge; +import net.datafaker.providers.base.AbstractProvider; import net.osgiliath.migrator.core.api.metamodel.model.MetamodelVertex; import net.osgiliath.migrator.core.api.transformers.JpaEntityColumnTransformer; -import net.osgiliath.migrator.core.api.transformers.MetamodelColumnCellTransformer; -import org.jgrapht.Graph; +import net.osgiliath.migrator.core.configuration.ColumnTransformationDefinition; +import java.lang.reflect.InvocationTargetException; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.time.temporal.ChronoUnit; import java.util.Optional; import java.util.Random; +import java.util.function.Function; public abstract class AbstractFaker extends JpaEntityColumnTransformer { - public AbstractFaker(MetamodelVertex metamodel, String columnName) { - super(metamodel, columnName); - } + private final ColumnTransformationDefinition columnTransformationDefinition; - protected String fake(String value) { - return getRandomInteger(value) - .orElseGet(() -> - getRandomLocalDate(value) - .orElseGet(() -> - getRandomString())); - } + public AbstractFaker(MetamodelVertex metamodel, ColumnTransformationDefinition columnTransformationDefinition) { + super(metamodel, columnTransformationDefinition.getColumnName()); + this.columnTransformationDefinition = columnTransformationDefinition; + } - private String getRandomString() { - Faker faker = new Faker(); - return new StringBuilder() - .append(faker.dragonBall().character()) - .toString(); - } + protected String fake(String value) { + return getRandomInteger(value) + .orElseGet(() -> + getRandomLocalDate(value) + .orElseGet(() -> + getRandomString())); + } - private Optional getRandomInteger(String value) { - if (value.length() > 0) { - try { - Integer inputAsInteger = Integer.parseInt(value); // i.e. 10 - Random random = new Random(); - Integer randomizerFactor = (inputAsInteger * 2) * 10; // i.e. 1110 - Integer randomResult = random.ints(randomizerFactor - inputAsInteger, randomizerFactor * inputAsInteger) - .findFirst() - .getAsInt(); - if (inputAsInteger > 0) { - if (randomResult < 0) { - randomResult = -randomResult; - } - } else { - if (randomResult > 0) { - randomResult = -randomResult; - } - } - return Optional.of(String.valueOf(randomResult)); - } catch (NumberFormatException nfe) { - // go to next step - } - } - return Optional.empty(); - } + private String getRandomString() { + Faker faker = new Faker().getFaker(); + if (columnTransformationDefinition.getOptions().containsKey("faker-class")) { + Class clzz = null; + try { + clzz = Class.forName(columnTransformationDefinition.getOptions().get("faker-class")); + AbstractProvider reg = faker.getProvider((Class) clzz, (Function) clzz.getDeclaredConstructor().newInstance(faker)); + return reg.getFaker().resolve(columnTransformationDefinition.getOptions().get("faker-function")); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } catch (InvocationTargetException e) { + throw new RuntimeException(e); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } + } + return new StringBuilder() + .append(faker.dragonBall().character()) + .toString(); + } - private Optional getRandomLocalDate(String value) { - for (SupportedLocalDateFormat supportedLocalDateFormat: SupportedLocalDateFormat.values()) { - try { - LocalDate date = LocalDate.parse(value, DateTimeFormatter.ofPattern(supportedLocalDateFormat.format)); - Random random = new Random(); - Integer bound = 200; - Integer randomResult = random.ints(100, bound) - .findFirst() - .getAsInt(); - return Optional.of(date.minus(randomResult, ChronoUnit.DAYS).format(DateTimeFormatter.ofPattern(supportedLocalDateFormat.format))); - } catch (DateTimeParseException dtpe) { + private Optional getRandomInteger(String value) { + if (value.length() > 0) { + try { + Integer inputAsInteger = Integer.parseInt(value); // i.e. 10 + Random random = new Random(); + Integer randomizerFactor = (inputAsInteger * 2) * 10; // i.e. 1110 + Integer randomResult = random.ints(randomizerFactor - inputAsInteger, randomizerFactor * inputAsInteger) + .findFirst() + .getAsInt(); + if (inputAsInteger > 0) { + if (randomResult < 0) { + randomResult = -randomResult; + } + } else { + if (randomResult > 0) { + randomResult = -randomResult; + } + } + return Optional.of(String.valueOf(randomResult)); + } catch (NumberFormatException nfe) { + // go to next step + } + } + return Optional.empty(); + } - } - } - return Optional.empty(); - } + private Optional getRandomLocalDate(String value) { + for (SupportedLocalDateFormat supportedLocalDateFormat : SupportedLocalDateFormat.values()) { + try { + LocalDate date = LocalDate.parse(value, DateTimeFormatter.ofPattern(supportedLocalDateFormat.format)); + Random random = new Random(); + Integer bound = 200; + Integer randomResult = random.ints(100, bound) + .findFirst() + .getAsInt(); + return Optional.of(date.minus(randomResult, ChronoUnit.DAYS).format(DateTimeFormatter.ofPattern(supportedLocalDateFormat.format))); + } catch (DateTimeParseException dtpe) { - private enum SupportedLocalDateFormat { - ISO("yyyy-MM-dd"), - DDMMYYYSLASH("dd/MM/yyyy"), - DDMMYYYDASH("dd-MM-yyyy"), - FULLTEXT("E, MMM dd yyyy"); + } + } + return Optional.empty(); + } - private final String format; + private enum SupportedLocalDateFormat { + ISO("yyyy-MM-dd"), + DDMMYYYSLASH("dd/MM/yyyy"), + DDMMYYYDASH("dd-MM-yyyy"), + FULLTEXT("E, MMM dd yyyy"); - SupportedLocalDateFormat(String format) { - this.format = format; - } - } + private final String format; + + SupportedLocalDateFormat(String format) { + this.format = format; + } + } } diff --git a/modules/faker/src/main/java/net/osgiliath/migrator/modules/faker/ColumnFaker.java b/modules/faker/src/main/java/net/osgiliath/migrator/modules/faker/ColumnFaker.java index 94aea098..69b10014 100644 --- a/modules/faker/src/main/java/net/osgiliath/migrator/modules/faker/ColumnFaker.java +++ b/modules/faker/src/main/java/net/osgiliath/migrator/modules/faker/ColumnFaker.java @@ -20,15 +20,14 @@ * #L% */ -import net.osgiliath.migrator.core.api.metamodel.model.FieldEdge; import net.osgiliath.migrator.core.api.metamodel.model.MetamodelVertex; -import org.jgrapht.Graph; +import net.osgiliath.migrator.core.configuration.ColumnTransformationDefinition; public class ColumnFaker extends AbstractFaker { - public ColumnFaker(MetamodelVertex metamodel, String columnName) { - super(metamodel, columnName); + public ColumnFaker(MetamodelVertex metamodel, ColumnTransformationDefinition columnTransformationDefinition) { + super(metamodel, columnTransformationDefinition); } @Override diff --git a/modules/faker/src/main/java/net/osgiliath/migrator/modules/faker/ColumnFakerFactorySequencer.java b/modules/faker/src/main/java/net/osgiliath/migrator/modules/faker/ColumnFakerFactorySequencer.java index 8a7eb776..2801bfa3 100644 --- a/modules/faker/src/main/java/net/osgiliath/migrator/modules/faker/ColumnFakerFactorySequencer.java +++ b/modules/faker/src/main/java/net/osgiliath/migrator/modules/faker/ColumnFakerFactorySequencer.java @@ -22,7 +22,8 @@ import net.osgiliath.migrator.core.api.metamodel.model.FieldEdge; import net.osgiliath.migrator.core.api.metamodel.model.MetamodelVertex; -import net.osgiliath.migrator.core.configuration.TransformationConfigurationDefinition; +import net.osgiliath.migrator.core.configuration.ColumnTransformationDefinition; +import net.osgiliath.migrator.core.configuration.SequencerDefinition; import net.osgiliath.migrator.core.api.model.ModelElement; import net.osgiliath.migrator.core.processing.FactorySequencer; import org.jgrapht.Graph; @@ -36,7 +37,7 @@ public boolean canHandle(Class beanClass) { } @Override - public Object createSequencerBean(Class beanClass, TransformationConfigurationDefinition definition, Graph graph, MetamodelVertex metamodelVertex, ModelElement entity, String columnName) { - return new ColumnFaker(metamodelVertex, columnName); + public Object createSequencerBean(Class beanClass, SequencerDefinition definition, Graph graph, MetamodelVertex metamodelVertex, ModelElement entity, ColumnTransformationDefinition columnTransformationDefinition) { + return new ColumnFaker(metamodelVertex, columnTransformationDefinition); } } diff --git a/readme.md b/readme.md index 277a6ce8..1e6a66fe 100644 --- a/readme.md +++ b/readme.md @@ -1,8 +1,10 @@ +![tests and snapshot deployment](https://github.com/OsgiliathEnterprise/data-migrator/actions/workflows/maven.yml/badge.svg) + # Welcome anonymous! ## Data Migrator tool -This tool aims to help the developer or business analyst to gather production data from a source datasource of any vendor, create the same structure in target database (same technology or other vendor or version, thanks to orm) then to execute the data transfer with a data value transformation sequence in between: standard anonymization (i.e. replacing column values with random values), custom scripted anonymization, e.g. executing tailor-made java custom business on top of spring data repositories and ai-driven or advanced customization. +This tool aims to help the developer or business analyst to gather production data from a source datasource of any vendor, create the same structure in target database (same technology or other vendor or version, thanks to orm) then to execute the data transfer with a data value transformation sequence in between: standard anonymization (i.e. replacing column values with random values), or custom scripted anonymization, e.g. executing tailor-made java custom business on top of spring data repositories or Tinkerpop's graph queries and even ai-driven. ## Procedure @@ -14,8 +16,8 @@ This tool aims to help the developer or business analyst to gather production da 6. Generate the java entities, java tables metamodel and repositories from the root of your generated project using the `./mvnw clean process-classes -Pentities-from-source-schema` command: doing so, new classes will appear on the `/target/generated-sources`directory. 7. Create your custom business logic (java code) on top of the spring-data-repositories and queries (optional if you use common modules that do ot need any customization). 8. Configure your anonymization sequence using the `src/main/resources/application.yml` property file with data-migrator sequence and sequencers. -9. Start your target database container (optional if you have access to the remote database) -10. create the target schema by executing the `./mvnw clean process-classes -Pcreate-target-schema` command. +9. Start your target database container (optional if you have access to the remote database). +10. Launch the sequencing: `./mvnw package && cd target && java .jar`. ## Additional useful commands diff --git a/sample-mono/pom.xml b/sample-mono/pom.xml index 98ffe281..4f1f6e29 100644 --- a/sample-mono/pom.xml +++ b/sample-mono/pom.xml @@ -16,7 +16,7 @@ ${project.groupId}.domain - + net.osgiliath.datamigrator @@ -124,27 +124,12 @@ ${java.version} ${java.version} - - -Aquerydsl.entityAccessors=true - -Aquerydsl.useFields=false - org.springframework.boot spring-boot-configuration-processor ${spring-boot.version} - - org.hibernate - hibernate-jpamodelgen - ${hibernate.version} - - - com.querydsl - querydsl-apt - ${querydsl.version} - jakarta - @@ -152,19 +137,19 @@ org.codehaus.mojo properties-maven-plugin - - initialize - - read-project-properties - - - - ${basedir}/database.properties - - - + + initialize + + read-project-properties + + + + ${basedir}/database.properties + + + - + org.apache.maven.plugins maven-surefire-plugin @@ -246,6 +231,37 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + ${java.version} + ${java.version} + + + -Aquerydsl.entityAccessors=true + -Aquerydsl.useFields=false + + + + org.springframework.boot + spring-boot-configuration-processor + ${spring-boot.version} + + + org.hibernate + hibernate-jpamodelgen + ${hibernate.version} + + + com.querydsl + querydsl-apt + ${querydsl.version} + jakarta + + + + com.google.code.maven-replacer-plugin replacer @@ -296,15 +312,18 @@ + org.apache.maven.plugins maven-antrun-plugin + 3.1.0 generate-sources - + - + @@ -341,8 +360,8 @@ - org.liquibase - liquibase-maven-plugin + org.liquibase + liquibase-maven-plugin generate-changelog @@ -353,7 +372,8 @@ - tables,views,columns,indexes,primarykeys,foreignkeys,uniqueconstraints + tables,views,columns,indexes,primarykeys,foreignkeys,uniqueconstraints + @@ -364,8 +384,8 @@ - org.liquibase - liquibase-maven-plugin + org.liquibase + liquibase-maven-plugin generate-changelog @@ -387,6 +407,37 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + ${java.version} + ${java.version} + + + -Aquerydsl.entityAccessors=true + -Aquerydsl.useFields=false + + + + org.springframework.boot + spring-boot-configuration-processor + ${spring-boot.version} + + + org.hibernate + hibernate-jpamodelgen + ${hibernate.version} + + + com.querydsl + querydsl-apt + ${querydsl.version} + jakarta + + + + io.fabric8 docker-maven-plugin @@ -405,31 +456,33 @@ - database - ${source.database.docker.image} - - - - - - - ${source.datasource.password} - ${source.datasource.defaultSchemaName} - ${source.datasource.username} - ${source.datasource.password} - - - ${source.database.port}:${source.database.port} - - - + database + ${source.database.docker.image} + + + + + + + ${source.datasource.password} + + ${source.datasource.defaultSchemaName} + + ${source.datasource.username} + ${source.datasource.password} + + + ${source.database.port}:${source.database.port} + + + @@ -592,9 +645,10 @@ generate-sources - + - + @@ -631,8 +685,8 @@ - org.liquibase - liquibase-maven-plugin + org.liquibase + liquibase-maven-plugin generate-changelog @@ -642,30 +696,39 @@ - - + + WARN - ${project.build.directory}/classes/config/liquibase-sink/master.xml - ${project.build.directory}/classes/config/liquibase-sink/changelog/${maven.build.timestamp}_diff.xml - ${project.build.directory}/classes/config/liquibase-sink/changelog/${maven.build.timestamp}_changelog.xml - ${project.build.directory}/classes/config/liquibase-sink/data - ${target.database.driver} - ${target.database.url} - ${target.datasource.defaultSchemaName} - ${target.datasource.username} - ${target.datasource.password} - - - - hibernate:spring:${jpa.domain.package}?dialect=${target.jdbc.dialect}&hibernate.physical_naming_strategy=org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy&hibernate.implicit_naming_strategy=org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy - true - - tables,views,columns,indexes,primarykeys,foreignkeys,uniqueconstraints,data - oauth_access_token, oauth_approvals, oauth_client_details, oauth_client_token, oauth_code, oauth_refresh_token - true - ${project.build.directory}/classes/config/liquibase/changelog-sink - + ${project.build.directory}/classes/config/liquibase-sink/master.xml + + + ${project.build.directory}/classes/config/liquibase-sink/changelog/${maven.build.timestamp}_diff.xml + + + ${project.build.directory}/classes/config/liquibase-sink/changelog/${maven.build.timestamp}_changelog.xml + + ${project.build.directory}/classes/config/liquibase-sink/data + ${target.database.driver} + ${target.database.url} + ${target.datasource.defaultSchemaName} + ${target.datasource.username} + ${target.datasource.password} + + + + hibernate:spring:${jpa.domain.package}?dialect=${target.jdbc.dialect}&hibernate.physical_naming_strategy=org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy&hibernate.implicit_naming_strategy=org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy + true + + tables,views,columns,indexes,primarykeys,foreignkeys,uniqueconstraints,data + + oauth_access_token, oauth_approvals, oauth_client_details, + oauth_client_token, oauth_code, oauth_refresh_token + + true + ${project.build.directory}/classes/config/liquibase/changelog-sink + + org.springframework.boot @@ -682,8 +745,8 @@ - org.liquibase - liquibase-maven-plugin + org.liquibase + liquibase-maven-plugin generate-changelog @@ -693,29 +756,37 @@ - - - /config/liquibase-sink/master.xml - ${project.build.directory}/classes/config/liquibase-sink/changelog/${maven.build.timestamp}_diff.xml - ${project.build.directory}/classes/config/liquibase-sink/changelog/${maven.build.timestamp}_changelog.xml - ${project.build.directory}/classes/config/liquibase-sink/data - ${project.build.directory}/classes/config/liquibase/changelog-sink - ${target.database.driver} - ${target.database.url} - ${target.datasource.defaultSchemaName} - ${target.datasource.username} - ${target.datasource.password} - - - - hibernate:spring:${jpa.domain.package}?dialect=${target.jdbc.dialect}&hibernate.physical_naming_strategy=org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy&hibernate.implicit_naming_strategy=org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy - true - - tables,views,columns,indexes,primarykeys,foreignkeys,uniqueconstraints,data - oauth_access_token, oauth_approvals, oauth_client_details, oauth_client_token, oauth_code, oauth_refresh_token - true - + + + /config/liquibase-sink/master.xml + + ${project.build.directory}/classes/config/liquibase-sink/changelog/${maven.build.timestamp}_diff.xml + + + ${project.build.directory}/classes/config/liquibase-sink/changelog/${maven.build.timestamp}_changelog.xml + + ${project.build.directory}/classes/config/liquibase-sink/data + ${project.build.directory}/classes/config/liquibase/changelog-sink + + ${target.database.driver} + ${target.database.url} + ${target.datasource.defaultSchemaName} + ${target.datasource.username} + ${target.datasource.password} + + + + hibernate:spring:${jpa.domain.package}?dialect=${target.jdbc.dialect}&hibernate.physical_naming_strategy=org.hibernate.boot.model.naming.CamelCaseToUnderscoresNamingStrategy&hibernate.implicit_naming_strategy=org.springframework.boot.orm.jpa.hibernate.SpringImplicitNamingStrategy + true + + tables,views,columns,indexes,primarykeys,foreignkeys,uniqueconstraints,data + + oauth_access_token, oauth_approvals, oauth_client_details, + oauth_client_token, oauth_code, oauth_refresh_token + + true + org.springframework.boot diff --git a/sample-mono/sample-mono.iml b/sample-mono/sample-mono.iml index 973a7f23..d7834b02 100644 --- a/sample-mono/sample-mono.iml +++ b/sample-mono/sample-mono.iml @@ -1,6 +1,6 @@ - \ No newline at end of file diff --git a/sample-mono/src/main/resources/application.yml b/sample-mono/src/main/resources/application.yml index 70852e66..3dcc765d 100644 --- a/sample-mono/src/main/resources/application.yml +++ b/sample-mono/src/main/resources/application.yml @@ -1,24 +1,31 @@ data-migrator: model-base-package: @jpa.domain.package@ sequence: -# - rows-minimize-1 -# - column-anonymize-1 -# - column-anonymize-2 + # - rows-minimize-1 + # - column-anonymize-1 + # - column-anonymize-2 - anonymizer-custom-1 sequencers: -# - name: rows-minimize-1 -# type: rows-minimize -# rows-to-keep: 10 -# - name: column-anonymize-1 -# type: column-anonymizers -# table: virtuals -# column: password -# faker: RockBand -# - name: column-anonymize-2 -# type: column-anonymizers -# table: virtuals -# column: biosspass -# faker: Disease + # - name: rows-minimize-1 + # type: rows-minimize + # rows-to-keep: 10 + # - name: column-anonymize-1 + # type: column-anonymizers + # table: virtuals + # column: password + # faker: RockBand + # - name: column-anonymize-2 + # type: column-anonymizers + # table: virtuals + # column: biosspass + # faker: Disease + # - name: column-anonymize-1 + # type: factory + # column-transformation-definitions: + # - column-name: firstName + # options: + # - faker-class: net.datafaker.providers.entertainment.Dragonball + # faker-function: character # https://www.datafaker.net/documentation/getting-started/ for options, will default to a dragonball character if not set - name: anonymizer-custom-1 type: bean @@ -48,10 +55,10 @@ spring: properties: hibernate: id.new_generator_mappings: true -# cache: -# use_second_level_cache: true -# use_query_cache: false -# region.factory_class: jcache + # cache: + # use_second_level_cache: true + # use_query_cache: false + # region.factory_class: jcache generate_statistics: false # modify batch size as necessary jdbc: diff --git a/sample-mono/src/test/java/net/osgiliath/FakerProcessingIT.java b/sample-mono/src/test/java/net/osgiliath/FakerProcessingIT.java index ae61553f..dc713e2d 100644 --- a/sample-mono/src/test/java/net/osgiliath/FakerProcessingIT.java +++ b/sample-mono/src/test/java/net/osgiliath/FakerProcessingIT.java @@ -2,8 +2,8 @@ import liquibase.exception.LiquibaseException; import liquibase.integration.spring.SpringLiquibase; -import net.osgiliath.datamigrator.sample.domain.Country; import net.osgiliath.datamigrator.sample.domain.Employee; +import net.osgiliath.datamigrator.sample.repository.EmployeeRepository; import net.osgiliath.migrator.core.api.metamodel.MetamodelScanner; import net.osgiliath.migrator.core.api.metamodel.model.FieldEdge; import net.osgiliath.migrator.core.api.metamodel.model.MetamodelVertex; @@ -12,8 +12,6 @@ import net.osgiliath.migrator.core.modelgraph.ModelGraphBuilder; import net.osgiliath.migrator.core.processing.SequenceProcessor; import net.osgiliath.migrator.sample.orchestration.DataMigratorApplication; -import net.osgiliath.datamigrator.sample.repository.CountryRepository; -import net.osgiliath.datamigrator.sample.repository.EmployeeRepository; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.jgrapht.Graph; import org.junit.jupiter.api.Test; @@ -36,16 +34,17 @@ import static org.assertj.core.api.Assertions.assertThat; @Testcontainers -@SpringBootTest(classes = { DataMigratorApplication.class }) +@SpringBootTest(classes = {DataMigratorApplication.class}) class FakerProcessingIT { static { - System.setProperty("liquibase.duplicateFileMode", "WARN"); + System.setProperty("liquibase.duplicateFileMode", "WARN"); } + private static final Logger logger = LoggerFactory.getLogger(FakerProcessingIT.class); @Container static MySQLContainer mySQLSourceContainer = new MySQLContainer(DockerImageName.parse("mysql:latest")); - // .withExposedPorts(64449); + // .withExposedPorts(64449); @Container static MySQLContainer mySQLTargetContainer = new MySQLContainer(DockerImageName.parse("mysql:latest")); @@ -67,12 +66,12 @@ static void mySQLProperties(DynamicPropertyRegistry registry) { registry.add("spring.datasource.sink.type", () -> "com.zaxxer.hikari.HikariDataSource"); registry.add("spring.datasource.sink.hikari.poolName", () -> "sinkHikari"); registry.add("spring.datasource.sink.hikari.auto-commit", () -> false); - DataSource ds = DataSourceBuilder.create() - .url(mySQLSourceContainer.getJdbcUrl()) - .username(mySQLSourceContainer.getUsername()) - .password(mySQLSourceContainer.getPassword()) - .driverClassName(mySQLSourceContainer.getDriverClassName()) - .build(); + DataSource ds = DataSourceBuilder.create() + .url(mySQLSourceContainer.getJdbcUrl()) + .username(mySQLSourceContainer.getUsername()) + .password(mySQLSourceContainer.getPassword()) + .driverClassName(mySQLSourceContainer.getDriverClassName()) + .build(); try { logger.warn("Starting Liquibase import"); SpringLiquibase liquibase = new SpringLiquibase(); @@ -114,8 +113,11 @@ void givenFedGraphWhenEntityProcessorAndSequenceProcessorIsCalledThenTargetDatab assertThat(employees).hasSize(9); List firstNames = employees.stream().map(Employee::getFirstName).toList(); assertThat(firstNames).doesNotContain("Shanny", "Chaz", "Horace", "Korbin", "Israel", "Javon", "Beryl", "Everett", "Destiny", "Sandrine"); - firstNames.stream().forEach(c -> - assertThat(c).isNotEmpty() + firstNames.stream().forEach(c -> { + + logger.warn("faked values {}", c); + assertThat(c).isNotEmpty(); + } ); } } diff --git a/sample-mono/src/test/resources/application.yml b/sample-mono/src/test/resources/application.yml index 7e82f680..bedb94e7 100644 --- a/sample-mono/src/test/resources/application.yml +++ b/sample-mono/src/test/resources/application.yml @@ -6,25 +6,28 @@ data-migrator: - column-anonymize-2 - anonymizer-custom-1 sequencers: -# - name: rows-minimize-1 -# type: rows-minimize -# rows-to-keep: 10 -# - name: column-anonymize-1 -# type: column-anonymizers -# table: virtuals -# column: password -# faker: RockBand -# - name: column-anonymize-2 -# type: column-anonymizers -# table: virtuals -# column: biosspass -# faker: Disease + # - name: rows-minimize-1 + # type: rows-minimize + # rows-to-keep: 10 + # - name: column-anonymize-1 + # type: column-anonymizers + # table: virtuals + # column: password + # faker: RockBand + # - name: column-anonymize-2 + # type: column-anonymizers + # table: virtuals + # column: biosspass + # faker: Disease - name: column-anonymize-1 type: factory transformer-class: net.osgiliath.migrator.modules.faker.ColumnFaker entity-class: Employee - columns: - - firstName + column-transformation-definitions: + - column-name: firstName + options: + - faker-class: net.datafaker.providers.entertainment.Dragonball + faker-function: character # https://www.datafaker.net/documentation/getting-started/ for options, will default to a dragonball character if not set - name: anonymizer-custom-1 type: bean @@ -58,9 +61,9 @@ spring: hibernate: id.new_generator_mappings: true # cache: -# use_second_level_cache: true -# use_query_cache: false -# region.factory_class: jcache + # use_second_level_cache: true + # use_query_cache: false + # region.factory_class: jcache generate_statistics: false # modify batch size as necessary jdbc: