Handle collisions between spec values and dictionary 225

tdrules-store-loader/pom.xml

@@ -62,6 +62,10 @@
 			<groupId>junit</groupId>
 			<artifactId>junit</artifactId>
 		</dependency>
+		<dependency>
+			<groupId>pl.pragmatists</groupId>
+			<artifactId>JUnitParams</artifactId>
+		</dependency>
 		<dependency>
 			<groupId>io.github.javiertuya</groupId>
 			<artifactId>portable-java</artifactId>

tdrules-store-loader/src/main/java/giis/tdrules/store/loader/EntityLoader.java

@@ -104,7 +104,7 @@ String loadValues(String entity, String[] attrNames, String[] attrValues) {
 			initializeGeneratedAttribute(null, gattrs, attr, currentEntity);
 
 		// Configures the attributes with specified values or symbols
-		setSpecifiedValues(gattrs, attrNames, attrValues);
+		setSpecifiedValues(gattrs, entity, attrNames, attrValues);
 		if (isArray)
 			setArrayKeyAttributes(gattrs);
 
@@ -138,18 +138,18 @@ private void initializeGeneratedAttribute(GeneratedAttribute parent, List<Genera
 		}
 	}
 
-	private void setSpecifiedValues(List<GeneratedAttribute> gattrs, String[] attrNames, String[] attrValues) {
+	private void setSpecifiedValues(List<GeneratedAttribute> gattrs, String entity, String[] attrNames, String[] attrValues) {
 		for (int i = 0; i < attrNames.length; i++) {
 			if (!attrNames[i].trim().equals("")) {
 				log.trace("setSpecifiedValues: attrName={} attrValue={}", attrNames[i], attrValues[i]);
 				GeneratedAttribute gattr = findGeneratedAttribute(gattrs, attrNames[i]);
 				log.trace("setSpecifiedValues:   before: {}", gattr);
-				setSpecifiedValuesForAttribute(gattr, attrValues[i]);
+				setSpecifiedValuesForAttribute(gattr, entity, attrValues[i]);
 				log.trace("setSpecifiedValues:    after: {}", gattr);
 			}
 		}
 	}
-	private void setSpecifiedValuesForAttribute(GeneratedAttribute gattr, String attrValue) {
+	private void setSpecifiedValuesForAttribute(GeneratedAttribute gattr, String entity, String attrValue) {
 		gattr.specValue = attrValue;
 		if (symbols.isSymbol(attrValue)) {
 			// The specified value is symbolic, it is a uid that will store its value in a symbol
@@ -162,7 +162,8 @@ private void setSpecifiedValuesForAttribute(GeneratedAttribute gattr, String att
 				gattr.genType = GenType.SPEC_FKSYM; // rid with symbolic value of a uid
 		} else {
 			gattr.genType = GenType.SPEC_USER;
-			gattr.genValue = attrValue;
+			// Even if specified, when using dictionaries it is possible some additional change because of collisions
+			gattr.genValue = config.attrGen.transformSpecValue(entity, gattr.attr.getName(), attrValue);
 		}
 		// If composite, as a value has been specified, remove descendants
 		if (!gattr.isPrimitive())

tdrules-store-loader/src/main/java/giis/tdrules/store/loader/IAttrGen.java

@@ -17,7 +17,7 @@ public interface IAttrGen {
 	void reset();
 
 	/**
-	 * Generates a string with the specified maximum lenght
+	 * Generates a string with the specified maximum length
 	 */
 	String generateString(String entityName, String attrName, int maxLength);
 
@@ -51,6 +51,16 @@ public interface IAttrGen {
 	 */
 	String generateCheckInConstraint(String[] allowedValues);
 
+	/**
+	 * Specified values are not generated, but this method must be invoked
+	 * because some times these values require some kind of transformation
+	 * and override this default implementation
+	 * (e.g. the case of dictionaries when the specified value matches any in the dictionary)
+	 */
+	default String transformSpecValue(String entityName, String attrName, String value) {
+		return value;
+	}
+
 	/**
 	 * Determines if a null value must be genrated for a given atribute with a given
 	 * probability.

tdrules-store-loader/src/main/java/giis/tdrules/store/loader/gen/DictionaryAttrGen.java

@@ -1,10 +1,15 @@
 package giis.tdrules.store.loader.gen;
 
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import giis.tdrules.store.loader.IConstraint;
 
 /**
@@ -19,13 +24,17 @@
  * desired configuration. All configuration methods are fluent.
  */
 public class DictionaryAttrGen extends DeterministicAttrGen {
+	private static final Logger log=LoggerFactory.getLogger(DictionaryAttrGen.class);
 
 	private SortedMap<String, DictionaryContainer> containers = new TreeMap<>();
 	private DictionaryContainer currentConfiguringContainer;
 
 	//Stores all configurations for a given coordinate
 	public class DictionaryContainer {
 		private String[] values;
+		// forbidden because some has been user specified
+		private Set<String> blacklist;
+
 		private int lastIndex = -1;
 
 		private String mask = "";
@@ -38,10 +47,11 @@ public class DictionaryContainer {
 
 		public void reset() {
 			lastIndex = -1;
+			blacklist= new HashSet<>();
 		}
 
 		public boolean hasValues() {
-			return values != null;
+			return values != null && this.values.length > 0;
 		}
 
 		public boolean hasMask() {
@@ -59,6 +69,15 @@ private String padValue(String value) {
 			return hasPad() ? String.format(padFormat, value).replace(' ', padChar) : value;
 		}
 
+		private int indexOf(String value) {
+			if (!this.hasValues()) 
+				return -1;
+			for (int i = 0; i < this.values.length; i++)
+				if (this.values[i].equals(value))
+					return i;
+			return -1;			
+		}
+
 		@Override
 		public String toString() {
 			return Arrays.asList(values).toString();
@@ -91,6 +110,7 @@ public DictionaryAttrGen with(String entityName, String attrName) {
 	 */
 	public DictionaryAttrGen dictionary(String... values) {
 		currentConfiguringContainer.values = values;
+		currentConfiguringContainer.blacklist = new HashSet<>();
 		return this;
 	}
 
@@ -132,19 +152,32 @@ private DictionaryContainer getDictionary(String entityName, String attrName) {
 	}
 
 	private String getNewStringFromDictionary(DictionaryContainer container) {
-		if (container.values == null)
+		if (!container.hasValues())
 			return null;
 		container.lastIndex++;
+		skipBlacklist(container);
+		return getStringFromDictionary(container, container.lastIndex);
+	}
+
+	private String getStringFromDictionary(DictionaryContainer container, int index) {
 		// if all strings have been generated, recycles the dictionary
-		int actualIndex = container.lastIndex % container.values.length;
-		int actualCycle = container.lastIndex / container.values.length;
+		int actualIndex = index % container.values.length;
+		int actualCycle = index / container.values.length;
 
 		String value = container.values[actualIndex];
 		if (actualCycle > 0) //to generate different values even if recycled
 			value += "-" + actualCycle;
 		return value;
 	}
 
+	private void skipBlacklist(DictionaryContainer container) {
+		for (int i=container.lastIndex; i<container.values.length; i++)
+			if (container.blacklist.contains(container.values[i]))
+				container.lastIndex++;
+			else
+				return;
+	}
+
 	private String getKey(String entityName, String attrName) {
 		return entityName.toLowerCase() + "." + attrName.toLowerCase();
 	}
@@ -184,5 +217,21 @@ public String generateNumber(IConstraint constraints, String entityName, String
 			value = container.maskValue(value);
 		return value;
 	}
+
+	@Override
+	public String transformSpecValue(String entityName, String attrName, String value) {
+		DictionaryContainer container = getDictionary(entityName, attrName);
+		if (container == null || !container.hasValues())
+			return value; // transparent because there are no values to choose in the dictionary
+
+		// Manage collisions between spec value and items in dictionary
+		int index = container.indexOf(value);
+		if (index >= 0) { // collision detected, the dictionary items is blackliste
+			container.blacklist.add(container.values[index]);
+			log.warn("Collision between specified value '{}' and an item in the dictionary, removing this item", value);
+		}
+
+		return value;
+	}
 
 }

tdrules-store-loader/src/test/java/test4giis/tdrules/store/loader/oa/TestOaDictionaryCollisions.java

@@ -0,0 +1,67 @@
+package test4giis.tdrules.store.loader.oa;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import junitparams.JUnitParamsRunner;
+import junitparams.Parameters;
+import giis.tdrules.openapi.model.TdAttribute;
+import giis.tdrules.openapi.model.TdSchema;
+import giis.tdrules.store.loader.DataLoader;
+import giis.tdrules.store.loader.gen.DictionaryAttrGen;
+import giis.tdrules.store.loader.oa.OaLocalAdapter;
+import giis.tdrules.openapi.model.TdEntity;
+
+/**
+ * Test data generation both with and without dictionaries is in
+ * TestOaLocalGeneration; here generation when there are collisions between
+ * specified values for a string attribute and values in the dictionary is
+ * tested (this causes these items removed from the dictionary)
+ */
+@RunWith(JUnitParamsRunner.class)
+public class TestOaDictionaryCollisions extends Base {
+
+	protected TdSchema getDataTypesModel() {
+		TdEntity entity = new TdEntity().name("ent")
+				.addAttributesItem(new TdAttribute().name("pk").datatype("integer").uid("true").notnull("true"))
+				.addAttributesItem(new TdAttribute().name("str").datatype("string").notnull("true"));
+		return new TdSchema().storetype("openapi").addEntitiesItem(entity);
+	}
+
+	protected DataLoader getGenerator(TdSchema model) {
+		return new DataLoader(model, new OaLocalAdapter())
+				.setAttrGen(new DictionaryAttrGen().with("ent", "str").dictionary("aa", "bb", "cc"));
+	}
+
+	@Test
+	@Parameters({ "_;xx;_;_;_;_ , aa;xx;bb;cc;aa-1;bb-1, no collision",
+			"bb;_;_;_;_;_ , bb;aa;cc;aa-1;bb-1;cc-1, spec before collision index",
+			"_;bb;_;_;_;_ , aa;bb;cc;aa-1;bb-1;cc-1, spec at collision index",
+			"_;aa;_;_;_;_ , aa;aa;bb;cc;aa-1;bb-1, spec after collision index (can't avoid)",
+			"_;_;_;aa;_;_ , aa;bb;cc;aa;aa-1;bb-1,   spec after collision index at recycle position",
+
+			"bb;_;bb;_;_;_ , bb;aa;bb;cc;aa-1;bb-1, multiple spec before collision index",
+			"_;bb;_;bb;_;_ , aa;bb;cc;bb;aa-1;bb-1, multiple spec at collision index",
+
+			"aa;_;_;_;_;_ , aa;bb;cc;aa-1;bb-1;cc-1, collision at first index",
+			"_;_;cc;_;_;_ , aa;bb;cc;aa-1;bb-1;cc-1, collision at last index",
+
+			"aa-1;_;_;_;_;_ , aa-1;aa;bb;cc;aa-1;bb-1, exclude collision with recycled items",
+			"_;_;_;aa-1;_;_ , aa;bb;cc;aa-1;aa-1;bb-1, exclude collision with recycled items",
+			})
+	public void testDictionaryCollisionWithSpecValue(String specs, String outs, String message) {
+		String[] spec = specs.split(";");
+		String[] out = outs.split(";");
+		DataLoader dtg = getGenerator(getDataTypesModel());
+		StringBuilder sb = new StringBuilder();
+		for (int i = 0; i < Math.max(spec.length, out.length); i++) {
+			spec[i] = spec[i].trim();
+			out[i] = out[i].trim();
+			dtg.load("ent", ("_".equals(spec[i]) ? "" : "str=" + spec[i]));
+			sb.append("\"ent\":{\"pk\":" + (i * 100 + 1) + ",\"str\":\"" + out[i] + "\"}\n");
+		}
+		assertEquals(message, sb.toString().trim(), dtg.getDataAdapter().getAllAsString());
+	}
+
+}