diff --git a/.classpath b/.classpath new file mode 100644 index 00000000..9fe03ad8 --- /dev/null +++ b/.classpath @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.project b/.project new file mode 100644 index 00000000..a73ac803 --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + Clustering + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.core.javanature + + diff --git a/CTD/.classpath b/CTD/.classpath index 50ec62a5..d5cd6b1c 100644 --- a/CTD/.classpath +++ b/CTD/.classpath @@ -6,17 +6,18 @@ + - + diff --git a/CTD/src/edu/isnap/hint/HintConfig.java b/CTD/src/edu/isnap/hint/HintConfig.java index 8c5e320b..39b48ef7 100644 --- a/CTD/src/edu/isnap/hint/HintConfig.java +++ b/CTD/src/edu/isnap/hint/HintConfig.java @@ -11,6 +11,9 @@ public abstract class HintConfig implements Serializable { /** If true, uses new version of SourceCheck with more global alignment */ public boolean sourceCheckV2 = false; + + /** If true, uses reference solutions to get high-level hints */ + public boolean useAnnotation = true; /** * Should return true if the hint generator can expect traces to keep consistent node IDs diff --git a/CTD/src/edu/isnap/hint/HintData.java b/CTD/src/edu/isnap/hint/HintData.java index 961fda3e..0454700d 100644 --- a/CTD/src/edu/isnap/hint/HintData.java +++ b/CTD/src/edu/isnap/hint/HintData.java @@ -1,6 +1,7 @@ package edu.isnap.hint; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import edu.isnap.ctd.hint.CTDHintGenerator; @@ -17,7 +18,15 @@ public class HintData { public final double minGrade; public final HintConfig config; private final List dataModels; + private LinkedHashMap referenceSolutions; + /** + * @param assignment the name of java file used in this assignment. Omit ".java". + * @param config + * @param minGrade + * @param consumer + * @param additionalConsumers + */ public HintData(String assignment, HintConfig config, double minGrade, IDataConsumer consumer, IDataConsumer... additionalConsumers) { this.assignment = assignment; @@ -29,6 +38,7 @@ public HintData(String assignment, HintConfig config, double minGrade, for (IDataConsumer con : additionalConsumers) { addDataModels(con.getRequiredData(this)); } + referenceSolutions = new LinkedHashMap<>(); } private void addDataModels(IDataModel[] models) { @@ -73,4 +83,12 @@ public CTDHintGenerator hintGenerator() { public HintHighlighter hintHighlighter() { return new HintHighlighter(this); } + + public LinkedHashMap getReferenceSolutions() { + return referenceSolutions; + } + + public Node addReferenceSoltion(int clusterID, Node solution) { + return referenceSolutions.put(clusterID, solution); + } } diff --git a/CTD/src/edu/isnap/hint/util/Alignment.java b/CTD/src/edu/isnap/hint/util/Alignment.java index 59807c39..463d90ba 100644 --- a/CTD/src/edu/isnap/hint/util/Alignment.java +++ b/CTD/src/edu/isnap/hint/util/Alignment.java @@ -25,6 +25,14 @@ public static double normAlignCost(String[] sequenceA, String[] sequenceB, int i } // Credit: http://introcs.cs.princeton.edu/java/96optimization/Diff.java.html + /** + * @param sequenceA + * @param sequenceB + * @param insCost + * @param delCost + * @param subCost + * @return Levenshtein distance between "sequenceA" and "sequenceB"?? + */ public static int alignCost(String[] sequenceA, String[] sequenceB, int insCost, int delCost, int subCost) { int[][] opt = createAlignmentMatrix(sequenceA, sequenceB, insCost, delCost, subCost, false); @@ -300,7 +308,7 @@ private static int correspondingIndexBefore(List pairs, List sequ return closestA; } - private static int[][] createAlignmentMatrix(String[] sequenceA, String[] sequenceB, + public static int[][] createAlignmentMatrix(String[] sequenceA, String[] sequenceB, int insCost, int delCost, int subCost, boolean flipInvalid) { // The penalties to apply int matchCost = 0; @@ -397,7 +405,7 @@ public static int[] reorderIndices(String[] from, String[] to, int[] toOrderGrou return toIndices; } - public static int getProgress(String[] from, String[] to, int orderReward, int unorderReward) { + public static double getProgress(String[] from, String[] to, int orderReward, int unorderReward) { return (int) Math.round(getProgress(from, to, orderReward, unorderReward, 0)); } @@ -417,14 +425,19 @@ public static double getProgress(String[] from, String[] to, int[] toOrderGroups private static double getProgress(String[] from, String[] to, int[] toOrderGroups, int orderReward, int unorderReward, double skipCost, int[] toIndices) { // TODO: This can and should be much more efficient + // toList[k] = "\0" if the k-th node in c(b_{rj}) is in c(a_{ri}) List toList = new LinkedList<>(Arrays.asList(to)); + // If indices[k] = l, k-th node in c(a_{ri}) is l-th node in c(b_{rj}) + // If indices[k] = -1, k-th node in c(a_{ri}) is not in c(b_{rj}) int[] indices = new int[from.length]; for (int i = 0; i < from.length; i++) { String item = from[i]; int index = toList.indexOf(item); - if (index >= 0) { - toList.set(index, "\0"); + if (index >= 0) { // i-th node in c(a_{ri}) is in c(b_{rj}) +// if (orderReward == 1 && unorderReward == 1) { + toList.set(index, "\0"); // prevents multiple matches +// } indices[i] = index; } else { indices[i] = -1; @@ -437,7 +450,7 @@ private static double getProgress(String[] from, String[] to, int[] toOrderGroup int lastIndex = -1; int maxIndex = -1; for (Integer index : indices) { - if (index < 0) continue; + if (index < 0) continue; // don't change reward if k-th node in c(a_{ri}) is not in c(b_{rj}) int adjIndex = index; int group; // System.out.println(index); @@ -487,7 +500,7 @@ private static double getProgress(String[] from, String[] to, int[] toOrderGroup * including duplicates */ public static int getMissingNodeCount(String[] from, String[] to) { - return to.length - getProgress(to, from, 1, 1); + return to.length - (int) getProgress(to, from, 1, 1); } public static void main(String[] args) { diff --git a/CTD/src/edu/isnap/node/Node.java b/CTD/src/edu/isnap/node/Node.java index e5e20a19..16da6f63 100644 --- a/CTD/src/edu/isnap/node/Node.java +++ b/CTD/src/edu/isnap/node/Node.java @@ -37,6 +37,8 @@ public abstract class Node extends StringHashable implements INode { public final String value; public final Node parent; public final List children = new ArrayList<>(); + private String studentID; + private String submissionTime; public transient Object tag; public final transient List canonicalizations = new ArrayList<>(); @@ -501,6 +503,10 @@ public String prettyPrintWithIDs() { return prettyPrint(false, new HashMap()); } + /** + * @return a list of strings of types obtained by depth-first iteration of + * the AST rooted at this node + */ public String[] depthFirstIteration() { String[] array = new String[treeSize()]; depthFirstIteration(array, 0); @@ -683,6 +689,11 @@ public static Node fromASTNode(ASTNode astNode, Node parent, NodeConstructor con Node node = constructor.constructNode(parent, type, astNode.value, astNode.id); node.readFromASTNode(astNode); node.tag = astNode; + if (astNode.annotation != null) { + node.writableAnnotations(); + TextHint hint = new TextHint(astNode.annotation, 1); + node.addTextHint(hint); + } for (ASTNode child : astNode.children()) { node.children.add(fromASTNode(child, node, constructor)); } @@ -697,4 +708,20 @@ protected void readFromASTNode(ASTNode node) { public String toString() { return prettyPrint(true); } + + public String getStudentID() { + return studentID; + } + + public void setStudentID(String studentID) { + this.studentID = studentID; + } + + public String getSubmissionTime() { + return submissionTime; + } + + public void setSubmissionTime(String submissionTime) { + this.submissionTime = submissionTime; + } } diff --git a/CTD/src/edu/isnap/node/TextualNode.java b/CTD/src/edu/isnap/node/TextualNode.java index 8a63604e..a166306c 100644 --- a/CTD/src/edu/isnap/node/TextualNode.java +++ b/CTD/src/edu/isnap/node/TextualNode.java @@ -1,15 +1,16 @@ package edu.isnap.node; -import java.util.Optional; -import org.json.JSONObject; +import java.util.Optional; +import org.json.JSONObject; import edu.isnap.node.ASTNode.SourceLocation; import edu.isnap.sourcecheck.NodeAlignment.Mapping; public abstract class TextualNode extends Node { public Optional correct = Optional.empty(); + public Optional cluster = Optional.empty(); private String source; // TODO: Protect @@ -49,7 +50,7 @@ public String getSource() { String rootSource = ((TextualNode) root()).source; if (startSourceLocation == null && endSourceLocation == null) return null; return rootSource.substring( - toIndex(rootSource, startSourceLocation), + toIndex(rootSource, startSourceLocation) - 1, toIndex(rootSource, endSourceLocation)); } @@ -104,12 +105,27 @@ public SourceLocation getLocationOfChildIndex(int index) { return null; } - public static TextualNode fromJSON(JSONObject jsonAST, String source, - NodeConstructor constructor) { + public static TextualNode fromJSON(JSONObject jsonAST, String source, NodeConstructor constructor) { ASTSnapshot astNode = ASTSnapshot.parse(jsonAST, source); TextualNode node = (TextualNode) fromASTNode(astNode, constructor); node.source = source; node.correct = Optional.of(astNode.isCorrect); return node; } + + /** + * Overloaded method to get clusterID + * @param jsonAST + * @param id + * @param source + * @param constructor + * @return + */ + public static TextualNode fromJSON(JSONObject jsonAST,String id, String source, NodeConstructor constructor) { + ASTSnapshot astNode = ASTSnapshot.parse(jsonAST, id, source); + TextualNode node = (TextualNode) fromASTNode(astNode, constructor); + node.source = source; + node.correct = Optional.of(astNode.isCorrect); + return node; + } } \ No newline at end of file diff --git a/CTD/src/edu/isnap/sourcecheck/ConsensusHintHighlighter.java b/CTD/src/edu/isnap/sourcecheck/ConsensusHintHighlighter.java index 9a29f5a3..08e06108 100644 --- a/CTD/src/edu/isnap/sourcecheck/ConsensusHintHighlighter.java +++ b/CTD/src/edu/isnap/sourcecheck/ConsensusHintHighlighter.java @@ -38,7 +38,7 @@ public List highlightConsensus(Node node) { double consensusThreshold = 0.6; List matches = NodeAlignment.findBestMatches( - node, solutions, getDistanceMeasure(config), config, nMatches); + node, getSolutions(), getDistanceMeasure(config), config, nMatches); final int n = matches.size(); final int thresh = (int) Math.ceil(n * consensusThreshold); diff --git a/CTD/src/edu/isnap/sourcecheck/HintHighlighter.java b/CTD/src/edu/isnap/sourcecheck/HintHighlighter.java index 799a0c90..cf80db02 100644 --- a/CTD/src/edu/isnap/sourcecheck/HintHighlighter.java +++ b/CTD/src/edu/isnap/sourcecheck/HintHighlighter.java @@ -50,7 +50,7 @@ public static enum Highlight { public final HintConfig config; public final HintData hintData; - final List solutions; + private final List solutions; public final static IDataConsumer DataConsumer = new IDataConsumer() { @Override @@ -106,12 +106,16 @@ public List highlight(Node node) { return highlight(node, mapping); } + /** + * @param node current student's code for which we show hints + * @return a list of edits with priorities + */ public List highlightWithPriorities(Node node) { Mapping mapping = findSolutionMapping(node); return highlightWithPriorities(node, mapping); } - private List highlightWithPriorities(Node node, Mapping mapping) { + public List highlightWithPriorities(Node node, Mapping mapping) { List hints = highlight(node, mapping); new HintPrioritizer(this).assignPriorities(mapping, hints); return hints; @@ -121,6 +125,12 @@ public List highlight(Node node, final Mapping mapping) { return highlight(node, mapping, true); } + /** + * @param node current student's code for which we show hints + * @param mapping + * @param reuseDeletedBlocks + * @return + */ public List highlight(Node node, final Mapping mapping, boolean reuseDeletedBlocks) { final List edits = new ArrayList<>(); @@ -555,6 +565,10 @@ public static DistanceMeasure getDistanceMeasure(HintConfig config) { return new ProgressDistanceMeasure(config); } + /** + * @param node current student's code for which we show hints + * @return the mapping from "node" to the closest solution + */ public Mapping findSolutionMapping(Node node) { long startTime = System.currentTimeMillis(); @@ -574,16 +588,21 @@ public Mapping findSolutionMapping(Node node) { return bestMatch; } + /** + * @param node current student's code for which we show hints + * @param maxReturned the number of the closest codes to return + * @return a mapping from "node" to maxReturned number of closest "solutions" + */ public List findBestMappings(Node node, int maxReturned) { DistanceMeasure dm = getDistanceMeasure(config); - List filteredSolutions = solutions; + List filteredSolutions = getSolutions(); RulesModel rulesModel = hintData.getModel(RulesModel.class); RuleSet ruleSet = rulesModel == null ? null : rulesModel.getRuleSet(); if (ruleSet != null) { RuleSet.trace = trace; - filteredSolutions = ruleSet.filterSolutions(solutions, node); + filteredSolutions = ruleSet.filterSolutions(getSolutions(), node); if (filteredSolutions.size() == 0) throw new RuntimeException("No solutions!"); } return NodeAlignment.findBestMatches(node, filteredSolutions, dm, config, maxReturned); @@ -775,7 +794,7 @@ public List highlightStringEdit(Node node) { double minDis = Double.MAX_VALUE; Node best = null; - for (Node solution : solutions) { + for (Node solution : getSolutions()) { double dis = Alignment.alignCost(nodeSeq, solution.depthFirstIteration()); if (dis < minDis) { best = solution; @@ -796,4 +815,8 @@ public List highlightStringEdit(Node node) { return highlight(node, mapping); } + + public List getSolutions() { + return solutions; + } } diff --git a/CTD/src/edu/isnap/sourcecheck/NodeAlignment.java b/CTD/src/edu/isnap/sourcecheck/NodeAlignment.java index 6238973f..ce268fae 100644 --- a/CTD/src/edu/isnap/sourcecheck/NodeAlignment.java +++ b/CTD/src/edu/isnap/sourcecheck/NodeAlignment.java @@ -72,6 +72,9 @@ public Mapping(Node from, Node to, HintConfig config) { } } + /** + * @return distance between "from" and "to"?? + */ public double cost() { return cost; } @@ -300,21 +303,22 @@ public Mapping calculateMapping(DistanceMeasure distanceMeasure) { return calculateMapping(distanceMeasure, null); } - private Mapping calculateMapping(DistanceMeasure distanceMeasure, Mapping previousMapping) { + public Mapping calculateMapping(DistanceMeasure distanceMeasure, Mapping previousMapping) { mapping = new Mapping(from, to, config); // If we're given a previous mapping, we use it to calculate value mappings if (previousMapping != null) mapping.calculateValueMappings(previousMapping); + // Behaves differently depending on previousMapping to.resetAnnotations(); ListMap fromMap = getChildMap(from, true); ListMap toMap = getChildMap(to, false); - for (String key : fromMap.keySet()) { + for (String key : fromMap.keySet()) { // Iterate over root paths from shortest to longest List fromNodes = fromMap.get(key); List toNodes = toMap.get(key); if (toNodes == null) { - // Continue if we have no toNodes to match + // Continue if we have no toNodes to match because the same root path isn't in "to" (i.e. B) continue; } @@ -322,8 +326,8 @@ private Mapping calculateMapping(DistanceMeasure distanceMeasure, Mapping previo // other containers e.g. Sprites, so we make sure only descendants of a given container // are aligned // TODO: Why do String IDs work so much faster than IDHashMap - ListMap fromContainers = new ListMap<>(); - ListMap toContainers = new ListMap<>(); + ListMap fromContainers = new ListMap<>(); // map root node or null to "from"? + ListMap toContainers = new ListMap<>(); // map root node or null to "to"? for (Node from : fromNodes) { Node container = getContainer(distanceMeasure, from); fromContainers.add(container == null ? null : container.id, from); @@ -334,6 +338,7 @@ private Mapping calculateMapping(DistanceMeasure distanceMeasure, Mapping previo } for (String containerID : fromContainers.keySet()) { + // Get child sequences? List containedFrom = fromContainers.get(containerID); List containedTo = toContainers.get(containerID); if (containedTo == null) continue; @@ -430,6 +435,13 @@ public double matchedOrphanReward(String type) { } }; + /** + * + * @param fromNodes child sequence of "from" node (i.e. a_{ri} in the paper) + * @param toNodes child sequence of "to" node (i.e. b_{ri} in the paper) + * @param distanceMeasure + * @param fromMap + */ private void align(List fromNodes, List toNodes, DistanceMeasure distanceMeasure, final ListMap fromMap) { String[][] fromStates = stateArray(fromNodes, true); @@ -439,7 +451,7 @@ private void align(List fromNodes, List toNodes, String type = fromNodes.get(0).type(); double minCost = Integer.MAX_VALUE; - double[][] costMatrix = new double[fromStates.length][toStates.length]; + double[][] costMatrix = new double[fromStates.length][toStates.length]; // pairwise distance CountMap costCounts = new CountMap<>(); for (int i = 0; i < fromStates.length; i++) { for (int j = 0; j < toStates.length; j++) { @@ -452,7 +464,7 @@ private void align(List fromNodes, List toNodes, minCost = Math.min(minCost, cost); } } - for (int i = 0; i < fromStates.length; i++) { + for (int i = 0; i < fromStates.length; i++) { // Break ties for (int j = 0; j < toStates.length; j++) { Node from = fromNodes.get(i), to = toNodes.get(j); double cost = costMatrix[i][j]; @@ -490,7 +502,7 @@ private void align(List fromNodes, List toNodes, } HungarianAlgorithm alg = new HungarianAlgorithm(costMatrix); - int[] matching = alg.execute(); + int[] matching = alg.execute(); // matching[k] = l means the k-th node in a_{ri} is matched with the l-th node in b_{rj} Set matchedTo = new HashSet<>(); // We pre-compute whether each of the from and to nodes have been previous put in the @@ -658,6 +670,14 @@ private boolean needsReorder(int[] reorders) { return needsReorder; } + /** + * @param a + * @param b + * @param config + * @param dm + * @return the difference between string representations of a and b obtained by + * depth-first iteration + */ public static double getSubCostEsitmate(Node a, Node b, HintConfig config, DistanceMeasure dm) { String[] aDFI = a.depthFirstIteration(); String[] bDFI = b.depthFirstIteration(); @@ -678,6 +698,11 @@ private String[][] stateArray(List nodes, boolean isFrom) { return states; } + /** + * + * @param nodes + * @return relative positions of annotations? + */ private int[][] orderGroups(List nodes) { int[][] orderGroups = new int[nodes.size()][]; for (int i = 0; i < orderGroups.length; i++) { @@ -717,6 +742,14 @@ private String parentNodeKey(Node node, boolean isFrom) { return Arrays.toString(list); } + /** + * @param from node of current student's code for which we show hints + * @param matches lists from which we find the closest + * @param distanceMeasure the definition of distance between source codes + * @param config + * @param maxReturned the number of the closest codes to return + * @return a list of the mappings from "from" to codes in "matches" + */ public static List findBestMatches(Node from, List matches, DistanceMeasure distanceMeasure, HintConfig config, int maxReturned) { List best = new LinkedList<>(); @@ -825,15 +858,17 @@ private Mapping align(DistanceMeasure dm, Mapping previousMapping) { } HungarianAlgorithm alg = new HungarianAlgorithm(costMatrix); - int[] matching = alg.execute(); + // i-th node in fromNodes is assigned to matching[i]-th node in toNodes + int[] matching = alg.execute(); for (int i = 0; i < fromNodes.size(); i++) { int j = matching[i]; - if (j < 0) continue; + if (j < 0) continue; // i-th node in fromNodes is unassigned // TODO: determine/config - if (costMatrix[i][j] > 1000) continue; - mapping.put(fromNodes.get(i), toNodes.get(j)); + // too large. Leave j-th node in toNodes unassigned + if (costMatrix[i][j] > 1000) continue; + mapping.put(fromNodes.get(i), toNodes.get(j)); // add assignment to mapping } } @@ -923,6 +958,12 @@ private void calculateMappingReward() { }); } + /** + * + * @param node + * @param isFrom + * @return a list of strings of types of the parent nodes staring from the closest parent + */ private String[] getRootPathArray(Node node, boolean isFrom) { String[] rp = new String[node.rootPathLength()]; int i = rp.length - 1; @@ -933,6 +974,13 @@ private String[] getRootPathArray(Node node, boolean isFrom) { return rp; } + /** + * + * @param node + * @param isFrom true if node is the current student's code + * @return ListMap (extends LinkedHashMap) that maps type or value strings in the AST + * to "node" + */ private ListMap getNodesByType(Node node, boolean isFrom) { final ListMap map = new ListMap<>(MapFactory.LinkedHashMapFactory); node.recurse(new Action() { diff --git a/CTD/src/edu/isnap/sourcecheck/edit/Suggestion.java b/CTD/src/edu/isnap/sourcecheck/edit/Suggestion.java index cb1f8b2c..bf5961ef 100644 --- a/CTD/src/edu/isnap/sourcecheck/edit/Suggestion.java +++ b/CTD/src/edu/isnap/sourcecheck/edit/Suggestion.java @@ -27,11 +27,11 @@ public Suggestion(EditHint hint, SourceLocation location, SuggestionType type, b @Override public int compareTo(Suggestion s) { - if (location == null) return -1; - if (s == null) return 1; + if (location == null) return 1; + if (s == null) return -1; int comp = location.compareTo(s.location); // If the source locations are different, return the later one (we work backwards) - if (comp != 0) return -comp; + if (comp != 0) return comp; // Otherwise, process starts before ends, so spans don't overlap if (start != s.start) return start ? 1 : -1; // Otherwise use the type order (reversed so the first is processed last) diff --git a/HighLevelHints/.classpath b/HighLevelHints/.classpath new file mode 100644 index 00000000..064a38d2 --- /dev/null +++ b/HighLevelHints/.classpath @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/HighLevelHints/.project b/HighLevelHints/.project new file mode 100644 index 00000000..988f4ee4 --- /dev/null +++ b/HighLevelHints/.project @@ -0,0 +1,30 @@ + + + HighLevelHints + + + + + + org.eclipse.wst.common.project.facet.core.builder + + + + + org.eclipse.jdt.core.javabuilder + + + + + org.eclipse.wst.validation.validationbuilder + + + + + + org.eclipse.jem.workbench.JavaEMFNature + org.eclipse.wst.common.modulecore.ModuleCoreNature + org.eclipse.jdt.core.javanature + org.eclipse.wst.common.project.facet.core.nature + + diff --git a/HighLevelHints/src/alignment/JavaSolutionConfig.java b/HighLevelHints/src/alignment/JavaSolutionConfig.java new file mode 100644 index 00000000..3a9359c7 --- /dev/null +++ b/HighLevelHints/src/alignment/JavaSolutionConfig.java @@ -0,0 +1,16 @@ +package alignment; + +import edu.isnap.hint.HintConfig.ValuesPolicy; +import edu.isnap.java.JavaHintConfig; + +public class JavaSolutionConfig extends JavaHintConfig { + public double baseReward = 2; + public double factor = 0.5; + public double outOfOrderReward = 1; + + public JavaSolutionConfig() { + super(); + progressMissingFactor = 0; + valuesPolicy = ValuesPolicy.MatchAllExactly; + } +} diff --git a/HighLevelHints/src/alignment/SolutionAlignment.java b/HighLevelHints/src/alignment/SolutionAlignment.java new file mode 100644 index 00000000..faa7a868 --- /dev/null +++ b/HighLevelHints/src/alignment/SolutionAlignment.java @@ -0,0 +1,239 @@ +package alignment; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.Set; + +import edu.isnap.hint.HintConfig; +import edu.isnap.hint.util.Alignment; +import edu.isnap.node.Node; +import edu.isnap.sourcecheck.NodeAlignment; + +public class SolutionAlignment extends Alignment { + + public static double getFullOrderReward(double d, double baseReward, double factor) { + double reward = 0.0; + for (int i = 0; i < d; i++) { + reward += Math.pow(baseReward, Math.pow(factor, i)); + } + return reward; + } + + public static int getMissingNodeCount(String[] from, String[] to) { + return (int) (to.length - getProgress(to, from, 1, 1, true)); + } + +// public static double getProgress(String[] from, String[] to, int orderReward, int unorderReward) { +// return getProgress(from, to, orderReward, unorderReward, false); +// } + + public static double getProgress(String[] from, String[] to, int[] toOrderGroups, int orderReward, int unorderReward) { + return getProgress(from, to, toOrderGroups, orderReward, unorderReward, new int[to.length], false); + } + + private static double getProgress(String[] from, String[] to, int orderReward, + int unorderReward, boolean missing) { + return getProgress(from, to, null, orderReward, unorderReward, new int[to.length], missing); + } + + static double getProgress(String[] from, String[] to, int[] toOrderGroups, + int orderReward, int unorderReward, int[] toIndices, boolean missing) { + // TODO: This can and should be much more efficient + // toList[k] = "\0" if the k-th node in c(b_{rj}) is in c(a_{ri}) + List toList = new LinkedList<>(Arrays.asList(to)); + + // If indices[k] = l, k-th node in c(a_{ri}) is l-th node in c(b_{rj}) + // If indices[k] = -1, k-th node in c(a_{ri}) is not in c(b_{rj}) + //int[] indices = new int[from.length]; + List indices = new ArrayList<>(from.length); + for (int i = 0; i < from.length; i++) { + String item = from[i]; + int index = toList.indexOf(item); + if (index >= 0) { // i-th node in c(a_{ri}) is in c(b_{rj}) + if (missing) { + toList.set(index, "\0"); // prevents multiple matches + } + //indices[i] = index; + indices.add(i, index); + } else { + //indices[i] = -1; + indices.add(i, index); + } + } + + Arrays.fill(toIndices, -1); + +// Queue> oneToOneIndices = new LinkedList<>(); +// oneToOneIndices.add(indices); +// if (!missing) { +// for (int i = 0; i < to.length; i++) { +// List fromIndices = new ArrayList<>(); +// List example = oneToOneIndices.peek(); +// for (int j = 0; j < example.size(); j++) { +// if (example.get(j) == i) { +// fromIndices.add(j); +// } +// } +// if (fromIndices.size() < 2) { +// continue; // no duplicates +// } +// // Multiple elements in "from" point to one element in "to" +// int queueSize = oneToOneIndices.size(); +// for (int j = 0; j < queueSize; j++) { +// ArrayList polled = (ArrayList) oneToOneIndices.poll(); +// for (int index : fromIndices) { +// List temp = new ArrayList<>(polled.size()); +// for (int h = 0; h < polled.size(); h++) { +// if (fromIndices.indexOf(polled.get(h)) != -1 && h != index) { +// temp.add(h, -1); +// }else { +// temp.add(h, polled.get(h)); +// } +// } +// oneToOneIndices.add(new ArrayList(temp)); +// } +// } +// } +// } + + double maxReward = Double.NEGATIVE_INFINITY; +// for (List filteredIndices : oneToOneIndices) { + + Set usedIndices = new HashSet<>(to.length); + + double reward = 0; + int lastIndex = -1; + int maxIndex = -1; + for (Integer index : indices) {//filteredIndices) { + if (index < 0) continue; // don't change reward if k-th node in c(a_{ri}) is not in c(b_{rj}) + int adjIndex = index; + int group; +// System.out.println(index); + if (toOrderGroups != null && (group = toOrderGroups[adjIndex]) > 0) { // always false for Java SourceCheck + // If the matched "to" item is in an order group (for which all items in the group + // are unordered), we should match i to the index of the earliest item in this group + // which comes after the last index, since the actual match could have been + // reordered to that index without loss of meaning + + // First check if the index can be decreased within the order group without going + // <= the max seen index (to avoid duplicate adjusted indices) + while (adjIndex > 0 && adjIndex - 1 > maxIndex && + toOrderGroups[adjIndex - 1] == group) { + adjIndex--; +// System.out.println("m-> " + adjIndex); + } + // Next check if the index is out of order and increasing it to maxIndex + 1 will + // make in order + int nextIndex = maxIndex + 1; + if (nextIndex < toOrderGroups.length && adjIndex <= lastIndex && + toOrderGroups[nextIndex] == group) { + adjIndex = nextIndex; +// System.out.println("p-> " + adjIndex); + } + + // Set the actual to-index used after adjustments above + if (index != adjIndex) { + toIndices[index] = adjIndex; + } + } + + if (to[adjIndex] != null) { + reward += adjIndex > lastIndex ? orderReward : unorderReward; + if (!missing && adjIndex - lastIndex > 1 && !to[adjIndex].equals(from[indices.indexOf(index)])) { + boolean skipped = true;// && !usedIndices.isEmpty(); + for (int idx = lastIndex + 1; idx < adjIndex; idx++) { + skipped = skipped && usedIndices.contains(idx); + } + if (skipped) reward -= 1; + } + } + usedIndices.add(adjIndex); + lastIndex = adjIndex; + maxIndex = Math.max(maxIndex, adjIndex); + } + if (reward > maxReward) maxReward = reward; +// } + + return maxReward; + } + + public static double getNormalizedMissingNodeCount(String[] from, String[] to, + double baseReward, double factor) { + return getFullOrderReward(to.length, baseReward, factor) + - getNormalizedProgress(to, from, null, baseReward, 1, factor, new int[to.length]); + } + + public static double getNormalizedProgress(String[] from, String[] to, int[] toOrderGroups, + double baseReward, double unorderReward, double factor, int[] toIndices) { + // TODO: This can and should be much more efficient + // toList[k] = "\0" if the k-th node in c(b_{rj}) is in c(a_{ri}) + List toList = new LinkedList<>(Arrays.asList(to)); + + // If indices[k] = l, k-th node in c(a_{ri}) is l-th node in c(b_{rj}) + // If indices[k] = -1, k-th node in c(a_{ri}) is not in c(b_{rj}) + int[] indices = new int[from.length]; + for (int i = 0; i < from.length; i++) { + String item = from[i]; + int index = toList.indexOf(item); + if (index >= 0) { // i-th node in c(a_{ri}) is in c(b_{rj}) + toList.set(index, "\0"); + indices[i] = index; + } else { + indices[i] = -1; + } + } + + Arrays.fill(toIndices, -1); + + double reward = 0; + int lastIndex = -1; + int maxIndex = -1; + for (Integer index : indices) { + if (index < 0) continue; // don't change reward if k-th node in c(a_{ri}) is not in c(b_{rj}) + int adjIndex = index; + int group; +// System.out.println(index); + if (toOrderGroups != null && (group = toOrderGroups[adjIndex]) > 0) { // always false for Java SourceCheck + // If the matched "to" item is in an order group (for which all items in the group + // are unordered), we should match i to the index of the earliest item in this group + // which comes after the last index, since the actual match could have been + // reordered to that index without loss of meaning + + // First check if the index can be decreased within the order group without going + // <= the max seen index (to avoid duplicate adjusted indices) + while (adjIndex > 0 && adjIndex - 1 > maxIndex && + toOrderGroups[adjIndex - 1] == group) { + adjIndex--; +// System.out.println("m-> " + adjIndex); + } + // Next check if the index is out of order and increasing it to maxIndex + 1 will + // make in order + int nextIndex = maxIndex + 1; + if (nextIndex < toOrderGroups.length && adjIndex <= lastIndex && + toOrderGroups[nextIndex] == group) { + adjIndex = nextIndex; +// System.out.println("p-> " + adjIndex); + } + + // Set the actual to-index used after adjustments above + if (index != adjIndex) { + toIndices[index] = adjIndex; + } + } + + if (to[adjIndex] != null) { + reward += adjIndex > lastIndex ? Math.pow(baseReward, Math.pow(factor, adjIndex)) : + Math.pow(unorderReward, Math.pow(factor, adjIndex)); + } + lastIndex = adjIndex; + maxIndex = Math.max(maxIndex, adjIndex); + } + + return reward; + } + +} diff --git a/HighLevelHints/src/alignment/SolutionDistanceMeasure.java b/HighLevelHints/src/alignment/SolutionDistanceMeasure.java new file mode 100644 index 00000000..ade47cce --- /dev/null +++ b/HighLevelHints/src/alignment/SolutionDistanceMeasure.java @@ -0,0 +1,48 @@ +package alignment; + +import edu.isnap.hint.HintConfig; +import edu.isnap.hint.util.Alignment; +import edu.isnap.node.Node; +import edu.isnap.sourcecheck.NodeAlignment.DistanceMeasure; +import edu.isnap.sourcecheck.NodeAlignment.ProgressDistanceMeasure; + +public class SolutionDistanceMeasure extends ProgressDistanceMeasure { + + public SolutionDistanceMeasure(HintConfig config) { + super(config); + // TODO Auto-generated constructor stub + } + + @Override + public double measure(Node from, String[] a, String[] b, int[] bOrderGroups) { +// if (this.config instanceof JavaSolutionConfig) { +// JavaSolutionConfig config = (JavaSolutionConfig) this.config; +// return SolutionAlignment.getNormalizedMissingNodeCount(a, b, config.baseReward, config.factor) +// + SolutionAlignment.getNormalizedMissingNodeCount(b, a, config.baseReward, config.factor) +// + SolutionAlignment.getFullOrderReward( +// SolutionAlignment.getProgress(a, b, 1, 1), config.baseReward, config.factor) +// - (SolutionAlignment.getNormalizedProgress( +// a, b, bOrderGroups, config.baseReward, config.outOfOrderReward, config.factor, new int[b.length])); +// + SolutionAlignment.getNormalizedProgress(b, a, null, config.baseReward, config.outOfOrderReward, config.factor, new int[a.length]) +// ) / 2; +// } + return SolutionAlignment.getMissingNodeCount(a, b) + SolutionAlignment.getMissingNodeCount(b, a) + + inOrderReward * SolutionAlignment.getProgress(a, b, 1, 1) + - Math.max(SolutionAlignment.getProgress(a, b, bOrderGroups, inOrderReward, outOfOrderReward, 0), + SolutionAlignment.getProgress(b, a, null, inOrderReward, outOfOrderReward, 0)); + } + + @Override + public double matchedOrphanReward(String type) { + return 0; + } + public static void main(String[] args) { + HintConfig temp = new JavaSolutionConfig(); + DistanceMeasure sol = new SolutionDistanceMeasure(temp); + String[] a = {"1", "2", "3", "4", "5"}; + String[] b = {"1", "2", "4", "3", "3", "5"}; + System.out.println(sol.measure(null, a, b, new int[b.length])); + System.out.println(sol.measure(null, b, a, new int[a.length])); + } + +} diff --git a/HighLevelHints/src/clustering/DBSCAN.java b/HighLevelHints/src/clustering/DBSCAN.java new file mode 100644 index 00000000..3c57bc70 --- /dev/null +++ b/HighLevelHints/src/clustering/DBSCAN.java @@ -0,0 +1,22 @@ +package clustering; + +import net.sf.javaml.core.*; +import net.sf.javaml.clustering.*; +import net.sf.javaml.distance.*; + +public class DBSCAN { + public static Dataset[] fit_predict(Dataset data) { + Clusterer dbscan = new DensityBasedSpatialClustering(); + return dbscan.cluster(data); + } + + public static Dataset[] fit_predict(Dataset data, double epsilon, int minPoints) { + Clusterer dbscan = new DensityBasedSpatialClustering(epsilon, minPoints); + return dbscan.cluster(data); + } + + public static Dataset[] fit_predict(Dataset data, double epsilon, int minPoints, DistanceMeasure dm) { + Clusterer dbscan = new DensityBasedSpatialClustering(epsilon, minPoints, dm); + return dbscan.cluster(data); + } +} diff --git a/HighLevelHints/src/clustering/PreComputed.java b/HighLevelHints/src/clustering/PreComputed.java new file mode 100644 index 00000000..45bd3166 --- /dev/null +++ b/HighLevelHints/src/clustering/PreComputed.java @@ -0,0 +1,59 @@ +package clustering; + +import net.sf.javaml.distance.DistanceMeasure; + +import java.util.Iterator; + +import net.sf.javaml.core.*; + +public class PreComputed implements DistanceMeasure { + private double max; + private double min; + private Dataset data; + + public PreComputed(Dataset data) { + this.data = data; + this.max = Double.NEGATIVE_INFINITY; + this.min = Double.POSITIVE_INFINITY; + for (Instance i : data) { + Iterator itr = i.iterator(); + while (itr.hasNext()) { + double value = itr.next(); + if (value > this.max) this.max = value; + if (value < this.min) this.min = value; + } + } + } + + @Override + public boolean compare(double arg0, double arg1) { + return arg0 >= arg1; + } + + @Override + public double getMaxValue() { + return this.max; + } + + @Override + public double getMinValue() { + return this.min; + } + + @Override + public double measure(Instance arg0, Instance arg1) { + int arg0Idx = data.indexOf(arg0); + int arg1Idx = data.indexOf(arg1); + + // Check if the distance matrix is symmetric + double arg0toarg1 = arg0.value(arg1Idx); + double arg1toarg0 = arg1.value(arg0Idx); + if (arg0toarg1 != arg1toarg0) { + return (arg0toarg1 + arg1toarg0) / 2; + //throw new RuntimeException("The precomputed distance matrix is not symmetric."); + } + return arg0toarg1; + } + +} + diff --git a/HighLevelHints/src/clustering/SolutionClusterer.java b/HighLevelHints/src/clustering/SolutionClusterer.java new file mode 100644 index 00000000..ed1dbacc --- /dev/null +++ b/HighLevelHints/src/clustering/SolutionClusterer.java @@ -0,0 +1,153 @@ +package clustering; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import alignment.JavaSolutionConfig; +import alignment.SolutionAlignment; +import alignment.SolutionDistanceMeasure; +import edu.isnap.node.JavaNode; +import edu.isnap.node.Node; +import edu.isnap.node.TextualNode; +import edu.isnap.python.SourceCodeHighlighter; +import edu.isnap.hint.HintConfig; +import edu.isnap.hint.HintData; +import edu.isnap.hint.util.NullStream; +import edu.isnap.java.JavaHintConfig; +import edu.isnap.sourcecheck.HintHighlighter; +import edu.isnap.sourcecheck.NodeAlignment; +import edu.isnap.sourcecheck.NodeAlignment.DistanceMeasure; +import edu.isnap.sourcecheck.NodeAlignment.Mapping; +import edu.isnap.sourcecheck.edit.EditHint; +import edu.isnap.sourcecheck.edit.Suggestion; +import net.sf.javaml.core.*; + +public class SolutionClusterer { + private List solutions; + private HintConfig solConfig = new JavaSolutionConfig(); + private String assignment; + + public SolutionClusterer(List solutions, String assignment) { + this.solutions = solutions; + this.assignment = assignment; + } + + public int getNumSolutions() { + return solutions.size(); + } + + public HashMap> clusterSolutions() { + // Sort solutions by studentID and timestamp + Collections.sort(solutions, new Comparator() { + @Override + public int compare(Node o1, Node o2) { + int studentID = o1.getStudentID().compareTo(o2.getStudentID()); + if (studentID != 0) return studentID; + return o1.getSubmissionTime().compareTo(o2.getSubmissionTime()); + } + }); + + // Map student ID and timestamp to a distance vector + HashMap> solutionMaps = new HashMap<>(); + + // Construct a distance matrix + double max = Double.NEGATIVE_INFINITY; + double min = Double.POSITIVE_INFINITY; + JavaHintConfig config = new JavaHintConfig(); + double[][] distanceMatrix = new double[solutions.size()][]; + for (int i = 0; i < solutions.size(); i++) { + TextualNode solution = (TextualNode) solutions.get(i); + double[] distances = getDistances(solution).stream().mapToDouble(d -> d).toArray(); +// double[] distances = new double[solutions.size()]; +// for (int j = 0; j < solutions.size(); j++) { +// List trace = new ArrayList<>(); +// trace.add(solutions.get(j)); +// HintData hintData = new HintData(assignment, config, 0, HintHighlighter.DataConsumer); +// hintData.addTrace(solutions.get(j).getStudentID(), trace); +// HintHighlighter highlighter = hintData.hintHighlighter(); +// +// highlighter.trace = NullStream.instance; +// +// List edits = highlighter.highlightWithPriorities(solution); +// List suggestions = SourceCodeHighlighter.getSuggestions(edits); +// int dist = suggestions.size(); +// distances[j] = dist; +// if (dist > max) max = dist; +// if (dist < min) min = dist; +// } + for (double dist : distances) { + if (dist > max) max = dist; + if (dist < min) min = dist; + } + distanceMatrix[i] = distances; + } + + // Check if the matrix is symmetric + for (int i = 0; i < distanceMatrix.length; i++) { + for (int j = 0; j < i; j++) { + if (distanceMatrix[i][j] != distanceMatrix[j][i]) { + System.err.println(distanceMatrix[i][j] + " is not " + distanceMatrix[j][i] + + " when i = " + i + ", j = " + j); + } + double smaller = Math.min(distanceMatrix[i][j], distanceMatrix[j][i]); + distanceMatrix[i][j] = smaller; + distanceMatrix[j][i] = smaller; + } + } + + // Normalize dataset + Dataset normalizedData = new DefaultDataset(); + for (int i = 0; i < solutions.size(); i++) { + TextualNode solution = (TextualNode) solutions.get(i); + Instance data = new SparseInstance(distanceMatrix[i]); + Instance normalized = data.minus(min).divide(max - min); + if (solutionMaps.get(solution.getStudentID()) == null) { + solutionMaps.put(solution.getStudentID(), new HashMap()); + } + solutionMaps.get(solution.getStudentID()).put(solution.getSubmissionTime(), normalized); + normalizedData.add(normalized); + } + + PreComputed precomputed = new PreComputed(normalizedData); + Dataset[] clusters = DBSCAN.fit_predict(normalizedData, 0.12, 6, precomputed); + for (int i = 0; i < clusters.length; i ++) { + for (Instance inst : clusters[i]) { + inst.setClassValue(String.valueOf(i)); + } + } + System.out.println("Finished Clustering!"); + return solutionMaps; + } + + private List getDistances(Node node) { + DistanceMeasure dm = new SolutionDistanceMeasure(solConfig); + + Mapping[] mappings = getMappingsFrom(node, solutions, dm, solConfig); + List distances = new ArrayList(); + for (Mapping map : mappings) { + distances.add(map.cost()); + } + + return distances; + } + + private static Mapping[] getMappingsFrom(Node from, List matches, + DistanceMeasure distanceMeasure, HintConfig config) { + boolean v2 = config.sourceCheckV2; + + Mapping[] mappings = new Mapping[matches.size()]; + for (int i = 0; i < mappings.length; i++) { + Node to = matches.get(i); + NodeAlignment align = new NodeAlignment(from, to, config); + Mapping mapping = v2 ? align.align(distanceMeasure) : + align.calculateMapping(distanceMeasure); + mappings[i] = mapping; + } + + return mappings; + } + +} \ No newline at end of file diff --git a/HintEvaluation/.classpath b/HintEvaluation/.classpath index ad2d3213..e2815455 100644 --- a/HintEvaluation/.classpath +++ b/HintEvaluation/.classpath @@ -24,5 +24,13 @@ + + + + + + + + diff --git a/HintEvaluation/pom.xml b/HintEvaluation/pom.xml index 641fde6c..ded0dd4b 100644 --- a/HintEvaluation/pom.xml +++ b/HintEvaluation/pom.xml @@ -28,11 +28,11 @@ - + de.cit-ec.tcs.alignment sequences diff --git a/HintEvaluation/src/edu/isnap/eval/java/FindCluster.java b/HintEvaluation/src/edu/isnap/eval/java/FindCluster.java new file mode 100644 index 00000000..a1d86528 --- /dev/null +++ b/HintEvaluation/src/edu/isnap/eval/java/FindCluster.java @@ -0,0 +1,351 @@ +package edu.isnap.eval.java; + +import org.apache.commons.lang.StringEscapeUtils; +import org.json.JSONObject; + +import edu.isnap.node.JavaNode; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Set; +import java.util.TreeMap; + +import com.github.javaparser.javaparser_symbol_solver_core.ASTParserJSON; + +import edu.isnap.node.Node; +import edu.isnap.hint.HintConfig; +import edu.isnap.hint.HintData; +import edu.isnap.hint.util.NullStream; + +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; +import java.io.Reader; +import edu.isnap.node.TextualNode; +import edu.isnap.node.ASTNode.SourceLocation; +import edu.isnap.sourcecheck.HintHighlighter; +import edu.isnap.sourcecheck.NodeAlignment.Mapping; +import edu.isnap.sourcecheck.edit.EditHint; +import edu.isnap.sourcecheck.edit.Insertion; +import edu.isnap.sourcecheck.edit.Suggestion; + +//import edu.isnap.hint.util.NullStream; +public class FindCluster { + public static int task; + public static int clusterid; + private static HashMap clusterMap = new HashMap<>(); + public static HintData hintData; + public static TreeMap hintTree; +// public static String dataDir = "../data/S20_3.3_OPE_Grading_Anon/3.3_OPE_Submissions-anonymized/"; //for local run + public static String dataDir = "./data/S20_3.3_OPE_Grading_Anon/3.3_OPE_Submissions-anonymized/"; + public static String separator = "@andrew.cmu.edu_data-consistency-ope_consistency-ope-task_"; + public static String[] assignments = {"BankUserConcurrentGet", "BankUserConcurrentPut", "BankUserMultiThreaded", "BankUserStrongConsistency"}; + public static String sourcePath = "/src/main/java/Project_OMP/BankUserSystem/"; // The path to the source file folder for each correct student submission + + public static String DELETE_START = "This code may be incorrect "; + public static String REPLACE_START = "This code may need to be replaced with something else :"; + public static String CANDIDATE_START = "This code is good, but it may be in the wrong place : "; + /** + * Export solution nodes with pre-computed clusterIDs as List + * @param inputCSV + * @return + */ + public static void getSolutionNodes(String inputCSV) throws IOException { + List csvRecords = readCSV(inputCSV); + LinkedHashMap> correctTraces = new LinkedHashMap<>(); + String assignment = assignments[task - 1]; + for (String[] record : csvRecords) { + String timestamp = record[0]; + String studentID = record[1]; + String isCorrect = record[10]; +// String isAnnotated = record[13]; + String clusterID = record[14]; + + if (isCorrect.toLowerCase().equals("true")) { + if (!clusterID.equals("")) { + File originalCode = new File(dataDir + studentID + separator + timestamp + sourcePath + assignment + ".java"); + String originalSourceCode = new String(Files.readAllBytes(originalCode.toPath())); + String jsonString = ASTParserJSON.toJSON(originalSourceCode); + JSONObject parsedTree = new JSONObject(jsonString); + + JavaNode node = (JavaNode)JavaNode.fromJSON(parsedTree, studentID, originalSourceCode, JavaNode::new); + + if (!correctTraces.containsKey(studentID)) { + clusterMap.put(studentID, (int)Float.parseFloat(clusterID)); + correctTraces.put(studentID , new ArrayList()); + } + correctTraces.get(studentID).add(node); + } + } + } + hintData = JavaImport.createHintData(assignment, correctTraces); + + assert correctTraces.size() != 0 : "No correct Solution Nodes exist"; + + } + + + + /** + * readCSV + * @param fileName + * @return + */ + private static List readCSV(String fileName) { + List csvRecords = new ArrayList<>(); + try (Reader reader = Files.newBufferedReader(Paths.get(fileName)); + CSVReader csvReader = new CSVReaderBuilder(reader).withSkipLines(1).build()) { + String[] nextLine; + while ((nextLine = csvReader.readNext()) != null) { + String projectID = nextLine[1]; + String sourceFileID = nextLine[2]; + String compileTime = nextLine[3]; + String filePath = nextLine[4]; + String sourceCode = nextLine[5]; + String diff = nextLine[6]; + String isLastDiff = nextLine[7]; + String sessionID = nextLine[8]; + String compileID = nextLine[9]; + String originalFileID = nextLine[10]; + String isCorrect = nextLine[11]; + String doesCompile = nextLine[12]; + String sourceCodeJSON = nextLine[13]; + String isAnnotated = nextLine[14]; + String clusterID = nextLine[15]; + String[] record = { projectID, sourceFileID, compileTime, filePath, sourceCode, diff, isLastDiff, + sessionID, compileID, originalFileID, isCorrect, doesCompile, sourceCodeJSON, isAnnotated, clusterID }; + csvRecords.add(record); + } + } catch (IOException e) { + e.printStackTrace(); + } + return csvRecords; + } + + + + /** + * Get the JavaNode for the new student code + * @param studentCodePath + * @return + */ + public static JavaNode preprocess(String studentCodePath) throws IOException { + File originalCode = new File(studentCodePath); + String originalSourceCode = new String(Files.readAllBytes(originalCode.toPath())); + String jsonString = ASTParserJSON.toJSON(originalSourceCode); + JSONObject parsedTree = new JSONObject(jsonString); + + JavaNode node = (JavaNode) JavaNode.fromJSON(parsedTree, originalSourceCode, JavaNode::new); + + return node; + + } + + + /** + * Returns Hints + * @param studentCodePath + * @param taskID + * @param level + * @param numHints + * @return + * @throws IOException + * + * Reference class files: + * path: /CTD/src/edu/isnap/sourcecheck/edit/EditHint.java + * + */ + public static int getClusterID(String studentCodePath, int taskID, int level, int numHints) throws IOException { + + String pth = dataDir + "input" + taskID + ".csv" ; + getSolutionNodes(pth); + TextualNode studentCode = (TextualNode) preprocess(studentCodePath); + + HintHighlighter highlighter = hintData.hintHighlighter(); + highlighter.trace = NullStream.instance; + Mapping mapping = highlighter.findSolutionMapping(studentCode); + List edits = highlighter.highlightWithPriorities(studentCode); +// String marked = studentCode.getSource(); +// List missing = new ArrayList<>(); + + List suggestions = new ArrayList<>(); + for (EditHint hint : edits) { + hint.addSuggestions(suggestions); + } + Collections.sort(suggestions, (s1, s2) -> s1.location.line - s2.location.line); + //suggestion has locations (line, col), hint and type + Set suggestionSet = new HashSet<>(); + for (Suggestion suggestion : suggestions) { + if (numHints < 0) break; + if (suggestion.start) { + SourceLocation location = suggestion.location; + EditHint ed = suggestion.hint; + if (!suggestionSet.contains(location.line)) { + suggestionSet.add(location.line); + + //low-level hints + if (level == 0) { + switch (suggestion.type) { + case DELETE: + System.out.println("line " + location.line + "," + location.col+ ":" + DELETE_START); +// marked = location.markSource(marked, DELETE_START); + break; + case MOVE: + System.out.println("line " + location.line + ":" + CANDIDATE_START); +// marked = location.markSource(marked, CANDIDATE_START); + break; + case REPLACE: + System.out.println("line " + location.line + ":" + REPLACE_START); +// marked = location.markSource(marked, REPLACE_START); + break; + case INSERT: + + String insertionLine = getInsertHint((Insertion)ed, mapping.config); + String insertionCode = getTextToInsert((Insertion)ed, mapping); + System.out.println("line " + location.line + ":" + insertionLine + "\n" +insertionCode); + System.out.println(); +// marked = location.markSource(marked, insertionCode); +// missing.add(getHumanReadableName((Insertion) ed, mapping.config)); + } +// if (!missing.isEmpty()) { +// marked += "\n\nYou may be missing the following:"; +// for (String m : missing) marked += m + "\n"; +// } +// System.out.println("low level:\n\n " + marked); + } + //high-level hints + if (level == 1) { + Integer key = hintTree.floorKey(location.line); + if (key == null) key = hintTree.ceilingKey(location.line); + String highLvl = hintTree.get(key); + System.out.println("line " + location.line + ":" + highLvl); + } + numHints--; + } + } + } + Node target = mapping.to; + return clusterMap.get(target.id); + } + + + /** + * + * @param insertion + * @param config + * @return + */ + private static String getInsertHint(Insertion insertion, HintConfig config) { + String hrName = getHumanReadableName(insertion, config); + String hint = "You may need to add " + hrName + " here"; + if (insertion.replaced != null) { + hint += ", instead of what you have."; + } else { + hint += "."; + } + return StringEscapeUtils.escapeHtml(hint); + } + + /** + * + * @param insertion + * @param config + * @return + */ + private static String getHumanReadableName(Insertion insertion, HintConfig config) { + String hrName = config.getHumanReadableName(insertion.pair); + if (insertion.replaced != null && insertion.replaced.hasType(insertion.type)) { + hrName = hrName.replaceAll("^(an?)", "$1 different"); +// System.out.println(hrName); + } + return hrName; + } + + /** + * + * @param insertion + * @param mapping + * @return + */ + private static String getTextToInsert(Insertion insertion, Mapping mapping) { + // TODO: Also need to handle newlines properly + Node mappedPair = insertion.pair.applyMapping(mapping); +// System.out.println("Pair:\n" + mappedPair); + String source = ((TextualNode) mappedPair).getSource(); + if (source != null) return source; + return insertion.pair.prettyPrint().replace("\n", ""); + } + + /** + * MAIN Function + * @param args + * @throws IOException + */ + public static void main (String[] args) throws IOException { +// String classpathStr = System.getProperty("java.class.path"); //Uncomment line to get the classpaths +// System.out.println("path: " + classpathStr); //Uncomment line to print the classpaths + String studentCodePath = args[0]; //path to student source code ".java" file + System.out.println("studentcodepath: " + studentCodePath); + String annotatedFilePath = args[1]; //"../data/S20_3.3_OPE_Grading_Anon/task4_annotated.txt" + System.out.println("annotatedFilePath: " + annotatedFilePath); + int taskID = Integer.parseInt(args[2]); // task ID + System.out.println("taskID: " + taskID); + int level = Integer.parseInt(args[3]); // high level hints = 1; low level hints = 0 + System.out.println("level: " + level); + int numHints = Integer.parseInt(args[4]); // max number of hints to show for any given snapshot + System.out.println("numHints: " + numHints); + + +// int taskID = 4; +// String studentCodePath = +//"../data/S20_3.3_OPE_Grading_Anon/3.3_OPE_Submissions-anonymized/13061@andrew.cmu.edu_data-consistency-ope_consistency-ope-task_20200303225053/src/main/java/Project_OMP/BankUserSystem/BankUserStrongConsistency.java"; +// int level = 0, numHints = 15; //level 1 high, level 0 low + + //option1 + //make jar file: https://cwiki.apache.org/confluence/display/MAVEN/Tutorial%3A+Build+a+JAR+file+with+Maven+in+5+minutes + + + //option2 + //can just right click project > run as > Run Configurations > java jre + + + //resolve maven error: + //step1: export M2_HOME="~/Downloads/apache-maven-3.6.3" + //step2: PATH="${M2_HOME}/bin:${PATH}" + //step3: export PATH + //step 4: mvn -version + + //To set default jvm path: +// On command line run - "/usr/libexec/java_home -V " +// Set the default version by "export JAVA_HOME=`/usr/libexec/java_home -v $version` " + + task = taskID; + hintTree = new TreeMap<>(); + + if (level == 1) { + File annotated = new File(annotatedFilePath); + String jsonString = new String(Files.readAllBytes(annotated.toPath())); + JSONObject hlvlHints = new JSONObject(jsonString); + @SuppressWarnings("unchecked") + Iterator k = hlvlHints.keys(); + while (k.hasNext()) { + String key = k.next(); + int line = Integer.parseInt(key); + String hint = hlvlHints.getString(key); + hintTree.put(line, hint); + } + } + + clusterid = getClusterID(studentCodePath, taskID, level, numHints); + System.out.println("cluster: " + clusterid); + + } + +} \ No newline at end of file diff --git a/HintEvaluation/src/edu/isnap/eval/java/GetClusters.java b/HintEvaluation/src/edu/isnap/eval/java/GetClusters.java new file mode 100644 index 00000000..eda24716 --- /dev/null +++ b/HintEvaluation/src/edu/isnap/eval/java/GetClusters.java @@ -0,0 +1,103 @@ +package edu.isnap.eval.java; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; + +import com.opencsv.CSVWriter; + +import clustering.SolutionClusterer; +import edu.isnap.node.JavaNode; +import edu.isnap.node.Node; +import edu.isnap.node.Node.Annotations; +import net.sf.javaml.core.Instance; + +public class GetClusters { + + public static void main(String[] args) throws IOException { + int task = 1; + // String dataDir = "../data/F19_Project_3_2/task" + task + "/"; + String dataDir = "../data/S20_3.3_OPE_Grading_Anon/3.3_OPE_Submissions-anonymized/"; // The path to the folder containing different students' source files + // String separator = "@andrew.cmu.edu_social-network_p32-task" + task + "_"; + String separator = "@andrew.cmu.edu_data-consistency-ope_consistency-ope-task_"; // You may not need this. This is useful when the names of folders for different students share separator string. + // String[] assignments = {"ProfileServlet", "FollowerServlet", "HomepageServlet", "TimelineServlet"}; + String[] assignments = {"BankUserConcurrentGet", "BankUserConcurrentPut", "BankUserMultiThreaded", "BankUserStrongConsistency"}; + String assignment = assignments[task - 1]; // The name of the source file + String sourcePath = "/src/main/java/Project_OMP/BankUserSystem/"; // The path to the source file folder for each student + String outFile = dataDir + "cluster_info_task" + task + ".csv"; + + HashMap> attempts = JavaImport.loadAssignment( + dataDir + "input_task" + task + ".csv", true, assignment, dataDir, separator, sourcePath); + + List correct = new ArrayList<>(); + LinkedHashMap annotated = new LinkedHashMap<>(); + for (String studentID : attempts.keySet()) { + List timestamps = new ArrayList<>(attempts.get(studentID).keySet()); + Collections.sort(timestamps); + JavaNode node = attempts.get(studentID).get(timestamps.get(timestamps.size() - 1)); + if (!node.readOnlyAnnotations().equals(Annotations.EMPTY)) { + annotated.put(node.cluster.get(), node); + } + + // If it was correct, then add it to the subset + if (node.correct.orElse(false)) { + correct.add(node); + } + } + + SolutionClusterer clusterer = new SolutionClusterer(correct, assignment); + HashMap> solutionMaps = clusterer.clusterSolutions(); + List students = new ArrayList(solutionMaps.keySet()); + Collections.sort(students); + + // Open a csv file + File clusterCSV = new File(outFile); + FileWriter outputfile = new FileWriter(clusterCSV); + + // create CSVWriter object filewriter object as parameter + CSVWriter writer = new CSVWriter(outputfile); + + // adding header to csv + List header = new ArrayList(); + header.add("StudentID"); + header.add("Timestamp"); + header.add("ClusterID"); + for (int i = 1; i <= clusterer.getNumSolutions(); i++) { + header.add("Dist_" + i); + } + String[] tempHeader = new String[header.size()]; + writer.writeNext(header.toArray(tempHeader)); + + // add data to csv + for (String student : students) { + HashMap timeMap = solutionMaps.get(student); + List timestamps = new ArrayList(timeMap.keySet()); + String timestamp = timestamps.get(0); + List data = new ArrayList(); + Instance inst = timeMap.get(timestamp); + String clusterID = (String) inst.classValue(); + data.add(student); // student id + data.add(timestamp); // timestamp + data.add(clusterID); // cluster id + + // Add distances + Iterator itr = inst.iterator(); + while (itr.hasNext()) { + data.add(itr.next().toString()); + } + String[] tempData = new String[data.size()]; + writer.writeNext(data.toArray(tempData)); + + } + + // closing writer connection + writer.close(); + } + +} diff --git a/HintEvaluation/src/edu/isnap/eval/java/JavaImport.java b/HintEvaluation/src/edu/isnap/eval/java/JavaImport.java index 322e66b0..f9c771ab 100644 --- a/HintEvaluation/src/edu/isnap/eval/java/JavaImport.java +++ b/HintEvaluation/src/edu/isnap/eval/java/JavaImport.java @@ -2,6 +2,7 @@ import java.io.File; import java.io.FileOutputStream; +import java.io.FileWriter; import java.io.IOException; import java.io.PrintStream; import java.io.PrintWriter; @@ -14,6 +15,8 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Optional; import java.util.Set; @@ -21,10 +24,13 @@ import org.json.JSONObject; -import com.esotericsoftware.kryo.Kryo; -import com.esotericsoftware.kryo.io.Output; import com.opencsv.CSVReader; import com.opencsv.CSVReaderBuilder; +import com.opencsv.CSVWriter; + +import clustering.SolutionClusterer; + +import com.github.javaparser.javaparser_symbol_solver_core.ASTParserJSON; import edu.isnap.hint.HintData; import edu.isnap.hint.SnapHintBuilder; @@ -32,167 +38,164 @@ import edu.isnap.node.JavaNode; import edu.isnap.node.Node; import edu.isnap.node.TextualNode; +import edu.isnap.node.Node.Annotations; import edu.isnap.python.SourceCodeHighlighter; import edu.isnap.sourcecheck.HintHighlighter; import edu.isnap.sourcecheck.edit.EditHint; -import edu.isnap.util.map.ListMap; +//import edu.isnap.util.map.ListMap; -public class JavaImport { +import net.sf.javaml.core.*; - static String DATA_DIR = "../data/blackbox/ClockDisplay/"; +public class JavaImport { public static void main(String[] args) throws IOException { - // Run generate hints to load data, generate hints for each student and print them out + // Run generate hints to load data, generate hints for each student and print + // them out // You need to update the file path to wherever you unzipped the data - -// Map> nodes = loadAssignment( -// DATA_DIR + "1__output_clock-display-ast.csv"); -// GrammarBuilder builder = new GrammarBuilder("java", new HashMap<>()); -// nodes.values().forEach(listMap -> listMap.values() -// .forEach(list -> list.forEach(n -> builder.add(n)))); -// System.out.println(builder.toJSON()); - - - PrintStream fileOut = new PrintStream(DATA_DIR + "output_hints.txt"); -// System.setOut(fileOut); - generateHintsForGS(DATA_DIR + "1__output_clock-display-ast-gs.csv", "ClockDisplay"); -// serializeHintData(DATA_DIR + "1__output_clock-display-ast.csv", "ClockDisplay", -// "../HintServer/WebContent/WEB-INF/data/ClockDisplay.hdata"); + int task = 4; + // String dataDir = "../data/F19_Project_3_2/task" + task + "/"; + String dataDir = "../data/S20_3.3_OPE_Grading_Anon/3.3_OPE_Submissions-anonymized/"; + // String separator = "@andrew.cmu.edu_social-network_p32-task" + task + "_"; + String separator = "@andrew.cmu.edu_data-consistency-ope_consistency-ope-task_"; + // String[] assignments = {"ProfileServlet", "FollowerServlet", "HomepageServlet", "TimelineServlet"}; + String[] assignments = {"BankUserConcurrentGet", "BankUserConcurrentPut", "BankUserMultiThreaded", "BankUserStrongConsistency"}; + String assignment = assignments[task - 1]; + String sourcePath = "/src/main/java/Project_OMP/BankUserSystem/"; // The path to the source file folder for each student + + generateHintsForGS(dataDir + "input_task" + task + ".csv", assignment, dataDir, separator, sourcePath); } - /*static HintData createHintData(String inputCSV, String assignment) throws IOException { - ListMap attempts = loadAssignment(inputCSV); - return createHintData(assignment, attempts); - }*/ + /* + * static HintData createHintData(String inputCSV, String assignment) throws + * IOException { LinkedHashMap> attempts = + * loadAssignment(inputCSV); return createHintData(assignment, attempts); } + */ - static HintData createHintData(String assignmentID, ListMap attempts) { + /** + * + * @param assignment the name of java file used in this assignment. Omit + * ".java". + * @param attempts Hashmap that maps student id's to LinkedHashMap's (i.e. + * each student has their own LinkedHashMap). Each + * LinkedHashMap maps timestamps to parsed JavaNodes + * + * @return + */ + static HintData createHintData(String assignment, LinkedHashMap> attempts) { JavaHintConfig config = new JavaHintConfig(); - HintData hintData = new HintData(assignmentID, config, 1, HintHighlighter.DataConsumer); - for (String attemptID : attempts.keySet()) { - List trace = attempts.get(attemptID).stream() - .map(node -> (Node) node) - .collect(Collectors.toList()); + HintData hintData = new HintData(assignment, config, 0, HintHighlighter.DataConsumer); + for (String studentID : attempts.keySet()) { + List trace = attempts.get(studentID).stream().map(node -> (Node) node).collect(Collectors.toList()); // Only needed for LOOCV - hintData.addTrace(attemptID, trace); + hintData.addTrace(studentID, trace); } return hintData; } - // Don't worry about this method for now - static void serializeHintData(String inputCSV, String assignment, String outputPath) - throws IOException { - ListMap attempts = loadAssignment(inputCSV, false).get("ClockDisplay.java"); - List toRemove = new ArrayList(); - // Remove incorrect attempts before serializing - for (String attemptID : attempts.keySet()) { - List attempt = attempts.get(attemptID); - if (attempt.size() == 0 || !attempt.get(attempt.size() - 1).correct.orElse(false)) { - toRemove.add(attemptID); - } - } - toRemove.forEach(attempts::remove); - HintData hintData = createHintData(assignment, attempts); - Kryo kryo = SnapHintBuilder.getKryo(); - Output output = new Output(new FileOutputStream(outputPath)); - kryo.writeObject(output, hintData); - output.close(); - } - - static void generateHints(String inputCSV, String assignment) throws IOException { - HashMap> filePathToattempts = loadAssignment(inputCSV, false); - - for (String filePath : filePathToattempts.keySet()) { - File startSourceFile = new File(DATA_DIR + "Start/" + filePath); - String startSource = ""; - if (startSourceFile.exists()) { - startSource = new String(Files.readAllBytes(startSourceFile.toPath())); - startSource = stripComments(startSource); - } + /* + * static void generateHints(String inputCSV, String assignment) throws + * IOException { HashMap>> + * filePathToattempts = loadAssignment( inputCSV, false, assignment); + * + * for (String filePath : filePathToattempts.keySet()) { File startSourceFile = + * new File(DATA_DIR + "Start/" + filePath); String startSource = ""; if + * (startSourceFile.exists()) { startSource = new + * String(Files.readAllBytes(startSourceFile.toPath())); startSource = + * stripComments(startSource); } + * + * // For now, just look at ClockDisplay, since NumberDisplay didn't have to be + * edited //if (!filePath.equals("ClockDisplay.java")) continue; + * LinkedHashMap> attempts = + * filePathToattempts.get(filePath); for (String student : attempts.keySet()) { + * // May want to change this to a random attempt, not just the first one, but + * you can // start with the first one JavaNode firstAttempt = + * attempts.get(student).get(0); if (firstAttempt.correct.orElse(false)) + * continue; + * + * LinkedHashMap> subset = new LinkedHashMap<>(); for + * (String attemptID : attempts.keySet()) { // Don't add this student to their + * own hint generation data if (attemptID.equals(student)) continue; // Get the + * sequence of snapshots over time List trace = + * attempts.get(attemptID); // If it was correct, then add it to the subset if + * (trace.get(trace.size() - 1).correct.orElse(false)) { // String + * solutionSource = stripComments( // trace.get(trace.size() - 1).getSource()); + * // System.out.println("Solution #: " + subset.size()); // + * System.out.println(Diff.diff(startSource, solutionSource, 2)); // + * System.out.println("--------------------------"); subset.put(attemptID, + * attempts.get(attemptID)); } } // if (1==1) break; + * System.out.println("Student: " + student); + * System.out.println("Building with: " + subset.size()); // We create a + * "HintData" object, which represents the data from which we generate // all + * hints HintData hintData = createHintData(assignment, subset); + * + * // Then we use this method to "highlight" the java source code using the + * SourceCheck // hints System.out.println( + * SourceCodeHighlighter.highlightSourceCode(hintData, firstAttempt)); + * + * break; } } } + */ - // For now, just look at ClockDisplay, since NumberDisplay didn't have to be edited - if (!filePath.equals("ClockDisplay.java")) continue; - ListMap attempts = filePathToattempts.get(filePath); - for (String student : attempts.keySet()) { - // May want to change this to a random attempt, not just the first one, but you can - // start with the first one - JavaNode firstAttempt = attempts.get(student).get(0); - if (firstAttempt.correct.orElse(false)) continue; + /** + * Exports html files highlighting hints + * + * @param inputCSV path to your input csv containing student id's, timestamps, + * correctness, etc + * @param assignment the name of java file used in this assignment. Omit + * ".java". + * @throws IOException + */ + static void generateHintsForGS(String inputCSV, String assignment, String dataDir, String separator, String sourcePath) throws IOException { + HashMap> attempts = loadAssignment(inputCSV, true, assignment, dataDir, separator, sourcePath); - ListMap subset = new ListMap<>(); - for (String attemptID : attempts.keySet()) { - // Don't add this student to their own hint generation data - if (attemptID.equals(student)) continue; - // Get the sequence of snapshots over time - List trace = attempts.get(attemptID); - // If it was correct, then add it to the subset - if (trace.get(trace.size() - 1).correct.orElse(false)) { -// String solutionSource = stripComments( -// trace.get(trace.size() - 1).getSource()); -// System.out.println("Solution #: " + subset.size()); -// System.out.println(Diff.diff(startSource, solutionSource, 2)); -// System.out.println("--------------------------"); - subset.put(attemptID, attempts.get(attemptID)); - } + // Maps student id's to their history of submissions. Only students + // who eventually got correct are considered + LinkedHashMap> correctTraces = new LinkedHashMap<>(); + // List of correct submissions + LinkedHashMap annotated = new LinkedHashMap<>(); + for (String studentID : attempts.keySet()) { + List trace = new ArrayList<>(); + List timestamps = new ArrayList<>(attempts.get(studentID).keySet()); + Collections.sort(timestamps); + for (String timestamp : timestamps) { + // Get the sequence of snapshots over time + JavaNode node = attempts.get(studentID).get(timestamp); +// if (node.correct.orElse(false)) { +// correct.add(node); +// } + trace.add(node); + if (!node.readOnlyAnnotations().equals(Annotations.EMPTY)) { + annotated.put(node.cluster.get(), node); } -// if (1==1) break; - System.out.println("Student: " + student); - System.out.println("Building with: " + subset.size()); - // We create a "HintData" object, which represents the data from which we generate - // all hints - HintData hintData = createHintData(assignment, subset); - - // Then we use this method to "highlight" the java source code using the SourceCheck - // hints - System.out.println( - SourceCodeHighlighter.highlightSourceCode(hintData, firstAttempt)); - - break; } - } - } - static void generateHintsForGS(String inputCSV, String assignment) throws IOException { - ListMap attempts = - loadAssignment(inputCSV, true).get("ClockDisplay.java"); - //System.out.println(attempts); - ListMap correct = new ListMap<>(); - for (String attemptID : attempts.keySet()) { - if(attemptID.startsWith("s") || attemptID.startsWith("r")) { - // Get the sequence of snapshots over time - List trace = attempts.get(attemptID); - // If it was correct, then add it to the subset - if (trace.get(trace.size() - 1).correct.orElse(false)) { - correct.put(attemptID, attempts.get(attemptID)); - } + // If it was correct, then add it to the subset + if (trace.get(trace.size() - 1).correct.orElse(false)) { + correctTraces.put(studentID, trace); } } - HintData hintData = createHintData(assignment, correct); - HintHighlighter highlighter = hintData.hintHighlighter(); + HintData hintData = createHintData(assignment, correctTraces); + + for (int clusterID : annotated.keySet()) { + hintData.addReferenceSoltion(clusterID, annotated.get(clusterID)); + } + + // TODO: Get the actual list from a .csv file, map project_id to the hint + // request + List csvRecords = readCSV(inputCSV); + for (String[] record : csvRecords) { + // for (String student : attempts.keySet()) { + String timestamp = record[0]; + String student = record[1]; + JavaNode hintRequest = attempts.get(student).get(timestamp); + String highlightedCode = SourceCodeHighlighter.highlightSourceCode(hintData, hintRequest); + highlightedCode = highlightedCode.replace("\n", "
\n"); + highlightedCode = "\n" + "\n" + highlightedCode; + PrintWriter out = new PrintWriter(dataDir + student + separator + timestamp + "/output_hints.html"); + out.println(highlightedCode); + out.close(); - // TODO: Get the actual list from a .csv file, map project_id to the hint request - ListMap goldStandardHintRequests = attempts; - new File(DATA_DIR+"GS").mkdirs(); - for (String student : goldStandardHintRequests.keySet()) { - if(student.startsWith("s") || student.startsWith("r")) { - continue; - } - JavaNode hintRequest = goldStandardHintRequests.get(student).get(0); - List edits = highlighter.highlightWithPriorities(hintRequest); - int i = 0; - for (EditHint hint : edits) { - Node copy = hintRequest.copy(); - EditHint.applyEdits(copy, Collections.singletonList(hint)); - double priority = hint.priority.consensus(); - JSONObject json = copy.toJSON(); - json.put("priority", priority); - String file = String.format("%s_%02d.json", student, i); - PrintWriter out = new PrintWriter(DATA_DIR+"GS/"+file); - out.println(json.toString()); - out.close(); - System.out.println(file + ": " + json.toString()); - i++; - } } } @@ -209,110 +212,108 @@ private static String stripComments(String source) { return String.join("\n", l); } - public static List readCSV(String fileName){ + public static List readCSV(String fileName) { List csvRecords = new ArrayList<>(); - try ( - Reader reader = Files.newBufferedReader(Paths.get(fileName)); - CSVReader csvReader = new CSVReaderBuilder(reader).withSkipLines(1).build() - ) { - String[] nextLine; - while ((nextLine = csvReader.readNext()) != null) { - String projectID = nextLine[0]; - String sourceFileID = nextLine[1]; - String compileTime = nextLine[2]; - String filePath = nextLine[3]; - String sourceCode = nextLine[4]; - String diff = nextLine[5]; - String isLastDiff = nextLine[6]; - String sessionID = nextLine[7]; - String compileID = nextLine[8]; - String originalFileID = nextLine[9]; - String isCorrect = nextLine[10]; - String doesCompile = nextLine[11]; - String sourceCodeJSON = nextLine[12]; - String[] record = {projectID, sourceFileID, compileTime, filePath, sourceCode, diff, - isLastDiff, sessionID, - compileID, originalFileID, isCorrect, doesCompile, sourceCodeJSON}; - csvRecords.add(record); - } - }catch (IOException e) { - e.printStackTrace(); - } + try (Reader reader = Files.newBufferedReader(Paths.get(fileName)); + CSVReader csvReader = new CSVReaderBuilder(reader).withSkipLines(1).build()) { + String[] nextLine; + while ((nextLine = csvReader.readNext()) != null) { + String projectID = nextLine[0]; + String sourceFileID = nextLine[1]; + String compileTime = nextLine[2]; + String filePath = nextLine[3]; + String sourceCode = nextLine[4]; + String diff = nextLine[5]; + String isLastDiff = nextLine[6]; + String sessionID = nextLine[7]; + String compileID = nextLine[8]; + String originalFileID = nextLine[9]; + String isCorrect = nextLine[10]; + String doesCompile = nextLine[11]; + String sourceCodeJSON = nextLine[12]; + String isAnnotated = nextLine[13]; + String clusterID = nextLine[14]; + String[] record = { projectID, sourceFileID, compileTime, filePath, sourceCode, diff, isLastDiff, + sessionID, compileID, originalFileID, isCorrect, doesCompile, sourceCodeJSON, isAnnotated, clusterID }; + csvRecords.add(record); + } + } catch (IOException e) { + e.printStackTrace(); + } return csvRecords; } private static String removeComments(String code) { - return String.join("\n", - Arrays.stream(code.split("\n")) - .filter(l -> !(l.trim().startsWith("*") || l.trim().startsWith("/**"))) - .collect(Collectors.toList()) - ); + return String.join("\n", Arrays.stream(code.split("\n")) + .filter(l -> !(l.trim().startsWith("*") || l.trim().startsWith("/**"))).collect(Collectors.toList())); } - static HashMap> loadAssignment(String inputCSV, boolean GS) - throws IOException { - HashMap> filePathToNodes = new HashMap<>(); - List csvRecords= readCSV(inputCSV); - Set numberDisplayProjects = new HashSet<>();; - String numberDisplayStartSource = null; - if(!GS) { - numberDisplayProjects = new HashSet<>(); - File startSourceFile = new File(DATA_DIR + "Start/NumberDisplay.java"); - numberDisplayStartSource = new String(Files.readAllBytes(startSourceFile.toPath()), - Charset.forName("UTF-8")); - numberDisplayStartSource = numberDisplayStartSource.replaceAll("\r", ""); - numberDisplayStartSource = removeComments(numberDisplayStartSource); - } + /** + * @param inputCSV path to your input csv containing student id's, timestamps, + * correctness, etc + * @param GS ignored + * @param assignment the name of java file used in this assignment. Omit + * ".java". + * + * @return Hashmap that maps student id's to LinkedHashMap's (i.e. each student + * has their own LinkedHashMap). Each LinkedHashMap maps timestamps to + * parsed JavaNodes + * + */ + static HashMap> loadAssignment(String inputCSV, boolean GS, + String assignment, String dataDir, String separator, String sourcePath) throws IOException { + HashMap> filePathToNodes = new HashMap<>(); + List csvRecords = readCSV(inputCSV); - for(String[] record: csvRecords) { - String projectID = record[0]; - String sourceCode = record[4]; - String sourceCodeJSON = record[12]; + for (String[] record : csvRecords) { + String timestamp = record[0]; + String studentID = record[1]; String isCorrect = record[10]; - String filePath = record[3]; - if(!GS) { - filePath = filePath.split("/")[1]; + String isAnnotated = record[13]; + String clusterID = record[14]; + + /* + * if (isCorrect.toLowerCase().equals("false")) { continue; } + */ + + File originalCode = new File(dataDir + studentID + separator + timestamp + sourcePath + assignment + ".java"); + String originalSourceCode = new String(Files.readAllBytes(originalCode.toPath())); + String jsonString; + if (isAnnotated.toLowerCase().equals("true")) { + File annotatedCode = new File(dataDir + studentID + separator + timestamp + "/" + assignment + ".json"); + jsonString = new String(Files.readAllBytes(annotatedCode.toPath())); + }else { + jsonString = ASTParserJSON.toJSON(originalSourceCode); } - else { - filePath = "ClockDisplay.java"; + + // TODO has to be commented out +// if ((studentID + separator + timestamp).equals("84895@andrew.cmu.edu_social-network_p32-task4_20191003042705") || +// (studentID + separator + timestamp).equals("69641@andrew.cmu.edu_social-network_p32-task4_20191013030137") || +// (studentID + separator + timestamp).equals("27319@andrew.cmu.edu_social-network_p32-task4_20191011031955")) { +// PrintWriter out = new PrintWriter(dataDir + studentID + separator + timestamp + "/" + assignment + ".json"); +// out.println(jsonString); +// out.close(); +// } + + JSONObject parsedTree = new JSONObject(jsonString); + JavaNode node = (JavaNode) JavaNode.fromJSON(parsedTree, originalSourceCode, JavaNode::new); + if (isCorrect.toLowerCase().equals("true")) {// || GS) { + node.correct = Optional.of(true); } - if(filePath.equals("ClockDisplay.java") || filePath.equals("NumberDisplay.java")) { - File testJson = new File(DATA_DIR + "ASTs/" + sourceCodeJSON); - if(GS) { - testJson = new File(DATA_DIR + "ASTsGS/" + sourceCodeJSON); - } - String json = new String(Files.readAllBytes(testJson.toPath())); - - if (!GS && filePath.contentEquals("NumberDisplay.java") && isCorrect.equals("True")) { - String source = removeComments(sourceCode); - if (!source.equals(numberDisplayStartSource)) { - numberDisplayProjects.add(projectID); - } - } - - JSONObject obj = new JSONObject(json); - JavaNode node = (JavaNode) TextualNode.fromJSON(obj, sourceCode, JavaNode::new); - if (isCorrect.toLowerCase().equals("true") || GS) { - node.correct = Optional.of(true); - } - - if(filePathToNodes.get(filePath) == null) { - filePathToNodes.put(filePath, new ListMap()); - } - filePathToNodes.get(filePath).add(projectID, node); + if (!clusterID.equals("")) { + node.cluster = Optional.of(Integer.parseInt(clusterID)); } - } - if(!GS) { - System.out.println("NDPs: " + numberDisplayProjects.size()); - } - // Remove all solutions that changed the NumberDisplay class (for now) - // TODO: At some point, we need to use both source files in hint generation... - for (String filePath : filePathToNodes.keySet()) { - for (String project : numberDisplayProjects) { - filePathToNodes.get(filePath).remove(project); + + node.setStudentID(studentID); + node.setSubmissionTime(timestamp); + + if (filePathToNodes.get(studentID) == null) { + filePathToNodes.put(studentID, new LinkedHashMap()); } + filePathToNodes.get(studentID).put(timestamp, node); } + return filePathToNodes; } } diff --git a/HintServer/.classpath b/HintServer/.classpath index 5f181e88..c653da6f 100644 --- a/HintServer/.classpath +++ b/HintServer/.classpath @@ -6,9 +6,6 @@ - - - @@ -22,6 +19,11 @@ + + + + + diff --git a/HintServer/.project b/HintServer/.project index 6347ce75..14cdfdd7 100644 --- a/HintServer/.project +++ b/HintServer/.project @@ -8,6 +8,7 @@ QualityScore CTD JavaParser + HighLevelHints diff --git a/HintServer/WebContent/WEB-INF/web.xml b/HintServer/WebContent/WEB-INF/web.xml index 33fddd5a..28261090 100644 --- a/HintServer/WebContent/WEB-INF/web.xml +++ b/HintServer/WebContent/WEB-INF/web.xml @@ -3,5 +3,13 @@ HintServer hints + java.html + java.htm + java.jsp + + +HighLelelHintServlet +HintServlet2 + \ No newline at end of file diff --git a/SnapParser/.classpath b/SnapParser/.classpath index f6be6232..0b8e865d 100644 --- a/SnapParser/.classpath +++ b/SnapParser/.classpath @@ -17,5 +17,6 @@ + diff --git a/iSnap/.classpath b/iSnap/.classpath index b97f49f4..72d03158 100644 --- a/iSnap/.classpath +++ b/iSnap/.classpath @@ -18,5 +18,6 @@ + diff --git a/iSnap/src/edu/isnap/python/SourceCodeHighlighter.java b/iSnap/src/edu/isnap/python/SourceCodeHighlighter.java index bbf79609..50e0815d 100644 --- a/iSnap/src/edu/isnap/python/SourceCodeHighlighter.java +++ b/iSnap/src/edu/isnap/python/SourceCodeHighlighter.java @@ -8,15 +8,20 @@ import edu.isnap.hint.HintConfig; import edu.isnap.hint.HintData; +import edu.isnap.hint.TextHint; import edu.isnap.hint.util.NullStream; import edu.isnap.node.ASTNode.SourceLocation; +import edu.isnap.node.Node.Annotations; import edu.isnap.node.Node; import edu.isnap.node.TextualNode; import edu.isnap.sourcecheck.HintHighlighter; +import edu.isnap.sourcecheck.NodeAlignment; +import edu.isnap.sourcecheck.NodeAlignment.DistanceMeasure; import edu.isnap.sourcecheck.NodeAlignment.Mapping; import edu.isnap.sourcecheck.edit.EditHint; import edu.isnap.sourcecheck.edit.Insertion; import edu.isnap.sourcecheck.edit.Suggestion; +import edu.isnap.sourcecheck.edit.Suggestion.SuggestionType; import edu.isnap.util.Diff; public class SourceCodeHighlighter { @@ -33,7 +38,17 @@ public class SourceCodeHighlighter { // public static String REORDER_START = ""; public static String SPAN_END = ""; - + + public static String DELETE_ANNOTATION = " System.out.println(((TextualNode) n).startSourceLocation)); - -// System.out.println("From:"); -// System.out.println(from); -// System.out.println("Target:"); -// System.out.println(target); - System.out.println("Node Diff:"); - System.out.println(Diff.diff(studentCode.prettyPrint(), target.prettyPrint(), 2)); -// System.out.println("Student source:"); -// System.out.println(studentCode.source); -// System.out.println("Target source:"); -// System.out.println(target.source); - System.out.println("Source Diff:"); - System.out.println(Diff.diff(studentCode.getSource(), target.getSource(), 2)); - mapping.printValueMappings(System.out); - edits.forEach(e -> System.out.printf("%.02f: %s\n", e.priority.consensus(), e)); - System.out.println(); - String marked = studentCode.getSource(); List suggestions = getSuggestions(edits); List missing = new ArrayList<>(); + + Mapping referenceMapping = null; + if (hintData.config.useAnnotation) { + int clusterID = 0; + boolean v2 = hintData.config.sourceCheckV2; + DistanceMeasure distanceMeasure = highlighter.getDistanceMeasure(); + + if (target.cluster.isPresent()) { + clusterID = target.cluster.get(); + } + TextualNode reference = (TextualNode) hintData.getReferenceSolutions().get(clusterID); + NodeAlignment align = new NodeAlignment(target, reference, hintData.config); + referenceMapping = v2 ? align.align(distanceMeasure) : align.calculateMapping(distanceMeasure); + } for (Suggestion suggestion : suggestions) { SourceLocation location = suggestion.location; - EditHint hint = suggestion.hint; + EditHint hint = suggestion.hint; if (!suggestion.start) { marked = location.markSource(marked, SPAN_END); continue; } - switch (suggestion.type) { - case DELETE: - marked = location.markSource(marked, DELETE_START); - break; - case MOVE: - marked = location.markSource(marked, CANDIDATE_START); - break; - case REPLACE: - marked = location.markSource(marked, REPLACE_START); - break; - case INSERT: - String insertionCode = getInsertHTML(mapping, hint); + if (referenceMapping != null) { + TextualNode ref = null; + switch (suggestion.type) { + case DELETE: // If reorder or deletion, hint.node to get "from" in studentCode and hint.parent to get its parent. hint.parent is from EditHint + TextualNode parent = (TextualNode) hint.parent; + ref = (TextualNode) referenceMapping.getFromMap().get(parent); // get corresponding parent node in reference + break; + case MOVE: // If insert, hint.candidate to get "from" in studentCode and hint.pair to get "to" + case REPLACE: // If insert, hint.candidate to get "from" in studentCode and hint.pair to get "to" + case INSERT: + Insertion ins = (Insertion) hint; // hint.candidate to get "from" in studentCode. If null, it's insertion. hint.pair to get "to" + TextualNode inserted = (TextualNode) ins.pair; // get node to be inserted + ref = (TextualNode) referenceMapping.getFromMap().get(inserted); // get corresponding node in reference + missing.add(getHumanReadableName((Insertion) hint, mapping.config)); + } + String insertionCode; + if (ref != null) { + Annotations annotations = ref.readOnlyAnnotations(); + if (!annotations.equals(Annotations.EMPTY)) { + List hints = annotations.getHints(); + insertionCode = getHTML(hints.get(0).text, suggestion.type); + }else { + insertionCode = getHTML(mapping, hint, suggestion.type); + } + }else { + insertionCode = getHTML(mapping, hint, suggestion.type); + } marked = location.markSource(marked, insertionCode); - missing.add(getHumanReadableName((Insertion) hint, mapping.config)); + }else { + switch (suggestion.type) { + case DELETE: + marked = location.markSource(marked, DELETE_START); + break; + case MOVE: + marked = location.markSource(marked, CANDIDATE_START); + break; + case REPLACE: + marked = location.markSource(marked, REPLACE_START); + break; + case INSERT: + String insertionCode = getInsertHTML(mapping, hint); + marked = location.markSource(marked, insertionCode); + missing.add(getHumanReadableName((Insertion) hint, mapping.config)); + } } + System.out.println(suggestion.type + ": " + suggestion.location); // System.out.println(marked); } @@ -101,11 +143,11 @@ public static String highlightSourceCode(HintData hintData, TextualNode studentC marked += ""; } - System.out.println(marked); + //System.out.println(marked); return marked; } - private static List getSuggestions(List edits) { + public static List getSuggestions(List edits) { List suggestions = new ArrayList<>(); for (EditHint hint : edits) { hint.addSuggestions(suggestions); @@ -116,10 +158,43 @@ private static List getSuggestions(List edits) { private static String getInsertHTML(Mapping mapping, EditHint editHint) { String hint = getInsertHint((Insertion)editHint, mapping.config); + return getInsertHTML(hint); + } + + private static String getInsertHTML(String hint) { String insertionCode = String.format("%s data-tooltip=\"%s\">%s%s", INSERT_START, hint, "\u2795", SPAN_END); return insertionCode; } + + private static String getHTML(Mapping mapping, EditHint editHint, SuggestionType type) { + switch (type) { + case DELETE: // If reorder or deletion, hint.node to get "from" in studentCode and hint.parent to get its parent. hint.parent is from EditHint + return DELETE_START; + case MOVE: // If insert, hint.candidate to get "from" in studentCode and hint.pair to get "to" + return CANDIDATE_START; + case REPLACE: // If insert, hint.candidate to get "from" in studentCode and hint.pair to get "to" + return REPLACE_START; + case INSERT: + String hint = getInsertHint((Insertion)editHint, mapping.config); + return String.format("%s data-tooltip=\"%s\">%s%s", INSERT_START, hint, "\u2795", SPAN_END); + } + return null; + } + + private static String getHTML(String hint, SuggestionType type) { + switch (type) { + case DELETE: // If reorder or deletion, hint.node to get "from" in studentCode and hint.parent to get its parent. hint.parent is from EditHint + return String.format("%s data-tooltip=\"%s\">%s", REPLACE_ANNOTATION, hint, "\u2795"); + case MOVE: // If insert, hint.candidate to get "from" in studentCode and hint.pair to get "to" + return String.format("%s data-tooltip=\"%s\">%s", CANDIDATE_ANNOTATION, hint, "\u2795"); + case REPLACE: // If insert, hint.candidate to get "from" in studentCode and hint.pair to get "to" + return String.format("%s data-tooltip=\"%s\">%s", REPLACE_ANNOTATION, hint, "\u2795"); + case INSERT: + return String.format("%s data-tooltip=\"%s\">%s%s", INSERT_START, hint, "\u2795", SPAN_END); + } + return null; + } private static String getInsertHint(Insertion insertion, HintConfig config) { String hrName = getHumanReadableName(insertion, config);