From 5f60780eea94a8740e252bd150783ca7b5463581 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 14:37:02 -0400 Subject: [PATCH 01/79] Initial work for generic merge experiment --- src/main/injectors/GenericMergeModule.groovy | 32 +++++++ .../GenericMergeDataCollector.groovy | 95 +++++++++++++++++++ .../GenericMergeDataOutputProcessor.groovy | 10 ++ 3 files changed, 137 insertions(+) create mode 100644 src/main/injectors/GenericMergeModule.groovy create mode 100644 src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy create mode 100644 src/main/services/outputProcessors/GenericMergeDataOutputProcessor.groovy diff --git a/src/main/injectors/GenericMergeModule.groovy b/src/main/injectors/GenericMergeModule.groovy new file mode 100644 index 000000000..aee3bade7 --- /dev/null +++ b/src/main/injectors/GenericMergeModule.groovy @@ -0,0 +1,32 @@ +package injectors + +import com.google.inject.AbstractModule +import com.google.inject.multibindings.Multibinder +import interfaces.CommitFilter +import interfaces.DataCollector +import interfaces.OutputProcessor +import interfaces.ProjectProcessor +import services.commitFilters.MutuallyModifiedFilesCommitFilter +import services.commitFilters.S3MCommitFilter +import services.dataCollectors.GenericMerge.GenericMergeDataCollector +import services.outputProcessors.GenericMergeDataOutputProcessor +import services.projectProcessors.DummyProjectProcessor +import services.util.ci.CIPlatform +import services.util.ci.TravisPlatform + +class GenericMergeModule extends AbstractModule { + @Override + protected void configure() { + Multibinder projectProcessorBinder = Multibinder.newSetBinder(binder(), ProjectProcessor.class) + projectProcessorBinder.addBinding().to(DummyProjectProcessor.class) + + Multibinder dataCollectorBinder = Multibinder.newSetBinder(binder(), DataCollector.class) + dataCollectorBinder.addBinding().to(GenericMergeDataCollector.class) + + Multibinder outputProcessorBinder = Multibinder.newSetBinder(binder(), OutputProcessor.class) + outputProcessorBinder.addBinding().to(GenericMergeDataOutputProcessor.class) + + bind(CommitFilter.class).to(MutuallyModifiedFilesCommitFilter.class) + bind(CIPlatform.class).to(TravisPlatform.class) + } +} diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy new file mode 100644 index 000000000..90abf008b --- /dev/null +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -0,0 +1,95 @@ +package services.dataCollectors.GenericMerge + +import interfaces.DataCollector +import org.apache.commons.io.FileUtils +import project.MergeCommit +import project.Project +import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector +import services.dataCollectors.S3MMergesCollector.SpreadsheetBuilder +import util.ProcessRunner + +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +class GenericMergeDataCollector implements DataCollector { + private static final GENERIC_MERGE_BINARY_PATH = "/Users/jpedroh/Projetos/msc/generic-merge/target/release/generic-merge" + private static final GENERIC_MERGE_REPORTS_PATH = "/Users/jpedroh/Projetos/msc/miningframework/output/reports/" + + @Override + void collectData(Project project, MergeCommit mergeCommit) { + List scenarios = MergeScenarioCollector.collectMergeScenarios(project, mergeCommit) + + Files.createDirectories(Paths.get("${GENERIC_MERGE_REPORTS_PATH}")); + def reportFile = new File("${GENERIC_MERGE_REPORTS_PATH}/${project.getName()}.csv"); + reportFile.createNewFile(); + + scenarios.stream().forEach { + def result = runToolInMergeScenario(it) + def line = "${project.getName()},${mergeCommit.getSHA()},${it.toAbsolutePath().toString()},${anyParentEqualsBase(it)},${result.result},${result.time}" + reportFile << "${line.replaceAll('\\\\', '/')}\n" + } + } + + private static anyParentEqualsBase(Path scenario) { + def leftEqualsBase = FileUtils.contentEquals( + new File("${scenario.toAbsolutePath()}/basejava"), + new File("${scenario.toAbsolutePath()}/leftjava") + ) + + def rightEqualsBase = FileUtils.contentEquals( + new File("${scenario.toAbsolutePath()}/basejava"), + new File("${scenario.toAbsolutePath()}/rightjava") + ) + + return leftEqualsBase || rightEqualsBase + } + + private static MergeScenarioExecution runToolInMergeScenario(Path scenario) { + def working_directory_path = scenario.toAbsolutePath().toString(); + + def processBuilder = ProcessRunner.buildProcess(working_directory_path); + processBuilder.command().addAll(getBuildParameters()) + + def startTime = System.nanoTime(); + def output = ProcessRunner.startProcess(processBuilder); + output.waitFor() + def endTime = System.nanoTime(); + + if (output.exitValue() > 1) { + println("Error while merging ${scenario.toAbsolutePath()}: ${output.getInputStream().readLines()}") + } + + def result = output.exitValue() == 0 ? ScenarioResult.SUCCESS_WITHOUT_CONFLICTS : output.exitValue() == 1 ? ScenarioResult.SUCCESS_WITH_CONFLICTS : ScenarioResult.TOOL_ERROR; + + return new MergeScenarioExecution(result, endTime - startTime); + } + + private static enum ScenarioResult { + SUCCESS_WITHOUT_CONFLICTS, + SUCCESS_WITH_CONFLICTS, + TOOL_ERROR + } + + private static class MergeScenarioExecution{ + ScenarioResult result; + int time; + + MergeScenarioExecution(ScenarioResult result, long time) { + this.result = result + this.time = time + } + } + + private static List getBuildParameters() { + def list = new ArrayList() + list.add(GENERIC_MERGE_BINARY_PATH) + list.add("--base-path=basejava") + list.add("--left-path=leftjava") + list.add("--right-path=rightjava") + list.add("--merge-path=merge.generic.java") + list.add("--language=java") + + return list + } +} diff --git a/src/main/services/outputProcessors/GenericMergeDataOutputProcessor.groovy b/src/main/services/outputProcessors/GenericMergeDataOutputProcessor.groovy new file mode 100644 index 000000000..5b4b39b45 --- /dev/null +++ b/src/main/services/outputProcessors/GenericMergeDataOutputProcessor.groovy @@ -0,0 +1,10 @@ +package services.outputProcessors + +import interfaces.OutputProcessor + +class GenericMergeDataOutputProcessor implements OutputProcessor{ + @Override + void processOutput() { + println "Processing output" + } +} From 08e59fb05e444e45551f4784a9cfdfba0af749b6 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 14:55:32 -0400 Subject: [PATCH 02/79] alter data type --- .../GenericMerge/GenericMergeDataCollector.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 90abf008b..7e6dcfbe7 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -5,7 +5,6 @@ import org.apache.commons.io.FileUtils import project.MergeCommit import project.Project import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector -import services.dataCollectors.S3MMergesCollector.SpreadsheetBuilder import util.ProcessRunner import java.nio.file.Files @@ -73,7 +72,7 @@ class GenericMergeDataCollector implements DataCollector { private static class MergeScenarioExecution{ ScenarioResult result; - int time; + long time; MergeScenarioExecution(ScenarioResult result, long time) { this.result = result From 3096869b61a666dfebdcd3167346f977805e2805 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 16:49:00 -0400 Subject: [PATCH 03/79] refactor: move classes along --- .../GenericMergeDataCollector.groovy | 79 +++++++------------ .../executors/GenericMergeToolExecutor.groovy | 37 +++++++++ .../executors/MergeToolExecutor.groovy | 15 ++++ 3 files changed, 80 insertions(+), 51 deletions(-) create mode 100644 src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy create mode 100644 src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 7e6dcfbe7..6463a38ca 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -4,16 +4,22 @@ import interfaces.DataCollector import org.apache.commons.io.FileUtils import project.MergeCommit import project.Project +import services.dataCollectors.GenericMerge.executors.GenericMergeToolExecutor +import services.dataCollectors.GenericMerge.executors.MergeToolExecutor import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector -import util.ProcessRunner import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths class GenericMergeDataCollector implements DataCollector { - private static final GENERIC_MERGE_BINARY_PATH = "/Users/jpedroh/Projetos/msc/generic-merge/target/release/generic-merge" - private static final GENERIC_MERGE_REPORTS_PATH = "/Users/jpedroh/Projetos/msc/miningframework/output/reports/" + private static final BASE_EXPERIMENT_PATH = "/usr/src/app" + private static final GENERIC_MERGE_REPORTS_PATH = "${BASE_EXPERIMENT_PATH}/output/reports/" + private final List mergeToolExecutors; + + GenericMergeDataCollector() { + this.mergeToolExecutors = List.of(new GenericMergeToolExecutor()) + } @Override void collectData(Project project, MergeCommit mergeCommit) { @@ -23,72 +29,43 @@ class GenericMergeDataCollector implements DataCollector { def reportFile = new File("${GENERIC_MERGE_REPORTS_PATH}/${project.getName()}.csv"); reportFile.createNewFile(); - scenarios.stream().forEach { - def result = runToolInMergeScenario(it) - def line = "${project.getName()},${mergeCommit.getSHA()},${it.toAbsolutePath().toString()},${anyParentEqualsBase(it)},${result.result},${result.time}" + scenarios.stream().flatMap { scenario -> mergeToolExecutors.stream().map { executor -> executor.runToolForMergeScenario(scenario) } + }.forEach { + def line = List.of(project.getName(), + mergeCommit.getSHA(), + it.scenario.toAbsolutePath().toString(), + anyParentEqualsBase(it.scenario).toString(), + it.result, + it.time).join(",") reportFile << "${line.replaceAll('\\\\', '/')}\n" } } private static anyParentEqualsBase(Path scenario) { - def leftEqualsBase = FileUtils.contentEquals( - new File("${scenario.toAbsolutePath()}/basejava"), - new File("${scenario.toAbsolutePath()}/leftjava") - ) + def leftEqualsBase = FileUtils.contentEquals(new File("${scenario.toAbsolutePath()}/basejava"), + new File("${scenario.toAbsolutePath()}/leftjava")) - def rightEqualsBase = FileUtils.contentEquals( - new File("${scenario.toAbsolutePath()}/basejava"), - new File("${scenario.toAbsolutePath()}/rightjava") - ) + def rightEqualsBase = FileUtils.contentEquals(new File("${scenario.toAbsolutePath()}/basejava"), + new File("${scenario.toAbsolutePath()}/rightjava")) return leftEqualsBase || rightEqualsBase } - private static MergeScenarioExecution runToolInMergeScenario(Path scenario) { - def working_directory_path = scenario.toAbsolutePath().toString(); - - def processBuilder = ProcessRunner.buildProcess(working_directory_path); - processBuilder.command().addAll(getBuildParameters()) - - def startTime = System.nanoTime(); - def output = ProcessRunner.startProcess(processBuilder); - output.waitFor() - def endTime = System.nanoTime(); - - if (output.exitValue() > 1) { - println("Error while merging ${scenario.toAbsolutePath()}: ${output.getInputStream().readLines()}") - } - - def result = output.exitValue() == 0 ? ScenarioResult.SUCCESS_WITHOUT_CONFLICTS : output.exitValue() == 1 ? ScenarioResult.SUCCESS_WITH_CONFLICTS : ScenarioResult.TOOL_ERROR; - - return new MergeScenarioExecution(result, endTime - startTime); - } - - private static enum ScenarioResult { + static enum MergeScenarioResult { SUCCESS_WITHOUT_CONFLICTS, SUCCESS_WITH_CONFLICTS, TOOL_ERROR } - private static class MergeScenarioExecution{ - ScenarioResult result; - long time; + static class MergeScenarioExecutionSummary { + public final Path scenario; + public final MergeScenarioResult result; + public final long time; - MergeScenarioExecution(ScenarioResult result, long time) { + MergeScenarioExecutionSummary(Path scenario, MergeScenarioResult result, long time) { + this.scenario = scenario this.result = result this.time = time } } - - private static List getBuildParameters() { - def list = new ArrayList() - list.add(GENERIC_MERGE_BINARY_PATH) - list.add("--base-path=basejava") - list.add("--left-path=leftjava") - list.add("--right-path=rightjava") - list.add("--merge-path=merge.generic.java") - list.add("--language=java") - - return list - } } diff --git a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy new file mode 100644 index 000000000..4be143896 --- /dev/null +++ b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy @@ -0,0 +1,37 @@ +package services.dataCollectors.GenericMerge.executors + +import services.dataCollectors.GenericMerge.GenericMergeDataCollector +import util.ProcessRunner + +import java.nio.file.Path + +class GenericMergeToolExecutor extends MergeToolExecutor { + private static final BASE_EXPERIMENT_PATH = "/usr/src/app" + private static final String GENERIC_MERGE_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/generic-merge" + + @Override + protected GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario) { + def working_directory_path = scenario.toAbsolutePath().toString(); + + def processBuilder = ProcessRunner.buildProcess(working_directory_path); + processBuilder.command().addAll(getBuildParameters()) + + def output = ProcessRunner.startProcess(processBuilder); + output.waitFor() + + if (output.exitValue() > 1) { + println("Error while merging ${scenario.toAbsolutePath()}: ${output.getInputStream().readLines()}") + } + + return output.exitValue() == 0 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS : output.exitValue() == 1 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS : GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR; + } + + private static List getBuildParameters() { + return List.of(GENERIC_MERGE_BINARY_PATH, + "--base-path=basejava", + "--left-path=leftjava", + "--right-path=rightjava", + "--merge-path=merge.generic.java", + "--language=java") + } +} diff --git a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy new file mode 100644 index 000000000..7bb91de34 --- /dev/null +++ b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy @@ -0,0 +1,15 @@ +package services.dataCollectors.GenericMerge.executors + +import services.dataCollectors.GenericMerge.GenericMergeDataCollector; + +import java.nio.file.Path; + +abstract class MergeToolExecutor { + static GenericMergeDataCollector.MergeScenarioExecutionSummary runToolForMergeScenario(Path scenario) { + def startTime = System.nanoTime(); + def result = executeTool(scenario) + return new GenericMergeDataCollector.MergeScenarioExecutionSummary(scenario, result, System.nanoTime() - startTime) + } + + protected abstract GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario); +} From 7b5abee66f163774df8143ef1fe0d26bbd364bae Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 16:56:16 -0400 Subject: [PATCH 04/79] feat: add tool --- .../GenericMerge/GenericMergeDataCollector.groovy | 4 +++- .../executors/GenericMergeToolExecutor.groovy | 14 ++++++++------ .../executors/MergeToolExecutor.groovy | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 6463a38ca..1718ce5f7 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -58,14 +58,16 @@ class GenericMergeDataCollector implements DataCollector { } static class MergeScenarioExecutionSummary { + public final String tool; public final Path scenario; public final MergeScenarioResult result; public final long time; - MergeScenarioExecutionSummary(Path scenario, MergeScenarioResult result, long time) { + MergeScenarioExecutionSummary(Path scenario, MergeScenarioResult result, long time, String tool) { this.scenario = scenario this.result = result this.time = time + this.tool = tool } } } diff --git a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy index 4be143896..7dbcf6483 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy @@ -27,11 +27,13 @@ class GenericMergeToolExecutor extends MergeToolExecutor { } private static List getBuildParameters() { - return List.of(GENERIC_MERGE_BINARY_PATH, - "--base-path=basejava", - "--left-path=leftjava", - "--right-path=rightjava", - "--merge-path=merge.generic.java", - "--language=java") + def list = new ArrayList() + list.add(GENERIC_MERGE_BINARY_PATH) + list.add("--base-path=basejava") + list.add("--left-path=leftjava") + list.add("--right-path=rightjava") + list.add("--merge-path=merge.generic.java") + list.add("--language=java") + return list; } } diff --git a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy index 7bb91de34..4369984f3 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy @@ -8,7 +8,7 @@ abstract class MergeToolExecutor { static GenericMergeDataCollector.MergeScenarioExecutionSummary runToolForMergeScenario(Path scenario) { def startTime = System.nanoTime(); def result = executeTool(scenario) - return new GenericMergeDataCollector.MergeScenarioExecutionSummary(scenario, result, System.nanoTime() - startTime) + return new GenericMergeDataCollector.MergeScenarioExecutionSummary(scenario, result, System.nanoTime() - startTime, this.getSimpleName()) } protected abstract GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario); From 7401ae0cb036721cd19ec3f055075a0af69b2c7b Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 17:00:04 -0400 Subject: [PATCH 05/79] refactor: move initialization --- .../GenericMergeDataCollector.groovy | 5 +---- .../executors/GenericMergeToolExecutor.groovy | 16 ++++++++-------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 1718ce5f7..584f482aa 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -15,11 +15,8 @@ import java.nio.file.Paths class GenericMergeDataCollector implements DataCollector { private static final BASE_EXPERIMENT_PATH = "/usr/src/app" private static final GENERIC_MERGE_REPORTS_PATH = "${BASE_EXPERIMENT_PATH}/output/reports/" - private final List mergeToolExecutors; + private final List mergeToolExecutors = List.of(new GenericMergeToolExecutor()); - GenericMergeDataCollector() { - this.mergeToolExecutors = List.of(new GenericMergeToolExecutor()) - } @Override void collectData(Project project, MergeCommit mergeCommit) { diff --git a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy index 7dbcf6483..6341e23b1 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy @@ -27,13 +27,13 @@ class GenericMergeToolExecutor extends MergeToolExecutor { } private static List getBuildParameters() { - def list = new ArrayList() - list.add(GENERIC_MERGE_BINARY_PATH) - list.add("--base-path=basejava") - list.add("--left-path=leftjava") - list.add("--right-path=rightjava") - list.add("--merge-path=merge.generic.java") - list.add("--language=java") - return list; + return List.of( + GENERIC_MERGE_BINARY_PATH, + "--base-path=basejava", + "--left-path=leftjava", + "--right-path=rightjava", + "--merge-path=merge.generic.java", + "--language=java" + ) } } From 679a13b201c059fa2fbfb7b8fc7addd59350a1b3 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 17:24:31 -0400 Subject: [PATCH 06/79] feat: change instatiantion --- .../GenericMerge/GenericMergeDataCollector.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 584f482aa..56de299da 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -15,8 +15,12 @@ import java.nio.file.Paths class GenericMergeDataCollector implements DataCollector { private static final BASE_EXPERIMENT_PATH = "/usr/src/app" private static final GENERIC_MERGE_REPORTS_PATH = "${BASE_EXPERIMENT_PATH}/output/reports/" - private final List mergeToolExecutors = List.of(new GenericMergeToolExecutor()); + private final List mergeToolExecutors; + GenericMergeDataCollector() { + this.mergeToolExecutors = new ArrayList() + this.mergeToolExecutors.add(new GenericMergeToolExecutor()) + } @Override void collectData(Project project, MergeCommit mergeCommit) { From bf1b8d7ee295986ad08425cb6702239622cc56fa Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 17:51:03 -0400 Subject: [PATCH 07/79] feat: add jdime collector --- .../GenericMergeDataCollector.groovy | 2 ++ .../executors/GenericMergeToolExecutor.groovy | 16 ++++----- .../executors/JDimeMergeToolExecutor.groovy | 35 +++++++++++++++++++ 3 files changed, 45 insertions(+), 8 deletions(-) create mode 100644 src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 56de299da..58a79070f 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -5,6 +5,7 @@ import org.apache.commons.io.FileUtils import project.MergeCommit import project.Project import services.dataCollectors.GenericMerge.executors.GenericMergeToolExecutor +import services.dataCollectors.GenericMerge.executors.JDimeMergeToolExecutor import services.dataCollectors.GenericMerge.executors.MergeToolExecutor import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector @@ -20,6 +21,7 @@ class GenericMergeDataCollector implements DataCollector { GenericMergeDataCollector() { this.mergeToolExecutors = new ArrayList() this.mergeToolExecutors.add(new GenericMergeToolExecutor()) + this.mergeToolExecutors.add(new JDimeMergeToolExecutor()) } @Override diff --git a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy index 6341e23b1..7dbcf6483 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy @@ -27,13 +27,13 @@ class GenericMergeToolExecutor extends MergeToolExecutor { } private static List getBuildParameters() { - return List.of( - GENERIC_MERGE_BINARY_PATH, - "--base-path=basejava", - "--left-path=leftjava", - "--right-path=rightjava", - "--merge-path=merge.generic.java", - "--language=java" - ) + def list = new ArrayList() + list.add(GENERIC_MERGE_BINARY_PATH) + list.add("--base-path=basejava") + list.add("--left-path=leftjava") + list.add("--right-path=rightjava") + list.add("--merge-path=merge.generic.java") + list.add("--language=java") + return list; } } diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy new file mode 100644 index 000000000..6e150078f --- /dev/null +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -0,0 +1,35 @@ +package services.dataCollectors.GenericMerge.executors + +import services.dataCollectors.GenericMerge.GenericMergeDataCollector +import util.ProcessRunner + +import java.nio.file.Path + +class JDimeMergeToolExecutor extends MergeToolExecutor { + private static final BASE_EXPERIMENT_PATH = "/usr/src/app" + private static final String JDIME_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/jdime/install/JDime/bin" + + @Override + protected GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario) { + def working_directory_path = scenario.toAbsolutePath().toString(); + + def processBuilder = ProcessRunner.buildProcess(JDIME_BINARY_PATH); + processBuilder.command().addAll(getBuildParameters(working_directory_path)) + + def output = ProcessRunner.startProcess(processBuilder); + output.waitFor() + + return output.exitValue() == 0 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS : output.exitValue() == 1 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS : GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR; + } + + private static List getBuildParameters(String basePath) { + def list = new ArrayList() + list.add("./JDime") + list.add("--mode=structured") + list.add("--output=${basePath}/merge.generic.java") + list.add("${basePath}/leftjava") + list.add("${basePath}/basejava") + list.add("${basePath}/rightjava") + return list; + } +} From a8ed95059506c7e0d33e36a5bddca42f8a651af7 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 17:59:14 -0400 Subject: [PATCH 08/79] chore: fix return codes --- .../GenericMerge/executors/JDimeMergeToolExecutor.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index 6e150078f..a647fef30 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -19,7 +19,7 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { def output = ProcessRunner.startProcess(processBuilder); output.waitFor() - return output.exitValue() == 0 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS : output.exitValue() == 1 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS : GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR; + return output.exitValue() == 0 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS : output.exitValue() <= 127 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS : GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR; } private static List getBuildParameters(String basePath) { From 5755b1e36e6941166beef9b76d8d0e75406c9d3c Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 18:21:55 -0400 Subject: [PATCH 09/79] feat: comment jdime --- .../GenericMerge/GenericMergeDataCollector.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 58a79070f..f35b4e94e 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -5,7 +5,7 @@ import org.apache.commons.io.FileUtils import project.MergeCommit import project.Project import services.dataCollectors.GenericMerge.executors.GenericMergeToolExecutor -import services.dataCollectors.GenericMerge.executors.JDimeMergeToolExecutor +//import services.dataCollectors.GenericMerge.executors.JDimeMergeToolExecutor import services.dataCollectors.GenericMerge.executors.MergeToolExecutor import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector @@ -21,7 +21,7 @@ class GenericMergeDataCollector implements DataCollector { GenericMergeDataCollector() { this.mergeToolExecutors = new ArrayList() this.mergeToolExecutors.add(new GenericMergeToolExecutor()) - this.mergeToolExecutors.add(new JDimeMergeToolExecutor()) +// this.mergeToolExecutors.add(new JDimeMergeToolExecutor()) } @Override From 69c1bdb8afb0c0bec322bd566d187357d3239fbd Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 18:27:27 -0400 Subject: [PATCH 10/79] feat: add identifier --- .../GenericMerge/executors/GenericMergeToolExecutor.groovy | 5 +++++ .../GenericMerge/executors/JDimeMergeToolExecutor.groovy | 5 +++++ .../GenericMerge/executors/MergeToolExecutor.groovy | 4 +++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy index 7dbcf6483..d959cd5d3 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy @@ -26,6 +26,11 @@ class GenericMergeToolExecutor extends MergeToolExecutor { return output.exitValue() == 0 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS : output.exitValue() == 1 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS : GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR; } + @Override + protected String getToolName() { + return "GENERIC_MERGE" + } + private static List getBuildParameters() { def list = new ArrayList() list.add(GENERIC_MERGE_BINARY_PATH) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index a647fef30..dd7319a96 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -22,6 +22,11 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { return output.exitValue() == 0 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS : output.exitValue() <= 127 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS : GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR; } + @Override + protected String getToolName() { + return "JDIME" + } + private static List getBuildParameters(String basePath) { def list = new ArrayList() list.add("./JDime") diff --git a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy index 4369984f3..1cdf8cc4f 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy @@ -8,8 +8,10 @@ abstract class MergeToolExecutor { static GenericMergeDataCollector.MergeScenarioExecutionSummary runToolForMergeScenario(Path scenario) { def startTime = System.nanoTime(); def result = executeTool(scenario) - return new GenericMergeDataCollector.MergeScenarioExecutionSummary(scenario, result, System.nanoTime() - startTime, this.getSimpleName()) + return new GenericMergeDataCollector.MergeScenarioExecutionSummary(scenario, result, System.nanoTime() - startTime, this.getToolName()) } protected abstract GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario); + + protected abstract String getToolName(); } From b7e553058ea72ace77df8b9d24eff89023ae0eed Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 18:45:33 -0400 Subject: [PATCH 11/79] fix: broken imports --- .../GenericMerge/GenericMergeDataCollector.groovy | 9 ++++++--- .../GenericMerge/executors/MergeToolExecutor.groovy | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index f35b4e94e..074308de6 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -5,7 +5,7 @@ import org.apache.commons.io.FileUtils import project.MergeCommit import project.Project import services.dataCollectors.GenericMerge.executors.GenericMergeToolExecutor -//import services.dataCollectors.GenericMerge.executors.JDimeMergeToolExecutor +import services.dataCollectors.GenericMerge.executors.JDimeMergeToolExecutor import services.dataCollectors.GenericMerge.executors.MergeToolExecutor import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector @@ -21,7 +21,7 @@ class GenericMergeDataCollector implements DataCollector { GenericMergeDataCollector() { this.mergeToolExecutors = new ArrayList() this.mergeToolExecutors.add(new GenericMergeToolExecutor()) -// this.mergeToolExecutors.add(new JDimeMergeToolExecutor()) + this.mergeToolExecutors.add(new JDimeMergeToolExecutor()) } @Override @@ -32,7 +32,10 @@ class GenericMergeDataCollector implements DataCollector { def reportFile = new File("${GENERIC_MERGE_REPORTS_PATH}/${project.getName()}.csv"); reportFile.createNewFile(); - scenarios.stream().flatMap { scenario -> mergeToolExecutors.stream().map { executor -> executor.runToolForMergeScenario(scenario) } + scenarios.stream().flatMap { scenario -> + mergeToolExecutors.stream().map { executor -> + executor.runToolForMergeScenario(scenario) + } }.forEach { def line = List.of(project.getName(), mergeCommit.getSHA(), diff --git a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy index 1cdf8cc4f..fb8b4c80e 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy @@ -5,7 +5,7 @@ import services.dataCollectors.GenericMerge.GenericMergeDataCollector; import java.nio.file.Path; abstract class MergeToolExecutor { - static GenericMergeDataCollector.MergeScenarioExecutionSummary runToolForMergeScenario(Path scenario) { + GenericMergeDataCollector.MergeScenarioExecutionSummary runToolForMergeScenario(Path scenario) { def startTime = System.nanoTime(); def result = executeTool(scenario) return new GenericMergeDataCollector.MergeScenarioExecutionSummary(scenario, result, System.nanoTime() - startTime, this.getToolName()) From b55d55c9eab6afbcca231b2124902b4e2d062298 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 18:49:49 -0400 Subject: [PATCH 12/79] fix: broken compilation --- .../GenericMergeDataCollector.groovy | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 074308de6..98a733a02 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -37,13 +37,16 @@ class GenericMergeDataCollector implements DataCollector { executor.runToolForMergeScenario(scenario) } }.forEach { - def line = List.of(project.getName(), - mergeCommit.getSHA(), - it.scenario.toAbsolutePath().toString(), - anyParentEqualsBase(it.scenario).toString(), - it.result, - it.time).join(",") - reportFile << "${line.replaceAll('\\\\', '/')}\n" + def list = new ArrayList(); + list.add(project.getName()) + list.add(it.tool) + list.add(mergeCommit.getSHA()) + list.add(it.scenario.toAbsolutePath().toString()) + list.add(anyParentEqualsBase(it.scenario).toString()) + list.add(it.result.toString()) + list.add(it.time.toString()) + + reportFile << "${list.join(",").replaceAll('\\\\', '/')}\n" } } From 0ac5b09826ee5062466c29ebff2aa0f349526704 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 20:42:49 -0400 Subject: [PATCH 13/79] inline process --- .../executors/JDimeMergeToolExecutor.groovy | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index dd7319a96..4ebd62110 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -13,8 +13,15 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { protected GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario) { def working_directory_path = scenario.toAbsolutePath().toString(); - def processBuilder = ProcessRunner.buildProcess(JDIME_BINARY_PATH); - processBuilder.command().addAll(getBuildParameters(working_directory_path)) + def processBuilder = ProcessRunner.buildProcess( + JDIME_BINARY_PATH, + "./JDime", + "--mode=structured", + "--output=${working_directory_path}/merge.generic.java", + "${working_directory_path}/leftjava", + "${working_directory_path}/basejava", + "${working_directory_path}/rightjava" + ) def output = ProcessRunner.startProcess(processBuilder); output.waitFor() From 8483972c4b69c779321951eeeca2f86f0306d30c Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 20:53:40 -0400 Subject: [PATCH 14/79] fix: return values --- .../GenericMerge/executors/JDimeMergeToolExecutor.groovy | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index 4ebd62110..add6a2fd7 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -16,6 +16,7 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { def processBuilder = ProcessRunner.buildProcess( JDIME_BINARY_PATH, "./JDime", + "-s", "--mode=structured", "--output=${working_directory_path}/merge.generic.java", "${working_directory_path}/leftjava", @@ -26,7 +27,12 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { def output = ProcessRunner.startProcess(processBuilder); output.waitFor() - return output.exitValue() == 0 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS : output.exitValue() <= 127 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS : GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR; + if (output.exitValue() == 0) { + return GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS + } else if (output.exitValue() >= 200) { + return GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR + } + return GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS } @Override From 5f2d718af2ffd33eff7747bb9897a5a133cc0569 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 21:09:14 -0400 Subject: [PATCH 15/79] fix file name and force write --- .../GenericMerge/executors/JDimeMergeToolExecutor.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index add6a2fd7..8a22e6708 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -16,9 +16,9 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { def processBuilder = ProcessRunner.buildProcess( JDIME_BINARY_PATH, "./JDime", - "-s", + "-sf", "--mode=structured", - "--output=${working_directory_path}/merge.generic.java", + "--output=${working_directory_path}/merge.jdime.java", "${working_directory_path}/leftjava", "${working_directory_path}/basejava", "${working_directory_path}/rightjava" From f2205fcb24f2f83a2c8d7c4d4b6594028e0ada1c Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 21:45:09 -0400 Subject: [PATCH 16/79] add logging on jdime error --- .../GenericMerge/executors/JDimeMergeToolExecutor.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index 8a22e6708..70251481d 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -30,6 +30,7 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { if (output.exitValue() == 0) { return GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS } else if (output.exitValue() >= 200) { + println("Error while merging ${scenario.toAbsolutePath()}: ${output.getInputStream().readLines()}") return GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR } return GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS From 46565f22dd2eae74d3c66103852931ac4e8b10ff Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 21:50:20 -0400 Subject: [PATCH 17/79] fix: use .java --- .../GenericMerge/executors/GenericMergeToolExecutor.groovy | 6 +++--- .../GenericMerge/executors/JDimeMergeToolExecutor.groovy | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy index d959cd5d3..b40fe1cb0 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy @@ -34,9 +34,9 @@ class GenericMergeToolExecutor extends MergeToolExecutor { private static List getBuildParameters() { def list = new ArrayList() list.add(GENERIC_MERGE_BINARY_PATH) - list.add("--base-path=basejava") - list.add("--left-path=leftjava") - list.add("--right-path=rightjava") + list.add("--base-path=base.java") + list.add("--left-path=left.java") + list.add("--right-path=right.java") list.add("--merge-path=merge.generic.java") list.add("--language=java") return list; diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index 70251481d..fa4a0f851 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -19,9 +19,9 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { "-sf", "--mode=structured", "--output=${working_directory_path}/merge.jdime.java", - "${working_directory_path}/leftjava", - "${working_directory_path}/basejava", - "${working_directory_path}/rightjava" + "${working_directory_path}/left.java", + "${working_directory_path}/base.java", + "${working_directory_path}/right.java" ) def output = ProcessRunner.startProcess(processBuilder); From d25b091cd3ac7bc9c22642635aeda2bae57e0c4f Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 22:03:36 -0400 Subject: [PATCH 18/79] Remove flags --- .../executors/JDimeMergeToolExecutor.groovy | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index fa4a0f851..e1a2d3778 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -16,7 +16,6 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { def processBuilder = ProcessRunner.buildProcess( JDIME_BINARY_PATH, "./JDime", - "-sf", "--mode=structured", "--output=${working_directory_path}/merge.jdime.java", "${working_directory_path}/left.java", @@ -27,28 +26,15 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { def output = ProcessRunner.startProcess(processBuilder); output.waitFor() - if (output.exitValue() == 0) { - return GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS - } else if (output.exitValue() >= 200) { + if (output.exitValue() >= 200) { println("Error while merging ${scenario.toAbsolutePath()}: ${output.getInputStream().readLines()}") return GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR } - return GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS + return GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS } @Override protected String getToolName() { return "JDIME" } - - private static List getBuildParameters(String basePath) { - def list = new ArrayList() - list.add("./JDime") - list.add("--mode=structured") - list.add("--output=${basePath}/merge.generic.java") - list.add("${basePath}/leftjava") - list.add("${basePath}/basejava") - list.add("${basePath}/rightjava") - return list; - } } From 1bacc9d18cdccac6ffdefb0318acb47bcf918643 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 22:04:05 -0400 Subject: [PATCH 19/79] add logging --- .../GenericMerge/executors/JDimeMergeToolExecutor.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index e1a2d3778..77a288194 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -24,7 +24,9 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { ) def output = ProcessRunner.startProcess(processBuilder); + println("Sarted merging ${scenario.toAbsolutePath()} with jdime") output.waitFor() + println("Finished merging ${scenario.toAbsolutePath()} with jdime") if (output.exitValue() >= 200) { println("Error while merging ${scenario.toAbsolutePath()}: ${output.getInputStream().readLines()}") From 32bb268d67fb4c1bc2c32d1977c8b041068f78d4 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 22:20:58 -0400 Subject: [PATCH 20/79] skip scenarios in which either one of the parents equals base --- .../GenericMergeDataCollector.groovy | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 98a733a02..75d9631aa 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -32,22 +32,24 @@ class GenericMergeDataCollector implements DataCollector { def reportFile = new File("${GENERIC_MERGE_REPORTS_PATH}/${project.getName()}.csv"); reportFile.createNewFile(); - scenarios.stream().flatMap { scenario -> - mergeToolExecutors.stream().map { executor -> - executor.runToolForMergeScenario(scenario) - } - }.forEach { - def list = new ArrayList(); - list.add(project.getName()) - list.add(it.tool) - list.add(mergeCommit.getSHA()) - list.add(it.scenario.toAbsolutePath().toString()) - list.add(anyParentEqualsBase(it.scenario).toString()) - list.add(it.result.toString()) - list.add(it.time.toString()) + scenarios.stream() + .filter { scenario -> !anyParentEqualsBase(scenario) } + .flatMap { scenario -> + mergeToolExecutors.stream().map { executor -> + executor.runToolForMergeScenario(scenario) + } + } + .forEach { + def list = new ArrayList(); + list.add(project.getName()) + list.add(it.tool) + list.add(mergeCommit.getSHA()) + list.add(it.scenario.toAbsolutePath().toString()) + list.add(it.result.toString()) + list.add(it.time.toString()) - reportFile << "${list.join(",").replaceAll('\\\\', '/')}\n" - } + reportFile << "${list.join(",").replaceAll('\\\\', '/')}\n" + } } private static anyParentEqualsBase(Path scenario) { From 03a68a47fc0e7b934c2d080adbf47f13b8c25a75 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Fri, 24 May 2024 22:25:47 -0400 Subject: [PATCH 21/79] fix extensions --- .../GenericMerge/GenericMergeDataCollector.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 75d9631aa..f2a599bc9 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -53,11 +53,11 @@ class GenericMergeDataCollector implements DataCollector { } private static anyParentEqualsBase(Path scenario) { - def leftEqualsBase = FileUtils.contentEquals(new File("${scenario.toAbsolutePath()}/basejava"), - new File("${scenario.toAbsolutePath()}/leftjava")) + def leftEqualsBase = FileUtils.contentEquals(new File("${scenario.toAbsolutePath()}/base.java"), + new File("${scenario.toAbsolutePath()}/left.java")) - def rightEqualsBase = FileUtils.contentEquals(new File("${scenario.toAbsolutePath()}/basejava"), - new File("${scenario.toAbsolutePath()}/rightjava")) + def rightEqualsBase = FileUtils.contentEquals(new File("${scenario.toAbsolutePath()}/base.java"), + new File("${scenario.toAbsolutePath()}/right.java")) return leftEqualsBase || rightEqualsBase } From ff41e734df8033e9248929d910553b3f1e8203c6 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sat, 25 May 2024 19:17:24 -0400 Subject: [PATCH 22/79] feat: add build request --- .../GenericMergeDataCollector.groovy | 35 +++++++++++++------ .../executors/JDimeMergeToolExecutor.groovy | 2 -- src/main/services/util/BuildRequester.groovy | 19 ++++++++-- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index f2a599bc9..89424fcac 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -8,6 +8,7 @@ import services.dataCollectors.GenericMerge.executors.GenericMergeToolExecutor import services.dataCollectors.GenericMerge.executors.JDimeMergeToolExecutor import services.dataCollectors.GenericMerge.executors.MergeToolExecutor import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector +import services.util.BuildRequester import java.nio.file.Files import java.nio.file.Path @@ -32,18 +33,20 @@ class GenericMergeDataCollector implements DataCollector { def reportFile = new File("${GENERIC_MERGE_REPORTS_PATH}/${project.getName()}.csv"); reportFile.createNewFile(); - scenarios.stream() - .filter { scenario -> !anyParentEqualsBase(scenario) } - .flatMap { scenario -> - mergeToolExecutors.stream().map { executor -> - executor.runToolForMergeScenario(scenario) - } - } - .forEach { + println "Starting execution of merge tools" + def mergeToolsExecutionResults = executeMergeTools(scenarios) + println "Finished execution of merge tools" + + println "Starting commit analysis" + BuildRequester.requestBuildWithRevision(project, mergeCommit, scenarios, "generic") + BuildRequester.requestBuildWithRevision(project, mergeCommit, scenarios, "jdime") + println "Finished commit analysis" + + mergeToolsExecutionResults.forEach { def list = new ArrayList(); list.add(project.getName()) - list.add(it.tool) list.add(mergeCommit.getSHA()) + list.add(it.tool) list.add(it.scenario.toAbsolutePath().toString()) list.add(it.result.toString()) list.add(it.time.toString()) @@ -52,14 +55,24 @@ class GenericMergeDataCollector implements DataCollector { } } - private static anyParentEqualsBase(Path scenario) { + private executeMergeTools(List scenarios) { + return scenarios.stream() + .filter { this::eitherParentDiffersFromBase } + .flatMap { scenario -> + mergeToolExecutors.stream().map { executor -> + executor.runToolForMergeScenario(scenario) + } + } + } + + private static boolean eitherParentDiffersFromBase(Path scenario) { def leftEqualsBase = FileUtils.contentEquals(new File("${scenario.toAbsolutePath()}/base.java"), new File("${scenario.toAbsolutePath()}/left.java")) def rightEqualsBase = FileUtils.contentEquals(new File("${scenario.toAbsolutePath()}/base.java"), new File("${scenario.toAbsolutePath()}/right.java")) - return leftEqualsBase || rightEqualsBase + return !leftEqualsBase && !rightEqualsBase } static enum MergeScenarioResult { diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index 77a288194..e1a2d3778 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -24,9 +24,7 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { ) def output = ProcessRunner.startProcess(processBuilder); - println("Sarted merging ${scenario.toAbsolutePath()} with jdime") output.waitFor() - println("Finished merging ${scenario.toAbsolutePath()} with jdime") if (output.exitValue() >= 200) { println("Error while merging ${scenario.toAbsolutePath()}: ${output.getInputStream().readLines()}") diff --git a/src/main/services/util/BuildRequester.groovy b/src/main/services/util/BuildRequester.groovy index c38b9caa3..1bd9e247d 100644 --- a/src/main/services/util/BuildRequester.groovy +++ b/src/main/services/util/BuildRequester.groovy @@ -49,6 +49,17 @@ final class BuildRequester { return getBuildLink(project, branchName) } + + static requestBuildWithRevision(Project project, MergeCommit mergeCommit, List mergeScenarios, String mergeTool) { + String toReplaceFile = "merge.${mergeTool}.java" + + String branchName = "${mergeCommit.getSHA().take(7)}-${mergeTool}" + + createBranchFromCommit(project, mergeCommit, branchName) + replaceFilesInProject(project, mergeCommit, mergeScenarios, toReplaceFile) + stageAndPushChanges(project, branchName, "Mining Framework Analysis") + } + private static String getBuildLink(Project project, String branchName) { String buildID = getBuildAttribute(project, 'id', branchName) return "https://travis-ci.com/${projectOwnerName}/${project.getName()}/builds/${buildID}" @@ -149,18 +160,22 @@ script: """ } - private static void stageAndPushChanges(Project project, String branchName) { + private static void stageAndPushChanges(Project project, String branchName, String commitMessage) { Path projectPath = Paths.get(project.getPath()) // Stage changes Utils.runGitCommand(projectPath, 'add', '.') // Commit changes - Utils.runGitCommand(projectPath, 'commit', '-m', 'S3M Handlers Analysis new branch') + Utils.runGitCommand(projectPath, 'commit', '-m', commitMessage) // Push changes Utils.runGitCommand(projectPath, 'push', '--set-upstream', 'origin', branchName, '--force-with-lease') } + private static void stageAndPushChanges(Project project, String branchName) { + stageAndPushChanges(project, branchName, 'S3M Handlers Analysis new branch') + } + } From 1fac42edbae9c8406681f3e0cd97c1d0f4661ca8 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sat, 25 May 2024 19:37:01 -0400 Subject: [PATCH 23/79] feat: move code --- .../GenericMerge/BuildRequester.groovy | 56 +++++++++++++++++++ .../GenericMergeDataCollector.groovy | 2 +- 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy new file mode 100644 index 000000000..66bc39ae1 --- /dev/null +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -0,0 +1,56 @@ +package services.dataCollectors.GenericMerge + +import project.MergeCommit +import project.Project +import services.util.Utils +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.StandardCopyOption + +class BuildRequester { + static requestBuildWithRevision(Project project, MergeCommit mergeCommit, List mergeScenarios, String mergeTool) { + String toReplaceFile = "merge.${mergeTool}.java" + + String branchName = "${mergeCommit.getSHA().take(7)}-${mergeTool}" + + createBranchFromCommit(project, mergeCommit, branchName) + replaceFilesInProject(project, mergeCommit, mergeScenarios, toReplaceFile) + stageAndPushChanges(project, branchName, "Mining Framework Analysis") + } + + private static void createBranchFromCommit(Project project, MergeCommit mergeCommit, String branchName) { + Path projectPath = Paths.get(project.getPath()) + + // Checkout to new branch + Utils.runGitCommand(projectPath, 'checkout', '-b', branchName, mergeCommit.getSHA()) + } + + private static void replaceFilesInProject(Project project, MergeCommit mergeCommit, List mergeScenarios, String toReplaceFile) { + mergeScenarios.stream() + .forEach(mergeScenario -> Files.copy(getSource(mergeScenario, toReplaceFile), getTarget(project, mergeCommit, mergeScenario), StandardCopyOption.REPLACE_EXISTING)) + } + + private static Path getSource(Path mergeScenario, String toReplaceFile) { + return mergeScenario.resolve(toReplaceFile) + } + + private static Path getTarget(Project project, MergeCommit mergeCommit, Path mergeScenario) { + Path projectPath = Paths.get(project.getPath()) + Path filePath = Utils.commitFilesPath(project, mergeCommit).relativize(mergeScenario) + return projectPath.resolve(filePath) + } + + private static void stageAndPushChanges(Project project, String branchName, String commitMessage) { + Path projectPath = Paths.get(project.getPath()) + + // Stage changes + Utils.runGitCommand(projectPath, 'add', '.') + + // Commit changes + Utils.runGitCommand(projectPath, 'commit', '-m', commitMessage) + + // Push changes + Utils.runGitCommand(projectPath, 'push', '--set-upstream', 'origin', branchName, '--force-with-lease') + } +} diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 89424fcac..d3e96c16b 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -8,7 +8,7 @@ import services.dataCollectors.GenericMerge.executors.GenericMergeToolExecutor import services.dataCollectors.GenericMerge.executors.JDimeMergeToolExecutor import services.dataCollectors.GenericMerge.executors.MergeToolExecutor import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector -import services.util.BuildRequester +import services.dataCollectors.GenericMerge.BuildRequester import java.nio.file.Files import java.nio.file.Path From 039401d3695e787b8fad0eecd9a017a0b92eda7f Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sat, 25 May 2024 19:52:27 -0400 Subject: [PATCH 24/79] feat: always run tools anyway --- .../GenericMerge/GenericMergeDataCollector.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index d3e96c16b..400a76c8e 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -46,6 +46,7 @@ class GenericMergeDataCollector implements DataCollector { def list = new ArrayList(); list.add(project.getName()) list.add(mergeCommit.getSHA()) + list.add(eitherParentDiffersFromBase(it.scenario).toString()) list.add(it.tool) list.add(it.scenario.toAbsolutePath().toString()) list.add(it.result.toString()) @@ -57,7 +58,6 @@ class GenericMergeDataCollector implements DataCollector { private executeMergeTools(List scenarios) { return scenarios.stream() - .filter { this::eitherParentDiffersFromBase } .flatMap { scenario -> mergeToolExecutors.stream().map { executor -> executor.runToolForMergeScenario(scenario) From 168b993bc2c838a0f7d99b005efd42b08a66254d Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sat, 25 May 2024 20:20:10 -0400 Subject: [PATCH 25/79] feat: skip copy if the file do not exist --- .../services/dataCollectors/GenericMerge/BuildRequester.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index 66bc39ae1..37d25ec8a 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -28,6 +28,7 @@ class BuildRequester { private static void replaceFilesInProject(Project project, MergeCommit mergeCommit, List mergeScenarios, String toReplaceFile) { mergeScenarios.stream() + .filter { Files.exists(getSource(it, toReplaceFile)) } .forEach(mergeScenario -> Files.copy(getSource(mergeScenario, toReplaceFile), getTarget(project, mergeCommit, mergeScenario), StandardCopyOption.REPLACE_EXISTING)) } From b1224e7e073a679a6217618e4550c5e69a6e9376 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sat, 25 May 2024 20:31:35 -0400 Subject: [PATCH 26/79] feat: add logging --- .../dataCollectors/GenericMerge/BuildRequester.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index 37d25ec8a..407ca040a 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -28,7 +28,11 @@ class BuildRequester { private static void replaceFilesInProject(Project project, MergeCommit mergeCommit, List mergeScenarios, String toReplaceFile) { mergeScenarios.stream() - .filter { Files.exists(getSource(it, toReplaceFile)) } + .filter { + def fileExists = Files.exists(getSource(it, toReplaceFile)); + println "Checking if ${getSource(it, toReplaceFile)} exists -> ${fileExists}" + return fileExists; + } .forEach(mergeScenario -> Files.copy(getSource(mergeScenario, toReplaceFile), getTarget(project, mergeCommit, mergeScenario), StandardCopyOption.REPLACE_EXISTING)) } From 647f23ec1731a073761af61faeff0ed8df39ec25 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sat, 25 May 2024 20:35:26 -0400 Subject: [PATCH 27/79] feat: force absolute --- .../dataCollectors/GenericMerge/BuildRequester.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index 407ca040a..7a2410292 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -29,8 +29,8 @@ class BuildRequester { private static void replaceFilesInProject(Project project, MergeCommit mergeCommit, List mergeScenarios, String toReplaceFile) { mergeScenarios.stream() .filter { - def fileExists = Files.exists(getSource(it, toReplaceFile)); - println "Checking if ${getSource(it, toReplaceFile)} exists -> ${fileExists}" + def fileExists = Files.exists(getSource(it, toReplaceFile).toAbsolutePath()); + println "Checking if ${getSource(it, toReplaceFile).toAbsolutePath()} exists -> ${fileExists}" return fileExists; } .forEach(mergeScenario -> Files.copy(getSource(mergeScenario, toReplaceFile), getTarget(project, mergeCommit, mergeScenario), StandardCopyOption.REPLACE_EXISTING)) From e6f27f55d98cabc7e3d18aa8f4506293e5368b25 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sat, 25 May 2024 20:43:39 -0400 Subject: [PATCH 28/79] use isFile() --- .../services/dataCollectors/GenericMerge/BuildRequester.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index 7a2410292..2ef64f00b 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -29,7 +29,7 @@ class BuildRequester { private static void replaceFilesInProject(Project project, MergeCommit mergeCommit, List mergeScenarios, String toReplaceFile) { mergeScenarios.stream() .filter { - def fileExists = Files.exists(getSource(it, toReplaceFile).toAbsolutePath()); + def fileExists = new File(getSource(it, toReplaceFile).toAbsolutePath()).isFile(); println "Checking if ${getSource(it, toReplaceFile).toAbsolutePath()} exists -> ${fileExists}" return fileExists; } From f063a7298259fa950aa1a2a1cc93fb80b4de9170 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sat, 25 May 2024 20:48:11 -0400 Subject: [PATCH 29/79] fiox: typing issue --- .../services/dataCollectors/GenericMerge/BuildRequester.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index 2ef64f00b..d706d1949 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -29,7 +29,7 @@ class BuildRequester { private static void replaceFilesInProject(Project project, MergeCommit mergeCommit, List mergeScenarios, String toReplaceFile) { mergeScenarios.stream() .filter { - def fileExists = new File(getSource(it, toReplaceFile).toAbsolutePath()).isFile(); + def fileExists = new File(getSource(it, toReplaceFile).toAbsolutePath().toString()).isFile(); println "Checking if ${getSource(it, toReplaceFile).toAbsolutePath()} exists -> ${fileExists}" return fileExists; } From a26d6209bd201c55281cbb9e3f5bfe66b65c66a1 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sat, 25 May 2024 20:50:41 -0400 Subject: [PATCH 30/79] force true --- .../services/dataCollectors/GenericMerge/BuildRequester.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index d706d1949..290af206a 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -29,6 +29,7 @@ class BuildRequester { private static void replaceFilesInProject(Project project, MergeCommit mergeCommit, List mergeScenarios, String toReplaceFile) { mergeScenarios.stream() .filter { + return true; def fileExists = new File(getSource(it, toReplaceFile).toAbsolutePath().toString()).isFile(); println "Checking if ${getSource(it, toReplaceFile).toAbsolutePath()} exists -> ${fileExists}" return fileExists; From 81e4f8a693f3f0e5436c1b4128b24bc82e7dfa1d Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sat, 25 May 2024 21:06:44 -0400 Subject: [PATCH 31/79] force true --- .../services/dataCollectors/GenericMerge/BuildRequester.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index 290af206a..43b4d8669 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -29,10 +29,9 @@ class BuildRequester { private static void replaceFilesInProject(Project project, MergeCommit mergeCommit, List mergeScenarios, String toReplaceFile) { mergeScenarios.stream() .filter { - return true; def fileExists = new File(getSource(it, toReplaceFile).toAbsolutePath().toString()).isFile(); println "Checking if ${getSource(it, toReplaceFile).toAbsolutePath()} exists -> ${fileExists}" - return fileExists; + return true; } .forEach(mergeScenario -> Files.copy(getSource(mergeScenario, toReplaceFile), getTarget(project, mergeCommit, mergeScenario), StandardCopyOption.REPLACE_EXISTING)) } From 95ffe024e29fe0158807a46b284d5975a7332c8a Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sat, 25 May 2024 21:21:53 -0400 Subject: [PATCH 32/79] feat: return --- .../dataCollectors/GenericMerge/BuildRequester.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index 43b4d8669..f01055578 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -30,8 +30,8 @@ class BuildRequester { mergeScenarios.stream() .filter { def fileExists = new File(getSource(it, toReplaceFile).toAbsolutePath().toString()).isFile(); - println "Checking if ${getSource(it, toReplaceFile).toAbsolutePath()} exists -> ${fileExists}" - return true; + println "Checking if ${getSource(it, toReplaceFile).toAbsolutePath()} exists -> ${fileExists} ${Files.isRegularFile(getSource(it, toReplaceFile))}" + return fileExists; } .forEach(mergeScenario -> Files.copy(getSource(mergeScenario, toReplaceFile), getTarget(project, mergeCommit, mergeScenario), StandardCopyOption.REPLACE_EXISTING)) } From c5cc57d36436d095725da3ac1e021d35dda627d9 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sat, 25 May 2024 21:28:11 -0400 Subject: [PATCH 33/79] allow all and skip failing --- .../GenericMerge/BuildRequester.groovy | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index f01055578..9e8efc3e1 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -28,12 +28,13 @@ class BuildRequester { private static void replaceFilesInProject(Project project, MergeCommit mergeCommit, List mergeScenarios, String toReplaceFile) { mergeScenarios.stream() - .filter { - def fileExists = new File(getSource(it, toReplaceFile).toAbsolutePath().toString()).isFile(); - println "Checking if ${getSource(it, toReplaceFile).toAbsolutePath()} exists -> ${fileExists} ${Files.isRegularFile(getSource(it, toReplaceFile))}" - return fileExists; - } - .forEach(mergeScenario -> Files.copy(getSource(mergeScenario, toReplaceFile), getTarget(project, mergeCommit, mergeScenario), StandardCopyOption.REPLACE_EXISTING)) + .forEach(mergeScenario -> { + try { + Files.copy(getSource(mergeScenario, toReplaceFile), getTarget(project, mergeCommit, mergeScenario), StandardCopyOption.REPLACE_EXISTING) + } catch (e) { + println "Error while copying ${getSource(mergeScenario, toReplaceFile)} ${e.getMessage()}" + } + }) } private static Path getSource(Path mergeScenario, String toReplaceFile) { From cccbf3966eb61e6714c23e5c5ab89a2d03a36870 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sun, 26 May 2024 10:47:41 -0400 Subject: [PATCH 34/79] enhance error message --- .../dataCollectors/GenericMerge/BuildRequester.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index 9e8efc3e1..baae7abc8 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -32,7 +32,9 @@ class BuildRequester { try { Files.copy(getSource(mergeScenario, toReplaceFile), getTarget(project, mergeCommit, mergeScenario), StandardCopyOption.REPLACE_EXISTING) } catch (e) { - println "Error while copying ${getSource(mergeScenario, toReplaceFile)} ${e.getMessage()}" + println "Error while copying ${getSource(mergeScenario, toReplaceFile)} to ${getTarget(project, mergeCommit, mergeScenario)}" + println e.cause.message + println e.message } }) } From eed49ed857a982140f9ebcfc806ee1009dc5c7d1 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sun, 26 May 2024 10:53:35 -0400 Subject: [PATCH 35/79] fix error logging --- .../services/dataCollectors/GenericMerge/BuildRequester.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index baae7abc8..461be78d5 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -33,7 +33,6 @@ class BuildRequester { Files.copy(getSource(mergeScenario, toReplaceFile), getTarget(project, mergeCommit, mergeScenario), StandardCopyOption.REPLACE_EXISTING) } catch (e) { println "Error while copying ${getSource(mergeScenario, toReplaceFile)} to ${getTarget(project, mergeCommit, mergeScenario)}" - println e.cause.message println e.message } }) From 8cfe67e5d6dfb7145b76224db7343762484b5c8a Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sun, 26 May 2024 11:03:16 -0400 Subject: [PATCH 36/79] fallback to native cp --- .../GenericMerge/BuildRequester.groovy | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index 461be78d5..297e9d9ef 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -3,6 +3,8 @@ package services.dataCollectors.GenericMerge import project.MergeCommit import project.Project import services.util.Utils +import util.ProcessRunner + import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths @@ -30,10 +32,17 @@ class BuildRequester { mergeScenarios.stream() .forEach(mergeScenario -> { try { - Files.copy(getSource(mergeScenario, toReplaceFile), getTarget(project, mergeCommit, mergeScenario), StandardCopyOption.REPLACE_EXISTING) + def process = ProcessRunner.buildProcess("/usr/src/app", + "cp", + getSource(mergeScenario, toReplaceFile).toAbsolutePath().toString(), + getTarget(project, mergeCommit, mergeScenario).toAbsolutePath().toString() + ) + println "Starting copy of file ${getSource(mergeScenario, toReplaceFile)} to ${getTarget(project, mergeCommit, mergeScenario)}" + process.start().waitFor() + println "Finished copy of file ${getSource(mergeScenario, toReplaceFile)} to ${getTarget(project, mergeCommit, mergeScenario)}" } catch (e) { println "Error while copying ${getSource(mergeScenario, toReplaceFile)} to ${getTarget(project, mergeCommit, mergeScenario)}" - println e.message + println e.toString() } }) } From 1a787b39d5e1f21cecbf9fe215e19356f9a50cdb Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sun, 26 May 2024 11:07:54 -0400 Subject: [PATCH 37/79] log exit code --- .../dataCollectors/GenericMerge/BuildRequester.groovy | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index 297e9d9ef..c5061006d 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -32,14 +32,15 @@ class BuildRequester { mergeScenarios.stream() .forEach(mergeScenario -> { try { - def process = ProcessRunner.buildProcess("/usr/src/app", - "cp", + def process = ProcessRunner.buildProcess( + "/usr/src/app", + "cp", getSource(mergeScenario, toReplaceFile).toAbsolutePath().toString(), getTarget(project, mergeCommit, mergeScenario).toAbsolutePath().toString() ) println "Starting copy of file ${getSource(mergeScenario, toReplaceFile)} to ${getTarget(project, mergeCommit, mergeScenario)}" - process.start().waitFor() - println "Finished copy of file ${getSource(mergeScenario, toReplaceFile)} to ${getTarget(project, mergeCommit, mergeScenario)}" + def statusCode = process.start().waitFor() + println "Finished copy of file with status: ${statusCode}" } catch (e) { println "Error while copying ${getSource(mergeScenario, toReplaceFile)} to ${getTarget(project, mergeCommit, mergeScenario)}" println e.toString() From 26b06c80b88c00665b2438580c99a0fa0c1ebf90 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sun, 26 May 2024 11:18:07 -0400 Subject: [PATCH 38/79] back into Files.copy --- .../GenericMerge/BuildRequester.groovy | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index c5061006d..231e72b25 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -31,20 +31,7 @@ class BuildRequester { private static void replaceFilesInProject(Project project, MergeCommit mergeCommit, List mergeScenarios, String toReplaceFile) { mergeScenarios.stream() .forEach(mergeScenario -> { - try { - def process = ProcessRunner.buildProcess( - "/usr/src/app", - "cp", - getSource(mergeScenario, toReplaceFile).toAbsolutePath().toString(), - getTarget(project, mergeCommit, mergeScenario).toAbsolutePath().toString() - ) - println "Starting copy of file ${getSource(mergeScenario, toReplaceFile)} to ${getTarget(project, mergeCommit, mergeScenario)}" - def statusCode = process.start().waitFor() - println "Finished copy of file with status: ${statusCode}" - } catch (e) { - println "Error while copying ${getSource(mergeScenario, toReplaceFile)} to ${getTarget(project, mergeCommit, mergeScenario)}" - println e.toString() - } + Files.copy(getSource(mergeScenario, toReplaceFile), getTarget(project, mergeCommit, mergeScenario)) }) } From 899a13a9ed339306ff607379b2b74c7acd415c1a Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Thu, 30 May 2024 15:48:49 -0400 Subject: [PATCH 39/79] Remove uneeded modifications --- src/main/services/util/BuildRequester.groovy | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/src/main/services/util/BuildRequester.groovy b/src/main/services/util/BuildRequester.groovy index 1bd9e247d..c38b9caa3 100644 --- a/src/main/services/util/BuildRequester.groovy +++ b/src/main/services/util/BuildRequester.groovy @@ -49,17 +49,6 @@ final class BuildRequester { return getBuildLink(project, branchName) } - - static requestBuildWithRevision(Project project, MergeCommit mergeCommit, List mergeScenarios, String mergeTool) { - String toReplaceFile = "merge.${mergeTool}.java" - - String branchName = "${mergeCommit.getSHA().take(7)}-${mergeTool}" - - createBranchFromCommit(project, mergeCommit, branchName) - replaceFilesInProject(project, mergeCommit, mergeScenarios, toReplaceFile) - stageAndPushChanges(project, branchName, "Mining Framework Analysis") - } - private static String getBuildLink(Project project, String branchName) { String buildID = getBuildAttribute(project, 'id', branchName) return "https://travis-ci.com/${projectOwnerName}/${project.getName()}/builds/${buildID}" @@ -160,22 +149,18 @@ script: """ } - private static void stageAndPushChanges(Project project, String branchName, String commitMessage) { + private static void stageAndPushChanges(Project project, String branchName) { Path projectPath = Paths.get(project.getPath()) // Stage changes Utils.runGitCommand(projectPath, 'add', '.') // Commit changes - Utils.runGitCommand(projectPath, 'commit', '-m', commitMessage) + Utils.runGitCommand(projectPath, 'commit', '-m', 'S3M Handlers Analysis new branch') // Push changes Utils.runGitCommand(projectPath, 'push', '--set-upstream', 'origin', branchName, '--force-with-lease') } - private static void stageAndPushChanges(Project project, String branchName) { - stageAndPushChanges(project, branchName, 'S3M Handlers Analysis new branch') - } - } From c113c42c887ac7c8d93333d96e2dd4e98b87460f Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 3 Jun 2024 13:38:25 -0300 Subject: [PATCH 40/79] feat: Add count of merge conflicts found --- .../GenericMerge/executors/JDimeMergeToolExecutor.groovy | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index e1a2d3778..abf372fd0 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -1,6 +1,7 @@ package services.dataCollectors.GenericMerge.executors import services.dataCollectors.GenericMerge.GenericMergeDataCollector +import services.util.MergeConflict import util.ProcessRunner import java.nio.file.Path @@ -30,6 +31,13 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { println("Error while merging ${scenario.toAbsolutePath()}: ${output.getInputStream().readLines()}") return GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR } + + def mergeConflictsCount = MergeConflict.getConflictsNumber(scenario.resolve("merge.jdime.java")); + println("Found ${mergeConflictsCount} while merging") + if (mergeConflictsCount > 0) { + return GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS + } + return GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS } From 245182e3d5d21673cc75bc5ceb779e9e11667bd4 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 3 Jun 2024 13:41:01 -0300 Subject: [PATCH 41/79] feat: skip build for now --- .../GenericMerge/GenericMergeDataCollector.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 400a76c8e..8e82ab77d 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -38,8 +38,8 @@ class GenericMergeDataCollector implements DataCollector { println "Finished execution of merge tools" println "Starting commit analysis" - BuildRequester.requestBuildWithRevision(project, mergeCommit, scenarios, "generic") - BuildRequester.requestBuildWithRevision(project, mergeCommit, scenarios, "jdime") +// BuildRequester.requestBuildWithRevision(project, mergeCommit, scenarios, "generic") +// BuildRequester.requestBuildWithRevision(project, mergeCommit, scenarios, "jdime") println "Finished commit analysis" mergeToolsExecutionResults.forEach { From 83357b8231d566b7158e44b275c922b3bf2e6391 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 3 Jun 2024 14:13:26 -0300 Subject: [PATCH 42/79] Ignore scenarios where one of the parents equals base --- .../GenericMergeDataCollector.groovy | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 8e82ab77d..f79bf310e 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -27,14 +27,22 @@ class GenericMergeDataCollector implements DataCollector { @Override void collectData(Project project, MergeCommit mergeCommit) { - List scenarios = MergeScenarioCollector.collectMergeScenarios(project, mergeCommit) + def scenarios = MergeScenarioCollector.collectMergeScenarios(project, mergeCommit) + .stream() + .filter { + eitherParentDiffersFromBase(it) + } Files.createDirectories(Paths.get("${GENERIC_MERGE_REPORTS_PATH}")); def reportFile = new File("${GENERIC_MERGE_REPORTS_PATH}/${project.getName()}.csv"); reportFile.createNewFile(); println "Starting execution of merge tools" - def mergeToolsExecutionResults = executeMergeTools(scenarios) + def mergeToolsExecutionResults = scenarios.flatMap { scenario -> + mergeToolExecutors.stream().map { executor -> + executor.runToolForMergeScenario(scenario) + } + } println "Finished execution of merge tools" println "Starting commit analysis" @@ -43,26 +51,17 @@ class GenericMergeDataCollector implements DataCollector { println "Finished commit analysis" mergeToolsExecutionResults.forEach { - def list = new ArrayList(); - list.add(project.getName()) - list.add(mergeCommit.getSHA()) - list.add(eitherParentDiffersFromBase(it.scenario).toString()) - list.add(it.tool) - list.add(it.scenario.toAbsolutePath().toString()) - list.add(it.result.toString()) - list.add(it.time.toString()) - - reportFile << "${list.join(",").replaceAll('\\\\', '/')}\n" - } - } - - private executeMergeTools(List scenarios) { - return scenarios.stream() - .flatMap { scenario -> - mergeToolExecutors.stream().map { executor -> - executor.runToolForMergeScenario(scenario) - } - } + def list = new ArrayList(); + list.add(project.getName()) + list.add(mergeCommit.getSHA()) + list.add(eitherParentDiffersFromBase(it.scenario).toString()) + list.add(it.tool) + list.add(it.scenario.toAbsolutePath().toString()) + list.add(it.result.toString()) + list.add(it.time.toString()) + + reportFile << "${list.join(",").replaceAll('\\\\', '/')}\n" + } } private static boolean eitherParentDiffersFromBase(Path scenario) { From c7343211946d1691d448e950a38d70e2c86797b9 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sun, 16 Jun 2024 00:26:07 -0300 Subject: [PATCH 43/79] chore: Add log4j for better logging --- build.gradle | 5 +++++ src/main/resources/log4j2.xml | 14 ++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 src/main/resources/log4j2.xml diff --git a/build.gradle b/build.gradle index 2b33a5e21..dde2a8a73 100644 --- a/build.gradle +++ b/build.gradle @@ -28,6 +28,11 @@ test { } } +run { + systemProperty "miningframework.generic_merge.base_experiment_path", System.getProperty("miningframework.generic_merge.base_experiment_path") + systemProperty "miningframework.generic_merge.merge_tool_executors_to_use", System.getProperty("miningframework.generic_merge.merge_tool_executors_to_use") +} + dependencies { implementation 'com.google.inject:guice:4.2.2' implementation 'commons-io:commons-io:2.6' diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml new file mode 100644 index 000000000..fc5a0fbb1 --- /dev/null +++ b/src/main/resources/log4j2.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file From 17bb73246b8f9e983776f8d49acdb7399e275f3d Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sun, 16 Jun 2024 00:27:25 -0300 Subject: [PATCH 44/79] feat: add check if result files are syntatically equivalent --- src/main/injectors/GenericMergeModule.groovy | 16 +++ .../GenericMergeDataCollector.groovy | 97 ++++++++++++++----- .../executors/GenericMergeToolExecutor.groovy | 17 ++-- .../executors/JDimeMergeToolExecutor.groovy | 23 ++--- .../executors/MergeToolExecutor.groovy | 7 +- 5 files changed, 116 insertions(+), 44 deletions(-) diff --git a/src/main/injectors/GenericMergeModule.groovy b/src/main/injectors/GenericMergeModule.groovy index aee3bade7..891d870a6 100644 --- a/src/main/injectors/GenericMergeModule.groovy +++ b/src/main/injectors/GenericMergeModule.groovy @@ -6,6 +6,8 @@ import interfaces.CommitFilter import interfaces.DataCollector import interfaces.OutputProcessor import interfaces.ProjectProcessor +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger import services.commitFilters.MutuallyModifiedFilesCommitFilter import services.commitFilters.S3MCommitFilter import services.dataCollectors.GenericMerge.GenericMergeDataCollector @@ -14,7 +16,12 @@ import services.projectProcessors.DummyProjectProcessor import services.util.ci.CIPlatform import services.util.ci.TravisPlatform +import java.nio.file.Files +import java.nio.file.Paths + class GenericMergeModule extends AbstractModule { + private static Logger LOG = LogManager.getLogger(GenericMergeModule.class) + @Override protected void configure() { Multibinder projectProcessorBinder = Multibinder.newSetBinder(binder(), ProjectProcessor.class) @@ -28,5 +35,14 @@ class GenericMergeModule extends AbstractModule { bind(CommitFilter.class).to(MutuallyModifiedFilesCommitFilter.class) bind(CIPlatform.class).to(TravisPlatform.class) + + createExecutionReportsFile() + } + + private static void createExecutionReportsFile() { + LOG.info("Creating Generic Merge report file") + Files.createDirectories(Paths.get(GenericMergeDataCollector.GENERIC_MERGE_REPORT_PATH)) + def reportFile = new File(GenericMergeDataCollector.GENERIC_MERGE_REPORT_FILE_NAME) + reportFile.createNewFile() } } diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index f79bf310e..be3f07fa7 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -8,62 +8,109 @@ import services.dataCollectors.GenericMerge.executors.GenericMergeToolExecutor import services.dataCollectors.GenericMerge.executors.JDimeMergeToolExecutor import services.dataCollectors.GenericMerge.executors.MergeToolExecutor import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector -import services.dataCollectors.GenericMerge.BuildRequester +import util.ProcessRunner import java.nio.file.Files import java.nio.file.Path -import java.nio.file.Paths + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; class GenericMergeDataCollector implements DataCollector { - private static final BASE_EXPERIMENT_PATH = "/usr/src/app" - private static final GENERIC_MERGE_REPORTS_PATH = "${BASE_EXPERIMENT_PATH}/output/reports/" + private static Logger LOG = LogManager.getLogger(GenericMergeDataCollector.class); + + private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") + private static final String GENERIC_MERGE_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/generic-merge" + public static final GENERIC_MERGE_REPORT_PATH = "${BASE_EXPERIMENT_PATH}/output/reports" + public static final GENERIC_MERGE_REPORT_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-execution.csv" + public static final MERGE_TOOL_EXECUTORS_TO_USE = System.getProperty("miningframework.generic_merge.merge_tool_executors_to_use", "generic_merge,jdime").split(",") + private final List mergeToolExecutors; GenericMergeDataCollector() { this.mergeToolExecutors = new ArrayList() - this.mergeToolExecutors.add(new GenericMergeToolExecutor()) - this.mergeToolExecutors.add(new JDimeMergeToolExecutor()) + if (MERGE_TOOL_EXECUTORS_TO_USE.contains("generic_merge")) { + LOG.debug("Registering Generic Merge as a merge tool executor") + this.mergeToolExecutors.add(new GenericMergeToolExecutor()) + } + if (MERGE_TOOL_EXECUTORS_TO_USE.contains("jdime")) { + LOG.debug("Registering jDime as a merge tool executor") + this.mergeToolExecutors.add(new JDimeMergeToolExecutor()) + } } @Override void collectData(Project project, MergeCommit mergeCommit) { - def scenarios = MergeScenarioCollector.collectMergeScenarios(project, mergeCommit) - .stream() - .filter { - eitherParentDiffersFromBase(it) - } - - Files.createDirectories(Paths.get("${GENERIC_MERGE_REPORTS_PATH}")); - def reportFile = new File("${GENERIC_MERGE_REPORTS_PATH}/${project.getName()}.csv"); - reportFile.createNewFile(); + def scenarios = filterScenariosForExecution(MergeScenarioCollector.collectMergeScenarios(project, mergeCommit)) - println "Starting execution of merge tools" + LOG.debug("Starting execution of merge tools on scenario") def mergeToolsExecutionResults = scenarios.flatMap { scenario -> mergeToolExecutors.stream().map { executor -> executor.runToolForMergeScenario(scenario) } } - println "Finished execution of merge tools" - - println "Starting commit analysis" -// BuildRequester.requestBuildWithRevision(project, mergeCommit, scenarios, "generic") -// BuildRequester.requestBuildWithRevision(project, mergeCommit, scenarios, "jdime") - println "Finished commit analysis" + LOG.debug("Finished execution of merge tools on scenario") mergeToolsExecutionResults.forEach { + def reportFile = new File(GENERIC_MERGE_REPORT_FILE_NAME) + + def resultFileName = it.tool === "GENERIC_MERGE" ? 'generic' : 'jdime' + def scenarioHasEquivalentResponse = areFilesSyntacticallyEquivalent( + it.scenario.resolve("merge.java"), + it.scenario.resolve("merge.${resultFileName}.java") + ) + def list = new ArrayList(); list.add(project.getName()) list.add(mergeCommit.getSHA()) - list.add(eitherParentDiffersFromBase(it.scenario).toString()) list.add(it.tool) list.add(it.scenario.toAbsolutePath().toString()) + list.add(it.output.toAbsolutePath().toString()) list.add(it.result.toString()) list.add(it.time.toString()) + list.add(scenarioHasEquivalentResponse.toString()) reportFile << "${list.join(",").replaceAll('\\\\', '/')}\n" } } + private static filterScenariosForExecution(List scenarios) { + return scenarios + .stream() + .filter { + def parentDiffersFromBase = eitherParentDiffersFromBase(it) + if (!parentDiffersFromBase) { + LOG.trace("Skipping scenario ${it.toString()} because one of the parents equals base") + } + return parentDiffersFromBase + } + } + + private static boolean areFilesSyntacticallyEquivalent(Path fileA, Path fileB) { + if (!Files.exists(fileA) || !Files.exists(fileB)) { + LOG.trace("Early returning because one of the files ${} do not exist") + return false; + } + + def process = ProcessRunner.buildProcess("./") + + def list = new ArrayList() + list.add(GENERIC_MERGE_BINARY_PATH) + list.add("--diff-only") + list.add("--left-path=${fileA.toAbsolutePath().toString()}".toString()) + list.add("--right-path=${fileB.toAbsolutePath().toString()}".toString()) + process.command().addAll(list) + + def output = ProcessRunner.startProcess(process) + output.waitFor() + + if (output.exitValue() > 1) { + LOG.warn("Error while running comparison between ${fileA.toString()} and ${fileB.toString()}: ${output.getInputStream().readLines()}") + } + + return output.exitValue() == 0; + } + private static boolean eitherParentDiffersFromBase(Path scenario) { def leftEqualsBase = FileUtils.contentEquals(new File("${scenario.toAbsolutePath()}/base.java"), new File("${scenario.toAbsolutePath()}/left.java")) @@ -83,11 +130,13 @@ class GenericMergeDataCollector implements DataCollector { static class MergeScenarioExecutionSummary { public final String tool; public final Path scenario; + public final Path output; public final MergeScenarioResult result; public final long time; - MergeScenarioExecutionSummary(Path scenario, MergeScenarioResult result, long time, String tool) { + MergeScenarioExecutionSummary(Path scenario, Path output, MergeScenarioResult result, long time, String tool) { this.scenario = scenario + this.output = output this.result = result this.time = time this.tool = tool diff --git a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy index b40fe1cb0..717a33626 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy @@ -1,26 +1,31 @@ package services.dataCollectors.GenericMerge.executors +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger import services.dataCollectors.GenericMerge.GenericMergeDataCollector import util.ProcessRunner import java.nio.file.Path class GenericMergeToolExecutor extends MergeToolExecutor { - private static final BASE_EXPERIMENT_PATH = "/usr/src/app" + private static Logger LOG = LogManager.getLogger(GenericMergeToolExecutor.class); + + private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") private static final String GENERIC_MERGE_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/generic-merge" @Override - protected GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario) { + protected GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario, Path outputFile) { def working_directory_path = scenario.toAbsolutePath().toString(); def processBuilder = ProcessRunner.buildProcess(working_directory_path); - processBuilder.command().addAll(getBuildParameters()) + processBuilder.command().addAll(getBuildParameters(outputFile)) def output = ProcessRunner.startProcess(processBuilder); output.waitFor() if (output.exitValue() > 1) { - println("Error while merging ${scenario.toAbsolutePath()}: ${output.getInputStream().readLines()}") + LOG.warn("Error while merging ${scenario.toAbsolutePath()}. Generic Merge exited with exitCode ${output.exitValue()}") + LOG.debug("Generic Merge output: ${output.getInputStream().readLines()}") } return output.exitValue() == 0 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS : output.exitValue() == 1 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS : GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR; @@ -31,13 +36,13 @@ class GenericMergeToolExecutor extends MergeToolExecutor { return "GENERIC_MERGE" } - private static List getBuildParameters() { + private static List getBuildParameters(Path outputFile) { def list = new ArrayList() list.add(GENERIC_MERGE_BINARY_PATH) list.add("--base-path=base.java") list.add("--left-path=left.java") list.add("--right-path=right.java") - list.add("--merge-path=merge.generic.java") + list.add("--merge-path=${outputFile.toAbsolutePath().toString()}".toString()) list.add("--language=java") return list; } diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index abf372fd0..02b672f2b 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -1,5 +1,7 @@ package services.dataCollectors.GenericMerge.executors +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger import services.dataCollectors.GenericMerge.GenericMergeDataCollector import services.util.MergeConflict import util.ProcessRunner @@ -7,33 +9,32 @@ import util.ProcessRunner import java.nio.file.Path class JDimeMergeToolExecutor extends MergeToolExecutor { - private static final BASE_EXPERIMENT_PATH = "/usr/src/app" + private static Logger LOG = LogManager.getLogger(JDimeMergeToolExecutor.class); + + private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") private static final String JDIME_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/jdime/install/JDime/bin" @Override - protected GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario) { + protected GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario, Path outputFile) { def working_directory_path = scenario.toAbsolutePath().toString(); - def processBuilder = ProcessRunner.buildProcess( - JDIME_BINARY_PATH, + def processBuilder = ProcessRunner.buildProcess(JDIME_BINARY_PATH, "./JDime", "--mode=structured", - "--output=${working_directory_path}/merge.jdime.java", + "--output=${outputFile.toAbsolutePath().toString()}".toString(), "${working_directory_path}/left.java", "${working_directory_path}/base.java", - "${working_directory_path}/right.java" - ) + "${working_directory_path}/right.java") def output = ProcessRunner.startProcess(processBuilder); output.waitFor() if (output.exitValue() >= 200) { - println("Error while merging ${scenario.toAbsolutePath()}: ${output.getInputStream().readLines()}") - return GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR + LOG.warn("Error while merging ${scenario.toAbsolutePath()}. jDime exited with exitCode ${output.exitValue()}") + LOG.debug("jDime output: ${output.getInputStream().readLines()}") } - def mergeConflictsCount = MergeConflict.getConflictsNumber(scenario.resolve("merge.jdime.java")); - println("Found ${mergeConflictsCount} while merging") + def mergeConflictsCount = MergeConflict.getConflictsNumber(outputFile); if (mergeConflictsCount > 0) { return GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS } diff --git a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy index fb8b4c80e..76439b9d5 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy @@ -7,11 +7,12 @@ import java.nio.file.Path; abstract class MergeToolExecutor { GenericMergeDataCollector.MergeScenarioExecutionSummary runToolForMergeScenario(Path scenario) { def startTime = System.nanoTime(); - def result = executeTool(scenario) - return new GenericMergeDataCollector.MergeScenarioExecutionSummary(scenario, result, System.nanoTime() - startTime, this.getToolName()) + def outputFilePath = scenario.resolve("merge.${getToolName().toLowerCase()}.java") + def result = executeTool(scenario, outputFilePath) + return new GenericMergeDataCollector.MergeScenarioExecutionSummary(scenario, outputFilePath, result, System.nanoTime() - startTime, this.getToolName()) } - protected abstract GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario); + protected abstract GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario, Path outputFile); protected abstract String getToolName(); } From 27a4edba3498f025426ca69fdc0a7e738cd0db30 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sun, 16 Jun 2024 00:32:19 -0300 Subject: [PATCH 45/79] fix: wrong check on generic merge --- .../GenericMergeDataCollector.groovy | 39 +++++++------------ 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index be3f07fa7..becefc8c3 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -2,6 +2,8 @@ package services.dataCollectors.GenericMerge import interfaces.DataCollector import org.apache.commons.io.FileUtils +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger import project.MergeCommit import project.Project import services.dataCollectors.GenericMerge.executors.GenericMergeToolExecutor @@ -13,11 +15,8 @@ import util.ProcessRunner import java.nio.file.Files import java.nio.file.Path -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - class GenericMergeDataCollector implements DataCollector { - private static Logger LOG = LogManager.getLogger(GenericMergeDataCollector.class); + private static Logger LOG = LogManager.getLogger(GenericMergeDataCollector.class) private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") private static final String GENERIC_MERGE_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/generic-merge" @@ -25,7 +24,7 @@ class GenericMergeDataCollector implements DataCollector { public static final GENERIC_MERGE_REPORT_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-execution.csv" public static final MERGE_TOOL_EXECUTORS_TO_USE = System.getProperty("miningframework.generic_merge.merge_tool_executors_to_use", "generic_merge,jdime").split(",") - private final List mergeToolExecutors; + private final List mergeToolExecutors GenericMergeDataCollector() { this.mergeToolExecutors = new ArrayList() @@ -44,23 +43,13 @@ class GenericMergeDataCollector implements DataCollector { def scenarios = filterScenariosForExecution(MergeScenarioCollector.collectMergeScenarios(project, mergeCommit)) LOG.debug("Starting execution of merge tools on scenario") - def mergeToolsExecutionResults = scenarios.flatMap { scenario -> - mergeToolExecutors.stream().map { executor -> - executor.runToolForMergeScenario(scenario) - } - } + def mergeToolsExecutionResults = scenarios.flatMap { scenario -> mergeToolExecutors.stream().map { executor -> executor.runToolForMergeScenario(scenario) } } LOG.debug("Finished execution of merge tools on scenario") mergeToolsExecutionResults.forEach { def reportFile = new File(GENERIC_MERGE_REPORT_FILE_NAME) - def resultFileName = it.tool === "GENERIC_MERGE" ? 'generic' : 'jdime' - def scenarioHasEquivalentResponse = areFilesSyntacticallyEquivalent( - it.scenario.resolve("merge.java"), - it.scenario.resolve("merge.${resultFileName}.java") - ) - - def list = new ArrayList(); + def list = new ArrayList() list.add(project.getName()) list.add(mergeCommit.getSHA()) list.add(it.tool) @@ -68,7 +57,7 @@ class GenericMergeDataCollector implements DataCollector { list.add(it.output.toAbsolutePath().toString()) list.add(it.result.toString()) list.add(it.time.toString()) - list.add(scenarioHasEquivalentResponse.toString()) + list.add(areFilesSyntacticallyEquivalent(it.scenario.resolve("merge.java"), it.output).toString()) reportFile << "${list.join(",").replaceAll('\\\\', '/')}\n" } @@ -89,7 +78,7 @@ class GenericMergeDataCollector implements DataCollector { private static boolean areFilesSyntacticallyEquivalent(Path fileA, Path fileB) { if (!Files.exists(fileA) || !Files.exists(fileB)) { LOG.trace("Early returning because one of the files ${} do not exist") - return false; + return false } def process = ProcessRunner.buildProcess("./") @@ -108,7 +97,7 @@ class GenericMergeDataCollector implements DataCollector { LOG.warn("Error while running comparison between ${fileA.toString()} and ${fileB.toString()}: ${output.getInputStream().readLines()}") } - return output.exitValue() == 0; + return output.exitValue() == 0 } private static boolean eitherParentDiffersFromBase(Path scenario) { @@ -128,11 +117,11 @@ class GenericMergeDataCollector implements DataCollector { } static class MergeScenarioExecutionSummary { - public final String tool; - public final Path scenario; - public final Path output; - public final MergeScenarioResult result; - public final long time; + public final String tool + public final Path scenario + public final Path output + public final MergeScenarioResult result + public final long time MergeScenarioExecutionSummary(Path scenario, Path output, MergeScenarioResult result, long time, String tool) { this.scenario = scenario From 7e4ef9f81c5f1c53f04de75cde49d4aca3cdad46 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sun, 16 Jun 2024 00:54:21 -0300 Subject: [PATCH 46/79] feat: tweak execution speed using parallelism --- build.gradle | 8 ++- .../GenericMergeDataCollector.groovy | 67 +++++++++++-------- .../executors/GenericMergeToolExecutor.groovy | 16 ++--- .../executors/JDimeMergeToolExecutor.groovy | 12 ++-- .../executors/MergeToolExecutor.groovy | 8 +-- 5 files changed, 63 insertions(+), 48 deletions(-) diff --git a/build.gradle b/build.gradle index dde2a8a73..0fcc5391a 100644 --- a/build.gradle +++ b/build.gradle @@ -28,9 +28,13 @@ test { } } +application { + applicationDefaultJvmArgs = ["-Dminingframework.generic_merge.base_experiment_path=/usr/src/app", "-Dminingframework.generic_merge.merge_tool_executors_to_use=generic_merge,jdime"] +} + run { - systemProperty "miningframework.generic_merge.base_experiment_path", System.getProperty("miningframework.generic_merge.base_experiment_path") - systemProperty "miningframework.generic_merge.merge_tool_executors_to_use", System.getProperty("miningframework.generic_merge.merge_tool_executors_to_use") + systemProperty "miningframework.generic_merge.base_experiment_path", System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") + systemProperty "miningframework.generic_merge.merge_tool_executors_to_use", System.getProperty("miningframework.generic_merge.merge_tool_executors_to_use", "generic_merge,jdime") } dependencies { diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index becefc8c3..d8c82330b 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -14,15 +14,16 @@ import util.ProcessRunner import java.nio.file.Files import java.nio.file.Path +import java.util.stream.Collectors class GenericMergeDataCollector implements DataCollector { private static Logger LOG = LogManager.getLogger(GenericMergeDataCollector.class) - private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") + private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path") private static final String GENERIC_MERGE_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/generic-merge" public static final GENERIC_MERGE_REPORT_PATH = "${BASE_EXPERIMENT_PATH}/output/reports" public static final GENERIC_MERGE_REPORT_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-execution.csv" - public static final MERGE_TOOL_EXECUTORS_TO_USE = System.getProperty("miningframework.generic_merge.merge_tool_executors_to_use", "generic_merge,jdime").split(",") + public static final MERGE_TOOL_EXECUTORS_TO_USE = System.getProperty("miningframework.generic_merge.merge_tool_executors_to_use").split(",") private final List mergeToolExecutors @@ -42,37 +43,39 @@ class GenericMergeDataCollector implements DataCollector { void collectData(Project project, MergeCommit mergeCommit) { def scenarios = filterScenariosForExecution(MergeScenarioCollector.collectMergeScenarios(project, mergeCommit)) - LOG.debug("Starting execution of merge tools on scenario") - def mergeToolsExecutionResults = scenarios.flatMap { scenario -> mergeToolExecutors.stream().map { executor -> executor.runToolForMergeScenario(scenario) } } - LOG.debug("Finished execution of merge tools on scenario") - - mergeToolsExecutionResults.forEach { - def reportFile = new File(GENERIC_MERGE_REPORT_FILE_NAME) - - def list = new ArrayList() - list.add(project.getName()) - list.add(mergeCommit.getSHA()) - list.add(it.tool) - list.add(it.scenario.toAbsolutePath().toString()) - list.add(it.output.toAbsolutePath().toString()) - list.add(it.result.toString()) - list.add(it.time.toString()) - list.add(areFilesSyntacticallyEquivalent(it.scenario.resolve("merge.java"), it.output).toString()) + LOG.trace("Starting execution of merge tools on scenario") + def mergeToolsExecutionResults = scenarios + .flatMap(scenario -> { + return mergeToolExecutors + .parallelStream() + .map(executor -> executor.runToolForMergeScenario(scenario)) + }) + LOG.trace("Finished execution of merge tools on scenario") + + LOG.trace("Starting write of results to report file") + def lines = mergeToolsExecutionResults.map(result -> getReportLine(project, mergeCommit, result)) + def reportFile = new File(GENERIC_MERGE_REPORT_FILE_NAME) + reportFile << lines.collect(Collectors.joining(System.lineSeparator())) << "\n" + LOG.trace("Finished write of results to report file") + } - reportFile << "${list.join(",").replaceAll('\\\\', '/')}\n" - } + private static getReportLine(Project project, MergeCommit mergeCommit, MergeScenarioExecutionSummary result) { + def list = new ArrayList() + list.add(project.getName()) + list.add(mergeCommit.getSHA()) + list.add(result.tool) + list.add(result.scenario.toAbsolutePath().toString()) + list.add(result.output.toAbsolutePath().toString()) + list.add(result.result.toString()) + list.add(result.time.toString()) + list.add(areFilesSyntacticallyEquivalent(result.scenario.resolve("merge.java"), result.output).toString()) + list.join(",").replaceAll('\\\\', '/') } private static filterScenariosForExecution(List scenarios) { return scenarios - .stream() - .filter { - def parentDiffersFromBase = eitherParentDiffersFromBase(it) - if (!parentDiffersFromBase) { - LOG.trace("Skipping scenario ${it.toString()} because one of the parents equals base") - } - return parentDiffersFromBase - } + .parallelStream() + .filter(GenericMergeDataCollector::eitherParentDiffersFromBase) } private static boolean areFilesSyntacticallyEquivalent(Path fileA, Path fileB) { @@ -107,6 +110,14 @@ class GenericMergeDataCollector implements DataCollector { def rightEqualsBase = FileUtils.contentEquals(new File("${scenario.toAbsolutePath()}/base.java"), new File("${scenario.toAbsolutePath()}/right.java")) + if (leftEqualsBase) { + LOG.trace("In scenario ${scenario.toString()} left equals base") + } + + if (rightEqualsBase) { + LOG.trace("In scenario ${scenario.toString()} right equals base") + } + return !leftEqualsBase && !rightEqualsBase } diff --git a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy index 717a33626..becdaf573 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy @@ -8,19 +8,19 @@ import util.ProcessRunner import java.nio.file.Path class GenericMergeToolExecutor extends MergeToolExecutor { - private static Logger LOG = LogManager.getLogger(GenericMergeToolExecutor.class); + private static Logger LOG = LogManager.getLogger(GenericMergeToolExecutor.class) - private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") + private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path") private static final String GENERIC_MERGE_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/generic-merge" @Override protected GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario, Path outputFile) { - def working_directory_path = scenario.toAbsolutePath().toString(); + def working_directory_path = scenario.toAbsolutePath().toString() - def processBuilder = ProcessRunner.buildProcess(working_directory_path); + def processBuilder = ProcessRunner.buildProcess(working_directory_path) processBuilder.command().addAll(getBuildParameters(outputFile)) - def output = ProcessRunner.startProcess(processBuilder); + def output = ProcessRunner.startProcess(processBuilder) output.waitFor() if (output.exitValue() > 1) { @@ -28,11 +28,11 @@ class GenericMergeToolExecutor extends MergeToolExecutor { LOG.debug("Generic Merge output: ${output.getInputStream().readLines()}") } - return output.exitValue() == 0 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS : output.exitValue() == 1 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS : GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR; + return output.exitValue() == 0 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS : output.exitValue() == 1 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS : GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR } @Override - protected String getToolName() { + String getToolName() { return "GENERIC_MERGE" } @@ -44,6 +44,6 @@ class GenericMergeToolExecutor extends MergeToolExecutor { list.add("--right-path=right.java") list.add("--merge-path=${outputFile.toAbsolutePath().toString()}".toString()) list.add("--language=java") - return list; + return list } } diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index 02b672f2b..562e99194 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -9,14 +9,14 @@ import util.ProcessRunner import java.nio.file.Path class JDimeMergeToolExecutor extends MergeToolExecutor { - private static Logger LOG = LogManager.getLogger(JDimeMergeToolExecutor.class); + private static Logger LOG = LogManager.getLogger(JDimeMergeToolExecutor.class) - private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") + private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path") private static final String JDIME_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/jdime/install/JDime/bin" @Override protected GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario, Path outputFile) { - def working_directory_path = scenario.toAbsolutePath().toString(); + def working_directory_path = scenario.toAbsolutePath().toString() def processBuilder = ProcessRunner.buildProcess(JDIME_BINARY_PATH, "./JDime", @@ -26,7 +26,7 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { "${working_directory_path}/base.java", "${working_directory_path}/right.java") - def output = ProcessRunner.startProcess(processBuilder); + def output = ProcessRunner.startProcess(processBuilder) output.waitFor() if (output.exitValue() >= 200) { @@ -34,7 +34,7 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { LOG.debug("jDime output: ${output.getInputStream().readLines()}") } - def mergeConflictsCount = MergeConflict.getConflictsNumber(outputFile); + def mergeConflictsCount = MergeConflict.getConflictsNumber(outputFile) if (mergeConflictsCount > 0) { return GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS } @@ -43,7 +43,7 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { } @Override - protected String getToolName() { + String getToolName() { return "JDIME" } } diff --git a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy index 76439b9d5..d9d80ec8b 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy @@ -1,12 +1,12 @@ package services.dataCollectors.GenericMerge.executors -import services.dataCollectors.GenericMerge.GenericMergeDataCollector; +import services.dataCollectors.GenericMerge.GenericMergeDataCollector -import java.nio.file.Path; +import java.nio.file.Path abstract class MergeToolExecutor { GenericMergeDataCollector.MergeScenarioExecutionSummary runToolForMergeScenario(Path scenario) { - def startTime = System.nanoTime(); + def startTime = System.nanoTime() def outputFilePath = scenario.resolve("merge.${getToolName().toLowerCase()}.java") def result = executeTool(scenario, outputFilePath) return new GenericMergeDataCollector.MergeScenarioExecutionSummary(scenario, outputFilePath, result, System.nanoTime() - startTime, this.getToolName()) @@ -14,5 +14,5 @@ abstract class MergeToolExecutor { protected abstract GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario, Path outputFile); - protected abstract String getToolName(); + abstract String getToolName(); } From 08c525ca8adf05481ad6ad5d4b12ab833d962d84 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sun, 16 Jun 2024 01:33:23 -0300 Subject: [PATCH 47/79] feat: override command --- .../GenericMerge/executors/JDimeMergeToolExecutor.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index 562e99194..a6788e834 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -21,7 +21,7 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { def processBuilder = ProcessRunner.buildProcess(JDIME_BINARY_PATH, "./JDime", "--mode=structured", - "--output=${outputFile.toAbsolutePath().toString()}".toString(), + "--output=${outputFile.toString()}".toString(), "${working_directory_path}/left.java", "${working_directory_path}/base.java", "${working_directory_path}/right.java") @@ -31,7 +31,7 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { if (output.exitValue() >= 200) { LOG.warn("Error while merging ${scenario.toAbsolutePath()}. jDime exited with exitCode ${output.exitValue()}") - LOG.debug("jDime output: ${output.getInputStream().readLines()}") + LOG.warn("jDime output: ${output.getInputStream().readLines()}") } def mergeConflictsCount = MergeConflict.getConflictsNumber(outputFile) From fbac7e39be4c3ed48eb0b3548420b35537eebf32 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sun, 16 Jun 2024 01:44:16 -0300 Subject: [PATCH 48/79] feat: bring back absolute path --- build.gradle | 6 +++--- .../GenericMerge/GenericMergeDataCollector.groovy | 4 ++-- .../GenericMerge/executors/GenericMergeToolExecutor.groovy | 2 +- .../GenericMerge/executors/JDimeMergeToolExecutor.groovy | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build.gradle b/build.gradle index 0fcc5391a..b15c8d3e0 100644 --- a/build.gradle +++ b/build.gradle @@ -28,9 +28,9 @@ test { } } -application { - applicationDefaultJvmArgs = ["-Dminingframework.generic_merge.base_experiment_path=/usr/src/app", "-Dminingframework.generic_merge.merge_tool_executors_to_use=generic_merge,jdime"] -} +//application { +// applicationDefaultJvmArgs = ["-Dminingframework.generic_merge.base_experiment_path=/usr/src/app", "-Dminingframework.generic_merge.merge_tool_executors_to_use=generic_merge,jdime"] +//} run { systemProperty "miningframework.generic_merge.base_experiment_path", System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index d8c82330b..3124b479a 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -19,11 +19,11 @@ import java.util.stream.Collectors class GenericMergeDataCollector implements DataCollector { private static Logger LOG = LogManager.getLogger(GenericMergeDataCollector.class) - private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path") + private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") private static final String GENERIC_MERGE_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/generic-merge" public static final GENERIC_MERGE_REPORT_PATH = "${BASE_EXPERIMENT_PATH}/output/reports" public static final GENERIC_MERGE_REPORT_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-execution.csv" - public static final MERGE_TOOL_EXECUTORS_TO_USE = System.getProperty("miningframework.generic_merge.merge_tool_executors_to_use").split(",") + public static final MERGE_TOOL_EXECUTORS_TO_USE = System.getProperty("miningframework.generic_merge.merge_tool_executors_to_use", "generic_merge,jdime").split(",") private final List mergeToolExecutors diff --git a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy index becdaf573..e4f357364 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy @@ -10,7 +10,7 @@ import java.nio.file.Path class GenericMergeToolExecutor extends MergeToolExecutor { private static Logger LOG = LogManager.getLogger(GenericMergeToolExecutor.class) - private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path") + private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") private static final String GENERIC_MERGE_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/generic-merge" @Override diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index a6788e834..a6d53c555 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -11,7 +11,7 @@ import java.nio.file.Path class JDimeMergeToolExecutor extends MergeToolExecutor { private static Logger LOG = LogManager.getLogger(JDimeMergeToolExecutor.class) - private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path") + private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") private static final String JDIME_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/jdime/install/JDime/bin" @Override @@ -21,7 +21,7 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { def processBuilder = ProcessRunner.buildProcess(JDIME_BINARY_PATH, "./JDime", "--mode=structured", - "--output=${outputFile.toString()}".toString(), + "--output=${outputFile.toAbsolutePath().toString()}".toString(), "${working_directory_path}/left.java", "${working_directory_path}/base.java", "${working_directory_path}/right.java") From 132f5a3d79861f76c9f628a5ec57d6e23409497b Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sun, 16 Jun 2024 01:59:02 -0300 Subject: [PATCH 49/79] fix: tweak debug and early return --- .../GenericMerge/executors/JDimeMergeToolExecutor.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index a6d53c555..9b50fc3cb 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -31,7 +31,8 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { if (output.exitValue() >= 200) { LOG.warn("Error while merging ${scenario.toAbsolutePath()}. jDime exited with exitCode ${output.exitValue()}") - LOG.warn("jDime output: ${output.getInputStream().readLines()}") + LOG.debug("jDime output: ${output.getInputStream().readLines()}") + return GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR } def mergeConflictsCount = MergeConflict.getConflictsNumber(outputFile) From 3f85aacd31e6341533395081557d5e803ba19160 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sun, 16 Jun 2024 17:49:41 -0300 Subject: [PATCH 50/79] feat: Alter invocation of generic merge to match new cli api --- .../GenericMerge/GenericMergeDataCollector.groovy | 3 ++- .../GenericMerge/executors/GenericMergeToolExecutor.groovy | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 3124b479a..2ade726c3 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -88,9 +88,10 @@ class GenericMergeDataCollector implements DataCollector { def list = new ArrayList() list.add(GENERIC_MERGE_BINARY_PATH) - list.add("--diff-only") + list.add("diff") list.add("--left-path=${fileA.toAbsolutePath().toString()}".toString()) list.add("--right-path=${fileB.toAbsolutePath().toString()}".toString()) + list.add("--language=java") process.command().addAll(list) def output = ProcessRunner.startProcess(process) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy index e4f357364..6f7f55b66 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy @@ -39,6 +39,7 @@ class GenericMergeToolExecutor extends MergeToolExecutor { private static List getBuildParameters(Path outputFile) { def list = new ArrayList() list.add(GENERIC_MERGE_BINARY_PATH) + list.add("merge") list.add("--base-path=base.java") list.add("--left-path=left.java") list.add("--right-path=right.java") From 0be4bbafbe73c4ecedcc9e396295692f7447c3d1 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 2 Jul 2024 20:29:03 -0300 Subject: [PATCH 51/79] feat: add fp/tp analysis code --- .../GenericMerge/BuildRequester.groovy | 10 +- .../GenericMergeDataCollector.groovy | 121 +++++++++++++----- 2 files changed, 96 insertions(+), 35 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index 231e72b25..561a1a202 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -1,9 +1,10 @@ package services.dataCollectors.GenericMerge +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger import project.MergeCommit import project.Project import services.util.Utils -import util.ProcessRunner import java.nio.file.Files import java.nio.file.Path @@ -11,8 +12,10 @@ import java.nio.file.Paths import java.nio.file.StandardCopyOption class BuildRequester { + private static Logger LOG = LogManager.getLogger(BuildRequester.class) + static requestBuildWithRevision(Project project, MergeCommit mergeCommit, List mergeScenarios, String mergeTool) { - String toReplaceFile = "merge.${mergeTool}.java" + String toReplaceFile = "merge.${mergeTool.toLowerCase()}.java" String branchName = "${mergeCommit.getSHA().take(7)}-${mergeTool}" @@ -31,7 +34,8 @@ class BuildRequester { private static void replaceFilesInProject(Project project, MergeCommit mergeCommit, List mergeScenarios, String toReplaceFile) { mergeScenarios.stream() .forEach(mergeScenario -> { - Files.copy(getSource(mergeScenario, toReplaceFile), getTarget(project, mergeCommit, mergeScenario)) + LOG.debug("Trying to copy " + getSource(mergeScenario, toReplaceFile) + " into " + getTarget(project, mergeCommit, mergeScenario)) + Files.copy(getSource(mergeScenario, toReplaceFile), getTarget(project, mergeCommit, mergeScenario), StandardCopyOption.REPLACE_EXISTING) }) } diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 2ade726c3..42924ecb8 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -41,22 +41,54 @@ class GenericMergeDataCollector implements DataCollector { @Override void collectData(Project project, MergeCommit mergeCommit) { - def scenarios = filterScenariosForExecution(MergeScenarioCollector.collectMergeScenarios(project, mergeCommit)) + def scenarios = filterScenariosForExecution(MergeScenarioCollector.collectMergeScenarios(project, mergeCommit)).collect(Collectors.toUnmodifiableList()) LOG.trace("Starting execution of merge tools on scenario") def mergeToolsExecutionResults = scenarios + .stream() .flatMap(scenario -> { return mergeToolExecutors .parallelStream() .map(executor -> executor.runToolForMergeScenario(scenario)) - }) + }).collect(Collectors.toUnmodifiableList()) LOG.trace("Finished execution of merge tools on scenario") - LOG.trace("Starting write of results to report file") - def lines = mergeToolsExecutionResults.map(result -> getReportLine(project, mergeCommit, result)) + // Aggregate scenario results by tool + def toolsCommitSummary = mergeToolsExecutionResults + .parallelStream() + .collect(Collectors.groupingBy(MergeScenarioExecutionSummary::getTool, + Collectors.collectingAndThen(Collectors.toList(), + MergeCommitExecutionSummary::fromFileResultsList) + )) + + + // Check which tools successfully integrated the scenario + def toolsInWhichIntegrationSucceeded = toolsCommitSummary + .entrySet() + .stream() + .filter { it -> it.value.result == MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS } + .map { x -> x.key } + .collect(Collectors.toUnmodifiableList()) + + // Are there any exclusive conflicts/tool errors? + if (toolsInWhichIntegrationSucceeded.size() != mergeToolExecutors.size()) { + LOG.info("At least one of the tools either reported a conflict or failed on the commit while the other did not") + toolsInWhichIntegrationSucceeded.forEach { tool -> { + def toolCommitSummary = toolsCommitSummary.get(tool) + if (toolCommitSummary.allScenariosMatch) { + LOG.info("Output of the tool " + tool + " fully matched the commit merge. Skipping build analysis") + } else { + LOG.info("Output of the tool " + tool + " did not fully matched the commit merge. Starting build analysis") + BuildRequester.requestBuildWithRevision(project, mergeCommit, scenarios, tool) + } + }} + } + + LOG.trace("Starting write of files results to report file") + def lines = mergeToolsExecutionResults.parallelStream().map(result -> getReportLine(project, mergeCommit, result)) def reportFile = new File(GENERIC_MERGE_REPORT_FILE_NAME) reportFile << lines.collect(Collectors.joining(System.lineSeparator())) << "\n" - LOG.trace("Finished write of results to report file") + LOG.trace("Finished write of files results to report file") } private static getReportLine(Project project, MergeCommit mergeCommit, MergeScenarioExecutionSummary result) { @@ -68,7 +100,7 @@ class GenericMergeDataCollector implements DataCollector { list.add(result.output.toAbsolutePath().toString()) list.add(result.result.toString()) list.add(result.time.toString()) - list.add(areFilesSyntacticallyEquivalent(result.scenario.resolve("merge.java"), result.output).toString()) + list.add(result.isEquivalentToOracle().toString()) list.join(",").replaceAll('\\\\', '/') } @@ -78,32 +110,6 @@ class GenericMergeDataCollector implements DataCollector { .filter(GenericMergeDataCollector::eitherParentDiffersFromBase) } - private static boolean areFilesSyntacticallyEquivalent(Path fileA, Path fileB) { - if (!Files.exists(fileA) || !Files.exists(fileB)) { - LOG.trace("Early returning because one of the files ${} do not exist") - return false - } - - def process = ProcessRunner.buildProcess("./") - - def list = new ArrayList() - list.add(GENERIC_MERGE_BINARY_PATH) - list.add("diff") - list.add("--left-path=${fileA.toAbsolutePath().toString()}".toString()) - list.add("--right-path=${fileB.toAbsolutePath().toString()}".toString()) - list.add("--language=java") - process.command().addAll(list) - - def output = ProcessRunner.startProcess(process) - output.waitFor() - - if (output.exitValue() > 1) { - LOG.warn("Error while running comparison between ${fileA.toString()} and ${fileB.toString()}: ${output.getInputStream().readLines()}") - } - - return output.exitValue() == 0 - } - private static boolean eitherParentDiffersFromBase(Path scenario) { def leftEqualsBase = FileUtils.contentEquals(new File("${scenario.toAbsolutePath()}/base.java"), new File("${scenario.toAbsolutePath()}/left.java")) @@ -134,6 +140,7 @@ class GenericMergeDataCollector implements DataCollector { public final Path output public final MergeScenarioResult result public final long time + public final boolean equivalentToOracle MergeScenarioExecutionSummary(Path scenario, Path output, MergeScenarioResult result, long time, String tool) { this.scenario = scenario @@ -141,6 +148,56 @@ class GenericMergeDataCollector implements DataCollector { this.result = result this.time = time this.tool = tool + this.equivalentToOracle = areFilesSyntacticallyEquivalent(scenario.resolve("merge.java"), output) + } + + String getTool() { + return tool + } + + boolean isEquivalentToOracle() { + return equivalentToOracle + } + + private static boolean areFilesSyntacticallyEquivalent(Path fileA, Path fileB) { + if (!Files.exists(fileA) || !Files.exists(fileB)) { + LOG.trace("Early returning because one of the files ${} do not exist") + return false + } + + def process = ProcessRunner.buildProcess("./") + + def list = new ArrayList() + list.add(GENERIC_MERGE_BINARY_PATH) + list.add("diff") + list.add("--left-path=${fileA.toAbsolutePath().toString()}".toString()) + list.add("--right-path=${fileB.toAbsolutePath().toString()}".toString()) + list.add("--language=java") + process.command().addAll(list) + + def output = ProcessRunner.startProcess(process) + output.waitFor() + + if (output.exitValue() > 1) { + LOG.warn("Error while running comparison between ${fileA.toString()} and ${fileB.toString()}: ${output.getInputStream().readLines()}") + } + + return output.exitValue() == 0 + } + } + + static class MergeCommitExecutionSummary { + public final MergeScenarioResult result + public final boolean allScenariosMatch + + private MergeCommitExecutionSummary(MergeScenarioResult result, boolean allScenariosMatch) { + this.result = result + this.allScenariosMatch = allScenariosMatch + } + + static MergeCommitExecutionSummary fromFileResultsList(List results) { + return new MergeCommitExecutionSummary(MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS, + results.stream().every { it -> it.isEquivalentToOracle() }) } } } From 91b382e7b107d12b2b26002173183a8a55e89e4f Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 2 Jul 2024 20:31:15 -0300 Subject: [PATCH 52/79] ci: uncomment buold gradle --- build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index b15c8d3e0..1c660e300 100644 --- a/build.gradle +++ b/build.gradle @@ -28,9 +28,9 @@ test { } } -//application { -// applicationDefaultJvmArgs = ["-Dminingframework.generic_merge.base_experiment_path=/usr/src/app", "-Dminingframework.generic_merge.merge_tool_executors_to_use=generic_merge,jdime"] -//} +application { + applicationDefaultJvmArgs = ["-Dminingframework.generic_merge.base_experiment_path=/usr/src/app", "-Dminingframework.generic_merge.merge_tool_executors_to_use=generic_merge,jdime"] +} run { systemProperty "miningframework.generic_merge.base_experiment_path", System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") From 5a0b5d3e7fcfaeb927adc413707251851e51ead0 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 2 Jul 2024 20:36:17 -0300 Subject: [PATCH 53/79] fix: do not use unmodifiablelist --- .../GenericMerge/GenericMergeDataCollector.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 42924ecb8..d26076b38 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -41,7 +41,7 @@ class GenericMergeDataCollector implements DataCollector { @Override void collectData(Project project, MergeCommit mergeCommit) { - def scenarios = filterScenariosForExecution(MergeScenarioCollector.collectMergeScenarios(project, mergeCommit)).collect(Collectors.toUnmodifiableList()) + def scenarios = filterScenariosForExecution(MergeScenarioCollector.collectMergeScenarios(project, mergeCommit)).collect(Collectors.toList()) LOG.trace("Starting execution of merge tools on scenario") def mergeToolsExecutionResults = scenarios @@ -50,7 +50,7 @@ class GenericMergeDataCollector implements DataCollector { return mergeToolExecutors .parallelStream() .map(executor -> executor.runToolForMergeScenario(scenario)) - }).collect(Collectors.toUnmodifiableList()) + }).collect(Collectors.toList()) LOG.trace("Finished execution of merge tools on scenario") // Aggregate scenario results by tool @@ -68,7 +68,7 @@ class GenericMergeDataCollector implements DataCollector { .stream() .filter { it -> it.value.result == MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS } .map { x -> x.key } - .collect(Collectors.toUnmodifiableList()) + .collect(Collectors.toList()) // Are there any exclusive conflicts/tool errors? if (toolsInWhichIntegrationSucceeded.size() != mergeToolExecutors.size()) { From 426f7af5ad058a623f5a54550310f644d53a3380 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 2 Jul 2024 21:18:11 -0300 Subject: [PATCH 54/79] fix: issues on result gathering --- .../GenericMergeDataCollector.groovy | 49 ++++++++++++++----- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index d26076b38..f902b8436 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -23,6 +23,7 @@ class GenericMergeDataCollector implements DataCollector { private static final String GENERIC_MERGE_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/generic-merge" public static final GENERIC_MERGE_REPORT_PATH = "${BASE_EXPERIMENT_PATH}/output/reports" public static final GENERIC_MERGE_REPORT_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-execution.csv" + public static final GENERIC_MERGE_REPORT_COMMITS_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-execution-commits.csv" public static final MERGE_TOOL_EXECUTORS_TO_USE = System.getProperty("miningframework.generic_merge.merge_tool_executors_to_use", "generic_merge,jdime").split(",") private final List mergeToolExecutors @@ -58,8 +59,7 @@ class GenericMergeDataCollector implements DataCollector { .parallelStream() .collect(Collectors.groupingBy(MergeScenarioExecutionSummary::getTool, Collectors.collectingAndThen(Collectors.toList(), - MergeCommitExecutionSummary::fromFileResultsList) - )) + MergeCommitExecutionSummary::fromFileResultsList))) // Check which tools successfully integrated the scenario @@ -73,15 +73,19 @@ class GenericMergeDataCollector implements DataCollector { // Are there any exclusive conflicts/tool errors? if (toolsInWhichIntegrationSucceeded.size() != mergeToolExecutors.size()) { LOG.info("At least one of the tools either reported a conflict or failed on the commit while the other did not") - toolsInWhichIntegrationSucceeded.forEach { tool -> { - def toolCommitSummary = toolsCommitSummary.get(tool) - if (toolCommitSummary.allScenariosMatch) { - LOG.info("Output of the tool " + tool + " fully matched the commit merge. Skipping build analysis") - } else { - LOG.info("Output of the tool " + tool + " did not fully matched the commit merge. Starting build analysis") - BuildRequester.requestBuildWithRevision(project, mergeCommit, scenarios, tool) + toolsInWhichIntegrationSucceeded.forEach { tool -> + { + def toolCommitSummary = toolsCommitSummary.get(tool) + if (toolCommitSummary.allScenariosMatch) { + LOG.info("Output of the tool " + tool + " fully matched the commit merge. Skipping build analysis") + } else { + LOG.info("Output of the tool " + tool + " did not fully matched the commit merge. Starting build analysis") + BuildRequester.requestBuildWithRevision(project, mergeCommit, scenarios, tool) + } } - }} + } + } else { + LOG.info("All the tools reported the same response") } LOG.trace("Starting write of files results to report file") @@ -89,6 +93,20 @@ class GenericMergeDataCollector implements DataCollector { def reportFile = new File(GENERIC_MERGE_REPORT_FILE_NAME) reportFile << lines.collect(Collectors.joining(System.lineSeparator())) << "\n" LOG.trace("Finished write of files results to report file") + + LOG.trace("Starting write of commit report") + def commitLines = toolsCommitSummary.entrySet().parallelStream().map { it -> + def list = new ArrayList() + list.add(project.getName()) + list.add(mergeCommit.getSHA()) + list.add(it.key) + list.add(it.value.result.toString()) + list.add(it.value.allScenariosMatch.toString()) + return list.join(",").replaceAll('\\\\', '/') + } + def commitReportFile = new File(GENERIC_MERGE_REPORT_COMMITS_FILE_NAME) + commitReportFile << commitLines.collect(Collectors.joining(System.lineSeparator())) << "\n" + LOG.trace("Finished write of commit report") } private static getReportLine(Project project, MergeCommit mergeCommit, MergeScenarioExecutionSummary result) { @@ -196,8 +214,15 @@ class GenericMergeDataCollector implements DataCollector { } static MergeCommitExecutionSummary fromFileResultsList(List results) { - return new MergeCommitExecutionSummary(MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS, - results.stream().every { it -> it.isEquivalentToOracle() }) + def result = results.stream() + .map { it -> it.result } + .reduce(MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS, MergeCommitExecutionSummary::validateResult) + def allMatchesOracle = results.stream().every { it -> it.isEquivalentToOracle() } + return new MergeCommitExecutionSummary(result, allMatchesOracle) + } + + private static validateResult(MergeScenarioResult prev, MergeScenarioResult cur) { + return cur != MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS ? cur : prev } } } From 590ad5db296a8550cc3b947b02c87e0dc4f45a09 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 2 Jul 2024 22:18:03 -0300 Subject: [PATCH 55/79] feat: add logging of git command --- src/main/services/util/Utils.groovy | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/services/util/Utils.groovy b/src/main/services/util/Utils.groovy index 6e4379abe..964b4f201 100644 --- a/src/main/services/util/Utils.groovy +++ b/src/main/services/util/Utils.groovy @@ -1,6 +1,8 @@ package services.util import app.MiningFramework +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger import project.MergeCommit import project.Project import util.ProcessRunner @@ -9,6 +11,7 @@ import java.nio.file.Path import java.nio.file.Paths final class Utils { + private static Logger LOG = LogManager.getLogger(Utils.class) /** * Runs a git command, waiting for it to finish. @@ -19,7 +22,10 @@ final class Utils { Process gitCommand = ProcessRunner.startProcess(buildGitCommand(repositoryPath, arguments)) gitCommand.getInputStream().eachLine { } - gitCommand.waitFor() + def exitCode = gitCommand.waitFor() + if (exitCode > 0) { + LOG.warn("An error occurred while running git command: ${output.getInputStream().readLines()}") + } } private static ProcessBuilder buildGitCommand(Path repositoryPath, String... arguments) { From f00a9038cbda01196c35f3db9b0d35c645366dd1 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 2 Jul 2024 22:34:54 -0300 Subject: [PATCH 56/79] fix: broken reference --- src/main/services/util/Utils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/services/util/Utils.groovy b/src/main/services/util/Utils.groovy index 964b4f201..af14a6cd9 100644 --- a/src/main/services/util/Utils.groovy +++ b/src/main/services/util/Utils.groovy @@ -24,7 +24,7 @@ final class Utils { } def exitCode = gitCommand.waitFor() if (exitCode > 0) { - LOG.warn("An error occurred while running git command: ${output.getInputStream().readLines()}") + LOG.warn("An error occurred while running git command: ${gitCommand.getInputStream().readLines()}") } } From 8c3713414f035a7df461a737b3dbed4211b4dcd0 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 2 Jul 2024 22:44:44 -0300 Subject: [PATCH 57/79] fix: tweak message --- src/main/services/util/Utils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/services/util/Utils.groovy b/src/main/services/util/Utils.groovy index af14a6cd9..0a8db7310 100644 --- a/src/main/services/util/Utils.groovy +++ b/src/main/services/util/Utils.groovy @@ -24,7 +24,7 @@ final class Utils { } def exitCode = gitCommand.waitFor() if (exitCode > 0) { - LOG.warn("An error occurred while running git command: ${gitCommand.getInputStream().readLines()}") + LOG.warn("Git command exited with error code ${exitCode}. Message: ${gitCommand.getErrorStream().readLines()}") } } From aa75f2b13f116f056a51673ded22282a9b076ed8 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 2 Jul 2024 22:53:06 -0300 Subject: [PATCH 58/79] fix: also print input stream --- src/main/services/util/Utils.groovy | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/services/util/Utils.groovy b/src/main/services/util/Utils.groovy index 0a8db7310..90bb59bb4 100644 --- a/src/main/services/util/Utils.groovy +++ b/src/main/services/util/Utils.groovy @@ -20,11 +20,9 @@ final class Utils { */ static void runGitCommand(Path repositoryPath, String... arguments) { Process gitCommand = ProcessRunner.startProcess(buildGitCommand(repositoryPath, arguments)) - gitCommand.getInputStream().eachLine { - } def exitCode = gitCommand.waitFor() if (exitCode > 0) { - LOG.warn("Git command exited with error code ${exitCode}. Message: ${gitCommand.getErrorStream().readLines()}") + LOG.warn("Git command exited with error code ${exitCode}.\n Error stream: ${gitCommand.getErrorStream().readLines()}\n Input stream: ${gitCommand.getInputStream().readLines()}") } } From d10db70c2659ee84d5752e66cea58cc5b8d33e7a Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 3 Jul 2024 19:41:56 -0300 Subject: [PATCH 59/79] feat: tweak logging --- .../GenericMerge/GenericMergeDataCollector.groovy | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index f902b8436..5b5dccd96 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -69,9 +69,10 @@ class GenericMergeDataCollector implements DataCollector { .filter { it -> it.value.result == MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS } .map { x -> x.key } .collect(Collectors.toList()) - - // Are there any exclusive conflicts/tool errors? - if (toolsInWhichIntegrationSucceeded.size() != mergeToolExecutors.size()) { + + if (toolsInWhichIntegrationSucceeded.size() == 0) { + LOG.info("Integration failed in all tools") + } else if (toolsInWhichIntegrationSucceeded.size() != mergeToolExecutors.size()) { LOG.info("At least one of the tools either reported a conflict or failed on the commit while the other did not") toolsInWhichIntegrationSucceeded.forEach { tool -> { From 5e31030f87998cc92fe37ad296d6c11f08a87b78 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 3 Jul 2024 23:56:00 -0300 Subject: [PATCH 60/79] feat: add sampling to increase reliability of performance --- .../GenericMergeDataCollector.groovy | 2 +- .../executors/MergeToolExecutor.groovy | 35 ++++++++++++++++--- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 5b5dccd96..c902c3899 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -69,7 +69,7 @@ class GenericMergeDataCollector implements DataCollector { .filter { it -> it.value.result == MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS } .map { x -> x.key } .collect(Collectors.toList()) - + if (toolsInWhichIntegrationSucceeded.size() == 0) { LOG.info("Integration failed in all tools") } else if (toolsInWhichIntegrationSucceeded.size() != mergeToolExecutors.size()) { diff --git a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy index d9d80ec8b..64d2e904e 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy @@ -1,15 +1,42 @@ package services.dataCollectors.GenericMerge.executors +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger import services.dataCollectors.GenericMerge.GenericMergeDataCollector import java.nio.file.Path abstract class MergeToolExecutor { + private static Logger LOG = LogManager.getLogger(MergeToolExecutor.class) + + public static final int NUMBER_OF_EXECUTIONS = 5 + GenericMergeDataCollector.MergeScenarioExecutionSummary runToolForMergeScenario(Path scenario) { - def startTime = System.nanoTime() - def outputFilePath = scenario.resolve("merge.${getToolName().toLowerCase()}.java") - def result = executeTool(scenario, outputFilePath) - return new GenericMergeDataCollector.MergeScenarioExecutionSummary(scenario, outputFilePath, result, System.nanoTime() - startTime, this.getToolName()) + LOG.trace("Starting execution of merge scenario with tool ${getToolName()}") + + List executionTimes = new ArrayList<>() + def outputFilePath = scenario.resolve("merge." + getToolName().toLowerCase() + ".java") + GenericMergeDataCollector.MergeScenarioResult result = null + + for (int i = 0; i < NUMBER_OF_EXECUTIONS; i++) { + LOG.trace("Starting execution ${i + 1} of ${NUMBER_OF_EXECUTIONS}") + long startTime = System.nanoTime() + result = executeTool(scenario, outputFilePath) + long endTime = System.nanoTime() + LOG.trace("Finished execution ${i + 1} of ${NUMBER_OF_EXECUTIONS} IN ${endTime - startTime} ns") + executionTimes.add(endTime - startTime) + } + + long averageTime = (long) (executionTimes.stream().reduce(0, (prev, cur) -> prev + cur) / executionTimes.size()) + + def summary = new GenericMergeDataCollector.MergeScenarioExecutionSummary(scenario, + outputFilePath, + result, + averageTime, + this.getToolName()) + + LOG.trace("Finished execution of merge scenario with tool ${summary.tool} in ${summary.time}ns with ${summary.result.toString()}") + return summary } protected abstract GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario, Path outputFile); From 882d66bfb6f41bc7a4ab298caffab513d53c35fc Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Thu, 4 Jul 2024 17:54:11 -0300 Subject: [PATCH 61/79] refactor: alter visibility of variable --- .../GenericMerge/executors/MergeToolExecutor.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy index 64d2e904e..9ed257528 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy @@ -9,7 +9,7 @@ import java.nio.file.Path abstract class MergeToolExecutor { private static Logger LOG = LogManager.getLogger(MergeToolExecutor.class) - public static final int NUMBER_OF_EXECUTIONS = 5 + private static final int NUMBER_OF_EXECUTIONS = 5 GenericMergeDataCollector.MergeScenarioExecutionSummary runToolForMergeScenario(Path scenario) { LOG.trace("Starting execution of merge scenario with tool ${getToolName()}") From bb757b6e770e85ceb22385bc743c04738abe6550 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 9 Jul 2024 14:29:17 -0300 Subject: [PATCH 62/79] feat: force jdime execution --- .../GenericMerge/executors/JDimeMergeToolExecutor.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index 9b50fc3cb..f7c424475 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -20,6 +20,7 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { def processBuilder = ProcessRunner.buildProcess(JDIME_BINARY_PATH, "./JDime", + "-f", "--mode=structured", "--output=${outputFile.toAbsolutePath().toString()}".toString(), "${working_directory_path}/left.java", From be6d7a914ba378d8d98d71d697318b9a50511957 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 16 Jul 2024 19:05:41 -0300 Subject: [PATCH 63/79] feat: attach build file to project --- .../GenericMerge/BuildRequester.groovy | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index 561a1a202..bcaece431 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -21,6 +21,7 @@ class BuildRequester { createBranchFromCommit(project, mergeCommit, branchName) replaceFilesInProject(project, mergeCommit, mergeScenarios, toReplaceFile) + createOrReplaceGithubActionsFile(project, mergeCommit) stageAndPushChanges(project, branchName, "Mining Framework Analysis") } @@ -39,6 +40,36 @@ class BuildRequester { }) } + private static void createOrReplaceGithubActionsFile(Project project, MergeCommit mergeCommit) { + def projectPath = Utils.commitFilesPath(project, mergeCommit).toAbsolutePath().toString() + def githubActionsFilePath = Paths.get("${projectPath}/.github/workflows/mining_framework.yml") + def githubActionsContent = """ +name: Mining Framework Check + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 1.8 + - run: ./gradlew assemble + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v1 + with: + java-version: 1.8 + - run: ./gradlew test +""" + + Files.write(githubActionsFilePath, githubActionsContent.getBytes()) + } + private static Path getSource(Path mergeScenario, String toReplaceFile) { return mergeScenario.resolve(toReplaceFile) } From 41554f0bf1d4442368e27a05c24a07e2a6d1ffe5 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 16 Jul 2024 20:49:46 -0300 Subject: [PATCH 64/79] fix: creation of github actiosn file --- .../GenericMerge/BuildRequester.groovy | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index bcaece431..198753719 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -21,7 +21,7 @@ class BuildRequester { createBranchFromCommit(project, mergeCommit, branchName) replaceFilesInProject(project, mergeCommit, mergeScenarios, toReplaceFile) - createOrReplaceGithubActionsFile(project, mergeCommit) + createOrReplaceGithubActionsFile(project) stageAndPushChanges(project, branchName, "Mining Framework Analysis") } @@ -40,14 +40,13 @@ class BuildRequester { }) } - private static void createOrReplaceGithubActionsFile(Project project, MergeCommit mergeCommit) { - def projectPath = Utils.commitFilesPath(project, mergeCommit).toAbsolutePath().toString() - def githubActionsFilePath = Paths.get("${projectPath}/.github/workflows/mining_framework.yml") + private static void createOrReplaceGithubActionsFile(Project project) { + LOG.debug("Starting creation of github actions file") + def githubActionsFilePath = "${Paths.get(project.getPath()).toAbsolutePath().toString()}/.github/workflows" + LOG.debug("Location of github actions folder ${githubActionsFilePath}") def githubActionsContent = """ name: Mining Framework Check - on: [push] - jobs: build: runs-on: ubuntu-latest @@ -66,8 +65,11 @@ jobs: java-version: 1.8 - run: ./gradlew test """ - - Files.write(githubActionsFilePath, githubActionsContent.getBytes()) + Files.createDirectories(Paths.get(githubActionsFilePath)) + def file = new File("${githubActionsFilePath}/mining_framework.yml") + file.createNewFile() + file.write(githubActionsContent) + LOG.debug("Finished creation of github actions file") } private static Path getSource(Path mergeScenario, String toReplaceFile) { From 2369a1c9b96541aa1a90a488322034ce29a58087 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 17 Jul 2024 16:38:55 -0300 Subject: [PATCH 65/79] feat: support multiple build tools gradle/mvn --- .../GenericMerge/BuildRequester.groovy | 62 ++++++++++++++++++- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index 198753719..a03b15695 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -44,6 +44,10 @@ class BuildRequester { LOG.debug("Starting creation of github actions file") def githubActionsFilePath = "${Paths.get(project.getPath()).toAbsolutePath().toString()}/.github/workflows" LOG.debug("Location of github actions folder ${githubActionsFilePath}") + + def buildSystem = getBuildSystemForProject(project) + LOG.debug("Using ${buildSystem.class.getSimpleName()} as build system for project ${project.getName()}") + def githubActionsContent = """ name: Mining Framework Check on: [push] @@ -55,7 +59,7 @@ jobs: - uses: actions/setup-java@v1 with: java-version: 1.8 - - run: ./gradlew assemble + - run: ${buildSystem.getBuildCommand()} test: runs-on: ubuntu-latest steps: @@ -63,7 +67,7 @@ jobs: - uses: actions/setup-java@v1 with: java-version: 1.8 - - run: ./gradlew test + - run: ${buildSystem.getTestCommand()} """ Files.createDirectories(Paths.get(githubActionsFilePath)) def file = new File("${githubActionsFilePath}/mining_framework.yml") @@ -94,4 +98,58 @@ jobs: // Push changes Utils.runGitCommand(projectPath, 'push', '--set-upstream', 'origin', branchName, '--force-with-lease') } + + private static interface BuildSystem { + String getBuildCommand() + String getTestCommand() + } + + private static class MavenBuildSystem implements BuildSystem { + @Override + String getBuildCommand() { + return "mvn package" + } + + @Override + String getTestCommand() { + return "mvn test" + } + } + + private static class GradleBuildSystem implements BuildSystem { + @Override + String getBuildCommand() { + return "./gradlew assemble" + } + + @Override + String getTestCommand() { + return "./gradlew test" + } + } + + private static class NoopBuildSystem implements BuildSystem { + @Override + String getBuildCommand() { + return "echo no build available" + } + + @Override + String getTestCommand() { + return "echo no test available" + } + } + + private static BuildSystem getBuildSystemForProject(Project project) { + File mavenFile = new File("${project.getPath()}/pom.xml") + File gradleFile = new File("${project.getPath()}/build.gradle") + + if (mavenFile.exists()) { + return new MavenBuildSystem() + } else if (gradleFile.exists()) { + return new GradleBuildSystem() + } else { + return new NoopBuildSystem() + } + } } From cf10777c160acbbaf68126bc8acb75ddc7b52134 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 17 Jul 2024 17:27:02 -0300 Subject: [PATCH 66/79] feat: Add execution reports log --- .../GenericMerge/BuildRequester.groovy | 15 +++++++++++++-- src/main/services/util/Utils.groovy | 8 ++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index a03b15695..2012ed88d 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -12,6 +12,9 @@ import java.nio.file.Paths import java.nio.file.StandardCopyOption class BuildRequester { + private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") + private static final BUILD_REQUESTER_REPORT_PATH = "${BASE_EXPERIMENT_PATH}/output/reports/generic-merge-execution-build-requests.csv" + private static Logger LOG = LogManager.getLogger(BuildRequester.class) static requestBuildWithRevision(Project project, MergeCommit mergeCommit, List mergeScenarios, String mergeTool) { @@ -22,7 +25,11 @@ class BuildRequester { createBranchFromCommit(project, mergeCommit, branchName) replaceFilesInProject(project, mergeCommit, mergeScenarios, toReplaceFile) createOrReplaceGithubActionsFile(project) - stageAndPushChanges(project, branchName, "Mining Framework Analysis") + def commitSha = stageAndPushChanges(project, branchName, "Mining Framework Analysis") + + def reportFile = new File(BUILD_REQUESTER_REPORT_PATH) + reportFile.createNewFile() + reportFile.append("${project.getName()},${branchName},${mergeTool},${commitSha}\n") } private static void createBranchFromCommit(Project project, MergeCommit mergeCommit, String branchName) { @@ -86,7 +93,7 @@ jobs: return projectPath.resolve(filePath) } - private static void stageAndPushChanges(Project project, String branchName, String commitMessage) { + private static String stageAndPushChanges(Project project, String branchName, String commitMessage) { Path projectPath = Paths.get(project.getPath()) // Stage changes @@ -94,9 +101,13 @@ jobs: // Commit changes Utils.runGitCommand(projectPath, 'commit', '-m', commitMessage) + def commitSha = Utils.runGitCommand(projectPath, 'rev-parse', 'HEAD'); + LOG.debug("Created commit with hash ${commitSha}") // Push changes Utils.runGitCommand(projectPath, 'push', '--set-upstream', 'origin', branchName, '--force-with-lease') + + return commitSha.get(0); } private static interface BuildSystem { diff --git a/src/main/services/util/Utils.groovy b/src/main/services/util/Utils.groovy index 90bb59bb4..574019d65 100644 --- a/src/main/services/util/Utils.groovy +++ b/src/main/services/util/Utils.groovy @@ -18,12 +18,16 @@ final class Utils { * @param repositoryPath * @param arguments */ - static void runGitCommand(Path repositoryPath, String... arguments) { + static List runGitCommand(Path repositoryPath, String... arguments) { Process gitCommand = ProcessRunner.startProcess(buildGitCommand(repositoryPath, arguments)) def exitCode = gitCommand.waitFor() + def commandOutput = gitCommand.getInputStream().readLines(); + if (exitCode > 0) { - LOG.warn("Git command exited with error code ${exitCode}.\n Error stream: ${gitCommand.getErrorStream().readLines()}\n Input stream: ${gitCommand.getInputStream().readLines()}") + LOG.warn("Git command exited with error code ${exitCode}.\n Error stream: ${gitCommand.getErrorStream().readLines()}\n Input stream: ${commandOutput}") } + + return commandOutput } private static ProcessBuilder buildGitCommand(Path repositoryPath, String... arguments) { From 81d613fbc9fcda2cbb1721c34eb968aa40d8cbbb Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Wed, 17 Jul 2024 21:09:04 -0300 Subject: [PATCH 67/79] fix: skip tests on build --- .../dataCollectors/GenericMerge/BuildRequester.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index 2012ed88d..22a31e35b 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -65,7 +65,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - run: ${buildSystem.getBuildCommand()} test: runs-on: ubuntu-latest @@ -73,7 +73,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - run: ${buildSystem.getTestCommand()} """ Files.createDirectories(Paths.get(githubActionsFilePath)) @@ -118,7 +118,7 @@ jobs: private static class MavenBuildSystem implements BuildSystem { @Override String getBuildCommand() { - return "mvn package" + return "mvn package -Dskiptests" } @Override From 28fde74ccf2b46604dada0a16c07576e5863a45d Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Sun, 21 Jul 2024 14:43:35 -0300 Subject: [PATCH 68/79] feat: use first run as a warmup --- .../GenericMerge/executors/MergeToolExecutor.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy index 9ed257528..1f4c92ffb 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy @@ -24,7 +24,10 @@ abstract class MergeToolExecutor { result = executeTool(scenario, outputFilePath) long endTime = System.nanoTime() LOG.trace("Finished execution ${i + 1} of ${NUMBER_OF_EXECUTIONS} IN ${endTime - startTime} ns") - executionTimes.add(endTime - startTime) + // If we're running more than one execution, we use the first one as a warm up + if (NUMBER_OF_EXECUTIONS == 1 || i > 0) { + executionTimes.add(endTime - startTime) + } } long averageTime = (long) (executionTimes.stream().reduce(0, (prev, cur) -> prev + cur) / executionTimes.size()) From 07b9d51dc0a046de0d6dc6dfac9beacfd93d47fc Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 22 Jul 2024 21:28:05 -0300 Subject: [PATCH 69/79] feat: Add format normalization on generic merge and initial merge file --- .../GenericMerge/FileFormatNormalizer.groovy | 30 +++++++++++++++++++ .../GenericMergeDataCollector.groovy | 6 ++++ .../executors/GenericMergeToolExecutor.groovy | 5 ++++ 3 files changed, 41 insertions(+) create mode 100644 src/main/services/dataCollectors/GenericMerge/FileFormatNormalizer.groovy diff --git a/src/main/services/dataCollectors/GenericMerge/FileFormatNormalizer.groovy b/src/main/services/dataCollectors/GenericMerge/FileFormatNormalizer.groovy new file mode 100644 index 000000000..59679c241 --- /dev/null +++ b/src/main/services/dataCollectors/GenericMerge/FileFormatNormalizer.groovy @@ -0,0 +1,30 @@ +package services.dataCollectors.GenericMerge + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import util.ProcessRunner + +import java.nio.file.Path + +class FileFormatNormalizer { + private static Logger LOG = LogManager.getLogger(FileFormatNormalizer.class) + + private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") + private static final String JDIME_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/jdime/install/JDime/bin" + + static void normalizeFileInPlace(Path file) { + def processBuilder = ProcessRunner.buildProcess(JDIME_BINARY_PATH, + "./JDime", + "-f", + "--mode=structured", + "--output=${file.toAbsolutePath().toString()}".toString(), + file.toAbsolutePath().toString(), + file.toAbsolutePath().toString(), + file.toAbsolutePath().toString()) + + def exitCode = ProcessRunner.startProcess(processBuilder).waitFor() + if (exitCode != 0) { + LOG.warn("File normalization failed with exit code ${exitCode}") + } + } +} diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index c902c3899..078eaa2ae 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -44,6 +44,12 @@ class GenericMergeDataCollector implements DataCollector { void collectData(Project project, MergeCommit mergeCommit) { def scenarios = filterScenariosForExecution(MergeScenarioCollector.collectMergeScenarios(project, mergeCommit)).collect(Collectors.toList()) + LOG.trace("Starting normalization of merge files on scenario") + scenarios.parallelStream().forEach { + scenario -> FileFormatNormalizer.normalizeFileInPlace(scenario.resolve("merge.java")) + } + LOG.trace("Finished normalization of merge files on scenario") + LOG.trace("Starting execution of merge tools on scenario") def mergeToolsExecutionResults = scenarios .stream() diff --git a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy index 6f7f55b66..d25466120 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy @@ -2,6 +2,7 @@ package services.dataCollectors.GenericMerge.executors import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger +import services.dataCollectors.GenericMerge.FileFormatNormalizer import services.dataCollectors.GenericMerge.GenericMergeDataCollector import util.ProcessRunner @@ -28,6 +29,10 @@ class GenericMergeToolExecutor extends MergeToolExecutor { LOG.debug("Generic Merge output: ${output.getInputStream().readLines()}") } + if (output.exitValue() == 0) { + FileFormatNormalizer.normalizeFileInPlace(outputFile) + } + return output.exitValue() == 0 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS : output.exitValue() == 1 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS : GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR } From ecb8e639fc926c9c772a9dddd63f9a1857f8aa94 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 22 Jul 2024 21:40:07 -0300 Subject: [PATCH 70/79] refactor: Move code around --- .../GenericMerge/FileSyntacticDiff.groovy | 41 +++++++++ .../GenericMergeDataCollector.groovy | 89 +------------------ .../executors/GenericMergeToolExecutor.groovy | 6 +- .../executors/JDimeMergeToolExecutor.groovy | 10 +-- .../executors/MergeToolExecutor.groovy | 11 +-- .../model/MergeCommitExecutionSummary.groovy | 23 +++++ .../MergeScenarioExecutionSummary.groovy | 32 +++++++ .../model/MergeScenarioResult.groovy | 7 ++ 8 files changed, 121 insertions(+), 98 deletions(-) create mode 100644 src/main/services/dataCollectors/GenericMerge/FileSyntacticDiff.groovy create mode 100644 src/main/services/dataCollectors/GenericMerge/model/MergeCommitExecutionSummary.groovy create mode 100644 src/main/services/dataCollectors/GenericMerge/model/MergeScenarioExecutionSummary.groovy create mode 100644 src/main/services/dataCollectors/GenericMerge/model/MergeScenarioResult.groovy diff --git a/src/main/services/dataCollectors/GenericMerge/FileSyntacticDiff.groovy b/src/main/services/dataCollectors/GenericMerge/FileSyntacticDiff.groovy new file mode 100644 index 000000000..92740d927 --- /dev/null +++ b/src/main/services/dataCollectors/GenericMerge/FileSyntacticDiff.groovy @@ -0,0 +1,41 @@ +package services.dataCollectors.GenericMerge + +import org.apache.logging.log4j.LogManager +import org.apache.logging.log4j.Logger +import util.ProcessRunner + +import java.nio.file.Files +import java.nio.file.Path + +class FileSyntacticDiff { + private static Logger LOG = LogManager.getLogger(FileSyntacticDiff.class) + + private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") + private static final String GENERIC_MERGE_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/generic-merge" + + static boolean areFilesSyntacticallyEquivalent(Path fileA, Path fileB) { + if (!Files.exists(fileA) || !Files.exists(fileB)) { + LOG.trace("Early returning because one of the files ${} do not exist") + return false + } + + def process = ProcessRunner.buildProcess("./") + + def list = new ArrayList() + list.add(GENERIC_MERGE_BINARY_PATH) + list.add("diff") + list.add("--left-path=${fileA.toAbsolutePath().toString()}".toString()) + list.add("--right-path=${fileB.toAbsolutePath().toString()}".toString()) + list.add("--language=java") + process.command().addAll(list) + + def output = ProcessRunner.startProcess(process) + output.waitFor() + + if (output.exitValue() > 1) { + LOG.warn("Error while running comparison between ${fileA.toString()} and ${fileB.toString()}: ${output.getInputStream().readLines()}") + } + + return output.exitValue() == 0 + } +} diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 078eaa2ae..e8f9a4556 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -9,10 +9,11 @@ import project.Project import services.dataCollectors.GenericMerge.executors.GenericMergeToolExecutor import services.dataCollectors.GenericMerge.executors.JDimeMergeToolExecutor import services.dataCollectors.GenericMerge.executors.MergeToolExecutor +import services.dataCollectors.GenericMerge.model.MergeCommitExecutionSummary +import services.dataCollectors.GenericMerge.model.MergeScenarioExecutionSummary +import services.dataCollectors.GenericMerge.model.MergeScenarioResult import services.dataCollectors.S3MMergesCollector.MergeScenarioCollector -import util.ProcessRunner -import java.nio.file.Files import java.nio.file.Path import java.util.stream.Collectors @@ -20,7 +21,6 @@ class GenericMergeDataCollector implements DataCollector { private static Logger LOG = LogManager.getLogger(GenericMergeDataCollector.class) private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") - private static final String GENERIC_MERGE_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/generic-merge" public static final GENERIC_MERGE_REPORT_PATH = "${BASE_EXPERIMENT_PATH}/output/reports" public static final GENERIC_MERGE_REPORT_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-execution.csv" public static final GENERIC_MERGE_REPORT_COMMITS_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-execution-commits.csv" @@ -45,8 +45,7 @@ class GenericMergeDataCollector implements DataCollector { def scenarios = filterScenariosForExecution(MergeScenarioCollector.collectMergeScenarios(project, mergeCommit)).collect(Collectors.toList()) LOG.trace("Starting normalization of merge files on scenario") - scenarios.parallelStream().forEach { - scenario -> FileFormatNormalizer.normalizeFileInPlace(scenario.resolve("merge.java")) + scenarios.parallelStream().forEach { scenario -> FileFormatNormalizer.normalizeFileInPlace(scenario.resolve("merge.java")) } LOG.trace("Finished normalization of merge files on scenario") @@ -152,84 +151,4 @@ class GenericMergeDataCollector implements DataCollector { return !leftEqualsBase && !rightEqualsBase } - - static enum MergeScenarioResult { - SUCCESS_WITHOUT_CONFLICTS, - SUCCESS_WITH_CONFLICTS, - TOOL_ERROR - } - - static class MergeScenarioExecutionSummary { - public final String tool - public final Path scenario - public final Path output - public final MergeScenarioResult result - public final long time - public final boolean equivalentToOracle - - MergeScenarioExecutionSummary(Path scenario, Path output, MergeScenarioResult result, long time, String tool) { - this.scenario = scenario - this.output = output - this.result = result - this.time = time - this.tool = tool - this.equivalentToOracle = areFilesSyntacticallyEquivalent(scenario.resolve("merge.java"), output) - } - - String getTool() { - return tool - } - - boolean isEquivalentToOracle() { - return equivalentToOracle - } - - private static boolean areFilesSyntacticallyEquivalent(Path fileA, Path fileB) { - if (!Files.exists(fileA) || !Files.exists(fileB)) { - LOG.trace("Early returning because one of the files ${} do not exist") - return false - } - - def process = ProcessRunner.buildProcess("./") - - def list = new ArrayList() - list.add(GENERIC_MERGE_BINARY_PATH) - list.add("diff") - list.add("--left-path=${fileA.toAbsolutePath().toString()}".toString()) - list.add("--right-path=${fileB.toAbsolutePath().toString()}".toString()) - list.add("--language=java") - process.command().addAll(list) - - def output = ProcessRunner.startProcess(process) - output.waitFor() - - if (output.exitValue() > 1) { - LOG.warn("Error while running comparison between ${fileA.toString()} and ${fileB.toString()}: ${output.getInputStream().readLines()}") - } - - return output.exitValue() == 0 - } - } - - static class MergeCommitExecutionSummary { - public final MergeScenarioResult result - public final boolean allScenariosMatch - - private MergeCommitExecutionSummary(MergeScenarioResult result, boolean allScenariosMatch) { - this.result = result - this.allScenariosMatch = allScenariosMatch - } - - static MergeCommitExecutionSummary fromFileResultsList(List results) { - def result = results.stream() - .map { it -> it.result } - .reduce(MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS, MergeCommitExecutionSummary::validateResult) - def allMatchesOracle = results.stream().every { it -> it.isEquivalentToOracle() } - return new MergeCommitExecutionSummary(result, allMatchesOracle) - } - - private static validateResult(MergeScenarioResult prev, MergeScenarioResult cur) { - return cur != MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS ? cur : prev - } - } } diff --git a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy index d25466120..9c6bcce95 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy @@ -3,7 +3,7 @@ package services.dataCollectors.GenericMerge.executors import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger import services.dataCollectors.GenericMerge.FileFormatNormalizer -import services.dataCollectors.GenericMerge.GenericMergeDataCollector +import services.dataCollectors.GenericMerge.model.MergeScenarioResult import util.ProcessRunner import java.nio.file.Path @@ -15,7 +15,7 @@ class GenericMergeToolExecutor extends MergeToolExecutor { private static final String GENERIC_MERGE_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/generic-merge" @Override - protected GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario, Path outputFile) { + protected MergeScenarioResult executeTool(Path scenario, Path outputFile) { def working_directory_path = scenario.toAbsolutePath().toString() def processBuilder = ProcessRunner.buildProcess(working_directory_path) @@ -33,7 +33,7 @@ class GenericMergeToolExecutor extends MergeToolExecutor { FileFormatNormalizer.normalizeFileInPlace(outputFile) } - return output.exitValue() == 0 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS : output.exitValue() == 1 ? GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS : GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR + return output.exitValue() == 0 ? MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS : output.exitValue() == 1 ? MergeScenarioResult.SUCCESS_WITH_CONFLICTS : MergeScenarioResult.TOOL_ERROR } @Override diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index f7c424475..42a37c24b 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -2,7 +2,7 @@ package services.dataCollectors.GenericMerge.executors import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger -import services.dataCollectors.GenericMerge.GenericMergeDataCollector +import services.dataCollectors.GenericMerge.model.MergeScenarioResult import services.util.MergeConflict import util.ProcessRunner @@ -15,7 +15,7 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { private static final String JDIME_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/jdime/install/JDime/bin" @Override - protected GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario, Path outputFile) { + protected MergeScenarioResult executeTool(Path scenario, Path outputFile) { def working_directory_path = scenario.toAbsolutePath().toString() def processBuilder = ProcessRunner.buildProcess(JDIME_BINARY_PATH, @@ -33,15 +33,15 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { if (output.exitValue() >= 200) { LOG.warn("Error while merging ${scenario.toAbsolutePath()}. jDime exited with exitCode ${output.exitValue()}") LOG.debug("jDime output: ${output.getInputStream().readLines()}") - return GenericMergeDataCollector.MergeScenarioResult.TOOL_ERROR + return MergeScenarioResult.TOOL_ERROR } def mergeConflictsCount = MergeConflict.getConflictsNumber(outputFile) if (mergeConflictsCount > 0) { - return GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITH_CONFLICTS + return MergeScenarioResult.SUCCESS_WITH_CONFLICTS } - return GenericMergeDataCollector.MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS + return MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS } @Override diff --git a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy index 1f4c92ffb..ce04d57f6 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy @@ -2,7 +2,8 @@ package services.dataCollectors.GenericMerge.executors import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger -import services.dataCollectors.GenericMerge.GenericMergeDataCollector +import services.dataCollectors.GenericMerge.model.MergeScenarioExecutionSummary +import services.dataCollectors.GenericMerge.model.MergeScenarioResult import java.nio.file.Path @@ -11,12 +12,12 @@ abstract class MergeToolExecutor { private static final int NUMBER_OF_EXECUTIONS = 5 - GenericMergeDataCollector.MergeScenarioExecutionSummary runToolForMergeScenario(Path scenario) { + MergeScenarioExecutionSummary runToolForMergeScenario(Path scenario) { LOG.trace("Starting execution of merge scenario with tool ${getToolName()}") List executionTimes = new ArrayList<>() def outputFilePath = scenario.resolve("merge." + getToolName().toLowerCase() + ".java") - GenericMergeDataCollector.MergeScenarioResult result = null + MergeScenarioResult result = null for (int i = 0; i < NUMBER_OF_EXECUTIONS; i++) { LOG.trace("Starting execution ${i + 1} of ${NUMBER_OF_EXECUTIONS}") @@ -32,7 +33,7 @@ abstract class MergeToolExecutor { long averageTime = (long) (executionTimes.stream().reduce(0, (prev, cur) -> prev + cur) / executionTimes.size()) - def summary = new GenericMergeDataCollector.MergeScenarioExecutionSummary(scenario, + def summary = new MergeScenarioExecutionSummary(scenario, outputFilePath, result, averageTime, @@ -42,7 +43,7 @@ abstract class MergeToolExecutor { return summary } - protected abstract GenericMergeDataCollector.MergeScenarioResult executeTool(Path scenario, Path outputFile); + protected abstract MergeScenarioResult executeTool(Path scenario, Path outputFile); abstract String getToolName(); } diff --git a/src/main/services/dataCollectors/GenericMerge/model/MergeCommitExecutionSummary.groovy b/src/main/services/dataCollectors/GenericMerge/model/MergeCommitExecutionSummary.groovy new file mode 100644 index 000000000..bc1f11d05 --- /dev/null +++ b/src/main/services/dataCollectors/GenericMerge/model/MergeCommitExecutionSummary.groovy @@ -0,0 +1,23 @@ +package services.dataCollectors.GenericMerge.model + +class MergeCommitExecutionSummary { + public final MergeScenarioResult result + public final boolean allScenariosMatch + + private MergeCommitExecutionSummary(MergeScenarioResult result, boolean allScenariosMatch) { + this.result = result + this.allScenariosMatch = allScenariosMatch + } + + static MergeCommitExecutionSummary fromFileResultsList(List results) { + def result = results.stream() + .map { it -> it.result } + .reduce(MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS, MergeCommitExecutionSummary::validateResult) + def allMatchesOracle = results.stream().every { it -> it.isEquivalentToOracle() } + return new MergeCommitExecutionSummary(result, allMatchesOracle) + } + + private static validateResult(MergeScenarioResult prev, MergeScenarioResult cur) { + return cur != MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS ? cur : prev + } +} diff --git a/src/main/services/dataCollectors/GenericMerge/model/MergeScenarioExecutionSummary.groovy b/src/main/services/dataCollectors/GenericMerge/model/MergeScenarioExecutionSummary.groovy new file mode 100644 index 000000000..f18deb00c --- /dev/null +++ b/src/main/services/dataCollectors/GenericMerge/model/MergeScenarioExecutionSummary.groovy @@ -0,0 +1,32 @@ +package services.dataCollectors.GenericMerge.model + + +import services.dataCollectors.GenericMerge.FileSyntacticDiff + +import java.nio.file.Path + +class MergeScenarioExecutionSummary { + public final String tool + public final Path scenario + public final Path output + public final MergeScenarioResult result + public final long time + public final boolean equivalentToOracle + + MergeScenarioExecutionSummary(Path scenario, Path output, MergeScenarioResult result, long time, String tool) { + this.scenario = scenario + this.output = output + this.result = result + this.time = time + this.tool = tool + this.equivalentToOracle = FileSyntacticDiff.areFilesSyntacticallyEquivalent(scenario.resolve("merge.java"), output) + } + + String getTool() { + return tool + } + + boolean isEquivalentToOracle() { + return equivalentToOracle + } +} diff --git a/src/main/services/dataCollectors/GenericMerge/model/MergeScenarioResult.groovy b/src/main/services/dataCollectors/GenericMerge/model/MergeScenarioResult.groovy new file mode 100644 index 000000000..3090aba50 --- /dev/null +++ b/src/main/services/dataCollectors/GenericMerge/model/MergeScenarioResult.groovy @@ -0,0 +1,7 @@ +package services.dataCollectors.GenericMerge.model + +enum MergeScenarioResult { + SUCCESS_WITHOUT_CONFLICTS, + SUCCESS_WITH_CONFLICTS, + TOOL_ERROR +} From c105955678c1818b2dbf68f0d55194ee2b81a397 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Mon, 22 Jul 2024 21:44:15 -0300 Subject: [PATCH 71/79] refactor: Create config class --- .../GenericMerge/BuildRequester.groovy | 10 ++++------ .../GenericMerge/FileFormatNormalizer.groovy | 5 +---- .../GenericMerge/FileSyntacticDiff.groovy | 5 +---- .../GenericMerge/GenericMergeConfig.groovy | 18 ++++++++++++++++++ .../GenericMergeDataCollector.groovy | 14 ++++---------- .../executors/GenericMergeToolExecutor.groovy | 6 ++---- .../executors/JDimeMergeToolExecutor.groovy | 6 ++---- .../executors/MergeToolExecutor.groovy | 11 +++++------ 8 files changed, 37 insertions(+), 38 deletions(-) create mode 100644 src/main/services/dataCollectors/GenericMerge/GenericMergeConfig.groovy diff --git a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy index 22a31e35b..2d4567cbd 100644 --- a/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy +++ b/src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy @@ -12,9 +12,6 @@ import java.nio.file.Paths import java.nio.file.StandardCopyOption class BuildRequester { - private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") - private static final BUILD_REQUESTER_REPORT_PATH = "${BASE_EXPERIMENT_PATH}/output/reports/generic-merge-execution-build-requests.csv" - private static Logger LOG = LogManager.getLogger(BuildRequester.class) static requestBuildWithRevision(Project project, MergeCommit mergeCommit, List mergeScenarios, String mergeTool) { @@ -27,7 +24,7 @@ class BuildRequester { createOrReplaceGithubActionsFile(project) def commitSha = stageAndPushChanges(project, branchName, "Mining Framework Analysis") - def reportFile = new File(BUILD_REQUESTER_REPORT_PATH) + def reportFile = new File(GenericMergeConfig.BUILD_REQUESTER_REPORT_PATH) reportFile.createNewFile() reportFile.append("${project.getName()},${branchName},${mergeTool},${commitSha}\n") } @@ -101,17 +98,18 @@ jobs: // Commit changes Utils.runGitCommand(projectPath, 'commit', '-m', commitMessage) - def commitSha = Utils.runGitCommand(projectPath, 'rev-parse', 'HEAD'); + def commitSha = Utils.runGitCommand(projectPath, 'rev-parse', 'HEAD') LOG.debug("Created commit with hash ${commitSha}") // Push changes Utils.runGitCommand(projectPath, 'push', '--set-upstream', 'origin', branchName, '--force-with-lease') - return commitSha.get(0); + return commitSha.get(0) } private static interface BuildSystem { String getBuildCommand() + String getTestCommand() } diff --git a/src/main/services/dataCollectors/GenericMerge/FileFormatNormalizer.groovy b/src/main/services/dataCollectors/GenericMerge/FileFormatNormalizer.groovy index 59679c241..c2a3d2554 100644 --- a/src/main/services/dataCollectors/GenericMerge/FileFormatNormalizer.groovy +++ b/src/main/services/dataCollectors/GenericMerge/FileFormatNormalizer.groovy @@ -9,11 +9,8 @@ import java.nio.file.Path class FileFormatNormalizer { private static Logger LOG = LogManager.getLogger(FileFormatNormalizer.class) - private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") - private static final String JDIME_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/jdime/install/JDime/bin" - static void normalizeFileInPlace(Path file) { - def processBuilder = ProcessRunner.buildProcess(JDIME_BINARY_PATH, + def processBuilder = ProcessRunner.buildProcess(GenericMergeConfig.JDIME_BINARY_PATH, "./JDime", "-f", "--mode=structured", diff --git a/src/main/services/dataCollectors/GenericMerge/FileSyntacticDiff.groovy b/src/main/services/dataCollectors/GenericMerge/FileSyntacticDiff.groovy index 92740d927..f6471494b 100644 --- a/src/main/services/dataCollectors/GenericMerge/FileSyntacticDiff.groovy +++ b/src/main/services/dataCollectors/GenericMerge/FileSyntacticDiff.groovy @@ -10,9 +10,6 @@ import java.nio.file.Path class FileSyntacticDiff { private static Logger LOG = LogManager.getLogger(FileSyntacticDiff.class) - private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") - private static final String GENERIC_MERGE_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/generic-merge" - static boolean areFilesSyntacticallyEquivalent(Path fileA, Path fileB) { if (!Files.exists(fileA) || !Files.exists(fileB)) { LOG.trace("Early returning because one of the files ${} do not exist") @@ -22,7 +19,7 @@ class FileSyntacticDiff { def process = ProcessRunner.buildProcess("./") def list = new ArrayList() - list.add(GENERIC_MERGE_BINARY_PATH) + list.add(GenericMergeConfig.GENERIC_MERGE_BINARY_PATH) list.add("diff") list.add("--left-path=${fileA.toAbsolutePath().toString()}".toString()) list.add("--right-path=${fileB.toAbsolutePath().toString()}".toString()) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeConfig.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeConfig.groovy new file mode 100644 index 000000000..03f1647c1 --- /dev/null +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeConfig.groovy @@ -0,0 +1,18 @@ +package services.dataCollectors.GenericMerge + +class GenericMergeConfig { + public static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") + + public static final MERGE_TOOL_EXECUTORS_TO_USE = System.getProperty("miningframework.generic_merge.merge_tool_executors_to_use", "generic_merge,jdime").split(",") + + public static final BUILD_REQUESTER_REPORT_PATH = "${BASE_EXPERIMENT_PATH}/output/reports/generic-merge-execution-build-requests.csv" + + public static final GENERIC_MERGE_REPORT_PATH = "${BASE_EXPERIMENT_PATH}/output/reports" + public static final GENERIC_MERGE_REPORT_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-execution.csv" + public static final GENERIC_MERGE_REPORT_COMMITS_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-execution-commits.csv" + + public static final String GENERIC_MERGE_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/generic-merge" + public static final String JDIME_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/jdime/install/JDime/bin" + + public static final int NUMBER_OF_EXECUTIONS = 5 +} diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index e8f9a4556..1182f588e 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -20,21 +20,15 @@ import java.util.stream.Collectors class GenericMergeDataCollector implements DataCollector { private static Logger LOG = LogManager.getLogger(GenericMergeDataCollector.class) - private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") - public static final GENERIC_MERGE_REPORT_PATH = "${BASE_EXPERIMENT_PATH}/output/reports" - public static final GENERIC_MERGE_REPORT_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-execution.csv" - public static final GENERIC_MERGE_REPORT_COMMITS_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-execution-commits.csv" - public static final MERGE_TOOL_EXECUTORS_TO_USE = System.getProperty("miningframework.generic_merge.merge_tool_executors_to_use", "generic_merge,jdime").split(",") - private final List mergeToolExecutors GenericMergeDataCollector() { this.mergeToolExecutors = new ArrayList() - if (MERGE_TOOL_EXECUTORS_TO_USE.contains("generic_merge")) { + if (GenericMergeConfig.MERGE_TOOL_EXECUTORS_TO_USE.contains("generic_merge")) { LOG.debug("Registering Generic Merge as a merge tool executor") this.mergeToolExecutors.add(new GenericMergeToolExecutor()) } - if (MERGE_TOOL_EXECUTORS_TO_USE.contains("jdime")) { + if (GenericMergeConfig.MERGE_TOOL_EXECUTORS_TO_USE.contains("jdime")) { LOG.debug("Registering jDime as a merge tool executor") this.mergeToolExecutors.add(new JDimeMergeToolExecutor()) } @@ -96,7 +90,7 @@ class GenericMergeDataCollector implements DataCollector { LOG.trace("Starting write of files results to report file") def lines = mergeToolsExecutionResults.parallelStream().map(result -> getReportLine(project, mergeCommit, result)) - def reportFile = new File(GENERIC_MERGE_REPORT_FILE_NAME) + def reportFile = new File(GenericMergeConfig.GENERIC_MERGE_REPORT_FILE_NAME) reportFile << lines.collect(Collectors.joining(System.lineSeparator())) << "\n" LOG.trace("Finished write of files results to report file") @@ -110,7 +104,7 @@ class GenericMergeDataCollector implements DataCollector { list.add(it.value.allScenariosMatch.toString()) return list.join(",").replaceAll('\\\\', '/') } - def commitReportFile = new File(GENERIC_MERGE_REPORT_COMMITS_FILE_NAME) + def commitReportFile = new File(GenericMergeConfig.GENERIC_MERGE_REPORT_COMMITS_FILE_NAME) commitReportFile << commitLines.collect(Collectors.joining(System.lineSeparator())) << "\n" LOG.trace("Finished write of commit report") } diff --git a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy index 9c6bcce95..6efc1f243 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy @@ -3,6 +3,7 @@ package services.dataCollectors.GenericMerge.executors import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger import services.dataCollectors.GenericMerge.FileFormatNormalizer +import services.dataCollectors.GenericMerge.GenericMergeConfig import services.dataCollectors.GenericMerge.model.MergeScenarioResult import util.ProcessRunner @@ -11,9 +12,6 @@ import java.nio.file.Path class GenericMergeToolExecutor extends MergeToolExecutor { private static Logger LOG = LogManager.getLogger(GenericMergeToolExecutor.class) - private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") - private static final String GENERIC_MERGE_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/generic-merge" - @Override protected MergeScenarioResult executeTool(Path scenario, Path outputFile) { def working_directory_path = scenario.toAbsolutePath().toString() @@ -43,7 +41,7 @@ class GenericMergeToolExecutor extends MergeToolExecutor { private static List getBuildParameters(Path outputFile) { def list = new ArrayList() - list.add(GENERIC_MERGE_BINARY_PATH) + list.add(GenericMergeConfig.GENERIC_MERGE_BINARY_PATH) list.add("merge") list.add("--base-path=base.java") list.add("--left-path=left.java") diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index 42a37c24b..2a7ab7483 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -2,6 +2,7 @@ package services.dataCollectors.GenericMerge.executors import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger +import services.dataCollectors.GenericMerge.GenericMergeConfig import services.dataCollectors.GenericMerge.model.MergeScenarioResult import services.util.MergeConflict import util.ProcessRunner @@ -11,14 +12,11 @@ import java.nio.file.Path class JDimeMergeToolExecutor extends MergeToolExecutor { private static Logger LOG = LogManager.getLogger(JDimeMergeToolExecutor.class) - private static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") - private static final String JDIME_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/jdime/install/JDime/bin" - @Override protected MergeScenarioResult executeTool(Path scenario, Path outputFile) { def working_directory_path = scenario.toAbsolutePath().toString() - def processBuilder = ProcessRunner.buildProcess(JDIME_BINARY_PATH, + def processBuilder = ProcessRunner.buildProcess(GenericMergeConfig.JDIME_BINARY_PATH, "./JDime", "-f", "--mode=structured", diff --git a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy index ce04d57f6..0c9902ba3 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy @@ -2,6 +2,7 @@ package services.dataCollectors.GenericMerge.executors import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger +import services.dataCollectors.GenericMerge.GenericMergeConfig import services.dataCollectors.GenericMerge.model.MergeScenarioExecutionSummary import services.dataCollectors.GenericMerge.model.MergeScenarioResult @@ -10,8 +11,6 @@ import java.nio.file.Path abstract class MergeToolExecutor { private static Logger LOG = LogManager.getLogger(MergeToolExecutor.class) - private static final int NUMBER_OF_EXECUTIONS = 5 - MergeScenarioExecutionSummary runToolForMergeScenario(Path scenario) { LOG.trace("Starting execution of merge scenario with tool ${getToolName()}") @@ -19,14 +18,14 @@ abstract class MergeToolExecutor { def outputFilePath = scenario.resolve("merge." + getToolName().toLowerCase() + ".java") MergeScenarioResult result = null - for (int i = 0; i < NUMBER_OF_EXECUTIONS; i++) { - LOG.trace("Starting execution ${i + 1} of ${NUMBER_OF_EXECUTIONS}") + for (int i = 0; i < GenericMergeConfig.NUMBER_OF_EXECUTIONS; i++) { + LOG.trace("Starting execution ${i + 1} of ${GenericMergeConfig.NUMBER_OF_EXECUTIONS}") long startTime = System.nanoTime() result = executeTool(scenario, outputFilePath) long endTime = System.nanoTime() - LOG.trace("Finished execution ${i + 1} of ${NUMBER_OF_EXECUTIONS} IN ${endTime - startTime} ns") + LOG.trace("Finished execution ${i + 1} of ${GenericMergeConfig.NUMBER_OF_EXECUTIONS} IN ${endTime - startTime} ns") // If we're running more than one execution, we use the first one as a warm up - if (NUMBER_OF_EXECUTIONS == 1 || i > 0) { + if (GenericMergeConfig.NUMBER_OF_EXECUTIONS == 1 || i > 0) { executionTimes.add(endTime - startTime) } } From 77009c01d518174c332db30724f4a6b478f2c61e Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 23 Jul 2024 07:40:29 -0300 Subject: [PATCH 72/79] feat: Remove normalization from execution to avoid incorrect runtime measure --- .../executors/GenericMergeToolExecutor.groovy | 10 +++++----- .../executors/JDimeMergeToolExecutor.groovy | 5 +++++ .../GenericMerge/executors/MergeToolExecutor.groovy | 7 +++++++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy index 6efc1f243..08be54737 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/GenericMergeToolExecutor.groovy @@ -2,7 +2,6 @@ package services.dataCollectors.GenericMerge.executors import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger -import services.dataCollectors.GenericMerge.FileFormatNormalizer import services.dataCollectors.GenericMerge.GenericMergeConfig import services.dataCollectors.GenericMerge.model.MergeScenarioResult import util.ProcessRunner @@ -27,13 +26,14 @@ class GenericMergeToolExecutor extends MergeToolExecutor { LOG.debug("Generic Merge output: ${output.getInputStream().readLines()}") } - if (output.exitValue() == 0) { - FileFormatNormalizer.normalizeFileInPlace(outputFile) - } - return output.exitValue() == 0 ? MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS : output.exitValue() == 1 ? MergeScenarioResult.SUCCESS_WITH_CONFLICTS : MergeScenarioResult.TOOL_ERROR } + @Override + protected boolean shouldSkipFileNormalization() { + return false + } + @Override String getToolName() { return "GENERIC_MERGE" diff --git a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy index 2a7ab7483..233740824 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/JDimeMergeToolExecutor.groovy @@ -42,6 +42,11 @@ class JDimeMergeToolExecutor extends MergeToolExecutor { return MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS } + @Override + protected boolean shouldSkipFileNormalization() { + return true + } + @Override String getToolName() { return "JDIME" diff --git a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy index 0c9902ba3..a05475591 100644 --- a/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy +++ b/src/main/services/dataCollectors/GenericMerge/executors/MergeToolExecutor.groovy @@ -2,6 +2,7 @@ package services.dataCollectors.GenericMerge.executors import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger +import services.dataCollectors.GenericMerge.FileFormatNormalizer import services.dataCollectors.GenericMerge.GenericMergeConfig import services.dataCollectors.GenericMerge.model.MergeScenarioExecutionSummary import services.dataCollectors.GenericMerge.model.MergeScenarioResult @@ -32,6 +33,10 @@ abstract class MergeToolExecutor { long averageTime = (long) (executionTimes.stream().reduce(0, (prev, cur) -> prev + cur) / executionTimes.size()) + if (result == MergeScenarioResult.SUCCESS_WITHOUT_CONFLICTS && !shouldSkipFileNormalization()) { + FileFormatNormalizer.normalizeFileInPlace(outputFilePath) + } + def summary = new MergeScenarioExecutionSummary(scenario, outputFilePath, result, @@ -44,5 +49,7 @@ abstract class MergeToolExecutor { protected abstract MergeScenarioResult executeTool(Path scenario, Path outputFile); + protected abstract boolean shouldSkipFileNormalization(); + abstract String getToolName(); } From 3b7a597e5983983df57f549568718b56e7750f1e Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 23 Jul 2024 07:50:07 -0300 Subject: [PATCH 73/79] feat: move to args --- src/main/arguments/ArgsParser.groovy | 5 +++++ src/main/arguments/Arguments.groovy | 20 ++++++++++++++----- .../GenericMerge/GenericMergeConfig.groovy | 6 ++++-- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/main/arguments/ArgsParser.groovy b/src/main/arguments/ArgsParser.groovy index 25a400dfb..2f8a3f1f2 100644 --- a/src/main/arguments/ArgsParser.groovy +++ b/src/main/arguments/ArgsParser.groovy @@ -35,6 +35,7 @@ class ArgsParser { this.cli.e(longOpt: 'extension', args: 1, argName: 'file extenson', 'Specify the file extension that should be used in the analysis (e.g. .rb, .ts, .java, .cpp. Default: .java)') this.cli.l(longOpt: 'language-separators', args: 1, argName: 'language syntactic separators', 'Specify the language separators that should be used in the analysis. Required for (and only considered when) running studies with the CSDiff tool. Default: \"{ } ( ) ; ,\"') this.cli.log(longOpt: 'log-level', args: 1, argName: 'log level', 'Specify the minimum log level: (OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE, ALL). Default: \"INFO\"') + this.cli.generic_merge_merge_tools_to_use(longOpt: 'generic-merge-merge-tools-to-use', argName: 'Merge tools to use', 'Merge tools that will be used in Generic Merge experiment') } Arguments parse(args) { @@ -130,6 +131,10 @@ class ArgsParser { if(this.options.log) { args.setLogLevel(Level.toLevel(this.options.log)) } + + if (this.options.generic_merge_merge_tools_to_use) { + args.setMergeToolsToUse(this.options.generic_merge_merge_tools_to_use.toString().split(",")) + } } private boolean repositoryExists(String repositoryURL) { diff --git a/src/main/arguments/Arguments.groovy b/src/main/arguments/Arguments.groovy index 0bda91758..fd84b2059 100644 --- a/src/main/arguments/Arguments.groovy +++ b/src/main/arguments/Arguments.groovy @@ -5,7 +5,7 @@ import org.apache.logging.log4j.Level import org.apache.logging.log4j.core.config.Configurator class Arguments { - + private String inputPath private String outputPath private String sinceDate @@ -19,6 +19,7 @@ class Arguments { private String syntacticSeparators private String fileExtension private Level logLevel + private String[] mergeToolsToUse Arguments() { // set the default values for all parameters isHelp = false @@ -33,9 +34,10 @@ class Arguments { syntacticSeparators = '{ } ( ) ; ,' fileExtension = 'java' logLevel = Level.INFO + mergeToolsToUse = ["generic_merge", "jdime"] } - void setNumOfThreads (int numOfThreads) { + void setNumOfThreads(int numOfThreads) { this.numOfThreads = numOfThreads } @@ -86,7 +88,7 @@ class Arguments { int getNumOfThreads() { return this.numOfThreads } - + String getInputPath() { return inputPath } @@ -111,7 +113,7 @@ class Arguments { return isHelp } - boolean getKeepProjects () { + boolean getKeepProjects() { return keepProjects } @@ -134,7 +136,7 @@ class Arguments { boolean providedAccessKey() { return accessKey.length() > 0 } - + boolean isPushCommandActive() { return !resultsRemoteRepositoryURL.equals('') } @@ -147,4 +149,12 @@ class Arguments { this.logLevel = logLevel Configurator.setRootLevel(logLevel) } + + String[] getMergeToolsToUse() { + return mergeToolsToUse + } + + void setMergeToolsToUse(String[] mergeToolsToUse) { + this.mergeToolsToUse = mergeToolsToUse + } } \ No newline at end of file diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeConfig.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeConfig.groovy index 03f1647c1..1161d2e50 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeConfig.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeConfig.groovy @@ -1,9 +1,11 @@ package services.dataCollectors.GenericMerge +import static app.MiningFramework.arguments + class GenericMergeConfig { - public static final BASE_EXPERIMENT_PATH = System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") + public static final BASE_EXPERIMENT_PATH = System.getProperty("user.dir") - public static final MERGE_TOOL_EXECUTORS_TO_USE = System.getProperty("miningframework.generic_merge.merge_tool_executors_to_use", "generic_merge,jdime").split(",") + public static final MERGE_TOOL_EXECUTORS_TO_USE = arguments.getMergeToolsToUse() public static final BUILD_REQUESTER_REPORT_PATH = "${BASE_EXPERIMENT_PATH}/output/reports/generic-merge-execution-build-requests.csv" From f2298c7c88d373d20194524f3884f84f7a91e336 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 23 Jul 2024 07:55:37 -0300 Subject: [PATCH 74/79] fix: Incorrect reference to variable --- src/main/injectors/GenericMergeModule.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/injectors/GenericMergeModule.groovy b/src/main/injectors/GenericMergeModule.groovy index 891d870a6..f2db32959 100644 --- a/src/main/injectors/GenericMergeModule.groovy +++ b/src/main/injectors/GenericMergeModule.groovy @@ -9,7 +9,7 @@ import interfaces.ProjectProcessor import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger import services.commitFilters.MutuallyModifiedFilesCommitFilter -import services.commitFilters.S3MCommitFilter +import services.dataCollectors.GenericMerge.GenericMergeConfig import services.dataCollectors.GenericMerge.GenericMergeDataCollector import services.outputProcessors.GenericMergeDataOutputProcessor import services.projectProcessors.DummyProjectProcessor @@ -41,8 +41,8 @@ class GenericMergeModule extends AbstractModule { private static void createExecutionReportsFile() { LOG.info("Creating Generic Merge report file") - Files.createDirectories(Paths.get(GenericMergeDataCollector.GENERIC_MERGE_REPORT_PATH)) - def reportFile = new File(GenericMergeDataCollector.GENERIC_MERGE_REPORT_FILE_NAME) + Files.createDirectories(Paths.get(GenericMergeConfig.GENERIC_MERGE_REPORT_PATH)) + def reportFile = new File(GenericMergeConfig.GENERIC_MERGE_REPORT_FILE_NAME) reportFile.createNewFile() } } From c5762ca3f75376a66198d25707e6c2881a05abe8 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 23 Jul 2024 08:02:51 -0300 Subject: [PATCH 75/79] fix: Remove commands --- src/main/arguments/Arguments.groovy | 10 ---------- .../GenericMerge/GenericMergeConfig.groovy | 4 ---- .../GenericMerge/GenericMergeDataCollector.groovy | 13 ++++--------- 3 files changed, 4 insertions(+), 23 deletions(-) diff --git a/src/main/arguments/Arguments.groovy b/src/main/arguments/Arguments.groovy index fd84b2059..7098fa792 100644 --- a/src/main/arguments/Arguments.groovy +++ b/src/main/arguments/Arguments.groovy @@ -19,7 +19,6 @@ class Arguments { private String syntacticSeparators private String fileExtension private Level logLevel - private String[] mergeToolsToUse Arguments() { // set the default values for all parameters isHelp = false @@ -34,7 +33,6 @@ class Arguments { syntacticSeparators = '{ } ( ) ; ,' fileExtension = 'java' logLevel = Level.INFO - mergeToolsToUse = ["generic_merge", "jdime"] } void setNumOfThreads(int numOfThreads) { @@ -149,12 +147,4 @@ class Arguments { this.logLevel = logLevel Configurator.setRootLevel(logLevel) } - - String[] getMergeToolsToUse() { - return mergeToolsToUse - } - - void setMergeToolsToUse(String[] mergeToolsToUse) { - this.mergeToolsToUse = mergeToolsToUse - } } \ No newline at end of file diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeConfig.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeConfig.groovy index 1161d2e50..b1750a1ee 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeConfig.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeConfig.groovy @@ -1,12 +1,8 @@ package services.dataCollectors.GenericMerge -import static app.MiningFramework.arguments - class GenericMergeConfig { public static final BASE_EXPERIMENT_PATH = System.getProperty("user.dir") - public static final MERGE_TOOL_EXECUTORS_TO_USE = arguments.getMergeToolsToUse() - public static final BUILD_REQUESTER_REPORT_PATH = "${BASE_EXPERIMENT_PATH}/output/reports/generic-merge-execution-build-requests.csv" public static final GENERIC_MERGE_REPORT_PATH = "${BASE_EXPERIMENT_PATH}/output/reports" diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 1182f588e..3e52c00f2 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -23,15 +23,10 @@ class GenericMergeDataCollector implements DataCollector { private final List mergeToolExecutors GenericMergeDataCollector() { - this.mergeToolExecutors = new ArrayList() - if (GenericMergeConfig.MERGE_TOOL_EXECUTORS_TO_USE.contains("generic_merge")) { - LOG.debug("Registering Generic Merge as a merge tool executor") - this.mergeToolExecutors.add(new GenericMergeToolExecutor()) - } - if (GenericMergeConfig.MERGE_TOOL_EXECUTORS_TO_USE.contains("jdime")) { - LOG.debug("Registering jDime as a merge tool executor") - this.mergeToolExecutors.add(new JDimeMergeToolExecutor()) - } + this.mergeToolExecutors = Arrays.asList([ + new GenericMergeToolExecutor(), + new JDimeMergeToolExecutor() + ]) } @Override From dc6fbf5df28980540156106f209dac71118a1e88 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 23 Jul 2024 08:07:03 -0300 Subject: [PATCH 76/79] chore: update build.gradle --- build.gradle | 9 --------- 1 file changed, 9 deletions(-) diff --git a/build.gradle b/build.gradle index 1c660e300..2b33a5e21 100644 --- a/build.gradle +++ b/build.gradle @@ -28,15 +28,6 @@ test { } } -application { - applicationDefaultJvmArgs = ["-Dminingframework.generic_merge.base_experiment_path=/usr/src/app", "-Dminingframework.generic_merge.merge_tool_executors_to_use=generic_merge,jdime"] -} - -run { - systemProperty "miningframework.generic_merge.base_experiment_path", System.getProperty("miningframework.generic_merge.base_experiment_path", "/usr/src/app") - systemProperty "miningframework.generic_merge.merge_tool_executors_to_use", System.getProperty("miningframework.generic_merge.merge_tool_executors_to_use", "generic_merge,jdime") -} - dependencies { implementation 'com.google.inject:guice:4.2.2' implementation 'commons-io:commons-io:2.6' From 4224a0d8c90bd34af455e8f5a5b3de6037742501 Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 23 Jul 2024 08:08:15 -0300 Subject: [PATCH 77/79] fix: remove params --- src/main/arguments/ArgsParser.groovy | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/arguments/ArgsParser.groovy b/src/main/arguments/ArgsParser.groovy index 2f8a3f1f2..25a400dfb 100644 --- a/src/main/arguments/ArgsParser.groovy +++ b/src/main/arguments/ArgsParser.groovy @@ -35,7 +35,6 @@ class ArgsParser { this.cli.e(longOpt: 'extension', args: 1, argName: 'file extenson', 'Specify the file extension that should be used in the analysis (e.g. .rb, .ts, .java, .cpp. Default: .java)') this.cli.l(longOpt: 'language-separators', args: 1, argName: 'language syntactic separators', 'Specify the language separators that should be used in the analysis. Required for (and only considered when) running studies with the CSDiff tool. Default: \"{ } ( ) ; ,\"') this.cli.log(longOpt: 'log-level', args: 1, argName: 'log level', 'Specify the minimum log level: (OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE, ALL). Default: \"INFO\"') - this.cli.generic_merge_merge_tools_to_use(longOpt: 'generic-merge-merge-tools-to-use', argName: 'Merge tools to use', 'Merge tools that will be used in Generic Merge experiment') } Arguments parse(args) { @@ -131,10 +130,6 @@ class ArgsParser { if(this.options.log) { args.setLogLevel(Level.toLevel(this.options.log)) } - - if (this.options.generic_merge_merge_tools_to_use) { - args.setMergeToolsToUse(this.options.generic_merge_merge_tools_to_use.toString().split(",")) - } } private boolean repositoryExists(String repositoryURL) { From 9ddb5ed1955d52d51cda78c8faac80da014510ac Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 23 Jul 2024 08:15:31 -0300 Subject: [PATCH 78/79] fix: Incorrect initialization --- .../GenericMerge/GenericMergeDataCollector.groovy | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index 3e52c00f2..cce7587f7 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -7,7 +7,6 @@ import org.apache.logging.log4j.Logger import project.MergeCommit import project.Project import services.dataCollectors.GenericMerge.executors.GenericMergeToolExecutor -import services.dataCollectors.GenericMerge.executors.JDimeMergeToolExecutor import services.dataCollectors.GenericMerge.executors.MergeToolExecutor import services.dataCollectors.GenericMerge.model.MergeCommitExecutionSummary import services.dataCollectors.GenericMerge.model.MergeScenarioExecutionSummary @@ -20,14 +19,10 @@ import java.util.stream.Collectors class GenericMergeDataCollector implements DataCollector { private static Logger LOG = LogManager.getLogger(GenericMergeDataCollector.class) - private final List mergeToolExecutors - - GenericMergeDataCollector() { - this.mergeToolExecutors = Arrays.asList([ - new GenericMergeToolExecutor(), - new JDimeMergeToolExecutor() - ]) - } + private final List mergeToolExecutors = Arrays.asList( + new GenericMergeToolExecutor(), + new JDimeMergeToolExecutor() + ) @Override void collectData(Project project, MergeCommit mergeCommit) { From 4c4ca263f34ef037145dc4e51fcc3248897e56ff Mon Sep 17 00:00:00 2001 From: Joao Duarte Date: Tue, 23 Jul 2024 08:33:58 -0300 Subject: [PATCH 79/79] fix: initialization --- .../dataCollectors/GenericMerge/GenericMergeDataCollector.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy index cce7587f7..91a04f44c 100644 --- a/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy +++ b/src/main/services/dataCollectors/GenericMerge/GenericMergeDataCollector.groovy @@ -7,6 +7,7 @@ import org.apache.logging.log4j.Logger import project.MergeCommit import project.Project import services.dataCollectors.GenericMerge.executors.GenericMergeToolExecutor +import services.dataCollectors.GenericMerge.executors.JDimeMergeToolExecutor import services.dataCollectors.GenericMerge.executors.MergeToolExecutor import services.dataCollectors.GenericMerge.model.MergeCommitExecutionSummary import services.dataCollectors.GenericMerge.model.MergeScenarioExecutionSummary