Merge pull request #191 from jpedroh/feat-generic-merge

Add study for Generic Merge tool

src/main/arguments/Arguments.groovy

@@ -5,7 +5,7 @@ import org.apache.logging.log4j.Level
 import org.apache.logging.log4j.core.config.Configurator
 
 class Arguments {
-    
+
     private String inputPath
     private String outputPath
     private String sinceDate
@@ -35,7 +35,7 @@ class Arguments {
         logLevel = Level.INFO
     }
 
-    void setNumOfThreads (int numOfThreads) {
+    void setNumOfThreads(int numOfThreads) {
         this.numOfThreads = numOfThreads
     }
 
@@ -86,7 +86,7 @@ class Arguments {
     int getNumOfThreads() {
         return this.numOfThreads
     }
-    
+
     String getInputPath() {
         return inputPath
     }
@@ -111,7 +111,7 @@ class Arguments {
         return isHelp
     }
 
-    boolean getKeepProjects () {
+    boolean getKeepProjects() {
         return keepProjects
     }
 
@@ -134,7 +134,7 @@ class Arguments {
     boolean providedAccessKey() {
         return accessKey.length() > 0
     }
-    
+
     boolean isPushCommandActive() {
         return !resultsRemoteRepositoryURL.equals('')
     }

src/main/injectors/GenericMergeModule.groovy

@@ -0,0 +1,48 @@
+package injectors
+
+import com.google.inject.AbstractModule
+import com.google.inject.multibindings.Multibinder
+import interfaces.CommitFilter
+import interfaces.DataCollector
+import interfaces.OutputProcessor
+import interfaces.ProjectProcessor
+import org.apache.logging.log4j.LogManager
+import org.apache.logging.log4j.Logger
+import services.commitFilters.MutuallyModifiedFilesCommitFilter
+import services.dataCollectors.GenericMerge.GenericMergeConfig
+import services.dataCollectors.GenericMerge.GenericMergeDataCollector
+import services.outputProcessors.GenericMergeDataOutputProcessor
+import services.projectProcessors.DummyProjectProcessor
+import services.util.ci.CIPlatform
+import services.util.ci.TravisPlatform
+
+import java.nio.file.Files
+import java.nio.file.Paths
+
+class GenericMergeModule extends AbstractModule {
+    private static Logger LOG = LogManager.getLogger(GenericMergeModule.class)
+
+    @Override
+    protected void configure() {
+        Multibinder<ProjectProcessor> projectProcessorBinder = Multibinder.newSetBinder(binder(), ProjectProcessor.class)
+        projectProcessorBinder.addBinding().to(DummyProjectProcessor.class)
+
+        Multibinder<DataCollector> dataCollectorBinder = Multibinder.newSetBinder(binder(), DataCollector.class)
+        dataCollectorBinder.addBinding().to(GenericMergeDataCollector.class)
+
+        Multibinder<OutputProcessor> outputProcessorBinder = Multibinder.newSetBinder(binder(), OutputProcessor.class)
+        outputProcessorBinder.addBinding().to(GenericMergeDataOutputProcessor.class)
+
+        bind(CommitFilter.class).to(MutuallyModifiedFilesCommitFilter.class)
+        bind(CIPlatform.class).to(TravisPlatform.class)
+
+        createExecutionReportsFile()
+    }
+
+    private static void createExecutionReportsFile() {
+        LOG.info("Creating Generic Merge report file")
+        Files.createDirectories(Paths.get(GenericMergeConfig.GENERIC_MERGE_REPORT_PATH))
+        def reportFile = new File(GenericMergeConfig.GENERIC_MERGE_REPORT_FILE_NAME)
+        reportFile.createNewFile()
+    }
+}

src/main/resources/log4j2.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="INFO">
+    <Appenders>
+        <Console name="console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n" />
+        </Console>
+    </Appenders>
+
+    <Loggers>
+        <Root level="info">
+            <AppenderRef ref="console" />
+        </Root>
+    </Loggers>
+</Configuration>

src/main/services/dataCollectors/GenericMerge/BuildRequester.groovy

@@ -0,0 +1,164 @@
+package services.dataCollectors.GenericMerge
+
+import org.apache.logging.log4j.LogManager
+import org.apache.logging.log4j.Logger
+import project.MergeCommit
+import project.Project
+import services.util.Utils
+
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
+import java.nio.file.StandardCopyOption
+
+class BuildRequester {
+    private static Logger LOG = LogManager.getLogger(BuildRequester.class)
+
+    static requestBuildWithRevision(Project project, MergeCommit mergeCommit, List<Path> mergeScenarios, String mergeTool) {
+        String toReplaceFile = "merge.${mergeTool.toLowerCase()}.java"
+
+        String branchName = "${mergeCommit.getSHA().take(7)}-${mergeTool}"
+
+        createBranchFromCommit(project, mergeCommit, branchName)
+        replaceFilesInProject(project, mergeCommit, mergeScenarios, toReplaceFile)
+        createOrReplaceGithubActionsFile(project)
+        def commitSha = stageAndPushChanges(project, branchName, "Mining Framework Analysis")
+
+        def reportFile = new File(GenericMergeConfig.BUILD_REQUESTER_REPORT_PATH)
+        reportFile.createNewFile()
+        reportFile.append("${project.getName()},${branchName},${mergeTool},${commitSha}\n")
+    }
+
+    private static void createBranchFromCommit(Project project, MergeCommit mergeCommit, String branchName) {
+        Path projectPath = Paths.get(project.getPath())
+
+        // Checkout to new branch
+        Utils.runGitCommand(projectPath, 'checkout', '-b', branchName, mergeCommit.getSHA())
+    }
+
+    private static void replaceFilesInProject(Project project, MergeCommit mergeCommit, List<Path> mergeScenarios, String toReplaceFile) {
+        mergeScenarios.stream()
+                .forEach(mergeScenario -> {
+                    LOG.debug("Trying to copy " + getSource(mergeScenario, toReplaceFile) + " into " + getTarget(project, mergeCommit, mergeScenario))
+                    Files.copy(getSource(mergeScenario, toReplaceFile), getTarget(project, mergeCommit, mergeScenario), StandardCopyOption.REPLACE_EXISTING)
+                })
+    }
+
+    private static void createOrReplaceGithubActionsFile(Project project) {
+        LOG.debug("Starting creation of github actions file")
+        def githubActionsFilePath = "${Paths.get(project.getPath()).toAbsolutePath().toString()}/.github/workflows"
+        LOG.debug("Location of github actions folder ${githubActionsFilePath}")
+
+        def buildSystem = getBuildSystemForProject(project)
+        LOG.debug("Using ${buildSystem.class.getSimpleName()} as build system for project ${project.getName()}")
+
+        def githubActionsContent = """
+name: Mining Framework Check
+on: [push]
+jobs:
+    build:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v2
+            - uses: actions/setup-java@v1
+              with:
+                java-version: 11
+            - run: ${buildSystem.getBuildCommand()}
+    test:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v2
+            - uses: actions/setup-java@v1
+              with:
+                java-version: 11
+            - run: ${buildSystem.getTestCommand()}
+"""
+        Files.createDirectories(Paths.get(githubActionsFilePath))
+        def file = new File("${githubActionsFilePath}/mining_framework.yml")
+        file.createNewFile()
+        file.write(githubActionsContent)
+        LOG.debug("Finished creation of github actions file")
+    }
+
+    private static Path getSource(Path mergeScenario, String toReplaceFile) {
+        return mergeScenario.resolve(toReplaceFile)
+    }
+
+    private static Path getTarget(Project project, MergeCommit mergeCommit, Path mergeScenario) {
+        Path projectPath = Paths.get(project.getPath())
+        Path filePath = Utils.commitFilesPath(project, mergeCommit).relativize(mergeScenario)
+        return projectPath.resolve(filePath)
+    }
+
+    private static String stageAndPushChanges(Project project, String branchName, String commitMessage) {
+        Path projectPath = Paths.get(project.getPath())
+
+        // Stage changes
+        Utils.runGitCommand(projectPath, 'add', '.')
+
+        // Commit changes
+        Utils.runGitCommand(projectPath, 'commit', '-m', commitMessage)
+        def commitSha = Utils.runGitCommand(projectPath, 'rev-parse', 'HEAD')
+        LOG.debug("Created commit with hash ${commitSha}")
+
+        // Push changes
+        Utils.runGitCommand(projectPath, 'push', '--set-upstream', 'origin', branchName, '--force-with-lease')
+
+        return commitSha.get(0)
+    }
+
+    private static interface BuildSystem {
+        String getBuildCommand()
+
+        String getTestCommand()
+    }
+
+    private static class MavenBuildSystem implements BuildSystem {
+        @Override
+        String getBuildCommand() {
+            return "mvn package -Dskiptests"
+        }
+
+        @Override
+        String getTestCommand() {
+            return "mvn test"
+        }
+    }
+
+    private static class GradleBuildSystem implements BuildSystem {
+        @Override
+        String getBuildCommand() {
+            return "./gradlew assemble"
+        }
+
+        @Override
+        String getTestCommand() {
+            return "./gradlew test"
+        }
+    }
+
+    private static class NoopBuildSystem implements BuildSystem {
+        @Override
+        String getBuildCommand() {
+            return "echo no build available"
+        }
+
+        @Override
+        String getTestCommand() {
+            return "echo no test available"
+        }
+    }
+
+    private static BuildSystem getBuildSystemForProject(Project project) {
+        File mavenFile = new File("${project.getPath()}/pom.xml")
+        File gradleFile = new File("${project.getPath()}/build.gradle")
+
+        if (mavenFile.exists()) {
+            return new MavenBuildSystem()
+        } else if (gradleFile.exists()) {
+            return new GradleBuildSystem()
+        } else {
+            return new NoopBuildSystem()
+        }
+    }
+}

src/main/services/dataCollectors/GenericMerge/FileFormatNormalizer.groovy

@@ -0,0 +1,27 @@
+package services.dataCollectors.GenericMerge
+
+import org.apache.logging.log4j.LogManager
+import org.apache.logging.log4j.Logger
+import util.ProcessRunner
+
+import java.nio.file.Path
+
+class FileFormatNormalizer {
+    private static Logger LOG = LogManager.getLogger(FileFormatNormalizer.class)
+
+    static void normalizeFileInPlace(Path file) {
+        def processBuilder = ProcessRunner.buildProcess(GenericMergeConfig.JDIME_BINARY_PATH,
+                "./JDime",
+                "-f",
+                "--mode=structured",
+                "--output=${file.toAbsolutePath().toString()}".toString(),
+                file.toAbsolutePath().toString(),
+                file.toAbsolutePath().toString(),
+                file.toAbsolutePath().toString())
+
+        def exitCode = ProcessRunner.startProcess(processBuilder).waitFor()
+        if (exitCode != 0) {
+            LOG.warn("File normalization failed with exit code ${exitCode}")
+        }
+    }
+}

src/main/services/dataCollectors/GenericMerge/FileSyntacticDiff.groovy

@@ -0,0 +1,38 @@
+package services.dataCollectors.GenericMerge
+
+import org.apache.logging.log4j.LogManager
+import org.apache.logging.log4j.Logger
+import util.ProcessRunner
+
+import java.nio.file.Files
+import java.nio.file.Path
+
+class FileSyntacticDiff {
+    private static Logger LOG = LogManager.getLogger(FileSyntacticDiff.class)
+
+    static boolean areFilesSyntacticallyEquivalent(Path fileA, Path fileB) {
+        if (!Files.exists(fileA) || !Files.exists(fileB)) {
+            LOG.trace("Early returning because one of the files ${} do not exist")
+            return false
+        }
+
+        def process = ProcessRunner.buildProcess("./")
+
+        def list = new ArrayList<String>()
+        list.add(GenericMergeConfig.GENERIC_MERGE_BINARY_PATH)
+        list.add("diff")
+        list.add("--left-path=${fileA.toAbsolutePath().toString()}".toString())
+        list.add("--right-path=${fileB.toAbsolutePath().toString()}".toString())
+        list.add("--language=java")
+        process.command().addAll(list)
+
+        def output = ProcessRunner.startProcess(process)
+        output.waitFor()
+
+        if (output.exitValue() > 1) {
+            LOG.warn("Error while running comparison between ${fileA.toString()} and ${fileB.toString()}: ${output.getInputStream().readLines()}")
+        }
+
+        return output.exitValue() == 0
+    }
+}

src/main/services/dataCollectors/GenericMerge/GenericMergeConfig.groovy

@@ -0,0 +1,16 @@
+package services.dataCollectors.GenericMerge
+
+class GenericMergeConfig {
+    public static final BASE_EXPERIMENT_PATH = System.getProperty("user.dir")
+
+    public static final BUILD_REQUESTER_REPORT_PATH = "${BASE_EXPERIMENT_PATH}/output/reports/generic-merge-execution-build-requests.csv"
+
+    public static final GENERIC_MERGE_REPORT_PATH = "${BASE_EXPERIMENT_PATH}/output/reports"
+    public static final GENERIC_MERGE_REPORT_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-execution.csv"
+    public static final GENERIC_MERGE_REPORT_COMMITS_FILE_NAME = "${GENERIC_MERGE_REPORT_PATH}/generic-merge-execution-commits.csv"
+
+    public static final String GENERIC_MERGE_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/generic-merge"
+    public static final String JDIME_BINARY_PATH = "${BASE_EXPERIMENT_PATH}/tools/jdime/install/JDime/bin"
+
+    public static final int NUMBER_OF_EXECUTIONS = 5
+}