From 318b36e6aa7c1cd469bfc68cd271ad59a55e603e Mon Sep 17 00:00:00 2001 From: md2manoppello Date: Wed, 31 Jul 2019 16:52:04 +0200 Subject: [PATCH 01/41] Supporting API migration use case --- KB-db/Dockerfile | 4 +- KB/Dockerfile | 6 +- KB/application.properties | 17 +- KB/src-rascal/org/maracas/Extractor.rsc | 143 +++++ KB/src-rascal/org/maracas/Maracas.java | 390 ++++++++++++ KB/src-rascal/org/maracas/Maracas.rsc | 34 ++ KB/src-rascal/org/maracas/RunAll.rsc | 171 ++++++ KB/src-rascal/org/maracas/Test.rsc | 5 + KB/src-rascal/org/maracas/config/Options.rsc | 21 + KB/src-rascal/org/maracas/data/Detection.java | 150 +++++ KB/src-rascal/org/maracas/delta/Delta.rsc | 121 ++++ .../org/maracas/delta/DeltaBuilder.rsc | 340 +++++++++++ KB/src-rascal/org/maracas/delta/Detector.rsc | 173 ++++++ KB/src-rascal/org/maracas/delta/Migration.rsc | 169 ++++++ .../org/maracas/delta/stats/Core.rsc | 28 + .../maracas/delta/stats/DeltaStatistics.rsc | 22 + .../delta/stats/DetectionStatistics.rsc | 19 + .../delta/stats/MigrationStatistics.rsc | 68 +++ .../org/maracas/delta/vis/Visualizer.rsc | 152 +++++ KB/src-rascal/org/maracas/io/File.rsc | 54 ++ .../org/maracas/io/properties/IO.java | 57 ++ .../org/maracas/io/properties/IO.rsc | 10 + .../org/maracas/io/properties/Parse.rsc | 6 + .../org/maracas/io/properties/Syntax.rsc | 19 + KB/src-rascal/org/maracas/lang/java/Load.rsc | 7 + .../org/maracas/lang/java/Syntax.rsc | 116 ++++ KB/src-rascal/org/maracas/lang/mj/Load.rsc | 11 + KB/src-rascal/org/maracas/lang/mj/Syntax.rsc | 119 ++++ .../org/maracas/lang/patl/Abstract.rsc | 24 + KB/src-rascal/org/maracas/lang/patl/Load.rsc | 12 + KB/src-rascal/org/maracas/lang/patl/Match.rsc | 50 ++ .../org/maracas/lang/patl/Syntax.rsc | 37 ++ KB/src-rascal/org/maracas/m3/Core.rsc | 399 +++++++++++++ KB/src-rascal/org/maracas/m3/M3Diff.rsc | 41 ++ KB/src-rascal/org/maracas/match/data/Data.rsc | 12 + .../org/maracas/match/fun/SetSimilarity.rsc | 7 + .../maracas/match/fun/StringSimilarity.rsc | 14 + .../match/fun/internal/StringSimilarity.java | 30 + .../match/matcher/DeclarationMatcher.rsc | 12 + .../maracas/match/matcher/GumTreeMatcher.rsc | 126 ++++ .../maracas/match/matcher/JaccardMatcher.rsc | 43 ++ .../match/matcher/LevenshteinMatcher.rsc | 43 ++ .../org/maracas/match/matcher/Matcher.rsc | 39 ++ .../org/maracas/match/matcher/Tree.rsc | 126 ++++ .../org/maracas/measure/java/Size.java | 51 ++ .../org/maracas/measure/java/Size.rsc | 78 +++ .../org/maracas/pipeline/quatlaas/Corpus.rsc | 96 +++ .../maracas/pipeline/quatlaas/data/matrix.csv | 78 +++ .../maracas/pipeline/sonarqube/Pipeline.rsc | 135 +++++ .../sonarqube/sourcemeter/SourceMeter.rsc | 90 +++ .../data/analyzer-java/detections.bin | Bin 0 -> 4260 bytes .../data/analyzer-java/migrations.bin | Bin 0 -> 15393 bytes .../data/analyzer-java/stats-detections.csv | 14 + .../data/analyzer-java/stats-migrations.csv | 14 + .../sonarqube/sourcemeter/data/bc.bin | Bin 0 -> 530033 bytes .../sourcemeter/data/core/detections.bin | Bin 0 -> 4147 bytes .../sourcemeter/data/core/migrations.bin | Bin 0 -> 7839 bytes .../data/core/stats-detections.csv | 14 + .../data/core/stats-migrations.csv | 14 + .../sonarqube/sourcemeter/data/delta.bin | Bin 0 -> 530033 bytes .../sourcemeter/data/r-scripts/delta.R | 34 ++ .../sourcemeter/data/sonar-plugin-api-4.2.jar | Bin 0 -> 594562 bytes .../sourcemeter/data/sonar-plugin-api-6.7.jar | Bin 0 -> 2546260 bytes ...emeter-analyzer-java-plugin-8.2-v6.7.1.jar | Bin 0 -> 1101041 bytes ...r-sourcemeter-analyzer-java-plugin-8.2.jar | Bin 0 -> 6187677 bytes ...nar-sourcemeter-core-plugin-8.2-v6.7.1.jar | Bin 0 -> 7438 bytes .../sonar-sourcemeter-core-plugin-8.2.jar | Bin 0 -> 1324381 bytes .../data/sourcemeter-plugins-8.2-sources.zip | Bin 0 -> 4696145 bytes .../sonarqube/sourcemeter/data/stats-bc.csv | 14 + .../sourcemeter/data/stats-delta.csv | 14 + KB/src-rascal/org/maracas/test/data/AST1.java | 16 + KB/src-rascal/org/maracas/test/data/AST2.java | 11 + KB/src-rascal/org/maracas/test/data/AST3.java | 17 + KB/src-rascal/org/maracas/test/data/AST4.java | 14 + KB/src-rascal/org/maracas/test/data/AST5.java | 14 + .../org/maracas/test/data/ast1CanonStr.bin | Bin 0 -> 371 bytes .../org/maracas/test/data/ast1Descendants.bin | Bin 0 -> 3390 bytes .../org/maracas/test/data/ast2CanonStr.bin | Bin 0 -> 310 bytes .../org/maracas/test/data/ast2Descendants.bin | Bin 0 -> 3103 bytes .../org/maracas/test/data/ast3CanonStr.bin | Bin 0 -> 460 bytes .../org/maracas/test/data/ast3Descendants.bin | Bin 0 -> 4044 bytes .../org/maracas/test/data/ast4CanonStr.bin | Bin 0 -> 371 bytes .../org/maracas/test/data/ast4Descendants.bin | Bin 0 -> 3390 bytes .../test/data/minimalbc-client.1.0.jar | Bin 0 -> 35283 bytes .../org/maracas/test/data/minimalbc.1.0.jar | Bin 0 -> 19943 bytes .../org/maracas/test/data/minimalbc.1.1.jar | Bin 0 -> 18630 bytes .../maracas/test/delta/DeltaBuilderTest.rsc | 287 +++++++++ .../org/maracas/test/delta/DetectorTest.rsc | 431 ++++++++++++++ .../maracas/test/delta/TestApiOldApiNew.rsc | 561 ++++++++++++++++++ .../test/input/ColoredTreeTestInput.rsc | 16 + .../org/maracas/test/m3/CoreTest.rsc | 184 ++++++ .../maracas/test/match/GumTreeMatcherTest.rsc | 30 + .../org/maracas/test/match/TreeTest.rsc | 121 ++++ .../org/maracas/test/util/MathTest.rsc | 18 + KB/src-rascal/org/maracas/util/Math.rsc | 18 + 95 files changed, 5825 insertions(+), 6 deletions(-) create mode 100644 KB/src-rascal/org/maracas/Extractor.rsc create mode 100644 KB/src-rascal/org/maracas/Maracas.java create mode 100644 KB/src-rascal/org/maracas/Maracas.rsc create mode 100644 KB/src-rascal/org/maracas/RunAll.rsc create mode 100644 KB/src-rascal/org/maracas/Test.rsc create mode 100644 KB/src-rascal/org/maracas/config/Options.rsc create mode 100644 KB/src-rascal/org/maracas/data/Detection.java create mode 100644 KB/src-rascal/org/maracas/delta/Delta.rsc create mode 100644 KB/src-rascal/org/maracas/delta/DeltaBuilder.rsc create mode 100644 KB/src-rascal/org/maracas/delta/Detector.rsc create mode 100644 KB/src-rascal/org/maracas/delta/Migration.rsc create mode 100644 KB/src-rascal/org/maracas/delta/stats/Core.rsc create mode 100644 KB/src-rascal/org/maracas/delta/stats/DeltaStatistics.rsc create mode 100644 KB/src-rascal/org/maracas/delta/stats/DetectionStatistics.rsc create mode 100644 KB/src-rascal/org/maracas/delta/stats/MigrationStatistics.rsc create mode 100644 KB/src-rascal/org/maracas/delta/vis/Visualizer.rsc create mode 100644 KB/src-rascal/org/maracas/io/File.rsc create mode 100644 KB/src-rascal/org/maracas/io/properties/IO.java create mode 100644 KB/src-rascal/org/maracas/io/properties/IO.rsc create mode 100644 KB/src-rascal/org/maracas/io/properties/Parse.rsc create mode 100644 KB/src-rascal/org/maracas/io/properties/Syntax.rsc create mode 100644 KB/src-rascal/org/maracas/lang/java/Load.rsc create mode 100644 KB/src-rascal/org/maracas/lang/java/Syntax.rsc create mode 100644 KB/src-rascal/org/maracas/lang/mj/Load.rsc create mode 100644 KB/src-rascal/org/maracas/lang/mj/Syntax.rsc create mode 100644 KB/src-rascal/org/maracas/lang/patl/Abstract.rsc create mode 100644 KB/src-rascal/org/maracas/lang/patl/Load.rsc create mode 100644 KB/src-rascal/org/maracas/lang/patl/Match.rsc create mode 100644 KB/src-rascal/org/maracas/lang/patl/Syntax.rsc create mode 100644 KB/src-rascal/org/maracas/m3/Core.rsc create mode 100644 KB/src-rascal/org/maracas/m3/M3Diff.rsc create mode 100644 KB/src-rascal/org/maracas/match/data/Data.rsc create mode 100644 KB/src-rascal/org/maracas/match/fun/SetSimilarity.rsc create mode 100644 KB/src-rascal/org/maracas/match/fun/StringSimilarity.rsc create mode 100644 KB/src-rascal/org/maracas/match/fun/internal/StringSimilarity.java create mode 100644 KB/src-rascal/org/maracas/match/matcher/DeclarationMatcher.rsc create mode 100644 KB/src-rascal/org/maracas/match/matcher/GumTreeMatcher.rsc create mode 100644 KB/src-rascal/org/maracas/match/matcher/JaccardMatcher.rsc create mode 100644 KB/src-rascal/org/maracas/match/matcher/LevenshteinMatcher.rsc create mode 100644 KB/src-rascal/org/maracas/match/matcher/Matcher.rsc create mode 100644 KB/src-rascal/org/maracas/match/matcher/Tree.rsc create mode 100644 KB/src-rascal/org/maracas/measure/java/Size.java create mode 100644 KB/src-rascal/org/maracas/measure/java/Size.rsc create mode 100644 KB/src-rascal/org/maracas/pipeline/quatlaas/Corpus.rsc create mode 100644 KB/src-rascal/org/maracas/pipeline/quatlaas/data/matrix.csv create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/Pipeline.rsc create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/SourceMeter.rsc create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/analyzer-java/detections.bin create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/analyzer-java/migrations.bin create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/analyzer-java/stats-detections.csv create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/analyzer-java/stats-migrations.csv create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/bc.bin create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/core/detections.bin create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/core/migrations.bin create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/core/stats-detections.csv create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/core/stats-migrations.csv create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/delta.bin create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/r-scripts/delta.R create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/sonar-plugin-api-4.2.jar create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/sonar-plugin-api-6.7.jar create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/sonar-sourcemeter-analyzer-java-plugin-8.2-v6.7.1.jar create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/sonar-sourcemeter-analyzer-java-plugin-8.2.jar create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/sonar-sourcemeter-core-plugin-8.2-v6.7.1.jar create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/sonar-sourcemeter-core-plugin-8.2.jar create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/sourcemeter-plugins-8.2-sources.zip create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/stats-bc.csv create mode 100644 KB/src-rascal/org/maracas/pipeline/sonarqube/sourcemeter/data/stats-delta.csv create mode 100644 KB/src-rascal/org/maracas/test/data/AST1.java create mode 100644 KB/src-rascal/org/maracas/test/data/AST2.java create mode 100644 KB/src-rascal/org/maracas/test/data/AST3.java create mode 100644 KB/src-rascal/org/maracas/test/data/AST4.java create mode 100644 KB/src-rascal/org/maracas/test/data/AST5.java create mode 100644 KB/src-rascal/org/maracas/test/data/ast1CanonStr.bin create mode 100644 KB/src-rascal/org/maracas/test/data/ast1Descendants.bin create mode 100644 KB/src-rascal/org/maracas/test/data/ast2CanonStr.bin create mode 100644 KB/src-rascal/org/maracas/test/data/ast2Descendants.bin create mode 100644 KB/src-rascal/org/maracas/test/data/ast3CanonStr.bin create mode 100644 KB/src-rascal/org/maracas/test/data/ast3Descendants.bin create mode 100644 KB/src-rascal/org/maracas/test/data/ast4CanonStr.bin create mode 100644 KB/src-rascal/org/maracas/test/data/ast4Descendants.bin create mode 100644 KB/src-rascal/org/maracas/test/data/minimalbc-client.1.0.jar create mode 100644 KB/src-rascal/org/maracas/test/data/minimalbc.1.0.jar create mode 100644 KB/src-rascal/org/maracas/test/data/minimalbc.1.1.jar create mode 100644 KB/src-rascal/org/maracas/test/delta/DeltaBuilderTest.rsc create mode 100644 KB/src-rascal/org/maracas/test/delta/DetectorTest.rsc create mode 100644 KB/src-rascal/org/maracas/test/delta/TestApiOldApiNew.rsc create mode 100644 KB/src-rascal/org/maracas/test/input/ColoredTreeTestInput.rsc create mode 100644 KB/src-rascal/org/maracas/test/m3/CoreTest.rsc create mode 100644 KB/src-rascal/org/maracas/test/match/GumTreeMatcherTest.rsc create mode 100644 KB/src-rascal/org/maracas/test/match/TreeTest.rsc create mode 100644 KB/src-rascal/org/maracas/test/util/MathTest.rsc create mode 100644 KB/src-rascal/org/maracas/util/Math.rsc diff --git a/KB-db/Dockerfile b/KB-db/Dockerfile index 51942ad..2db1ce8 100644 --- a/KB-db/Dockerfile +++ b/KB-db/Dockerfile @@ -11,8 +11,8 @@ FROM mongo:3.4 # Get download dump archive -ADD http://ci3.castalia.camp/dl/M30/KB_CROSSMINER.gz /home -RUN mv /home/KB_CROSSMINER.gz /home/CROSSMINER.gz +ADD http://ci3.castalia.camp/dl/CROSSMINER.201907313.gz /home +RUN mv /home/CROSSMINER.201907313.gz /home/CROSSMINER.gz WORKDIR home diff --git a/KB/Dockerfile b/KB/Dockerfile index e658a1d..b5df05d 100644 --- a/KB/Dockerfile +++ b/KB/Dockerfile @@ -7,11 +7,11 @@ LABEL description="Image with the Knowledge base image" WORKDIR /home/root COPY application.properties application.properties - +COPY src-rascal/ /home/root # Downloads last jar build by the continuous integration server RUN wget http://ci5.castalia.camp:8080/job/scava-knowledge-base/job/dev/lastSuccessfulBuild/artifact/knowledge-base/org.eclipse.scava.knowledgebase/target/scava.knoledgebase.jar -ADD http://ci3.castalia.camp/dl/index.tar.gz /home/root/ -RUN tar -xzf /home/root/index.tar.gz +ADD http://ci3.castalia.camp/dl/Indexes.tar.gz /home/root/ +RUN tar -xzf /home/root/Indexes.tar.gz # Update the jar file with the Docker specific configurions RUN jar -uf scava.knoledgebase.jar application.properties diff --git a/KB/application.properties b/KB/application.properties index 112f6da..efd8c30 100644 --- a/KB/application.properties +++ b/KB/application.properties @@ -31,4 +31,19 @@ sorecommender.questionBoostValue = 0.5 sorecommender.answerBoostValue = 0.5 sorecommender.importerBoostLibrariesPath = libraries.txt versionrecommender.releasefile = release_all.csv -migration.maven.path = "/User/juri/Desktop/juri/" \ No newline at end of file +migration.maven.path = /home/root/maven_repo/ +CROSSIndexRecommender.hitsCodeBasedQuery=20 +CROSSIndexRecommender.hitsNLBasedQuery=20 +CROSSIndexRecommender.fieldForCodeBasedQuery="plain_text" +CROSSIndexRecommender.fieldForNLBasedQuery="plain_text" +CROSSIndexRecommender.fieldRecommendation="plain_text" +migration.deltas.path = /home/root/deltas/ +migration.maven.max.training.clients = 10 +crossindexrecommender.hitsCodeBasedQuery=20 +crossindexrecommender.hitsNLBasedQuery=20 +crossindexrecommender.fieldForCodeBasedQuery=plain_text +crossindexrecommender.fieldForNLBasedQuery=plain_text +crossindexrecommender.fieldRecommendation=plain_text +crossindexrecommender.elasticsearch.hostname=elasticsearch +crossindexrecommender.elasticsearch.scheme=https +crossindexrecommender.elasticsearch.port=9200 \ No newline at end of file diff --git a/KB/src-rascal/org/maracas/Extractor.rsc b/KB/src-rascal/org/maracas/Extractor.rsc new file mode 100644 index 0000000..9fefd90 --- /dev/null +++ b/KB/src-rascal/org/maracas/Extractor.rsc @@ -0,0 +1,143 @@ +module org::maracas::Extractor +import lang::java::m3::AST; +import lang::java::m3::Core; +import analysis::m3::TypeSymbol; + +import org::maracas::Maracas; +import org::maracas::RunAll; +import org::maracas::delta::Detector; +import org::maracas::delta::Delta; +import org::maracas::delta::vis::Visualizer; +import ValueIO; +import ParseTree; + +import IO; +import Relation; +import Set; +import String; +import List; + +void methodBreakingChangesExporter(loc delta, loc report=|file:///Users/juri/Desktop/report_spring_data_mongo/|, loc output=|file:///Users/juri/Desktop/rascal-TEST.txt|) { + Delta d = parseDeltaFile(delta); + Delta mbc = methodDelta(d); + writeHtml(report + "Delta.html", d); + writeFile(|file:///Users/juri/Desktop/delta.txt|,mbc); + writeFile(output, ""); + println("________REMOVED____________"); + for(m <- mbc.removed){ + appendToFile(output,"#null\n"); + println("."); + } + println("________CHANGE PUBLIC TO PRIVATE________"); + for(m<- mbc.accessModifiers){ + if(m.mapping[0]==\public()) + appendToFile(output,"#null\n"); + println("."); + } + println("_______RENAMED____________"); + for(m <- mbc.renamed){ + appendToFile(output,"#\n"); + println("."); + } + println("_______MOVED_________"); + for(m <- mbc.moved){ + appendToFile(output,"#\n"); + println("."); + } + println("_______CHANGE PARAM LIST_________"); + for(m <- mbc.paramLists){ + appendToFile(output,")#)\n"); + println("."); + } + println("_______CHANGE RETURN TYPE_________"); + for(m <- mbc.types){ + appendToFile(output,")#)\n"); + println("."); + } + for(m <- mbc.deprecated) { + appendToFile(output,"#null\n"); + println("."); + } +} + +void myRunAll(loc libs=|file:///Users/ochoa/Documents/cwi/crossminer/data/api-migration-dataset/sonarqube/erase/|, loc clients=|file:///Users/ochoa/Documents/cwi/crossminer/data/api-migration-dataset/sonarqube/clients/|,loc output=|file:///Users/ochoa/Desktop/results.txt|){ + + set[loc] libraries = walkJARs(libs); + for (lib1 <- libraries){ + for (lib2 <- libraries) + if (lib1 != lib2){ + try { + str v1 = lib1.path[findLast(lib1.path,"/")+1..-4]; + v1 = replaceLast(v1, "sonar-plugin-api-","sonar-plugin-api__"); + loc clients_loc = clients + v1; + if(exists(clients_loc)){ + println("Computing Delta model "); + + Delta d; + if(!exists(libs + "Delta" + lib1.file + "-" + lib2.file + ".delta")) + d = delta(lib1, lib2); + else + d = parseDeltaFile(clients_loc); + println("Pruning breaking changes..."); + d = breakingChanges(d); + + set[loc] clients = walkJARs(clients_loc); + int i = 0; + int count = size(clients); + + for (client <- clients) { + try { + i = i + 1; + println("[/] Computing detection model for ... "); + + M3 m = createM3FromJar(client); + set[Detection] detects = detections(m, d); + + if (size(detects) > 0) + writeBinaryValueFile(clients_loc + "detection" + lib1.file + "-" + lib2.file + (client.file + ".detection"), detects); + writeFile(output, ",,, "); + } + catch e: + writeFile(output, ",,,-1 \n \n \n"); + + } + } + } + catch e: + writeFile(output, ",,-1,-1 \n \n \n"); + } + } +} + + +void allVersionsDeltas(loc libs=|file:///Users/juri/development/git/aethereal/aethereal/dataset/libraries/|, loc report=|file:///Users/juri/Desktop/report_spring_data_mongo/|, loc output=|file:///Users/juri/Desktop/rascal-TEST.txt|){ + set[loc] libraries = walkJARs(libs); + for (lib1 <- libraries){ + for (lib2 <- libraries) + if(lib1 != lib2) + try{ + println("Computing: "); + methodBreakingChangesExporter(lib1, lib2); + } + catch: + println("\t###ERROR: "); + } +} + + +void methodBreakingChangesExporter(loc libV1, loc libV2, loc report=|file:///Users/juri/Desktop/report_sonarcube/|, loc output=|file:///Users/juri/Desktop/rascal-TEST.txt|) { + println(report + ("Delta" + libV1.file + "-" + libV2.file + ".delta")); + if(!exists(report + ("Delta" + libV1.file + "-" + libV2.file + ".delta"))){ + Delta d = delta(libV1, libV2); + writeFile(|file:///Users/juri/Desktop/delta.txt|,d); + writeFile(output, ""); + writeBinaryValueFile(report + ("Delta" + libV1.file + "-" + libV2.file + ".delta"), d); + writeHtml(report + ("Delta" + libV1.file + "-" + libV2.file + ".html"), d); + } +} + +Delta parseDeltaFile(loc report) { + loc entry = report; + Delta deltas = readBinaryValueFile(#Delta, entry); + return deltas; +} \ No newline at end of file diff --git a/KB/src-rascal/org/maracas/Maracas.java b/KB/src-rascal/org/maracas/Maracas.java new file mode 100644 index 0000000..59983ae --- /dev/null +++ b/KB/src-rascal/org/maracas/Maracas.java @@ -0,0 +1,390 @@ +package org.maracas; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintWriter; +import java.net.URISyntaxException; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Enumeration; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarInputStream; + +import org.maracas.data.Detection; +import org.rascalmpl.debug.IRascalMonitor; +import org.rascalmpl.interpreter.Evaluator; +import org.rascalmpl.interpreter.NullRascalMonitor; +import org.rascalmpl.interpreter.env.GlobalEnvironment; +import org.rascalmpl.interpreter.env.ModuleEnvironment; +import org.rascalmpl.interpreter.load.StandardLibraryContributor; +import org.rascalmpl.values.ValueFactoryFactory; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; + +import io.usethesource.vallang.IBool; +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.ISet; +import io.usethesource.vallang.ISourceLocation; +import io.usethesource.vallang.IString; +import io.usethesource.vallang.ITuple; +import io.usethesource.vallang.IValueFactory; + +public class Maracas { + private IValueFactory vf = ValueFactoryFactory.getValueFactory(); + private Evaluator evaluator = createRascalEvaluator(vf); + private IRascalMonitor mon = new NullRascalMonitor(); + + /** + * Run the full Maracas pipeline (build M3 models -> build Delta model + * -> build Detections model), cf. RunAll.rsc. Output of the analysis is + * serialized in {@code report}. + * + * @param libPath1 Absolute path to the JAR of the 1st version of the library + * @param libPath2 Absolute path to the JAR of the 2nd version of the library + * @param clients Absolute path to the directory containing all clients of + * libPath1 + * @param report Absolute path to the output directory where analysis results + * are serialized + */ + public void runAll(String libPath1, String libPath2, String clients, String report) { + ISourceLocation loc1 = vf.sourceLocation(libPath1); + ISourceLocation loc2 = vf.sourceLocation(libPath2); + ISourceLocation locClients = vf.sourceLocation(clients); + ISourceLocation locReport = vf.sourceLocation(report); + + evaluator.call("runAll", loc1, loc2, locClients, locReport, vf.bool(true), vf.bool(true)); + } + + /** + * Parses the analysis results to map to each client JAR of a library its + * detection models + * + * @param Absolute path to the output directory where analysis results of + * {@code runAll} were serialized. + * @return A Multimap associating to each client JAR the detections that were + * found + */ + public Multimap parseDetections(String report) { + Multimap result = ArrayListMultimap.create(); + ISourceLocation locReport = vf.sourceLocation(report); + + ISet allDetections = (ISet) evaluator.call("parseDetectionFiles", locReport); + allDetections.forEach(e -> { + // tuple[str, set[Detection]] + ITuple t = (ITuple) e; + String jar = ((IString) t.get(0)).toString(); + ISet detections = (ISet) t.get(1); + + detections.forEach(d -> { + result.put(jar, Detection.fromRascalDetection((IConstructor) d)); + }); + }); + + return result; + } + + /** + * The method computes a M3 model from a JAR file and stores the model in a given + * path. + * + * @param pathJar: absolute path to the JAR file of the project + * @param pathM3: absolute path to the file where the M3 model should be stored + * (including the name of the file and its extension --.m3--) + * @return true if the M3 model was correctly computed and stored in the given + * location; false otherwise + */ + public boolean storeM3(String pathJar, String pathM3) { + ISourceLocation locJar = vf.sourceLocation(pathJar); + ISourceLocation locM3 = vf.sourceLocation(pathM3); + + IBool store = (IBool) evaluator.call("storeM3", locJar, locM3); + return store.getValue(); + } + + /** + * The method computes a M3 model from a JAR file and stores the model in a given + * path. + * + * @param locDir: absolute path to the project source code + * @param pathM3: absolute path to the file where the M3 model should be stored + * (including the name of the file and its extension --.m3--) + * @return true if the M3 model was correctly computed and stored in the given + * location; false otherwise + */ + public boolean storeM3FromDir(String pathDir, String pathM3) { + ISourceLocation locDir = vf.sourceLocation(pathDir); + ISourceLocation locM3 = vf.sourceLocation(pathM3); + + IBool store = (IBool) evaluator.call("storeM3FromDir", locDir, locM3); + return store.getValue(); + } + + /** + * The method computes a Delta model between the old and new M3 models of a + * library. Then, it stores the model in a given path. + * + * @param pathM3OldAPI: absolute path to the M3 model of the library old version + * @param pathM3NewAPI: absolute path to the M3 model of the library new version + * @param pathDelta: absolute path to the file where the Delta model should be + * stored (including the name of the file and its extension --.delta--) + * @return true if the Delta model was correctly computed and stored in the + * given location; false otherwise + */ + public boolean storeDelta(String pathM3OldAPI, String pathM3NewAPI, String pathDelta) { + ISourceLocation locM3OldAPI = vf.sourceLocation(pathM3OldAPI); + ISourceLocation locM3NewAPI = vf.sourceLocation(pathM3NewAPI); + ISourceLocation locDelta = vf.sourceLocation(pathDelta); + + IBool store = (IBool) evaluator.call("storeDelta", locM3OldAPI, locM3NewAPI, locDelta); + return store.getValue(); + } + + /** + * The method returns a list of Detections from a client M3 model and a library + * Delta model. + * + * @param pathM3Client: absolute path to the M3 model of a client project + * @param pathDelta: absolute path to the file where of the library Delta model + * @return list of Detections + */ + public List getDetections(String pathM3Client, String pathDelta) { + ISourceLocation locM3Client = vf.sourceLocation(pathM3Client); + ISourceLocation locDelta = vf.sourceLocation(pathDelta); + List detections = new ArrayList(); + + ISet allDetections = (ISet) evaluator.call("getDetections", locM3Client, locDelta); + allDetections.forEach(d -> { + detections.add(Detection.fromRascalDetection((IConstructor) d)); + }); + return detections; + } + + /** + * The method returns a string with the code of the declaration from the + * path of a previously stored M3 model and a string representing the + * Rascal logical location of the declaration. The M3 model must be created + * from a source directory, not from a JAR file. + * + * @param pathM3FromDir: absolute path to the M3 model of a project + * @param decl: Rascal logical location of the declaration (e.g. + * “|java+method:///com/google/common/primitives/UnsignedLongs/toString(long)|”) + * @return string with the source code of the declaration + */ + public String getCodeFromM3(String pathM3FromDir, String decl) { + try { + ISourceLocation locDecl = getDeclSourceLocation(decl); + ISourceLocation locM3FromDir = vf.sourceLocation(pathM3FromDir); + + IString source = (IString) evaluator.call("getCodeFromM3", locM3FromDir, locDecl); + return source.getValue(); + } + catch (URISyntaxException e) { + e.printStackTrace(); + } + return ""; + } + + /** + * The method returns a string with the code of the declaration from + * the absolute path of a source directory and a string representing + * the Rascal logical location of the declaration. + * + * @param pathSourceDir: absolute path to the source directory of a project + * @param decl: Rascal logical location of the declaration (e.g. + * “|java+method:///com/google/common/primitives/UnsignedLongs/toString(long)|”) + * @return string with the source code of the declaration + */ + public String getCodeFromSourceDir(String pathSourceDir, String decl) { + try { + ISourceLocation locDecl = getDeclSourceLocation(decl); + ISourceLocation locSourceDir = vf.sourceLocation(pathSourceDir); + + IString source = (IString) evaluator.call("getCodeFromSourceDir", locSourceDir, locDecl); + return source.getValue(); + } + catch (URISyntaxException e) { + e.printStackTrace(); + } + return ""; + } + + /** + * The method returns a string with the code of the declaration from the + * absolute path of a JAR sources file, a destination path where unzipped + * files must be copied, and a string representing the Rascal logical + * location of the declaration. + * + * @param pathSourceJar: absolute path to the JAR sources file + * @param sourceDir: absolute path to the directory where the source files + * must be copied + * @param decl: Rascal logical location of the declaration (e.g. + * “|java+method:///com/google/common/primitives/UnsignedLongs/toString(long)|”) + * @return string with the source code of the declaration + */ + public String getCodeFromSourceJar(String pathSourceJar, String sourceDir, String decl) { + try { + ISourceLocation locDecl = getDeclSourceLocation(decl); + ISourceLocation locSourceDir = vf.sourceLocation(sourceDir); + + boolean unzipped = unzipJar(pathSourceJar, sourceDir); + if (unzipped) { + IString source = (IString) evaluator.call("getCodeFromSourceDir", locSourceDir, locDecl); + return source.getValue(); + } + } + catch (URISyntaxException e) { + e.printStackTrace(); + } + return ""; + } + + /** + * Unzips a JAR file given the absolute path to the JAR and the directory + * where content should be stracted. + * + * @param pathJar: absolute path to JAR file + * @param pathDir: absolute path to directory where JAR content should be + * extracted + * @return true if the JAR file was successfully unzziped; false otherwise. + */ + public boolean unzipJar(String pathJar, String pathDir) { + try { + File file = new File(File.separator + pathJar); + JarFile fileJar = new JarFile(file); + Enumeration entries = fileJar.entries(); + + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + File fileEntry = new File(File.separator + pathDir + File.separator + entry.getName()); + + if (entry.isDirectory()) { + fileEntry.mkdirs(); + } + else { + InputStream is = fileJar.getInputStream(entry); + FileOutputStream os = new FileOutputStream(fileEntry); + + while (is.available() > 0) { + os.write(is.read()); + } + + os.close(); + is.close(); + } + } + + fileJar.close(); + return true; + } + catch (IOException e) { + e.printStackTrace(); + } + + return false; + } + + private boolean unzipJarAlt(String pathJar, String pathDir) { + try { + FileInputStream fileStream = new FileInputStream(File.separator + pathJar); + JarInputStream jarStream = new JarInputStream(fileStream); + JarEntry entry = jarStream.getNextJarEntry(); + + while (entry != null) { + File fileEntry = new File(File.separator + pathDir + File.separator + entry.getName()); + + if (entry.isDirectory()) { + fileEntry.mkdirs(); + } + else { + FileOutputStream os = new FileOutputStream(fileEntry); + + while (jarStream.available() > 0) { + os.write(jarStream.read()); + } + + os.close(); + } + entry = jarStream.getNextJarEntry(); + } + + jarStream.close(); + return true; + } + catch (IOException e) { + e.printStackTrace(); + } + + return false; + } + + private ISourceLocation getDeclSourceLocation(String decl) throws URISyntaxException { + String scheme = getScheme(decl); + String path = getPath(decl); + ISourceLocation locDecl = vf.sourceLocation(scheme, "", path); + return locDecl; + } + + private String getScheme(String rascalLoc) { + int endIndex = rascalLoc.indexOf(":"); + return rascalLoc.substring(1, endIndex); + } + + private String getPath(String rascalLoc) { + int beginIndex = rascalLoc.indexOf("///") + 3; + int endIndex = rascalLoc.length() - 1; + return rascalLoc.substring(beginIndex, endIndex); + } + + public static void main(String[] args) { + if (args.length != 4) { + System.err.println("Usage: maracas "); + return; + } + + Maracas m = new Maracas(); + String lib1 = args[0]; + String lib2 = args[1]; + String clients = args[2]; + String report = args[3]; + + // Build BreakingChanges/Detections models in 'report' + m.runAll(lib1, lib2, clients, report); + + // Parse the Detections models and build the Multimap + Multimap detections = m.parseDetections(report + "/detection"); + System.out.println(detections.size() + " usages found."); + + // Example usage: list usages of @Deprecated methods + for (String client : detections.keySet()) { + Collection clientDetections = detections.get(client); + + clientDetections.stream().filter(d -> d.getType().equals(Detection.Type.DEPRECATED)).forEach(d -> { + System.out.println(String.format("%s uses method %s which has been deprecated", d.getClientLocation(), + d.getOldLibraryLocation())); + }); + } + } + + private Evaluator createRascalEvaluator(IValueFactory vf) { + GlobalEnvironment heap = new GlobalEnvironment(); + ModuleEnvironment module = new ModuleEnvironment("$maracas$", heap); + PrintWriter stderr = new PrintWriter(System.err); + PrintWriter stdout = new PrintWriter(System.out); + Evaluator eval = new Evaluator(vf, stderr, stdout, module, heap); + + eval.addRascalSearchPathContributor(StandardLibraryContributor.getInstance()); + //eval.addRascalSearchPath(vf.sourceLocation( + // Paths.get("/Users/juri/development/git/maracas/maracas/" + "src").toAbsolutePath().toString())); + eval.addRascalSearchPath(vf.sourceLocation(Paths.get("src-rascal").toAbsolutePath().toString())); + eval.doImport(mon, "org::maracas::RunAll"); + + return eval; + } +} diff --git a/KB/src-rascal/org/maracas/Maracas.rsc b/KB/src-rascal/org/maracas/Maracas.rsc new file mode 100644 index 0000000..5facbca --- /dev/null +++ b/KB/src-rascal/org/maracas/Maracas.rsc @@ -0,0 +1,34 @@ +module org::maracas::Maracas + +import IO; +import lang::java::m3::Core; +import org::maracas::delta::Delta; +import org::maracas::delta::DeltaBuilder; +import org::maracas::delta::Detector; +import org::maracas::delta::Migration; +import ValueIO; + + +Delta delta(loc oldAPI, loc newAPI) + = createDelta(createM3FromJar(oldAPI), createM3FromJar(newAPI)); + +Delta delta(M3 oldAPI, M3 newAPI) + = createDelta(oldAPI, newAPI); + +Delta classDelta(Delta delta) + = getClassDelta(delta); + +Delta methodDelta(Delta delta) + = getMethodDelta(delta); + +Delta fieldDelta(Delta delta) + = getFieldDelta(delta); + +set[Detection] detections(loc oldClient, Delta delta) + = detections(createM3FromJar(oldClient), delta); + +set[Detection] detections(M3 oldClient, Delta delta) + = detections(oldClient, delta); + +set[Migration] migrations(loc newClient, set[Detection] detects) + = migrations(newClient, detects); \ No newline at end of file diff --git a/KB/src-rascal/org/maracas/RunAll.rsc b/KB/src-rascal/org/maracas/RunAll.rsc new file mode 100644 index 0000000..a69a706 --- /dev/null +++ b/KB/src-rascal/org/maracas/RunAll.rsc @@ -0,0 +1,171 @@ +/* + * Temporary, for FOCUS purposes. + */ +module org::maracas::RunAll + +import IO; +import ValueIO; +import String; +import Set; +import org::maracas::Maracas; +import org::maracas::delta::Delta; +import org::maracas::delta::Migration; +import org::maracas::delta::vis::Visualizer; +import lang::java::m3::Core; +import org::maracas::delta::Detector; + +void runAll(loc libv1, loc libv2, loc clients, loc report, bool serializeDelta, bool serializeHtml) { + set[loc] clients = walkJARs(clients); + int count = size(clients); + + println("Computing Delta model..."); + Delta d = delta(libv1, libv2); + + println("Pruning breaking changes..."); + d = breakingChanges(d); + + if (serializeDelta) { + println("Serializing Delta model..."); + writeBinaryValueFile(report + "Delta.delta", d); + } + + if (serializeHtml) { + println("Serializing HTML report..."); + writeHtml(report + "Delta.html", d); + } + + int i = 0; + for (client <- clients) { + i = i + 1; + println("[/] Computing detection model for ... "); + + M3 m3 = createM3FromJar(client); + set[Detection] detects = detections(m3, d); + + if (size(detects) > 0) + writeBinaryValueFile(report + "detection" + (client.file + ".detection"), detects); + } +} + +void runAllMigrations(loc report, loc clients) { + set[Detection] ds = {}; + + for (str e <- listEntries(report + "detection")) { + //ds += { d | d <- readBinaryValueFile(#set[Detection], report + "detection" + e), + // d.typ != removed() }; // Not interested in those ones, yet + ds += readBinaryValueFile(#set[Detection], report + "detection" + e); + } + + println("Found detection models"); + + set[Migration] ms = buildMigrations(ds, clients); + + println("Found migrations"); + + writeBinaryValueFile(report + "Migrations.migration", ms); + + println("HTML report..."); + + writeHtml(report + "Migrations.html", ms); +} + +rel[str, set[Detection]] parseDetectionFiles(loc report) { + rel[str, set[Detection]] result = {}; + + for (e <- listEntries(report), endsWith(e, ".detection")) { + if (/^\.jar\.detection$/ := e) { + loc entry = report + e; + set[Detection] detects = readBinaryValueFile(#set[Detection], entry); + result[name] += detects; + } else + throw "Shouldn\'t be here"; + } + + return result; +} + +Delta parseDeltaFile(loc report) { + loc d = report + "Delta.delta"; + + if (isFile(d)) + return readBinaryValueFile(#Delta, d); + else throw " not found"; +} + +set[loc] walkJARs(loc dataset) { + set[loc] result = {}; + + for (e <- listEntries(dataset)) { + loc entry = dataset + e; + + if (isDirectory(entry)) + result += walkJARs(entry); + else if (endsWith(e, ".jar")) + result += entry; + }; + + return result; +} + +set[Detection] getDetections(loc oldClient, loc delt) { + M3 clientM3 = readM3(oldClient); + Delta delta = readBinaryValueFile(#Delta, delt); + return detections(clientM3, delta); +} + +bool storeDelta(loc m3OldAPI, loc m3NewAPI, loc delt) { + try { + M3 oldM3 = readM3(m3OldAPI); + M3 newM3 = readM3(m3NewAPI); + + Delta d = delta(oldM3, newM3); + writeBinaryValueFile(delt, d); + return true; + } + catch : + return false; +} + +bool storeM3(loc projectJar, loc projectM3) { + try { + M3 m = createM3FromJar(projectJar); + writeBinaryValueFile(projectM3, m); + return true; + } + catch e : + println(e); + return false; +} + +bool storeM3FromDir(loc projectDir, loc projectM3) { + try { + M3 m = createM3FromDirectory(projectDir); + writeBinaryValueFile(projectM3, m); + return true; + } + catch e : + println(e); + return false; +} + +str getCodeFromM3(loc m3FromDir, loc decl) { + M3 m = readM3(m3FromDir); + return getDeclLoc(m, decl); +} + +str getCodeFromSourceDir(loc sourceDir, loc decl) { + M3 m = createM3FromDirectory(sourceDir); + return getDeclLoc(m, decl); +} + +private str getDeclLoc(M3 m, loc decl) { + set[loc] physical = m.declarations[decl]; + + if (physical != {}) { + return readFile(getOneFrom(physical)); + } + return ""; +} + +private M3 readM3(loc m3Path) + = readBinaryValueFile(#M3, m3Path); \ No newline at end of file diff --git a/KB/src-rascal/org/maracas/Test.rsc b/KB/src-rascal/org/maracas/Test.rsc new file mode 100644 index 0000000..8b93cf9 --- /dev/null +++ b/KB/src-rascal/org/maracas/Test.rsc @@ -0,0 +1,5 @@ +module org::maracas::Test + +void getM3(loc m3Loc){ + +} diff --git a/KB/src-rascal/org/maracas/config/Options.rsc b/KB/src-rascal/org/maracas/config/Options.rsc new file mode 100644 index 0000000..79ec80a --- /dev/null +++ b/KB/src-rascal/org/maracas/config/Options.rsc @@ -0,0 +1,21 @@ +module org::maracas::config::Options + + +// ------------------------------- +// Keys +// ------------------------------- + +public str DEP_MATCHES_LOAD = "deprecatedMatchesLoad"; +public str DEP_MATCHES_LOC = "deprecatedMatchesLoc"; +public str DEP_MATCHERS = "deprecatedMatchers"; +public str MATCHERS = "matchers"; + + +// ------------------------------- +// Values +// ------------------------------- + +public str MATCH_LOAD = "load"; +public str MATCH_SIGNATURE = "signature"; +public str MATCH_LEVENSHTEIN = "levenshtein"; +public str MATCH_JACCARD = "jaccard"; \ No newline at end of file diff --git a/KB/src-rascal/org/maracas/data/Detection.java b/KB/src-rascal/org/maracas/data/Detection.java new file mode 100644 index 0000000..282864b --- /dev/null +++ b/KB/src-rascal/org/maracas/data/Detection.java @@ -0,0 +1,150 @@ +package org.maracas.data; + +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.IReal; +import io.usethesource.vallang.ISourceLocation; +import io.usethesource.vallang.ITuple; +import io.usethesource.vallang.IValue; + +public class Detection { + public enum Type { + ACCESS_MODIFIER, FINAL_MODIFIER, STATIC_MODIFIER, ABSTRACT_MODIFIER, DEPRECATED, RENAMED, MOVED, REMOVED, + PARAMS_LIST, RETURN_TYPE, TYPE, EXTENDS, IMPLEMENTS, ADDED + } + + private String clientLocation; + public void setClientLocation(String clientLocation) { + this.clientLocation = clientLocation; + } + + + public void setOldLibraryLocation(String oldLibraryLocation) { + this.oldLibraryLocation = oldLibraryLocation; + } + + + public void setNewLibraryLocation(String newLibraryLocation) { + this.newLibraryLocation = newLibraryLocation; + } + + + public void setType(Type type) { + this.type = type; + } + + + public void setScore(double score) { + this.score = score; + } + + private String oldLibraryLocation; + private String newLibraryLocation; + private Type type; + private double score; + + public Detection(String clientLocation, String oldLibraryLocation, String newLibraryLocation, Type type, double score) { + this.clientLocation = clientLocation; + this.oldLibraryLocation = oldLibraryLocation; + this.newLibraryLocation = newLibraryLocation; + this.type = type; + this.score = score; + } + + + public static Detection fromRascalDetection(IConstructor detection) { + // detection(loc elem, loc used, tuple[&T from, &T to, real conf, str m], BCTyp + // typ) + String client = ((ISourceLocation) detection.get(1)).toString(); + String oldLibrary = ((ISourceLocation) detection.get(2)).toString(); + String newLibrary = ""; + ITuple mapping = (ITuple) detection.get(3); + IValue to = mapping.get(1); + double score = ((IReal) mapping.get(2)).doubleValue(); + IConstructor bcType = (IConstructor) detection.get(4); + + Type type; + switch (bcType.getName()) { + case "accessModifiers": + type = Type.ACCESS_MODIFIER; + newLibrary = oldLibrary; + break; + case "finalModifiers": + type = Type.FINAL_MODIFIER; + newLibrary = oldLibrary; + break; + case "staticModifiers": + type = Type.STATIC_MODIFIER; + newLibrary = oldLibrary; + break; + case "abstractModifiers": + type = Type.ABSTRACT_MODIFIER; + newLibrary = oldLibrary; + break; + case "deprecated": + type = Type.DEPRECATED; + newLibrary = ((ISourceLocation) to).toString(); + break; + case "renamed": + type = Type.RENAMED; + newLibrary = ((ISourceLocation) to).toString(); + break; + case "moved": + type = Type.MOVED; + newLibrary = ((ISourceLocation) to).toString(); + break; + case "removed": + type = Type.REMOVED; + break; + case "paramLists": + type = Type.PARAMS_LIST; + // Compute it + break; + case "types": + type = Type.TYPE; + newLibrary = oldLibrary; + break; + case "extends": + type = Type.EXTENDS; + newLibrary = oldLibrary; + break; + case "implements": + type = Type.IMPLEMENTS; + newLibrary = oldLibrary; + break; + case "added": + type = Type.ADDED; + newLibrary = newLibrary; + break; + default: + throw new RuntimeException("Unexpected BCType"); + } + + return new Detection(client, oldLibrary, newLibrary, type, score); + } + + public String getClientLocation() { + return clientLocation; + } + + public String getOldLibraryLocation() { + return oldLibraryLocation; + } + + public String getNewLibraryLocation() { + return newLibraryLocation; + } + + public Type getType() { + return type; + } + + @Override + public String toString() { + return String.format("[%s] %s uses %s, replaced with %s [%.2f]", type, clientLocation, oldLibraryLocation, + newLibraryLocation, getScore()); + } + + public double getScore() { + return score; + } +} diff --git a/KB/src-rascal/org/maracas/delta/Delta.rsc b/KB/src-rascal/org/maracas/delta/Delta.rsc new file mode 100644 index 0000000..0f8e552 --- /dev/null +++ b/KB/src-rascal/org/maracas/delta/Delta.rsc @@ -0,0 +1,121 @@ +module org::maracas::delta::Delta + +import lang::java::m3::AST; +import lang::java::m3::Core; +import org::maracas::m3::Core; +import Relation; +import Set; + +data Delta ( + rel[loc elem, Mapping[Modifier] mapping] accessModifiers = {}, + rel[loc elem, Mapping[Modifier] mapping] finalModifiers = {}, + rel[loc elem, Mapping[Modifier] mapping] staticModifiers = {}, + rel[loc elem, Mapping[Modifier] mapping] abstractModifiers = {}, + rel[loc elem, Mapping[list[TypeSymbol]] mapping] paramLists = {}, + rel[loc elem, Mapping[TypeSymbol] mapping] types = {}, + rel[loc elem, Mapping[loc] mapping] extends = {}, + rel[loc elem, Mapping[set[loc]] mapping] implements = {}, + rel[loc elem, Mapping[loc] mapping] deprecated = {}, + rel[loc elem, Mapping[loc] mapping] renamed = {}, + rel[loc elem, Mapping[loc] mapping] moved = {}, + rel[loc elem, Mapping[loc] mapping] removed = {}, + rel[loc elem, Mapping[loc] mapping] added = {}, + map[str, str] options = ()) + = delta(tuple[loc from, loc to] id); + +alias Mapping[&T] + = tuple [ + &T from, + &T to, + real conf, + str method + ]; + +Delta getClassDelta(Delta delta) + = getFilteredDelta(delta, isType); + +Delta getMethodDelta(Delta delta) + = getFilteredDelta(delta, isMethod); + +Delta getFieldDelta(Delta delta) + = getFilteredDelta(delta, isField); + +private Delta getFilteredDelta(Delta delta, bool (loc) fun) { + delta.accessModifiers = { m | m <- delta.accessModifiers, fun(m.elem) }; + delta.finalModifiers = { m | m <- delta.finalModifiers, fun(m.elem) }; + delta.staticModifiers = { m | m <- delta.staticModifiers, fun(m.elem) }; + delta.abstractModifiers = { m | m <- delta.abstractModifiers, fun(m.elem) }; + delta.paramLists = { m | m <- delta.paramLists, fun(m.elem) }; + delta.types = { m | m <- delta.types, fun(m.elem) }; + delta.extends = { m | m <- delta.extends, fun(m.elem) }; + delta.implements = { m | m <- delta.implements, fun(m.elem) }; + delta.deprecated = { m | m <- delta.deprecated, fun(m.elem) }; + delta.renamed = { m | m <- delta.renamed, fun(m.elem) }; + delta.moved = { m | m <- delta.moved, fun(m.elem) }; + delta.removed = { m | m <- delta.removed, fun(m.elem) }; + delta.added = { m | m <- delta.added, fun(m.elem) }; + + return delta; +} + +tuple[loc, Mapping[&T]] buildDeltaMapping(loc elem, &T from, &T to, real score, str meth) + = ; + +tuple[loc, Mapping[&T]] buildDeltaMapping(loc elem, Mapping[&T] mapping) + = ; + +Mapping[&T] buildMapping(&T from, &T to, real score, str meth) + = ; + +Delta breakingChanges(Delta delta) { + M3 m3from = createM3(delta.id.from); + M3 m3to = createM3(delta.id.to); + + delta.accessModifiers = { | <- delta.accessModifiers, isAPI(m3from, e), !isAPI(m3to, e) }; + // Only problematic when going from static to non-static + delta.finalModifiers = { | <- delta.finalModifiers, isAPI(m3from, e), m[1] == \final() }; + // Just a warning if miscalling a static method in a non-static way + delta.staticModifiers = { | <- delta.staticModifiers, isAPI(m3from, e), m[0] == \static() }; + delta.abstractModifiers = { | <- delta.abstractModifiers, isAPI(m3from, e), m[1] == \abstract() }; + // May not be BC do to subtyping, autoboxing, etc. + delta.paramLists = { | <- delta.paramLists, isAPI(m3from, e) }; + // May not be BC do to subtyping, autoboxing, etc. + delta.types = { | <- delta.types, isAPI(m3from, e) }; + // May not be BC if replaced by a subtype / other cases + delta.extends = { | <- delta.extends, isAPI(m3from, e) }; + // May not be BC if replaced by a subtype / other cases + delta.implements = { | <- delta.implements, isAPI(m3from, e) }; + // Included as BC for now, though not really + delta.deprecated = { | <- delta.deprecated, isAPI(m3from, e) }; + delta.renamed = { | <- delta.renamed, isAPI(m3from, e) }; + delta.moved = { | <- delta.moved, isAPI(m3from, e) }; + delta.removed = { | <- delta.removed, isAPI(m3from, e) }; + delta.added = { | <- delta.added, mayBeHookMethod(m3to, e) }; + + return delta; +} + +bool isAPI(M3 m, loc elem) { // FIXME: check the whole visibility path for nested elements + if (isType(elem) || isEnum(elem)) // public or default, ie. package-private + return \public() in m.modifiers[elem]; + if (isMethod(elem) || isField(elem)) // private, protected, public, or default + return {\public(), \protected()} & m.modifiers[elem] != {} + && isAPI(m, getOneFrom(invert(m.containment)[elem])); + return false; +} + +bool mayBeHookMethod(M3 m, loc elem) { + if (!isMethod(elem)) + return false; + + set[loc] types = invert(m.containment)[elem]; + if (types != {}) { + loc typ = getOneFrom(types); + return isAPI(m, typ) // The method belongs to a public type + && \abstract() in m.modifiers[elem]; // and the method is abstract + // (don't need other checks, since if the method + // is abstract, the type is too, hence it cannot + // be final) + } + return false; +} diff --git a/KB/src-rascal/org/maracas/delta/DeltaBuilder.rsc b/KB/src-rascal/org/maracas/delta/DeltaBuilder.rsc new file mode 100644 index 0000000..fcce8ad --- /dev/null +++ b/KB/src-rascal/org/maracas/delta/DeltaBuilder.rsc @@ -0,0 +1,340 @@ +module org::maracas::delta::DeltaBuilder + +import IO; +import Boolean; +import lang::java::m3::AST; +import lang::java::m3::Core; +import lang::java::m3::TypeSymbol; +import org::maracas::config::Options; +import org::maracas::delta::Delta; +import org::maracas::io::properties::IO; +import org::maracas::m3::Core; +import org::maracas::m3::M3Diff; +import org::maracas::match::matcher::Matcher; +import org::maracas::match::matcher::JaccardMatcher; +import org::maracas::match::matcher::LevenshteinMatcher; +import Relation; +import Set; +import String; + + +Delta createDelta(M3 from, M3 to, loc optionsFile = |project://maracas/config/config.properties|) { + M3Diff diff = createM3Diff(from, to); + Delta d = delta(); + + println("--------------------------"); + println("Computing options"); + //d.options = readProperties(optionsFile); + println("--------------------------"); + println("Computing accessModifiers"); + d.accessModifiers = accessModifiers(diff); + println("--------------------------"); + println("Computing finalModifiers"); + d.finalModifiers = finalModifiers(diff); + println("--------------------------"); + println("Computing staticModifiers"); + d.staticModifiers = staticModifiers(diff); + println("--------------------------"); + println("Computing abstractModifiers"); + d.abstractModifiers = abstractModifiers(diff); + println("--------------------------"); + println("Computing paramLists"); + d.paramLists = paramLists(diff); + println("--------------------------"); + println("Computing types"); + d.types = returnTypes(diff) + types(diff); + println("--------------------------"); + println("Computing extends"); + d.extends = extends(diff); + println("--------------------------"); + println("Computing implements"); + d.implements = implements(diff); + println("--------------------------"); + println("Computing deprecated"); + d.deprecated = deprecated(diff, d); + println("--------------------------"); + println("Computing renamed"); + d.renamed = renamed(diff, d); + println("--------------------------"); + println("Computing moved"); + d.moved = moved(diff, d); + println("--------------------------"); + println("Computing removed"); + d.removed = removed(diff, d); + println("--------------------------"); + println("Computing added"); + d.added = added(diff, d); + + return d; +} + + +/* + * Identifying changes in access modifiers + */ +private rel[loc, Mapping[Modifier]] accessModifiers(M3Diff diff) { + result = {}; + // The confidence of the mapping is 1 if the signature is the same + for ( <- diff.additions.modifiers, isTargetMember(e), isAccessModifier(addMod)) { + for (remMod <- diff.removals.modifiers[e], isAccessModifier(remMod)) { + result += buildDeltaMapping(e, remMod, addMod, 1.0, MATCH_SIGNATURE); + } + } + + return result; +} + +private bool isAccessModifier(Modifier m) { + accMods = { + \defaultAccess(), + \public(), + \private(), + \protected() + }; + return m in accMods; +} + +/* + * Identifying changes in final, static, and abstract modifiers + */ +private rel[loc, Mapping[Modifier]] finalModifiers(M3Diff diff) + = changedModifier(diff, \final()); + +private rel[loc, Mapping[Modifier]] staticModifiers(M3Diff diff) + = changedModifier(diff, \static()); + +private rel[loc, Mapping[Modifier]] abstractModifiers(M3Diff diff) + = changedModifier(diff, \abstract()); + +// The confidence of the mapping is 1 if the signature is the same +private rel[loc, Mapping[Modifier]] changedModifier(M3Diff diff, Modifier modifier) + = { buildDeltaMapping(elem, \default(), modif, 1.0, MATCH_SIGNATURE) + | <- diff.additions.modifiers, elem notin diff.addedDecls, isTargetMemberExclInterface(elem), modif := modifier } + + { buildDeltaMapping(elem, modif, \default(), 1.0, MATCH_SIGNATURE) + | <- diff.removals.modifiers, elem notin diff.removedDecls, isTargetMemberExclInterface(elem), modif := modifier }; + + +private rel[loc, Mapping[loc]] deprecated(M3Diff diff, Delta delta) { + load = DEP_MATCHES_LOAD in delta.options && fromString(delta.options[DEP_MATCHES_LOAD]); + + // TODO: check that we only load matches from a certain kind (i.e. class, method, field) + if (load) { + url = |file://| + delta.options[DEP_MATCHES_LOC]; + matches = loadMatches(url); + return { buildDeltaMapping(m.from, m) | m <- matches }; + } + + // For now, only mark @Deprecated elements + return { buildDeltaMapping(e, e, e, 1.0, MATCH_SIGNATURE) + | <- diff.additions.annotations, isTargetMember(e), + a == |java+interface:///java/lang/Deprecated|}; +} + + +/* + * Identifying renamed elements + */ +private rel[loc, Mapping[loc]] renamed(M3Diff diff, Delta delta) { + diff = filterDiffRenamed(diff, delta); + result = {}; + + for (elem <- diff.removedDecls, isTargetMember(elem)) { + invCont = invert(diff.from.containment); + + if (invCont[elem] != {}) { + // In type cases we need the owner package instead of its compilation unit. + cont = getNonCUContainer(elem, diff.from); + elemsSameCont = {}; + + for (a <- diff.to.containment[cont], a in diff.addedDecls) { + a = getNonCUChild(a, diff.to); + if (a.scheme == elem.scheme) { + elemsSameCont += a; + } + } + + if (elemsSameCont != {}) { + diffTemp = diff; + diffTemp.removedDecls = { elem }; + diffTemp.addedDecls = elemsSameCont; + result += applyMatchers(diffTemp, delta.options, MATCHERS); + } + } + + else { + c = diff.from.containment; + p = c[elem]; + println(elem); + } + + } + + + //for ( <- removals.containment, elem in diff.removedDecls, isTargetMember(elem)) { + // // In type cases we need the owner package instead of its compilation unit. + // if (isCompilationUnit(cont)) { + // cont = getOneFrom(invert(removals.containment)[cont]); + // } + // + // elemsSameCont = {}; + // for (a <- additions.containment[cont], a in diff.addedDecls) { + // if (isCompilationUnit(a)) { + // a = getOneFrom(additions.containment[a]); + // } + // if (a.scheme == elem.scheme) { + // elemsSameCont += a; + // } + // } + // + // if (elemsSameCont != {}) { + // diffTemp = diff; + // diffTemp.removedDecls = { elem }; + // diffTemp.addedDecls = elemsSameCont; + // result += applyMatchers(diffTemp, delta.options, MATCHERS); + // } + //} + return result; +} + +private M3Diff filterDiffRenamed(M3Diff diff, Delta delta) { + diff.removedDecls + = diff.removedDecls + - ((!isEmpty(delta.paramLists)) ? delta.paramLists.elem : {}); + return diff; +} + + +private rel[loc, Mapping[loc]] moved(M3Diff diff, Delta delta) { + diff = filterDiffMoved(diff, delta); + return applyMatchers(diff, delta.options, MATCHERS); +} + +private M3Diff filterDiffMoved(M3Diff diff, Delta delta) { + diff.removedDecls + = diff.removedDecls + - ((!isEmpty(delta.paramLists)) ? delta.paramLists.elem : {}) + - ((!isEmpty(delta.renamed)) ? delta.renamed.mapping<0> : {}); + diff.addedDecls + = diff.addedDecls + - ((!isEmpty(delta.renamed)) ? delta.renamed.mapping<1> : {}); + + return diff; +} + +/* + * Identifying removed elements + */ +// FIXME: match signature? +private rel[loc, Mapping[loc]] removed(M3Diff diff, Delta delta) + = { buildDeltaMapping(e, e, |unknown:///|, 1.0, MATCH_SIGNATURE) + | e <- diff.removedDecls, isTargetMember(e) }; + + +private rel[loc, Mapping[loc]] added(M3Diff diff, Delta delta) + = { buildDeltaMapping(e, |unknown:///|, e, 1.0, MATCH_SIGNATURE) + | e <- diff.addedDecls, isTargetMember(e) }; + + +// Default matcher: Jaccard +rel[loc, Mapping[loc]] applyMatchers(M3Diff diff, map[str,str] options, str option) { + Matcher currentMatcher = matcher(jaccardMatch); + matchers = (option in options) ? split(",", options[option]) : []; + threshold = ("