From c14ae99fee506348c06e9540a24322eab26b8f7c Mon Sep 17 00:00:00 2001 From: Alexander Milster Date: Tue, 21 Nov 2023 11:54:56 +0100 Subject: [PATCH] Rewritten python tests with new syntax. --- .../TestSourceIgnoredLinesCollector.java | 2 +- .../de/jplag/python3/PyhtonLanguageTest.java | 33 +++++ .../python3/PythonLanguageCoverageTest.java | 116 ------------------ .../python3/PythonLanguageSequenceTest.java | 38 ------ 4 files changed, 34 insertions(+), 155 deletions(-) create mode 100644 languages/python-3/src/test/java/de/jplag/python3/PyhtonLanguageTest.java delete mode 100644 languages/python-3/src/test/java/de/jplag/python3/PythonLanguageCoverageTest.java delete mode 100644 languages/python-3/src/test/java/de/jplag/python3/PythonLanguageSequenceTest.java diff --git a/language-testutils/src/test/java/de/jplag/testutils/datacollector/TestSourceIgnoredLinesCollector.java b/language-testutils/src/test/java/de/jplag/testutils/datacollector/TestSourceIgnoredLinesCollector.java index 06a67cbb7..2f6dcea47 100644 --- a/language-testutils/src/test/java/de/jplag/testutils/datacollector/TestSourceIgnoredLinesCollector.java +++ b/language-testutils/src/test/java/de/jplag/testutils/datacollector/TestSourceIgnoredLinesCollector.java @@ -41,7 +41,7 @@ public void ignoreLinesByContains(String content) { public void ignoreMultipleLines(String startMarker, String endMarker) { boolean inMultilineIgnore = false; - for (int i = 0; i < this.relevantLines.size(); i++) { + for (int i = 0; i < this.originalSource.length; i++) { String line = this.originalSource[i]; if (!inMultilineIgnore) { if (line.trim().startsWith(startMarker)) { diff --git a/languages/python-3/src/test/java/de/jplag/python3/PyhtonLanguageTest.java b/languages/python-3/src/test/java/de/jplag/python3/PyhtonLanguageTest.java new file mode 100644 index 000000000..9b40c5e77 --- /dev/null +++ b/languages/python-3/src/test/java/de/jplag/python3/PyhtonLanguageTest.java @@ -0,0 +1,33 @@ +package de.jplag.python3; + +import de.jplag.testutils.LanguageModuleTest; +import de.jplag.testutils.datacollector.TestDataCollector; +import de.jplag.testutils.datacollector.TestSourceIgnoredLinesCollector; + +public class PyhtonLanguageTest extends LanguageModuleTest { + public PyhtonLanguageTest() { + super(new PythonLanguage(), Python3TokenType.class); + } + + @Override + protected void collectTestData(TestDataCollector collector) { + collector.testFile("test_utils.py").testCoverages(); + + collector.testFile("base_features.py", "streams.py").testSourceCoverage(); + + collector.testFile("log.py").testSourceCoverage().testTokenSequence(Python3TokenType.IMPORT, Python3TokenType.ASSIGN, Python3TokenType.ARRAY, + Python3TokenType.APPLY); + + collector.testFile("unicode.py").testSourceCoverage().testTokenSequence(Python3TokenType.ASSIGN); + } + + @Override + protected void configureIgnoredLines(TestSourceIgnoredLinesCollector collector) { + collector.ignoreMultipleLines("\"\"\""); + + collector.ignoreLinesByPrefix("else:"); + collector.ignoreLinesByPrefix("elif"); + collector.ignoreLinesByPrefix("#"); + collector.ignoreLinesByPrefix("pass"); + } +} diff --git a/languages/python-3/src/test/java/de/jplag/python3/PythonLanguageCoverageTest.java b/languages/python-3/src/test/java/de/jplag/python3/PythonLanguageCoverageTest.java deleted file mode 100644 index ada546e4b..000000000 --- a/languages/python-3/src/test/java/de/jplag/python3/PythonLanguageCoverageTest.java +++ /dev/null @@ -1,116 +0,0 @@ -package de.jplag.python3; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.util.*; -import java.util.stream.Collectors; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; - -import de.jplag.ParsingException; -import de.jplag.Token; -import de.jplag.TokenType; -import de.jplag.util.FileUtils; - -public class PythonLanguageCoverageTest { - private static final File testFileLocation = Path.of("src", "test", "resources", "de", "jplag", "python3").toFile(); - private static final String[] tokenCoverageFileNames = {"test_utils.py"}; - - private static final String[] lineIgnorePrefixes = {"else:", "elif", "#", "pass"}; - private static final String multilineStringMarker = "\"\"\""; - - private final PythonLanguage language = new PythonLanguage(); - - @ParameterizedTest - @MethodSource("collectSourceCoverageTestFiles") - void testSourceCoverage(File testFile) throws ParsingException, IOException { - List tokens = language.parse(Set.of(testFile)); - List lines = new ArrayList<>(getRelevantSourceFiles(testFile)); - - tokens.forEach(token -> lines.remove((Object) token.getLine())); - - if (!lines.isEmpty()) { - Assertions.fail("There were uncovered lines in: " + testFile.getPath() + "\n" + lines); - } - } - - @ParameterizedTest - @MethodSource("collectTokenCoverageTestFiles") - void testTokenCoverage(File testFile) throws ParsingException { - Set foundTokens = language.parse(Set.of(testFile)).stream().map(Token::getType).collect(Collectors.toSet()); - List allTokens = new ArrayList<>(List.of(Python3TokenType.values())); - - allTokens.removeAll(foundTokens); - - if (!allTokens.isEmpty()) { - Assertions.fail("There are token, that were not found in: " + testFile.getPath() + "\n" + allTokens); - } - } - - @ParameterizedTest - @MethodSource("collectSourceCoverageTestFiles") - void testMonotoneTokenOrder(File testFile) throws ParsingException { - List tokens = language.parse(Set.of(testFile)); - - for (int i = 0; i < tokens.size() - 2; i++) { - if (tokens.get(i).getLine() > tokens.get(i + 1).getLine()) { - Assertions.fail("Token " + tokens.get(i).getType() + " reported in line " + tokens.get(i).getLine() - + " has a bigger line index, than token " + tokens.get(i + 1).getType() + " in line " + tokens.get(i + 1).getLine()); - } - } - } - - public static List collectSourceCoverageTestFiles() { - return Arrays.asList(Objects.requireNonNull(testFileLocation.listFiles())); - } - - public static List collectTokenCoverageTestFiles() { - return Arrays.stream(tokenCoverageFileNames).map(it -> new File(testFileLocation, it)).toList(); - } - - private List getRelevantSourceFiles(File file) throws IOException { - List lines = FileUtils.readFileContent(file).lines().toList(); - List relevantLineIndices = new ArrayList<>(); - - boolean inMultilineString = false; - - for (int i = 0; i < lines.size(); i++) { - String line = lines.get(i); - if (!inMultilineString) { - if (isLineInlineIgnore(line)) { - continue; - } - - if (isMultilineStringStart(line)) { - inMultilineString = true; - continue; - } - } else { - if (isMultilineStringEnd(line)) { - inMultilineString = false; - } - continue; - } - - relevantLineIndices.add(i + 1); - } - - return relevantLineIndices; - } - - private boolean isLineInlineIgnore(String line) { - return line.isBlank() || Arrays.stream(lineIgnorePrefixes).anyMatch(it -> line.trim().startsWith(it)) - || (line.trim().startsWith(multilineStringMarker) && line.trim().substring(3).contains(multilineStringMarker)); - } - - private boolean isMultilineStringStart(String line) { - return line.trim().startsWith(multilineStringMarker); - } - - private boolean isMultilineStringEnd(String line) { - return line.trim().contains(multilineStringMarker); - } -} diff --git a/languages/python-3/src/test/java/de/jplag/python3/PythonLanguageSequenceTest.java b/languages/python-3/src/test/java/de/jplag/python3/PythonLanguageSequenceTest.java deleted file mode 100644 index 4ab5344da..000000000 --- a/languages/python-3/src/test/java/de/jplag/python3/PythonLanguageSequenceTest.java +++ /dev/null @@ -1,38 +0,0 @@ -package de.jplag.python3; - -import java.io.File; -import java.nio.file.Path; -import java.util.List; -import java.util.Set; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -import de.jplag.ParsingException; -import de.jplag.SharedTokenType; -import de.jplag.Token; -import de.jplag.TokenType; - -public class PythonLanguageSequenceTest { - private static final File testFileLocation = Path.of("src", "test", "resources", "de", "jplag", "python3").toFile(); - private final PythonLanguage language = new PythonLanguage(); - - @ParameterizedTest - @MethodSource("provideFileData") - void testTokenSequences(File file, TokenType[] sequence) throws ParsingException { - TokenType[] fileTokens = language.parse(Set.of(file)).stream().map(Token::getType).toArray(TokenType[]::new); - - Assertions.assertArrayEquals(sequence, fileTokens); - } - - public static List provideFileData() { - TokenType[] logTokens = new TokenType[] {Python3TokenType.IMPORT, Python3TokenType.ASSIGN, Python3TokenType.ARRAY, Python3TokenType.APPLY, - SharedTokenType.FILE_END}; - TokenType[] unicodeTokens = new TokenType[] {Python3TokenType.ASSIGN, SharedTokenType.FILE_END}; - - return List.of(Arguments.of(new File(testFileLocation, "log.py"), logTokens), - Arguments.of(new File(testFileLocation, "unicode.py"), unicodeTokens)); - } -}