From 55f142f6d070ca7414b11a62510d46eaff509575 Mon Sep 17 00:00:00 2001 From: vityaman Date: Sat, 27 Jul 2024 13:29:58 +0300 Subject: [PATCH 1/8] #132 Support multiple grammar tests Signed-off-by: vityaman --- ports/cpp/.vscode/launch.json | 2 +- ports/cpp/cmake/Testing.cmake | 18 ++- ports/cpp/test/CMakeLists.txt | 1 + ports/cpp/test/whitebox/CMakeLists.txt | 1 + ports/cpp/test/whitebox/Whitebox.g4 | 138 +++++++++++++++++++++++ ports/cpp/test/whitebox/WhiteboxTest.cpp | 3 + 6 files changed, 158 insertions(+), 5 deletions(-) create mode 100644 ports/cpp/test/whitebox/CMakeLists.txt create mode 100644 ports/cpp/test/whitebox/Whitebox.g4 create mode 100644 ports/cpp/test/whitebox/WhiteboxTest.cpp diff --git a/ports/cpp/.vscode/launch.json b/ports/cpp/.vscode/launch.json index afd6056..87c8d48 100644 --- a/ports/cpp/.vscode/launch.json +++ b/ports/cpp/.vscode/launch.json @@ -5,7 +5,7 @@ "type": "lldb", "request": "launch", "name": "Debug Test", - "program": "${workspaceFolder}/build/test/${fileDirnameBasename}/antlr4-c3-test", + "program": "${workspaceFolder}/build/test/${fileDirnameBasename}/antlr4-c3-test-${fileDirnameBasename}", "args": [], "cwd": "${workspaceFolder}", "preLaunchTask": "Build All" diff --git a/ports/cpp/cmake/Testing.cmake b/ports/cpp/cmake/Testing.cmake index fb2c5ef..f8f2f72 100644 --- a/ports/cpp/cmake/Testing.cmake +++ b/ports/cpp/cmake/Testing.cmake @@ -1,4 +1,9 @@ macro(define_grammar_test grammar) + get_filename_component( + ANTLR4C3_CURRENT_DIR_NAME + ${CMAKE_CURRENT_LIST_DIR} NAME + ) + antlr_generate( ${CMAKE_CURRENT_LIST_DIR}/${grammar} ${CMAKE_CURRENT_BINARY_DIR} @@ -11,19 +16,24 @@ macro(define_grammar_test grammar) ${CMAKE_CURRENT_BINARY_DIR}/*.cpp ) - add_executable(${PROJECT_NAME}-test ${SOURCE}) + set( + ANTLR4C3_TEST_TARGET + ${PROJECT_NAME}-test-${ANTLR4C3_CURRENT_DIR_NAME} + ) + + add_executable(${ANTLR4C3_TEST_TARGET} ${SOURCE}) target_include_directories( - ${PROJECT_NAME}-test PRIVATE + ${ANTLR4C3_TEST_TARGET} PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ) target_link_libraries( - ${PROJECT_NAME}-test PRIVATE + ${ANTLR4C3_TEST_TARGET} PRIVATE ${PROJECT_NAME} GTest::gtest_main GTest::gmock ) - gtest_discover_tests(${PROJECT_NAME}-test) + gtest_discover_tests(${ANTLR4C3_TEST_TARGET}) endmacro() diff --git a/ports/cpp/test/CMakeLists.txt b/ports/cpp/test/CMakeLists.txt index 74efe32..145ff3c 100644 --- a/ports/cpp/test/CMakeLists.txt +++ b/ports/cpp/test/CMakeLists.txt @@ -3,3 +3,4 @@ add_compile_options(-Wno-unused-parameter) enable_testing() add_subdirectory(expr) +add_subdirectory(whitebox) diff --git a/ports/cpp/test/whitebox/CMakeLists.txt b/ports/cpp/test/whitebox/CMakeLists.txt new file mode 100644 index 0000000..f753029 --- /dev/null +++ b/ports/cpp/test/whitebox/CMakeLists.txt @@ -0,0 +1 @@ +define_grammar_test(Whitebox.g4) diff --git a/ports/cpp/test/whitebox/Whitebox.g4 b/ports/cpp/test/whitebox/Whitebox.g4 new file mode 100644 index 0000000..4acb400 --- /dev/null +++ b/ports/cpp/test/whitebox/Whitebox.g4 @@ -0,0 +1,138 @@ +grammar Whitebox; + +// $antlr-format columnLimit 100, minEmptyLines 1, maxEmptyLinesToKeep 1, useTab false +// $antlr-format reflowComments false, breakBeforeBraces false +// $antlr-format keepEmptyLinesAtTheStartOfBlocks false, allowShortRulesOnASingleLine false +// $antlr-format alignSemicolons hanging, alignColons hanging, alignTrailingComments true + +test1 + : rule1 ADIPISCING + ; + +rule1 + : rule2 CONSECTETUR + ; + +rule2 + : LOREM rule3 rule5 SIT* AMET? + ; + +rule3 + : rule4 DOLOR? + ; + +rule4 + : IPSUM? + ; + +rule5 + : + ; + +test2 + : rule7 ADIPISCING + ; + +rule7 + : rule8 CONSECTETUR + ; + +rule8 + : LOREM rule11 rule9 SIT* AMET? + ; + +rule9 + : rule10 DOLOR? + ; + +rule10 + : IPSUM? + ; + +rule11 + : + ; + +test3 + : LOREM IPSUM? rule13 AMET+ CONSECTETUR + ; + +rule13 + : (DOLOR | SIT)* + ; + +test4 + : LOREM (rule15 | rule16) + ; + +rule15 + : IPSUM DOLOR SIT + ; + +rule16 + : IPSUM DOLOR AMET + ; + +test5 + : LOREM (rule15 | rule16) + ; + +rule18 + : IPSUM DOLOR (SIT | CONSECTETUR) + ; + +rule19 + : IPSUM DOLOR AMET + ; + +test6 + : LOREM (rule15 | rule16) + ; + +rule21 + : IPSUM DOLOR SIT + ; + +rule22 + : IPSUM DOLOR (AMET | CONSECTETUR) + ; + +test7 + : LOREM (IPSUM DOLOR SIT | IPSUM DOLOR AMET) + ; + +test8 + : LOREM (IPSUM DOLOR SIT AMET | IPSUM DOLOR SIT CONSECTETUR) + ; + +LOREM + : 'LOREM' + ; + +IPSUM + : 'IPSUM' + ; + +DOLOR + : 'DOLOR' + ; + +SIT + : 'SIT' + ; + +AMET + : 'AMET' + ; + +CONSECTETUR + : 'CONSECTETUR' + ; + +ADIPISCING + : 'ADIPISCING' + ; + +WS + : [ \n\r\t] -> skip + ; diff --git a/ports/cpp/test/whitebox/WhiteboxTest.cpp b/ports/cpp/test/whitebox/WhiteboxTest.cpp new file mode 100644 index 0000000..7787c2b --- /dev/null +++ b/ports/cpp/test/whitebox/WhiteboxTest.cpp @@ -0,0 +1,3 @@ +#include + +TEST(HelloWorld, Nothing) {} From dd0562ccf47906713de87a6c787c30ae7dd7f3d6 Mon Sep 17 00:00:00 2001 From: vityaman Date: Sat, 27 Jul 2024 13:45:31 +0300 Subject: [PATCH 2/8] #132 Put testing utility to separate module Signed-off-by: vityaman --- ports/cpp/cmake/Testing.cmake | 1 + ports/cpp/test/expr/ExprTest.cpp | 76 +++++------------------- ports/cpp/test/utility/AntlrPipeline.hpp | 37 ++++++++++++ ports/cpp/test/utility/Maps.hpp | 16 +++++ ports/cpp/test/utility/Testing.hpp | 14 +++++ ports/cpp/test/whitebox/WhiteboxTest.cpp | 5 +- 6 files changed, 88 insertions(+), 61 deletions(-) create mode 100644 ports/cpp/test/utility/AntlrPipeline.hpp create mode 100644 ports/cpp/test/utility/Maps.hpp create mode 100644 ports/cpp/test/utility/Testing.hpp diff --git a/ports/cpp/cmake/Testing.cmake b/ports/cpp/cmake/Testing.cmake index f8f2f72..8646464 100644 --- a/ports/cpp/cmake/Testing.cmake +++ b/ports/cpp/cmake/Testing.cmake @@ -26,6 +26,7 @@ macro(define_grammar_test grammar) target_include_directories( ${ANTLR4C3_TEST_TARGET} PRIVATE ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_LIST_DIR}/.. ) target_link_libraries( diff --git a/ports/cpp/test/expr/ExprTest.cpp b/ports/cpp/test/expr/ExprTest.cpp index 4fb8f2e..78446c5 100644 --- a/ports/cpp/test/expr/ExprTest.cpp +++ b/ports/cpp/test/expr/ExprTest.cpp @@ -1,57 +1,21 @@ #include -#include - -#include -#include #include #include -#include -#include - -class CountingErrorListener final : public antlr4::BaseErrorListener { -public: - void syntaxError(antlr4::Recognizer *recognizer, - antlr4::Token *offendingSymbol, std::size_t line, - std::size_t charPositionInLine, const std::string &msg, - std::exception_ptr e) override {} - - std::size_t GetErrorCount() const { return errorCount; } - -private: - std::size_t errorCount = 0; -}; +#include -struct AntlrPipeline { - AntlrPipeline(std::string_view text) - : chars(text), lexer(&chars), tokens(&lexer), parser(&tokens) {} +namespace c3::test { - antlr4::ANTLRInputStream chars; - ExprLexer lexer; - antlr4::CommonTokenStream tokens; - ExprParser parser; +struct ExprGrammar { + using Lexer = ExprLexer; + using Parser = ExprParser; }; -template std::vector Keys(const std::map &map) { - std::vector keys; - for (const auto &[key, value] : map) { - keys.emplace_back(key); - } - return keys; -} - -using testing::ElementsAre; -using testing::UnorderedElementsAre; - TEST(SimpleExpressionParser, MostSimpleSetup) { - CountingErrorListener listener; - - AntlrPipeline pipeline("var c = a + b()"); - pipeline.parser.addErrorListener(&listener); - + AntlrPipeline pipeline("var c = a + b()"); pipeline.parser.expression(); - EXPECT_EQ(listener.GetErrorCount(), 0); + EXPECT_EQ(pipeline.listener.GetErrorCount(), 0); c3::CodeCompletionCore completion(&pipeline.parser); const auto collectCandidatesAt = [&](std::size_t tokenIndex) { @@ -109,13 +73,9 @@ TEST(SimpleExpressionParser, MostSimpleSetup) { } TEST(SimpleExpressionParser, TypicalSetup) { - CountingErrorListener listener; - - AntlrPipeline pipeline("var c = a + b()"); - pipeline.parser.addErrorListener(&listener); - + AntlrPipeline pipeline("var c = a + b()"); pipeline.parser.expression(); - EXPECT_EQ(listener.GetErrorCount(), 0); + EXPECT_EQ(pipeline.listener.GetErrorCount(), 0); c3::CodeCompletionCore completion(&pipeline.parser); completion.ignoredTokens = { @@ -178,12 +138,9 @@ TEST(SimpleExpressionParser, TypicalSetup) { } TEST(SimpleExpressionParser, RecursivePreferredRule) { - AntlrPipeline pipeline("var c = a + b"); - - CountingErrorListener listener; - pipeline.parser.addErrorListener(&listener); + AntlrPipeline pipeline("var c = a + b"); pipeline.parser.expression(); - EXPECT_EQ(listener.GetErrorCount(), 0); + EXPECT_EQ(pipeline.listener.GetErrorCount(), 0); c3::CodeCompletionCore completion(&pipeline.parser); completion.preferredRules = {ExprParser::RuleSimpleExpression}; @@ -230,12 +187,9 @@ TEST(SimpleExpressionParser, RecursivePreferredRule) { } TEST(SimpleExpressionParser, CandidateRulesWithDifferentStartTokens) { - AntlrPipeline pipeline("var c = a + b"); - - CountingErrorListener listener; - pipeline.parser.addErrorListener(&listener); + AntlrPipeline pipeline("var c = a + b"); pipeline.parser.expression(); - EXPECT_EQ(listener.GetErrorCount(), 0); + EXPECT_EQ(pipeline.listener.GetErrorCount(), 0); c3::CodeCompletionCore completion(&pipeline.parser); completion.preferredRules = { @@ -272,4 +226,6 @@ TEST(SimpleExpressionParser, CandidateRulesWithDifferentStartTokens) { // The start token of the variableRef rule begins at token 'a'. EXPECT_EQ(candidates.rules[ExprParser::RuleVariableRef].startTokenIndex, 6); } -} \ No newline at end of file +} + +} // namespace c3::test diff --git a/ports/cpp/test/utility/AntlrPipeline.hpp b/ports/cpp/test/utility/AntlrPipeline.hpp new file mode 100644 index 0000000..15d5fd4 --- /dev/null +++ b/ports/cpp/test/utility/AntlrPipeline.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include + +#include +#include + +namespace c3::test { + +class CountingErrorListener final : public antlr4::BaseErrorListener { +public: + void syntaxError(antlr4::Recognizer *recognizer, + antlr4::Token *offendingSymbol, std::size_t line, + std::size_t charPositionInLine, const std::string &msg, + std::exception_ptr e) override {} + + std::size_t GetErrorCount() const { return errorCount; } + +private: + std::size_t errorCount = 0; +}; + +template struct AntlrPipeline { + AntlrPipeline(std::string_view text) + : chars(text), lexer(&chars), tokens(&lexer), parser(&tokens) { + parser.removeErrorListeners(); + parser.addErrorListener(&listener); + } + + antlr4::ANTLRInputStream chars; + Grammar::Lexer lexer; + antlr4::CommonTokenStream tokens; + Grammar::Parser parser; + CountingErrorListener listener; +}; + +} // namespace c3::test diff --git a/ports/cpp/test/utility/Maps.hpp b/ports/cpp/test/utility/Maps.hpp new file mode 100644 index 0000000..dfcb893 --- /dev/null +++ b/ports/cpp/test/utility/Maps.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include +#include + +namespace c3::test { + +template std::vector Keys(const std::map &map) { + std::vector keys; + for (const auto &[key, value] : map) { + keys.emplace_back(key); + } + return keys; +} + +} // namespace c3::test diff --git a/ports/cpp/test/utility/Testing.hpp b/ports/cpp/test/utility/Testing.hpp new file mode 100644 index 0000000..c9b4cfe --- /dev/null +++ b/ports/cpp/test/utility/Testing.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include +#include + +#include "AntlrPipeline.hpp" +#include "Maps.hpp" + +namespace c3::test { + +using testing::ElementsAre; +using testing::UnorderedElementsAre; + +} diff --git a/ports/cpp/test/whitebox/WhiteboxTest.cpp b/ports/cpp/test/whitebox/WhiteboxTest.cpp index 7787c2b..ba538ba 100644 --- a/ports/cpp/test/whitebox/WhiteboxTest.cpp +++ b/ports/cpp/test/whitebox/WhiteboxTest.cpp @@ -1,3 +1,6 @@ #include -TEST(HelloWorld, Nothing) {} +TEST(WhiteboxGrammarTests, + CaretAtTransitionToRuleWithNonExhaustiveFollowSetOptionalTokens) { + // +} From 87426bd8a317f6910d6535dcea59290bffbc921c Mon Sep 17 00:00:00 2001 From: vityaman Date: Sat, 27 Jul 2024 14:09:54 +0300 Subject: [PATCH 3/8] #132 Port Whitebox grammar test Signed-off-by: vityaman --- ports/cpp/test/utility/AntlrPipeline.hpp | 4 +- ports/cpp/test/utility/Extension.hpp | 22 ++++++ ports/cpp/test/utility/Testing.hpp | 3 +- ports/cpp/test/whitebox/WhiteboxTest.cpp | 89 +++++++++++++++++++++++- 4 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 ports/cpp/test/utility/Extension.hpp diff --git a/ports/cpp/test/utility/AntlrPipeline.hpp b/ports/cpp/test/utility/AntlrPipeline.hpp index 15d5fd4..2e423d5 100644 --- a/ports/cpp/test/utility/AntlrPipeline.hpp +++ b/ports/cpp/test/utility/AntlrPipeline.hpp @@ -12,7 +12,9 @@ class CountingErrorListener final : public antlr4::BaseErrorListener { void syntaxError(antlr4::Recognizer *recognizer, antlr4::Token *offendingSymbol, std::size_t line, std::size_t charPositionInLine, const std::string &msg, - std::exception_ptr e) override {} + std::exception_ptr e) override { + errorCount += 1; + } std::size_t GetErrorCount() const { return errorCount; } diff --git a/ports/cpp/test/utility/Extension.hpp b/ports/cpp/test/utility/Extension.hpp new file mode 100644 index 0000000..b06c9d9 --- /dev/null +++ b/ports/cpp/test/utility/Extension.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include + +namespace c3::test { + +inline auto Candidates(CodeCompletionCore &completion, + std::size_t caretTokenIndex) { + return completion.collectCandidates(caretTokenIndex, + /*context=*/nullptr, /*size_t=*/0, + /*cancel=*/nullptr); +} + +inline auto Candidates(CodeCompletionCore &completion, + std::size_t caretTokenIndex, + antlr4::ParserRuleContext *context) { + return completion.collectCandidates(caretTokenIndex, + /*context=*/context, /*size_t=*/0, + /*cancel=*/nullptr); +} + +} // namespace c3::test \ No newline at end of file diff --git a/ports/cpp/test/utility/Testing.hpp b/ports/cpp/test/utility/Testing.hpp index c9b4cfe..ea412c4 100644 --- a/ports/cpp/test/utility/Testing.hpp +++ b/ports/cpp/test/utility/Testing.hpp @@ -4,6 +4,7 @@ #include #include "AntlrPipeline.hpp" +#include "Extension.hpp" #include "Maps.hpp" namespace c3::test { @@ -11,4 +12,4 @@ namespace c3::test { using testing::ElementsAre; using testing::UnorderedElementsAre; -} +} // namespace c3::test diff --git a/ports/cpp/test/whitebox/WhiteboxTest.cpp b/ports/cpp/test/whitebox/WhiteboxTest.cpp index ba538ba..ce54b2c 100644 --- a/ports/cpp/test/whitebox/WhiteboxTest.cpp +++ b/ports/cpp/test/whitebox/WhiteboxTest.cpp @@ -1,6 +1,89 @@ -#include +#include + +#include +#include + +#include + +namespace c3::test { + +struct WhiteboxGrammar { + using Lexer = WhiteboxLexer; + using Parser = WhiteboxParser; +}; + +/// (optional tokens) +TEST(WhiteboxGrammarTests, CaretAtTransitionToRuleWithNonExhaustiveFollowSet) { + AntlrPipeline pipeline("LOREM "); + auto *ctx = pipeline.parser.test1(); + EXPECT_EQ(pipeline.listener.GetErrorCount(), 1); + + c3::CodeCompletionCore completion(&pipeline.parser); + auto candidates = Candidates(completion, 1, ctx); + + EXPECT_THAT(Keys(candidates.tokens), + UnorderedElementsAre(WhiteboxLexer::IPSUM, WhiteboxLexer::DOLOR, + WhiteboxLexer::SIT, WhiteboxLexer::AMET, + WhiteboxLexer::CONSECTETUR)); +} + +/// (epsilon-only transition to rule end) +TEST(WhiteboxGrammarTests, CaretAtTransitionToRuleWithEmptyFollowSet) { + AntlrPipeline pipeline("LOREM "); + auto *ctx = pipeline.parser.test2(); + EXPECT_EQ(pipeline.listener.GetErrorCount(), 1); + + c3::CodeCompletionCore completion(&pipeline.parser); + auto candidates = Candidates(completion, 1, ctx); + + EXPECT_THAT(Keys(candidates.tokens), + UnorderedElementsAre(WhiteboxLexer::IPSUM, WhiteboxLexer::DOLOR, + WhiteboxLexer::SIT, WhiteboxLexer::AMET, + WhiteboxLexer::CONSECTETUR)); +} + +TEST(WhiteboxGrammarTests, CaretAtOneOfMultiplePossibleStates) { + for (const auto index : {4, 5, 6, 7}) { + AntlrPipeline pipeline("LOREM IPSUM "); + + auto *ctx = [&] -> antlr4::ParserRuleContext * { + switch (index) { + case 4: + return pipeline.parser.test4(); + case 5: + return pipeline.parser.test5(); + case 6: + return pipeline.parser.test6(); + case 7: + return pipeline.parser.test7(); + default: + std::abort(); + } + }(); + + c3::CodeCompletionCore completion(&pipeline.parser); + auto candidates = Candidates(completion, 2, ctx); + + EXPECT_THAT(Keys(candidates.tokens), + UnorderedElementsAre(WhiteboxLexer::DOLOR)); + EXPECT_THAT(candidates.tokens[WhiteboxLexer::DOLOR], + UnorderedElementsAre()); + } +} TEST(WhiteboxGrammarTests, - CaretAtTransitionToRuleWithNonExhaustiveFollowSetOptionalTokens) { - // + CaretAtOneOfMultiplePossibleStatesWithCommonFollowList) { + AntlrPipeline pipeline("LOREM IPSUM "); + + auto *ctx = pipeline.parser.test8(); + + c3::CodeCompletionCore completion(&pipeline.parser); + auto candidates = Candidates(completion, 2, ctx); + + EXPECT_THAT(Keys(candidates.tokens), + UnorderedElementsAre(WhiteboxLexer::DOLOR)); + EXPECT_THAT(candidates.tokens[WhiteboxLexer::DOLOR], + UnorderedElementsAre(WhiteboxLexer::SIT)); } + +} // namespace c3::test From dc868b388090ecddc49a68a6de0c03e9ea37a17f Mon Sep 17 00:00:00 2001 From: vityaman Date: Sat, 27 Jul 2024 22:22:46 +0300 Subject: [PATCH 4/8] #132 Port C++14 grammar test Signed-off-by: vityaman --- ports/cpp/test/CMakeLists.txt | 1 + ports/cpp/test/cpp14/CMakeLists.txt | 1 + ports/cpp/test/cpp14/CPP14.g4 | 1930 +++++++++++++++++ ports/cpp/test/cpp14/Cpp14Test.cpp | 405 ++++ .../utility/{Maps.hpp => Collections.hpp} | 8 + ports/cpp/test/utility/Testing.hpp | 5 +- 6 files changed, 2349 insertions(+), 1 deletion(-) create mode 100644 ports/cpp/test/cpp14/CMakeLists.txt create mode 100644 ports/cpp/test/cpp14/CPP14.g4 create mode 100644 ports/cpp/test/cpp14/Cpp14Test.cpp rename ports/cpp/test/utility/{Maps.hpp => Collections.hpp} (61%) diff --git a/ports/cpp/test/CMakeLists.txt b/ports/cpp/test/CMakeLists.txt index 145ff3c..51b1f8f 100644 --- a/ports/cpp/test/CMakeLists.txt +++ b/ports/cpp/test/CMakeLists.txt @@ -4,3 +4,4 @@ add_compile_options(-Wno-unused-parameter) enable_testing() add_subdirectory(expr) add_subdirectory(whitebox) +add_subdirectory(cpp14) diff --git a/ports/cpp/test/cpp14/CMakeLists.txt b/ports/cpp/test/cpp14/CMakeLists.txt new file mode 100644 index 0000000..56879b4 --- /dev/null +++ b/ports/cpp/test/cpp14/CMakeLists.txt @@ -0,0 +1 @@ +define_grammar_test(CPP14.g4) diff --git a/ports/cpp/test/cpp14/CPP14.g4 b/ports/cpp/test/cpp14/CPP14.g4 new file mode 100644 index 0000000..ec8d0b0 --- /dev/null +++ b/ports/cpp/test/cpp14/CPP14.g4 @@ -0,0 +1,1930 @@ +/******************************************************************************* + * The MIT License (MIT) + * + * Copyright (c) 2015 Camilo Sanchez (Camiloasc1) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and + * associated documentation files (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, publish, distribute, + * sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or + * substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT + * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * **************************************************************************** + */ + +/******************************************************************************* + * C++14 Grammar for ANTLR v4 + * + * Based on n4140 draft paper https://github.com/cplusplus/draft/blob/master/papers/n4140.pdf and + * http://www.nongnu.org/hcb/ + * + * Possible Issues: + * + * Input must avoid conditional compilation blocks (this grammar ignores any preprocessor directive) + * GCC extensions not yet supported (do not try to parse the preprocessor output) Right angle + * bracket (C++11) - Solution '>>' and '>>=' are not tokens, only '>' Lexer issue with + * pure-specifier rule ('0' token) - Solution in embedded code Change it to match the target + * language you want in line 1097 or verify inside your listeners/visitors Java: + * if($val.text.compareTo("0")!=0) throw new InputMismatchException(this); JavaScript: + * + * Python2: + * + * Python3: + * + * C#: + * + * **************************************************************************** + */ +grammar CPP14; + +// $antlr-format columnLimit 100, minEmptyLines 1, maxEmptyLinesToKeep 1, useTab false +// $antlr-format reflowComments false, breakBeforeBraces false +// $antlr-format keepEmptyLinesAtTheStartOfBlocks false, allowShortRulesOnASingleLine false +// $antlr-format alignSemicolons hanging, alignColons hanging, alignTrailingComments true + +/*Basic concepts*/ +translationunit + : declarationseq? EOF + ; + +/*Expressions*/ +primaryexpression + : literal + | This + | '(' expression ')' + | idexpression + | lambdaexpression + ; + +idexpression + : unqualifiedid + | qualifiedid + ; + +unqualifiedid + : Identifier + | operatorfunctionid + | conversionfunctionid + | literaloperatorid + | '~' classname + | '~' decltypespecifier + | templateid + ; + +qualifiedid + : nestednamespecifier Template? unqualifiedid + ; + +nestednamespecifier + : '::' + | typename '::' + | namespacename '::' + | decltypespecifier '::' + | nestednamespecifier Identifier '::' + | nestednamespecifier Template? simpletemplateid '::' + ; + +lambdaexpression + : lambdaintroducer lambdadeclarator? compoundstatement + ; + +lambdaintroducer + : '[' lambdacapture? ']' + ; + +lambdacapture + : capturedefault + | capturelist + | capturedefault ',' capturelist + ; + +capturedefault + : '&' + | '=' + ; + +capturelist + : capture '...'? + | capturelist ',' capture '...'? + ; + +capture + : simplecapture + | initcapture + ; + +simplecapture + : Identifier + | '&' Identifier + | This + ; + +initcapture + : Identifier initializer + | '&' Identifier initializer + ; + +lambdadeclarator + : '(' parameterdeclarationclause ')' Mutable? exceptionspecification? attributespecifierseq? + trailingreturntype? + ; + +postfixexpression + : primaryexpression + | postfixexpression '[' expression ']' + | postfixexpression '[' bracedinitlist ']' + | postfixexpression '(' expressionlist? ')' + | simpletypespecifier '(' expressionlist? ')' + | typenamespecifier '(' expressionlist? ')' + | simpletypespecifier bracedinitlist + | typenamespecifier bracedinitlist + | postfixexpression '.' Template? idexpression + | postfixexpression '->' Template? idexpression + | postfixexpression '.' pseudodestructorname + | postfixexpression '->' pseudodestructorname + | postfixexpression '++' + | postfixexpression '--' + | Dynamic_cast '<' typeid '>' '(' expression ')' + | Static_cast '<' typeid '>' '(' expression ')' + | Reinterpret_cast '<' typeid '>' '(' expression ')' + | Const_cast '<' typeid '>' '(' expression ')' + | Typeid '(' expression ')' + | Typeid '(' typeid ')' + ; + +expressionlist + : initializerlist + ; + +pseudodestructorname + : nestednamespecifier? typename '::' '~' typename + | nestednamespecifier Template simpletemplateid '::' '~' typename + | nestednamespecifier? '~' typename + | '~' decltypespecifier + ; + +unaryexpression + : postfixexpression + | '++' castexpression + | '--' castexpression + | unaryoperator castexpression + | Sizeof unaryexpression + | Sizeof '(' typeid ')' + | Sizeof '...' '(' Identifier ')' + | Alignof '(' typeid ')' + | noexceptexpression + | newexpression + | deleteexpression + ; + +unaryoperator + : '|' + | '*' + | '&' + | '+' + | '!' + | '~' + | '-' + ; + +newexpression + : '::'? New newplacement? newtypeid newinitializer? + | '::'? New newplacement? '(' typeid ')' newinitializer? + ; + +newplacement + : '(' expressionlist ')' + ; + +newtypeid + : typespecifierseq newdeclarator? + ; + +newdeclarator + : ptroperator newdeclarator? + | noptrnewdeclarator + ; + +noptrnewdeclarator + : '[' expression ']' attributespecifierseq? + | noptrnewdeclarator '[' constantexpression ']' attributespecifierseq? + ; + +newinitializer + : '(' expressionlist? ')' + | bracedinitlist + ; + +deleteexpression + : '::'? Delete castexpression + | '::'? Delete '[' ']' castexpression + ; + +noexceptexpression + : Noexcept '(' expression ')' + ; + +castexpression + : unaryexpression + | '(' typeid ')' castexpression + ; + +pmexpression + : castexpression + | pmexpression '.*' castexpression + | pmexpression '->*' castexpression + ; + +multiplicativeexpression + : pmexpression + | multiplicativeexpression '*' pmexpression + | multiplicativeexpression '/' pmexpression + | multiplicativeexpression '%' pmexpression + ; + +additiveexpression + : multiplicativeexpression + | additiveexpression '+' multiplicativeexpression + | additiveexpression '-' multiplicativeexpression + ; + +shiftexpression + : additiveexpression + | shiftexpression '<<' additiveexpression + | shiftexpression rightShift additiveexpression + ; + +relationalexpression + : shiftexpression + | relationalexpression '<' shiftexpression + | relationalexpression '>' shiftexpression + | relationalexpression '<=' shiftexpression + | relationalexpression '>=' shiftexpression + ; + +equalityexpression + : relationalexpression + | equalityexpression '==' relationalexpression + | equalityexpression '!=' relationalexpression + ; + +andexpression + : equalityexpression + | andexpression '&' equalityexpression + ; + +exclusiveorexpression + : andexpression + | exclusiveorexpression '^' andexpression + ; + +inclusiveorexpression + : exclusiveorexpression + | inclusiveorexpression '|' exclusiveorexpression + ; + +logicalandexpression + : inclusiveorexpression + | logicalandexpression '&&' inclusiveorexpression + ; + +logicalorexpression + : logicalandexpression + | logicalorexpression '||' logicalandexpression + ; + +conditionalexpression + : logicalorexpression + | logicalorexpression '?' expression ':' assignmentexpression + ; + +assignmentexpression + : conditionalexpression + | logicalorexpression assignmentoperator initializerclause + | throwexpression + ; + +assignmentoperator + : '=' + | '*=' + | '/=' + | '%=' + | '+=' + | '-=' + | rightShiftAssign + | '<<=' + | '&=' + | '^=' + | '|=' + ; + +expression + : assignmentexpression + | expression ',' assignmentexpression + ; + +constantexpression + : conditionalexpression + ; + +/*Statements*/ +statement + : labeledstatement + | attributespecifierseq? expressionstatement + | attributespecifierseq? compoundstatement + | attributespecifierseq? selectionstatement + | attributespecifierseq? iterationstatement + | attributespecifierseq? jumpstatement + | declarationstatement + | attributespecifierseq? tryblock + ; + +labeledstatement + : attributespecifierseq? Identifier ':' statement + | attributespecifierseq? Case constantexpression ':' statement + | attributespecifierseq? Default ':' statement + ; + +expressionstatement + : expression? ';' + ; + +compoundstatement + : '{' statementseq? '}' + ; + +statementseq + : statement + | statementseq statement + ; + +selectionstatement + : If '(' condition ')' statement + | If '(' condition ')' statement Else statement + | Switch '(' condition ')' statement + ; + +condition + : expression + | attributespecifierseq? declspecifierseq declarator '=' initializerclause + | attributespecifierseq? declspecifierseq declarator bracedinitlist + ; + +iterationstatement + : While '(' condition ')' statement + | Do statement While '(' expression ')' ';' + | For '(' forinitstatement condition? ';' expression? ')' statement + | For '(' forrangedeclaration ':' forrangeinitializer ')' statement + ; + +forinitstatement + : expressionstatement + | simpledeclaration + ; + +forrangedeclaration + : attributespecifierseq? declspecifierseq declarator + ; + +forrangeinitializer + : expression + | bracedinitlist + ; + +jumpstatement + : Break ';' + | Continue ';' + | Return expression? ';' + | Return bracedinitlist ';' + | Goto Identifier ';' + ; + +declarationstatement + : blockdeclaration + ; + +/*Declarations*/ +declarationseq + : declaration + | declarationseq declaration + ; + +declaration + : blockdeclaration + | functiondefinition + | templatedeclaration + | explicitinstantiation + | explicitspecialization + | linkagespecification + | namespacedefinition + | emptydeclaration + | attributedeclaration + ; + +blockdeclaration + : simpledeclaration + | asmdefinition + | namespacealiasdefinition + | usingdeclaration + | usingdirective + | static_assertdeclaration + | aliasdeclaration + | opaqueenumdeclaration + ; + +aliasdeclaration + : Using Identifier attributespecifierseq? '=' typeid ';' + ; + +simpledeclaration + : declspecifierseq? initdeclaratorlist? ';' + | attributespecifierseq declspecifierseq? initdeclaratorlist ';' + ; + +static_assertdeclaration + : Static_assert '(' constantexpression ',' Stringliteral ')' ';' + ; + +emptydeclaration + : ';' + ; + +attributedeclaration + : attributespecifierseq ';' + ; + +declspecifier + : storageclassspecifier + | typespecifier + | functionspecifier + | Friend + | Typedef + | Constexpr + ; + +declspecifierseq + : declspecifier attributespecifierseq? + | declspecifier declspecifierseq + ; + +storageclassspecifier + : Register + | Static + | Thread_local + | Extern + | Mutable + ; + +functionspecifier + : Inline + | Virtual + | Explicit + ; + +typedefname + : Identifier + ; + +typespecifier + : trailingtypespecifier + | classspecifier + | enumspecifier + ; + +trailingtypespecifier + : simpletypespecifier + | elaboratedtypespecifier + | typenamespecifier + | cvqualifier + ; + +typespecifierseq + : typespecifier attributespecifierseq? + | typespecifier typespecifierseq + ; + +trailingtypespecifierseq + : trailingtypespecifier attributespecifierseq? + | trailingtypespecifier trailingtypespecifierseq + ; + +simpletypespecifier + : nestednamespecifier? typename + | nestednamespecifier Template simpletemplateid + | Char + | Char16 + | Char32 + | Wchar + | Bool + | Short + | Int + | Long + | Signed + | Unsigned + | Float + | Double + | Void + | Auto + | decltypespecifier + ; + +typename + : classname + | enumname + | typedefname + | simpletemplateid + ; + +decltypespecifier + : Decltype '(' expression ')' + | Decltype '(' Auto ')' + ; + +elaboratedtypespecifier + : classkey attributespecifierseq? nestednamespecifier? Identifier + | classkey simpletemplateid + | classkey nestednamespecifier Template? simpletemplateid + | Enum nestednamespecifier? Identifier + ; + +enumname + : Identifier + ; + +enumspecifier + : enumhead '{' enumeratorlist? '}' + | enumhead '{' enumeratorlist ',' '}' + ; + +enumhead + : enumkey attributespecifierseq? Identifier? enumbase? + | enumkey attributespecifierseq? nestednamespecifier Identifier enumbase? + ; + +opaqueenumdeclaration + : enumkey attributespecifierseq? Identifier enumbase? ';' + ; + +enumkey + : Enum + | Enum Class + | Enum Struct + ; + +enumbase + : ':' typespecifierseq + ; + +enumeratorlist + : enumeratordefinition + | enumeratorlist ',' enumeratordefinition + ; + +enumeratordefinition + : enumerator + | enumerator '=' constantexpression + ; + +enumerator + : Identifier + ; + +namespacename + : originalnamespacename + | namespacealias + ; + +originalnamespacename + : Identifier + ; + +namespacedefinition + : namednamespacedefinition + | unnamednamespacedefinition + ; + +namednamespacedefinition + : originalnamespacedefinition + | extensionnamespacedefinition + ; + +originalnamespacedefinition + : Inline? Namespace Identifier '{' namespacebody '}' + ; + +extensionnamespacedefinition + : Inline? Namespace originalnamespacename '{' namespacebody '}' + ; + +unnamednamespacedefinition + : Inline? Namespace '{' namespacebody '}' + ; + +namespacebody + : declarationseq? + ; + +namespacealias + : Identifier + ; + +namespacealiasdefinition + : Namespace Identifier '=' qualifiednamespacespecifier ';' + ; + +qualifiednamespacespecifier + : nestednamespecifier? namespacename + ; + +usingdeclaration + : Using Typename? nestednamespecifier unqualifiedid ';' + | Using '::' unqualifiedid ';' + ; + +usingdirective + : attributespecifierseq? Using Namespace nestednamespecifier? namespacename ';' + ; + +asmdefinition + : Asm '(' Stringliteral ')' ';' + ; + +linkagespecification + : Extern Stringliteral '{' declarationseq? '}' + | Extern Stringliteral declaration + ; + +attributespecifierseq + : attributespecifier + | attributespecifierseq attributespecifier + ; + +attributespecifier + : '[' '[' attributelist ']' ']' + | alignmentspecifier + ; + +alignmentspecifier + : Alignas '(' typeid '...'? ')' + | Alignas '(' constantexpression '...'? ')' + ; + +attributelist + : attribute? + | attributelist ',' attribute? + | attribute '...' + | attributelist ',' attribute '...' + ; + +attribute + : attributetoken attributeargumentclause? + ; + +attributetoken + : Identifier + | attributescopedtoken + ; + +attributescopedtoken + : attributenamespace '::' Identifier + ; + +attributenamespace + : Identifier + ; + +attributeargumentclause + : '(' balancedtokenseq ')' + ; + +balancedtokenseq + : balancedtoken? + | balancedtokenseq balancedtoken + ; + +balancedtoken + : '(' balancedtokenseq ')' + | '[' balancedtokenseq ']' + | '{' balancedtokenseq '}' + /*any token other than a parenthesis , a bracket , or a brace*/ + ; + +/*Declarators*/ +initdeclaratorlist + : initdeclarator + | initdeclaratorlist ',' initdeclarator + ; + +initdeclarator + : declarator initializer? + ; + +declarator + : ptrdeclarator + | noptrdeclarator parametersandqualifiers trailingreturntype + ; + +ptrdeclarator + : noptrdeclarator + | ptroperator ptrdeclarator + ; + +noptrdeclarator + : declaratorid attributespecifierseq? + | noptrdeclarator parametersandqualifiers + | noptrdeclarator '[' constantexpression? ']' attributespecifierseq? + | '(' ptrdeclarator ')' + ; + +parametersandqualifiers + : '(' parameterdeclarationclause ')' cvqualifierseq? refqualifier? exceptionspecification? + attributespecifierseq? + ; + +trailingreturntype + : '->' trailingtypespecifierseq abstractdeclarator? + ; + +ptroperator + : '*' attributespecifierseq? cvqualifierseq? + | '&' attributespecifierseq? + | '&&' attributespecifierseq? + | nestednamespecifier '*' attributespecifierseq? cvqualifierseq? + ; + +cvqualifierseq + : cvqualifier cvqualifierseq? + ; + +cvqualifier + : Const + | Volatile + ; + +refqualifier + : '&' + | '&&' + ; + +declaratorid + : '...'? idexpression + ; + +typeid + : typespecifierseq abstractdeclarator? + ; + +abstractdeclarator + : ptrabstractdeclarator + | noptrabstractdeclarator? parametersandqualifiers trailingreturntype + | abstractpackdeclarator + ; + +ptrabstractdeclarator + : noptrabstractdeclarator + | ptroperator ptrabstractdeclarator? + ; + +noptrabstractdeclarator + : noptrabstractdeclarator parametersandqualifiers + | parametersandqualifiers + | noptrabstractdeclarator '[' constantexpression? ']' attributespecifierseq? + | '[' constantexpression? ']' attributespecifierseq? + | '(' ptrabstractdeclarator ')' + ; + +abstractpackdeclarator + : noptrabstractpackdeclarator + | ptroperator abstractpackdeclarator + ; + +noptrabstractpackdeclarator + : noptrabstractpackdeclarator parametersandqualifiers + | noptrabstractpackdeclarator '[' constantexpression? ']' attributespecifierseq? + | '...' + ; + +parameterdeclarationclause + : parameterdeclarationlist? '...'? + | parameterdeclarationlist ',' '...' + ; + +parameterdeclarationlist + : parameterdeclaration + | parameterdeclarationlist ',' parameterdeclaration + ; + +parameterdeclaration + : attributespecifierseq? declspecifierseq declarator + | attributespecifierseq? declspecifierseq declarator '=' initializerclause + | attributespecifierseq? declspecifierseq abstractdeclarator? + | attributespecifierseq? declspecifierseq abstractdeclarator? '=' initializerclause + ; + +functiondefinition + : attributespecifierseq? declspecifierseq? declarator virtspecifierseq? functionbody + ; + +functionbody + : ctorinitializer? compoundstatement + | functiontryblock + | '=' Default ';' + | '=' Delete ';' + ; + +initializer + : braceorequalinitializer + | '(' expressionlist ')' + ; + +braceorequalinitializer + : '=' initializerclause + | bracedinitlist + ; + +initializerclause + : assignmentexpression + | bracedinitlist + ; + +initializerlist + : initializerclause '...'? + | initializerlist ',' initializerclause '...'? + ; + +bracedinitlist + : '{' initializerlist ','? '}' + | '{' '}' + ; + +/*Classes*/ +classname + : Identifier + | simpletemplateid + ; + +classspecifier + : classhead '{' memberspecification? '}' + ; + +classhead + : classkey attributespecifierseq? classheadname classvirtspecifier? baseclause? + | classkey attributespecifierseq? baseclause? + ; + +classheadname + : nestednamespecifier? classname + ; + +classvirtspecifier + : Final + ; + +classkey + : Class + | Struct + | Union + ; + +memberspecification + : memberdeclaration memberspecification? + | accessspecifier ':' memberspecification? + ; + +memberdeclaration + : attributespecifierseq? declspecifierseq? memberdeclaratorlist? ';' + | functiondefinition + | usingdeclaration + | static_assertdeclaration + | templatedeclaration + | aliasdeclaration + | emptydeclaration + ; + +memberdeclaratorlist + : memberdeclarator + | memberdeclaratorlist ',' memberdeclarator + ; + +memberdeclarator + : declarator virtspecifierseq? purespecifier? + | declarator braceorequalinitializer? + | Identifier? attributespecifierseq? ':' constantexpression + ; + +virtspecifierseq + : virtspecifier + | virtspecifierseq virtspecifier + ; + +virtspecifier + : Override + | Final + ; + +/* +purespecifier: + '=' '0'//Conflicts with the lexer + ; + */ +purespecifier + : Assign val = Octalliteral { if ($val.text != "0") throw InputMismatchException(this); } + ; + +/*Derived classes*/ +baseclause + : ':' basespecifierlist + ; + +basespecifierlist + : basespecifier '...'? + | basespecifierlist ',' basespecifier '...'? + ; + +basespecifier + : attributespecifierseq? basetypespecifier + | attributespecifierseq? Virtual accessspecifier? basetypespecifier + | attributespecifierseq? accessspecifier Virtual? basetypespecifier + ; + +classordecltype + : nestednamespecifier? classname + | decltypespecifier + ; + +basetypespecifier + : classordecltype + ; + +accessspecifier + : Private + | Protected + | Public + ; + +/*Special member functions*/ +conversionfunctionid + : Operator conversiontypeid + ; + +conversiontypeid + : typespecifierseq conversiondeclarator? + ; + +conversiondeclarator + : ptroperator conversiondeclarator? + ; + +ctorinitializer + : ':' meminitializerlist + ; + +meminitializerlist + : meminitializer '...'? + | meminitializer '...'? ',' meminitializerlist + ; + +meminitializer + : meminitializerid '(' expressionlist? ')' + | meminitializerid bracedinitlist + ; + +meminitializerid + : classordecltype + | Identifier + ; + +/*Overloading*/ +operatorfunctionid + : Operator operator + ; + +literaloperatorid + : Operator Stringliteral Identifier + | Operator Userdefinedstringliteral + ; + +/*Templates*/ +templatedeclaration + : Template '<' templateparameterlist '>' declaration + ; + +templateparameterlist + : templateparameter + | templateparameterlist ',' templateparameter + ; + +templateparameter + : typeparameter + | parameterdeclaration + ; + +typeparameter + : Class '...'? Identifier? + | Class Identifier? '=' typeid + | Typename '...'? Identifier? + | Typename Identifier? '=' typeid + | Template '<' templateparameterlist '>' Class '...'? Identifier? + | Template '<' templateparameterlist '>' Class Identifier? '=' idexpression + ; + +simpletemplateid + : templatename '<' templateargumentlist? '>' + ; + +templateid + : simpletemplateid + | operatorfunctionid '<' templateargumentlist? '>' + | literaloperatorid '<' templateargumentlist? '>' + ; + +templatename + : Identifier + ; + +templateargumentlist + : templateargument '...'? + | templateargumentlist ',' templateargument '...'? + ; + +templateargument + : typeid + | constantexpression + | idexpression + ; + +typenamespecifier + : Typename nestednamespecifier Identifier + | Typename nestednamespecifier Template? simpletemplateid + ; + +explicitinstantiation + : Extern? Template declaration + ; + +explicitspecialization + : Template '<' '>' declaration + ; + +/*Exception handling*/ +tryblock + : Try compoundstatement handlerseq + ; + +functiontryblock + : Try ctorinitializer? compoundstatement handlerseq + ; + +handlerseq + : handler handlerseq? + ; + +handler + : Catch '(' exceptiondeclaration ')' compoundstatement + ; + +exceptiondeclaration + : attributespecifierseq? typespecifierseq declarator + | attributespecifierseq? typespecifierseq abstractdeclarator? + | '...' + ; + +throwexpression + : Throw assignmentexpression? + ; + +exceptionspecification + : dynamicexceptionspecification + | noexceptspecification + ; + +dynamicexceptionspecification + : Throw '(' typeidlist? ')' + ; + +typeidlist + : typeid '...'? + | typeidlist ',' typeid '...'? + ; + +noexceptspecification + : Noexcept '(' constantexpression ')' + | Noexcept + ; + +/*Preprocessing directives*/ + +MultiLineMacro + : '#' (~[\n]*? '\\' '\r'? '\n')+ ~[\n]+ -> channel(HIDDEN) + ; + +Directive + : '#' ~[\n]* -> channel(HIDDEN) + ; + +/*Lexer*/ + +/*Keywords*/ +Alignas + : 'alignas' + ; + +Alignof + : 'alignof' + ; + +Asm + : 'asm' + ; + +Auto + : 'auto' + ; + +Bool + : 'bool' + ; + +Break + : 'break' + ; + +Case + : 'case' + ; + +Catch + : 'catch' + ; + +Char + : 'char' + ; + +Char16 + : 'char16_t' + ; + +Char32 + : 'char32_t' + ; + +Class + : 'class' + ; + +Const + : 'const' + ; + +Constexpr + : 'constexpr' + ; + +Const_cast + : 'const_cast' + ; + +Continue + : 'continue' + ; + +Decltype + : 'decltype' + ; + +Default + : 'default' + ; + +Delete + : 'delete' + ; + +Do + : 'do' + ; + +Double + : 'double' + ; + +Dynamic_cast + : 'dynamic_cast' + ; + +Else + : 'else' + ; + +Enum + : 'enum' + ; + +Explicit + : 'explicit' + ; + +Export + : 'export' + ; + +Extern + : 'extern' + ; + +False + : 'false' + ; + +Final + : 'final' + ; + +Float + : 'float' + ; + +For + : 'for' + ; + +Friend + : 'friend' + ; + +Goto + : 'goto' + ; + +If + : 'if' + ; + +Inline + : 'inline' + ; + +Int + : 'int' + ; + +Long + : 'long' + ; + +Mutable + : 'mutable' + ; + +Namespace + : 'namespace' + ; + +New + : 'new' + ; + +Noexcept + : 'noexcept' + ; + +Nullptr + : 'nullptr' + ; + +Operator + : 'operator' + ; + +Override + : 'override' + ; + +Private + : 'private' + ; + +Protected + : 'protected' + ; + +Public + : 'public' + ; + +Register + : 'register' + ; + +Reinterpret_cast + : 'reinterpret_cast' + ; + +Return + : 'return' + ; + +Short + : 'short' + ; + +Signed + : 'signed' + ; + +Sizeof + : 'sizeof' + ; + +Static + : 'static' + ; + +Static_assert + : 'static_assert' + ; + +Static_cast + : 'static_cast' + ; + +Struct + : 'struct' + ; + +Switch + : 'switch' + ; + +Template + : 'template' + ; + +This + : 'this' + ; + +Thread_local + : 'thread_local' + ; + +Throw + : 'throw' + ; + +True + : 'true' + ; + +Try + : 'try' + ; + +Typedef + : 'typedef' + ; + +Typeid + : 'typeid' + ; + +Typename + : 'typename' + ; + +Union + : 'union' + ; + +Unsigned + : 'unsigned' + ; + +Using + : 'using' + ; + +Virtual + : 'virtual' + ; + +Void + : 'void' + ; + +Volatile + : 'volatile' + ; + +Wchar + : 'wchar_t' + ; + +While + : 'while' + ; + +/*Operators*/ +LeftParen + : '(' + ; + +RightParen + : ')' + ; + +LeftBracket + : '[' + ; + +RightBracket + : ']' + ; + +LeftBrace + : '{' + ; + +RightBrace + : '}' + ; + +Plus + : '+' + ; + +Minus + : '-' + ; + +Star + : '*' + ; + +Div + : '/' + ; + +Mod + : '%' + ; + +Caret + : '^' + ; + +And + : '&' + ; + +Or + : '|' + ; + +Tilde + : '~' + ; + +Not + : '!' + ; + +Assign + : '=' + ; + +Less + : '<' + ; + +Greater + : '>' + ; + +PlusAssign + : '+=' + ; + +MinusAssign + : '-=' + ; + +StarAssign + : '*=' + ; + +DivAssign + : '/=' + ; + +ModAssign + : '%=' + ; + +XorAssign + : '^=' + ; + +AndAssign + : '&=' + ; + +OrAssign + : '|=' + ; + +LeftShift + : '<<' + ; + +rightShift + : + //'>>' + Greater Greater + ; + +LeftShiftAssign + : '<<=' + ; + +rightShiftAssign + : + //'>>=' + Greater Greater Assign + ; + +Equal + : '==' + ; + +NotEqual + : '!=' + ; + +LessEqual + : '<=' + ; + +GreaterEqual + : '>=' + ; + +AndAnd + : '&&' + ; + +OrOr + : '||' + ; + +PlusPlus + : '++' + ; + +MinusMinus + : '--' + ; + +Comma + : ',' + ; + +ArrowStar + : '->*' + ; + +Arrow + : '->' + ; + +Question + : '?' + ; + +Colon + : ':' + ; + +Doublecolon + : '::' + ; + +Semi + : ';' + ; + +Dot + : '.' + ; + +DotStar + : '.*' + ; + +Ellipsis + : '...' + ; + +operator + : New + | Delete + | New '[' ']' + | Delete '[' ']' + | '+' + | '-' + | '*' + | '/' + | '%' + | '^' + | '&' + | '|' + | '~' + | '!' + | '=' + | '<' + | '>' + | '+=' + | '-=' + | '*=' + | '/=' + | '%=' + | '^=' + | '&=' + | '|=' + | '<<' + | rightShift + | rightShiftAssign + | '<<=' + | '==' + | '!=' + | '<=' + | '>=' + | '&&' + | '||' + | '++' + | '--' + | ',' + | '->*' + | '->' + | '(' ')' + | '[' ']' + ; + +/*Lexer*/ +fragment Hexquad + : HEXADECIMALDIGIT HEXADECIMALDIGIT HEXADECIMALDIGIT HEXADECIMALDIGIT + ; + +fragment Universalcharactername + : '\\u' Hexquad + | '\\U' Hexquad Hexquad + ; + +Identifier + : + /* + Identifiernondigit + | Identifier Identifiernondigit + | Identifier DIGIT + */ Identifiernondigit (Identifiernondigit | DIGIT)* + ; + +fragment Identifiernondigit + : NONDIGIT + | Universalcharactername + /* other implementation defined characters*/ + ; + +fragment NONDIGIT + : [a-zA-Z_] + ; + +fragment DIGIT + : [0-9] + ; + +literal + : Integerliteral + | Characterliteral + | Floatingliteral + | Stringliteral + | booleanliteral + | pointerliteral + | userdefinedliteral + ; + +Integerliteral + : Decimalliteral Integersuffix? + | Octalliteral Integersuffix? + | Hexadecimalliteral Integersuffix? + | Binaryliteral Integersuffix? + ; + +Decimalliteral + : NONZERODIGIT ('\''? DIGIT)* + ; + +Octalliteral + : '0' ('\''? OCTALDIGIT)* + ; + +Hexadecimalliteral + : ('0x' | '0X') HEXADECIMALDIGIT ('\''? HEXADECIMALDIGIT)* + ; + +Binaryliteral + : ('0b' | '0B') BINARYDIGIT ('\''? BINARYDIGIT)* + ; + +fragment NONZERODIGIT + : [1-9] + ; + +fragment OCTALDIGIT + : [0-7] + ; + +fragment HEXADECIMALDIGIT + : [0-9a-fA-F] + ; + +fragment BINARYDIGIT + : [01] + ; + +Integersuffix + : Unsignedsuffix Longsuffix? + | Unsignedsuffix Longlongsuffix? + | Longsuffix Unsignedsuffix? + | Longlongsuffix Unsignedsuffix? + ; + +fragment Unsignedsuffix + : [uU] + ; + +fragment Longsuffix + : [lL] + ; + +fragment Longlongsuffix + : 'll' + | 'LL' + ; + +Characterliteral + : '\'' Cchar+ '\'' + | 'u' '\'' Cchar+ '\'' + | 'U' '\'' Cchar+ '\'' + | 'L' '\'' Cchar+ '\'' + ; + +fragment Cchar + : ~['\\\r\n] + | Escapesequence + | Universalcharactername + ; + +fragment Escapesequence + : Simpleescapesequence + | Octalescapesequence + | Hexadecimalescapesequence + ; + +fragment Simpleescapesequence + : '\\\'' + | '\\"' + | '\\?' + | '\\\\' + | '\\a' + | '\\b' + | '\\f' + | '\\n' + | '\\r' + | '\\t' + | '\\v' + ; + +fragment Octalescapesequence + : '\\' OCTALDIGIT + | '\\' OCTALDIGIT OCTALDIGIT + | '\\' OCTALDIGIT OCTALDIGIT OCTALDIGIT + ; + +fragment Hexadecimalescapesequence + : '\\x' HEXADECIMALDIGIT+ + ; + +Floatingliteral + : Fractionalconstant Exponentpart? Floatingsuffix? + | Digitsequence Exponentpart Floatingsuffix? + ; + +fragment Fractionalconstant + : Digitsequence? '.' Digitsequence + | Digitsequence '.' + ; + +fragment Exponentpart + : 'e' SIGN? Digitsequence + | 'E' SIGN? Digitsequence + ; + +fragment SIGN + : [+-] + ; + +fragment Digitsequence + : DIGIT ('\''? DIGIT)* + ; + +fragment Floatingsuffix + : [flFL] + ; + +Stringliteral + : Encodingprefix? '"' Schar* '"' + | Encodingprefix? 'R' Rawstring + ; + +fragment Encodingprefix + : 'u8' + | 'u' + | 'U' + | 'L' + ; + +fragment Schar + : ~["\\\r\n] + | Escapesequence + | Universalcharactername + ; + +fragment Rawstring /* '"' dcharsequence? '(' rcharsequence? ')' dcharsequence? '"' */ + : '"' .*? '(' .*? ')' .*? '"' + ; + +booleanliteral + : False + | True + ; + +pointerliteral + : Nullptr + ; + +userdefinedliteral + : Userdefinedintegerliteral + | Userdefinedfloatingliteral + | Userdefinedstringliteral + | Userdefinedcharacterliteral + ; + +Userdefinedintegerliteral + : Decimalliteral Udsuffix + | Octalliteral Udsuffix + | Hexadecimalliteral Udsuffix + | Binaryliteral Udsuffix + ; + +Userdefinedfloatingliteral + : Fractionalconstant Exponentpart? Udsuffix + | Digitsequence Exponentpart Udsuffix + ; + +Userdefinedstringliteral + : Stringliteral Udsuffix + ; + +Userdefinedcharacterliteral + : Characterliteral Udsuffix + ; + +fragment Udsuffix + : Identifier + ; + +Whitespace + : [ \t]+ -> skip + ; + +Newline + : ('\r' '\n'? | '\n') -> skip + ; + +BlockComment + : '/*' .*? '*/' -> skip + ; + +LineComment + : '//' ~[\r\n]* -> skip + ; diff --git a/ports/cpp/test/cpp14/Cpp14Test.cpp b/ports/cpp/test/cpp14/Cpp14Test.cpp new file mode 100644 index 0000000..7ac3b95 --- /dev/null +++ b/ports/cpp/test/cpp14/Cpp14Test.cpp @@ -0,0 +1,405 @@ +#include + +#include +#include + +#include + +#include +#include + +namespace c3::test { + +struct Cpp14Grammar { + using Lexer = CPP14Lexer; + using Parser = CPP14Parser; +}; + +TEST(CPP14Parser, SimpleExample) { + // We are trying here to get useful code completion candidates without + // adjusting the grammar in any way. We use the grammar as downloaded from the + // ANTLR grammar directory and set up the c3 engine instead in a way that + // still returns useful info. This limits us somewhat. + const auto *source = "class A {\n" + "public:\n" + " void test() {\n" + " }\n" + "};\n"; + AntlrPipeline pipeline(source); + pipeline.parser.translationunit(); + EXPECT_EQ(pipeline.listener.GetErrorCount(), 0); + + CodeCompletionCore completion(&pipeline.parser); + + // Ignore operators and the generic ID token. + completion.ignoredTokens = { + CPP14Lexer::Identifier, CPP14Lexer::LeftParen, CPP14Lexer::RightParen, + CPP14Lexer::Operator, CPP14Lexer::Star, CPP14Lexer::And, + CPP14Lexer::AndAnd, CPP14Lexer::LeftBracket, CPP14Lexer::Ellipsis, + CPP14Lexer::Doublecolon, CPP14Lexer::Semi, + }; + + // For a C++ grammar you can of course get many candidates of all kind. For + // this test we focus only on a few, namely namespace, class and variable + // references. For variable references there is no own rule, only an + // "idexpression" as part of the primary expression. + completion.preferredRules = { + CPP14Parser::RuleClassname, + CPP14Parser::RuleNamespacename, + CPP14Parser::RuleIdexpression, + }; + + { + // 1) At the input start. + auto candidates = Candidates(completion, 0); + + EXPECT_THAT( + Keys(candidates.tokens), + UnorderedElementsAre( + CPP14Lexer::Extern, CPP14Lexer::Mutable, CPP14Lexer::Register, + CPP14Lexer::Static, CPP14Lexer::Thread_local, CPP14Lexer::Decltype, + CPP14Lexer::Char, CPP14Lexer::Char16, CPP14Lexer::Char32, + CPP14Lexer::Wchar, CPP14Lexer::Bool, CPP14Lexer::Short, + CPP14Lexer::Int, CPP14Lexer::Long, CPP14Lexer::Signed, + CPP14Lexer::Unsigned, CPP14Lexer::Float, CPP14Lexer::Double, + CPP14Lexer::Void, CPP14Lexer::Auto, CPP14Lexer::Class, + CPP14Lexer::Struct, CPP14Lexer::Union, CPP14Lexer::Enum, + CPP14Lexer::Typename, CPP14Lexer::Const, CPP14Lexer::Volatile, + CPP14Lexer::Explicit, CPP14Lexer::Inline, CPP14Lexer::Virtual, + CPP14Lexer::Friend, CPP14Lexer::Typedef, CPP14Lexer::Constexpr, + CPP14Lexer::Alignas, CPP14Lexer::Asm, CPP14Lexer::Namespace, + CPP14Lexer::Using, CPP14Lexer::Static_assert, CPP14Lexer::Template, + CPP14Lexer::EOF)); + + EXPECT_THAT(Keys(candidates.rules), + UnorderedElementsAre(CPP14Parser::RuleClassname, + CPP14Parser::RuleNamespacename, + CPP14Parser::RuleIdexpression)); + + EXPECT_THAT( + candidates.rules[CPP14Parser::RuleNamespacename].ruleList, + ElementsAre( + CPP14Parser::RuleTranslationunit, CPP14Parser::RuleDeclarationseq, + CPP14Parser::RuleDeclaration, CPP14Parser::RuleFunctiondefinition, + CPP14Parser::RuleDeclarator, CPP14Parser::RulePtrdeclarator, + CPP14Parser::RulePtroperator, + CPP14Parser::RuleNestednamespecifier)); + + EXPECT_THAT( + candidates.rules[CPP14Parser::RuleClassname].ruleList, + ElementsAre( + CPP14Parser::RuleTranslationunit, CPP14Parser::RuleDeclarationseq, + CPP14Parser::RuleDeclaration, CPP14Parser::RuleFunctiondefinition, + CPP14Parser::RuleDeclarator, CPP14Parser::RulePtrdeclarator, + CPP14Parser::RulePtroperator, CPP14Parser::RuleNestednamespecifier, + CPP14Parser::RuleTypename)); + } + for (auto translateRulesTopDown : {false, true}) { + // 2) Within the method body. + // Note when counting token indexes: the C++14 grammar skips all + // whitespaces, hence there are no tokens for them. + completion.translateRulesTopDown = translateRulesTopDown; + auto candidates = Candidates(completion, 10); + + const std::vector idexpressionStack = { + CPP14Parser::RuleTranslationunit, + CPP14Parser::RuleDeclarationseq, + CPP14Parser::RuleDeclaration, + CPP14Parser::RuleBlockdeclaration, // TS: +- `RuleFunctiondefinition` + CPP14Parser::RuleSimpledeclaration, // TS: -- + CPP14Parser::RuleDeclspecifierseq, + CPP14Parser::RuleDeclspecifier, + CPP14Parser::RuleTypespecifier, + CPP14Parser::RuleClassspecifier, + CPP14Parser::RuleMemberspecification, + CPP14Parser::RuleMemberspecification, + CPP14Parser::RuleMemberdeclaration, + + CPP14Parser::RuleMemberdeclaratorlist, + CPP14Parser::RuleMemberdeclarator, + CPP14Parser::RuleBraceorequalinitializer, + CPP14Parser::RuleBracedinitlist, + CPP14Parser::RuleInitializerlist, + CPP14Parser::RuleInitializerclause, + + CPP14Parser::RuleAssignmentexpression, + CPP14Parser::RuleLogicalorexpression, + CPP14Parser::RuleLogicalandexpression, + CPP14Parser::RuleInclusiveorexpression, + CPP14Parser::RuleExclusiveorexpression, + CPP14Parser::RuleAndexpression, + CPP14Parser::RuleEqualityexpression, + CPP14Parser::RuleRelationalexpression, + CPP14Parser::RuleShiftexpression, + CPP14Parser::RuleAdditiveexpression, + CPP14Parser::RuleMultiplicativeexpression, + CPP14Parser::RulePmexpression, + CPP14Parser::RuleCastexpression, + CPP14Parser::RuleUnaryexpression, + CPP14Parser::RulePostfixexpression, + CPP14Parser::RulePrimaryexpression, + }; + + EXPECT_THAT(Keys(candidates.rules), + UnorderedElementsAre(CPP14Parser::RuleClassname, + CPP14Parser::RuleNamespacename, + CPP14Parser::RuleIdexpression)); + + EXPECT_THAT(candidates.rules[CPP14Parser::RuleIdexpression].ruleList, + ElementsAreArray(idexpressionStack)); + + EXPECT_THAT(candidates.rules[CPP14Parser::RuleClassname].ruleList, + ElementsAreArray([&] { + auto stack = idexpressionStack; + stack.pop_back(); + for (auto rule : { + CPP14Parser::RuleSimpletypespecifier, + CPP14Parser::RuleNestednamespecifier, + CPP14Parser::RuleTypename, + }) { + stack.emplace_back(rule); + } + return stack; + }())); + + EXPECT_THAT(candidates.rules[CPP14Parser::RuleNamespacename].ruleList, + ElementsAreArray([&] { + auto stack = idexpressionStack; + stack.pop_back(); + for (auto rule : { + CPP14Parser::RuleSimpletypespecifier, + CPP14Parser::RuleNestednamespecifier, + }) { + stack.emplace_back(rule); + } + return stack; + }())); + } + { + // 2) Within the method body. + // Note when counting token indexes: the C++14 grammar skips all + // whitespaces, hence there are no tokens for them. + completion.translateRulesTopDown = true; + auto candidates = Candidates(completion, 10); + + EXPECT_EQ(candidates.tokens.size(), 82); + + EXPECT_THAT(Keys(candidates.tokens), + IsSupersetOf({ + CPP14Lexer::If, + CPP14Lexer::This, + CPP14Lexer::New, + CPP14Lexer::Case, + CPP14Lexer::While, + CPP14Lexer::Throw, + // Fixing issue #12 causes this to be included that was + // previously not returned + CPP14Lexer::Decltype, + })); + + EXPECT_FALSE(candidates.tokens.contains(CPP14Lexer::Override)); + EXPECT_FALSE(candidates.tokens.contains(CPP14Lexer::Export)); + EXPECT_FALSE(candidates.tokens.contains(CPP14Lexer::Private)); + EXPECT_FALSE(candidates.tokens.contains(CPP14Lexer::Protected)); + } +} + +TEST(CPP14Parser, SimpleCppExampleWithErrorsInInput) { + const auto *source = "class A {\n" + "public:\n" + " void test() {\n" + " if ()" + " }\n" + "};\n"; + AntlrPipeline pipeline(source); + pipeline.parser.translationunit(); + EXPECT_EQ(pipeline.listener.GetErrorCount(), 3); + + CodeCompletionCore completion(&pipeline.parser); + + // Ignore operators and the generic ID token. + completion.ignoredTokens = { + CPP14Lexer::Identifier, + // Let parentheses show up in this test, so + // CPP14Lexer.LeftParen, + // CPP14Lexer.RightParen, + CPP14Lexer::Operator, + CPP14Lexer::Star, + CPP14Lexer::And, + CPP14Lexer::AndAnd, + CPP14Lexer::LeftBracket, + CPP14Lexer::Ellipsis, + CPP14Lexer::Doublecolon, + CPP14Lexer::Semi, + }; + + completion.preferredRules = { + CPP14Parser::RuleClassname, + CPP14Parser::RuleNamespacename, + CPP14Parser::RuleIdexpression, + }; + + { + // At the opening parenthesis. + auto candidates = Candidates(completion, 11); + + EXPECT_THAT(Keys(candidates.tokens), + UnorderedElementsAre(CPP14Lexer::LeftParen)); + } + { + // At the closing parenthesis -> again everything in an expression allowed + // (no control flow this time, though). + auto candidates = Candidates(completion, 12); + + EXPECT_EQ(candidates.tokens.size(), 65); + + EXPECT_EQ(candidates.tokens.contains(CPP14Lexer::If), false); + EXPECT_EQ(candidates.tokens.contains(CPP14Lexer::This), true); + EXPECT_EQ(candidates.tokens.contains(CPP14Lexer::New), true); + EXPECT_EQ(candidates.tokens.contains(CPP14Lexer::Case), false); + EXPECT_EQ(candidates.tokens.contains(CPP14Lexer::While), false); + EXPECT_EQ(candidates.tokens.contains(CPP14Lexer::Throw), true); + + EXPECT_EQ(candidates.tokens.contains(CPP14Lexer::Override), false); + EXPECT_EQ(candidates.tokens.contains(CPP14Lexer::Export), false); + EXPECT_EQ(candidates.tokens.contains(CPP14Lexer::Private), false); + EXPECT_EQ(candidates.tokens.contains(CPP14Lexer::Protected), false); + + // Fixing issue #12 causes this to be included that + // was previously not returned + EXPECT_EQ(candidates.tokens.contains(CPP14Lexer::Decltype), true); + } + { + // After the error position -> no suggestions. + auto candidates = Candidates(completion, 13); + + EXPECT_EQ(candidates.tokens.size(), 0); + EXPECT_EQ(candidates.rules.size(), 0); + } +} + +TEST(CPP14Parser, RealCppFile) { + { + auto actual = // + std::filesystem::current_path() // + | std::views::reverse // + | std::views::take(5) // + | std::views::reverse // + | std::views::transform([](auto word) { return word.string(); }) // + | std::ranges::to(); + std::vector expected{"ports", "cpp", "build", "test", "cpp14"}; + EXPECT_EQ(actual, expected); + } + + const auto source = [] { + // Assume we are at antlr4-c3/ports/cpp/build + std::ifstream file("../../../tests/Parser.cpp"); + std::string content((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + return content; + }(); + + AntlrPipeline pipeline(source); + pipeline.parser.translationunit(); + EXPECT_EQ(pipeline.listener.GetErrorCount(), 0); + + CodeCompletionCore completion(&pipeline.parser); + + // Ignore operators and the generic ID token. + completion.ignoredTokens = { + CPP14Lexer::Identifier, CPP14Lexer::LeftParen, CPP14Lexer::RightParen, + CPP14Lexer::Operator, CPP14Lexer::Star, CPP14Lexer::And, + CPP14Lexer::AndAnd, CPP14Lexer::LeftBracket, CPP14Lexer::Ellipsis, + CPP14Lexer::Doublecolon, CPP14Lexer::Semi, + }; + + completion.preferredRules = { + CPP14Parser::RuleClassname, + CPP14Parser::RuleNamespacename, + CPP14Parser::RuleIdexpression, + }; + + std::vector idexpressionStack = { + CPP14Parser::RuleTranslationunit, + CPP14Parser::RuleDeclarationseq, + CPP14Parser::RuleDeclaration, + CPP14Parser::RuleFunctiondefinition, + // These are in TypeScript version, but not in C++ port + // CPP14Parser::RuleFunctionbody, + // CPP14Parser::RuleCompoundstatement, + // CPP14Parser::RuleStatementseq, + // CPP14Parser::RuleStatement, + // CPP14Parser::RuleDeclarationstatement, + // CPP14Parser::RuleBlockdeclaration, + // CPP14Parser::RuleSimpledeclaration, + // CPP14Parser::RuleInitdeclaratorlist, + // CPP14Parser::RuleInitdeclarator, + CPP14Parser::RuleDeclarator, + CPP14Parser::RuleNoptrdeclarator, + CPP14Parser::RuleDeclaratorid, + }; + + std::vector classnameStack = + Concat(idexpressionStack, { + CPP14Parser::RuleIdexpression, + CPP14Parser::RuleQualifiedid, + CPP14Parser::RuleNestednamespecifier, + CPP14Parser::RuleTypename, + }); + + std::vector namespacenameStack = + Concat(idexpressionStack, { + CPP14Parser::RuleIdexpression, + CPP14Parser::RuleQualifiedid, + CPP14Parser::RuleNestednamespecifier, + }); + + { + auto candidates = Candidates(completion, 3469); + + EXPECT_THAT(Keys(candidates.rules), + UnorderedElementsAre(CPP14Parser::RuleClassname, + CPP14Parser::RuleNamespacename, + CPP14Parser::RuleIdexpression)); + + EXPECT_THAT(candidates.rules[CPP14Parser::RuleIdexpression].ruleList, + ElementsAreArray(idexpressionStack)); + } + { + // We should receive more specific rules when translating top down. + + completion.translateRulesTopDown = true; + auto candidates = Candidates(completion, 3469); + + EXPECT_THAT(candidates.rules[CPP14Parser::RuleClassname].ruleList, + ElementsAreArray(classnameStack)); + + EXPECT_THAT(candidates.rules[CPP14Parser::RuleNamespacename].ruleList, + ElementsAreArray(namespacenameStack)); + + // We are starting a primary expression in a function body, so everything + // related to expressions and control flow is allowed here. We only check + // for a few possible keywords. + EXPECT_EQ(candidates.tokens.size(), 40); // TS: 82 + + { // TS: at each statement in this block must be EXPECT_TRUE + EXPECT_FALSE(candidates.tokens.contains(CPP14Lexer::If)); + EXPECT_FALSE(candidates.tokens.contains(CPP14Lexer::This)); + EXPECT_FALSE(candidates.tokens.contains(CPP14Lexer::New)); + EXPECT_FALSE(candidates.tokens.contains(CPP14Lexer::Case)); + EXPECT_FALSE(candidates.tokens.contains(CPP14Lexer::While)); + EXPECT_FALSE(candidates.tokens.contains(CPP14Lexer::Throw)); + } + + // Fixing issue #12 causes this to be included that + // was previously not returned. + EXPECT_TRUE(candidates.tokens.contains(CPP14Lexer::Decltype)); + + EXPECT_FALSE(candidates.tokens.contains(CPP14Lexer::Override)); + EXPECT_FALSE(candidates.tokens.contains(CPP14Lexer::Export)); + EXPECT_FALSE(candidates.tokens.contains(CPP14Lexer::Private)); + EXPECT_FALSE(candidates.tokens.contains(CPP14Lexer::Protected)); + } +} + +} // namespace c3::test diff --git a/ports/cpp/test/utility/Maps.hpp b/ports/cpp/test/utility/Collections.hpp similarity index 61% rename from ports/cpp/test/utility/Maps.hpp rename to ports/cpp/test/utility/Collections.hpp index dfcb893..64cfdbd 100644 --- a/ports/cpp/test/utility/Maps.hpp +++ b/ports/cpp/test/utility/Collections.hpp @@ -13,4 +13,12 @@ template std::vector Keys(const std::map &map) { return keys; } +template +std::vector Concat(std::vector lhs, const std::vector &rhs) { + for (const auto &element : rhs) { + lhs.emplace_back(element); + } + return lhs; +} + } // namespace c3::test diff --git a/ports/cpp/test/utility/Testing.hpp b/ports/cpp/test/utility/Testing.hpp index ea412c4..41a30c5 100644 --- a/ports/cpp/test/utility/Testing.hpp +++ b/ports/cpp/test/utility/Testing.hpp @@ -5,11 +5,14 @@ #include "AntlrPipeline.hpp" #include "Extension.hpp" -#include "Maps.hpp" +#include "Collections.hpp" namespace c3::test { using testing::ElementsAre; +using testing::ElementsAreArray; +using testing::IsSupersetOf; using testing::UnorderedElementsAre; +using testing::UnorderedElementsAreArray; } // namespace c3::test From 38ca2f8da523eb2b8da89bbaea231f0c681e181b Mon Sep 17 00:00:00 2001 From: vityaman Date: Sun, 28 Jul 2024 10:00:11 +0300 Subject: [PATCH 5/8] #132 Run GitHub Workflow only when required Signed-off-by: vityaman --- .github/workflows/cmake.yml | 2 ++ .github/workflows/nodejs.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 7fd10f6..3e0c08b 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -3,8 +3,10 @@ name: CMake on: push: branches: ["main"] + paths: ["ports/cpp/**"] pull_request: branches: ["main"] + paths: ["ports/cpp/**"] jobs: build: diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 680410f..3c7b989 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -6,8 +6,10 @@ name: Build & Test on: push: branches: [ main ] + paths-ignore: ["ports/**"] pull_request: branches: [ main ] + paths-ignore: ["ports/**"] jobs: build: From 9369f788e04319b41f8961038abe2949b4ba8375 Mon Sep 17 00:00:00 2001 From: vityaman Date: Sun, 28 Jul 2024 10:13:51 +0300 Subject: [PATCH 6/8] #132 Migrate to C++ 20 Signed-off-by: vityaman --- .github/workflows/cmake.yml | 6 ++++-- ports/cpp/README.md | 2 +- ports/cpp/cmake/CompileOptions.cmake | 2 +- ports/cpp/test/cpp14/Cpp14Test.cpp | 11 ++--------- ports/cpp/test/whitebox/WhiteboxTest.cpp | 2 +- 5 files changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 3e0c08b..5b1498f 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -16,8 +16,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Install Build Essential - run: sudo apt update && sudo apt install build-essential + - name: Install dependencies + run: | + sudo apt update + sudo apt install build-essential - name: Setup ccache uses: hendrikmuhs/ccache-action@v1.2 with: diff --git a/ports/cpp/README.md b/ports/cpp/README.md index 83e1d18..6614ca6 100644 --- a/ports/cpp/README.md +++ b/ports/cpp/README.md @@ -14,7 +14,7 @@ Please see the parent [README.md](../../readme.md) for an explanation of the lib ## Requirements -- [C++ 23 standard](https://en.cppreference.com/w/cpp/23) to compile sources. +- [C++ 20 standard](https://en.cppreference.com/w/cpp/20) to compile sources. - [ANTLRv4 C++ Runtime](https://github.com/antlr/antlr4/tree/4.13.1/runtime/Cpp) to compile sources. diff --git a/ports/cpp/cmake/CompileOptions.cmake b/ports/cpp/cmake/CompileOptions.cmake index f3c0c44..3d0dfc9 100644 --- a/ports/cpp/cmake/CompileOptions.cmake +++ b/ports/cpp/cmake/CompileOptions.cmake @@ -1,4 +1,4 @@ -set(CMAKE_CXX_STANDARD 23) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/ports/cpp/test/cpp14/Cpp14Test.cpp b/ports/cpp/test/cpp14/Cpp14Test.cpp index 7ac3b95..7b1a647 100644 --- a/ports/cpp/test/cpp14/Cpp14Test.cpp +++ b/ports/cpp/test/cpp14/Cpp14Test.cpp @@ -280,15 +280,8 @@ TEST(CPP14Parser, SimpleCppExampleWithErrorsInInput) { TEST(CPP14Parser, RealCppFile) { { - auto actual = // - std::filesystem::current_path() // - | std::views::reverse // - | std::views::take(5) // - | std::views::reverse // - | std::views::transform([](auto word) { return word.string(); }) // - | std::ranges::to(); - std::vector expected{"ports", "cpp", "build", "test", "cpp14"}; - EXPECT_EQ(actual, expected); + const auto path = std::filesystem::current_path().string(); + EXPECT_TRUE(path.ends_with("ports/cpp/build/test/cpp14")); } const auto source = [] { diff --git a/ports/cpp/test/whitebox/WhiteboxTest.cpp b/ports/cpp/test/whitebox/WhiteboxTest.cpp index ce54b2c..09ca3c5 100644 --- a/ports/cpp/test/whitebox/WhiteboxTest.cpp +++ b/ports/cpp/test/whitebox/WhiteboxTest.cpp @@ -46,7 +46,7 @@ TEST(WhiteboxGrammarTests, CaretAtOneOfMultiplePossibleStates) { for (const auto index : {4, 5, 6, 7}) { AntlrPipeline pipeline("LOREM IPSUM "); - auto *ctx = [&] -> antlr4::ParserRuleContext * { + auto *ctx = [&]() -> antlr4::ParserRuleContext * { switch (index) { case 4: return pipeline.parser.test4(); From 7e0a9c99b7bd202c5fc9d094f0a8a7ff8c4037a5 Mon Sep 17 00:00:00 2001 From: vityaman Date: Sun, 28 Jul 2024 11:09:02 +0300 Subject: [PATCH 7/8] #132 Run CMake GitHub Workflow on workflows change Signed-off-by: vityaman --- .github/workflows/cmake.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 5b1498f..28458b2 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -3,10 +3,14 @@ name: CMake on: push: branches: ["main"] - paths: ["ports/cpp/**"] + paths: + - "ports/cpp/**" + - ".github/workflows/*" pull_request: branches: ["main"] - paths: ["ports/cpp/**"] + paths: + - "ports/cpp/**" + - ".github/workflows/*" jobs: build: From 23ba1111c7e770003b683943bc2b4e52f5969585 Mon Sep 17 00:00:00 2001 From: vityaman Date: Sun, 28 Jul 2024 13:32:21 +0300 Subject: [PATCH 8/8] #132 Add default parameters to collectCandidates Signed-off-by: vityaman --- .../source/antlr4-c3/CodeCompletionCore.hpp | 2 +- ports/cpp/test/cpp14/Cpp14Test.cpp | 17 +++--- ports/cpp/test/expr/ExprTest.cpp | 55 ++++++------------- ports/cpp/test/utility/Extension.hpp | 22 -------- ports/cpp/test/utility/Testing.hpp | 1 - ports/cpp/test/whitebox/WhiteboxTest.cpp | 8 +-- 6 files changed, 29 insertions(+), 76 deletions(-) delete mode 100644 ports/cpp/test/utility/Extension.hpp diff --git a/ports/cpp/source/antlr4-c3/CodeCompletionCore.hpp b/ports/cpp/source/antlr4-c3/CodeCompletionCore.hpp index ce023ea..0792085 100644 --- a/ports/cpp/source/antlr4-c3/CodeCompletionCore.hpp +++ b/ports/cpp/source/antlr4-c3/CodeCompletionCore.hpp @@ -181,7 +181,7 @@ class CodeCompletionCore * @returns The collection of completion candidates. If cancelled or timed out, the returned collection will have its 'cancelled' * value set to true and the collected candidates may be incomplete. */ - CandidatesCollection collectCandidates(size_t caretTokenIndex, antlr4::ParserRuleContext * context, size_t timeoutMS, std::atomic * cancel); + CandidatesCollection collectCandidates(size_t caretTokenIndex, antlr4::ParserRuleContext * context = nullptr, size_t timeoutMS = 0, std::atomic * cancel = nullptr); diff --git a/ports/cpp/test/cpp14/Cpp14Test.cpp b/ports/cpp/test/cpp14/Cpp14Test.cpp index 7b1a647..8af94bf 100644 --- a/ports/cpp/test/cpp14/Cpp14Test.cpp +++ b/ports/cpp/test/cpp14/Cpp14Test.cpp @@ -6,7 +6,6 @@ #include #include -#include namespace c3::test { @@ -51,7 +50,7 @@ TEST(CPP14Parser, SimpleExample) { { // 1) At the input start. - auto candidates = Candidates(completion, 0); + auto candidates = completion.collectCandidates(0); EXPECT_THAT( Keys(candidates.tokens), @@ -99,7 +98,7 @@ TEST(CPP14Parser, SimpleExample) { // Note when counting token indexes: the C++14 grammar skips all // whitespaces, hence there are no tokens for them. completion.translateRulesTopDown = translateRulesTopDown; - auto candidates = Candidates(completion, 10); + auto candidates = completion.collectCandidates(10); const std::vector idexpressionStack = { CPP14Parser::RuleTranslationunit, @@ -180,7 +179,7 @@ TEST(CPP14Parser, SimpleExample) { // Note when counting token indexes: the C++14 grammar skips all // whitespaces, hence there are no tokens for them. completion.translateRulesTopDown = true; - auto candidates = Candidates(completion, 10); + auto candidates = completion.collectCandidates(10); EXPECT_EQ(candidates.tokens.size(), 82); @@ -241,7 +240,7 @@ TEST(CPP14Parser, SimpleCppExampleWithErrorsInInput) { { // At the opening parenthesis. - auto candidates = Candidates(completion, 11); + auto candidates = completion.collectCandidates(11); EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(CPP14Lexer::LeftParen)); @@ -249,7 +248,7 @@ TEST(CPP14Parser, SimpleCppExampleWithErrorsInInput) { { // At the closing parenthesis -> again everything in an expression allowed // (no control flow this time, though). - auto candidates = Candidates(completion, 12); + auto candidates = completion.collectCandidates(12); EXPECT_EQ(candidates.tokens.size(), 65); @@ -271,7 +270,7 @@ TEST(CPP14Parser, SimpleCppExampleWithErrorsInInput) { } { // After the error position -> no suggestions. - auto candidates = Candidates(completion, 13); + auto candidates = completion.collectCandidates(13); EXPECT_EQ(candidates.tokens.size(), 0); EXPECT_EQ(candidates.rules.size(), 0); @@ -348,7 +347,7 @@ TEST(CPP14Parser, RealCppFile) { }); { - auto candidates = Candidates(completion, 3469); + auto candidates = completion.collectCandidates(3469); EXPECT_THAT(Keys(candidates.rules), UnorderedElementsAre(CPP14Parser::RuleClassname, @@ -362,7 +361,7 @@ TEST(CPP14Parser, RealCppFile) { // We should receive more specific rules when translating top down. completion.translateRulesTopDown = true; - auto candidates = Candidates(completion, 3469); + auto candidates = completion.collectCandidates(3469); EXPECT_THAT(candidates.rules[CPP14Parser::RuleClassname].ruleList, ElementsAreArray(classnameStack)); diff --git a/ports/cpp/test/expr/ExprTest.cpp b/ports/cpp/test/expr/ExprTest.cpp index 78446c5..01edd87 100644 --- a/ports/cpp/test/expr/ExprTest.cpp +++ b/ports/cpp/test/expr/ExprTest.cpp @@ -18,15 +18,10 @@ TEST(SimpleExpressionParser, MostSimpleSetup) { EXPECT_EQ(pipeline.listener.GetErrorCount(), 0); c3::CodeCompletionCore completion(&pipeline.parser); - const auto collectCandidatesAt = [&](std::size_t tokenIndex) { - return completion.collectCandidates(tokenIndex, - /*context=*/nullptr, /*size_t=*/0, - /*cancel=*/nullptr); - }; { // 1) At the input start. - auto candidates = collectCandidatesAt(0); + auto candidates = completion.collectCandidates(0); EXPECT_THAT( Keys(candidates.tokens), UnorderedElementsAre(ExprLexer::VAR, ExprLexer::LET, ExprLexer::ID)); @@ -40,17 +35,17 @@ TEST(SimpleExpressionParser, MostSimpleSetup) { // 2) On the first whitespace. In real implementations you would do some // additional checks where in the whitespace the caret is, as the outcome is // different depending on that position. - auto candidates = collectCandidatesAt(1); + auto candidates = completion.collectCandidates(1); EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(ExprLexer::ID)); } { // 3) On the variable name ('c'). - auto candidates = collectCandidatesAt(2); + auto candidates = completion.collectCandidates(2); EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(ExprLexer::ID)); } { // 4) On the equal sign (ignoring whitespace positions from now on). - auto candidates = collectCandidatesAt(4); + auto candidates = completion.collectCandidates(4); EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(ExprLexer::EQUAL)); } @@ -58,13 +53,13 @@ TEST(SimpleExpressionParser, MostSimpleSetup) { // 5) On the variable reference 'a'. But since we have not configure the c3 // engine to return us var refs (or function refs for that matter) we only // get an ID here. - auto candidates = collectCandidatesAt(6); + auto candidates = completion.collectCandidates(6); EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(ExprLexer::ID)); } { // 6) On the '+' operator. Usually you would not show operators as // candidates, but we have not set up the c3 engine yet to not return them. - auto candidates = collectCandidatesAt(8); + auto candidates = completion.collectCandidates(8); EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(ExprLexer::PLUS, ExprLexer::MINUS, ExprLexer::MULTIPLY, ExprLexer::DIVIDE, @@ -87,15 +82,9 @@ TEST(SimpleExpressionParser, TypicalSetup) { ExprParser::RuleVariableRef, }; - const auto collectCandidatesAt = [&](std::size_t tokenIndex) { - return completion.collectCandidates(tokenIndex, - /*context=*/nullptr, /*size_t=*/0, - /*cancel=*/nullptr); - }; - { // 1) At the input start. - auto candidates = collectCandidatesAt(0); + auto candidates = completion.collectCandidates(0); EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(ExprLexer::VAR, ExprLexer::LET)); @@ -105,17 +94,17 @@ TEST(SimpleExpressionParser, TypicalSetup) { } { // 2) On the variable name ('c'). - auto candidates = collectCandidatesAt(2); + auto candidates = completion.collectCandidates(2); EXPECT_EQ(candidates.tokens.size(), 0); } { // 4) On the equal sign. - auto candidates = collectCandidatesAt(4); + auto candidates = completion.collectCandidates(4); EXPECT_EQ(candidates.tokens.size(), 0); } { // 5) On the variable reference 'a'. - auto candidates = collectCandidatesAt(6); + auto candidates = completion.collectCandidates(6); EXPECT_EQ(candidates.tokens.size(), 0); // Here we get 2 rule indexes, derived from 2 different IDs possible at this // caret position. These are what we told the engine above to be preferred @@ -129,7 +118,7 @@ TEST(SimpleExpressionParser, TypicalSetup) { { // 6) On the whitespace just after the variable reference 'a' (but it could // still be a function reference!). - auto candidates = collectCandidatesAt(7); + auto candidates = completion.collectCandidates(7); EXPECT_EQ(candidates.tokens.size(), 0); EXPECT_THAT(Keys(candidates.rules), UnorderedElementsAre(ExprParser::RuleFunctionRef)); @@ -145,15 +134,9 @@ TEST(SimpleExpressionParser, RecursivePreferredRule) { c3::CodeCompletionCore completion(&pipeline.parser); completion.preferredRules = {ExprParser::RuleSimpleExpression}; - const auto collectCandidatesAt = [&](std::size_t tokenIndex) { - return completion.collectCandidates(tokenIndex, - /*context=*/nullptr, /*size_t=*/0, - /*cancel=*/nullptr); - }; - { // 1) On the variable reference 'a'. - auto candidates = collectCandidatesAt(6); + auto candidates = completion.collectCandidates(6); EXPECT_THAT(Keys(candidates.rules), UnorderedElementsAre(ExprParser::RuleSimpleExpression)); // The start token of the simpleExpression rule begins at token 'a'. @@ -163,7 +146,7 @@ TEST(SimpleExpressionParser, RecursivePreferredRule) { { // 2) On the variable reference 'b'. completion.translateRulesTopDown = false; - auto candidates = collectCandidatesAt(10); + auto candidates = completion.collectCandidates(10); EXPECT_THAT(Keys(candidates.rules), UnorderedElementsAre(ExprParser::RuleSimpleExpression)); // When translateRulesTopDown is false, startTokenIndex should match the @@ -175,7 +158,7 @@ TEST(SimpleExpressionParser, RecursivePreferredRule) { { // 3) On the variable reference 'b' topDown preferred rules. completion.translateRulesTopDown = true; - auto candidates = collectCandidatesAt(10); + auto candidates = completion.collectCandidates(10); EXPECT_THAT(Keys(candidates.rules), UnorderedElementsAre(ExprParser::RuleSimpleExpression)); // When translateRulesTopDown is true, startTokenIndex should match the @@ -198,15 +181,9 @@ TEST(SimpleExpressionParser, CandidateRulesWithDifferentStartTokens) { }; completion.translateRulesTopDown = true; - const auto collectCandidatesAt = [&](std::size_t tokenIndex) { - return completion.collectCandidates(tokenIndex, - /*context=*/nullptr, /*size_t=*/0, - /*cancel=*/nullptr); - }; - { // 1) On the token 'var'. - auto candidates = collectCandidatesAt(0); + auto candidates = completion.collectCandidates(0); EXPECT_THAT(Keys(candidates.rules), UnorderedElementsAre(ExprParser::RuleAssignment, ExprParser::RuleVariableRef)); @@ -217,7 +194,7 @@ TEST(SimpleExpressionParser, CandidateRulesWithDifferentStartTokens) { } { // 2) On the variable reference 'a'. - auto candidates = collectCandidatesAt(6); + auto candidates = completion.collectCandidates(6); EXPECT_THAT(Keys(candidates.rules), UnorderedElementsAre(ExprParser::RuleAssignment, ExprParser::RuleVariableRef)); diff --git a/ports/cpp/test/utility/Extension.hpp b/ports/cpp/test/utility/Extension.hpp deleted file mode 100644 index b06c9d9..0000000 --- a/ports/cpp/test/utility/Extension.hpp +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include - -namespace c3::test { - -inline auto Candidates(CodeCompletionCore &completion, - std::size_t caretTokenIndex) { - return completion.collectCandidates(caretTokenIndex, - /*context=*/nullptr, /*size_t=*/0, - /*cancel=*/nullptr); -} - -inline auto Candidates(CodeCompletionCore &completion, - std::size_t caretTokenIndex, - antlr4::ParserRuleContext *context) { - return completion.collectCandidates(caretTokenIndex, - /*context=*/context, /*size_t=*/0, - /*cancel=*/nullptr); -} - -} // namespace c3::test \ No newline at end of file diff --git a/ports/cpp/test/utility/Testing.hpp b/ports/cpp/test/utility/Testing.hpp index 41a30c5..81b1b20 100644 --- a/ports/cpp/test/utility/Testing.hpp +++ b/ports/cpp/test/utility/Testing.hpp @@ -4,7 +4,6 @@ #include #include "AntlrPipeline.hpp" -#include "Extension.hpp" #include "Collections.hpp" namespace c3::test { diff --git a/ports/cpp/test/whitebox/WhiteboxTest.cpp b/ports/cpp/test/whitebox/WhiteboxTest.cpp index 09ca3c5..e43f6a8 100644 --- a/ports/cpp/test/whitebox/WhiteboxTest.cpp +++ b/ports/cpp/test/whitebox/WhiteboxTest.cpp @@ -19,7 +19,7 @@ TEST(WhiteboxGrammarTests, CaretAtTransitionToRuleWithNonExhaustiveFollowSet) { EXPECT_EQ(pipeline.listener.GetErrorCount(), 1); c3::CodeCompletionCore completion(&pipeline.parser); - auto candidates = Candidates(completion, 1, ctx); + auto candidates = completion.collectCandidates(1, ctx); EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(WhiteboxLexer::IPSUM, WhiteboxLexer::DOLOR, @@ -34,7 +34,7 @@ TEST(WhiteboxGrammarTests, CaretAtTransitionToRuleWithEmptyFollowSet) { EXPECT_EQ(pipeline.listener.GetErrorCount(), 1); c3::CodeCompletionCore completion(&pipeline.parser); - auto candidates = Candidates(completion, 1, ctx); + auto candidates = completion.collectCandidates(1, ctx); EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(WhiteboxLexer::IPSUM, WhiteboxLexer::DOLOR, @@ -62,7 +62,7 @@ TEST(WhiteboxGrammarTests, CaretAtOneOfMultiplePossibleStates) { }(); c3::CodeCompletionCore completion(&pipeline.parser); - auto candidates = Candidates(completion, 2, ctx); + auto candidates = completion.collectCandidates(2, ctx); EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(WhiteboxLexer::DOLOR)); @@ -78,7 +78,7 @@ TEST(WhiteboxGrammarTests, auto *ctx = pipeline.parser.test8(); c3::CodeCompletionCore completion(&pipeline.parser); - auto candidates = Candidates(completion, 2, ctx); + auto candidates = completion.collectCandidates(2, ctx); EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(WhiteboxLexer::DOLOR));