From e1480a02af1ab35eac93d2b74507a3225336cf81 Mon Sep 17 00:00:00 2001 From: EvilBeaver Date: Tue, 10 Oct 2023 16:53:46 +0300 Subject: [PATCH 1/8] =?UTF-8?q?=D0=9F=D0=BE=D0=B4=D0=B4=D0=B5=D1=80=D0=B6?= =?UTF-8?q?=D0=BA=D0=B0=20=D0=BF=D0=B0=D1=80=D1=81=D0=B5=D1=80=D0=BE=D0=BC?= =?UTF-8?q?=20=D0=90=D1=81=D0=B8=D0=BD=D1=85=20=D0=B8=20=D0=96=D0=B4=D0=B0?= =?UTF-8?q?=D1=82=D1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/OneScript.Language/LanguageDef.cs | 32 +-- src/OneScript.Language/LexemTrie.cs | 213 ------------------ .../LexicalAnalysis/Token.cs | 2 + .../SyntaxAnalysis/AstNodes/MethodNode.cs | 2 + .../AstNodes/UnaryOperationNode.cs | 2 +- .../SyntaxAnalysis/BslSyntaxWalker.cs | 10 + .../SyntaxAnalysis/DefaultBslParser.cs | 62 ++++- .../SyntaxAnalysis/LocalizedErrors.cs | 5 + .../SyntaxAnalysis/NodeKind.cs | 3 +- .../SyntaxAnalysis/RegionDirectiveHandler.cs | 4 +- .../OneScript.Language.Tests/ParserTests.cs | 67 ++++++ .../OneScript.Language.Tests/TestAstNode.cs | 7 +- .../TreeValidatorExtensions.cs | 35 +++ .../OneScript.Language.Tests/TrieTests.cs | 6 +- 14 files changed, 198 insertions(+), 252 deletions(-) delete mode 100644 src/OneScript.Language/LexemTrie.cs diff --git a/src/OneScript.Language/LanguageDef.cs b/src/OneScript.Language/LanguageDef.cs index fadd827f3..25cf484cc 100644 --- a/src/OneScript.Language/LanguageDef.cs +++ b/src/OneScript.Language/LanguageDef.cs @@ -17,16 +17,13 @@ public static class LanguageDef static readonly Dictionary _priority = new Dictionary(); public const int MAX_OPERATION_PRIORITY = 8; - private static readonly LexemTrie _stringToToken = new LexemTrie(); + private static readonly IdentifiersTrie _stringToToken = new IdentifiersTrie(); - private static readonly LexemTrie _undefined = new LexemTrie(); - private static readonly LexemTrie _booleans = new LexemTrie(); - private static readonly LexemTrie _logicalOp = new LexemTrie(); + private static readonly IdentifiersTrie _undefined = new IdentifiersTrie(); + private static readonly IdentifiersTrie _booleans = new IdentifiersTrie(); + private static readonly IdentifiersTrie _logicalOp = new IdentifiersTrie(); - private static readonly LexemTrie _preprocRegion = new LexemTrie(); - private static readonly LexemTrie _preprocEndRegion = new LexemTrie(); - - private static readonly LexemTrie _preprocImport = new LexemTrie(); + private static readonly IdentifiersTrie _preprocImport = new IdentifiersTrie(); const int BUILTINS_INDEX = (int)Token.ByValParam; @@ -134,6 +131,8 @@ static LanguageDef() AddToken(Token.Equal, "="); AddToken(Token.Semicolon, ";"); AddToken(Token.Question, "?"); + AddToken(Token.Tilde, "~"); + AddToken(Token.Colon, ":"); #endregion @@ -216,11 +215,6 @@ static LanguageDef() #endregion - _preprocRegion.Add("Область",true); - _preprocRegion.Add("Region", true); - _preprocEndRegion.Add("КонецОбласти", true); - _preprocEndRegion.Add("EndRegion", true); - _preprocImport.Add("Использовать", true); _preprocImport.Add("Use", true); } @@ -410,18 +404,6 @@ public static bool IsLogicalOperatorString(string content) return _logicalOp.TryGetValue(content, out var nodeIsFilled) && nodeIsFilled; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool IsPreprocRegion(string value) - { - return _preprocRegion.TryGetValue(value, out var nodeIsFilled) && nodeIsFilled; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool IsPreprocEndRegion(string value) - { - return _preprocEndRegion.TryGetValue(value, out var nodeIsFilled) && nodeIsFilled; - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsImportDirective(string value) { diff --git a/src/OneScript.Language/LexemTrie.cs b/src/OneScript.Language/LexemTrie.cs deleted file mode 100644 index 5263b4e15..000000000 --- a/src/OneScript.Language/LexemTrie.cs +++ /dev/null @@ -1,213 +0,0 @@ -/*---------------------------------------------------------- -This Source Code Form is subject to the terms of the -Mozilla Public License, v.2.0. If a copy of the MPL -was not distributed with this file, You can obtain one -at http://mozilla.org/MPL/2.0/. -----------------------------------------------------------*/ - -using System.Collections.Generic; -using System.Diagnostics; - -namespace OneScript.Language -{ - public class LexemTrie - { - private class TrieNode - { - public char UCase; - public char LCase; - - public TrieNode next; - public TrieNode sibling; - public T value; - } - - private static int _alphabetLength; - private TrieNode[] _alphabet; - private int _count; - - static LexemTrie() - { - var ru = "абвгдеёжзийклмнопрстуфхцчшщьыъэюя"; - var en = "abcdefghijklmnopqrstuvwxyz"; - var symbols = @"+-*/\()[].,<>=;?%0123456789"; - - var all = symbols + - ru + - ru.ToUpper() + - en + - en.ToUpper(); - - _alphabetLength = all.Length; - } - - public LexemTrie() - { - _alphabet = new TrieNode[_alphabetLength]; - } - - public int Count => _count; - - private static int GetIndex(char c) - { - var code = (int) c; - - if (code >= 1040 && code <= 1103) - { - return code - 960; - } - - if (code >= 40 && code <= 57) - { - return code - 39; - } - - if (code >= 59 && code <= 63) - { - return code - 40; - } - - if (code >= 65 && code <= 93) - { - return code - 41; - } - - if (code >= 97 && code <= 122) - { - return code - 44; - } - - switch (c) - { - case '%': - return 0; - case 'Ё': - return 79; - case 'ё': - return 144; - } - - return -1; - } - - private TrieNode GetValueNode(string key) - { - var index = GetIndex(key[0]); - if (index == -1) - return null; - - var node = _alphabet[index]; - if (node == null) - { - node = new TrieNode(); - node.LCase = char.ToLower(key[0]); - node.UCase = char.ToUpper(key[0]); - _alphabet[GetIndex(node.LCase)] = node; - _alphabet[GetIndex(node.UCase)] = node; - } - - for (int i = 1; i < key.Length; i++) - { - var current = node; - node = node.next; - if (node == null) - { - var newNode = new TrieNode(); - newNode.LCase = char.ToLower(key[i]); - newNode.UCase = char.ToUpper(key[i]); - current.next = newNode; - node = newNode; - } - else if (node.LCase != key[i] && node.UCase != key[i]) - { - var insert = node.sibling; - while (insert != null) - { - if (insert.LCase == key[i] || insert.UCase == key[i]) - { - node = insert; - break; - } - - node = insert; - insert = insert.sibling; - } - - if (insert == null) - { - var newNode = new TrieNode(); - newNode.LCase = char.ToLower(key[i]); - newNode.UCase = char.ToUpper(key[i]); - node.sibling = newNode; - node = newNode; - } - } - } - - return node; - } - - public void Add(string key, T value) - { - var node = GetValueNode(key); - Debug.Assert(node != null); - - node.value = value; - ++_count; - } - - public T Get(string key) - { - var node = FindNode(key); - if (node == null) - { - throw new KeyNotFoundException(); - } - - return node.value; - } - - private TrieNode FindNode(string key) - { - var index = GetIndex(key[0]); - if (index == -1 || _alphabet[index] == null) - return null; - - var node = _alphabet[index]; - for (int i = 1; i < key.Length; i++) - { - node = node.next; - if (node == null) - return null; - - while(node.LCase != key[i] && node.UCase != key[i]) - { - node = node.sibling; - if(node == null) - return null; - } - } - - return node; - } - - public bool TryGetValue(string key, out T value) - { - var node = FindNode(key); - if (node == null) - { - value = default(T); - return false; - } - - value = node.value; - return true; - } - - public T this[string key] - { - get => Get(key); - set => Add(key, value); - } - } -} diff --git a/src/OneScript.Language/LexicalAnalysis/Token.cs b/src/OneScript.Language/LexicalAnalysis/Token.cs index fc3fd7824..0a78f9df1 100644 --- a/src/OneScript.Language/LexicalAnalysis/Token.cs +++ b/src/OneScript.Language/LexicalAnalysis/Token.cs @@ -43,6 +43,8 @@ public enum Token RemoveHandler, Async, Await, + Tilde, + Colon, // operators Plus, diff --git a/src/OneScript.Language/SyntaxAnalysis/AstNodes/MethodNode.cs b/src/OneScript.Language/SyntaxAnalysis/AstNodes/MethodNode.cs index d7de41a27..07aa269a3 100644 --- a/src/OneScript.Language/SyntaxAnalysis/AstNodes/MethodNode.cs +++ b/src/OneScript.Language/SyntaxAnalysis/AstNodes/MethodNode.cs @@ -17,6 +17,8 @@ public MethodNode() : base(NodeKind.Method) { } + public bool IsAsync { get; set; } + public MethodSignatureNode Signature { get; private set; } public BslSyntaxNode MethodBody { get; private set; } diff --git a/src/OneScript.Language/SyntaxAnalysis/AstNodes/UnaryOperationNode.cs b/src/OneScript.Language/SyntaxAnalysis/AstNodes/UnaryOperationNode.cs index dddcaf793..05e9b80cd 100644 --- a/src/OneScript.Language/SyntaxAnalysis/AstNodes/UnaryOperationNode.cs +++ b/src/OneScript.Language/SyntaxAnalysis/AstNodes/UnaryOperationNode.cs @@ -13,7 +13,7 @@ public class UnaryOperationNode : NonTerminalNode { public Token Operation { get; } - public UnaryOperationNode(Lexem operation) : base(NodeKind.UnaryOperation) + public UnaryOperationNode(Lexem operation) : base(NodeKind.UnaryOperation, operation) { Operation = operation.Token; } diff --git a/src/OneScript.Language/SyntaxAnalysis/BslSyntaxWalker.cs b/src/OneScript.Language/SyntaxAnalysis/BslSyntaxWalker.cs index fbeae0c73..855ead38a 100644 --- a/src/OneScript.Language/SyntaxAnalysis/BslSyntaxWalker.cs +++ b/src/OneScript.Language/SyntaxAnalysis/BslSyntaxWalker.cs @@ -158,10 +158,20 @@ protected virtual void VisitStatement(BslSyntaxNode statement) VisitGlobalProcedureCall(statement as CallNode); else if (statement.Kind == NodeKind.DereferenceOperation) VisitProcedureDereference(statement); + else if (statement.Kind == NodeKind.UnaryOperation && statement is UnaryOperationNode + { + Operation: Token.Await + } unaryOp) + VisitGlobalAwaitCall(unaryOp); else DefaultVisit(statement); } + private void VisitGlobalAwaitCall(UnaryOperationNode awaitStatement) + { + VisitStatement(awaitStatement.Children[0]); + } + protected virtual void VisitAssignment(BslSyntaxNode assignment) { var left = assignment.Children[0]; diff --git a/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs b/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs index 87e99f83f..5ab6a8f3e 100644 --- a/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs +++ b/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs @@ -27,6 +27,7 @@ public class DefaultBslParser private bool _isMethodsDefined; private bool _isStatementsDefined; private bool _isInFunctionScope; + private bool _isInAsyncMethod; private bool _lastDereferenceIsWritable; private readonly Stack _tokenStack = new Stack(); @@ -286,7 +287,8 @@ private void BuildMethodsSection() { if (_lastExtractedLexem.Type != LexemType.Annotation && _lastExtractedLexem.Token != Token.Procedure - && _lastExtractedLexem.Token != Token.Function) + && _lastExtractedLexem.Token != Token.Function + && _lastExtractedLexem.Token != Token.Async) { return; } @@ -301,7 +303,7 @@ private void BuildMethodsSection() while (true) { BuildAnnotations(); - if (_lastExtractedLexem.Token == Token.Procedure || _lastExtractedLexem.Token == Token.Function) + if (IsStartOfMethod(_lastExtractedLexem)) { if (!sectionExist) { @@ -324,14 +326,26 @@ private void BuildMethodsSection() } } + private static bool IsStartOfMethod(in Lexem lex) + { + return lex.Token == Token.Async || lex.Token == Token.Procedure || lex.Token == Token.Function; + } + private void BuildMethod() { - Debug.Assert(_lastExtractedLexem.Token == Token.Procedure || _lastExtractedLexem.Token == Token.Function); + Debug.Assert(IsStartOfMethod(_lastExtractedLexem)); var method = _nodeContext.AddChild(new MethodNode()); ApplyAnnotations(method); PushContext(method); + if (_lastExtractedLexem.Token == Token.Async) + { + method.IsAsync = true; + _isInAsyncMethod = true; + NextLexem(); + } + try { BuildMethodSignature(); @@ -345,6 +359,7 @@ private void BuildMethod() _isInFunctionScope = false; _inMethodScope = false; _isStatementsDefined = false; + _isInAsyncMethod = false; PopContext(); } } @@ -382,9 +397,8 @@ private void BuildMethodBody() private void BuildMethodSignature() { - var isFunction = _lastExtractedLexem.Token == Token.Function; - var signature = _nodeContext.AddChild(new MethodSignatureNode(_lastExtractedLexem)); + var isFunction = _lastExtractedLexem.Token == Token.Function; CreateChild(signature, isFunction? NodeKind.Function : NodeKind.Procedure, _lastExtractedLexem); _isInFunctionScope = isFunction; NextLexem(); @@ -676,7 +690,10 @@ private void BuildComplexStructureStatement() case Token.AddHandler: case Token.RemoveHandler: BuildEventHandlerOperation(_lastExtractedLexem.Token); - break; + break; + case Token.Await: + BuildGlobalCallAwaitOperator(); + break; default: var expected = _tokenStack.Peek(); AddError(LocalizedErrors.TokenExpected(expected)); @@ -684,6 +701,35 @@ private void BuildComplexStructureStatement() } } + private void BuildGlobalCallAwaitOperator() + { + Debug.Assert(_lastExtractedLexem.Token == Token.Await); + + _nodeContext.AddChild(TerminalNode()); + } + + + private BslSyntaxNode BuildExpressionAwaitOperator(in Lexem lexem) + { + Debug.Assert(_lastExtractedLexem.Token == Token.Await); + CheckAsyncMethod(); + var awaitOperator = new UnaryOperationNode(lexem); + NextLexem(); + + var call = BuildGlobalCall(_lastExtractedLexem); + awaitOperator.AddChild(call); + + return awaitOperator; + } + + private void CheckAsyncMethod() + { + if (!_isInAsyncMethod) + { + AddError(LocalizedErrors.AwaitMustBeInAsyncMethod(), false); + } + } + private void BuildIfStatement() { var condition = _nodeContext.AddChild(new ConditionNode(_lastExtractedLexem)); @@ -1330,6 +1376,10 @@ private BslSyntaxNode TerminalNode() { node = BuildGlobalCall(_lastExtractedLexem); } + else if (_lastExtractedLexem.Token == Token.Await) + { + node = BuildExpressionAwaitOperator(_lastExtractedLexem); + } else { AddError(LocalizedErrors.ExpressionSyntax()); diff --git a/src/OneScript.Language/SyntaxAnalysis/LocalizedErrors.cs b/src/OneScript.Language/SyntaxAnalysis/LocalizedErrors.cs index b2f8b73fe..b069f1656 100644 --- a/src/OneScript.Language/SyntaxAnalysis/LocalizedErrors.cs +++ b/src/OneScript.Language/SyntaxAnalysis/LocalizedErrors.cs @@ -60,6 +60,11 @@ public static CodeError ExportedLocalVar(string varName) $"Local variable can't be exported ({varName})"); } + public static CodeError AwaitMustBeInAsyncMethod() => Create( + "Оператор Ждать (Await) может употребляться только в асинхронных процедурах или функциях", + "Operator Await can be used only in async procedures or functions" + ); + public static CodeError LiteralExpected() => Create("Ожидается константа", "Constant expected"); public static CodeError NumberExpected() => Create("Ожидается числовая константа", "Numeric constant expected"); diff --git a/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs b/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs index bc76bca70..855402c39 100644 --- a/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs +++ b/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs @@ -61,6 +61,7 @@ public enum NodeKind RemoveHandler, Preprocessor, Import, - TopLevelExpression + TopLevelExpression, + Async } } \ No newline at end of file diff --git a/src/OneScript.Language/SyntaxAnalysis/RegionDirectiveHandler.cs b/src/OneScript.Language/SyntaxAnalysis/RegionDirectiveHandler.cs index 90a05ad03..d659ce4da 100644 --- a/src/OneScript.Language/SyntaxAnalysis/RegionDirectiveHandler.cs +++ b/src/OneScript.Language/SyntaxAnalysis/RegionDirectiveHandler.cs @@ -12,8 +12,8 @@ namespace OneScript.Language.SyntaxAnalysis { public class RegionDirectiveHandler : DirectiveHandlerBase { - private readonly LexemTrie _preprocRegion = new LexemTrie(); - private readonly LexemTrie _preprocEndRegion = new LexemTrie(); + private readonly IdentifiersTrie _preprocRegion = new IdentifiersTrie(); + private readonly IdentifiersTrie _preprocEndRegion = new IdentifiersTrie(); private int _regionsNesting = 0; diff --git a/src/Tests/OneScript.Language.Tests/ParserTests.cs b/src/Tests/OneScript.Language.Tests/ParserTests.cs index ec6144928..afd6cdf04 100644 --- a/src/Tests/OneScript.Language.Tests/ParserTests.cs +++ b/src/Tests/OneScript.Language.Tests/ParserTests.cs @@ -1168,6 +1168,73 @@ public void TestLocalExportVar() }); } + [Fact] + public void TestAsyncProcedure() + { + var code = + @"Асинх Процедура Проц1() + Перем Переменная; + КонецПроцедуры"; + + var node = ParseModuleAndGetValidator(code); + node.Is(NodeKind.MethodsSection); + + var methodNode = node.NextChild().Is(NodeKind.Method); + + methodNode.CurrentNode.RealNode.As().IsAsync.Should().BeTrue(); + } + + [Fact] + public void TestAsyncFunction() + { + var code = + @"Асинх Функция Ф1() + Перем Переменная; + КонецФункции"; + + var node = ParseModuleAndGetValidator(code); + node.Is(NodeKind.MethodsSection); + + var methodNode = node.NextChild().Is(NodeKind.Method); + + methodNode.CurrentNode.RealNode.As().IsAsync.Should().BeTrue(); + } + + [Fact] + public void TestAwaitPriority() + { + var code = + @"Асинх Процедура Проц1() + А = Ждать Вызов().Поле; + КонецПроцедуры"; + + var node = ParseModuleAndGetValidator(code); + node.Is(NodeKind.MethodsSection); + + var method = node.NextChild().Is(NodeKind.Method); + var expression = method.DownTo(NodeKind.Assignment) + .NextChildIs(NodeKind.Identifier) + .NextChild(); + + expression.CurrentNode.RealNode.As().Operation.Should().Be(Token.Await); + expression + .NextChildIs(NodeKind.DereferenceOperation) + .NoMoreChildren(); + } + + [Fact] + public void TestAwaitMustBeInAsyncOnly() + { + var code = + @"Процедура Проц1() + Ждать Операция(); + КонецПроцедуры"; + + CatchParsingError(code, errors => + { + errors.Single().Description.Should().Contain("Await"); + }); + } private static void CatchParsingError(string code) { diff --git a/src/Tests/OneScript.Language.Tests/TestAstNode.cs b/src/Tests/OneScript.Language.Tests/TestAstNode.cs index 613371016..e909e7952 100644 --- a/src/Tests/OneScript.Language.Tests/TestAstNode.cs +++ b/src/Tests/OneScript.Language.Tests/TestAstNode.cs @@ -31,7 +31,7 @@ public TestAstNode(BslSyntaxNode node) BinaryOperationNode binary => binary.Operation.ToString(), UnaryOperationNode unary => unary.Operation.ToString(), PreprocessorDirectiveNode preproc => preproc.DirectiveName, - _ => Value + _ => nonTerm.ToString() }; } else if(node is TerminalNode term) @@ -39,6 +39,11 @@ public TestAstNode(BslSyntaxNode node) _childrenLazy = new Lazy>(new TestAstNode[0]); Value = term.Lexem.Content; } + else + { + _childrenLazy = new Lazy>(new TestAstNode[0]); + Value = node.ToString(); + } } public string Value { get; set; } diff --git a/src/Tests/OneScript.Language.Tests/TreeValidatorExtensions.cs b/src/Tests/OneScript.Language.Tests/TreeValidatorExtensions.cs index 93f319b0d..1b246c6cb 100644 --- a/src/Tests/OneScript.Language.Tests/TreeValidatorExtensions.cs +++ b/src/Tests/OneScript.Language.Tests/TreeValidatorExtensions.cs @@ -27,6 +27,41 @@ public static SyntaxTreeValidator Is(this SyntaxTreeValidator validator, NodeKin validator.CurrentNode.Kind.Should().Be(type); return validator; } + + public static SyntaxTreeValidator DownTo(this SyntaxTreeValidator validator, NodeKind type) + { + var node = DownTo(validator.CurrentNode, type); + if (node == null) + { + throw new Exception("No such child: " + type); + } + + return new SyntaxTreeValidator(node); + } + + private static TestAstNode DownTo(this TestAstNode node, NodeKind type) + { + if (node.Kind == type) + return node; + + if (node.Children.Count == 0) + return null; + + foreach (var childNode in node.ChildrenList) + { + if (childNode.Kind == type) + return childNode; + + if (childNode.ChildrenList.Count == 0) + continue; + + var result = DownTo(childNode, type); + if (result != null) + return result; + } + + return null; + } public static void Is(this TestAstNode node, NodeKind type) { diff --git a/src/Tests/OneScript.Language.Tests/TrieTests.cs b/src/Tests/OneScript.Language.Tests/TrieTests.cs index f6c137daa..0c476e7d0 100644 --- a/src/Tests/OneScript.Language.Tests/TrieTests.cs +++ b/src/Tests/OneScript.Language.Tests/TrieTests.cs @@ -12,9 +12,9 @@ namespace OneScript.Language.Tests public class TrieTests { [Fact] - public void LexemTrieAdd() + public void IdentifiersTrieAdd() { - var t = new LexemTrie(); + var t = new IdentifiersTrie(); t.Add("Иван", 0); t.Add("Иволга", 1); @@ -28,7 +28,7 @@ public void LexemTrieAdd() [Fact] public void Tokens() { - var t = new LexemTrie(); + var t = new IdentifiersTrie(); t.Add("иначе", 1); t.Add("иначеесли", 2); From a45e044665b082745ad5e9ff4ebc444a7ebd4171 Mon Sep 17 00:00:00 2001 From: EvilBeaver Date: Tue, 10 Oct 2023 16:53:46 +0300 Subject: [PATCH 2/8] =?UTF-8?q?=D0=9F=D0=BE=D0=B4=D0=B4=D0=B5=D1=80=D0=B6?= =?UTF-8?q?=D0=BA=D0=B0=20=D0=BF=D0=B0=D1=80=D1=81=D0=B5=D1=80=D0=BE=D0=BC?= =?UTF-8?q?=20=D0=90=D1=81=D0=B8=D0=BD=D1=85=20=D0=B8=20=D0=96=D0=B4=D0=B0?= =?UTF-8?q?=D1=82=D1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/OneScript.Language/LanguageDef.cs | 32 +-- src/OneScript.Language/LexemTrie.cs | 213 ------------------ .../LexicalAnalysis/Token.cs | 2 + .../SyntaxAnalysis/AstNodes/MethodNode.cs | 2 + .../AstNodes/UnaryOperationNode.cs | 2 +- .../SyntaxAnalysis/BslSyntaxWalker.cs | 10 + .../SyntaxAnalysis/DefaultBslParser.cs | 62 ++++- .../SyntaxAnalysis/LocalizedErrors.cs | 5 + .../SyntaxAnalysis/NodeKind.cs | 3 +- .../SyntaxAnalysis/RegionDirectiveHandler.cs | 4 +- .../OneScript.Language.Tests/ParserTests.cs | 67 ++++++ .../OneScript.Language.Tests/TestAstNode.cs | 7 +- .../TreeValidatorExtensions.cs | 35 +++ .../OneScript.Language.Tests/TrieTests.cs | 6 +- 14 files changed, 198 insertions(+), 252 deletions(-) delete mode 100644 src/OneScript.Language/LexemTrie.cs diff --git a/src/OneScript.Language/LanguageDef.cs b/src/OneScript.Language/LanguageDef.cs index fadd827f3..25cf484cc 100644 --- a/src/OneScript.Language/LanguageDef.cs +++ b/src/OneScript.Language/LanguageDef.cs @@ -17,16 +17,13 @@ public static class LanguageDef static readonly Dictionary _priority = new Dictionary(); public const int MAX_OPERATION_PRIORITY = 8; - private static readonly LexemTrie _stringToToken = new LexemTrie(); + private static readonly IdentifiersTrie _stringToToken = new IdentifiersTrie(); - private static readonly LexemTrie _undefined = new LexemTrie(); - private static readonly LexemTrie _booleans = new LexemTrie(); - private static readonly LexemTrie _logicalOp = new LexemTrie(); + private static readonly IdentifiersTrie _undefined = new IdentifiersTrie(); + private static readonly IdentifiersTrie _booleans = new IdentifiersTrie(); + private static readonly IdentifiersTrie _logicalOp = new IdentifiersTrie(); - private static readonly LexemTrie _preprocRegion = new LexemTrie(); - private static readonly LexemTrie _preprocEndRegion = new LexemTrie(); - - private static readonly LexemTrie _preprocImport = new LexemTrie(); + private static readonly IdentifiersTrie _preprocImport = new IdentifiersTrie(); const int BUILTINS_INDEX = (int)Token.ByValParam; @@ -134,6 +131,8 @@ static LanguageDef() AddToken(Token.Equal, "="); AddToken(Token.Semicolon, ";"); AddToken(Token.Question, "?"); + AddToken(Token.Tilde, "~"); + AddToken(Token.Colon, ":"); #endregion @@ -216,11 +215,6 @@ static LanguageDef() #endregion - _preprocRegion.Add("Область",true); - _preprocRegion.Add("Region", true); - _preprocEndRegion.Add("КонецОбласти", true); - _preprocEndRegion.Add("EndRegion", true); - _preprocImport.Add("Использовать", true); _preprocImport.Add("Use", true); } @@ -410,18 +404,6 @@ public static bool IsLogicalOperatorString(string content) return _logicalOp.TryGetValue(content, out var nodeIsFilled) && nodeIsFilled; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool IsPreprocRegion(string value) - { - return _preprocRegion.TryGetValue(value, out var nodeIsFilled) && nodeIsFilled; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool IsPreprocEndRegion(string value) - { - return _preprocEndRegion.TryGetValue(value, out var nodeIsFilled) && nodeIsFilled; - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsImportDirective(string value) { diff --git a/src/OneScript.Language/LexemTrie.cs b/src/OneScript.Language/LexemTrie.cs deleted file mode 100644 index 5263b4e15..000000000 --- a/src/OneScript.Language/LexemTrie.cs +++ /dev/null @@ -1,213 +0,0 @@ -/*---------------------------------------------------------- -This Source Code Form is subject to the terms of the -Mozilla Public License, v.2.0. If a copy of the MPL -was not distributed with this file, You can obtain one -at http://mozilla.org/MPL/2.0/. -----------------------------------------------------------*/ - -using System.Collections.Generic; -using System.Diagnostics; - -namespace OneScript.Language -{ - public class LexemTrie - { - private class TrieNode - { - public char UCase; - public char LCase; - - public TrieNode next; - public TrieNode sibling; - public T value; - } - - private static int _alphabetLength; - private TrieNode[] _alphabet; - private int _count; - - static LexemTrie() - { - var ru = "абвгдеёжзийклмнопрстуфхцчшщьыъэюя"; - var en = "abcdefghijklmnopqrstuvwxyz"; - var symbols = @"+-*/\()[].,<>=;?%0123456789"; - - var all = symbols + - ru + - ru.ToUpper() + - en + - en.ToUpper(); - - _alphabetLength = all.Length; - } - - public LexemTrie() - { - _alphabet = new TrieNode[_alphabetLength]; - } - - public int Count => _count; - - private static int GetIndex(char c) - { - var code = (int) c; - - if (code >= 1040 && code <= 1103) - { - return code - 960; - } - - if (code >= 40 && code <= 57) - { - return code - 39; - } - - if (code >= 59 && code <= 63) - { - return code - 40; - } - - if (code >= 65 && code <= 93) - { - return code - 41; - } - - if (code >= 97 && code <= 122) - { - return code - 44; - } - - switch (c) - { - case '%': - return 0; - case 'Ё': - return 79; - case 'ё': - return 144; - } - - return -1; - } - - private TrieNode GetValueNode(string key) - { - var index = GetIndex(key[0]); - if (index == -1) - return null; - - var node = _alphabet[index]; - if (node == null) - { - node = new TrieNode(); - node.LCase = char.ToLower(key[0]); - node.UCase = char.ToUpper(key[0]); - _alphabet[GetIndex(node.LCase)] = node; - _alphabet[GetIndex(node.UCase)] = node; - } - - for (int i = 1; i < key.Length; i++) - { - var current = node; - node = node.next; - if (node == null) - { - var newNode = new TrieNode(); - newNode.LCase = char.ToLower(key[i]); - newNode.UCase = char.ToUpper(key[i]); - current.next = newNode; - node = newNode; - } - else if (node.LCase != key[i] && node.UCase != key[i]) - { - var insert = node.sibling; - while (insert != null) - { - if (insert.LCase == key[i] || insert.UCase == key[i]) - { - node = insert; - break; - } - - node = insert; - insert = insert.sibling; - } - - if (insert == null) - { - var newNode = new TrieNode(); - newNode.LCase = char.ToLower(key[i]); - newNode.UCase = char.ToUpper(key[i]); - node.sibling = newNode; - node = newNode; - } - } - } - - return node; - } - - public void Add(string key, T value) - { - var node = GetValueNode(key); - Debug.Assert(node != null); - - node.value = value; - ++_count; - } - - public T Get(string key) - { - var node = FindNode(key); - if (node == null) - { - throw new KeyNotFoundException(); - } - - return node.value; - } - - private TrieNode FindNode(string key) - { - var index = GetIndex(key[0]); - if (index == -1 || _alphabet[index] == null) - return null; - - var node = _alphabet[index]; - for (int i = 1; i < key.Length; i++) - { - node = node.next; - if (node == null) - return null; - - while(node.LCase != key[i] && node.UCase != key[i]) - { - node = node.sibling; - if(node == null) - return null; - } - } - - return node; - } - - public bool TryGetValue(string key, out T value) - { - var node = FindNode(key); - if (node == null) - { - value = default(T); - return false; - } - - value = node.value; - return true; - } - - public T this[string key] - { - get => Get(key); - set => Add(key, value); - } - } -} diff --git a/src/OneScript.Language/LexicalAnalysis/Token.cs b/src/OneScript.Language/LexicalAnalysis/Token.cs index fc3fd7824..0a78f9df1 100644 --- a/src/OneScript.Language/LexicalAnalysis/Token.cs +++ b/src/OneScript.Language/LexicalAnalysis/Token.cs @@ -43,6 +43,8 @@ public enum Token RemoveHandler, Async, Await, + Tilde, + Colon, // operators Plus, diff --git a/src/OneScript.Language/SyntaxAnalysis/AstNodes/MethodNode.cs b/src/OneScript.Language/SyntaxAnalysis/AstNodes/MethodNode.cs index d7de41a27..07aa269a3 100644 --- a/src/OneScript.Language/SyntaxAnalysis/AstNodes/MethodNode.cs +++ b/src/OneScript.Language/SyntaxAnalysis/AstNodes/MethodNode.cs @@ -17,6 +17,8 @@ public MethodNode() : base(NodeKind.Method) { } + public bool IsAsync { get; set; } + public MethodSignatureNode Signature { get; private set; } public BslSyntaxNode MethodBody { get; private set; } diff --git a/src/OneScript.Language/SyntaxAnalysis/AstNodes/UnaryOperationNode.cs b/src/OneScript.Language/SyntaxAnalysis/AstNodes/UnaryOperationNode.cs index dddcaf793..05e9b80cd 100644 --- a/src/OneScript.Language/SyntaxAnalysis/AstNodes/UnaryOperationNode.cs +++ b/src/OneScript.Language/SyntaxAnalysis/AstNodes/UnaryOperationNode.cs @@ -13,7 +13,7 @@ public class UnaryOperationNode : NonTerminalNode { public Token Operation { get; } - public UnaryOperationNode(Lexem operation) : base(NodeKind.UnaryOperation) + public UnaryOperationNode(Lexem operation) : base(NodeKind.UnaryOperation, operation) { Operation = operation.Token; } diff --git a/src/OneScript.Language/SyntaxAnalysis/BslSyntaxWalker.cs b/src/OneScript.Language/SyntaxAnalysis/BslSyntaxWalker.cs index fbeae0c73..855ead38a 100644 --- a/src/OneScript.Language/SyntaxAnalysis/BslSyntaxWalker.cs +++ b/src/OneScript.Language/SyntaxAnalysis/BslSyntaxWalker.cs @@ -158,10 +158,20 @@ protected virtual void VisitStatement(BslSyntaxNode statement) VisitGlobalProcedureCall(statement as CallNode); else if (statement.Kind == NodeKind.DereferenceOperation) VisitProcedureDereference(statement); + else if (statement.Kind == NodeKind.UnaryOperation && statement is UnaryOperationNode + { + Operation: Token.Await + } unaryOp) + VisitGlobalAwaitCall(unaryOp); else DefaultVisit(statement); } + private void VisitGlobalAwaitCall(UnaryOperationNode awaitStatement) + { + VisitStatement(awaitStatement.Children[0]); + } + protected virtual void VisitAssignment(BslSyntaxNode assignment) { var left = assignment.Children[0]; diff --git a/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs b/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs index 87e99f83f..5ab6a8f3e 100644 --- a/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs +++ b/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs @@ -27,6 +27,7 @@ public class DefaultBslParser private bool _isMethodsDefined; private bool _isStatementsDefined; private bool _isInFunctionScope; + private bool _isInAsyncMethod; private bool _lastDereferenceIsWritable; private readonly Stack _tokenStack = new Stack(); @@ -286,7 +287,8 @@ private void BuildMethodsSection() { if (_lastExtractedLexem.Type != LexemType.Annotation && _lastExtractedLexem.Token != Token.Procedure - && _lastExtractedLexem.Token != Token.Function) + && _lastExtractedLexem.Token != Token.Function + && _lastExtractedLexem.Token != Token.Async) { return; } @@ -301,7 +303,7 @@ private void BuildMethodsSection() while (true) { BuildAnnotations(); - if (_lastExtractedLexem.Token == Token.Procedure || _lastExtractedLexem.Token == Token.Function) + if (IsStartOfMethod(_lastExtractedLexem)) { if (!sectionExist) { @@ -324,14 +326,26 @@ private void BuildMethodsSection() } } + private static bool IsStartOfMethod(in Lexem lex) + { + return lex.Token == Token.Async || lex.Token == Token.Procedure || lex.Token == Token.Function; + } + private void BuildMethod() { - Debug.Assert(_lastExtractedLexem.Token == Token.Procedure || _lastExtractedLexem.Token == Token.Function); + Debug.Assert(IsStartOfMethod(_lastExtractedLexem)); var method = _nodeContext.AddChild(new MethodNode()); ApplyAnnotations(method); PushContext(method); + if (_lastExtractedLexem.Token == Token.Async) + { + method.IsAsync = true; + _isInAsyncMethod = true; + NextLexem(); + } + try { BuildMethodSignature(); @@ -345,6 +359,7 @@ private void BuildMethod() _isInFunctionScope = false; _inMethodScope = false; _isStatementsDefined = false; + _isInAsyncMethod = false; PopContext(); } } @@ -382,9 +397,8 @@ private void BuildMethodBody() private void BuildMethodSignature() { - var isFunction = _lastExtractedLexem.Token == Token.Function; - var signature = _nodeContext.AddChild(new MethodSignatureNode(_lastExtractedLexem)); + var isFunction = _lastExtractedLexem.Token == Token.Function; CreateChild(signature, isFunction? NodeKind.Function : NodeKind.Procedure, _lastExtractedLexem); _isInFunctionScope = isFunction; NextLexem(); @@ -676,7 +690,10 @@ private void BuildComplexStructureStatement() case Token.AddHandler: case Token.RemoveHandler: BuildEventHandlerOperation(_lastExtractedLexem.Token); - break; + break; + case Token.Await: + BuildGlobalCallAwaitOperator(); + break; default: var expected = _tokenStack.Peek(); AddError(LocalizedErrors.TokenExpected(expected)); @@ -684,6 +701,35 @@ private void BuildComplexStructureStatement() } } + private void BuildGlobalCallAwaitOperator() + { + Debug.Assert(_lastExtractedLexem.Token == Token.Await); + + _nodeContext.AddChild(TerminalNode()); + } + + + private BslSyntaxNode BuildExpressionAwaitOperator(in Lexem lexem) + { + Debug.Assert(_lastExtractedLexem.Token == Token.Await); + CheckAsyncMethod(); + var awaitOperator = new UnaryOperationNode(lexem); + NextLexem(); + + var call = BuildGlobalCall(_lastExtractedLexem); + awaitOperator.AddChild(call); + + return awaitOperator; + } + + private void CheckAsyncMethod() + { + if (!_isInAsyncMethod) + { + AddError(LocalizedErrors.AwaitMustBeInAsyncMethod(), false); + } + } + private void BuildIfStatement() { var condition = _nodeContext.AddChild(new ConditionNode(_lastExtractedLexem)); @@ -1330,6 +1376,10 @@ private BslSyntaxNode TerminalNode() { node = BuildGlobalCall(_lastExtractedLexem); } + else if (_lastExtractedLexem.Token == Token.Await) + { + node = BuildExpressionAwaitOperator(_lastExtractedLexem); + } else { AddError(LocalizedErrors.ExpressionSyntax()); diff --git a/src/OneScript.Language/SyntaxAnalysis/LocalizedErrors.cs b/src/OneScript.Language/SyntaxAnalysis/LocalizedErrors.cs index b2f8b73fe..b069f1656 100644 --- a/src/OneScript.Language/SyntaxAnalysis/LocalizedErrors.cs +++ b/src/OneScript.Language/SyntaxAnalysis/LocalizedErrors.cs @@ -60,6 +60,11 @@ public static CodeError ExportedLocalVar(string varName) $"Local variable can't be exported ({varName})"); } + public static CodeError AwaitMustBeInAsyncMethod() => Create( + "Оператор Ждать (Await) может употребляться только в асинхронных процедурах или функциях", + "Operator Await can be used only in async procedures or functions" + ); + public static CodeError LiteralExpected() => Create("Ожидается константа", "Constant expected"); public static CodeError NumberExpected() => Create("Ожидается числовая константа", "Numeric constant expected"); diff --git a/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs b/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs index bc76bca70..855402c39 100644 --- a/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs +++ b/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs @@ -61,6 +61,7 @@ public enum NodeKind RemoveHandler, Preprocessor, Import, - TopLevelExpression + TopLevelExpression, + Async } } \ No newline at end of file diff --git a/src/OneScript.Language/SyntaxAnalysis/RegionDirectiveHandler.cs b/src/OneScript.Language/SyntaxAnalysis/RegionDirectiveHandler.cs index 90a05ad03..d659ce4da 100644 --- a/src/OneScript.Language/SyntaxAnalysis/RegionDirectiveHandler.cs +++ b/src/OneScript.Language/SyntaxAnalysis/RegionDirectiveHandler.cs @@ -12,8 +12,8 @@ namespace OneScript.Language.SyntaxAnalysis { public class RegionDirectiveHandler : DirectiveHandlerBase { - private readonly LexemTrie _preprocRegion = new LexemTrie(); - private readonly LexemTrie _preprocEndRegion = new LexemTrie(); + private readonly IdentifiersTrie _preprocRegion = new IdentifiersTrie(); + private readonly IdentifiersTrie _preprocEndRegion = new IdentifiersTrie(); private int _regionsNesting = 0; diff --git a/src/Tests/OneScript.Language.Tests/ParserTests.cs b/src/Tests/OneScript.Language.Tests/ParserTests.cs index ec6144928..afd6cdf04 100644 --- a/src/Tests/OneScript.Language.Tests/ParserTests.cs +++ b/src/Tests/OneScript.Language.Tests/ParserTests.cs @@ -1168,6 +1168,73 @@ public void TestLocalExportVar() }); } + [Fact] + public void TestAsyncProcedure() + { + var code = + @"Асинх Процедура Проц1() + Перем Переменная; + КонецПроцедуры"; + + var node = ParseModuleAndGetValidator(code); + node.Is(NodeKind.MethodsSection); + + var methodNode = node.NextChild().Is(NodeKind.Method); + + methodNode.CurrentNode.RealNode.As().IsAsync.Should().BeTrue(); + } + + [Fact] + public void TestAsyncFunction() + { + var code = + @"Асинх Функция Ф1() + Перем Переменная; + КонецФункции"; + + var node = ParseModuleAndGetValidator(code); + node.Is(NodeKind.MethodsSection); + + var methodNode = node.NextChild().Is(NodeKind.Method); + + methodNode.CurrentNode.RealNode.As().IsAsync.Should().BeTrue(); + } + + [Fact] + public void TestAwaitPriority() + { + var code = + @"Асинх Процедура Проц1() + А = Ждать Вызов().Поле; + КонецПроцедуры"; + + var node = ParseModuleAndGetValidator(code); + node.Is(NodeKind.MethodsSection); + + var method = node.NextChild().Is(NodeKind.Method); + var expression = method.DownTo(NodeKind.Assignment) + .NextChildIs(NodeKind.Identifier) + .NextChild(); + + expression.CurrentNode.RealNode.As().Operation.Should().Be(Token.Await); + expression + .NextChildIs(NodeKind.DereferenceOperation) + .NoMoreChildren(); + } + + [Fact] + public void TestAwaitMustBeInAsyncOnly() + { + var code = + @"Процедура Проц1() + Ждать Операция(); + КонецПроцедуры"; + + CatchParsingError(code, errors => + { + errors.Single().Description.Should().Contain("Await"); + }); + } private static void CatchParsingError(string code) { diff --git a/src/Tests/OneScript.Language.Tests/TestAstNode.cs b/src/Tests/OneScript.Language.Tests/TestAstNode.cs index 613371016..e909e7952 100644 --- a/src/Tests/OneScript.Language.Tests/TestAstNode.cs +++ b/src/Tests/OneScript.Language.Tests/TestAstNode.cs @@ -31,7 +31,7 @@ public TestAstNode(BslSyntaxNode node) BinaryOperationNode binary => binary.Operation.ToString(), UnaryOperationNode unary => unary.Operation.ToString(), PreprocessorDirectiveNode preproc => preproc.DirectiveName, - _ => Value + _ => nonTerm.ToString() }; } else if(node is TerminalNode term) @@ -39,6 +39,11 @@ public TestAstNode(BslSyntaxNode node) _childrenLazy = new Lazy>(new TestAstNode[0]); Value = term.Lexem.Content; } + else + { + _childrenLazy = new Lazy>(new TestAstNode[0]); + Value = node.ToString(); + } } public string Value { get; set; } diff --git a/src/Tests/OneScript.Language.Tests/TreeValidatorExtensions.cs b/src/Tests/OneScript.Language.Tests/TreeValidatorExtensions.cs index 93f319b0d..1b246c6cb 100644 --- a/src/Tests/OneScript.Language.Tests/TreeValidatorExtensions.cs +++ b/src/Tests/OneScript.Language.Tests/TreeValidatorExtensions.cs @@ -27,6 +27,41 @@ public static SyntaxTreeValidator Is(this SyntaxTreeValidator validator, NodeKin validator.CurrentNode.Kind.Should().Be(type); return validator; } + + public static SyntaxTreeValidator DownTo(this SyntaxTreeValidator validator, NodeKind type) + { + var node = DownTo(validator.CurrentNode, type); + if (node == null) + { + throw new Exception("No such child: " + type); + } + + return new SyntaxTreeValidator(node); + } + + private static TestAstNode DownTo(this TestAstNode node, NodeKind type) + { + if (node.Kind == type) + return node; + + if (node.Children.Count == 0) + return null; + + foreach (var childNode in node.ChildrenList) + { + if (childNode.Kind == type) + return childNode; + + if (childNode.ChildrenList.Count == 0) + continue; + + var result = DownTo(childNode, type); + if (result != null) + return result; + } + + return null; + } public static void Is(this TestAstNode node, NodeKind type) { diff --git a/src/Tests/OneScript.Language.Tests/TrieTests.cs b/src/Tests/OneScript.Language.Tests/TrieTests.cs index f6c137daa..0c476e7d0 100644 --- a/src/Tests/OneScript.Language.Tests/TrieTests.cs +++ b/src/Tests/OneScript.Language.Tests/TrieTests.cs @@ -12,9 +12,9 @@ namespace OneScript.Language.Tests public class TrieTests { [Fact] - public void LexemTrieAdd() + public void IdentifiersTrieAdd() { - var t = new LexemTrie(); + var t = new IdentifiersTrie(); t.Add("Иван", 0); t.Add("Иволга", 1); @@ -28,7 +28,7 @@ public void LexemTrieAdd() [Fact] public void Tokens() { - var t = new LexemTrie(); + var t = new IdentifiersTrie(); t.Add("иначе", 1); t.Add("иначеесли", 2); From 022a03a44c3cc55ed9885eacb52f11b39a0890f5 Mon Sep 17 00:00:00 2001 From: Andrei Ovsiankin Date: Wed, 18 Oct 2023 09:51:36 +0300 Subject: [PATCH 3/8] =?UTF-8?q?=D0=A4=D0=B8=D0=BA=D1=81=20=D0=BE=D0=BF?= =?UTF-8?q?=D0=B5=D1=80=D0=B0=D1=82=D0=BE=D1=80=D0=B0=20=D0=96=D0=B4=D0=B0?= =?UTF-8?q?=D1=82=D1=8C=20=D0=B2=20=D0=BD=D0=B5=D0=B0=D1=81=D0=B8=D0=BD?= =?UTF-8?q?=D1=85=D1=80=D0=BE=D0=BD=D0=BD=D0=BE=D0=BC=20=D0=BA=D0=BE=D0=BD?= =?UTF-8?q?=D1=82=D0=B5=D0=BA=D1=81=D1=82=D0=B5=20=D0=B8=20=D1=82=D0=B5?= =?UTF-8?q?=D1=81=D1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../SyntaxAnalysis/DefaultBslParser.cs | 12 +++++++++--- .../OneScript.Language.Tests/ParserTests.cs | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs b/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs index 5ab6a8f3e..41debb371 100644 --- a/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs +++ b/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs @@ -1536,11 +1536,17 @@ private void NewObjectStaticConstructor(NonTerminalNode node) } } - #endregion - + #endregion + private void NextLexem() { - _lastExtractedLexem = _lexer.NextLexem(); + var localLexem = _lexer.NextLexem(); + if (localLexem.Token == Token.Await && !_isInAsyncMethod) + { + localLexem.Token = Token.NotAToken; + } + + _lastExtractedLexem = localLexem; } private bool NextExpected(Token expected) diff --git a/src/Tests/OneScript.Language.Tests/ParserTests.cs b/src/Tests/OneScript.Language.Tests/ParserTests.cs index afd6cdf04..5e596d862 100644 --- a/src/Tests/OneScript.Language.Tests/ParserTests.cs +++ b/src/Tests/OneScript.Language.Tests/ParserTests.cs @@ -1235,6 +1235,22 @@ public void TestAwaitMustBeInAsyncOnly() errors.Single().Description.Should().Contain("Await"); }); } + + [Fact] + public void AwaitISNotKeywordInNonAsyncContext() + { + var code = + @"Процедура Проц1() + А = Ждать; + КонецПроцедуры"; + + var validator = ParseModuleAndGetValidator(code) + .DownTo(NodeKind.Assignment); + + validator + .NextChildIs(NodeKind.Identifier) + .NextChildIs(NodeKind.Identifier); + } private static void CatchParsingError(string code) { From baa7237a0f5ff943ffc36d688ddd768e9138c6a7 Mon Sep 17 00:00:00 2001 From: Andrei Ovsiankin Date: Wed, 18 Oct 2023 10:52:52 +0300 Subject: [PATCH 4/8] =?UTF-8?q?=D0=A3=D0=BB=D1=83=D1=87=D1=88=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D1=8F=20=D1=82=D0=B5=D1=81=D1=82=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/OneScript.Language/OneScript.Language.csproj | 1 + src/Tests/OneScript.Language.Tests/ParserTests.cs | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/OneScript.Language/OneScript.Language.csproj b/src/OneScript.Language/OneScript.Language.csproj index b4a2f7bda..404476e0e 100644 --- a/src/OneScript.Language/OneScript.Language.csproj +++ b/src/OneScript.Language/OneScript.Language.csproj @@ -22,6 +22,7 @@ true false + DEBUG;TRACE diff --git a/src/Tests/OneScript.Language.Tests/ParserTests.cs b/src/Tests/OneScript.Language.Tests/ParserTests.cs index 5e596d862..20364a784 100644 --- a/src/Tests/OneScript.Language.Tests/ParserTests.cs +++ b/src/Tests/OneScript.Language.Tests/ParserTests.cs @@ -1237,7 +1237,7 @@ public void TestAwaitMustBeInAsyncOnly() } [Fact] - public void AwaitISNotKeywordInNonAsyncContext() + public void AwaitIsNotKeywordInNonAsyncContext() { var code = @"Процедура Проц1() @@ -1251,6 +1251,17 @@ public void AwaitISNotKeywordInNonAsyncContext() .NextChildIs(NodeKind.Identifier) .NextChildIs(NodeKind.Identifier); } + + [Fact] + public void AwaitRequiresExpression() + { + var code = + @"Асинх Процедура Проц1() + А = Ждать; + КонецПроцедуры"; + + CatchParsingError(code, err => err.Single().ErrorId.Should().Be("ExpressionExpected")); + } private static void CatchParsingError(string code) { From 9deefae6426c06d50a627c2c9dcdb5e1b2a055a8 Mon Sep 17 00:00:00 2001 From: Andrei Ovsiankin Date: Wed, 18 Oct 2023 14:07:25 +0300 Subject: [PATCH 5/8] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=90=D1=81=D0=B8=D0=BD=D1=85/?= =?UTF-8?q?=D0=96=D0=B4=D0=B0=D1=82=D1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/OneScript.Language/LanguageDef.cs | 2 +- .../AstNodes/ErrorTerminalNode.cs | 18 ++++ .../SyntaxAnalysis/DefaultBslParser.cs | 100 +++++++++++------- .../SyntaxAnalysis/NodeKind.cs | 1 - .../OneScript.Language.Tests/ParserTests.cs | 78 +++++++++++++- 5 files changed, 158 insertions(+), 41 deletions(-) create mode 100644 src/OneScript.Language/SyntaxAnalysis/AstNodes/ErrorTerminalNode.cs diff --git a/src/OneScript.Language/LanguageDef.cs b/src/OneScript.Language/LanguageDef.cs index 25cf484cc..e506baecf 100644 --- a/src/OneScript.Language/LanguageDef.cs +++ b/src/OneScript.Language/LanguageDef.cs @@ -286,7 +286,7 @@ public static bool IsUnaryOperator(Token token) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool IsLiteral(ref Lexem lex) + public static bool IsLiteral(in Lexem lex) { return lex.Type == LexemType.StringLiteral || lex.Type == LexemType.NumberLiteral diff --git a/src/OneScript.Language/SyntaxAnalysis/AstNodes/ErrorTerminalNode.cs b/src/OneScript.Language/SyntaxAnalysis/AstNodes/ErrorTerminalNode.cs new file mode 100644 index 000000000..b3a0a3deb --- /dev/null +++ b/src/OneScript.Language/SyntaxAnalysis/AstNodes/ErrorTerminalNode.cs @@ -0,0 +1,18 @@ +using OneScript.Language.LexicalAnalysis; + +namespace OneScript.Language.SyntaxAnalysis.AstNodes +{ + /// + /// Нода ошибочного синтаксиса + /// + public class ErrorTerminalNode : TerminalNode + { + public ErrorTerminalNode() : base(NodeKind.Unknown) + { + } + + public ErrorTerminalNode(Lexem lexem) : base(NodeKind.Unknown, lexem) + { + } + } +} \ No newline at end of file diff --git a/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs b/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs index 41debb371..384f3ae31 100644 --- a/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs +++ b/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs @@ -485,7 +485,7 @@ private bool BuildDefaultParameterValue(NonTerminalNode param, NodeKind nodeKind NextLexem(); } - if (LanguageDef.IsLiteral(ref _lastExtractedLexem)) + if (LanguageDef.IsLiteral(_lastExtractedLexem)) { string literalText = _lastExtractedLexem.Content; if (hasSign) @@ -709,19 +709,32 @@ private void BuildGlobalCallAwaitOperator() } - private BslSyntaxNode BuildExpressionAwaitOperator(in Lexem lexem) + private BslSyntaxNode BuildExpressionAwaitOperator(Lexem lexem) { Debug.Assert(_lastExtractedLexem.Token == Token.Await); - CheckAsyncMethod(); - var awaitOperator = new UnaryOperationNode(lexem); - NextLexem(); - var call = BuildGlobalCall(_lastExtractedLexem); - awaitOperator.AddChild(call); + NextLexem(); - return awaitOperator; + var argument = SelectTerminalNode(_lastExtractedLexem, false); + if (argument != default) + { + CheckAsyncMethod(); + var awaitOperator = new UnaryOperationNode(lexem); + awaitOperator.AddChild(argument); + return awaitOperator; + } + else if (!_isInAsyncMethod) + { + // это просто переменная Ждать или метод Ждать + return CallOrVariable(lexem); + } + else + { + AddError(LocalizedErrors.ExpressionSyntax()); + return new ErrorTerminalNode(_lastExtractedLexem); + } } - + private void CheckAsyncMethod() { if (!_isInAsyncMethod) @@ -1065,10 +1078,15 @@ private void BuildAssignment(NonTerminalNode batch) private BslSyntaxNode BuildGlobalCall(Lexem identifier) { - var target = NodeBuilder.CreateNode(NodeKind.Identifier, identifier); NextLexem(); - if (_lastExtractedLexem.Token != Token.OpenPar) + return CallOrVariable(identifier); + } + + private BslSyntaxNode CallOrVariable(Lexem identifier) + { + var target = NodeBuilder.CreateNode(NodeKind.Identifier, identifier); + if (_lastExtractedLexem.Token != Token.OpenPar) { _lastDereferenceIsWritable = true; // одиночный идентификатор } @@ -1143,7 +1161,6 @@ private void BuildCallArgument(NonTerminalNode argsList) } } - [MethodImpl(MethodImplOptions.AggressiveInlining)] private void BuildLastDefaultArg(NonTerminalNode argsList) { NextLexem(); @@ -1299,9 +1316,18 @@ private BslSyntaxNode BuildExpressionUpTo(NonTerminalNode parent, Token stopToke else { if (_lastExtractedLexem.Token == Token.EndOfText) + { AddError(LocalizedErrors.UnexpectedEof()); + } else - AddError(LocalizedErrors.ExpressionSyntax()); + { + SkipToNextStatement(new []{stopToken}); + AddError(LocalizedErrors.ExpressionSyntax(), false); + if (_lastExtractedLexem.Token == stopToken) + { + NextLexem(); + } + } node = default; } @@ -1322,7 +1348,6 @@ private void BuildOptionalExpression(NonTerminalNode parent, Token stopToken) #region Operators - [MethodImpl(MethodImplOptions.AggressiveInlining)] private static BslSyntaxNode MakeBinaryOperationNode(BslSyntaxNode firstArg, BslSyntaxNode secondArg, in Lexem lexem) { var node = new BinaryOperationNode(lexem); @@ -1331,7 +1356,6 @@ private static BslSyntaxNode MakeBinaryOperationNode(BslSyntaxNode firstArg, Bsl return node; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] private BslSyntaxNode BuildParenthesis() { if (_lastExtractedLexem.Token == Token.OpenPar) @@ -1353,38 +1377,45 @@ private BslSyntaxNode BuildParenthesis() #endregion private BslSyntaxNode TerminalNode() + { + BslSyntaxNode node = SelectTerminalNode(_lastExtractedLexem, true); + if (node == default) + { + AddError(LocalizedErrors.ExpressionSyntax()); + } + + return node; + } + + private BslSyntaxNode SelectTerminalNode(in Lexem currentLexem, bool supportAwait) { BslSyntaxNode node = default; - if (LanguageDef.IsLiteral(ref _lastExtractedLexem)) + if (LanguageDef.IsLiteral(currentLexem)) { - node = NodeBuilder.CreateNode(NodeKind.Constant, _lastExtractedLexem); + node = NodeBuilder.CreateNode(NodeKind.Constant, currentLexem); NextLexem(); } - else if (LanguageDef.IsUserSymbol(in _lastExtractedLexem)) + else if (LanguageDef.IsUserSymbol(currentLexem)) { - node = BuildGlobalCall(_lastExtractedLexem); + node = BuildGlobalCall(currentLexem); } - else if(_lastExtractedLexem.Token == Token.NewObject) + else if(currentLexem.Token == Token.NewObject) { node = BuildNewObjectCreation(); } - else if (_lastExtractedLexem.Token == Token.Question) + else if (currentLexem.Token == Token.Question) { node = BuildQuestionOperator(); } - else if (LanguageDef.IsBuiltInFunction(_lastExtractedLexem.Token)) - { - node = BuildGlobalCall(_lastExtractedLexem); - } - else if (_lastExtractedLexem.Token == Token.Await) + else if (LanguageDef.IsBuiltInFunction(currentLexem.Token)) { - node = BuildExpressionAwaitOperator(_lastExtractedLexem); + node = BuildGlobalCall(currentLexem); } - else + else if (supportAwait && currentLexem.Token == Token.Await) { - AddError(LocalizedErrors.ExpressionSyntax()); + node = BuildExpressionAwaitOperator(currentLexem); } - + return node; } @@ -1538,15 +1569,10 @@ private void NewObjectStaticConstructor(NonTerminalNode node) #endregion + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void NextLexem() { - var localLexem = _lexer.NextLexem(); - if (localLexem.Token == Token.Await && !_isInAsyncMethod) - { - localLexem.Token = Token.NotAToken; - } - - _lastExtractedLexem = localLexem; + _lastExtractedLexem = _lexer.NextLexem(); } private bool NextExpected(Token expected) diff --git a/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs b/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs index 855402c39..f8b8b0cb5 100644 --- a/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs +++ b/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs @@ -62,6 +62,5 @@ public enum NodeKind Preprocessor, Import, TopLevelExpression, - Async } } \ No newline at end of file diff --git a/src/Tests/OneScript.Language.Tests/ParserTests.cs b/src/Tests/OneScript.Language.Tests/ParserTests.cs index 20364a784..f3495dc1a 100644 --- a/src/Tests/OneScript.Language.Tests/ParserTests.cs +++ b/src/Tests/OneScript.Language.Tests/ParserTests.cs @@ -1237,7 +1237,7 @@ public void TestAwaitMustBeInAsyncOnly() } [Fact] - public void AwaitIsNotKeywordInNonAsyncContext() + public void AwaitIsNotKeywordInNonAsyncContext_Variable() { var code = @"Процедура Проц1() @@ -1252,6 +1252,69 @@ public void AwaitIsNotKeywordInNonAsyncContext() .NextChildIs(NodeKind.Identifier); } + [Fact] + public void AwaitIsNotKeywordInNonAsyncContext_Method() + { + var code = + @"Процедура Проц1() + А = Ждать(); + КонецПроцедуры"; + + var validator = ParseModuleAndGetValidator(code) + .DownTo(NodeKind.Assignment); + + validator + .NextChildIs(NodeKind.Identifier) + .NextChildIs(NodeKind.GlobalCall); + } + + [Fact] + public void AwaitIsNotKeywordInNonAsyncContext_PropertyName() + { + var code = + @"Процедура Проц1() + А = П.Ждать; + КонецПроцедуры"; + + var validator = ParseModuleAndGetValidator(code) + .DownTo(NodeKind.DereferenceOperation); + + validator + .NextChildIs(NodeKind.Identifier) + .NextChildIs(NodeKind.Identifier); + } + + [Fact] + public void AwaitIsNotKeywordInNonAsyncContext_MemberMethodName() + { + var code = + @"Процедура Проц1() + А = П.Ждать(); + КонецПроцедуры"; + + var validator = ParseModuleAndGetValidator(code) + .DownTo(NodeKind.DereferenceOperation); + + validator + .NextChildIs(NodeKind.Identifier) + .NextChildIs(NodeKind.MethodCall); + } + + [Fact] + public void Await_In_NonAsync_Expression_Fails() + { + var code = + @"Процедура Проц1() + Если Ждать ВтороеСлово Тогда + КонецЕсли; + КонецПроцедуры"; + + CatchParsingError(code, err => + { + err.Single().ErrorId.Should().Be("AwaitMustBeInAsyncMethod"); + }); + } + [Fact] public void AwaitRequiresExpression() { @@ -1260,7 +1323,18 @@ public void AwaitRequiresExpression() А = Ждать; КонецПроцедуры"; - CatchParsingError(code, err => err.Single().ErrorId.Should().Be("ExpressionExpected")); + CatchParsingError(code, err => err.Single().ErrorId.Should().Be("ExpressionSyntax")); + } + + [Fact] + public void DoubleAwaitIsForbidden() + { + var code = + @"Асинх Процедура Проц1() + А = Ждать Ждать Б; + КонецПроцедуры"; + + CatchParsingError(code, err => err.Single().ErrorId.Should().Be("ExpressionSyntax")); } private static void CatchParsingError(string code) From 8fa327fa33bdc37fe4d38bd9f47c0066b4d09e10 Mon Sep 17 00:00:00 2001 From: Andrei Ovsiankin Date: Thu, 19 Oct 2023 11:08:21 +0300 Subject: [PATCH 6/8] =?UTF-8?q?=D0=9B=D0=B5=D0=BA=D1=81=D0=B5=D1=80=20?= =?UTF-8?q?=D0=B4=D0=BB=D1=8F=20=D0=BC=D0=B5=D1=82=D0=BE=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/OneScript.Language/LanguageDef.cs | 2 - .../LexicalAnalysis/FullSourceLexer.cs | 5 +++ .../LexicalAnalysis/LabelLexerState.cs | 39 +++++++++++++++++++ .../LexicalAnalysis/LexemType.cs | 2 + .../LexicalAnalysis/Token.cs | 4 +- src/OneScript.Language/SpecialChars.cs | 2 + .../OneScript.Language.Tests/LexerTests.cs | 24 ++++++++++-- 7 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 src/OneScript.Language/LexicalAnalysis/LabelLexerState.cs diff --git a/src/OneScript.Language/LanguageDef.cs b/src/OneScript.Language/LanguageDef.cs index e506baecf..7f925859b 100644 --- a/src/OneScript.Language/LanguageDef.cs +++ b/src/OneScript.Language/LanguageDef.cs @@ -131,8 +131,6 @@ static LanguageDef() AddToken(Token.Equal, "="); AddToken(Token.Semicolon, ";"); AddToken(Token.Question, "?"); - AddToken(Token.Tilde, "~"); - AddToken(Token.Colon, ":"); #endregion diff --git a/src/OneScript.Language/LexicalAnalysis/FullSourceLexer.cs b/src/OneScript.Language/LexicalAnalysis/FullSourceLexer.cs index 72aed2aac..0fc6c88c2 100644 --- a/src/OneScript.Language/LexicalAnalysis/FullSourceLexer.cs +++ b/src/OneScript.Language/LexicalAnalysis/FullSourceLexer.cs @@ -23,6 +23,7 @@ public class FullSourceLexer : ILexer private readonly LexerState _commentState = new CommentLexerState(); private readonly LexerState _annotationState = new AnnotationLexerState(); private readonly LexerState _directiveState = new PreprocessorDirectiveLexerState(); + private readonly LexerState _labelState = new LabelLexerState(); private readonly FixedLexerState _fixedState = new FixedLexerState(); @@ -119,6 +120,10 @@ private void SelectState() { _state = _annotationState; } + else if (cs == SpecialChars.Tilde) + { + _state = _labelState; + } else { var cp = _iterator.GetErrorPosition(); diff --git a/src/OneScript.Language/LexicalAnalysis/LabelLexerState.cs b/src/OneScript.Language/LexicalAnalysis/LabelLexerState.cs new file mode 100644 index 000000000..17dc88f49 --- /dev/null +++ b/src/OneScript.Language/LexicalAnalysis/LabelLexerState.cs @@ -0,0 +1,39 @@ +using System.Diagnostics; +using OneScript.Localization; + +namespace OneScript.Language.LexicalAnalysis +{ + public class LabelLexerState : LexerState + { + private static BilingualString MESSAGE_NAME_EXPECTED = new BilingualString( + "Ожидается имя метки", + "Label name expected" + ); + + WordLexerState _wordExtractor = new WordLexerState(); + + public override Lexem ReadNextLexem(SourceCodeIterator iterator) + { + Debug.Assert(iterator.CurrentSymbol == SpecialChars.Tilde); + + var start = new CodeRange(iterator.CurrentLine, iterator.CurrentColumn); + iterator.MoveNext(); + if (!iterator.MoveToContent()) + throw CreateExceptionOnCurrentLine(MESSAGE_NAME_EXPECTED.ToString(), iterator); + + if (!char.IsLetter(iterator.CurrentSymbol)) + throw CreateExceptionOnCurrentLine(MESSAGE_NAME_EXPECTED.ToString(), iterator); + + var result = _wordExtractor.ReadNextLexem(iterator); + result.Type = LexemType.LabelRef; + if (iterator.CurrentSymbol == SpecialChars.Colon) + { + result.Type = LexemType.Label; + iterator.MoveNext(); + } + + result.Location = start; + return result; + } + } +} \ No newline at end of file diff --git a/src/OneScript.Language/LexicalAnalysis/LexemType.cs b/src/OneScript.Language/LexicalAnalysis/LexemType.cs index 48f8c5fc6..86723c470 100644 --- a/src/OneScript.Language/LexicalAnalysis/LexemType.cs +++ b/src/OneScript.Language/LexicalAnalysis/LexemType.cs @@ -22,6 +22,8 @@ public enum LexemType PreprocessorDirective, Annotation, Comment, + Label, + LabelRef, EndOfText } } diff --git a/src/OneScript.Language/LexicalAnalysis/Token.cs b/src/OneScript.Language/LexicalAnalysis/Token.cs index 0a78f9df1..5b4d91f1d 100644 --- a/src/OneScript.Language/LexicalAnalysis/Token.cs +++ b/src/OneScript.Language/LexicalAnalysis/Token.cs @@ -43,9 +43,7 @@ public enum Token RemoveHandler, Async, Await, - Tilde, - Colon, - + // operators Plus, Minus, diff --git a/src/OneScript.Language/SpecialChars.cs b/src/OneScript.Language/SpecialChars.cs index fb46c4f42..cfc96e761 100644 --- a/src/OneScript.Language/SpecialChars.cs +++ b/src/OneScript.Language/SpecialChars.cs @@ -18,6 +18,8 @@ public static class SpecialChars public const char QuestionMark = '?'; public const char Preprocessor = '#'; public const char Annotation = '&'; + public const char Tilde = '~'; + public const char Colon = ':'; public static bool IsOperatorChar(char symbol) { diff --git a/src/Tests/OneScript.Language.Tests/LexerTests.cs b/src/Tests/OneScript.Language.Tests/LexerTests.cs index 0d047c52e..37552c9c8 100644 --- a/src/Tests/OneScript.Language.Tests/LexerTests.cs +++ b/src/Tests/OneScript.Language.Tests/LexerTests.cs @@ -586,12 +586,12 @@ public void Syntax_Error_Handling() Assert.Equal("Б", lex.Content); } - [Fact(Skip = "Рефакторинг")] + [Fact] public void New_Exception_Shows_Negative_Line_And_Column() { - //var e = new ScriptException(); - //Assert.True(e.LineNumber == -1); - //Assert.True(e.ColumnNumber == -1); + var e = new ScriptException("fake"); + Assert.True(e.LineNumber == -1); + Assert.True(e.ColumnNumber == -1); } [Fact] @@ -633,6 +633,22 @@ public void Lexer_Ignores_Comments() Assert.Equal("value", lex.Content); } + [Fact] + public void Leer_Extracts_Labels() + { + string code = "~ImALabel: ~LabelRef;"; + var lexer = GetLexerForCode(code); + + var lex = lexer.NextLexem(); + + Assert.Equal(LexemType.Label, lex.Type); + Assert.Equal("ImALabel", lex.Content); + + lex = lexer.NextLexem(); + Assert.Equal(LexemType.LabelRef, lex.Type); + Assert.Equal("LabelRef", lex.Content); + } + private ILexer GetLexerForCode(string code) { return new FullSourceLexer From 47357677e184a0327ef59c0a54861ae316e6367d Mon Sep 17 00:00:00 2001 From: Andrei Ovsiankin Date: Thu, 19 Oct 2023 11:56:10 +0300 Subject: [PATCH 7/8] =?UTF-8?q?=D0=9F=D0=BE=D0=B4=D0=B4=D0=B5=D1=80=D0=B6?= =?UTF-8?q?=D0=BA=D0=B0=20=D0=BC=D0=B5=D1=82=D0=BE=D0=BA=20=D0=B2=20=D0=BF?= =?UTF-8?q?=D0=B0=D1=80=D1=81=D0=B5=D1=80=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/OneScript.Language/LanguageDef.cs | 1 + .../LexicalAnalysis/LabelLexerState.cs | 16 +++++++++- .../LexicalAnalysis/Token.cs | 1 + .../SyntaxAnalysis/AstNodes/LabelNode.cs | 14 ++++++++ .../SyntaxAnalysis/BslSyntaxWalker.cs | 10 ++++++ .../SyntaxAnalysis/DefaultBslParser.cs | 32 +++++++++++++++++++ .../SyntaxAnalysis/LocalizedErrors.cs | 6 ++++ .../SyntaxAnalysis/NodeKind.cs | 2 ++ .../Compiler/ModuleCompiler.cs | 15 +++++++++ .../Compiler/StackMachineCodeGenerator.cs | 14 ++++++++ .../OneScript.Language.Tests/LexerTests.cs | 5 +-- .../OneScript.Language.Tests/ParserTests.cs | 32 +++++++++++++++++++ 12 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 src/OneScript.Language/SyntaxAnalysis/AstNodes/LabelNode.cs diff --git a/src/OneScript.Language/LanguageDef.cs b/src/OneScript.Language/LanguageDef.cs index 7f925859b..ecbbb9c76 100644 --- a/src/OneScript.Language/LanguageDef.cs +++ b/src/OneScript.Language/LanguageDef.cs @@ -107,6 +107,7 @@ static LanguageDef() AddToken(Token.RemoveHandler, "УдалитьОбработчик", "RemoveHandler"); AddToken(Token.Async, "Асинх", "Async"); AddToken(Token.Await, "Ждать", "Await"); + AddToken(Token.Goto, "Перейти", "Goto"); #endregion diff --git a/src/OneScript.Language/LexicalAnalysis/LabelLexerState.cs b/src/OneScript.Language/LexicalAnalysis/LabelLexerState.cs index 17dc88f49..d9734fb91 100644 --- a/src/OneScript.Language/LexicalAnalysis/LabelLexerState.cs +++ b/src/OneScript.Language/LexicalAnalysis/LabelLexerState.cs @@ -9,6 +9,11 @@ public class LabelLexerState : LexerState "Ожидается имя метки", "Label name expected" ); + + private static BilingualString INVALID_LABEL = new BilingualString( + "Неверно задана метка", + "Invalid label definition" + ); WordLexerState _wordExtractor = new WordLexerState(); @@ -25,11 +30,20 @@ public override Lexem ReadNextLexem(SourceCodeIterator iterator) throw CreateExceptionOnCurrentLine(MESSAGE_NAME_EXPECTED.ToString(), iterator); var result = _wordExtractor.ReadNextLexem(iterator); + if (!LanguageDef.IsUserSymbol(result)) + { + throw CreateExceptionOnCurrentLine(INVALID_LABEL.ToString(), iterator); + } + result.Type = LexemType.LabelRef; if (iterator.CurrentSymbol == SpecialChars.Colon) { result.Type = LexemType.Label; - iterator.MoveNext(); + var tail = iterator.ReadToLineEnd(); + if (tail.Trim().Length != 0) + { + throw CreateExceptionOnCurrentLine(INVALID_LABEL.ToString(), iterator); + } } result.Location = start; diff --git a/src/OneScript.Language/LexicalAnalysis/Token.cs b/src/OneScript.Language/LexicalAnalysis/Token.cs index 5b4d91f1d..aca7738f1 100644 --- a/src/OneScript.Language/LexicalAnalysis/Token.cs +++ b/src/OneScript.Language/LexicalAnalysis/Token.cs @@ -43,6 +43,7 @@ public enum Token RemoveHandler, Async, Await, + Goto, // operators Plus, diff --git a/src/OneScript.Language/SyntaxAnalysis/AstNodes/LabelNode.cs b/src/OneScript.Language/SyntaxAnalysis/AstNodes/LabelNode.cs new file mode 100644 index 000000000..f26c8adc5 --- /dev/null +++ b/src/OneScript.Language/SyntaxAnalysis/AstNodes/LabelNode.cs @@ -0,0 +1,14 @@ +using OneScript.Language.LexicalAnalysis; + +namespace OneScript.Language.SyntaxAnalysis.AstNodes +{ + public class LabelNode : LineMarkerNode + { + public LabelNode(Lexem labelLexem) : base(labelLexem.Location, NodeKind.Label) + { + LabelName = labelLexem.Content; + } + + public string LabelName { get; } + } +} \ No newline at end of file diff --git a/src/OneScript.Language/SyntaxAnalysis/BslSyntaxWalker.cs b/src/OneScript.Language/SyntaxAnalysis/BslSyntaxWalker.cs index 855ead38a..7c2b352da 100644 --- a/src/OneScript.Language/SyntaxAnalysis/BslSyntaxWalker.cs +++ b/src/OneScript.Language/SyntaxAnalysis/BslSyntaxWalker.cs @@ -51,9 +51,19 @@ private void CreateVisitors() _nodeVisitors[(int)NodeKind.RemoveHandler] = VisitHandlerOperation; _nodeVisitors[(int)NodeKind.NewObject] = (x) => VisitNewObjectCreation((NewObjectNode)x); _nodeVisitors[(int)NodeKind.Preprocessor] = (x) => VisitPreprocessorDirective((PreprocessorDirectiveNode)x); + _nodeVisitors[(int)NodeKind.Goto] = (x) => VisitGotoNode((NonTerminalNode)x); + _nodeVisitors[(int)NodeKind.Label] = (x) => VisitLabelNode((LabelNode)x); } + protected virtual void VisitGotoNode(NonTerminalNode node) + { + } + + protected virtual void VisitLabelNode(LabelNode node) + { + } + protected void SetDefaultVisitorFor(NodeKind kind, Action action) { _nodeVisitors[(int)kind] = action; diff --git a/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs b/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs index 384f3ae31..9701bb69b 100644 --- a/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs +++ b/src/OneScript.Language/SyntaxAnalysis/DefaultBslParser.cs @@ -621,6 +621,12 @@ private void BuildCodeBatch(params Token[] endTokens) continue; } + if (_lastExtractedLexem.Type == LexemType.Label) + { + DefineLabel(_lastExtractedLexem); + continue; + } + if (_lastExtractedLexem.Type != LexemType.Identifier && _lastExtractedLexem.Token != Token.EndOfText) { AddError(LocalizedErrors.UnexpectedOperation()); @@ -642,6 +648,13 @@ private void BuildCodeBatch(params Token[] endTokens) PopStructureToken(); } + private void DefineLabel(Lexem label) + { + var node = new LabelNode(label); + _nodeContext.AddChild(node); + NextLexem(); + } + #region Statements private void BuildStatement() @@ -694,6 +707,9 @@ private void BuildComplexStructureStatement() case Token.Await: BuildGlobalCallAwaitOperator(); break; + case Token.Goto: + BuildGotoOperator(); + break; default: var expected = _tokenStack.Peek(); AddError(LocalizedErrors.TokenExpected(expected)); @@ -734,6 +750,22 @@ private BslSyntaxNode BuildExpressionAwaitOperator(Lexem lexem) return new ErrorTerminalNode(_lastExtractedLexem); } } + + private void BuildGotoOperator() + { + var gotoNode = new NonTerminalNode(NodeKind.Goto, _lastExtractedLexem); + NextLexem(); + + if (_lastExtractedLexem.Type != LexemType.LabelRef) + { + AddError(LocalizedErrors.LabelNameExpected()); + } + + gotoNode.AddChild(new LabelNode(_lastExtractedLexem)); + NextLexem(); + + _nodeContext.AddChild(gotoNode); + } private void CheckAsyncMethod() { diff --git a/src/OneScript.Language/SyntaxAnalysis/LocalizedErrors.cs b/src/OneScript.Language/SyntaxAnalysis/LocalizedErrors.cs index b069f1656..e0b96014f 100644 --- a/src/OneScript.Language/SyntaxAnalysis/LocalizedErrors.cs +++ b/src/OneScript.Language/SyntaxAnalysis/LocalizedErrors.cs @@ -43,6 +43,9 @@ private static CodeError Create(string ru, string en, [CallerMemberName] string public static CodeError IdentifierExpected() => Create("Ожидается идентификатор", "Identifier expecting"); + + public static CodeError LabelNameExpected() + => Create("Ожидается имя метки", "Label name expected"); public static CodeError ExpressionSyntax() => Create("Ошибка в выражении", "Expression syntax error"); @@ -139,5 +142,8 @@ public static CodeError DuplicateMethodDefinition(string methodName) => public static CodeError SymbolNotFound(string symbol) => Create($"Неизвестный символ: {symbol}", $"Symbol not found {symbol}"); + + public static CodeError AsyncMethodsNotSupported() => + Create("Асинхронные методы не поддерживаются", "Async methods aren't supported"); } } \ No newline at end of file diff --git a/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs b/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs index f8b8b0cb5..975941278 100644 --- a/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs +++ b/src/OneScript.Language/SyntaxAnalysis/NodeKind.cs @@ -62,5 +62,7 @@ public enum NodeKind Preprocessor, Import, TopLevelExpression, + Label, + Goto } } \ No newline at end of file diff --git a/src/OneScript.Native/Compiler/ModuleCompiler.cs b/src/OneScript.Native/Compiler/ModuleCompiler.cs index 07c200a31..61acb2044 100644 --- a/src/OneScript.Native/Compiler/ModuleCompiler.cs +++ b/src/OneScript.Native/Compiler/ModuleCompiler.cs @@ -5,6 +5,7 @@ This Source Code Form is subject to the terms of the at http://mozilla.org/MPL/2.0/. ----------------------------------------------------------*/ +using System; using System.Linq; using OneScript.Compilation; using OneScript.Compilation.Binding; @@ -166,8 +167,22 @@ protected override void VisitModuleVariable(VariableDefinitionNode varNode) } } + protected override void VisitGotoNode(NonTerminalNode node) + { + throw new NotSupportedException(); + } + + protected override void VisitLabelNode(LabelNode node) + { + throw new NotSupportedException(); + } + protected override void VisitMethod(MethodNode methodNode) { + if (methodNode.IsAsync) + { + AddError(LocalizedErrors.AsyncMethodsNotSupported(), methodNode.Location); + } var methodSymbol = Symbols.GetScope(Symbols.ScopeCount - 1).Methods[methodNode.Signature.MethodName]; var methodInfo = (BslNativeMethodInfo)methodSymbol.Method; diff --git a/src/ScriptEngine/Compiler/StackMachineCodeGenerator.cs b/src/ScriptEngine/Compiler/StackMachineCodeGenerator.cs index c37e0938b..e361f4e9f 100644 --- a/src/ScriptEngine/Compiler/StackMachineCodeGenerator.cs +++ b/src/ScriptEngine/Compiler/StackMachineCodeGenerator.cs @@ -224,9 +224,23 @@ private static string[] GetVariableNames(SymbolScope localCtx) return localCtx.Variables.Select(v => v.Name).ToArray(); } + + protected override void VisitGotoNode(NonTerminalNode node) + { + throw new NotSupportedException(); + } + + protected override void VisitLabelNode(LabelNode node) + { + throw new NotSupportedException(); + } protected override void VisitMethod(MethodNode methodNode) { + if (methodNode.IsAsync) + { + AddError(LocalizedErrors.AsyncMethodsNotSupported(), methodNode.Location); + } var signature = methodNode.Signature; var methodBuilder = NewMethod(); diff --git a/src/Tests/OneScript.Language.Tests/LexerTests.cs b/src/Tests/OneScript.Language.Tests/LexerTests.cs index 37552c9c8..2f13ab36c 100644 --- a/src/Tests/OneScript.Language.Tests/LexerTests.cs +++ b/src/Tests/OneScript.Language.Tests/LexerTests.cs @@ -634,9 +634,10 @@ public void Lexer_Ignores_Comments() } [Fact] - public void Leer_Extracts_Labels() + public void Lexer_Extracts_Labels() { - string code = "~ImALabel: ~LabelRef;"; + string code = "~ImALabel:\n" + + " ~LabelRef;"; var lexer = GetLexerForCode(code); var lex = lexer.NextLexem(); diff --git a/src/Tests/OneScript.Language.Tests/ParserTests.cs b/src/Tests/OneScript.Language.Tests/ParserTests.cs index f3495dc1a..41ed6f52a 100644 --- a/src/Tests/OneScript.Language.Tests/ParserTests.cs +++ b/src/Tests/OneScript.Language.Tests/ParserTests.cs @@ -1336,6 +1336,38 @@ public void DoubleAwaitIsForbidden() CatchParsingError(code, err => err.Single().ErrorId.Should().Be("ExpressionSyntax")); } + + [Fact] + public void Labels_Can_Appear_In_CodeBlocks() + { + var code = + @"А = 1; + ~Метка: + Б = 2; + "; + + var validator = ParseBatchAndGetValidator(code); + + validator.NextChildIs(NodeKind.Assignment); + validator.NextChildIs(NodeKind.Label); + validator.NextChildIs(NodeKind.Assignment); + } + + [Fact] + public void Goto_Can_Appear_In_CodeBlocks() + { + var code = + @"А = 1; + Перейти ~Метка; + Б = 2; + "; + + var validator = ParseBatchAndGetValidator(code); + + validator.NextChildIs(NodeKind.Assignment); + validator.NextChildIs(NodeKind.Goto); + validator.NextChildIs(NodeKind.Assignment); + } private static void CatchParsingError(string code) { From 7246307e627303336f84be61dd0946af01f525e1 Mon Sep 17 00:00:00 2001 From: Andrei Ovsiankin Date: Thu, 19 Oct 2023 19:08:27 +0300 Subject: [PATCH 8/8] =?UTF-8?q?=D0=A3=D0=B4=D0=B0=D0=BB=D0=B5=D0=BD=20?= =?UTF-8?q?=D1=81=D1=82=D0=B0=D1=80=D1=8B=D0=B9=20=D1=84=D0=B0=D0=B9=D0=BB?= =?UTF-8?q?=20=D1=81=D0=B1=D0=BE=D1=80=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Build.csproj | 248 ------------------------------------------------ src/1Script.sln | 1 - 2 files changed, 249 deletions(-) delete mode 100644 Build.csproj diff --git a/Build.csproj b/Build.csproj deleted file mode 100644 index d7273cda1..000000000 --- a/Build.csproj +++ /dev/null @@ -1,248 +0,0 @@ - - - - - 1.1.0 - $(BUILD_NUMBER) - 0 - - $(MSBuildProjectDirectory)/built - $(MSBuildProjectDirectory)/src/1Script.sln - Release - - - - - x86 - bin32 - - - Any CPU - bin - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - $(ArtifactsRoot)/tmp - $(TempFolder)/lib - $(TempFolder)/bin - $(TempFolder)/examples - $(TempFolder)/doc - $(ArtifactsRoot)\vscode\ - $(ArtifactsRoot)/mddoc - - - - - - - - - - - - - - - - - - - - - - $(TempFolder)/bin - - - $(TempFolder)/bin32 - - - - - - - $(TempFolder)/bin - - - $(TempFolder)/bin32 - - - - - - - - - $(MSBuildProjectDirectory)/src/VSCode.DebugAdapter/bin/$(Configuration)/net461 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - "$(InnoSetupPath)\iscc.exe" - $(ArtifactsRoot) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/1Script.sln b/src/1Script.sln index 9da3cfeaf..058492ea8 100644 --- a/src/1Script.sln +++ b/src/1Script.sln @@ -17,7 +17,6 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{D14BF321-348C-46B8-B96A-43A22BA7AC10}" ProjectSection(SolutionItems) = preProject - ..\Build.csproj = ..\Build.csproj oscommon.targets = oscommon.targets ..\Build_Core.csproj = ..\Build_Core.csproj ..\Jenkinsfile = ..\Jenkinsfile