Merge pull request #8358 from FirebirdSQL/work/isql-frontend-parser

Refactor ISQL creating FrontendParser class

builds/win32/msvc15/isql_static.vcxproj

@@ -24,6 +24,7 @@
     <ClCompile Include="..\..\..\gen\isql\extract.cpp" />
     <ClCompile Include="..\..\..\src\common\fb_exception.cpp" />
     <ClCompile Include="..\..\..\src\isql\FrontendLexer.cpp" />
+    <ClCompile Include="..\..\..\src\isql\FrontendParser.cpp" />
     <ClCompile Include="..\..\..\src\isql\InputDevices.cpp" />
     <ClCompile Include="..\..\..\gen\isql\isql.cpp" />
     <ClCompile Include="..\..\..\src\isql\iutils.cpp" />
@@ -40,6 +41,7 @@
     <ClInclude Include="..\..\..\src\isql\Extender.h" />
     <ClInclude Include="..\..\..\src\isql\extra_proto.h" />
     <ClInclude Include="..\..\..\src\isql\FrontendLexer.h" />
+    <ClInclude Include="..\..\..\src\isql\FrontendParser.h" />
     <ClInclude Include="..\..\..\src\isql\InputDevices.h" />
     <ClInclude Include="..\..\..\src\isql\isql.h" />
     <ClInclude Include="..\..\..\src\isql\isql_proto.h" />

builds/win32/msvc15/isql_static.vcxproj.filters

@@ -30,6 +30,9 @@
     <ClCompile Include="..\..\..\src\isql\FrontendLexer.cpp">
       <Filter>ISQL files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\src\isql\FrontendParser.cpp">
+      <Filter>ISQL files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\src\isql\InputDevices.cpp">
       <Filter>ISQL files</Filter>
     </ClCompile>
@@ -73,6 +76,9 @@
     <ClInclude Include="..\..\..\src\isql\FrontendLexer.h">
       <Filter>Header files</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\..\src\isql\FrontendParser.h">
+      <Filter>Header files</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\..\src\isql\InputDevices.h">
       <Filter>Header files</Filter>
     </ClInclude>

builds/win32/msvc15/isql_test.vcxproj

@@ -177,6 +177,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\..\src\isql\tests\FrontendLexerTest.cpp" />
+    <ClCompile Include="..\..\..\src\isql\tests\FrontendParserTest.cpp" />
     <ClCompile Include="..\..\..\src\isql\tests\ISqlTest.cpp" />
   </ItemGroup>
   <ItemGroup>

builds/win32/msvc15/isql_test.vcxproj.filters

@@ -10,6 +10,9 @@
     <ClCompile Include="..\..\..\src\isql\tests\FrontendLexerTest.cpp">
       <Filter>source</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\src\isql\tests\FrontendParserTest.cpp">
+      <Filter>source</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\src\isql\tests\ISqlTest.cpp">
       <Filter>source</Filter>
     </ClCompile>

src/common/StdHelper.h

@@ -0,0 +1,42 @@
+/*
+ *  The contents of this file are subject to the Initial
+ *  Developer's Public License Version 1.0 (the "License");
+ *  you may not use this file except in compliance with the
+ *  License. You may obtain a copy of the License at
+ *  http://www.ibphoenix.com/main.nfs?a=ibphoenix&page=ibp_idpl.
+ *
+ *  Software distributed under the License is distributed AS IS,
+ *  WITHOUT WARRANTY OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing rights
+ *  and limitations under the License.
+ *
+ *  The Original Code was created by Adriano dos Santos Fernandes
+ *  for the Firebird Open Source RDBMS project.
+ *
+ *  Copyright (c) 2024 Adriano dos Santos Fernandes <adrianosf at gmail.com>
+ *  and all contributors signed below.
+ *
+ *  All Rights Reserved.
+ *  Contributor(s): ______________________________________.
+ *
+ */
+
+#ifndef FB_COMMON_STD_HELPER_H
+#define FB_COMMON_STD_HELPER_H
+
+namespace Firebird {
+
+// To be used with std::visit
+
+template <typename... Ts>
+struct StdVisitOverloads : Ts...
+{
+	using Ts::operator()...;
+};
+
+template <typename... Ts>
+StdVisitOverloads(Ts...) -> StdVisitOverloads<Ts...>;
+
+}	// namespace Firebird
+
+#endif	// FB_COMMON_STD_HELPER_H

src/common/classes/MetaString.h

@@ -93,6 +93,30 @@ class MetaString
 	int compare(const AbstractString& s) const { return compare(s.c_str(), s.length()); }
 	int compare(const MetaString& m) const { return memcmp(data, m.data, MAX_SQL_IDENTIFIER_SIZE); }
 
+	string toQuotedString() const
+	{
+		string s;
+
+		if (hasData())
+		{
+			s.reserve(count + 2);
+
+			s.append("\"");
+
+			for (const auto c : *this)
+			{
+				if (c == '"')
+					s.append("\"");
+
+				s.append(&c, 1);
+			}
+
+			s.append("\"");
+		}
+
+		return s;
+	}
+
 	bool operator==(const char* s) const { return compare(s) == 0; }
 	bool operator!=(const char* s) const { return compare(s) != 0; }
 	bool operator==(const AbstractString& s) const { return compare(s) == 0; }

src/isql/FrontendLexer.cpp

@@ -28,9 +28,7 @@
 #include <cctype>
 
 
-static std::string trim(std::string_view str);
-
-static std::string trim(std::string_view str)
+std::string FrontendLexer::trim(std::string_view str)
 {
 	auto finish = str.end();
 	auto start = str.begin();
@@ -142,7 +140,7 @@ std::variant<FrontendLexer::SingleStatement, FrontendLexer::IncompleteTokenError
 
 		while (pos < end)
 		{
-			if (end - pos >= term.length() && std::equal(term.begin(), term.end(), pos))
+			if (std::size_t(end - pos) >= term.length() && std::equal(term.begin(), term.end(), pos))
 			{
 				const auto initialStatement = std::string(buffer.cbegin(), pos);
 				pos += term.length();
@@ -189,28 +187,85 @@ FrontendLexer::Token FrontendLexer::getToken()
 
 	switch (toupper(*pos))
 	{
-		case '(':
-			token.type = Token::TYPE_OPEN_PAREN;
+		case ';':
+		case '.':
+			token.type = Token::TYPE_OTHER;
 			token.processedText = *pos++;
 			break;
 
-		case ')':
-			token.type = Token::TYPE_CLOSE_PAREN;
-			token.processedText = *pos++;
+		default:
+			while (pos != end && !fb_utils::isspace(*pos))
+				++pos;
+
+			token.processedText = std::string(start, pos);
+			std::transform(token.processedText.begin(), token.processedText.end(),
+				token.processedText.begin(), toupper);
 			break;
+	}
+
+	token.rawText = std::string(start, pos);
+
+	return token;
+}
+
+FrontendLexer::Token FrontendLexer::getNameToken()
+{
+	skipSpacesAndComments();
+
+	Token token;
+
+	if (pos >= end)
+	{
+		token.type = Token::TYPE_EOF;
+		return token;
+	}
+
+	if (const auto optStringToken = getStringToken(); optStringToken.has_value())
+		return optStringToken.value();
+
+	/*** Revert to strict parsing with schemas support branch.
+	const auto start = pos;
+	bool first = true;
+
+	while (pos < end)
+	{
+		const auto c = *pos++;
+
+		if (!((c >= 'A' && c <= 'Z') ||
+			  (c >= 'a' && c <= 'z') ||
+			  c == '{' ||
+			  c == '}' ||
+			  (!first && c >= '0' && c <= '9') ||
+			  (!first && c == '$') ||
+			  (!first && c == '_')))
+		{
+			if (!first)
+				--pos;
 
-		case ',':
-			token.type = Token::TYPE_COMMA;
-			token.processedText = *pos++;
 			break;
+		}
 
+		first = false;
+	}
+
+	token.processedText = token.rawText = std::string(start, pos);
+	std::transform(token.processedText.begin(), token.processedText.end(),
+		token.processedText.begin(), toupper);
+
+	return token;
+	***/
+
+	const auto start = pos;
+
+	switch (toupper(*pos))
+	{
 		case ';':
 			token.type = Token::TYPE_OTHER;
 			token.processedText = *pos++;
 			break;
 
 		default:
-			while (pos != end && !fb_utils::isspace(*pos))
+			while (pos != end && !fb_utils::isspace(*pos) && *pos != '.')
 				++pos;
 
 			token.processedText = std::string(start, pos);
@@ -226,11 +281,10 @@ FrontendLexer::Token FrontendLexer::getToken()
 
 std::optional<FrontendLexer::Token> FrontendLexer::getStringToken()
 {
-	Token token;
-
 	if (pos >= end)
 		return std::nullopt;
 
+	Token token;
 	const auto start = pos;
 
 	switch (toupper(*pos))

src/isql/FrontendLexer.h

@@ -39,15 +39,18 @@ class FrontendLexer
 			TYPE_EOF,
 			TYPE_STRING,
 			TYPE_META_STRING,
-			TYPE_OPEN_PAREN,
-			TYPE_CLOSE_PAREN,
-			TYPE_COMMA,
 			TYPE_OTHER
 		};
 
 		Type type = TYPE_OTHER;
 		std::string rawText;
 		std::string processedText;
+
+		std::string getProcessedString() const
+		{
+			return type == FrontendLexer::Token::TYPE_STRING || type == FrontendLexer::Token::TYPE_META_STRING ?
+				processedText : rawText;
+		}
 	};
 
 	struct SingleStatement
@@ -74,6 +77,7 @@ class FrontendLexer
 	FrontendLexer& operator=(const FrontendLexer&) = delete;
 
 public:
+	static std::string trim(std::string_view str);
 	static std::string stripComments(std::string_view statement);
 
 public:
@@ -87,6 +91,11 @@ class FrontendLexer
 		return pos;
 	}
 
+	void setPos(std::string::const_iterator newPos)
+	{
+		pos = newPos;
+	}
+
 	void rewind()
 	{
 		deletePos = buffer.begin();
@@ -97,7 +106,9 @@ class FrontendLexer
 	void appendBuffer(std::string_view newBuffer);
 	void reset();
 	std::variant<SingleStatement, FrontendLexer::IncompleteTokenError> getSingleStatement(std::string_view term);
+
 	Token getToken();
+	Token getNameToken();
 
 private:
 	std::optional<Token> getStringToken();