Skip to content

Commit

Permalink
Antlr4 sql flag (ydb-platform#9091)
Browse files Browse the repository at this point in the history
  • Loading branch information
@vitstn
vitstn authored Sep 11, 2024
1 parent 8d11d04 commit a837a20
Show file tree
Hide file tree
Showing 4 changed files with 26 additions and 10 deletions.
6 changes: 6 additions & 0 deletions ydb/library/yql/sql/settings/translation_settings.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,10 @@ namespace NSQLTranslation {
return false;
}

if (settings.Flags.contains("Antlr4")) {
settings.Antlr4Parser = true;
}

TSplitDelimiters lineDelimiters("\n\r");
TDelimitersSplit linesSplit(query, lineDelimiters);
auto lineIterator = linesSplit.Iterator();
Expand Down Expand Up @@ -126,6 +130,8 @@ namespace NSQLTranslation {
settings.AnsiLexer = true;
} else if (value == "antlr4_parser") {
settings.Antlr4Parser = true;
} else if (value == "antlr3_parser") {
settings.Antlr4Parser = false;
} else if (value == "syntax_pg") {
settings.PgParser = true;
} else {
Expand Down
17 changes: 8 additions & 9 deletions ydb/library/yql/tools/yqlrun/http/yql_server.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ class TOptPipelineConfigurator : public IPipelineConfigurator, TLogLevelPromoute
IOutputStream* Stream;
};

NSQLTranslation::TTranslationSettings GetTranslationSettings(const THolder<TGatewaysConfig>& gatewaysConfig) {
NSQLTranslation::TTranslationSettings GetTranslationSettings(const THashSet<TString>& sqlFlags) {
static const THashMap<TString, TString> clusters = {
{ "plato", TString(YtProviderName) },
{ "plato_rtmr", TString(RtmrProviderName) },
Expand All @@ -151,9 +151,7 @@ NSQLTranslation::TTranslationSettings GetTranslationSettings(const THolder<TGate
settings.SyntaxVersion = 1;
settings.InferSyntaxVersion = true;
settings.V0Behavior = NSQLTranslation::EV0Behavior::Report;
if (gatewaysConfig && gatewaysConfig->HasSqlCore()) {
settings.Flags.insert(gatewaysConfig->GetSqlCore().GetTranslationFlags().begin(), gatewaysConfig->GetSqlCore().GetTranslationFlags().end());
}
settings.Flags = sqlFlags;
return settings;
}

Expand Down Expand Up @@ -326,7 +324,7 @@ YQL_ACTION(Parse)

bool parsed = (options & TYqlAction::YqlProgram)
? prg->ParseYql()
: prg->ParseSql(GetTranslationSettings(YqlServer.GatewaysConfig));
: prg->ParseSql(GetTranslationSettings(YqlServer.SqlFlags));

if (parsed) {
ui32 prettyFlg = TAstPrintFlags::PerLine | TAstPrintFlags::ShortQuote;
Expand All @@ -353,7 +351,7 @@ YQL_ACTION(Compile)
TProgramPtr prg = MakeFileProgram(program, YqlServer, {}, {}, tmpDir.Name());
prg->SetParametersYson(parameters);

bool noError = (options & TYqlAction::YqlProgram) ? prg->ParseYql() : prg->ParseSql(GetTranslationSettings(YqlServer.GatewaysConfig));
bool noError = (options & TYqlAction::YqlProgram) ? prg->ParseYql() : prg->ParseSql(GetTranslationSettings(YqlServer.SqlFlags));
noError = noError && prg->Compile(GetUsername());

if (options & (EOptions::PrintAst | EOptions::PrintExpr)) {
Expand Down Expand Up @@ -387,7 +385,7 @@ YQL_ACTION(OptimizeOrValidateFile)
TTempDir tmpDir;
TProgramPtr prg = MakeFileProgram(program, input, attr, inputFile, outputFile, YqlServer, tmpDir.Name());

bool noError = (options & TYqlAction::YqlProgram) ? prg->ParseYql() : prg->ParseSql(GetTranslationSettings(YqlServer.GatewaysConfig));
bool noError = (options & TYqlAction::YqlProgram) ? prg->ParseYql() : prg->ParseSql(GetTranslationSettings(YqlServer.SqlFlags));

prg->SetParametersYson(parameters);
prg->SetDiagnosticFormat(NYson::EYsonFormat::Pretty);
Expand Down Expand Up @@ -462,7 +460,7 @@ YQL_ACTION(FileRun)
TTempDir tmpDir;
TProgramPtr prg = MakeFileProgram(program, input, attr, inputFile, outputFile, YqlServer, tmpDir.Name());

bool noError = (options & TYqlAction::YqlProgram) ? prg->ParseYql() : prg->ParseSql(GetTranslationSettings(YqlServer.GatewaysConfig));
bool noError = (options & TYqlAction::YqlProgram) ? prg->ParseYql() : prg->ParseSql(GetTranslationSettings(YqlServer.SqlFlags));

prg->SetDiagnosticFormat(NYson::EYsonFormat::Pretty);
prg->SetParametersYson(parameters);
Expand Down Expand Up @@ -634,13 +632,14 @@ TAutoPtr<TYqlServer> CreateYqlServer(
ui64 nextUniqueId,
TUserDataTable filesMapping,
THolder<TGatewaysConfig>&& gatewaysConfig,
const THashSet<TString>& sqlFlags,
IModuleResolver::TPtr modules,
IUdfResolver::TPtr udfResolver,
TFileStoragePtr fileStorage)
{
TAutoPtr<TYqlServer> server = new TYqlServer(
config, functionRegistry, udfIndex, nextUniqueId,
std::move(filesMapping), std::move(gatewaysConfig), modules, udfResolver, fileStorage);
std::move(filesMapping), std::move(gatewaysConfig), sqlFlags, modules, udfResolver, fileStorage);

server->RegisterAction<TYqlActionPaste>("/api/yql/paste");
server->RegisterAction<TYqlActionParse>("/api/yql/parse");
Expand Down
4 changes: 4 additions & 0 deletions ydb/library/yql/tools/yqlrun/http/yql_server.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ class TYqlServer: private TNonCopyable
ui64 nextUniqueId,
TUserDataTable filesMapping,
THolder<TGatewaysConfig>&& gatewaysConfig,
const THashSet<TString>& sqlFlags,
IModuleResolver::TPtr modules,
IUdfResolver::TPtr udfResolver,
TFileStoragePtr fileStorage)
Expand All @@ -46,6 +47,7 @@ class TYqlServer: private TNonCopyable
, NextUniqueId(nextUniqueId)
, FilesMapping(std::move(filesMapping))
, GatewaysConfig(std::move(gatewaysConfig))
, SqlFlags(sqlFlags)
, Modules(modules)
, UdfResolver(udfResolver)
, FileStorage(fileStorage)
Expand All @@ -72,6 +74,7 @@ class TYqlServer: private TNonCopyable
ui64 NextUniqueId;
TUserDataTable FilesMapping;
const THolder<TGatewaysConfig> GatewaysConfig;
const THashSet<TString> SqlFlags;
IModuleResolver::TPtr Modules;
IUdfResolver::TPtr UdfResolver;
TFileStoragePtr FileStorage;
Expand All @@ -84,6 +87,7 @@ TAutoPtr<TYqlServer> CreateYqlServer(
ui64 nextUniqueId,
TUserDataTable filesMapping,
THolder<TGatewaysConfig>&& gatewaysConfig,
const THashSet<TString>& sqlFlags,
IModuleResolver::TPtr modules = nullptr,
IUdfResolver::TPtr udfResolver = nullptr,
TFileStoragePtr fileStorage = nullptr);
Expand Down
9 changes: 8 additions & 1 deletion ydb/library/yql/tools/yqlrun/yqlrun.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -935,6 +935,7 @@ int RunUI(int argc, const char* argv[])

THashMap<TString, TString> clusterMapping;
clusterMapping["plato"] = YtProviderName;
THashSet<TString> sqlFlags;

NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default();
opts.AddLongOption('u', "udf", "Load shared library with UDF by given path").AppendTo(&udfsPaths);
Expand All @@ -951,6 +952,7 @@ int RunUI(int argc, const char* argv[])
opts.AddLongOption("gateways-cfg", "gateways configuration file").Optional().RequiredArgument("FILE").StoreResult(&gatewaysCfgFile);
opts.AddLongOption("fs-cfg", "fs configuration file").Optional().RequiredArgument("FILE").StoreResult(&fsCfgFile);
opts.AddLongOption("pg-ext", "pg extensions config file").StoreResult(&pgExtConfig);
opts.AddLongOption("sql-flags", "SQL translator pragma flags").SplitHandler(&sqlFlags, ',');

TServerConfig config;
config.SetAssetsPath("http/www");
Expand Down Expand Up @@ -993,6 +995,10 @@ int RunUI(int argc, const char* argv[])
if (!gatewaysConfig) {
return -1;
}

if (gatewaysConfig->HasSqlCore()) {
sqlFlags.insert(gatewaysConfig->GetSqlCore().GetTranslationFlags().begin(), gatewaysConfig->GetSqlCore().GetTranslationFlags().end());
}
}

THolder<TFileStorageConfig> fsConfig;
Expand Down Expand Up @@ -1028,7 +1034,7 @@ int RunUI(int argc, const char* argv[])
return -1;
}

moduleResolver = std::make_shared<TModuleResolver>(std::move(modules), ctx.NextUniqueId, clusterMapping, THashSet<TString>());
moduleResolver = std::make_shared<TModuleResolver>(std::move(modules), ctx.NextUniqueId, clusterMapping, sqlFlags);
} else {
if (!GetYqlDefaultModuleResolver(ctx, moduleResolver, clusterMapping)) {
Cerr << "Errors loading default YQL libraries:" << Endl;
Expand Down Expand Up @@ -1058,6 +1064,7 @@ int RunUI(int argc, const char* argv[])
funcRegistry.Get(), udfIndex, ctx.NextUniqueId,
userData,
std::move(gatewaysConfig),
sqlFlags,
moduleResolver, udfResolver, fileStorage);
server->Start();
server->Wait();
Expand Down

0 comments on commit a837a20

Please sign in to comment.