From fb9afbafc7dfe226b9db54d4923bfb8839635274 Mon Sep 17 00:00:00 2001 From: Anton Bobukh Date: Tue, 18 Jun 2024 16:02:57 -0700 Subject: [PATCH] [gRPC] Update the code generator for Python to produce typed handlers (#8326) * Move `namer.h` and `idl_namer.h` to `include/codegen` so they can be reused from `grpc` dirqectory. * [gRPC] Update the Python generator to produce typed handlers and Python stubs if requested. * [gRPC] Document the newly added compiler flags. --- CMakeLists.txt | 2 + docs/source/Compiler.md | 43 ++- grpc/src/compiler/BUILD.bazel | 2 + grpc/src/compiler/python_generator.cc | 445 +++++++++++++++++++------- grpc/src/compiler/python_generator.h | 19 +- include/codegen/BUILD.bazel | 17 +- include/codegen/idl_namer.h | 179 +++++++++++ include/codegen/namer.h | 270 ++++++++++++++++ include/codegen/python.h | 48 +++ include/flatbuffers/idl.h | 6 +- src/BUILD.bazel | 2 + src/flatc.cpp | 8 + src/idl_gen_grpc.cpp | 54 +--- src/idl_gen_python.cpp | 50 +-- src/idl_namer.h | 181 +---------- src/namer.h | 272 +--------------- tests/PythonTest.sh | 1 + tests/service_test.fbs | 11 + tests/service_test_generated.py | 58 ++++ tests/service_test_generated.pyi | 26 ++ tests/service_test_grpc.fb.py | 97 ++++++ tests/service_test_grpc.fb.pyi | 27 ++ 22 files changed, 1165 insertions(+), 653 deletions(-) create mode 100644 include/codegen/idl_namer.h create mode 100644 include/codegen/namer.h create mode 100644 tests/service_test.fbs create mode 100644 tests/service_test_generated.py create mode 100644 tests/service_test_generated.pyi create mode 100644 tests/service_test_grpc.fb.py create mode 100644 tests/service_test_grpc.fb.pyi diff --git a/CMakeLists.txt b/CMakeLists.txt index 755d2b47326..e7038f89d73 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -183,6 +183,8 @@ set(FlatBuffers_Compiler_SRCS src/bfbs_gen_lua.h src/bfbs_gen_nim.h src/bfbs_namer.h + include/codegen/idl_namer.h + include/codegen/namer.h include/codegen/python.h include/codegen/python.cc include/flatbuffers/code_generators.h diff --git a/docs/source/Compiler.md b/docs/source/Compiler.md index ff71378b526..a1a095a4d24 100644 --- a/docs/source/Compiler.md +++ b/docs/source/Compiler.md @@ -96,10 +96,10 @@ Additional options: - `--scoped-enums` : Use C++11 style scoped and strongly typed enums in generated C++. This also implies `--no-prefix`. - + - `--no-emit-min-max-enum-values` : Disable generation of MIN and MAX enumerated values for scoped enums and prefixed enums. - + - `--gen-includes` : (deprecated), this is the default behavior. If the original behavior is required (no include statements) use `--no-includes.` @@ -238,5 +238,44 @@ Additional options: - `--python-typing` : Generate Python type annotations +Additional gRPC options: + +- `--grpc-filename-suffix`: `[C++]` An optional suffix for the generated + files' names. For example, compiling gRPC for C++ with + `--grpc-filename-suffix=.fbs` will generate `{name}.fbs.h` and + `{name}.fbs.cc` files. + +- `--grpc-additional-header`: `[C++]` Additional headers to include in the + generated files. + +- `--grpc-search-path`: `[C++]` An optional prefix for the gRPC runtime path. + For example, compiling gRPC for C++ with `--grpc-search-path=some/path` will + generate the following includes: + + ```cpp + #include "some/path/grpcpp/impl/codegen/async_stream.h" + #include "some/path/grpcpp/impl/codegen/async_unary_call.h" + #include "some/path/grpcpp/impl/codegen/method_handler.h" + ... + ``` + +- `--grpc-use-system-headers`: `[C++]` Whether to generate `#include
` + instead of `#include "header.h"` for all headers when compiling gRPC for + C++. For example, compiling gRPC for C++ with `--grpc-use-system-headers` + will generate the following includes: + + ```cpp + #include + #include + #include + ... + ``` + + NOTE: This option can be negated with `--no-grpc-use-system-headers`. + +- `--grpc-python-typed-handlers`: `[Python]` Whether to generate the typed + handlers that use the generated Python classes instead of raw bytes for + requests/responses. + NOTE: short-form options for generators are deprecated, use the long form whenever possible. diff --git a/grpc/src/compiler/BUILD.bazel b/grpc/src/compiler/BUILD.bazel index 0efa9560c2d..a73a79ba45d 100644 --- a/grpc/src/compiler/BUILD.bazel +++ b/grpc/src/compiler/BUILD.bazel @@ -95,6 +95,8 @@ cc_library( visibility = ["//visibility:private"], deps = [ "//:flatbuffers", + "//include/codegen:namer", + "//include/codegen:python", ], ) diff --git a/grpc/src/compiler/python_generator.cc b/grpc/src/compiler/python_generator.cc index d5f69e20e71..91203a28024 100644 --- a/grpc/src/compiler/python_generator.cc +++ b/grpc/src/compiler/python_generator.cc @@ -16,136 +16,365 @@ * */ +#include "src/compiler/python_generator.h" + +#include +#include #include +#include #include +#include +#include "codegen/idl_namer.h" +#include "codegen/namer.h" +#include "codegen/python.h" +#include "flatbuffers/idl.h" #include "flatbuffers/util.h" -#include "src/compiler/python_generator.h" -namespace grpc_python_generator { +namespace flatbuffers { +namespace python { +namespace grpc { namespace { +bool ClientStreaming(const RPCCall *method) { + const Value *val = method->attributes.Lookup("streaming"); + return val != nullptr && (val->constant == "client" || val->constant == "bidi"); +} -static grpc::string GenerateMethodType(const grpc_generator::Method *method) { +bool ServerStreaming(const RPCCall *method) { + const Value *val = method->attributes.Lookup("streaming"); + return val != nullptr && (val->constant == "server" || val->constant == "bidi"); +} - if (method->NoStreaming()) - return "unary_unary"; +void FormatImports(std::stringstream &ss, const Imports &imports) { + std::set modules; + std::map> names_by_module; + for (const Import &import : imports.imports) { + if (import.IsLocal()) continue; // skip all local imports + if (import.name == "") { + modules.insert(import.module); + } else { + names_by_module[import.module].insert(import.name); + } + } - if (method->ServerStreaming()) - return "unary_stream"; + for (const std::string &module : modules) { + ss << "import " << module << '\n'; + } + ss << '\n'; + for (const auto &import : names_by_module) { + ss << "from " << import.first << " import "; + size_t i = 0; + for (const std::string &name : import.second) { + if (i > 0) ss << ", "; + ss << name; + ++i; + } + ss << '\n'; + } + ss << "\n\n"; +} - if (method->ClientStreaming()) - return "stream_unary"; +bool SaveStub(const std::string &filename, const Imports &imports, + const std::string &content) { + std::stringstream ss; + ss << "# Generated by the gRPC FlatBuffers compiler. DO NOT EDIT!\n" + << '\n' + << "from __future__ import annotations\n" + << '\n'; + FormatImports(ss, imports); + ss << content << '\n'; - return "stream_stream"; + EnsureDirExists(StripFileName(filename)); + return flatbuffers::SaveFile(filename.c_str(), ss.str(), false); } -grpc::string GenerateMethodInput(const grpc_generator::Method *method) { - - if (method->NoStreaming() || method->ServerStreaming()) - return "self, request, context"; +bool SaveService(const std::string &filename, const Imports &imports, + const std::string &content) { + std::stringstream ss; + ss << "# Generated by the gRPC FlatBuffers compiler. DO NOT EDIT!\n" << '\n'; + FormatImports(ss, imports); + ss << content << '\n'; - return "self, request_iterator, context"; + EnsureDirExists(StripFileName(filename)); + return flatbuffers::SaveFile(filename.c_str(), ss.str(), false); } -void GenerateStub(const grpc_generator::Service *service, - grpc_generator::Printer *printer, - std::map *dictonary) { - auto vars = *dictonary; - printer->Print(vars, "class $ServiceName$Stub(object):\n"); - printer->Indent(); - printer->Print("\"\"\" Interface exported by the server. \"\"\""); - printer->Print("\n\n"); - printer->Print("def __init__(self, channel):\n"); - printer->Indent(); - printer->Print("\"\"\" Constructor. \n\n"); - printer->Print("Args: \n"); - printer->Print("channel: A grpc.Channel. \n"); - printer->Print("\"\"\"\n\n"); - - for (int j = 0; j < service->method_count(); j++) { - auto method = service->method(j); - vars["MethodName"] = method->name(); - vars["MethodType"] = GenerateMethodType(&*method); - printer->Print(vars, "self.$MethodName$ = channel.$MethodType$(\n"); - printer->Indent(); - printer->Print(vars, "\"/$PATH$$ServiceName$/$MethodName$\"\n"); - printer->Print(")\n"); - printer->Outdent(); - printer->Print("\n"); +class BaseGenerator { + protected: + BaseGenerator(const Parser &parser, const Namer::Config &config, + const std::string &path, const Version &version) + : parser_{parser}, + namer_{WithFlagOptions(config, parser.opts, path), Keywords(version)}, + version_{version} {} + + protected: + std::string ModuleForFile(const std::string &file) const { + std::string module = parser_.opts.include_prefix + StripExtension(file) + + parser_.opts.filename_suffix; + std::replace(module.begin(), module.end(), '/', '.'); + return module; } - printer->Outdent(); - printer->Outdent(); - printer->Print("\n"); -} -void GenerateServicer(const grpc_generator::Service *service, - grpc_generator::Printer *printer, - std::map *dictonary) { - auto vars = *dictonary; - printer->Print(vars, "class $ServiceName$Servicer(object):\n"); - printer->Indent(); - printer->Print("\"\"\" Interface exported by the server. \"\"\""); - printer->Print("\n\n"); - - for (int j = 0; j < service->method_count(); j++) { - auto method = service->method(j); - vars["MethodName"] = method->name(); - vars["MethodInput"] = GenerateMethodInput(&*method); - printer->Print(vars, "def $MethodName$($MethodInput$):\n"); - printer->Indent(); - printer->Print("context.set_code(grpc.StatusCode.UNIMPLEMENTED)\n"); - printer->Print("context.set_details('Method not implemented!')\n"); - printer->Print("raise NotImplementedError('Method not implemented!')\n"); - printer->Outdent(); - printer->Print("\n\n"); + template + std::string ModuleFor(const T *def) const { + if (parser_.opts.one_file) return ModuleForFile(def->file); + return namer_.NamespacedType(*def); } - printer->Outdent(); - printer->Print("\n"); -} + const Parser &parser_; + const IdlNamer namer_; + const Version version_; +}; + +class StubGenerator : public BaseGenerator { + public: + StubGenerator(const Parser &parser, const std::string &path, + const Version &version) + : BaseGenerator(parser, kStubConfig, path, version) {} + + bool Generate() { + Imports imports; + std::stringstream stub; + for (const ServiceDef *service : parser_.services_.vec) { + Generate(stub, service, &imports); + } + + std::string filename = + namer_.config_.output_path + + StripPath(StripExtension(parser_.file_being_parsed_)) + "_grpc" + + parser_.opts.grpc_filename_suffix + namer_.config_.filename_extension; + + return SaveStub(filename, imports, stub.str()); + } + + private: + void Generate(std::stringstream &ss, const ServiceDef *service, + Imports *imports) { + imports->Import("grpc"); + + ss << "class " << service->name << "Stub(object):\n" + << " def __init__(self, channel: grpc.Channel) -> None: ...\n"; + + for (const RPCCall *method : service->calls.vec) { + std::string request = "bytes"; + std::string response = "bytes"; + + if (parser_.opts.grpc_python_typed_handlers) { + request = namer_.Type(*method->request); + response = namer_.Type(*method->response); + + imports->Import(ModuleFor(method->request), request); + imports->Import(ModuleFor(method->response), response); + } + + ss << " def " << method->name << "(self, "; + if (ClientStreaming(method)) { + imports->Import("typing"); + ss << "request_iterator: typing.Iterator[" << request << "]"; + } else { + ss << "request: " << request; + } + ss << ") -> "; + if (ServerStreaming(method)) { + imports->Import("typing"); + ss << "typing.Iterator[" << response << "]"; + } else { + ss << response; + } + ss << ": ...\n"; + } + + ss << "\n\n"; + ss << "class " << service->name << "Servicer(object):\n"; + + for (const RPCCall *method : service->calls.vec) { + std::string request = "bytes"; + std::string response = "bytes"; + + if (parser_.opts.grpc_python_typed_handlers) { + request = namer_.Type(*method->request); + response = namer_.Type(*method->response); + + imports->Import(ModuleFor(method->request), request); + imports->Import(ModuleFor(method->response), response); + } + + ss << " def " << method->name << "(self, "; + if (ClientStreaming(method)) { + imports->Import("typing"); + ss << "request_iterator: typing.Iterator[" << request << "]"; + } else { + ss << "request: " << request; + } + ss << ", context: grpc.ServicerContext) -> "; + if (ServerStreaming(method)) { + imports->Import("typing"); + ss << "typing.Iterator[" << response << "]"; + } else { + ss << response; + } + ss << ": ...\n"; + } -void GenerateRegister(const grpc_generator::Service *service, - grpc_generator::Printer *printer, - std::map *dictonary) { - auto vars = *dictonary; - printer->Print(vars, "def add_$ServiceName$Servicer_to_server(servicer, server):\n"); - printer->Indent(); - printer->Print("rpc_method_handlers = {\n"); - printer->Indent(); - for (int j = 0; j < service->method_count(); j++) { - auto method = service->method(j); - vars["MethodName"] = method->name(); - vars["MethodType"] = GenerateMethodType(&*method); - printer->Print(vars, "'$MethodName$': grpc.$MethodType$_rpc_method_handler(\n"); - printer->Indent(); - printer->Print(vars, "servicer.$MethodName$\n"); - printer->Outdent(); - printer->Print("),\n"); + ss << '\n' + << '\n' + << "def add_" << service->name + << "Servicer_to_server(servicer: " << service->name + << "Servicer, server: grpc.Server) -> None: ...\n"; } - printer->Outdent(); - printer->Print("}\n"); - printer->Print(vars, "generic_handler = grpc.method_handlers_generic_handler(\n"); - printer->Indent(); - printer->Print(vars, "'$PATH$$ServiceName$', rpc_method_handlers)\n"); - printer->Outdent(); - printer->Print("server.add_generic_rpc_handlers((generic_handler,))"); - printer->Outdent(); - printer->Print("\n"); +}; + +class ServiceGenerator : public BaseGenerator { + public: + ServiceGenerator(const Parser &parser, const std::string &path, + const Version &version) + : BaseGenerator(parser, kConfig, path, version) {} + + bool Generate() { + Imports imports; + std::stringstream ss; + + imports.Import("flatbuffers"); + + if (parser_.opts.grpc_python_typed_handlers) { + ss << "def _serialize_to_bytes(table):\n" + << " buf = table._tab.Bytes\n" + << " n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0)\n" + << " if table._tab.Pos != n:\n" + << " raise ValueError('must be a top-level table')\n" + << " return bytes(buf)\n" + << '\n' + << '\n'; + } + + for (const ServiceDef *service : parser_.services_.vec) { + GenerateStub(ss, service, &imports); + GenerateServicer(ss, service, &imports); + GenerateRegister(ss, service, &imports); + } + + std::string filename = + namer_.config_.output_path + + StripPath(StripExtension(parser_.file_being_parsed_)) + "_grpc" + + parser_.opts.grpc_filename_suffix + namer_.config_.filename_extension; + + return SaveService(filename, imports, ss.str()); + } + + private: + void GenerateStub(std::stringstream &ss, const ServiceDef *service, + Imports *imports) { + ss << "class " << service->name << "Stub"; + if (version_.major != 3) ss << "(object)"; + ss << ":\n" + << " '''Interface exported by the server.'''\n" + << '\n' + << " def __init__(self, channel):\n" + << " '''Constructor.\n" + << '\n' + << " Args:\n" + << " channel: A grpc.Channel.\n" + << " '''\n" + << '\n'; + + for (const RPCCall *method : service->calls.vec) { + std::string response = namer_.Type(*method->response); + + imports->Import(ModuleFor(method->response), response); + + ss << " self." << method->name << " = channel." + << (ClientStreaming(method) ? "stream" : "unary") << "_" + << (ServerStreaming(method) ? "stream" : "unary") << "(\n" + << " method='/" + << service->defined_namespace->GetFullyQualifiedName(service->name) + << "/" << method->name << "'"; + + if (parser_.opts.grpc_python_typed_handlers) { + ss << ",\n" + << " request_serializer=_serialize_to_bytes,\n" + << " response_deserializer=" << response << ".GetRootAs"; + } + ss << ")\n\n"; + } + + ss << '\n'; + } + + void GenerateServicer(std::stringstream &ss, const ServiceDef *service, + Imports *imports) { + imports->Import("grpc"); + + ss << "class " << service->name << "Servicer"; + if (version_.major != 3) ss << "(object)"; + ss << ":\n" + << " '''Interface exported by the server.'''\n" + << '\n'; + + for (const RPCCall *method : service->calls.vec) { + const std::string request_param = + ClientStreaming(method) ? "request_iterator" : "request"; + ss << " def " << method->name << "(self, " << request_param + << ", context):\n" + << " context.set_code(grpc.StatusCode.UNIMPLEMENTED)\n" + << " context.set_details('Method not implemented!')\n" + << " raise NotImplementedError('Method not implemented!')\n" + << '\n'; + } + + ss << '\n'; + } + + void GenerateRegister(std::stringstream &ss, const ServiceDef *service, + Imports *imports) { + imports->Import("grpc"); + + ss << "def add_" << service->name + << "Servicer_to_server(servicer, server):\n" + << " rpc_method_handlers = {\n"; + + for (const RPCCall *method : service->calls.vec) { + std::string request = namer_.Type(*method->request); + + imports->Import(ModuleFor(method->request), request); + + ss << " '" << method->name << "': grpc." + << (ClientStreaming(method) ? "stream" : "unary") << "_" + << (ServerStreaming(method) ? "stream" : "unary") + << "_rpc_method_handler(\n" + << " servicer." << method->name; + + if (parser_.opts.grpc_python_typed_handlers) { + ss << ",\n" + << " request_deserializer=" << request << ".GetRootAs,\n" + << " response_serializer=_serialize_to_bytes"; + } + ss << "),\n"; + } + ss << " }\n" + << '\n' + << " generic_handler = grpc.method_handlers_generic_handler(\n" + << " '" + << service->defined_namespace->GetFullyQualifiedName(service->name) + << "', rpc_method_handlers)\n" + << '\n' + << " server.add_generic_rpc_handlers((generic_handler,))\n" + << '\n'; + } +}; +} // namespace + +bool Generate(const Parser &parser, const std::string &path, + const Version &version) { + ServiceGenerator generator{parser, path, version}; + return generator.Generate(); } -} // namespace - -grpc::string Generate(grpc_generator::File *file, - const grpc_generator::Service *service) { - grpc::string output; - std::map vars; - vars["PATH"] = file->package(); - if (!file->package().empty()) { vars["PATH"].append("."); } - vars["ServiceName"] = service->name(); - auto printer = file->CreatePrinter(&output); - GenerateStub(service, &*printer, &vars); - GenerateServicer(service, &*printer, &vars); - GenerateRegister(service, &*printer, &vars); - return output; + +bool GenerateStub(const Parser &parser, const std::string &path, + const Version &version) { + StubGenerator generator{parser, path, version}; + return generator.Generate(); } -} // namespace grpc_python_generator +} // namespace grpc +} // namespace python +} // namespace flatbuffers diff --git a/grpc/src/compiler/python_generator.h b/grpc/src/compiler/python_generator.h index 40d29aada55..6335ecc9fbe 100644 --- a/grpc/src/compiler/python_generator.h +++ b/grpc/src/compiler/python_generator.h @@ -19,14 +19,21 @@ #ifndef GRPC_INTERNAL_COMPILER_PYTHON_GENERATOR_H #define GRPC_INTERNAL_COMPILER_PYTHON_GENERATOR_H -#include +#include -#include "src/compiler/schema_interface.h" +#include "codegen/python.h" +#include "flatbuffers/idl.h" -namespace grpc_python_generator { +namespace flatbuffers { +namespace python { +namespace grpc { +bool Generate(const Parser &parser, const std::string &path, + const Version &version); -grpc::string Generate(grpc_generator::File *file, - const grpc_generator::Service *service); -} // namespace grpc_python_generator +bool GenerateStub(const Parser &parser, const std::string &path, + const Version &version); +} // namespace grpc +} // namespace python +} // namespace flatbuffers #endif // GRPC_INTERNAL_COMPILER_PYTHON_GENERATOR_H diff --git a/include/codegen/BUILD.bazel b/include/codegen/BUILD.bazel index 0063e8b6e8d..196181b411b 100644 --- a/include/codegen/BUILD.bazel +++ b/include/codegen/BUILD.bazel @@ -15,10 +15,25 @@ filegroup( visibility = ["//visibility:public"], ) +cc_library( + name = "namer", + hdrs = [ + "idl_namer.h", + "namer.h", + ], + strip_include_prefix = "/include", + visibility = ["//:__subpackages__"], + deps = ["//:runtime_cc"], +) + cc_library( name = "python", srcs = ["python.cc"], hdrs = ["python.h"], strip_include_prefix = "/include", - visibility = ["//src:__subpackages__"], + visibility = [ + "//grpc:__subpackages__", + "//src:__subpackages__", + ], + deps = [":namer"], ) diff --git a/include/codegen/idl_namer.h b/include/codegen/idl_namer.h new file mode 100644 index 00000000000..dd2fe3b8afe --- /dev/null +++ b/include/codegen/idl_namer.h @@ -0,0 +1,179 @@ +#ifndef FLATBUFFERS_INCLUDE_CODEGEN_IDL_NAMER_H_ +#define FLATBUFFERS_INCLUDE_CODEGEN_IDL_NAMER_H_ + +#include "codegen/namer.h" +#include "flatbuffers/idl.h" + +namespace flatbuffers { + +// Provides Namer capabilities to types defined in the flatbuffers IDL. +class IdlNamer : public Namer { + public: + explicit IdlNamer(Config config, std::set keywords) + : Namer(config, std::move(keywords)) {} + + using Namer::Constant; + using Namer::Directories; + using Namer::Field; + using Namer::File; + using Namer::Function; + using Namer::Method; + using Namer::Namespace; + using Namer::NamespacedType; + using Namer::ObjectType; + using Namer::Type; + using Namer::Variable; + using Namer::Variant; + + std::string Constant(const FieldDef &d) const { return Constant(d.name); } + + // Types are always structs or enums so we can only expose these two + // overloads. + std::string Type(const StructDef &d) const { return Type(d.name); } + std::string Type(const EnumDef &d) const { return Type(d.name); } + + std::string Function(const Definition &s) const { return Function(s.name); } + std::string Function(const std::string& prefix, const Definition &s) const { + return Function(prefix + s.name); + } + + std::string Field(const FieldDef &s) const { return Field(s.name); } + std::string Field(const FieldDef &d, const std::string &s) const { + return Field(d.name + "_" + s); + } + + std::string Variable(const FieldDef &s) const { return Variable(s.name); } + + std::string Variable(const StructDef &s) const { return Variable(s.name); } + + std::string Variant(const EnumVal &s) const { return Variant(s.name); } + + std::string EnumVariant(const EnumDef &e, const EnumVal &v) const { + return Type(e) + config_.enum_variant_seperator + Variant(v); + } + + std::string ObjectType(const StructDef &d) const { + return ObjectType(d.name); + } + std::string ObjectType(const EnumDef &d) const { return ObjectType(d.name); } + + std::string Method(const FieldDef &d, const std::string &suffix) const { + return Method(d.name, suffix); + } + std::string Method(const std::string &prefix, const StructDef &d) const { + return Method(prefix, d.name); + } + std::string Method(const std::string &prefix, const FieldDef &d) const { + return Method(prefix, d.name); + } + std::string Method(const std::string &prefix, const FieldDef &d, + const std::string &suffix) const { + return Method(prefix, d.name, suffix); + } + + std::string Namespace(const struct Namespace &ns) const { + return Namespace(ns.components); + } + + std::string NamespacedEnumVariant(const EnumDef &e, const EnumVal &v) const { + return NamespacedString(e.defined_namespace, EnumVariant(e, v)); + } + + std::string NamespacedType(const Definition &def) const { + return NamespacedString(def.defined_namespace, Type(def.name)); + } + + std::string NamespacedObjectType(const Definition &def) const { + return NamespacedString(def.defined_namespace, ObjectType(def.name)); + } + + std::string Directories(const struct Namespace &ns, + SkipDir skips = SkipDir::None, + Case input_case = Case::kUpperCamel) const { + return Directories(ns.components, skips, input_case); + } + + // Legacy fields do not really follow the usual config and should be + // considered for deprecation. + + std::string LegacyRustNativeVariant(const EnumVal &v) const { + return ConvertCase(EscapeKeyword(v.name), Case::kUpperCamel); + } + + std::string LegacyRustFieldOffsetName(const FieldDef &field) const { + return "VT_" + ConvertCase(EscapeKeyword(field.name), Case::kAllUpper); + } + std::string LegacyRustUnionTypeOffsetName(const FieldDef &field) const { + return "VT_" + ConvertCase(EscapeKeyword(field.name + "_type"), Case::kAllUpper); + } + + + std::string LegacySwiftVariant(const EnumVal &ev) const { + auto name = ev.name; + if (isupper(name.front())) { + std::transform(name.begin(), name.end(), name.begin(), CharToLower); + } + return EscapeKeyword(ConvertCase(name, Case::kLowerCamel)); + } + + // Also used by Kotlin, lol. + std::string LegacyJavaMethod2(const std::string &prefix, const StructDef &sd, + const std::string &suffix) const { + return prefix + sd.name + suffix; + } + + std::string LegacyKotlinVariant(EnumVal &ev) const { + // Namer assumes the input case is snake case which is wrong... + return ConvertCase(EscapeKeyword(ev.name), Case::kLowerCamel); + } + // Kotlin methods escapes keywords after case conversion but before + // prefixing and suffixing. + std::string LegacyKotlinMethod(const std::string &prefix, const FieldDef &d, + const std::string &suffix) const { + return prefix + ConvertCase(EscapeKeyword(d.name), Case::kUpperCamel) + + suffix; + } + std::string LegacyKotlinMethod(const std::string &prefix, const StructDef &d, + const std::string &suffix) const { + return prefix + ConvertCase(EscapeKeyword(d.name), Case::kUpperCamel) + + suffix; + } + + // This is a mix of snake case and keep casing, when Ts should be using + // lower camel case. + std::string LegacyTsMutateMethod(const FieldDef& d) { + return "mutate_" + d.name; + } + + std::string LegacyRustUnionTypeMethod(const FieldDef &d) { + // assert d is a union + // d should convert case but not escape keywords due to historical reasons + return ConvertCase(d.name, config_.fields, Case::kLowerCamel) + "_type"; + } + + private: + std::string NamespacedString(const struct Namespace *ns, + const std::string &str) const { + std::string ret; + if (ns != nullptr) { ret += Namespace(ns->components); } + if (!ret.empty()) ret += config_.namespace_seperator; + return ret + str; + } +}; + +// This is a temporary helper function for code generators to call until all +// flag-overriding logic into flatc.cpp +inline Namer::Config WithFlagOptions(const Namer::Config &input, + const IDLOptions &opts, + const std::string &path) { + Namer::Config result = input; + result.object_prefix = opts.object_prefix; + result.object_suffix = opts.object_suffix; + result.output_path = path; + result.filename_suffix = opts.filename_suffix; + return result; +} + +} // namespace flatbuffers + +#endif // FLATBUFFERS_INCLUDE_CODEGEN_IDL_NAMER_H_ diff --git a/include/codegen/namer.h b/include/codegen/namer.h new file mode 100644 index 00000000000..e8b4286307b --- /dev/null +++ b/include/codegen/namer.h @@ -0,0 +1,270 @@ +#ifndef FLATBUFFERS_INCLUDE_CODEGEN_NAMER_H_ +#define FLATBUFFERS_INCLUDE_CODEGEN_NAMER_H_ + +#include "flatbuffers/util.h" + +namespace flatbuffers { + +// Options for Namer::File. +enum class SkipFile { + None = 0, + Suffix = 1, + Extension = 2, + SuffixAndExtension = 3, +}; +inline SkipFile operator&(SkipFile a, SkipFile b) { + return static_cast(static_cast(a) & static_cast(b)); +} +// Options for Namer::Directories +enum class SkipDir { + None = 0, + // Skip prefixing the -o $output_path. + OutputPath = 1, + // Skip trailing path seperator. + TrailingPathSeperator = 2, + OutputPathAndTrailingPathSeparator = 3, +}; +inline SkipDir operator&(SkipDir a, SkipDir b) { + return static_cast(static_cast(a) & static_cast(b)); +} + +// `Namer` applies style configuration to symbols in generated code. It manages +// casing, escapes keywords, and object API naming. +// TODO: Refactor all code generators to use this. +class Namer { + public: + struct Config { + // Symbols in code. + + // Case style for flatbuffers-defined types. + // e.g. `class TableA {}` + Case types; + // Case style for flatbuffers-defined constants. + // e.g. `uint64_t ENUM_A_MAX`; + Case constants; + // Case style for flatbuffers-defined methods. + // e.g. `class TableA { int field_a(); }` + Case methods; + // Case style for flatbuffers-defined functions. + // e.g. `TableA* get_table_a_root()`; + Case functions; + // Case style for flatbuffers-defined fields. + // e.g. `struct Struct { int my_field; }` + Case fields; + // Case style for flatbuffers-defined variables. + // e.g. `int my_variable = 2` + Case variables; + // Case style for flatbuffers-defined variants. + // e.g. `enum class Enum { MyVariant, }` + Case variants; + // Seperator for qualified enum names. + // e.g. `Enum::MyVariant` uses `::`. + std::string enum_variant_seperator; + + // Configures, when formatting code, whether symbols are checked against + // keywords and escaped before or after case conversion. It does not make + // sense to do so before, but its legacy behavior. :shrug: + // TODO(caspern): Deprecate. + enum class Escape { + BeforeConvertingCase, + AfterConvertingCase, + }; + Escape escape_keywords; + + // Namespaces + + // e.g. `namespace my_namespace {}` + Case namespaces; + // The seperator between namespaces in a namespace path. + std::string namespace_seperator; + + // Object API. + // Native versions flatbuffers types have this prefix. + // e.g. "" (it's usually empty string) + std::string object_prefix; + // Native versions flatbuffers types have this suffix. + // e.g. "T" + std::string object_suffix; + + // Keywords. + // Prefix used to escape keywords. It is usually empty string. + std::string keyword_prefix; + // Suffix used to escape keywords. It is usually "_". + std::string keyword_suffix; + + // Files. + + // Case style for filenames. e.g. `foo_bar_generated.rs` + Case filenames; + // Case style for directories, e.g. `output_files/foo_bar/baz/` + Case directories; + // The directory within which we will generate files. + std::string output_path; + // Suffix for generated file names, e.g. "_generated". + std::string filename_suffix; + // Extension for generated files, e.g. ".cpp" or ".rs". + std::string filename_extension; + }; + Namer(Config config, std::set keywords) + : config_(config), keywords_(std::move(keywords)) {} + + virtual ~Namer() {} + + template std::string Method(const T &s) const { + return Method(s.name); + } + + virtual std::string Method(const std::string &pre, + const std::string &mid, + const std::string &suf) const { + return Format(pre + "_" + mid + "_" + suf, config_.methods); + } + virtual std::string Method(const std::string &pre, + const std::string &suf) const { + return Format(pre + "_" + suf, config_.methods); + } + virtual std::string Method(const std::string &s) const { + return Format(s, config_.methods); + } + + virtual std::string Constant(const std::string &s) const { + return Format(s, config_.constants); + } + + virtual std::string Function(const std::string &s) const { + return Format(s, config_.functions); + } + + virtual std::string Variable(const std::string &s) const { + return Format(s, config_.variables); + } + + template + std::string Variable(const std::string &p, const T &s) const { + return Format(p + "_" + s.name, config_.variables); + } + virtual std::string Variable(const std::string &p, + const std::string &s) const { + return Format(p + "_" + s, config_.variables); + } + + virtual std::string Namespace(const std::string &s) const { + return Format(s, config_.namespaces); + } + + virtual std::string Namespace(const std::vector &ns) const { + std::string result; + for (auto it = ns.begin(); it != ns.end(); it++) { + if (it != ns.begin()) result += config_.namespace_seperator; + result += Namespace(*it); + } + return result; + } + + virtual std::string NamespacedType(const std::vector &ns, + const std::string &s) const { + return (ns.empty() ? "" : (Namespace(ns) + config_.namespace_seperator)) + + Type(s); + } + + // Returns `filename` with the right casing, suffix, and extension. + virtual std::string File(const std::string &filename, + SkipFile skips = SkipFile::None) const { + const bool skip_suffix = (skips & SkipFile::Suffix) != SkipFile::None; + const bool skip_ext = (skips & SkipFile::Extension) != SkipFile::None; + return ConvertCase(filename, config_.filenames, Case::kUpperCamel) + + (skip_suffix ? "" : config_.filename_suffix) + + (skip_ext ? "" : config_.filename_extension); + } + template + std::string File(const T &f, SkipFile skips = SkipFile::None) const { + return File(f.name, skips); + } + + // Formats `directories` prefixed with the output_path and joined with the + // right seperator. Output path prefixing and the trailing separator may be + // skiped using `skips`. + // Callers may want to use `EnsureDirExists` with the result. + // input_case is used to tell how to modify namespace. e.g. kUpperCamel will + // add a underscode between case changes, so MyGame turns into My_Game + // (depending also on the output_case). + virtual std::string Directories(const std::vector &directories, + SkipDir skips = SkipDir::None, + Case input_case = Case::kUpperCamel) const { + const bool skip_output_path = + (skips & SkipDir::OutputPath) != SkipDir::None; + const bool skip_trailing_seperator = + (skips & SkipDir::TrailingPathSeperator) != SkipDir::None; + std::string result = skip_output_path ? "" : config_.output_path; + for (auto d = directories.begin(); d != directories.end(); d++) { + result += ConvertCase(*d, config_.directories, input_case); + result.push_back(kPathSeparator); + } + if (skip_trailing_seperator && !result.empty()) result.pop_back(); + return result; + } + + virtual std::string EscapeKeyword(const std::string &name) const { + if (keywords_.find(name) == keywords_.end()) { + return name; + } else { + return config_.keyword_prefix + name + config_.keyword_suffix; + } + } + + virtual std::string Type(const std::string &s) const { + return Format(s, config_.types); + } + virtual std::string Type(const std::string &t, const std::string &s) const { + return Format(t + "_" + s, config_.types); + } + + virtual std::string ObjectType(const std::string &s) const { + return config_.object_prefix + Type(s) + config_.object_suffix; + } + + virtual std::string Field(const std::string &s) const { + return Format(s, config_.fields); + } + + virtual std::string Variant(const std::string &s) const { + return Format(s, config_.variants); + } + + virtual std::string Format(const std::string &s, Case casing) const { + if (config_.escape_keywords == Config::Escape::BeforeConvertingCase) { + return ConvertCase(EscapeKeyword(s), casing, Case::kLowerCamel); + } else { + return EscapeKeyword(ConvertCase(s, casing, Case::kLowerCamel)); + } + } + + // Denamespaces a string (e.g. The.Quick.Brown.Fox) by returning the last part + // after the `delimiter` (Fox) and placing the rest in `namespace_prefix` + // (The.Quick.Brown). + virtual std::string Denamespace(const std::string &s, + std::string &namespace_prefix, + const char delimiter = '.') const { + const size_t pos = s.find_last_of(delimiter); + if (pos == std::string::npos) { + namespace_prefix = ""; + return s; + } + namespace_prefix = s.substr(0, pos); + return s.substr(pos + 1); + } + + // Same as above, but disregards the prefix. + virtual std::string Denamespace(const std::string &s, + const char delimiter = '.') const { + std::string prefix; + return Denamespace(s, prefix, delimiter); + } + + const Config config_; + const std::set keywords_; +}; + +} // namespace flatbuffers + +#endif // FLATBUFFERS_INCLUDE_CODEGEN_NAMER_H_ diff --git a/include/codegen/python.h b/include/codegen/python.h index 7d57c4f37da..778eacfc4e5 100644 --- a/include/codegen/python.h +++ b/include/codegen/python.h @@ -6,8 +6,56 @@ #include #include +#include "codegen/namer.h" + namespace flatbuffers { namespace python { +static const Namer::Config kConfig = { + /*types=*/Case::kKeep, + /*constants=*/Case::kScreamingSnake, + /*methods=*/Case::kUpperCamel, + /*functions=*/Case::kUpperCamel, + /*fields=*/Case::kLowerCamel, + /*variable=*/Case::kLowerCamel, + /*variants=*/Case::kKeep, + /*enum_variant_seperator=*/".", + /*escape_keywords=*/Namer::Config::Escape::AfterConvertingCase, + /*namespaces=*/Case::kKeep, // Packages in python. + /*namespace_seperator=*/".", + /*object_prefix=*/"", + /*object_suffix=*/"T", + /*keyword_prefix=*/"", + /*keyword_suffix=*/"_", + /*filenames=*/Case::kKeep, + /*directories=*/Case::kKeep, + /*output_path=*/"", + /*filename_suffix=*/"", + /*filename_extension=*/".py", +}; + +static const Namer::Config kStubConfig = { + /*types=*/Case::kKeep, + /*constants=*/Case::kScreamingSnake, + /*methods=*/Case::kUpperCamel, + /*functions=*/Case::kUpperCamel, + /*fields=*/Case::kLowerCamel, + /*variables=*/Case::kLowerCamel, + /*variants=*/Case::kKeep, + /*enum_variant_seperator=*/".", + /*escape_keywords=*/Namer::Config::Escape::AfterConvertingCase, + /*namespaces=*/Case::kKeep, // Packages in python. + /*namespace_seperator=*/".", + /*object_prefix=*/"", + /*object_suffix=*/"T", + /*keyword_prefix=*/"", + /*keyword_suffix=*/"_", + /*filenames=*/Case::kKeep, + /*directories=*/Case::kKeep, + /*output_path=*/"", + /*filename_suffix=*/"", + /*filename_extension=*/".pyi", +}; + // `Version` represent a Python version. // // The zero value (i.e. `Version{}`) represents both Python2 and Python3. diff --git a/include/flatbuffers/idl.h b/include/flatbuffers/idl.h index 7306b0b272a..c84999a5485 100644 --- a/include/flatbuffers/idl.h +++ b/include/flatbuffers/idl.h @@ -784,6 +784,9 @@ struct IDLOptions { std::string grpc_search_path; std::vector grpc_additional_headers; + /******************************* Python gRPC ********************************/ + bool grpc_python_typed_handlers; + IDLOptions() : gen_jvmstatic(false), use_flexbuffers(false), @@ -857,7 +860,8 @@ struct IDLOptions { set_empty_strings_to_null(true), set_empty_vectors_to_null(true), grpc_filename_suffix(".fb"), - grpc_use_system_headers(true) {} + grpc_use_system_headers(true), + grpc_python_typed_handlers(false) {} }; // This encapsulates where the parser is in the current source file. diff --git a/src/BUILD.bazel b/src/BUILD.bazel index c2b1f741929..4b9fb7a8156 100644 --- a/src/BUILD.bazel +++ b/src/BUILD.bazel @@ -89,6 +89,7 @@ cc_library( visibility = ["//:__pkg__"], deps = [ ":flatbuffers", + "//include/codegen:namer", ], ) @@ -155,6 +156,7 @@ cc_library( "//grpc/src/compiler:python_generator", "//grpc/src/compiler:swift_generator", "//grpc/src/compiler:ts_generator", + "//include/codegen:namer", "//include/codegen:python", ], ) diff --git a/src/flatc.cpp b/src/flatc.cpp index 00d1236998f..1aee9a62c93 100644 --- a/src/flatc.cpp +++ b/src/flatc.cpp @@ -268,6 +268,8 @@ const static FlatCOption flatc_options[] = { { "", "grpc-use-system-headers", "", "Use <> for headers included from the generated code." }, { "", "grpc-search-path", "PATH", "Prefix to any gRPC includes." }, + { "", "grpc-python-typed-handlers", "", + "The handlers will use the generated classes rather than raw bytes." }, }; auto cmp = [](FlatCOption a, FlatCOption b) { return a.long_opt < b.long_opt; }; @@ -720,6 +722,12 @@ FlatCOptions FlatCompiler::ParseFromCommandLineArguments(int argc, } else if (arg == "--no-grpc-use-system-headers" || arg == "--grpc-use-system-headers=false") { opts.grpc_use_system_headers = false; + } else if (arg == "--grpc-python-typed-handlers" || + arg == "--grpc-python-typed-handlers=true") { + opts.grpc_python_typed_handlers = true; + } else if (arg == "--no-grpc-python-typed-handlers" || + arg == "--grpc-python-typed-handlers=false") { + opts.grpc_python_typed_handlers = false; } else { if (arg == "--proto") { opts.proto_mode = true; } diff --git a/src/idl_gen_grpc.cpp b/src/idl_gen_grpc.cpp index e7c3753effc..4764e25a450 100644 --- a/src/idl_gen_grpc.cpp +++ b/src/idl_gen_grpc.cpp @@ -435,47 +435,8 @@ bool GenerateJavaGRPC(const Parser &parser, const std::string &path, return JavaGRPCGenerator(parser, path, file_name).generate(); } -class PythonGRPCGenerator : public flatbuffers::BaseGenerator { - private: - CodeWriter code_; - - public: - PythonGRPCGenerator(const Parser &parser, const std::string &filename) - : BaseGenerator(parser, "", filename, "", "" /*Unused*/, "swift") {} - - bool generate() { - code_.Clear(); - code_ += - "# Generated by the gRPC Python protocol compiler plugin. " - "DO NOT EDIT!\n"; - code_ += "import grpc\n"; - - FlatBufFile file(parser_, file_name_, FlatBufFile::kLanguagePython); - - for (int i = 0; i < file.service_count(); i++) { - auto service = file.service(i); - code_ += grpc_python_generator::Generate(&file, service.get()); - } - const auto final_code = code_.ToString(); - const auto filename = GenerateFileName(); - return SaveFile(filename.c_str(), final_code, false); - } - - std::string GenerateFileName() { - std::string namespace_dir; - auto &namespaces = parser_.namespaces_.back()->components; - for (auto it = namespaces.begin(); it != namespaces.end(); ++it) { - if (it != namespaces.begin()) namespace_dir += kPathSeparator; - namespace_dir += *it; - } - std::string grpc_py_filename = namespace_dir; - if (!namespace_dir.empty()) grpc_py_filename += kPathSeparator; - return grpc_py_filename + file_name_ + "_grpc_fb.py"; - } -}; - -bool GeneratePythonGRPC(const Parser &parser, const std::string & /*path*/, - const std::string &file_name) { +bool GeneratePythonGRPC(const Parser &parser, const std::string &path, + const std::string & /*file_name*/) { int nservices = 0; for (auto it = parser.services_.vec.begin(); it != parser.services_.vec.end(); ++it) { @@ -483,7 +444,16 @@ bool GeneratePythonGRPC(const Parser &parser, const std::string & /*path*/, } if (!nservices) return true; - return PythonGRPCGenerator(parser, file_name).generate(); + flatbuffers::python::Version version{parser.opts.python_version}; + if (!version.IsValid()) return false; + + if (!flatbuffers::python::grpc::Generate(parser, path, version)) { + return false; + } + if (parser.opts.python_typing) { + return flatbuffers::python::grpc::GenerateStub(parser, path, version); + } + return true; } class SwiftGRPCGenerator : public flatbuffers::BaseGenerator { diff --git a/src/idl_gen_python.cpp b/src/idl_gen_python.cpp index 0041df4c19e..1c814cd53cf 100644 --- a/src/idl_gen_python.cpp +++ b/src/idl_gen_python.cpp @@ -29,12 +29,12 @@ #include #include +#include "codegen/idl_namer.h" #include "codegen/python.h" #include "flatbuffers/code_generators.h" #include "flatbuffers/flatbuffers.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" -#include "idl_namer.h" namespace flatbuffers { namespace python { @@ -44,52 +44,6 @@ namespace { typedef std::pair ImportMapEntry; typedef std::set ImportMap; -static Namer::Config PythonDefaultConfig() { - return { /*types=*/Case::kKeep, - /*constants=*/Case::kScreamingSnake, - /*methods=*/Case::kUpperCamel, - /*functions=*/Case::kUpperCamel, - /*fields=*/Case::kLowerCamel, - /*variable=*/Case::kLowerCamel, - /*variants=*/Case::kKeep, - /*enum_variant_seperator=*/".", - /*escape_keywords=*/Namer::Config::Escape::AfterConvertingCase, - /*namespaces=*/Case::kKeep, // Packages in python. - /*namespace_seperator=*/".", - /*object_prefix=*/"", - /*object_suffix=*/"T", - /*keyword_prefix=*/"", - /*keyword_suffix=*/"_", - /*filenames=*/Case::kKeep, - /*directories=*/Case::kKeep, - /*output_path=*/"", - /*filename_suffix=*/"", - /*filename_extension=*/".py" }; -} - -static Namer::Config kStubConfig = { - /*types=*/Case::kKeep, - /*constants=*/Case::kScreamingSnake, - /*methods=*/Case::kUpperCamel, - /*functions=*/Case::kUpperCamel, - /*fields=*/Case::kLowerCamel, - /*variables=*/Case::kLowerCamel, - /*variants=*/Case::kKeep, - /*enum_variant_seperator=*/".", - /*escape_keywords=*/Namer::Config::Escape::AfterConvertingCase, - /*namespaces=*/Case::kKeep, // Packages in python. - /*namespace_seperator=*/".", - /*object_prefix=*/"", - /*object_suffix=*/"T", - /*keyword_prefix=*/"", - /*keyword_suffix=*/"_", - /*filenames=*/Case::kKeep, - /*directories=*/Case::kKeep, - /*output_path=*/"", - /*filename_suffix=*/"", - /*filename_extension=*/".pyi", -}; - // Hardcode spaces per indentation. static const CommentConfig def_comment = { nullptr, "#", nullptr }; static const std::string Indent = " "; @@ -662,7 +616,7 @@ class PythonGenerator : public BaseGenerator { : BaseGenerator(parser, path, file_name, "" /* not used */, "" /* not used */, "py"), float_const_gen_("float('nan')", "float('inf')", "float('-inf')"), - namer_(WithFlagOptions(PythonDefaultConfig(), parser.opts, path), + namer_(WithFlagOptions(kConfig, parser.opts, path), Keywords(version)) {} // Most field accessors need to retrieve and test the field offset first, diff --git a/src/idl_namer.h b/src/idl_namer.h index 9a7fdb8e368..dc829e78642 100644 --- a/src/idl_namer.h +++ b/src/idl_namer.h @@ -1,179 +1,6 @@ -#ifndef FLATBUFFERS_IDL_NAMER -#define FLATBUFFERS_IDL_NAMER +#ifndef FLATBUFFERS_IDL_NAMER_H_ +#define FLATBUFFERS_IDL_NAMER_H_ -#include "flatbuffers/idl.h" -#include "namer.h" +#include "codegen/idl_namer.h" -namespace flatbuffers { - -// Provides Namer capabilities to types defined in the flatbuffers IDL. -class IdlNamer : public Namer { - public: - explicit IdlNamer(Config config, std::set keywords) - : Namer(config, std::move(keywords)) {} - - using Namer::Constant; - using Namer::Directories; - using Namer::Field; - using Namer::File; - using Namer::Function; - using Namer::Method; - using Namer::Namespace; - using Namer::NamespacedType; - using Namer::ObjectType; - using Namer::Type; - using Namer::Variable; - using Namer::Variant; - - std::string Constant(const FieldDef &d) const { return Constant(d.name); } - - // Types are always structs or enums so we can only expose these two - // overloads. - std::string Type(const StructDef &d) const { return Type(d.name); } - std::string Type(const EnumDef &d) const { return Type(d.name); } - - std::string Function(const Definition &s) const { return Function(s.name); } - std::string Function(const std::string& prefix, const Definition &s) const { - return Function(prefix + s.name); - } - - std::string Field(const FieldDef &s) const { return Field(s.name); } - std::string Field(const FieldDef &d, const std::string &s) const { - return Field(d.name + "_" + s); - } - - std::string Variable(const FieldDef &s) const { return Variable(s.name); } - - std::string Variable(const StructDef &s) const { return Variable(s.name); } - - std::string Variant(const EnumVal &s) const { return Variant(s.name); } - - std::string EnumVariant(const EnumDef &e, const EnumVal &v) const { - return Type(e) + config_.enum_variant_seperator + Variant(v); - } - - std::string ObjectType(const StructDef &d) const { - return ObjectType(d.name); - } - std::string ObjectType(const EnumDef &d) const { return ObjectType(d.name); } - - std::string Method(const FieldDef &d, const std::string &suffix) const { - return Method(d.name, suffix); - } - std::string Method(const std::string &prefix, const StructDef &d) const { - return Method(prefix, d.name); - } - std::string Method(const std::string &prefix, const FieldDef &d) const { - return Method(prefix, d.name); - } - std::string Method(const std::string &prefix, const FieldDef &d, - const std::string &suffix) const { - return Method(prefix, d.name, suffix); - } - - std::string Namespace(const struct Namespace &ns) const { - return Namespace(ns.components); - } - - std::string NamespacedEnumVariant(const EnumDef &e, const EnumVal &v) const { - return NamespacedString(e.defined_namespace, EnumVariant(e, v)); - } - - std::string NamespacedType(const Definition &def) const { - return NamespacedString(def.defined_namespace, Type(def.name)); - } - - std::string NamespacedObjectType(const Definition &def) const { - return NamespacedString(def.defined_namespace, ObjectType(def.name)); - } - - std::string Directories(const struct Namespace &ns, - SkipDir skips = SkipDir::None, - Case input_case = Case::kUpperCamel) const { - return Directories(ns.components, skips, input_case); - } - - // Legacy fields do not really follow the usual config and should be - // considered for deprecation. - - std::string LegacyRustNativeVariant(const EnumVal &v) const { - return ConvertCase(EscapeKeyword(v.name), Case::kUpperCamel); - } - - std::string LegacyRustFieldOffsetName(const FieldDef &field) const { - return "VT_" + ConvertCase(EscapeKeyword(field.name), Case::kAllUpper); - } - std::string LegacyRustUnionTypeOffsetName(const FieldDef &field) const { - return "VT_" + ConvertCase(EscapeKeyword(field.name + "_type"), Case::kAllUpper); - } - - - std::string LegacySwiftVariant(const EnumVal &ev) const { - auto name = ev.name; - if (isupper(name.front())) { - std::transform(name.begin(), name.end(), name.begin(), CharToLower); - } - return EscapeKeyword(ConvertCase(name, Case::kLowerCamel)); - } - - // Also used by Kotlin, lol. - std::string LegacyJavaMethod2(const std::string &prefix, const StructDef &sd, - const std::string &suffix) const { - return prefix + sd.name + suffix; - } - - std::string LegacyKotlinVariant(EnumVal &ev) const { - // Namer assumes the input case is snake case which is wrong... - return ConvertCase(EscapeKeyword(ev.name), Case::kLowerCamel); - } - // Kotlin methods escapes keywords after case conversion but before - // prefixing and suffixing. - std::string LegacyKotlinMethod(const std::string &prefix, const FieldDef &d, - const std::string &suffix) const { - return prefix + ConvertCase(EscapeKeyword(d.name), Case::kUpperCamel) + - suffix; - } - std::string LegacyKotlinMethod(const std::string &prefix, const StructDef &d, - const std::string &suffix) const { - return prefix + ConvertCase(EscapeKeyword(d.name), Case::kUpperCamel) + - suffix; - } - - // This is a mix of snake case and keep casing, when Ts should be using - // lower camel case. - std::string LegacyTsMutateMethod(const FieldDef& d) { - return "mutate_" + d.name; - } - - std::string LegacyRustUnionTypeMethod(const FieldDef &d) { - // assert d is a union - // d should convert case but not escape keywords due to historical reasons - return ConvertCase(d.name, config_.fields, Case::kLowerCamel) + "_type"; - } - - private: - std::string NamespacedString(const struct Namespace *ns, - const std::string &str) const { - std::string ret; - if (ns != nullptr) { ret += Namespace(ns->components); } - if (!ret.empty()) ret += config_.namespace_seperator; - return ret + str; - } -}; - -// This is a temporary helper function for code generators to call until all -// flag-overriding logic into flatc.cpp -inline Namer::Config WithFlagOptions(const Namer::Config &input, - const IDLOptions &opts, - const std::string &path) { - Namer::Config result = input; - result.object_prefix = opts.object_prefix; - result.object_suffix = opts.object_suffix; - result.output_path = path; - result.filename_suffix = opts.filename_suffix; - return result; -} - -} // namespace flatbuffers - -#endif // FLATBUFFERS_IDL_NAMER +#endif // FLATBUFFERS_IDL_NAMER_H_ diff --git a/src/namer.h b/src/namer.h index 097d4490bcd..1bb3a087f89 100644 --- a/src/namer.h +++ b/src/namer.h @@ -1,270 +1,6 @@ -#ifndef FLATBUFFERS_NAMER -#define FLATBUFFERS_NAMER +#ifndef FLATBUFFERS_NAMER_H_ +#define FLATBUFFERS_NAMER_H_ -#include "flatbuffers/util.h" +#include "codegen/namer.h" -namespace flatbuffers { - -// Options for Namer::File. -enum class SkipFile { - None = 0, - Suffix = 1, - Extension = 2, - SuffixAndExtension = 3, -}; -inline SkipFile operator&(SkipFile a, SkipFile b) { - return static_cast(static_cast(a) & static_cast(b)); -} -// Options for Namer::Directories -enum class SkipDir { - None = 0, - // Skip prefixing the -o $output_path. - OutputPath = 1, - // Skip trailing path seperator. - TrailingPathSeperator = 2, - OutputPathAndTrailingPathSeparator = 3, -}; -inline SkipDir operator&(SkipDir a, SkipDir b) { - return static_cast(static_cast(a) & static_cast(b)); -} - -// `Namer` applies style configuration to symbols in generated code. It manages -// casing, escapes keywords, and object API naming. -// TODO: Refactor all code generators to use this. -class Namer { - public: - struct Config { - // Symbols in code. - - // Case style for flatbuffers-defined types. - // e.g. `class TableA {}` - Case types; - // Case style for flatbuffers-defined constants. - // e.g. `uint64_t ENUM_A_MAX`; - Case constants; - // Case style for flatbuffers-defined methods. - // e.g. `class TableA { int field_a(); }` - Case methods; - // Case style for flatbuffers-defined functions. - // e.g. `TableA* get_table_a_root()`; - Case functions; - // Case style for flatbuffers-defined fields. - // e.g. `struct Struct { int my_field; }` - Case fields; - // Case style for flatbuffers-defined variables. - // e.g. `int my_variable = 2` - Case variables; - // Case style for flatbuffers-defined variants. - // e.g. `enum class Enum { MyVariant, }` - Case variants; - // Seperator for qualified enum names. - // e.g. `Enum::MyVariant` uses `::`. - std::string enum_variant_seperator; - - // Configures, when formatting code, whether symbols are checked against - // keywords and escaped before or after case conversion. It does not make - // sense to do so before, but its legacy behavior. :shrug: - // TODO(caspern): Deprecate. - enum class Escape { - BeforeConvertingCase, - AfterConvertingCase, - }; - Escape escape_keywords; - - // Namespaces - - // e.g. `namespace my_namespace {}` - Case namespaces; - // The seperator between namespaces in a namespace path. - std::string namespace_seperator; - - // Object API. - // Native versions flatbuffers types have this prefix. - // e.g. "" (it's usually empty string) - std::string object_prefix; - // Native versions flatbuffers types have this suffix. - // e.g. "T" - std::string object_suffix; - - // Keywords. - // Prefix used to escape keywords. It is usually empty string. - std::string keyword_prefix; - // Suffix used to escape keywords. It is usually "_". - std::string keyword_suffix; - - // Files. - - // Case style for filenames. e.g. `foo_bar_generated.rs` - Case filenames; - // Case style for directories, e.g. `output_files/foo_bar/baz/` - Case directories; - // The directory within which we will generate files. - std::string output_path; - // Suffix for generated file names, e.g. "_generated". - std::string filename_suffix; - // Extension for generated files, e.g. ".cpp" or ".rs". - std::string filename_extension; - }; - Namer(Config config, std::set keywords) - : config_(config), keywords_(std::move(keywords)) {} - - virtual ~Namer() {} - - template std::string Method(const T &s) const { - return Method(s.name); - } - - virtual std::string Method(const std::string &pre, - const std::string &mid, - const std::string &suf) const { - return Format(pre + "_" + mid + "_" + suf, config_.methods); - } - virtual std::string Method(const std::string &pre, - const std::string &suf) const { - return Format(pre + "_" + suf, config_.methods); - } - virtual std::string Method(const std::string &s) const { - return Format(s, config_.methods); - } - - virtual std::string Constant(const std::string &s) const { - return Format(s, config_.constants); - } - - virtual std::string Function(const std::string &s) const { - return Format(s, config_.functions); - } - - virtual std::string Variable(const std::string &s) const { - return Format(s, config_.variables); - } - - template - std::string Variable(const std::string &p, const T &s) const { - return Format(p + "_" + s.name, config_.variables); - } - virtual std::string Variable(const std::string &p, - const std::string &s) const { - return Format(p + "_" + s, config_.variables); - } - - virtual std::string Namespace(const std::string &s) const { - return Format(s, config_.namespaces); - } - - virtual std::string Namespace(const std::vector &ns) const { - std::string result; - for (auto it = ns.begin(); it != ns.end(); it++) { - if (it != ns.begin()) result += config_.namespace_seperator; - result += Namespace(*it); - } - return result; - } - - virtual std::string NamespacedType(const std::vector &ns, - const std::string &s) const { - return (ns.empty() ? "" : (Namespace(ns) + config_.namespace_seperator)) + - Type(s); - } - - // Returns `filename` with the right casing, suffix, and extension. - virtual std::string File(const std::string &filename, - SkipFile skips = SkipFile::None) const { - const bool skip_suffix = (skips & SkipFile::Suffix) != SkipFile::None; - const bool skip_ext = (skips & SkipFile::Extension) != SkipFile::None; - return ConvertCase(filename, config_.filenames, Case::kUpperCamel) + - (skip_suffix ? "" : config_.filename_suffix) + - (skip_ext ? "" : config_.filename_extension); - } - template - std::string File(const T &f, SkipFile skips = SkipFile::None) const { - return File(f.name, skips); - } - - // Formats `directories` prefixed with the output_path and joined with the - // right seperator. Output path prefixing and the trailing separator may be - // skiped using `skips`. - // Callers may want to use `EnsureDirExists` with the result. - // input_case is used to tell how to modify namespace. e.g. kUpperCamel will - // add a underscode between case changes, so MyGame turns into My_Game - // (depending also on the output_case). - virtual std::string Directories(const std::vector &directories, - SkipDir skips = SkipDir::None, - Case input_case = Case::kUpperCamel) const { - const bool skip_output_path = - (skips & SkipDir::OutputPath) != SkipDir::None; - const bool skip_trailing_seperator = - (skips & SkipDir::TrailingPathSeperator) != SkipDir::None; - std::string result = skip_output_path ? "" : config_.output_path; - for (auto d = directories.begin(); d != directories.end(); d++) { - result += ConvertCase(*d, config_.directories, input_case); - result.push_back(kPathSeparator); - } - if (skip_trailing_seperator && !result.empty()) result.pop_back(); - return result; - } - - virtual std::string EscapeKeyword(const std::string &name) const { - if (keywords_.find(name) == keywords_.end()) { - return name; - } else { - return config_.keyword_prefix + name + config_.keyword_suffix; - } - } - - virtual std::string Type(const std::string &s) const { - return Format(s, config_.types); - } - virtual std::string Type(const std::string &t, const std::string &s) const { - return Format(t + "_" + s, config_.types); - } - - virtual std::string ObjectType(const std::string &s) const { - return config_.object_prefix + Type(s) + config_.object_suffix; - } - - virtual std::string Field(const std::string &s) const { - return Format(s, config_.fields); - } - - virtual std::string Variant(const std::string &s) const { - return Format(s, config_.variants); - } - - virtual std::string Format(const std::string &s, Case casing) const { - if (config_.escape_keywords == Config::Escape::BeforeConvertingCase) { - return ConvertCase(EscapeKeyword(s), casing, Case::kLowerCamel); - } else { - return EscapeKeyword(ConvertCase(s, casing, Case::kLowerCamel)); - } - } - - // Denamespaces a string (e.g. The.Quick.Brown.Fox) by returning the last part - // after the `delimiter` (Fox) and placing the rest in `namespace_prefix` - // (The.Quick.Brown). - virtual std::string Denamespace(const std::string &s, - std::string &namespace_prefix, - const char delimiter = '.') const { - const size_t pos = s.find_last_of(delimiter); - if (pos == std::string::npos) { - namespace_prefix = ""; - return s; - } - namespace_prefix = s.substr(0, pos); - return s.substr(pos + 1); - } - - // Same as above, but disregards the prefix. - virtual std::string Denamespace(const std::string &s, - const char delimiter = '.') const { - std::string prefix; - return Denamespace(s, prefix, delimiter); - } - - const Config config_; - const std::set keywords_; -}; - -} // namespace flatbuffers - -#endif // FLATBUFFERS_NAMER +#endif // FLATBUFFERS_NAMER_H_ diff --git a/tests/PythonTest.sh b/tests/PythonTest.sh index e7199b8770b..647f3daf158 100755 --- a/tests/PythonTest.sh +++ b/tests/PythonTest.sh @@ -27,6 +27,7 @@ ${test_dir}/../flatc -p -o ${gen_code_path} -I include_test monster_test.fbs --g ${test_dir}/../flatc -p -o ${gen_code_path} -I include_test monster_extra.fbs --gen-object-api --python-typing --gen-compare ${test_dir}/../flatc -p -o ${gen_code_path} -I include_test arrays_test.fbs --gen-object-api --python-typing ${test_dir}/../flatc -p -o ${gen_code_path} -I include_test nested_union_test.fbs --gen-object-api --python-typing +${test_dir}/../flatc -p -o ${gen_code_path} -I include_test service_test.fbs --grpc --grpc-python-typed-handlers --python-typing --no-python-gen-numpy --gen-onefile # Syntax: run_tests # diff --git a/tests/service_test.fbs b/tests/service_test.fbs new file mode 100644 index 00000000000..6a28f9f9030 --- /dev/null +++ b/tests/service_test.fbs @@ -0,0 +1,11 @@ +namespace example; + +table HelloRequest {} +table HelloResponse {} + +rpc_service HelloService { + Hello(HelloRequest):HelloResponse; + StreamClient(HelloRequest):HelloResponse (streaming: "client"); + StreamServer(HelloRequest):HelloResponse (streaming: "server"); + Stream(HelloRequest):HelloResponse (streaming: "bidi"); +} diff --git a/tests/service_test_generated.py b/tests/service_test_generated.py new file mode 100644 index 00000000000..1fa9b099a07 --- /dev/null +++ b/tests/service_test_generated.py @@ -0,0 +1,58 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: example + +import flatbuffers +from typing import Any +class HelloRequest(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAs(cls, buf, offset: int = 0): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = HelloRequest() + x.Init(buf, n + offset) + return x + + @classmethod + def GetRootAsHelloRequest(cls, buf, offset=0): + """This method is deprecated. Please switch to GetRootAs.""" + return cls.GetRootAs(buf, offset) + # HelloRequest + def Init(self, buf: bytes, pos: int): + self._tab = flatbuffers.table.Table(buf, pos) + +def HelloRequestStart(builder: flatbuffers.Builder): + builder.StartObject(0) + +def HelloRequestEnd(builder: flatbuffers.Builder) -> int: + return builder.EndObject() + + + +class HelloResponse(object): + __slots__ = ['_tab'] + + @classmethod + def GetRootAs(cls, buf, offset: int = 0): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = HelloResponse() + x.Init(buf, n + offset) + return x + + @classmethod + def GetRootAsHelloResponse(cls, buf, offset=0): + """This method is deprecated. Please switch to GetRootAs.""" + return cls.GetRootAs(buf, offset) + # HelloResponse + def Init(self, buf: bytes, pos: int): + self._tab = flatbuffers.table.Table(buf, pos) + +def HelloResponseStart(builder: flatbuffers.Builder): + builder.StartObject(0) + +def HelloResponseEnd(builder: flatbuffers.Builder) -> int: + return builder.EndObject() + + + diff --git a/tests/service_test_generated.pyi b/tests/service_test_generated.pyi new file mode 100644 index 00000000000..2189a94c693 --- /dev/null +++ b/tests/service_test_generated.pyi @@ -0,0 +1,26 @@ +from __future__ import annotations + +import flatbuffers + +import flatbuffers +import typing + +uoffset: typing.TypeAlias = flatbuffers.number_types.UOffsetTFlags.py_type + +class HelloRequest(object): + @classmethod + def GetRootAs(cls, buf: bytes, offset: int) -> HelloRequest: ... + @classmethod + def GetRootAsHelloRequest(cls, buf: bytes, offset: int) -> HelloRequest: ... + def Init(self, buf: bytes, pos: int) -> None: ... +def HelloRequestStart(builder: flatbuffers.Builder) -> None: ... +def HelloRequestEnd(builder: flatbuffers.Builder) -> uoffset: ... +class HelloResponse(object): + @classmethod + def GetRootAs(cls, buf: bytes, offset: int) -> HelloResponse: ... + @classmethod + def GetRootAsHelloResponse(cls, buf: bytes, offset: int) -> HelloResponse: ... + def Init(self, buf: bytes, pos: int) -> None: ... +def HelloResponseStart(builder: flatbuffers.Builder) -> None: ... +def HelloResponseEnd(builder: flatbuffers.Builder) -> uoffset: ... + diff --git a/tests/service_test_grpc.fb.py b/tests/service_test_grpc.fb.py new file mode 100644 index 00000000000..27284ea9486 --- /dev/null +++ b/tests/service_test_grpc.fb.py @@ -0,0 +1,97 @@ +# Generated by the gRPC FlatBuffers compiler. DO NOT EDIT! + +import flatbuffers +import grpc + +from service_test_generated import HelloRequest, HelloResponse + + +def _serialize_to_bytes(table): + buf = table._tab.Bytes + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, 0) + if table._tab.Pos != n: + raise ValueError('must be a top-level table') + return bytes(buf) + + +class HelloServiceStub(object): + '''Interface exported by the server.''' + + def __init__(self, channel): + '''Constructor. + + Args: + channel: A grpc.Channel. + ''' + + self.Hello = channel.unary_unary( + method='/example.HelloService/Hello', + request_serializer=_serialize_to_bytes, + response_deserializer=HelloResponse.GetRootAs) + + self.StreamClient = channel.stream_unary( + method='/example.HelloService/StreamClient', + request_serializer=_serialize_to_bytes, + response_deserializer=HelloResponse.GetRootAs) + + self.StreamServer = channel.unary_stream( + method='/example.HelloService/StreamServer', + request_serializer=_serialize_to_bytes, + response_deserializer=HelloResponse.GetRootAs) + + self.Stream = channel.stream_stream( + method='/example.HelloService/Stream', + request_serializer=_serialize_to_bytes, + response_deserializer=HelloResponse.GetRootAs) + + +class HelloServiceServicer(object): + '''Interface exported by the server.''' + + def Hello(self, request, context): + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def StreamClient(self, request_iterator, context): + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def StreamServer(self, request, context): + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Stream(self, request_iterator, context): + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_HelloServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'Hello': grpc.unary_unary_rpc_method_handler( + servicer.Hello, + request_deserializer=HelloRequest.GetRootAs, + response_serializer=_serialize_to_bytes), + 'StreamClient': grpc.stream_unary_rpc_method_handler( + servicer.StreamClient, + request_deserializer=HelloRequest.GetRootAs, + response_serializer=_serialize_to_bytes), + 'StreamServer': grpc.unary_stream_rpc_method_handler( + servicer.StreamServer, + request_deserializer=HelloRequest.GetRootAs, + response_serializer=_serialize_to_bytes), + 'Stream': grpc.stream_stream_rpc_method_handler( + servicer.Stream, + request_deserializer=HelloRequest.GetRootAs, + response_serializer=_serialize_to_bytes), + } + + generic_handler = grpc.method_handlers_generic_handler( + 'example.HelloService', rpc_method_handlers) + + server.add_generic_rpc_handlers((generic_handler,)) + + diff --git a/tests/service_test_grpc.fb.pyi b/tests/service_test_grpc.fb.pyi new file mode 100644 index 00000000000..d0f38aa4f3c --- /dev/null +++ b/tests/service_test_grpc.fb.pyi @@ -0,0 +1,27 @@ +# Generated by the gRPC FlatBuffers compiler. DO NOT EDIT! + +from __future__ import annotations + +import grpc +import typing + +from service_test_generated import HelloRequest, HelloResponse + + +class HelloServiceStub(object): + def __init__(self, channel: grpc.Channel) -> None: ... + def Hello(self, request: HelloRequest) -> HelloResponse: ... + def StreamClient(self, request_iterator: typing.Iterator[HelloRequest]) -> HelloResponse: ... + def StreamServer(self, request: HelloRequest) -> typing.Iterator[HelloResponse]: ... + def Stream(self, request_iterator: typing.Iterator[HelloRequest]) -> typing.Iterator[HelloResponse]: ... + + +class HelloServiceServicer(object): + def Hello(self, request: HelloRequest, context: grpc.ServicerContext) -> HelloResponse: ... + def StreamClient(self, request_iterator: typing.Iterator[HelloRequest], context: grpc.ServicerContext) -> HelloResponse: ... + def StreamServer(self, request: HelloRequest, context: grpc.ServicerContext) -> typing.Iterator[HelloResponse]: ... + def Stream(self, request_iterator: typing.Iterator[HelloRequest], context: grpc.ServicerContext) -> typing.Iterator[HelloResponse]: ... + + +def add_HelloServiceServicer_to_server(servicer: HelloServiceServicer, server: grpc.Server) -> None: ... +