From 90f7537f51fb4ba97644a61eef291639e3a807ce Mon Sep 17 00:00:00 2001 From: Horia Culea Date: Tue, 30 Apr 2024 15:22:10 +0200 Subject: [PATCH 01/12] Add experimental typeshare support for Python Thanks to the prior work of @adriangb: https://github.com/1Password/typeshare/pull/25 --- Cargo.lock | 23 ++ cli/src/config.rs | 7 + cli/src/main.rs | 10 +- core/Cargo.toml | 3 + core/src/language/mod.rs | 6 +- core/src/language/python.rs | 776 ++++++++++++++++++++++++++++++++++++ 6 files changed, 821 insertions(+), 4 deletions(-) create mode 100644 core/src/language/python.rs diff --git a/Cargo.lock b/Cargo.lock index 58cddc9f..5d4c84f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -139,6 +139,15 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "convert_case" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec182b0ca2f35d8fc196cf3404988fd8b8c739a4d270ff118a398feb0cbec1ca" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation-sys" version = "0.8.3" @@ -581,6 +590,12 @@ dependencies = [ "serde", ] +[[package]] +name = "topological-sort" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea68304e134ecd095ac6c3574494fc62b909f416c4fca77e440530221e549d3d" + [[package]] name = "typeshare" version = "1.0.2" @@ -618,6 +633,7 @@ name = "typeshare-core" version = "1.9.2" dependencies = [ "anyhow", + "convert_case", "expect-test", "itertools", "joinery", @@ -627,6 +643,7 @@ dependencies = [ "quote", "syn", "thiserror", + "topological-sort", ] [[package]] @@ -635,6 +652,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcc811dc4066ac62f84f11307873c4850cb653bfa9b1719cee2bd2204a4bc5dd" +[[package]] +name = "unicode-segmentation" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + [[package]] name = "walkdir" version = "2.3.2" diff --git a/cli/src/config.rs b/cli/src/config.rs index 9305a9b5..2cb64e03 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -9,6 +9,12 @@ use std::{ const DEFAULT_CONFIG_FILE_NAME: &str = "typeshare.toml"; +#[derive(Default, Serialize, Deserialize, PartialEq, Eq, Debug)] +#[serde(default)] +pub struct PythonParams { + pub type_mappings: HashMap, +} + #[derive(Default, Serialize, Deserialize, Debug, PartialEq, Eq)] #[serde(default)] pub struct KotlinParams { @@ -59,6 +65,7 @@ pub(crate) struct Config { pub typescript: TypeScriptParams, pub kotlin: KotlinParams, pub scala: ScalaParams, + pub python: PythonParams, #[cfg(feature = "go")] pub go: GoParams, } diff --git a/cli/src/main.rs b/cli/src/main.rs index 066b0c1d..de32d153 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -8,9 +8,9 @@ use ignore::types::TypesBuilder; use ignore::WalkBuilder; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::{fs, path::Path}; -use typeshare_core::language::GenericConstraints; #[cfg(feature = "go")] use typeshare_core::language::Go; +use typeshare_core::language::{GenericConstraints, Python}; use typeshare_core::{ language::{Kotlin, Language, Scala, SupportedLanguage, Swift, TypeScript}, parser::ParsedData, @@ -35,10 +35,10 @@ const ARG_OUTPUT_FILE: &str = "output-file"; const ARG_FOLLOW_LINKS: &str = "follow-links"; #[cfg(feature = "go")] -const AVAILABLE_LANGUAGES: [&str; 5] = ["kotlin", "scala", "swift", "typescript", "go"]; +const AVAILABLE_LANGUAGES: [&str; 6] = ["kotlin", "scala", "swift", "typescript", "go", "python"]; #[cfg(not(feature = "go"))] -const AVAILABLE_LANGUAGES: [&str; 4] = ["kotlin", "scala", "swift", "typescript"]; +const AVAILABLE_LANGUAGES: [&str; 5] = ["kotlin", "scala", "swift", "typescript", "python"]; fn build_command() -> Command<'static> { command!("typeshare") @@ -225,6 +225,10 @@ fn main() { type_mappings: config.typescript.type_mappings, ..Default::default() }), + Some(SupportedLanguage::Python) => Box::new(Python { + type_mappings: config.python.type_mappings, + ..Default::default() + }), #[cfg(feature = "go")] Some(SupportedLanguage::Go) => Box::new(Go { package: config.go.package, diff --git a/core/Cargo.toml b/core/Cargo.toml index 6dcddf02..892c5ffd 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -15,6 +15,9 @@ thiserror = "1.0.40" itertools = "0.10" lazy_format = "1.8" joinery = "2" +topological-sort = { version = "0.2.2"} +once_cell = { version = "1"} +convert_case = { version = "0.6.0"} [dev-dependencies] anyhow = "1" diff --git a/core/src/language/mod.rs b/core/src/language/mod.rs index c96ba5ac..37b523f3 100644 --- a/core/src/language/mod.rs +++ b/core/src/language/mod.rs @@ -9,6 +9,7 @@ use std::{collections::HashMap, fmt::Debug, io::Write, str::FromStr}; mod go; mod kotlin; +mod python; mod scala; mod swift; mod typescript; @@ -16,6 +17,7 @@ mod typescript; use crate::rust_types::{RustType, RustTypeFormatError, SpecialRustType}; pub use go::Go; pub use kotlin::Kotlin; +pub use python::Python; pub use scala::Scala; pub use swift::GenericConstraints; pub use swift::Swift; @@ -30,13 +32,14 @@ pub enum SupportedLanguage { Scala, Swift, TypeScript, + Python, } impl SupportedLanguage { /// Returns an iterator over all supported language variants. pub fn all_languages() -> impl Iterator { use SupportedLanguage::*; - [Go, Kotlin, Scala, Swift, TypeScript].into_iter() + [Go, Kotlin, Scala, Swift, TypeScript, Python].into_iter() } } @@ -50,6 +53,7 @@ impl FromStr for SupportedLanguage { "scala" => Ok(Self::Scala), "swift" => Ok(Self::Swift), "typescript" => Ok(Self::TypeScript), + "python" => Ok(Self::Python), _ => Err(ParseError::UnsupportedLanguage(s.into())), } } diff --git a/core/src/language/python.rs b/core/src/language/python.rs new file mode 100644 index 00000000..6554754b --- /dev/null +++ b/core/src/language/python.rs @@ -0,0 +1,776 @@ +use crate::rust_types::{RustType, RustTypeFormatError, SpecialRustType}; +use crate::{ + language::Language, + rust_types::{RustEnum, RustEnumVariant, RustField, RustStruct, RustTypeAlias}, +}; +use once_cell::sync::Lazy; +use std::cell::RefCell; +use std::collections::hash_map::Entry; +use std::collections::HashSet; +use std::hash::Hash; +use std::{collections::HashMap, io::Write}; + +use convert_case::{Case, Casing}; +use topological_sort::TopologicalSort; + +#[derive(Debug, Default)] +pub struct Module { + // HashMap + imports: HashMap>, + // HashMap> + // Used to lay out runtime references in the module + // such that it can be read top to bottom + globals: HashMap>, + type_variables: HashSet, +} + +#[derive(Debug)] +struct GenerationError(String); + +impl Module { + // Idempotently insert an import + fn add_import(&mut self, module: String, identifier: String) { + self.imports.entry(module).or_default().insert(identifier); + } + fn add_global(&mut self, identifier: String, deps: Vec) { + match self.globals.entry(identifier) { + Entry::Occupied(mut e) => e.get_mut().extend_from_slice(&deps), + Entry::Vacant(e) => { + e.insert(deps); + } + } + } + fn add_type_var(&mut self, name: String) { + self.add_import("typing".to_string(), "TypeVar".to_string()); + self.type_variables.insert(name); + } + fn get_type_vars(&mut self, n: usize) -> Vec { + let vars: Vec = (0..n) + .into_iter() + .map(|i| { + if i == 0 { + "T".to_string() + } else { + format!("T{}", i) + } + }) + .collect(); + vars.iter().for_each(|tv| self.add_type_var(tv.clone())); + vars + } + // Rust lets you declare type aliases before the struct they point to. + // But in Python we need the struct to come first. + // So we need to topologically sort the globals so that type aliases + // always come _after_ the struct/enum they point to. + fn topologically_sorted_globals(&self) -> Result, GenerationError> { + let mut ts: TopologicalSort = TopologicalSort::new(); + for (identifier, dependencies) in &self.globals { + for dependency in dependencies { + ts.add_dependency(dependency.clone(), identifier.clone()) + } + } + let mut res: Vec = Vec::new(); + loop { + let mut level = ts.pop_all(); + level.sort(); + res.extend_from_slice(&level); + if level.is_empty() { + if !ts.is_empty() { + return Err(GenerationError("Cyclical runtime dependency".to_string())); + } + break; + } + } + let existing: HashSet<&String> = HashSet::from_iter(res.iter()); + let mut missing: Vec = self + .globals + .iter() + .map(|(k, _)| k.clone()) + .filter(|k| !existing.contains(k)) + .collect(); + missing.sort(); + res.extend(missing); + Ok(res) + } +} + +#[derive(Debug, Clone)] +enum ParsedRusthThing<'a> { + Struct(&'a RustStruct), + Enum(&'a RustEnum), + TypeAlias(&'a RustTypeAlias), +} + +// Collect unique type vars from an enum field +// Since we explode enums into unions of types, we need to extract all of the generics +// used by each individual field +// We do this by exploring each field's type and comparing against the generics used by the enum +// itself +fn collect_generics_for_variant(variant_type: &RustType, generics: &[String]) -> Vec { + let mut all = vec![]; + match variant_type { + RustType::Generic { id, parameters } => { + if generics.contains(id) { + all.push(id.clone()) + } + // Recurse into the params for the case of `Foo(HashMap)` + for param in parameters { + all.extend(collect_generics_for_variant(param, generics)) + } + } + RustType::Simple { id } => { + if generics.contains(id) { + all.push(id.clone()) + } + } + RustType::Special(special) => match &special { + SpecialRustType::HashMap(key_type, value_type) => { + all.extend(collect_generics_for_variant(key_type, generics)); + all.extend(collect_generics_for_variant(value_type, generics)); + } + SpecialRustType::Option(some_type) => { + all.extend(collect_generics_for_variant(some_type, generics)); + } + SpecialRustType::Vec(value_type) => { + all.extend(collect_generics_for_variant(value_type, generics)); + } + _ => {} + }, + } + // Remove any duplicates + // E.g. Foo(HashMap) should only produce a single type var + dedup(&mut all); + all +} + +fn dedup(v: &mut Vec) { + // note the Copy constraint + let mut uniques = HashSet::new(); + v.retain(|e| uniques.insert(e.clone())); +} + +/// All information needed to generate Python type-code +#[derive(Default)] +pub struct Python { + /// Mappings from Rust type names to Python type names + pub type_mappings: HashMap, + pub module: RefCell, +} + +impl Language for Python { + fn type_map(&mut self) -> &HashMap { + &self.type_mappings + } + fn generate_types( + &mut self, + w: &mut dyn Write, + data: &crate::parser::ParsedData, + ) -> std::io::Result<()> { + let mut globals: Vec; + { + let mut module = self.module.borrow_mut(); + for alias in &data.aliases { + let thing = ParsedRusthThing::TypeAlias(alias); + let identifier = self.get_identifier(thing); + match &alias.r#type { + RustType::Generic { id, parameters: _ } => { + module.add_global(identifier, vec![id.clone()]) + } + RustType::Simple { id } => module.add_global(identifier, vec![id.clone()]), + RustType::Special(_) => {} + } + } + for strct in &data.structs { + let thing = ParsedRusthThing::Struct(strct); + let identifier = self.get_identifier(thing); + module.add_global(identifier, vec![]); + } + for enm in &data.enums { + let thing = ParsedRusthThing::Enum(enm); + let identifier = self.get_identifier(thing); + module.add_global(identifier, vec![]); + } + globals = data + .aliases + .iter() + .map(ParsedRusthThing::TypeAlias) + .chain(data.structs.iter().map(ParsedRusthThing::Struct)) + .chain(data.enums.iter().map(ParsedRusthThing::Enum)) + .collect(); + let sorted_identifiers = module.topologically_sorted_globals().unwrap(); + globals.sort_by(|a, b| { + let identifier_a = self.get_identifier(a.clone()); + let identifier_b = self.get_identifier(b.clone()); + let pos_a = sorted_identifiers + .iter() + .position(|o| o.eq(&identifier_a)) + .unwrap_or(0); + let pos_b = sorted_identifiers + .iter() + .position(|o| o.eq(&identifier_b)) + .unwrap_or(0); + pos_a.cmp(&pos_b) + }); + } + let mut body: Vec = Vec::new(); + for thing in globals { + match thing { + ParsedRusthThing::Enum(e) => self.write_enum(&mut body, e)?, + ParsedRusthThing::Struct(rs) => self.write_struct(&mut body, rs)?, + ParsedRusthThing::TypeAlias(t) => self.write_type_alias(&mut body, t)?, + }; + } + self.begin_file(w)?; + let _ = w.write(&body)?; + Ok(()) + } + + fn format_generic_type( + &mut self, + base: &String, + parameters: &[RustType], + generic_types: &[String], + ) -> Result { + if let Some(mapped) = self.type_map().get(base) { + Ok(mapped.into()) + } else { + let parameters: Result, RustTypeFormatError> = parameters + .iter() + .map(|p| self.format_type(p, generic_types)) + .collect(); + let parameters = parameters?; + Ok(format!( + "{}{}", + self.format_simple_type(base, generic_types)?, + (!parameters.is_empty()) + .then(|| format!("[{}]", parameters.join(", "))) + .unwrap_or_default() + )) + } + } + + fn format_simple_type( + &mut self, + base: &String, + _generic_types: &[String], + ) -> Result { + self.add_imports(base); + Ok(if let Some(mapped) = self.type_map().get(base) { + mapped.into() + } else { + base.into() + }) + } + + fn format_special_type( + &mut self, + special_ty: &SpecialRustType, + generic_types: &[String], + ) -> Result { + match special_ty { + SpecialRustType::Vec(rtype) => { + self.module + .borrow_mut() + .add_import("typing".to_string(), "List".to_string()); + Ok(format!("List[{}]", self.format_type(rtype, generic_types)?)) + } + // We add optionality above the type formatting level + SpecialRustType::Option(rtype) => self.format_type(rtype, generic_types), + SpecialRustType::HashMap(rtype1, rtype2) => { + self.module + .borrow_mut() + .add_import("typing".to_string(), "Dict".to_string()); + Ok(format!( + "Dict[{}, {}]", + match rtype1.as_ref() { + RustType::Simple { id } if generic_types.contains(id) => { + return Err(RustTypeFormatError::GenericKeyForbiddenInTS(id.clone())); + } + _ => self.format_type(rtype1, generic_types)?, + }, + self.format_type(rtype2, generic_types)? + )) + } + SpecialRustType::Unit => Ok("None".into()), + SpecialRustType::String => Ok("str".into()), + SpecialRustType::I8 + | SpecialRustType::U8 + | SpecialRustType::I16 + | SpecialRustType::U16 + | SpecialRustType::I32 + | SpecialRustType::U32 + | SpecialRustType::I54 + | SpecialRustType::U53 => Ok("int".into()), + SpecialRustType::F32 | SpecialRustType::F64 => Ok("float".into()), + SpecialRustType::Bool => Ok("bool".into()), + SpecialRustType::U64 + | SpecialRustType::I64 + | SpecialRustType::ISize + | SpecialRustType::USize => { + panic!("64 bit types not allowed in Typeshare") + } + SpecialRustType::Array(_, _) => todo!(), + SpecialRustType::Slice(_) => todo!(), + SpecialRustType::Char => todo!(), + } + } + + fn begin_file(&mut self, w: &mut dyn Write) -> std::io::Result<()> { + let module = self.module.borrow(); + let mut type_var_names: Vec = module.type_variables.iter().cloned().collect(); + type_var_names.sort(); + let type_vars: Vec = type_var_names + .iter() + .map(|name| format!("{} = TypeVar(\"{}\")", name, name)) + .collect(); + let mut imports = vec![]; + for (import_module, identifiers) in &module.imports { + let mut identifier_vec = identifiers.iter().cloned().collect::>(); + identifier_vec.sort(); + imports.push(format!( + "from {} import {}", + import_module, + identifier_vec.join(", ") + )) + } + imports.sort(); + writeln!(w, "\"\"\"")?; + writeln!(w, " Generated by typeshare {}", env!("CARGO_PKG_VERSION"))?; + writeln!(w, "\"\"\"")?; + writeln!(w, "from __future__ import annotations\n").unwrap(); + writeln!(w, "{}\n", imports.join("\n"))?; + match type_vars.is_empty() { + true => writeln!(w).unwrap(), + false => writeln!(w, "{}\n\n", type_vars.join("\n")).unwrap(), + }; + Ok(()) + } + + fn write_type_alias(&mut self, w: &mut dyn Write, ty: &RustTypeAlias) -> std::io::Result<()> { + let r#type = self + .format_type(&ty.r#type, ty.generic_types.as_slice()) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + + writeln!( + w, + "{}{} = {}\n\n", + ty.id.renamed, + (!ty.generic_types.is_empty()) + .then(|| format!("[{}]", ty.generic_types.join(", "))) + .unwrap_or_default(), + r#type, + )?; + + self.write_comments(w, true, &ty.comments, 1)?; + + Ok(()) + } + + fn write_struct(&mut self, w: &mut dyn Write, rs: &RustStruct) -> std::io::Result<()> { + { + let mut module = self.module.borrow_mut(); + rs.generic_types + .iter() + .cloned() + .for_each(|v| module.add_type_var(v)) + } + let bases = match rs.generic_types.is_empty() { + true => "BaseModel".to_string(), + false => { + self.module + .borrow_mut() + .add_import("pydantic.generics".to_string(), "GenericModel".to_string()); + self.module + .borrow_mut() + .add_import("typing".to_string(), "Generic".to_string()); + format!("GenericModel, Generic[{}]", rs.generic_types.join(", ")) + } + }; + writeln!(w, "class {}({}):", rs.id.renamed, bases,)?; + + self.write_comments(w, true, &rs.comments, 1)?; + + rs.fields + .iter() + .try_for_each(|f| self.write_field(w, f, rs.generic_types.as_slice()))?; + + if rs.fields.is_empty() { + write!(w, " pass")? + } + write!(w, "\n\n")?; + self.module + .borrow_mut() + .add_import("pydantic".to_string(), "BaseModel".to_string()); + Ok(()) + } + + fn write_enum(&mut self, w: &mut dyn Write, e: &RustEnum) -> std::io::Result<()> { + // Make a suitable name for an anonymous struct enum variant + let make_anonymous_struct_name = + |variant_name: &str| format!("{}{}Inner", &e.shared().id.original, variant_name); + + // Generate named types for any anonymous struct variants of this enum + self.write_types_for_anonymous_structs(w, e, &make_anonymous_struct_name)?; + + match e { + // Write all the unit variants out (there can only be unit variants in + // this case) + RustEnum::Unit(shared) => { + self.module + .borrow_mut() + .add_import("typing".to_string(), "Literal".to_string()); + write!( + w, + "{} = Literal[{}]", + shared.id.renamed, + shared + .variants + .iter() + .map(|v| format!( + "\"{}\"", + match v { + RustEnumVariant::Unit(v) => { + v.id.renamed.clone() + } + _ => panic!(), + } + )) + .collect::>() + .join(", ") + )?; + write!(w, "\n\n").unwrap(); + } + // Write all the algebraic variants out (all three variant types are possible + // here) + RustEnum::Algebraic { + tag_key, + content_key, + shared, + .. + } => { + { + let mut module = self.module.borrow_mut(); + shared + .generic_types + .iter() + .cloned() + .for_each(|v| module.add_type_var(v)) + } + let mut variants: Vec<(String, Vec)> = Vec::new(); + shared.variants.iter().for_each(|variant| { + match variant { + RustEnumVariant::Unit(unit_variant) => { + self.module + .borrow_mut() + .add_import("typing".to_string(), "Literal".to_string()); + let variant_name = + format!("{}{}", shared.id.original, unit_variant.id.original); + variants.push((variant_name.clone(), vec![])); + writeln!(w, "class {}:", variant_name).unwrap(); + writeln!( + w, + " {}: Literal[\"{}\"]", + tag_key, unit_variant.id.renamed + ) + .unwrap(); + } + RustEnumVariant::Tuple { + ty, + shared: variant_shared, + } => { + self.module + .borrow_mut() + .add_import("typing".to_string(), "Literal".to_string()); + let variant_name = + format!("{}{}", shared.id.original, variant_shared.id.original); + match ty { + RustType::Generic { id: _, parameters } => { + // This variant has generics, include them in the class def + let mut generic_parameters: Vec = parameters + .iter() + .flat_map(|p| { + collect_generics_for_variant(p, &shared.generic_types) + }) + .collect(); + dedup(&mut generic_parameters); + let type_vars = self + .module + .borrow_mut() + .get_type_vars(generic_parameters.len()); + variants.push((variant_name.clone(), type_vars)); + { + let mut module = self.module.borrow_mut(); + if generic_parameters.is_empty() { + module.add_import( + "pydantic".to_string(), + "BaseModel".to_string(), + ); + writeln!(w, "class {}(BaseModel):", variant_name) + .unwrap(); + } else { + module.add_import( + "typing".to_string(), + "Generic".to_string(), + ); + module.add_import( + "pydantic.generics".to_string(), + "GenericModel".to_string(), + ); + writeln!( + w, + "class {}(GenericModel, Generic[{}]):", + // note: generics is always unique (a single item) + variant_name, + generic_parameters.join(", ") + ) + .unwrap(); + } + } + } + other => { + let mut generics = vec![]; + if let RustType::Simple { id } = other { + // This could be a bare generic + if shared.generic_types.contains(id) { + generics = vec![id.clone()]; + } + } + variants.push((variant_name.clone(), generics.clone())); + { + let mut module = self.module.borrow_mut(); + if generics.is_empty() { + module.add_import( + "pydantic".to_string(), + "BaseModel".to_string(), + ); + writeln!(w, "class {}(BaseModel):", variant_name) + .unwrap(); + } else { + module.add_import( + "typing".to_string(), + "Generic".to_string(), + ); + module.add_import( + "pydantic.generics".to_string(), + "GenericModel".to_string(), + ); + writeln!( + w, + "class {}(GenericModel, Generic[{}]):", + // note: generics is always unique (a single item) + variant_name, + generics.join(", ") + ) + .unwrap(); + } + } + } + }; + writeln!( + w, + " {}: Literal[\"{}\"]", + tag_key, variant_shared.id.renamed + ) + .unwrap(); + writeln!( + w, + " {}: {}", + content_key, + match ty { + RustType::Simple { id } => id.to_owned(), + RustType::Special(special_ty) => self + .format_special_type(special_ty, &shared.generic_types) + .unwrap(), + RustType::Generic { id, parameters } => { + self.format_generic_type(id, parameters, &[]).unwrap() + } + } + ) + .unwrap(); + write!(w, "\n\n").unwrap(); + } + RustEnumVariant::AnonymousStruct { + shared: variant_shared, + fields, + } => { + let num_generic_parameters = fields + .iter() + .flat_map(|f| { + collect_generics_for_variant(&f.ty, &shared.generic_types) + }) + .count(); + let type_vars = self + .module + .borrow_mut() + .get_type_vars(num_generic_parameters); + let name = make_anonymous_struct_name(&variant_shared.id.original); + variants.push((name, type_vars)); + } + }; + }); + writeln!( + w, + "{} = {}", + shared.id.original, + variants + .iter() + .map(|(name, parameters)| match parameters.is_empty() { + true => name.clone(), + false => format!("{}[{}]", name, parameters.join(", ")), + }) + .collect::>() + .join(" | ") + ) + .unwrap(); + self.write_comments(w, true, &e.shared().comments, 0)?; + writeln!(w).unwrap(); + } + }; + Ok(()) + } +} + +impl Python { + pub fn new(type_mappings: HashMap) -> Self { + let mut mappings = type_mappings; + mappings.insert("DateTime".to_string(), "datetime".to_string()); + mappings.insert("Url".to_string(), "AnyUrl".to_string()); + Python { + type_mappings: mappings, + module: RefCell::new(Module::default()), + } + } + fn add_imports(&self, tp: &str) { + match tp { + "Url" => { + self.module + .borrow_mut() + .add_import("pydantic.networks".to_string(), "AnyUrl".to_string()); + } + "DateTime" => { + self.module + .borrow_mut() + .add_import("datetime".to_string(), "datetime".to_string()); + } + _ => {} + } + } + + fn get_identifier(&self, thing: ParsedRusthThing) -> String { + match thing { + ParsedRusthThing::TypeAlias(alias) => alias.id.original.clone(), + ParsedRusthThing::Struct(strct) => strct.id.original.clone(), + ParsedRusthThing::Enum(enm) => match enm { + RustEnum::Unit(u) => u.id.original.clone(), + RustEnum::Algebraic { + tag_key: _, + content_key: _, + shared, + } => shared.id.original.clone(), + }, + } + } + + fn write_field( + &mut self, + w: &mut dyn Write, + field: &RustField, + generic_types: &[String], + ) -> std::io::Result<()> { + let mut python_type = self + .format_type(&field.ty, generic_types) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + if field.ty.is_optional() || field.has_default { + python_type = format!("Optional[{}]", python_type); + self.module + .borrow_mut() + .add_import("typing".to_string(), "Optional".to_string()); + } + let mut default = None; + if field.has_default { + default = Some("None".to_string()) + } + let python_field_name = python_property_aware_rename(&field.id.original); + python_type = match python_field_name == field.id.renamed { + true => python_type, + false => { + self.module + .borrow_mut() + .add_import("typing".to_string(), "Annotated".to_string()); + self.module + .borrow_mut() + .add_import("pydantic".to_string(), "Field".to_string()); + format!( + "Annotated[{}, Field(alias=\"{}\")]", + python_type, field.id.renamed + ) + } + }; + match default { + Some(default) => writeln!( + w, + " {}: {} = {}", + python_field_name, python_type, default, + )?, + None => writeln!(w, " {}: {}", python_field_name, python_type)?, + } + + self.write_comments(w, true, &field.comments, 1)?; + Ok(()) + } + + fn write_comments( + &self, + w: &mut dyn Write, + is_docstring: bool, + comments: &[String], + indent_level: usize, + ) -> std::io::Result<()> { + // Only attempt to write a comment if there are some, otherwise we're Ok() + let indent = " ".repeat(indent_level); + if !comments.is_empty() { + let comment: String = { + if is_docstring { + format!( + "{indent}\"\"\"\n{indented_comments}\n{indent}\"\"\"", + indent = indent, + indented_comments = comments + .iter() + .map(|v| format!("{}{}", indent, v)) + .collect::>() + .join("\n"), + ) + } else { + comments + .iter() + .map(|v| format!("{}# {}", indent, v)) + .collect::>() + .join("\n") + } + }; + writeln!(w, "{}", comment)?; + } + Ok(()) + } +} + +static PYTHON_KEYWORDS: Lazy> = Lazy::new(|| { + HashSet::from_iter( + vec![ + "False", "None", "True", "and", "as", "assert", "async", "await", "break", "class", + "continue", "def", "del", "elif", "else", "except", "finally", "for", "from", "global", + "if", "import", "in", "is", "lambda", "nonlocal", "not", "or", "pass", "raise", + "return", "try", "while", "with", "yield", + ] + .iter() + .map(|v| v.to_string()), + ) +}); + +fn python_property_aware_rename(name: &str) -> String { + let snake_name = name.to_case(Case::Snake); + match PYTHON_KEYWORDS.contains(&snake_name) { + true => format!("{}_", name), + false => snake_name, + } +} From ee5673c1b1f57cf869ef2cd910f8a232fa542517 Mon Sep 17 00:00:00 2001 From: Horia Culea Date: Tue, 28 May 2024 16:45:57 +0200 Subject: [PATCH 02/12] Resolve clippy errors --- core/src/language/python.rs | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/core/src/language/python.rs b/core/src/language/python.rs index 6554754b..b00f83be 100644 --- a/core/src/language/python.rs +++ b/core/src/language/python.rs @@ -25,7 +25,7 @@ pub struct Module { } #[derive(Debug)] -struct GenerationError(String); +struct GenerationError; impl Module { // Idempotently insert an import @@ -46,7 +46,6 @@ impl Module { } fn get_type_vars(&mut self, n: usize) -> Vec { let vars: Vec = (0..n) - .into_iter() .map(|i| { if i == 0 { "T".to_string() @@ -76,7 +75,7 @@ impl Module { res.extend_from_slice(&level); if level.is_empty() { if !ts.is_empty() { - return Err(GenerationError("Cyclical runtime dependency".to_string())); + return Err(GenerationError); } break; } @@ -84,8 +83,8 @@ impl Module { let existing: HashSet<&String> = HashSet::from_iter(res.iter()); let mut missing: Vec = self .globals - .iter() - .map(|(k, _)| k.clone()) + .keys() + .cloned() .filter(|k| !existing.contains(k)) .collect(); missing.sort(); @@ -154,6 +153,7 @@ fn dedup(v: &mut Vec) { pub struct Python { /// Mappings from Rust type names to Python type names pub type_mappings: HashMap, + /// The Python module for the generated code. pub module: RefCell, } @@ -631,15 +631,6 @@ impl Language for Python { } impl Python { - pub fn new(type_mappings: HashMap) -> Self { - let mut mappings = type_mappings; - mappings.insert("DateTime".to_string(), "datetime".to_string()); - mappings.insert("Url".to_string(), "AnyUrl".to_string()); - Python { - type_mappings: mappings, - module: RefCell::new(Module::default()), - } - } fn add_imports(&self, tp: &str) { match tp { "Url" => { From dfd6522e0a953de9cf4b76fc9b28e1cb2cfa6ae5 Mon Sep 17 00:00:00 2001 From: Horia Culea Date: Tue, 28 May 2024 16:59:50 +0200 Subject: [PATCH 03/12] Resolve merge conflicts with latest main --- cli/src/main.rs | 5 +---- core/src/language/python.rs | 8 ++++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index 29060c78..2927bf5c 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -16,10 +16,7 @@ use std::collections::HashMap; use typeshare_core::language::Go; use typeshare_core::language::{GenericConstraints, Python}; use typeshare_core::{ - language::{ - CrateName, Kotlin, Language, Scala, SupportedLanguage, Swift, - TypeScript, - }, + language::{CrateName, Kotlin, Language, Scala, SupportedLanguage, Swift, TypeScript}, parser::ParsedData, }; use writer::write_generated; diff --git a/core/src/language/python.rs b/core/src/language/python.rs index 9a2c2c8a..1c2ea0d5 100644 --- a/core/src/language/python.rs +++ b/core/src/language/python.rs @@ -1,3 +1,4 @@ +use crate::parser::ParsedData; use crate::rust_types::{RustType, RustTypeFormatError, SpecialRustType}; use crate::{ language::Language, @@ -8,12 +9,10 @@ use std::cell::RefCell; use std::collections::hash_map::Entry; use std::collections::HashSet; use std::hash::Hash; -use crate::parser::ParsedData; use std::{collections::HashMap, io::Write}; use super::CrateTypes; - use convert_case::{Case, Casing}; use topological_sort::TopologicalSort; @@ -88,7 +87,8 @@ impl Module { let mut missing: Vec = self .globals .keys() - .filter(|&k| !existing.contains(k)).cloned() + .filter(|&k| !existing.contains(k)) + .cloned() .collect(); missing.sort(); res.extend(missing); @@ -632,7 +632,7 @@ impl Language for Python { }; Ok(()) } - + fn write_imports( &mut self, _writer: &mut dyn Write, From 0c149147258d5bb49168f697d8205c6dff35aedd Mon Sep 17 00:00:00 2001 From: Horia Culea Date: Tue, 28 May 2024 17:07:11 +0200 Subject: [PATCH 04/12] Add python as an option for the --lang flag --- cli/src/args.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/args.rs b/cli/src/args.rs index 8bf11d01..61f363e8 100644 --- a/cli/src/args.rs +++ b/cli/src/args.rs @@ -19,10 +19,10 @@ pub const ARG_OUTPUT_FOLDER: &str = "output-folder"; pub const ARG_FOLLOW_LINKS: &str = "follow-links"; #[cfg(feature = "go")] -const AVAILABLE_LANGUAGES: [&str; 5] = ["kotlin", "scala", "swift", "typescript", "go"]; +const AVAILABLE_LANGUAGES: [&str; 6] = ["kotlin", "scala", "swift", "typescript", "go", "python"]; #[cfg(not(feature = "go"))] -const AVAILABLE_LANGUAGES: [&str; 4] = ["kotlin", "scala", "swift", "typescript"]; +const AVAILABLE_LANGUAGES: [&str; 5] = ["kotlin", "scala", "swift", "typescript", "python"]; /// Parse command line arguments. pub(crate) fn build_command() -> Command<'static> { From 439783b4e5ad4439c80839fa72275edb87ad9de5 Mon Sep 17 00:00:00 2001 From: Horia Culea Date: Mon, 10 Jun 2024 14:58:10 +0200 Subject: [PATCH 05/12] Add support for todo and priorly unsupported types --- core/src/language/python.rs | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/core/src/language/python.rs b/core/src/language/python.rs index 1c2ea0d5..13ae99c6 100644 --- a/core/src/language/python.rs +++ b/core/src/language/python.rs @@ -272,7 +272,9 @@ impl Language for Python { generic_types: &[String], ) -> Result { match special_ty { - SpecialRustType::Vec(rtype) => { + SpecialRustType::Vec(rtype) + | SpecialRustType::Array(rtype, _) + | SpecialRustType::Slice(rtype) => { self.module .borrow_mut() .add_import("typing".to_string(), "List".to_string()); @@ -296,7 +298,7 @@ impl Language for Python { )) } SpecialRustType::Unit => Ok("None".into()), - SpecialRustType::String => Ok("str".into()), + SpecialRustType::String | SpecialRustType::Char => Ok("str".into()), SpecialRustType::I8 | SpecialRustType::U8 | SpecialRustType::I16 @@ -304,18 +306,13 @@ impl Language for Python { | SpecialRustType::I32 | SpecialRustType::U32 | SpecialRustType::I54 - | SpecialRustType::U53 => Ok("int".into()), - SpecialRustType::F32 | SpecialRustType::F64 => Ok("float".into()), - SpecialRustType::Bool => Ok("bool".into()), - SpecialRustType::U64 + | SpecialRustType::U53 + | SpecialRustType::U64 | SpecialRustType::I64 | SpecialRustType::ISize - | SpecialRustType::USize => { - panic!("64 bit types not allowed in Typeshare") - } - SpecialRustType::Array(_, _) => todo!(), - SpecialRustType::Slice(_) => todo!(), - SpecialRustType::Char => todo!(), + | SpecialRustType::USize => Ok("int".into()), + SpecialRustType::F32 | SpecialRustType::F64 => Ok("float".into()), + SpecialRustType::Bool => Ok("bool".into()), } } From d7f12d0f494dffca7992ca74241bbf8dc7097b81 Mon Sep 17 00:00:00 2001 From: Horia Culea Date: Mon, 10 Jun 2024 15:07:59 +0200 Subject: [PATCH 06/12] Leverage mutable self to remove RefCell around Module --- core/src/language/python.rs | 65 ++++++++++++------------------------- 1 file changed, 20 insertions(+), 45 deletions(-) diff --git a/core/src/language/python.rs b/core/src/language/python.rs index 13ae99c6..be43a160 100644 --- a/core/src/language/python.rs +++ b/core/src/language/python.rs @@ -5,7 +5,6 @@ use crate::{ rust_types::{RustEnum, RustEnumVariant, RustField, RustStruct, RustTypeAlias}, }; use once_cell::sync::Lazy; -use std::cell::RefCell; use std::collections::hash_map::Entry; use std::collections::HashSet; use std::hash::Hash; @@ -157,7 +156,7 @@ pub struct Python { /// Mappings from Rust type names to Python type names pub type_mappings: HashMap, /// The Python module for the generated code. - pub module: RefCell, + pub module: Module, } impl Language for Python { @@ -172,27 +171,26 @@ impl Language for Python { ) -> std::io::Result<()> { let mut globals: Vec; { - let mut module = self.module.borrow_mut(); for alias in &data.aliases { let thing = ParsedRusthThing::TypeAlias(alias); let identifier = self.get_identifier(thing); match &alias.r#type { RustType::Generic { id, parameters: _ } => { - module.add_global(identifier, vec![id.clone()]) + self.module.add_global(identifier, vec![id.clone()]) } - RustType::Simple { id } => module.add_global(identifier, vec![id.clone()]), + RustType::Simple { id } => self.module.add_global(identifier, vec![id.clone()]), RustType::Special(_) => {} } } for strct in &data.structs { let thing = ParsedRusthThing::Struct(strct); let identifier = self.get_identifier(thing); - module.add_global(identifier, vec![]); + self.module.add_global(identifier, vec![]); } for enm in &data.enums { let thing = ParsedRusthThing::Enum(enm); let identifier = self.get_identifier(thing); - module.add_global(identifier, vec![]); + self.module.add_global(identifier, vec![]); } globals = data .aliases @@ -201,7 +199,7 @@ impl Language for Python { .chain(data.structs.iter().map(ParsedRusthThing::Struct)) .chain(data.enums.iter().map(ParsedRusthThing::Enum)) .collect(); - let sorted_identifiers = module.topologically_sorted_globals().unwrap(); + let sorted_identifiers = self.module.topologically_sorted_globals().unwrap(); globals.sort_by(|a, b| { let identifier_a = self.get_identifier(a.clone()); let identifier_b = self.get_identifier(b.clone()); @@ -276,7 +274,6 @@ impl Language for Python { | SpecialRustType::Array(rtype, _) | SpecialRustType::Slice(rtype) => { self.module - .borrow_mut() .add_import("typing".to_string(), "List".to_string()); Ok(format!("List[{}]", self.format_type(rtype, generic_types)?)) } @@ -284,7 +281,6 @@ impl Language for Python { SpecialRustType::Option(rtype) => self.format_type(rtype, generic_types), SpecialRustType::HashMap(rtype1, rtype2) => { self.module - .borrow_mut() .add_import("typing".to_string(), "Dict".to_string()); Ok(format!( "Dict[{}, {}]", @@ -317,15 +313,14 @@ impl Language for Python { } fn begin_file(&mut self, w: &mut dyn Write, _parsed_data: &ParsedData) -> std::io::Result<()> { - let module = self.module.borrow(); - let mut type_var_names: Vec = module.type_variables.iter().cloned().collect(); + let mut type_var_names: Vec = self.module.type_variables.iter().cloned().collect(); type_var_names.sort(); let type_vars: Vec = type_var_names .iter() .map(|name| format!("{} = TypeVar(\"{}\")", name, name)) .collect(); let mut imports = vec![]; - for (import_module, identifiers) in &module.imports { + for (import_module, identifiers) in &self.module.imports { let mut identifier_vec = identifiers.iter().cloned().collect::>(); identifier_vec.sort(); imports.push(format!( @@ -369,20 +364,17 @@ impl Language for Python { fn write_struct(&mut self, w: &mut dyn Write, rs: &RustStruct) -> std::io::Result<()> { { - let mut module = self.module.borrow_mut(); rs.generic_types .iter() .cloned() - .for_each(|v| module.add_type_var(v)) + .for_each(|v| self.module.add_type_var(v)) } let bases = match rs.generic_types.is_empty() { true => "BaseModel".to_string(), false => { self.module - .borrow_mut() .add_import("pydantic.generics".to_string(), "GenericModel".to_string()); self.module - .borrow_mut() .add_import("typing".to_string(), "Generic".to_string()); format!("GenericModel, Generic[{}]", rs.generic_types.join(", ")) } @@ -400,7 +392,6 @@ impl Language for Python { } write!(w, "\n\n")?; self.module - .borrow_mut() .add_import("pydantic".to_string(), "BaseModel".to_string()); Ok(()) } @@ -418,7 +409,6 @@ impl Language for Python { // this case) RustEnum::Unit(shared) => { self.module - .borrow_mut() .add_import("typing".to_string(), "Literal".to_string()); write!( w, @@ -450,19 +440,17 @@ impl Language for Python { .. } => { { - let mut module = self.module.borrow_mut(); shared .generic_types .iter() .cloned() - .for_each(|v| module.add_type_var(v)) + .for_each(|v| self.module.add_type_var(v)) } let mut variants: Vec<(String, Vec)> = Vec::new(); shared.variants.iter().for_each(|variant| { match variant { RustEnumVariant::Unit(unit_variant) => { self.module - .borrow_mut() .add_import("typing".to_string(), "Literal".to_string()); let variant_name = format!("{}{}", shared.id.original, unit_variant.id.original); @@ -480,7 +468,6 @@ impl Language for Python { shared: variant_shared, } => { self.module - .borrow_mut() .add_import("typing".to_string(), "Literal".to_string()); let variant_name = format!("{}{}", shared.id.original, variant_shared.id.original); @@ -494,26 +481,23 @@ impl Language for Python { }) .collect(); dedup(&mut generic_parameters); - let type_vars = self - .module - .borrow_mut() - .get_type_vars(generic_parameters.len()); + let type_vars = + self.module.get_type_vars(generic_parameters.len()); variants.push((variant_name.clone(), type_vars)); { - let mut module = self.module.borrow_mut(); if generic_parameters.is_empty() { - module.add_import( + self.module.add_import( "pydantic".to_string(), "BaseModel".to_string(), ); writeln!(w, "class {}(BaseModel):", variant_name) .unwrap(); } else { - module.add_import( + self.module.add_import( "typing".to_string(), "Generic".to_string(), ); - module.add_import( + self.module.add_import( "pydantic.generics".to_string(), "GenericModel".to_string(), ); @@ -538,20 +522,19 @@ impl Language for Python { } variants.push((variant_name.clone(), generics.clone())); { - let mut module = self.module.borrow_mut(); if generics.is_empty() { - module.add_import( + self.module.add_import( "pydantic".to_string(), "BaseModel".to_string(), ); writeln!(w, "class {}(BaseModel):", variant_name) .unwrap(); } else { - module.add_import( + self.module.add_import( "typing".to_string(), "Generic".to_string(), ); - module.add_import( + self.module.add_import( "pydantic.generics".to_string(), "GenericModel".to_string(), ); @@ -600,10 +583,7 @@ impl Language for Python { collect_generics_for_variant(&f.ty, &shared.generic_types) }) .count(); - let type_vars = self - .module - .borrow_mut() - .get_type_vars(num_generic_parameters); + let type_vars = self.module.get_type_vars(num_generic_parameters); let name = make_anonymous_struct_name(&variant_shared.id.original); variants.push((name, type_vars)); } @@ -640,16 +620,14 @@ impl Language for Python { } impl Python { - fn add_imports(&self, tp: &str) { + fn add_imports(&mut self, tp: &str) { match tp { "Url" => { self.module - .borrow_mut() .add_import("pydantic.networks".to_string(), "AnyUrl".to_string()); } "DateTime" => { self.module - .borrow_mut() .add_import("datetime".to_string(), "datetime".to_string()); } _ => {} @@ -683,7 +661,6 @@ impl Python { if field.ty.is_optional() || field.has_default { python_type = format!("Optional[{}]", python_type); self.module - .borrow_mut() .add_import("typing".to_string(), "Optional".to_string()); } let mut default = None; @@ -695,10 +672,8 @@ impl Python { true => python_type, false => { self.module - .borrow_mut() .add_import("typing".to_string(), "Annotated".to_string()); self.module - .borrow_mut() .add_import("pydantic".to_string(), "Field".to_string()); format!( "Annotated[{}, Field(alias=\"{}\")]", From 203981804ad24aa8205edaa14daa50d0e7936e7c Mon Sep 17 00:00:00 2001 From: Horia Culea Date: Mon, 10 Jun 2024 15:32:55 +0200 Subject: [PATCH 07/12] Fix ParsedRusthThing typo --- core/src/language/python.rs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/core/src/language/python.rs b/core/src/language/python.rs index be43a160..48234910 100644 --- a/core/src/language/python.rs +++ b/core/src/language/python.rs @@ -96,7 +96,7 @@ impl Module { } #[derive(Debug, Clone)] -enum ParsedRusthThing<'a> { +enum ParsedRustThing<'a> { Struct(&'a RustStruct), Enum(&'a RustEnum), TypeAlias(&'a RustTypeAlias), @@ -169,10 +169,10 @@ impl Language for Python { _imports: &CrateTypes, data: ParsedData, ) -> std::io::Result<()> { - let mut globals: Vec; + let mut globals: Vec; { for alias in &data.aliases { - let thing = ParsedRusthThing::TypeAlias(alias); + let thing = ParsedRustThing::TypeAlias(alias); let identifier = self.get_identifier(thing); match &alias.r#type { RustType::Generic { id, parameters: _ } => { @@ -183,21 +183,21 @@ impl Language for Python { } } for strct in &data.structs { - let thing = ParsedRusthThing::Struct(strct); + let thing = ParsedRustThing::Struct(strct); let identifier = self.get_identifier(thing); self.module.add_global(identifier, vec![]); } for enm in &data.enums { - let thing = ParsedRusthThing::Enum(enm); + let thing = ParsedRustThing::Enum(enm); let identifier = self.get_identifier(thing); self.module.add_global(identifier, vec![]); } globals = data .aliases .iter() - .map(ParsedRusthThing::TypeAlias) - .chain(data.structs.iter().map(ParsedRusthThing::Struct)) - .chain(data.enums.iter().map(ParsedRusthThing::Enum)) + .map(ParsedRustThing::TypeAlias) + .chain(data.structs.iter().map(ParsedRustThing::Struct)) + .chain(data.enums.iter().map(ParsedRustThing::Enum)) .collect(); let sorted_identifiers = self.module.topologically_sorted_globals().unwrap(); globals.sort_by(|a, b| { @@ -217,9 +217,9 @@ impl Language for Python { let mut body: Vec = Vec::new(); for thing in globals { match thing { - ParsedRusthThing::Enum(e) => self.write_enum(&mut body, e)?, - ParsedRusthThing::Struct(rs) => self.write_struct(&mut body, rs)?, - ParsedRusthThing::TypeAlias(t) => self.write_type_alias(&mut body, t)?, + ParsedRustThing::Enum(e) => self.write_enum(&mut body, e)?, + ParsedRustThing::Struct(rs) => self.write_struct(&mut body, rs)?, + ParsedRustThing::TypeAlias(t) => self.write_type_alias(&mut body, t)?, }; } self.begin_file(w, &data)?; @@ -634,11 +634,11 @@ impl Python { } } - fn get_identifier(&self, thing: ParsedRusthThing) -> String { + fn get_identifier(&self, thing: ParsedRustThing) -> String { match thing { - ParsedRusthThing::TypeAlias(alias) => alias.id.original.clone(), - ParsedRusthThing::Struct(strct) => strct.id.original.clone(), - ParsedRusthThing::Enum(enm) => match enm { + ParsedRustThing::TypeAlias(alias) => alias.id.original.clone(), + ParsedRustThing::Struct(strct) => strct.id.original.clone(), + ParsedRustThing::Enum(enm) => match enm { RustEnum::Unit(u) => u.id.original.clone(), RustEnum::Algebraic { tag_key: _, From c9f4be42b0415c1844db7a95d4d2604072e7fb0c Mon Sep 17 00:00:00 2001 From: Horia Culea Date: Mon, 5 Aug 2024 11:32:14 +0200 Subject: [PATCH 08/12] Use serialization alias in renaming fields in Python --- core/src/language/python.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/language/python.rs b/core/src/language/python.rs index 48234910..2c10a85f 100644 --- a/core/src/language/python.rs +++ b/core/src/language/python.rs @@ -676,7 +676,7 @@ impl Python { self.module .add_import("pydantic".to_string(), "Field".to_string()); format!( - "Annotated[{}, Field(alias=\"{}\")]", + "Annotated[{}, Field(serialization_alias=\"{}\")]", python_type, field.id.renamed ) } From 459f596d304ca6f3d75a74f8c7af35306e3fab80 Mon Sep 17 00:00:00 2001 From: AndyTitu Date: Tue, 20 Aug 2024 13:25:02 +0300 Subject: [PATCH 09/12] Fix python serializing and deserialzing using pydantic Field aliasing and ConfigDict --- core/src/language/python.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/core/src/language/python.rs b/core/src/language/python.rs index 2c10a85f..9c562e54 100644 --- a/core/src/language/python.rs +++ b/core/src/language/python.rs @@ -383,6 +383,8 @@ impl Language for Python { self.write_comments(w, true, &rs.comments, 1)?; + handle_model_config(w, &mut self.module, rs); + rs.fields .iter() .try_for_each(|f| self.write_field(w, f, rs.generic_types.as_slice()))?; @@ -676,7 +678,7 @@ impl Python { self.module .add_import("pydantic".to_string(), "Field".to_string()); format!( - "Annotated[{}, Field(serialization_alias=\"{}\")]", + "Annotated[{}, Field(alias=\"{}\")]", python_type, field.id.renamed ) } @@ -749,3 +751,15 @@ fn python_property_aware_rename(name: &str) -> String { false => snake_name, } } + +// If at least one field from within a class is changed when the serde rename is used (a.k.a the field has 2 words) then we must use aliasing and we must also use a config dict at the top level of the class. +fn handle_model_config(w: &mut dyn Write, python_module: &mut Module, rs: &RustStruct) { + let visibly_renamed_field = rs.fields.iter().find(|f| { + let python_field_name = python_property_aware_rename(&f.id.original); + python_field_name != f.id.renamed + }); + if visibly_renamed_field.is_some() { + python_module.add_import("pydantic".to_string(), "ConfigDict".to_string()); + let _ = writeln!(w, "model_config = ConfigDict(populate_by_name=True)"); + }; +} From fde8907a20ad9774d865eb647dd43bbafe9f9a35 Mon Sep 17 00:00:00 2001 From: AndyTitu Date: Tue, 20 Aug 2024 14:28:45 +0300 Subject: [PATCH 10/12] Fix indentation of writing config dict --- core/src/language/python.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/language/python.rs b/core/src/language/python.rs index 9c562e54..bf982783 100644 --- a/core/src/language/python.rs +++ b/core/src/language/python.rs @@ -760,6 +760,6 @@ fn handle_model_config(w: &mut dyn Write, python_module: &mut Module, rs: &RustS }); if visibly_renamed_field.is_some() { python_module.add_import("pydantic".to_string(), "ConfigDict".to_string()); - let _ = writeln!(w, "model_config = ConfigDict(populate_by_name=True)"); + let _ = writeln!(w, " model_config = ConfigDict(populate_by_name=True)\n"); }; } From 6e50bb7de230d9fb3e75fc2064a0f4b4f6abf2d8 Mon Sep 17 00:00:00 2001 From: Amanda Yu Date: Tue, 20 Aug 2024 16:55:49 -0400 Subject: [PATCH 11/12] added None default to Optional python type --- core/src/language/python.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/core/src/language/python.rs b/core/src/language/python.rs index bf982783..4983e81e 100644 --- a/core/src/language/python.rs +++ b/core/src/language/python.rs @@ -660,8 +660,13 @@ impl Python { let mut python_type = self .format_type(&field.ty, generic_types) .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; + let python_field_name = python_property_aware_rename(&field.id.original); if field.ty.is_optional() || field.has_default { - python_type = format!("Optional[{}]", python_type); + let mut add_none_str: &str = ""; + if python_field_name == field.id.renamed{ + add_none_str = " = None"; + } + python_type = format!("Optional[{}]{}", python_type, add_none_str); self.module .add_import("typing".to_string(), "Optional".to_string()); } @@ -669,8 +674,7 @@ impl Python { if field.has_default { default = Some("None".to_string()) } - let python_field_name = python_property_aware_rename(&field.id.original); - python_type = match python_field_name == field.id.renamed { + python_type = match python_field_name == field.id.renamed{ true => python_type, false => { self.module @@ -678,7 +682,7 @@ impl Python { self.module .add_import("pydantic".to_string(), "Field".to_string()); format!( - "Annotated[{}, Field(alias=\"{}\")]", + "Annotated[{}, Field(alias=\"{}\")] = None", python_type, field.id.renamed ) } From b709aff9a259b413c71b17b351314d05487ac928 Mon Sep 17 00:00:00 2001 From: Amanda Yu Date: Mon, 26 Aug 2024 21:58:12 -0400 Subject: [PATCH 12/12] fixed optional field default --- core/src/language/python.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/core/src/language/python.rs b/core/src/language/python.rs index 4983e81e..46d7cd52 100644 --- a/core/src/language/python.rs +++ b/core/src/language/python.rs @@ -661,12 +661,10 @@ impl Python { .format_type(&field.ty, generic_types) .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?; let python_field_name = python_property_aware_rename(&field.id.original); + let mut add_none_default: bool = false; if field.ty.is_optional() || field.has_default { - let mut add_none_str: &str = ""; - if python_field_name == field.id.renamed{ - add_none_str = " = None"; - } - python_type = format!("Optional[{}]{}", python_type, add_none_str); + add_none_default = true; + python_type = format!("Optional[{}]", python_type); self.module .add_import("typing".to_string(), "Optional".to_string()); } @@ -682,15 +680,18 @@ impl Python { self.module .add_import("pydantic".to_string(), "Field".to_string()); format!( - "Annotated[{}, Field(alias=\"{}\")] = None", + "Annotated[{}, Field(alias=\"{}\")]", python_type, field.id.renamed ) } }; + if add_none_default { + python_type = format!("{python_type} = None"); + } match default { Some(default) => writeln!( w, - " {}: {} = {}", + " {}: {} = none python default {}", python_field_name, python_type, default, )?, None => writeln!(w, " {}: {}", python_field_name, python_type)?,