WIP. Test gen doc from macros. to_date

datafusion/functions/src/datetime/to_date.rs

@@ -26,9 +26,42 @@ use datafusion_expr::scalar_doc_sections::DOC_SECTION_DATETIME;
 use datafusion_expr::{
     ColumnarValue, Documentation, ScalarUDFImpl, Signature, Volatility,
 };
+use datafusion_macros::udf_doc;
 use std::any::Any;
 use std::sync::OnceLock;
 
+#[udf_doc(
+    doc_section(include = "true", label = "Time and Date Functions"),
+    description = r"Converts a value to a date (`YYYY-MM-DD`).
+Supports strings, integer and double types as input.
+Strings are parsed as YYYY-MM-DD (e.g. '2023-07-20') if no [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)s are provided.
+Integers and doubles are interpreted as days since the unix epoch (`1970-01-01T00:00:00Z`).
+Returns the corresponding date.
+Note: `to_date` returns Date32, which represents its values as the number of days since unix epoch(`1970-01-01`) stored as signed 32 bit value. The largest supported date value is `9999-12-31`.",
+    syntax_example = "to_date('2017-05-31', '%Y-%m-%d')",
+    sql_example = "```sql\n\
+> select to_date('2023-01-31');\n\
++-----------------------------+\n\
+| to_date(Utf8(\"2023-01-31\")) |\n\
++-----------------------------+\n\
+| 2023-01-31                  |\n\
++-----------------------------+\n\
+> select to_date('2023/01/31', '%Y-%m-%d', '%Y/%m/%d');\n\
++---------------------------------------------------------------+\n\
+| to_date(Utf8(\"2023/01/31\"),Utf8(\"%Y-%m-%d\"),Utf8(\"%Y/%m/%d\")) |\n\
++---------------------------------------------------------------+\n\
+| 2023-01-31                                                    |\n\
++---------------------------------------------------------------+\n\
+```\n\n\
+Additional examples can be found [here](https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/to_date.rs)",
+    standard_argument(name = "expression", expression_type = "String"),
+    argument(
+        name = "format_n",
+        description = r"Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order
+  they appear with the first successful one being returned. If none of the formats successfully parse the expression
+  an error will be returned."
+    )
+)]
 #[derive(Debug)]
 pub struct ToDateFunc {
     signature: Signature,
@@ -376,4 +409,10 @@ mod tests {
             );
         }
     }
+
+    #[test]
+    fn test_doc() {
+        let dt = ToDateFunc::new();
+        println!("{:?}", dt.documentation_test());
+    }
 }

datafusion/functions/src/math/log.rs

@@ -17,7 +17,6 @@
 
 //! Math function: `log()`.
 
-use datafusion_macros::udf_doc;
 use std::any::Any;
 use std::sync::{Arc, OnceLock};
 
@@ -38,7 +37,6 @@ use datafusion_expr::{
 };
 use datafusion_expr::{ScalarUDFImpl, Signature, Volatility};
 
-#[udf_doc(description = "log_description", example = "log_example")]
 #[derive(Debug)]
 pub struct LogFunc {
     signature: Signature,
@@ -59,8 +57,10 @@ fn get_log_doc() -> &'static Documentation {
             .with_description("Returns the base-x logarithm of a number. Can either provide a specified base, or if omitted then takes the base-10 of a number.")
             .with_syntax_example(r#"log(base, numeric_expression)
 log(numeric_expression)"#)
-            .with_standard_argument("base", "Base numeric")
-            .with_standard_argument("numeric_expression", "Numeric")
+            .with_argument("base",
+                           "Base numeric expression to operate on. Can be a constant, column, or function, and any combination of arithmetic operators.")
+            .with_argument("numeric_expression",
+                           "Numeric expression to operate on. Can be a constant, column, or function, and any combination of arithmetic operators.")
             .build()
             .unwrap()
     })
@@ -185,6 +185,10 @@ impl ScalarUDFImpl for LogFunc {
         Ok(ColumnarValue::Array(arr))
     }
 
+    fn documentation(&self) -> Option<&Documentation> {
+        Some(get_log_doc())
+    }
+
     /// Simplify the `log` function by the relevant rules:
     /// 1. Log(a, 1) ===> 0
     /// 2. Log(a, Power(a, b)) ===> b
@@ -262,7 +266,6 @@ mod tests {
     use datafusion_common::DFSchema;
     use datafusion_expr::execution_props::ExecutionProps;
     use datafusion_expr::simplify::SimplifyContext;
-    use datafusion_pre_macros::DocumentationTest;
 
     #[test]
     fn test_log_f64() {
@@ -471,16 +474,4 @@ mod tests {
             SortProperties::Unordered
         );
     }
-
-    #[test]
-    fn test_doc() {
-        let log = LogFunc::new();
-        assert_eq!(
-            log.documentation_test(),
-            Some(DocumentationTest {
-                description: "log_description".to_string(),
-                syntax_example: "log_example".to_string(),
-            })
-        );
-    }
 }

datafusion/macros/Cargo.toml

@@ -37,7 +37,7 @@ proc-macro = true
 
 [dependencies]
 quote = "1.0.37"
-syn = "2.0.79"
+syn = { version = "2.0.79", features = ["full"] }
 proc-macro2 = "1.0"
 datafusion-pre-macros = { workspace = true }
 

datafusion/macros/src/lib.rs

@@ -22,41 +22,104 @@ use syn::{parse_macro_input, DeriveInput, LitStr};
 
 #[proc_macro_attribute]
 pub fn udf_doc(args: TokenStream, input: TokenStream) -> TokenStream {
+    let mut doc_section_include: Option<LitStr> = None;
+    let mut doc_section_lbl: Option<LitStr> = None;
+    let mut doc_section_desc: Option<LitStr> = None;
+
     let mut description: Option<LitStr> = None;
-    let mut example: Option<LitStr> = None;
+    let mut syntax_example: Option<LitStr> = None;
+    let mut sql_example: Option<LitStr> = None;
+    let mut standard_args: Vec<(Option<LitStr>, Option<LitStr>)> = vec![];
+    let mut udf_args: Vec<(Option<LitStr>, Option<LitStr>)> = vec![];
 
     let parser = syn::meta::parser(|meta| {
-        if meta.path.is_ident("description") {
+        if meta.path.is_ident("doc_section") {
+            meta.parse_nested_meta(|meta| {
+                //dbg!(meta.path);
+                if meta.path.is_ident("include") {
+                    doc_section_include = meta.value()?.parse()?;
+                    return Ok(());
+                } else if meta.path.is_ident("label") {
+                    doc_section_lbl = meta.value()?.parse()?;
+                    return Ok(());
+                } else if meta.path.is_ident("description") {
+                    doc_section_desc = meta.value()?.parse()?;
+                    return Ok(());
+                }
+                Ok(())
+            })
+        } else if meta.path.is_ident("description") {
             description = Some(meta.value()?.parse()?);
             Ok(())
-        } else if meta.path.is_ident("example") {
-            example = Some(meta.value()?.parse()?);
+        } else if meta.path.is_ident("syntax_example") {
+            syntax_example = Some(meta.value()?.parse()?);
+            Ok(())
+        } else if meta.path.is_ident("sql_example") {
+            sql_example = Some(meta.value()?.parse()?);
             Ok(())
+        } else if meta.path.is_ident("standard_argument") {
+            let mut standard_arg: (Option<LitStr>, Option<LitStr>) = (None, None);
+            meta.parse_nested_meta(|meta| {
+                if meta.path.is_ident("name") {
+                    standard_arg.0 = meta.value()?.parse()?;
+                    return Ok(());
+                } else if meta.path.is_ident("expression_type") {
+                    standard_arg.1 = meta.value()?.parse()?;
+                    return Ok(());
+                }
+                standard_args.push(standard_arg.clone());
+                Ok(())
+            })
+        } else if meta.path.is_ident("argument") {
+            let mut arg: (Option<LitStr>, Option<LitStr>) = (None, None);
+            meta.parse_nested_meta(|meta| {
+                if meta.path.is_ident("name") {
+                    arg.0 = meta.value()?.parse()?;
+                    return Ok(());
+                } else if meta.path.is_ident("description") {
+                    arg.1 = meta.value()?.parse()?;
+                    return Ok(());
+                }
+                udf_args.push(arg.clone());
+                Ok(())
+            })
         } else {
             Err(meta.error("unsupported property"))
         }
     });
     parse_macro_input!(args with parser);
-    eprintln!("description={description:?} example={example:?}");
 
     // Parse the input struct
     let input = parse_macro_input!(input as DeriveInput);
     let name = input.clone().ident;
 
-    //eprintln!("input={input:?}");
+    eprintln!("doc_section_include=cc{doc_section_include:?}cc");
+    let doc_section_include: bool = doc_section_include.unwrap().value().parse().unwrap();
 
     let expanded = quote! {
         #input
 
         use datafusion_pre_macros::DocumentationTest;
+        use datafusion_pre_macros::DocSectionTest;
+        use datafusion_pre_macros::DocumentationBuilderTest;
+
+        static DOCUMENTATION_TEST: OnceLock<DocumentationTest> = OnceLock::new();
 
         impl #name {
-                fn documentation_test(&self) -> Option<DocumentationTest> {
-                    Some(DocumentationTest { description: #description.to_string(), syntax_example: #example.to_string() })
+                fn documentation_test(&self) -> Option<&DocumentationTest> {
+                    Some(DOCUMENTATION_TEST.get_or_init(|| {
+                        DocumentationTest::builder()
+                        .with_doc_section(DocSectionTest { include: #doc_section_include, label: #doc_section_lbl, description: Some("") })
+                        .with_description(#description.to_string())
+                        .with_syntax_example(#syntax_example.to_string())
+                        .build()
+                    }))
             }
         }
     };
 
+    eprintln!("{}", expanded);
+
     // Return the generated code
     TokenStream::from(expanded)
 }