feat: Added Special Functions Documentation Page

datafusion/core/src/bin/print_functions_docs.rs

@@ -17,8 +17,8 @@
 
 use datafusion::execution::SessionStateDefaults;
 use datafusion_expr::{
-    aggregate_doc_sections, scalar_doc_sections, window_doc_sections, AggregateUDF,
-    DocSection, Documentation, ScalarUDF, WindowUDF,
+    aggregate_doc_sections, scalar_doc_sections, special_doc_sections,
+    window_doc_sections, AggregateUDF, DocSection, Documentation, ScalarUDF, WindowUDF,
 };
 use hashbrown::HashSet;
 use itertools::Itertools;
@@ -35,7 +35,7 @@ fn main() {
 
     if args.len() != 2 {
         panic!(
-            "Usage: {} type (one of 'aggregate', 'scalar', 'window')",
+            "Usage: {} type (one of 'aggregate', 'scalar', 'special', 'window')",
             args[0]
         );
     }
@@ -44,6 +44,7 @@ fn main() {
     let docs = match function_type.as_str() {
         "aggregate" => print_aggregate_docs(),
         "scalar" => print_scalar_docs(),
+        "special" => print_special_docs(),
         "window" => print_window_docs(),
         _ => {
             panic!("Unknown function type: {}", function_type)
@@ -73,6 +74,17 @@ fn print_scalar_docs() -> String {
     print_docs(providers, scalar_doc_sections::doc_sections())
 }
 
+fn print_special_docs() -> String {
+    let mut providers: Vec<Box<dyn DocProvider>> = vec![];
+
+    // Iterates through the default_scalar_functions to retrieve the special functions
+    for f in SessionStateDefaults::default_scalar_functions() {
+        providers.push(Box::new(f.as_ref().clone()));
+    }
+
+    print_docs(providers, special_doc_sections::doc_sections())
+}
+
 fn print_window_docs() -> String {
     let mut providers: Vec<Box<dyn DocProvider>> = vec![];
 

datafusion/expr/src/lib.rs

@@ -35,6 +35,7 @@ mod table_source;
 mod udaf;
 mod udf;
 mod udf_docs;
+mod udsf;
 mod udwf;
 
 pub mod conditional_expressions;
@@ -96,6 +97,7 @@ pub use udaf::{
 };
 pub use udf::{scalar_doc_sections, ScalarUDF, ScalarUDFImpl};
 pub use udf_docs::{DocSection, Documentation, DocumentationBuilder};
+pub use udsf::special_doc_sections;
 pub use udwf::{window_doc_sections, ReversedUDWF, WindowUDF, WindowUDFImpl};
 pub use window_frame::{WindowFrame, WindowFrameBound, WindowFrameUnits};
 

datafusion/expr/src/udf.rs

@@ -767,7 +767,6 @@ pub mod scalar_doc_sections {
             DOC_SECTION_DATETIME,
             DOC_SECTION_ARRAY,
             DOC_SECTION_STRUCT,
-            DOC_SECTION_MAP,
             DOC_SECTION_HASHING,
             DOC_SECTION_OTHER,
         ]
@@ -826,12 +825,6 @@ The following regular expression functions are supported:"#,
         description: None,
     };
 
-    pub const DOC_SECTION_MAP: DocSection = DocSection {
-        include: true,
-        label: "Map Functions",
-        description: None,
-    };
-
     pub const DOC_SECTION_HASHING: DocSection = DocSection {
         include: true,
         label: "Hashing Functions",

datafusion/expr/src/udf_docs.rs

@@ -147,23 +147,29 @@ impl DocumentationBuilder {
 
     /// Add a standard "expression" argument to the documentation
     ///
-    /// This is similar to  [`Self::with_argument`] except that  a standard
-    /// description is appended to the end: `"Can be a constant, column, or
-    /// function, and any combination of arithmetic operators."`
-    ///
-    /// The argument is rendered like
+    /// The argument is rendered like below if Some() is passed through:
     ///
     /// ```text
     /// <arg_name>:
     ///   <expression_type> expression to operate on. Can be a constant, column, or function, and any combination of operators.
     /// ```
+    ///
+    /// The argument is rendered like below if None is passed through:
+    ///
+    ///  ```text
+    /// <arg_name>:
+    ///   The expression to operate on. Can be a constant, column, or function, and any combination of operators.
+    /// ```
     pub fn with_standard_argument(
         self,
         arg_name: impl Into<String>,
-        expression_type: impl AsRef<str>,
+        expression_type: Option<&str>,
     ) -> Self {
-        let expression_type = expression_type.as_ref();
-        self.with_argument(arg_name, format!("{expression_type} expression to operate on. Can be a constant, column, or function, and any combination of operators."))
+        let description = format!(
+            "{} expression to operate on. Can be a constant, column, or function, and any combination of operators.",
+            expression_type.unwrap_or("The")
+        );
+        self.with_argument(arg_name, description)
     }
 
     pub fn with_related_udf(mut self, related_udf: impl Into<String>) -> Self {

datafusion/expr/src/udsf.rs

@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! [`SpecialUDF`]: Special User Defined Functions
+
+pub mod special_doc_sections {
+    use crate::DocSection;
+
+    pub fn doc_sections() -> Vec<DocSection> {
+        vec![DOC_SECTION_MAP]
+    }
+
+    pub const DOC_SECTION_MAP: DocSection = DocSection {
+        include: true,
+        label: "Map Functions",
+        description: None,
+    };
+}

datafusion/functions-aggregate/src/approx_distinct.rs

@@ -332,7 +332,7 @@ fn get_approx_distinct_doc() -> &'static Documentation {
 +-----------------------------------+
 ```"#, 
             )
-            .with_standard_argument("expression", "The")
+            .with_standard_argument("expression", None)
             .build()
             .unwrap()
     })

datafusion/functions-aggregate/src/approx_median.rs

@@ -145,7 +145,7 @@ fn get_approx_median_doc() -> &'static Documentation {
 +-----------------------------------+
 ```"#,
             )
-            .with_standard_argument("expression", "The")
+            .with_standard_argument("expression", None)
             .build()
             .unwrap()
     })

datafusion/functions-aggregate/src/approx_percentile_cont.rs

@@ -293,7 +293,7 @@ fn get_approx_percentile_cont_doc() -> &'static Documentation {
 | 65.0                                            |
 +-------------------------------------------------+
 ```"#)
-            .with_standard_argument("expression", "The")
+            .with_standard_argument("expression", None)
             .with_argument("percentile", "Percentile to compute. Must be a float value between 0 and 1 (inclusive).")
             .with_argument("centroids", "Number of centroids to use in the t-digest algorithm. _Default is 100_. A higher number results in more accurate approximation but requires more memory.")
             .build()

datafusion/functions-aggregate/src/approx_percentile_cont_with_weight.rs

@@ -179,7 +179,7 @@ fn get_approx_percentile_cont_with_weight_doc() -> &'static Documentation {
 +----------------------------------------------------------------------+
 ```"#,
             )
-            .with_standard_argument("expression", "The")
+            .with_standard_argument("expression", None)
             .with_argument("weight", "Expression to use as weight. Can be a constant, column, or function, and any combination of arithmetic operators.")
             .with_argument("percentile", "Percentile to compute. Must be a float value between 0 and 1 (inclusive).")
             .build()

datafusion/functions-aggregate/src/array_agg.rs

@@ -168,7 +168,7 @@ fn get_array_agg_doc() -> &'static Documentation {
 +-----------------------------------------------+
 ```"#, 
             )
-            .with_standard_argument("expression", "The")
+            .with_standard_argument("expression", None)
             .build()
             .unwrap()
     })

datafusion/functions-aggregate/src/average.rs

@@ -263,7 +263,7 @@ fn get_avg_doc() -> &'static Documentation {
 +---------------------------+
 ```"#,
             )
-            .with_standard_argument("expression", "The")
+            .with_standard_argument("expression", None)
             .build()
             .unwrap()
     })

datafusion/functions-aggregate/src/bit_and_or_xor.rs

@@ -142,7 +142,7 @@ fn get_bit_and_doc() -> &'static Documentation {
             .with_doc_section(DOC_SECTION_GENERAL)
             .with_description("Computes the bitwise AND of all non-null input values.")
             .with_syntax_example("bit_and(expression)")
-            .with_standard_argument("expression", "Integer")
+            .with_standard_argument("expression", Some("Integer"))
             .build()
             .unwrap()
     })
@@ -156,7 +156,7 @@ fn get_bit_or_doc() -> &'static Documentation {
             .with_doc_section(DOC_SECTION_GENERAL)
             .with_description("Computes the bitwise OR of all non-null input values.")
             .with_syntax_example("bit_or(expression)")
-            .with_standard_argument("expression", "Integer")
+            .with_standard_argument("expression", Some("Integer"))
             .build()
             .unwrap()
     })
@@ -172,7 +172,7 @@ fn get_bit_xor_doc() -> &'static Documentation {
                 "Computes the bitwise exclusive OR of all non-null input values.",
             )
             .with_syntax_example("bit_xor(expression)")
-            .with_standard_argument("expression", "Integer")
+            .with_standard_argument("expression", Some("Integer"))
             .build()
             .unwrap()
     })

datafusion/functions-aggregate/src/bool_and_or.rs

@@ -201,7 +201,7 @@ fn get_bool_and_doc() -> &'static Documentation {
 +----------------------------+
 ```"#,
             )
-            .with_standard_argument("expression", "The")
+            .with_standard_argument("expression", None)
             .build()
             .unwrap()
     })
@@ -350,7 +350,7 @@ fn get_bool_or_doc() -> &'static Documentation {
 +----------------------------+
 ```"#,
             )
-            .with_standard_argument("expression", "The")
+            .with_standard_argument("expression", None)
             .build()
             .unwrap()
     })

datafusion/functions-aggregate/src/correlation.rs

@@ -134,8 +134,8 @@ fn get_corr_doc() -> &'static Documentation {
 +--------------------------------+
 ```"#,
             )
-            .with_standard_argument("expression1", "First")
-            .with_standard_argument("expression2", "Second")
+            .with_standard_argument("expression1", Some("First"))
+            .with_standard_argument("expression2", Some("Second"))
             .build()
             .unwrap()
     })

datafusion/functions-aggregate/src/count.rs

@@ -357,7 +357,7 @@ fn get_count_doc() -> &'static Documentation {
 | 120              |
 +------------------+
 ```"#)
-            .with_standard_argument("expression", "The")
+            .with_standard_argument("expression", None)
             .build()
             .unwrap()
     })

datafusion/functions-aggregate/src/covariance.rs

@@ -150,8 +150,8 @@ fn get_covar_samp_doc() -> &'static Documentation {
 +-----------------------------------+
 ```"#,
             )
-            .with_standard_argument("expression1", "First")
-            .with_standard_argument("expression2", "Second")
+            .with_standard_argument("expression1", Some("First"))
+            .with_standard_argument("expression2", Some("Second"))
             .build()
             .unwrap()
     })
@@ -248,8 +248,8 @@ fn get_covar_pop_doc() -> &'static Documentation {
 +-----------------------------------+
 ```"#,
             )
-            .with_standard_argument("expression1", "First")
-            .with_standard_argument("expression2", "Second")
+            .with_standard_argument("expression1", Some("First"))
+            .with_standard_argument("expression2", Some("Second"))
             .build()
             .unwrap()
     })

datafusion/functions-aggregate/src/first_last.rs

@@ -191,7 +191,7 @@ fn get_first_value_doc() -> &'static Documentation {
 +-----------------------------------------------+
 ```"#,
             )
-            .with_standard_argument("expression", "The")
+            .with_standard_argument("expression", None)
             .build()
             .unwrap()
     })
@@ -519,7 +519,7 @@ fn get_last_value_doc() -> &'static Documentation {
 +-----------------------------------------------+
 ```"#,
             )
-            .with_standard_argument("expression", "The")
+            .with_standard_argument("expression", None)
             .build()
             .unwrap()
     })

datafusion/functions-aggregate/src/median.rs

@@ -177,7 +177,7 @@ fn get_median_doc() -> &'static Documentation {
 +----------------------+
 ```"#,
             )
-            .with_standard_argument("expression", "The")
+            .with_standard_argument("expression", None)
             .build()
             .unwrap()
     })

datafusion/functions-aggregate/src/min_max.rs

@@ -357,7 +357,7 @@ fn get_max_doc() -> &'static Documentation {
 +----------------------+
 ```"#,
             )
-            .with_standard_argument("expression", "The")
+            .with_standard_argument("expression", None)
             .build()
             .unwrap()
     })
@@ -1187,7 +1187,7 @@ fn get_min_doc() -> &'static Documentation {
 +----------------------+
 ```"#,
             )
-            .with_standard_argument("expression", "The")
+            .with_standard_argument("expression", None)
             .build()
             .unwrap()
     })

datafusion/functions-aggregate/src/nth_value.rs

@@ -191,7 +191,7 @@ fn get_nth_value_doc() -> &'static Documentation {
 | 2       | 45000  | 45000                   |
 +---------+--------+-------------------------+
 ```"#)
-            .with_standard_argument("expression", "The column or expression to retrieve the nth value from.")
+            .with_argument("expression", "The column or expression to retrieve the nth value from.")
             .with_argument("n", "The position (nth) of the value to retrieve, based on the ordering.")
             .build()
             .unwrap()

datafusion/functions-aggregate/src/stddev.rs

@@ -158,7 +158,7 @@ fn get_stddev_doc() -> &'static Documentation {
 +----------------------+
 ```"#,
             )
-            .with_standard_argument("expression", "The")
+            .with_standard_argument("expression", None)
             .build()
             .unwrap()
     })
@@ -282,7 +282,7 @@ fn get_stddev_pop_doc() -> &'static Documentation {
 +--------------------------+
 ```"#,
             )
-            .with_standard_argument("expression", "The")
+            .with_standard_argument("expression", None)
             .build()
             .unwrap()
     })

datafusion/functions-aggregate/src/sum.rs

@@ -260,7 +260,7 @@ fn get_sum_doc() -> &'static Documentation {
 +-----------------------+
 ```"#,
             )
-            .with_standard_argument("expression", "The")
+            .with_standard_argument("expression", None)
             .build()
             .unwrap()
     })

datafusion/functions-aggregate/src/variance.rs

@@ -153,7 +153,7 @@ fn get_variance_sample_doc() -> &'static Documentation {
                 "Returns the statistical sample variance of a set of numbers.",
             )
             .with_syntax_example("var(expression)")
-            .with_standard_argument("expression", "Numeric")
+            .with_standard_argument("expression", Some("Numeric"))
             .build()
             .unwrap()
     })
@@ -259,7 +259,7 @@ fn get_variance_population_doc() -> &'static Documentation {
                 "Returns the statistical population variance of a set of numbers.",
             )
             .with_syntax_example("var_pop(expression)")
-            .with_standard_argument("expression", "Numeric")
+            .with_standard_argument("expression", Some("Numeric"))
             .build()
             .unwrap()
     })