wip

Cargo.toml

@@ -38,7 +38,7 @@ arrow-buffer = { version = "43.0.0", default-features = false }
 arrow-flight = { version = "43.0.0", features = ["flight-sql-experimental"] }
 arrow-schema = { version = "43.0.0", default-features = false }
 parquet = { version = "43.0.0", features = ["arrow", "async", "object_store"] }
-sqlparser = { version = "0.35", features = ["visitor"] }
+sqlparser = { git = "https://github.com/chunshao90/sqlparser-rs", rev = "9de47d3a1686d39a1fe8613fc24264ca943676d2" , features=["visitor"]}
 
 [profile.release]
 codegen-units = 1

datafusion/core/tests/sql/order.rs

@@ -48,7 +48,9 @@ async fn sort_with_lots_of_repetition_values() -> Result<()> {
 async fn create_external_table_with_order() -> Result<()> {
     let ctx = SessionContext::new();
     let sql = "CREATE EXTERNAL TABLE dt (a_id integer, a_str string, a_bool boolean) STORED AS CSV WITH ORDER (a_id ASC) LOCATION 'file://path/to/table';";
-    let LogicalPlan::Ddl(DdlStatement::CreateExternalTable(cmd)) = ctx.state().create_logical_plan(sql).await? else {
+    let LogicalPlan::Ddl(DdlStatement::CreateExternalTable(cmd)) =
+        ctx.state().create_logical_plan(sql).await?
+    else {
         panic!("Wrong command")
     };
 

datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs

@@ -412,7 +412,9 @@ impl<'a, S: SimplifyInfo> TreeNodeRewriter for Simplifier<'a, S> {
             }) if list.len() == 1
                 && matches!(list.first(), Some(Expr::ScalarSubquery { .. })) =>
             {
-                let Expr::ScalarSubquery(subquery) = list.remove(0) else { unreachable!() };
+                let Expr::ScalarSubquery(subquery) = list.remove(0) else {
+                    unreachable!()
+                };
                 Expr::InSubquery(InSubquery::new(expr, subquery, negated))
             }
 

datafusion/optimizer/src/simplify_expressions/regex.rs

@@ -203,7 +203,9 @@ fn anchored_literal_to_expr(v: &[Hir]) -> Option<Expr> {
     match v.len() {
         2 => Some(lit("")),
         3 => {
-            let HirKind::Literal(l) = v[1].kind() else { return None };
+            let HirKind::Literal(l) = v[1].kind() else {
+                return None;
+            };
             like_str_from_literal(l).map(lit)
         }
         _ => None,

datafusion/sql/src/expr/mod.rs

@@ -288,6 +288,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
                 expr,
                 substring_from,
                 substring_for,
+               ..
             } => self.sql_substring_to_expr(expr, substring_from, substring_for, schema, planner_context),
 
             #[cfg(not(feature = "unicode_expressions"))]

datafusion/sql/src/expr/substring.rs

@@ -60,6 +60,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
                     expr,
                     substring_from: None,
                     substring_for: None,
+                    special: false,
                 };
 
                 return Err(DataFusionError::Plan(format!(

datafusion/sql/src/planner.rs

@@ -306,20 +306,20 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
 
     fn convert_simple_data_type(&self, sql_type: &SQLDataType) -> Result<DataType> {
         match sql_type {
-            SQLDataType::Boolean => Ok(DataType::Boolean),
-            SQLDataType::TinyInt(_) => Ok(DataType::Int8),
+            SQLDataType::Boolean|SQLDataType::Bool => Ok(DataType::Boolean),
+            SQLDataType::TinyInt(_)|SQLDataType::Int2(_)|SQLDataType::Int4(_)|SQLDataType::Int8(_) => Ok(DataType::Int8),
             SQLDataType::SmallInt(_) => Ok(DataType::Int16),
             SQLDataType::Int(_) | SQLDataType::Integer(_) => Ok(DataType::Int32),
             SQLDataType::BigInt(_) => Ok(DataType::Int64),
-            SQLDataType::UnsignedTinyInt(_) => Ok(DataType::UInt8),
+            SQLDataType::UnsignedTinyInt(_)|SQLDataType::UnsignedInt2(_)|SQLDataType::UnsignedInt4(_)|SQLDataType::UnsignedInt8(_) => Ok(DataType::UInt8),
             SQLDataType::UnsignedSmallInt(_) => Ok(DataType::UInt16),
             SQLDataType::UnsignedInt(_) | SQLDataType::UnsignedInteger(_) => {
                 Ok(DataType::UInt32)
             }
             SQLDataType::UnsignedBigInt(_) => Ok(DataType::UInt64),
-            SQLDataType::Float(_) => Ok(DataType::Float32),
+            SQLDataType::Float(_)|SQLDataType::Float4 => Ok(DataType::Float32),
             SQLDataType::Real => Ok(DataType::Float32),
-            SQLDataType::Double | SQLDataType::DoublePrecision => Ok(DataType::Float64),
+            SQLDataType::Double | SQLDataType::DoublePrecision |SQLDataType::Float8=> Ok(DataType::Float64),
             SQLDataType::Char(_)
             | SQLDataType::Varchar(_)
             | SQLDataType::Text

datafusion/sql/src/set_expr.rs

@@ -36,8 +36,10 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
                 set_quantifier,
             } => {
                 let all = match set_quantifier {
-                    SetQuantifier::All => true,
-                    SetQuantifier::Distinct | SetQuantifier::None => false,
+                    SetQuantifier::All | SetQuantifier::AllByName => true,
+                    SetQuantifier::Distinct
+                    | SetQuantifier::None
+                    | SetQuantifier::ByName => false,
                 };
 
                 let left_plan = self.set_expr_to_plan(*left, planner_context)?;

datafusion/sql/src/statement.rs

@@ -248,6 +248,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
                 cascade,
                 restrict: _,
                 purge: _,
+                temporary: _,
             } => {
                 // We don't support cascade and purge for now.
                 // nor do we support multiple object names
@@ -430,7 +431,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
                 self.delete_to_plan(table_name, selection)
             }
 
-            Statement::StartTransaction { modes } => {
+            Statement::StartTransaction { modes, .. } => {
                 let isolation_level: ast::TransactionIsolationLevel = modes
                     .iter()
                     .filter_map(|m: &ast::TransactionMode| match m {
@@ -514,10 +515,10 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
                 "DELETE FROM only supports single table, got: joins".to_string(),
             ));
         }
-        let TableFactor::Table{name, ..} = table_factor.relation else {
+        let TableFactor::Table { name, .. } = table_factor.relation else {
             return Err(DataFusionError::NotImplemented(format!(
                 "DELETE FROM only supports single table, got: {table_factor:?}"
-            )))
+            )));
         };
 
         Ok(name)

datafusion/substrait/src/logical_plan/consumer.rs

@@ -560,9 +560,9 @@ pub async fn from_substrait_sorts(
             Some(k) => match k {
                 Direction(d) => {
                     let Some(direction) = SortDirection::from_i32(*d) else {
-                        return Err(DataFusionError::NotImplemented(
-                            format!("Unsupported Substrait SortDirection value {d}"),
-                        ))
+                        return Err(DataFusionError::NotImplemented(format!(
+                            "Unsupported Substrait SortDirection value {d}"
+                        )));
                     };
 
                     match direction {
@@ -1300,27 +1300,27 @@ async fn make_datafusion_like(
     }
 
     let Some(ArgType::Value(expr_substrait)) = &f.arguments[0].arg_type else {
-        return Err(DataFusionError::NotImplemented(
-            format!("Invalid arguments type for `{fn_name}` expr")
-        ))
+        return Err(DataFusionError::NotImplemented(format!(
+            "Invalid arguments type for `{fn_name}` expr"
+        )));
     };
     let expr = from_substrait_rex(expr_substrait, input_schema, extensions)
         .await?
         .as_ref()
         .clone();
     let Some(ArgType::Value(pattern_substrait)) = &f.arguments[1].arg_type else {
-        return Err(DataFusionError::NotImplemented(
-            format!("Invalid arguments type for `{fn_name}` expr")
-        ))
+        return Err(DataFusionError::NotImplemented(format!(
+            "Invalid arguments type for `{fn_name}` expr"
+        )));
     };
     let pattern = from_substrait_rex(pattern_substrait, input_schema, extensions)
         .await?
         .as_ref()
         .clone();
     let Some(ArgType::Value(escape_char_substrait)) = &f.arguments[2].arg_type else {
-        return Err(DataFusionError::NotImplemented(
-            format!("Invalid arguments type for `{fn_name}` expr")
-        ))
+        return Err(DataFusionError::NotImplemented(format!(
+            "Invalid arguments type for `{fn_name}` expr"
+        )));
     };
     let escape_char_expr =
         from_substrait_rex(escape_char_substrait, input_schema, extensions)
@@ -1330,7 +1330,7 @@ async fn make_datafusion_like(
     let Expr::Literal(ScalarValue::Utf8(escape_char)) = escape_char_expr else {
         return Err(DataFusionError::Substrait(format!(
             "Expect Utf8 literal for escape char, but found {escape_char_expr:?}",
-        )))
+        )));
     };
 
     Ok(Arc::new(Expr::Like(Like {

datafusion/substrait/src/logical_plan/producer.rs

@@ -1644,7 +1644,10 @@ mod test {
         println!("Checking round trip of {scalar:?}");
 
         let substrait = to_substrait_literal(&scalar)?;
-        let Expression { rex_type: Some(RexType::Literal(substrait_literal)) } = substrait else {
+        let Expression {
+            rex_type: Some(RexType::Literal(substrait_literal)),
+        } = substrait
+        else {
             panic!("Expected Literal expression, got {substrait:?}");
         };