implement nested identifier access (#12614)

* adding struct_extract function

* changing logics

* Revert "adding struct_extract function"

This reverts commit 00a12bc.

* fix clippy

* optimize

Author: Lordworms

datafusion/functions/src/core/planner.rs

@@ -17,7 +17,7 @@
 
 use arrow::datatypes::Field;
 use datafusion_common::Result;
-use datafusion_common::{not_impl_err, Column, DFSchema, ScalarValue, TableReference};
+use datafusion_common::{Column, DFSchema, ScalarValue, TableReference};
 use datafusion_expr::expr::ScalarFunction;
 use datafusion_expr::planner::{ExprPlanner, PlannerResult, RawDictionaryExpr};
 use datafusion_expr::{lit, Expr};
@@ -70,19 +70,20 @@ impl ExprPlanner for CoreFunctionPlanner {
         qualifier: Option<&TableReference>,
         nested_names: &[String],
     ) -> Result<PlannerResult<Vec<Expr>>> {
-        // TODO: remove when can support multiple nested identifiers
-        if nested_names.len() > 1 {
-            return not_impl_err!(
-                "Nested identifiers not yet supported for column {}",
-                Column::from((qualifier, field)).quoted_flat_name()
-            );
+        let col = Expr::Column(Column::from((qualifier, field)));
+
+        // Start with the base column expression
+        let mut expr = col;
+
+        // Iterate over nested_names and create nested get_field expressions
+        for nested_name in nested_names {
+            let get_field_args = vec![expr, lit(ScalarValue::from(nested_name.clone()))];
+            expr = Expr::ScalarFunction(ScalarFunction::new_udf(
+                crate::core::get_field(),
+                get_field_args,
+            ));
         }
-        let nested_name = nested_names[0].to_string();
 
-        let col = Expr::Column(Column::from((qualifier, field)));
-        let get_field_args = vec![col, lit(ScalarValue::from(nested_name))];
-        Ok(PlannerResult::Planned(Expr::ScalarFunction(
-            ScalarFunction::new_udf(crate::core::get_field(), get_field_args),
-        )))
+        Ok(PlannerResult::Planned(expr))
     }
 }

datafusion/sql/src/expr/identifier.rs

@@ -19,8 +19,8 @@ use arrow_schema::Field;
 use sqlparser::ast::{Expr as SQLExpr, Ident};
 
 use datafusion_common::{
-    internal_err, not_impl_err, plan_datafusion_err, Column, DFSchema, DataFusionError,
-    Result, TableReference,
+    internal_err, not_impl_err, plan_datafusion_err, plan_err, Column, DFSchema,
+    DataFusionError, Result, TableReference,
 };
 use datafusion_expr::planner::PlannerResult;
 use datafusion_expr::{Case, Expr};
@@ -113,13 +113,6 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
                 .map(|id| self.ident_normalizer.normalize(id))
                 .collect::<Vec<_>>();
 
-            // Currently not supporting more than one nested level
-            // Though ideally once that support is in place, this code should work with it
-            // TODO: remove when can support multiple nested identifiers
-            if ids.len() > 5 {
-                return not_impl_err!("Compound identifier: {ids:?}");
-            }
-
             let search_result = search_dfschema(&ids, schema);
             match search_result {
                 // found matching field with spare identifier(s) for nested field(s) in structure
@@ -142,9 +135,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
                             }
                         }
                     }
-                    not_impl_err!(
-                        "Compound identifiers not supported by ExprPlanner: {ids:?}"
-                    )
+                    plan_err!("could not parse compound identifier from {ids:?}")
                 }
                 // found matching field with no spare identifier(s)
                 Some((field, qualifier, _nested_names)) => {

datafusion/sqllogictest/test_files/struct.slt

@@ -282,3 +282,94 @@ drop table values;
 
 statement ok
 drop table struct_values;
+
+statement ok
+CREATE OR REPLACE VIEW complex_view AS 
+SELECT {
+    'user': {
+        'info': {
+            'personal': {
+                'name': 'John Doe',
+                'age': 30,
+                'email': '[email protected]'
+            },
+            'address': {
+                'street': '123 Main St',
+                'city': 'Anytown',
+                'country': 'Countryland',
+                'coordinates': [40.7128, -74.0060]
+            }
+        },
+        'preferences': {
+            'theme': 'dark',
+            'notifications': true,
+            'languages': ['en', 'es', 'fr']
+        },
+        'stats': {
+            'logins': 42,
+            'last_active': '2023-09-15',
+            'scores': [85, 92, 78, 95],
+            'achievements': {
+                'badges': ['early_bird', 'top_contributor'],
+                'levels': {
+                    'beginner': true,
+                    'intermediate': true,
+                    'advanced': false
+                }
+            }
+        }
+    },
+    'metadata': {
+        'version': '1.0',
+        'created_at': '2023-09-01T12:00:00Z'
+    },
+    'deep_nested': {
+        'level1': {
+            'level2': {
+                'level3': {
+                    'level4': {
+                        'level5': {
+                            'level6': {
+                                'level7': {
+                                    'level8': {
+                                        'level9': {
+                                            'level10': 'You reached the bottom!'
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+} AS complex_data;
+
+query T
+SELECT complex_data.user.info.personal.name FROM complex_view;
+----
+John Doe
+
+query I
+SELECT complex_data.user.info.personal.age FROM complex_view;
+----
+30
+
+query T
+SELECT complex_data.user.info.address.city FROM complex_view;
+----
+Anytown
+
+query T
+SELECT complex_data.user.preferences.languages[2] FROM complex_view;
+----
+es
+
+query T
+SELECT complex_data.deep_nested.level1.level2.level3.level4.level5.level6.level7.level8.level9.level10 FROM complex_view;
+----
+You reached the bottom!
+
+statement ok
+drop view complex_view;