Skip to content

Commit b23e3cd

Browse files
alambalamb
committed
Support EXPLAIN FORMAT <format>
1 parent dcd4568 commit b23e3cd

File tree

11 files changed

+777
-207
lines changed

11 files changed

+777
-207
lines changed

‎datafusion/core/src/execution/session_state.rs

+2
Original file line numberDiff line numberDiff line change
@@ -577,6 +577,7 @@ impl SessionState {
577577
return Ok(LogicalPlan::Explain(Explain {
578578
verbose: e.verbose,
579579
plan: Arc::clone(&e.plan),
580+
explain_format: e.explain_format.clone(),
580581
stringified_plans,
581582
schema: Arc::clone(&e.schema),
582583
logical_optimization_succeeded: false,
@@ -612,6 +613,7 @@ impl SessionState {
612613

613614
Ok(LogicalPlan::Explain(Explain {
614615
verbose: e.verbose,
616+
explain_format: e.explain_format.clone(),
615617
plan,
616618
stringified_plans,
617619
schema: Arc::clone(&e.schema),

‎datafusion/core/src/physical_planner.rs

+210-154
Large diffs are not rendered by default.

‎datafusion/expr/src/logical_plan/builder.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ use crate::{
4646
};
4747

4848
use super::dml::InsertOp;
49-
use super::plan::ColumnUnnestList;
49+
use super::plan::{ColumnUnnestList, ExplainFormat};
5050
use arrow::compute::can_cast_types;
5151
use arrow::datatypes::{DataType, Field, Fields, Schema, SchemaRef};
5252
use datafusion_common::display::ToStringifiedPlan;
@@ -1211,6 +1211,7 @@ impl LogicalPlanBuilder {
12111211
Ok(Self::new(LogicalPlan::Explain(Explain {
12121212
verbose,
12131213
plan: self.plan,
1214+
explain_format: ExplainFormat::Indent,
12141215
stringified_plans,
12151216
schema,
12161217
logical_optimization_succeeded: false,

‎datafusion/expr/src/logical_plan/mod.rs

+3-3
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,9 @@ pub use ddl::{
3838
pub use dml::{DmlStatement, WriteOp};
3939
pub use plan::{
4040
projection_schema, Aggregate, Analyze, ColumnUnnestList, DescribeTable, Distinct,
41-
DistinctOn, EmptyRelation, Explain, Extension, FetchType, Filter, Join,
42-
JoinConstraint, JoinType, Limit, LogicalPlan, Partitioning, PlanType, Projection,
43-
RecursiveQuery, Repartition, SkipType, Sort, StringifiedPlan, Subquery,
41+
DistinctOn, EmptyRelation, Explain, ExplainFormat, Extension, FetchType, Filter,
42+
Join, JoinConstraint, JoinType, Limit, LogicalPlan, Partitioning, PlanType,
43+
Projection, RecursiveQuery, Repartition, SkipType, Sort, StringifiedPlan, Subquery,
4444
SubqueryAlias, TableScan, ToStringifiedPlan, Union, Unnest, Values, Window,
4545
};
4646
pub use statement::{

‎datafusion/expr/src/logical_plan/plan.rs

+157
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ use std::cmp::Ordering;
2121
use std::collections::{BTreeMap, HashMap, HashSet};
2222
use std::fmt::{self, Debug, Display, Formatter};
2323
use std::hash::{Hash, Hasher};
24+
use std::str::FromStr;
2425
use std::sync::{Arc, LazyLock};
2526

2627
use super::dml::CopyTo;
@@ -1084,6 +1085,7 @@ impl LogicalPlan {
10841085
Ok(LogicalPlan::Explain(Explain {
10851086
verbose: e.verbose,
10861087
plan: Arc::new(input),
1088+
explain_format: e.explain_format.clone(),
10871089
stringified_plans: e.stringified_plans.clone(),
10881090
schema: Arc::clone(&e.schema),
10891091
logical_optimization_succeeded: e.logical_optimization_succeeded,
@@ -2924,12 +2926,167 @@ impl PartialOrd for DescribeTable {
29242926
}
29252927
}
29262928

2929+
/// Output formats for controlling for Explain plans
2930+
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
2931+
pub enum ExplainFormat {
2932+
/// Indent mode
2933+
///
2934+
/// Example:
2935+
/// ```text
2936+
/// > explain format indent select x from values (1) t(x);
2937+
/// +---------------+-----------------------------------------------------+
2938+
/// | plan_type | plan |
2939+
/// +---------------+-----------------------------------------------------+
2940+
/// | logical_plan | SubqueryAlias: t |
2941+
/// | | Projection: column1 AS x |
2942+
/// | | Values: (Int64(1)) |
2943+
/// | physical_plan | ProjectionExec: expr=[column1@0 as x] |
2944+
/// | | DataSourceExec: partitions=1, partition_sizes=[1] |
2945+
/// | | |
2946+
/// +---------------+-----------------------------------------------------+
2947+
/// ```
2948+
Indent,
2949+
/// Tree mode
2950+
///
2951+
/// Example:
2952+
/// ```text
2953+
/// > explain format tree select x from values (1) t(x);
2954+
/// +---------------+-------------------------------+
2955+
/// | plan_type | plan |
2956+
/// +---------------+-------------------------------+
2957+
/// | physical_plan | ┌───────────────────────────┐ |
2958+
/// | | │ ProjectionExec │ |
2959+
/// | | │ -------------------- │ |
2960+
/// | | │ x: column1@0 │ |
2961+
/// | | └─────────────┬─────────────┘ |
2962+
/// | | ┌─────────────┴─────────────┐ |
2963+
/// | | │ DataSourceExec │ |
2964+
/// | | │ -------------------- │ |
2965+
/// | | │ bytes: 128 │ |
2966+
/// | | │ format: memory │ |
2967+
/// | | │ rows: 1 │ |
2968+
/// | | └───────────────────────────┘ |
2969+
/// | | |
2970+
/// +---------------+-------------------------------+
2971+
/// ```
2972+
Tree,
2973+
/// Postgres Json mode
2974+
///
2975+
/// A displayable structure that produces plan in postgresql JSON format.
2976+
///
2977+
/// Users can use this format to visualize the plan in existing plan
2978+
/// visualization tools, for example [dalibo](https://explain.dalibo.com/)
2979+
///
2980+
/// Example:
2981+
/// ```text
2982+
/// > explain format pgjson select x from values (1) t(x);
2983+
/// +--------------+--------------------------------------+
2984+
/// | plan_type | plan |
2985+
/// +--------------+--------------------------------------+
2986+
/// | logical_plan | [ |
2987+
/// | | { |
2988+
/// | | "Plan": { |
2989+
/// | | "Alias": "t", |
2990+
/// | | "Node Type": "Subquery", |
2991+
/// | | "Output": [ |
2992+
/// | | "x" |
2993+
/// | | ], |
2994+
/// | | "Plans": [ |
2995+
/// | | { |
2996+
/// | | "Expressions": [ |
2997+
/// | | "column1 AS x" |
2998+
/// | | ], |
2999+
/// | | "Node Type": "Projection", |
3000+
/// | | "Output": [ |
3001+
/// | | "x" |
3002+
/// | | ], |
3003+
/// | | "Plans": [ |
3004+
/// | | { |
3005+
/// | | "Node Type": "Values", |
3006+
/// | | "Output": [ |
3007+
/// | | "column1" |
3008+
/// | | ], |
3009+
/// | | "Plans": [], |
3010+
/// | | "Values": "(Int64(1))" |
3011+
/// | | } |
3012+
/// | | ] |
3013+
/// | | } |
3014+
/// | | ] |
3015+
/// | | } |
3016+
/// | | } |
3017+
/// | | ] |
3018+
/// +--------------+--------------------------------------+
3019+
/// ```
3020+
PostgresJSON,
3021+
/// Graphviz mode
3022+
///
3023+
/// Example:
3024+
/// ```text
3025+
/// > explain format graphviz select x from values (1) t(x);
3026+
/// +--------------+------------------------------------------------------------------------+
3027+
/// | plan_type | plan |
3028+
/// +--------------+------------------------------------------------------------------------+
3029+
/// | logical_plan | |
3030+
/// | | // Begin DataFusion GraphViz Plan, |
3031+
/// | | // display it online here: https://dreampuf.github.io/GraphvizOnline |
3032+
/// | | |
3033+
/// | | digraph { |
3034+
/// | | subgraph cluster_1 |
3035+
/// | | { |
3036+
/// | | graph[label="LogicalPlan"] |
3037+
/// | | 2[shape=box label="SubqueryAlias: t"] |
3038+
/// | | 3[shape=box label="Projection: column1 AS x"] |
3039+
/// | | 2 -> 3 [arrowhead=none, arrowtail=normal, dir=back] |
3040+
/// | | 4[shape=box label="Values: (Int64(1))"] |
3041+
/// | | 3 -> 4 [arrowhead=none, arrowtail=normal, dir=back] |
3042+
/// | | } |
3043+
/// | | subgraph cluster_5 |
3044+
/// | | { |
3045+
/// | | graph[label="Detailed LogicalPlan"] |
3046+
/// | | 6[shape=box label="SubqueryAlias: t\nSchema: [x:Int64;N]"] |
3047+
/// | | 7[shape=box label="Projection: column1 AS x\nSchema: [x:Int64;N]"] |
3048+
/// | | 6 -> 7 [arrowhead=none, arrowtail=normal, dir=back] |
3049+
/// | | 8[shape=box label="Values: (Int64(1))\nSchema: [column1:Int64;N]"] |
3050+
/// | | 7 -> 8 [arrowhead=none, arrowtail=normal, dir=back] |
3051+
/// | | } |
3052+
/// | | } |
3053+
/// | | // End DataFusion GraphViz Plan |
3054+
/// | | |
3055+
/// +--------------+------------------------------------------------------------------------+
3056+
/// ```
3057+
Graphviz,
3058+
}
3059+
3060+
/// Implement parsing strings to `ExplainFormat`
3061+
impl FromStr for ExplainFormat {
3062+
type Err = DataFusionError;
3063+
3064+
fn from_str(format: &str) -> std::result::Result<Self, Self::Err> {
3065+
match format.to_lowercase().as_str() {
3066+
"indent" => Ok(ExplainFormat::Indent),
3067+
"tree" => Ok(ExplainFormat::Tree),
3068+
"pgjson" => Ok(ExplainFormat::PostgresJSON),
3069+
"graphviz" => Ok(ExplainFormat::Graphviz),
3070+
_ => {
3071+
plan_err!("Invalid explain format. Expected 'indent', 'tree', 'pgjson' or 'graphviz'. Got '{format}'")
3072+
}
3073+
}
3074+
}
3075+
}
3076+
29273077
/// Produces a relation with string representations of
29283078
/// various parts of the plan
3079+
///
3080+
/// See [the documentation] for more information
3081+
///
3082+
/// [the documentation]: https://datafusion.apache.org/user-guide/sql/explain.html
29293083
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
29303084
pub struct Explain {
29313085
/// Should extra (detailed, intermediate plans) be included?
29323086
pub verbose: bool,
3087+
/// Output format for explain, if specified.
3088+
/// If none, defaults to `text`
3089+
pub explain_format: ExplainFormat,
29333090
/// The logical plan that is being EXPLAIN'd
29343091
pub plan: Arc<LogicalPlan>,
29353092
/// Represent the various stages plans have gone through

‎datafusion/expr/src/logical_plan/tree_node.rs

+2
Original file line numberDiff line numberDiff line change
@@ -202,13 +202,15 @@ impl TreeNode for LogicalPlan {
202202
.update_data(LogicalPlan::Distinct),
203203
LogicalPlan::Explain(Explain {
204204
verbose,
205+
explain_format: format,
205206
plan,
206207
stringified_plans,
207208
schema,
208209
logical_optimization_succeeded,
209210
}) => plan.map_elements(f)?.update_data(|plan| {
210211
LogicalPlan::Explain(Explain {
211212
verbose,
213+
explain_format: format,
212214
plan,
213215
stringified_plans,
214216
schema,

‎datafusion/physical-plan/src/display.rs

+2-18
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,12 @@
1919
//! [`crate::displayable`] for examples of how to format
2020
2121
use std::collections::{BTreeMap, HashMap};
22+
use std::fmt;
2223
use std::fmt::Formatter;
23-
use std::{fmt, str::FromStr};
2424

2525
use arrow::datatypes::SchemaRef;
2626

2727
use datafusion_common::display::{GraphvizBuilder, PlanType, StringifiedPlan};
28-
use datafusion_common::DataFusionError;
2928
use datafusion_expr::display_schema;
3029
use datafusion_physical_expr::LexOrdering;
3130

@@ -39,7 +38,7 @@ pub enum DisplayFormatType {
3938
/// Default, compact format. Example: `FilterExec: c12 < 10.0`
4039
///
4140
/// This format is designed to provide a detailed textual description
42-
/// of all rele
41+
/// of all parts of the plan.
4342
Default,
4443
/// Verbose, showing all available details.
4544
///
@@ -79,21 +78,6 @@ pub enum DisplayFormatType {
7978
TreeRender,
8079
}
8180

82-
impl FromStr for DisplayFormatType {
83-
type Err = DataFusionError;
84-
85-
fn from_str(s: &str) -> Result<Self, Self::Err> {
86-
match s.to_lowercase().as_str() {
87-
"indent" => Ok(Self::Default),
88-
"tree" => Ok(Self::TreeRender),
89-
_ => Err(DataFusionError::Configuration(format!(
90-
"Invalid explain format: {}",
91-
s
92-
))),
93-
}
94-
}
95-
}
96-
9781
/// Wraps an `ExecutionPlan` with various methods for formatting
9882
///
9983
///

‎datafusion/sql/src/parser.rs

+41-3
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@
1616
// under the License.
1717

1818
//! [`DFParser`]: DataFusion SQL Parser based on [`sqlparser`]
19+
//!
20+
//! This parser implements DataFusion specific statements such as
21+
//! `CREATE EXTERNAL TABLE`
1922
2023
use std::collections::VecDeque;
2124
use std::fmt;
@@ -43,12 +46,23 @@ fn parse_file_type(s: &str) -> Result<String, ParserError> {
4346
Ok(s.to_uppercase())
4447
}
4548

46-
/// DataFusion specific EXPLAIN (needed so we can EXPLAIN datafusion
47-
/// specific COPY and other statements)
49+
/// DataFusion specific `EXPLAIN`
50+
///
51+
/// Syntax:
52+
/// ```sql
53+
/// EXPLAIN <ANALYZE> <VERBOSE> [FORMAT format] statement
54+
///```
4855
#[derive(Debug, Clone, PartialEq, Eq)]
4956
pub struct ExplainStatement {
57+
/// `EXPLAIN ANALYZE ..`
5058
pub analyze: bool,
59+
/// `EXPLAIN .. VERBOSE ..`
5160
pub verbose: bool,
61+
/// `EXPLAIN .. FORMAT `
62+
pub format: Option<String>,
63+
/// The statement to analyze. Note this is a DataFusion [`Statement`] (not a
64+
/// [`sqlparser::ast::Statement`] so that we can `EXPLAIN` `COPY` and other
65+
/// DataFusion specific statements
5266
pub statement: Box<Statement>,
5367
}
5468

@@ -57,6 +71,7 @@ impl fmt::Display for ExplainStatement {
5771
let Self {
5872
analyze,
5973
verbose,
74+
format,
6075
statement,
6176
} = self;
6277

@@ -67,6 +82,9 @@ impl fmt::Display for ExplainStatement {
6782
if *verbose {
6883
write!(f, "VERBOSE ")?;
6984
}
85+
if let Some(format) = format.as_ref() {
86+
write!(f, "FORMAT {format} ")?;
87+
}
7088

7189
write!(f, "{statement}")
7290
}
@@ -446,7 +464,6 @@ impl<'a> DFParser<'a> {
446464
self.parse_copy()
447465
}
448466
Keyword::EXPLAIN => {
449-
// (TODO parse all supported statements)
450467
self.parser.next_token(); // EXPLAIN
451468
self.parse_explain()
452469
}
@@ -620,15 +637,35 @@ impl<'a> DFParser<'a> {
620637
pub fn parse_explain(&mut self) -> Result<Statement, ParserError> {
621638
let analyze = self.parser.parse_keyword(Keyword::ANALYZE);
622639
let verbose = self.parser.parse_keyword(Keyword::VERBOSE);
640+
let format = self.parse_explain_format()?;
641+
623642
let statement = self.parse_statement()?;
624643

625644
Ok(Statement::Explain(ExplainStatement {
626645
statement: Box::new(statement),
627646
analyze,
628647
verbose,
648+
format,
629649
}))
630650
}
631651

652+
pub fn parse_explain_format(&mut self) -> Result<Option<String>, ParserError> {
653+
if !self.parser.parse_keyword(Keyword::FORMAT) {
654+
return Ok(None);
655+
}
656+
657+
let next_token = self.parser.next_token();
658+
let format = match next_token.token {
659+
Token::Word(w) => Ok(w.value),
660+
Token::SingleQuotedString(w) => Ok(w),
661+
Token::DoubleQuotedString(w) => Ok(w),
662+
_ => self
663+
.parser
664+
.expected("an explain format such as TREE", next_token),
665+
}?;
666+
Ok(Some(format))
667+
}
668+
632669
/// Parse a SQL `CREATE` statement handling `CREATE EXTERNAL TABLE`
633670
pub fn parse_create(&mut self) -> Result<Statement, ParserError> {
634671
if self.parser.parse_keyword(Keyword::EXTERNAL) {
@@ -1543,6 +1580,7 @@ mod tests {
15431580
let expected = Statement::Explain(ExplainStatement {
15441581
analyze,
15451582
verbose,
1583+
format: None,
15461584
statement: Box::new(expected_copy),
15471585
});
15481586
assert_eq!(verified_stmt(sql), expected);

0 commit comments

Comments
 (0)