Skip to content

Commit e8c0aaa

Browse files
RachelintRachelint
committed
Merge branch 'main' into vectorize-append-value
2 parents 7a1ed90 + 89e96b4 commit e8c0aaa

File tree

239 files changed

+6460
-1768
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

239 files changed

+6460
-1768
lines changed

‎CHANGELOG.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
under the License.
1818
-->
1919

20-
* [DataFusion CHANGELOG](./datafusion/CHANGELOG.md)
20+
Change logs for each release can be found [here](dev/changelog).
21+
2122

2223
For older versions, see [apache/arrow/CHANGELOG.md](https://github.com/apache/arrow/blob/master/CHANGELOG.md).

‎Cargo.toml

+26-24
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ members = [
2626
"datafusion/expr",
2727
"datafusion/expr-common",
2828
"datafusion/execution",
29+
"datafusion/ffi",
2930
"datafusion/functions",
3031
"datafusion/functions-aggregate",
3132
"datafusion/functions-aggregate-common",
@@ -59,7 +60,7 @@ license = "Apache-2.0"
5960
readme = "README.md"
6061
repository = "https://github.com/apache/datafusion"
6162
rust-version = "1.79"
62-
version = "42.1.0"
63+
version = "42.2.0"
6364

6465
[workspace.dependencies]
6566
# We turn off default-features for some dependencies here so the workspaces which inherit them can
@@ -92,29 +93,30 @@ bytes = "1.4"
9293
chrono = { version = "0.4.38", default-features = false }
9394
ctor = "0.2.0"
9495
dashmap = "6.0.1"
95-
datafusion = { path = "datafusion/core", version = "42.1.0", default-features = false }
96-
datafusion-catalog = { path = "datafusion/catalog", version = "42.1.0" }
97-
datafusion-common = { path = "datafusion/common", version = "42.1.0", default-features = false }
98-
datafusion-common-runtime = { path = "datafusion/common-runtime", version = "42.1.0" }
99-
datafusion-execution = { path = "datafusion/execution", version = "42.1.0" }
100-
datafusion-expr = { path = "datafusion/expr", version = "42.1.0" }
101-
datafusion-expr-common = { path = "datafusion/expr-common", version = "42.1.0" }
102-
datafusion-functions = { path = "datafusion/functions", version = "42.1.0" }
103-
datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "42.1.0" }
104-
datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "42.1.0" }
105-
datafusion-functions-nested = { path = "datafusion/functions-nested", version = "42.1.0" }
106-
datafusion-functions-window = { path = "datafusion/functions-window", version = "42.1.0" }
107-
datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "42.1.0" }
108-
datafusion-optimizer = { path = "datafusion/optimizer", version = "42.1.0", default-features = false }
109-
datafusion-physical-expr = { path = "datafusion/physical-expr", version = "42.1.0", default-features = false }
110-
datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "42.1.0", default-features = false }
111-
datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "42.1.0" }
112-
datafusion-physical-plan = { path = "datafusion/physical-plan", version = "42.1.0" }
113-
datafusion-proto = { path = "datafusion/proto", version = "42.1.0" }
114-
datafusion-proto-common = { path = "datafusion/proto-common", version = "42.1.0" }
115-
datafusion-sql = { path = "datafusion/sql", version = "42.1.0" }
116-
datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "42.1.0" }
117-
datafusion-substrait = { path = "datafusion/substrait", version = "42.1.0" }
96+
datafusion = { path = "datafusion/core", version = "42.2.0", default-features = false }
97+
datafusion-catalog = { path = "datafusion/catalog", version = "42.2.0" }
98+
datafusion-common = { path = "datafusion/common", version = "42.2.0", default-features = false }
99+
datafusion-common-runtime = { path = "datafusion/common-runtime", version = "42.2.0" }
100+
datafusion-execution = { path = "datafusion/execution", version = "42.2.0" }
101+
datafusion-expr = { path = "datafusion/expr", version = "42.2.0" }
102+
datafusion-expr-common = { path = "datafusion/expr-common", version = "42.2.0" }
103+
datafusion-ffi = { path = "datafusion/ffi", version = "42.2.0" }
104+
datafusion-functions = { path = "datafusion/functions", version = "42.2.0" }
105+
datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "42.2.0" }
106+
datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "42.2.0" }
107+
datafusion-functions-nested = { path = "datafusion/functions-nested", version = "42.2.0" }
108+
datafusion-functions-window = { path = "datafusion/functions-window", version = "42.2.0" }
109+
datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "42.2.0" }
110+
datafusion-optimizer = { path = "datafusion/optimizer", version = "42.2.0", default-features = false }
111+
datafusion-physical-expr = { path = "datafusion/physical-expr", version = "42.2.0", default-features = false }
112+
datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "42.2.0", default-features = false }
113+
datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "42.2.0" }
114+
datafusion-physical-plan = { path = "datafusion/physical-plan", version = "42.2.0" }
115+
datafusion-proto = { path = "datafusion/proto", version = "42.2.0" }
116+
datafusion-proto-common = { path = "datafusion/proto-common", version = "42.2.0" }
117+
datafusion-sql = { path = "datafusion/sql", version = "42.2.0" }
118+
datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "42.2.0" }
119+
datafusion-substrait = { path = "datafusion/substrait", version = "42.2.0" }
118120
doc-comment = "0.3"
119121
env_logger = "0.11"
120122
futures = "0.3"

‎benchmarks/src/bin/external_aggr.rs

+1-6
Original file line numberDiff line numberDiff line change
@@ -193,12 +193,7 @@ impl ExternalAggrConfig {
193193
) -> Result<Vec<QueryResult>> {
194194
let query_name =
195195
format!("Q{query_id}({})", human_readable_size(mem_limit as usize));
196-
let mut config = self.common.config();
197-
config
198-
.options_mut()
199-
.execution
200-
.parquet
201-
.schema_force_view_types = self.common.force_view_types;
196+
let config = self.common.config();
202197
let runtime_config = RuntimeConfig::new()
203198
.with_memory_pool(Arc::new(FairSpillPool::new(mem_limit as usize)))
204199
.build_arc()?;

‎benchmarks/src/clickbench.rs

-1
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,6 @@ impl RunOpt {
119119
let mut config = self.common.config();
120120
{
121121
let parquet_options = &mut config.options_mut().execution.parquet;
122-
parquet_options.schema_force_view_types = self.common.force_view_types;
123122
// The hits_partitioned dataset specifies string columns
124123
// as binary due to how it was written. Force it to strings
125124
parquet_options.binary_as_string = true;

‎benchmarks/src/imdb/run.rs

+1-7
Original file line numberDiff line numberDiff line change
@@ -305,11 +305,7 @@ impl RunOpt {
305305
.config()
306306
.with_collect_statistics(!self.disable_statistics);
307307
config.options_mut().optimizer.prefer_hash_join = self.prefer_hash_join;
308-
config
309-
.options_mut()
310-
.execution
311-
.parquet
312-
.schema_force_view_types = self.common.force_view_types;
308+
313309
let ctx = SessionContext::new_with_config(config);
314310

315311
// register tables
@@ -517,7 +513,6 @@ mod tests {
517513
partitions: Some(2),
518514
batch_size: 8192,
519515
debug: false,
520-
force_view_types: false,
521516
};
522517
let opt = RunOpt {
523518
query: Some(query),
@@ -551,7 +546,6 @@ mod tests {
551546
partitions: Some(2),
552547
batch_size: 8192,
553548
debug: false,
554-
force_view_types: false,
555549
};
556550
let opt = RunOpt {
557551
query: Some(query),

‎benchmarks/src/sort.rs

+3-3
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ use crate::util::{AccessLogOpt, BenchmarkRun, CommonOpt};
2222

2323
use arrow::util::pretty;
2424
use datafusion::common::Result;
25-
use datafusion::physical_expr::PhysicalSortExpr;
25+
use datafusion::physical_expr::{LexOrdering, LexOrderingRef, PhysicalSortExpr};
2626
use datafusion::physical_plan::collect;
2727
use datafusion::physical_plan::sorts::sort::SortExec;
2828
use datafusion::prelude::{SessionConfig, SessionContext};
@@ -170,13 +170,13 @@ impl RunOpt {
170170

171171
async fn exec_sort(
172172
ctx: &SessionContext,
173-
expr: &[PhysicalSortExpr],
173+
expr: LexOrderingRef<'_>,
174174
test_file: &TestParquetFile,
175175
debug: bool,
176176
) -> Result<(usize, std::time::Duration)> {
177177
let start = Instant::now();
178178
let scan = test_file.create_scan(ctx, None).await?;
179-
let exec = Arc::new(SortExec::new(expr.to_owned(), scan));
179+
let exec = Arc::new(SortExec::new(LexOrdering::new(expr.to_owned()), scan));
180180
let task_ctx = ctx.task_ctx();
181181
let result = collect(exec, task_ctx).await?;
182182
let elapsed = start.elapsed();

‎benchmarks/src/tpch/run.rs

-7
Original file line numberDiff line numberDiff line change
@@ -120,11 +120,6 @@ impl RunOpt {
120120
.config()
121121
.with_collect_statistics(!self.disable_statistics);
122122
config.options_mut().optimizer.prefer_hash_join = self.prefer_hash_join;
123-
config
124-
.options_mut()
125-
.execution
126-
.parquet
127-
.schema_force_view_types = self.common.force_view_types;
128123
let ctx = SessionContext::new_with_config(config);
129124

130125
// register tables
@@ -345,7 +340,6 @@ mod tests {
345340
partitions: Some(2),
346341
batch_size: 8192,
347342
debug: false,
348-
force_view_types: false,
349343
};
350344
let opt = RunOpt {
351345
query: Some(query),
@@ -379,7 +373,6 @@ mod tests {
379373
partitions: Some(2),
380374
batch_size: 8192,
381375
debug: false,
382-
force_view_types: false,
383376
};
384377
let opt = RunOpt {
385378
query: Some(query),

‎benchmarks/src/util/options.rs

-5
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,6 @@ pub struct CommonOpt {
3737
/// Activate debug mode to see more details
3838
#[structopt(short, long)]
3939
pub debug: bool,
40-
41-
/// If true, will use StringView/BinaryViewArray instead of String/BinaryArray
42-
/// when reading ParquetFiles
43-
#[structopt(long)]
44-
pub force_view_types: bool,
4540
}
4641

4742
impl CommonOpt {

0 commit comments

Comments
 (0)