@@ -21,7 +21,9 @@ pub use mutable_vector::*;
21
21
22
22
#[ macro_use]
23
23
pub mod util;
24
+ pub mod parquet_support;
24
25
pub mod read;
26
+ pub mod schema_adapter;
25
27
26
28
use std:: task:: Poll ;
27
29
use std:: { boxed:: Box , ptr:: NonNull , sync:: Arc } ;
@@ -40,27 +42,28 @@ use jni::{
40
42
} ,
41
43
} ;
42
44
45
+ use self :: util:: jni:: TypePromotionInfo ;
43
46
use crate :: execution:: operators:: ExecutionError ;
44
47
use crate :: execution:: utils:: SparkArrowConvert ;
45
48
use crate :: parquet:: data_type:: AsBytes ;
49
+ use crate :: parquet:: schema_adapter:: SparkSchemaAdapterFactory ;
46
50
use arrow:: buffer:: { Buffer , MutableBuffer } ;
47
51
use arrow_array:: { Array , RecordBatch } ;
48
52
use datafusion:: datasource:: listing:: PartitionedFile ;
49
53
use datafusion:: datasource:: physical_plan:: parquet:: ParquetExecBuilder ;
50
54
use datafusion:: datasource:: physical_plan:: FileScanConfig ;
51
55
use datafusion:: physical_plan:: ExecutionPlan ;
52
- use datafusion_comet_spark_expr:: { EvalMode , SparkCastOptions , SparkSchemaAdapterFactory } ;
56
+ use datafusion_comet_spark_expr:: EvalMode ;
53
57
use datafusion_common:: config:: TableParquetOptions ;
54
58
use datafusion_execution:: { SendableRecordBatchStream , TaskContext } ;
55
59
use futures:: { poll, StreamExt } ;
56
60
use jni:: objects:: { JBooleanArray , JByteArray , JLongArray , JPrimitiveArray , JString , ReleaseMode } ;
57
61
use jni:: sys:: jstring;
58
62
use parquet:: arrow:: arrow_reader:: ParquetRecordBatchReader ;
63
+ use parquet_support:: SparkParquetOptions ;
59
64
use read:: ColumnReader ;
60
65
use util:: jni:: { convert_column_descriptor, convert_encoding, deserialize_schema, get_file_path} ;
61
66
62
- use self :: util:: jni:: TypePromotionInfo ;
63
-
64
67
/// Parquet read context maintained across multiple JNI calls.
65
68
struct Context {
66
69
pub column_reader : ColumnReader ,
@@ -679,13 +682,13 @@ pub unsafe extern "system" fn Java_org_apache_comet_parquet_Native_initRecordBat
679
682
table_parquet_options. global . pushdown_filters = true ;
680
683
table_parquet_options. global . reorder_filters = true ;
681
684
682
- let mut spark_cast_options = SparkCastOptions :: new ( EvalMode :: Legacy , "UTC" , false ) ;
683
- spark_cast_options . allow_cast_unsigned_ints = true ;
685
+ let mut spark_parquet_options = SparkParquetOptions :: new ( EvalMode :: Legacy , "UTC" , false ) ;
686
+ spark_parquet_options . allow_cast_unsigned_ints = true ;
684
687
685
688
let builder2 = ParquetExecBuilder :: new ( file_scan_config)
686
689
. with_table_parquet_options ( table_parquet_options)
687
690
. with_schema_adapter_factory ( Arc :: new ( SparkSchemaAdapterFactory :: new (
688
- spark_cast_options ,
691
+ spark_parquet_options ,
689
692
) ) ) ;
690
693
691
694
//TODO: (ARROW NATIVE) - predicate pushdown??
0 commit comments