diff --git a/s3-fast-list/Cargo.toml b/s3-fast-list/Cargo.toml index dcfdc29..b5f83d6 100644 --- a/s3-fast-list/Cargo.toml +++ b/s3-fast-list/Cargo.toml @@ -19,10 +19,10 @@ hyper-rustls = "0.24.2" aws-smithy-runtime = { version = "1.1.1", features = ["client", "connector-hyper-0-14-x"] } aws-smithy-runtime-api = "1.1.1" aws-smithy-async = "1.1.1" -arrow = "49.0.0" -arrow-array = "49.0.0" -arrow-schema = "49.0.0" -parquet = { version = "49.0.0", features = ["async"] } +arrow = "54.3.1" +arrow-array = "54.3.1" +arrow-schema = "54.3.1" +parquet = { version = "54.3.1", features = ["async"] } futures = "0.3.30" ctrlc = "3.4.2" chrono = "0.4.31" diff --git a/s3-fast-list/src/data_map.rs b/s3-fast-list/src/data_map.rs index 43631a0..80614a1 100644 --- a/s3-fast-list/src/data_map.rs +++ b/s3-fast-list/src/data_map.rs @@ -106,7 +106,7 @@ impl PrefixMap { let buffer_size = 100 * MB; let f = tokio::fs::File::create(output_file).await?; let writer = tokio::io::BufWriter::with_capacity(buffer_size, f); - let mut parquet = utils::AsyncParquetOutput::new(writer, buffer_size); + let mut parquet = utils::AsyncParquetOutput::new(writer); let hash = self.inner.read().await; diff --git a/s3-fast-list/src/utils.rs b/s3-fast-list/src/utils.rs index 1970029..0aba8be 100644 --- a/s3-fast-list/src/utils.rs +++ b/s3-fast-list/src/utils.rs @@ -17,7 +17,7 @@ pub struct AsyncParquetOutput { impl AsyncParquetOutput { - pub fn new(buf_wr: W, buffer_size: usize) -> Self { + pub fn new(buf_wr: W) -> Self { // define fields let field_key = Field::new("Key", DataType::Utf8, false); @@ -37,7 +37,7 @@ impl AsyncParquetOutput { .build(); // build writer - let writer = AsyncArrowWriter::try_new(buf_wr, Arc::clone(&schema_ref), buffer_size, Some(writer_props.clone())).unwrap(); + let writer = AsyncArrowWriter::try_new(buf_wr, Arc::clone(&schema_ref), Some(writer_props.clone())).unwrap(); Self { schema_ref: schema_ref,