chore: Use twox-hash 2.0 xxhash64 oneshot api instead of custom implementation (#1041)

Author: NoeB

LICENSE.txt

@@ -210,31 +210,4 @@ This project includes code from Apache Aurora.
 
 Copyright: 2016 The Apache Software Foundation.
 Home page: https://aurora.apache.org/
-License: http://www.apache.org/licenses/LICENSE-2.0
-
---------------------------------------------------------------------------------
-
-This project includes software from the twox-hash project
-https://github.com/shepmaster/twox-hash
-
-The MIT License (MIT)
-
-Copyright (c) 2015 Jake Goulding
-
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in
- all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- THE SOFTWARE.
+License: http://www.apache.org/licenses/LICENSE-2.0

NOTICE.txt

@@ -4,9 +4,6 @@ Copyright 2024 The Apache Software Foundation
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
 
-This product includes software from the twox-hash project (MIT License)
-https://github.com/shepmaster/twox-hash
-
 This product includes software developed at
 Apache Gluten (https://github.com/apache/incubator-gluten/)
 Specifically:

native/Cargo.lock

@@ -19,7 +19,7 @@ use std::{cmp, mem, ptr};
 
 /// This is a copy of the `rdxsort-rs` crate, with the following changes:
 /// - removed `Rdx` implementations for all types except for i64 which is the packed representation
-/// of row addresses and partition ids from Spark.
+///   of row addresses and partition ids from Spark.
 
 pub trait Rdx {
     /// Sets the number of buckets used by the generic implementation.

native/core/src/execution/sort.rs

@@ -48,7 +48,7 @@ pub trait DataPageBuilder {
 ///   - add_def_levels()
 ///   - add_values() for normal data page / add_indices() for dictionary data page
 ///   - consume()
-/// in order to populate and obtain a data page.
+///     in order to populate and obtain a data page.
 pub struct DataPageBuilderImpl {
     desc: ColumnDescPtr,
     encoding: Option<Encoding>,

native/core/src/parquet/util/test_common/page_util.rs

@@ -39,12 +39,13 @@ chrono-tz = { workspace = true }
 num = { workspace = true }
 regex = { workspace = true }
 thiserror = { workspace = true }
+twox-hash = "2.0.0"
 
 [dev-dependencies]
 arrow-data = {workspace = true}
 criterion = "0.5.1"
 rand = { workspace = true}
-twox-hash = "1.6.3"
+
 
 [lib]
 name = "datafusion_comet_spark_expr"

native/spark-expr/Cargo.toml

@@ -51,12 +51,12 @@ fn criterion_benchmark(c: &mut Criterion) {
         if i % 7 == 0 {
             c2.append_null();
         } else {
-            c2.append_value(&format!("string {i}"));
+            c2.append_value(format!("string {i}"));
         }
         if i % 9 == 0 {
             c3.append_null();
         } else {
-            c3.append_value(&format!("other string {i}"));
+            c3.append_value(format!("other string {i}"));
         }
     }
     let c1 = Arc::new(c1.finish());

native/spark-expr/benches/conditional.rs

@@ -1568,9 +1568,7 @@ fn get_timestamp_values<T: TimeZone>(
     timestamp_type: &str,
     tz: &T,
 ) -> SparkResult<Option<i64>> {
-    let values: Vec<_> = value
-        .split(|c| c == 'T' || c == '-' || c == ':' || c == '.')
-        .collect();
+    let values: Vec<_> = value.split(['T', '-', ':', '.']).collect();
     let year = values[0].parse::<i32>().unwrap_or_default();
     let month = values.get(1).map_or(1, |m| m.parse::<u32>().unwrap_or(1));
     let day = values.get(2).map_or(1, |d| d.parse::<u32>().unwrap_or(1));

native/spark-expr/src/cast.rs

@@ -33,7 +33,6 @@ mod temporal;
 pub mod timezone;
 mod to_json;
 pub mod utils;
-mod xxhash64;
 
 pub use cast::{spark_cast, Cast};
 pub use error::{SparkError, SparkResult};

native/spark-expr/src/lib.rs

@@ -22,6 +22,7 @@ use arrow::{
     datatypes::{ArrowNativeTypeOp, UInt16Type, UInt32Type, UInt64Type, UInt8Type},
 };
 use std::sync::Arc;
+use twox_hash::XxHash64;
 
 use datafusion::{
     arrow::{
@@ -34,7 +35,10 @@ use datafusion::{
     error::{DataFusionError, Result},
 };
 
-use crate::xxhash64::spark_compatible_xxhash64;
+#[inline]
+pub(crate) fn spark_compatible_xxhash64<T: AsRef<[u8]>>(data: T, seed: u64) -> u64 {
+    XxHash64::oneshot(seed, data.as_ref())
+}
 
 /// Spark-compatible murmur3 hash function
 #[inline]

native/spark-expr/src/spark_hash.rs

@@ -16,5 +16,5 @@
 # under the License.
 
 [toolchain]
-channel = "1.79"
+channel = "1.81"
 components = ["rustfmt", "clippy", "rust-analyzer"]