diff --git a/native/Cargo.lock b/native/Cargo.lock index 3c2b7c6052..ce7dc7f7f9 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -999,6 +999,7 @@ dependencies = [ "snap", "tempfile", "thiserror 2.0.12", + "tikv-jemallocator", "tokio", "url", "zstd", @@ -3862,6 +3863,26 @@ dependencies = [ "ordered-float", ] +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.0+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3c60906412afa9c2b5b5a48ca6a5abe5736aec9eb48ad05037a677e52e4e2d" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cec5ff18518d81584f477e9bfdf957f5bb0979b0bac3af4ca30b5b3ae2d2865" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "tiny-keccak" version = "2.0.2" diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index fd88057943..e50226935e 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -39,6 +39,7 @@ arrow = { workspace = true } parquet = { workspace = true, default-features = false, features = ["experimental"] } futures = { workspace = true } mimalloc = { version = "*", default-features = false, optional = true } +tikv-jemallocator = { version = "0.6.0", optional = true, features = ["disable_initial_exec_tls"] } tokio = { version = "1", features = ["rt-multi-thread"] } async-trait = { workspace = true } log = "0.4" @@ -51,13 +52,13 @@ snap = "1.1" # we disable default features in lz4_flex to force the use of the faster unsafe encoding and decoding implementation lz4_flex = { version = "0.11.3", default-features = false } zstd = "0.13.3" -rand = { workspace = true} +rand = { workspace = true } num = { workspace = true } bytes = { workspace = true } tempfile = "3.8.0" itertools = "0.14.0" paste = "1.0.14" -datafusion = { workspace = true } +datafusion = { workspace = true } once_cell = "1.18.0" regex = { workspace = true } crc32fast = "1.3.2" @@ -80,7 +81,8 @@ datafusion-functions-nested = { version = "47.0.0" } [features] default = [] -hdfs=["datafusion-comet-objectstore-hdfs"] +hdfs = ["datafusion-comet-objectstore-hdfs"] +jemalloc = ["tikv-jemallocator"] # exclude optional packages from cargo machete verifications [package.metadata.cargo-machete] diff --git a/native/core/src/lib.rs b/native/core/src/lib.rs index 4180751477..c6491eebec 100644 --- a/native/core/src/lib.rs +++ b/native/core/src/lib.rs @@ -36,9 +36,13 @@ use log4rs::{ encode::pattern::PatternEncoder, Config, }; +use once_cell::sync::OnceCell; + +#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] +use tikv_jemallocator::Jemalloc; + #[cfg(feature = "mimalloc")] use mimalloc::MiMalloc; -use once_cell::sync::OnceCell; use errors::{try_unwrap_or_throw, CometError, CometResult}; @@ -50,6 +54,10 @@ pub mod execution; mod jvm_bridge; pub mod parquet; +#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] +#[global_allocator] +static GLOBAL: Jemalloc = Jemalloc; + #[cfg(feature = "mimalloc")] #[global_allocator] static GLOBAL: MiMalloc = MiMalloc;