Skip to content

Commit 3554e06

Browse files
committed
tests
Signed-off-by: CalvinNeo <[email protected]>
1 parent 00e3324 commit 3554e06

File tree

8 files changed

+178
-23
lines changed

8 files changed

+178
-23
lines changed

proxy_components/engine_store_ffi/src/core/forward_raft/region.rs

+4-2
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ impl<T: Transport + 'static, ER: RaftEngine> ProxyForwarder<T, ER> {
1313
}
1414

1515
pub fn on_region_changed(&self, ob_region: &Region, e: RegionChangeEvent, r: StateRole) {
16-
self.engine_store_server_helper.maybe_jemalloc_register_alloc();
16+
self.engine_store_server_helper
17+
.maybe_jemalloc_register_alloc();
1718
let region_id = ob_region.get_id();
1819
if e == RegionChangeEvent::Destroy {
1920
info!(
@@ -105,7 +106,8 @@ impl<T: Transport + 'static, ER: RaftEngine> ProxyForwarder<T, ER> {
105106
}
106107

107108
pub fn on_role_change(&self, ob_region: &Region, r: &RoleChange) {
108-
self.engine_store_server_helper.maybe_jemalloc_register_alloc();
109+
self.engine_store_server_helper
110+
.maybe_jemalloc_register_alloc();
109111
let region_id = ob_region.get_id();
110112
let is_replicated = !r.initialized;
111113
let is_fap_enabled = if let Some(b) = self.engine.proxy_ext.config_set.as_ref() {

proxy_components/mock-engine-store/src/mock_store/mock_engine_store_server.rs

+16-13
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
// Copyright 2022 TiKV Project Authors. Licensed under Apache-2.0.
22

3-
use std::{cell::RefCell, pin::Pin, sync::atomic::Ordering};
4-
use std::sync::Mutex;
3+
use std::{
4+
cell::RefCell,
5+
pin::Pin,
6+
sync::{atomic::Ordering, Mutex},
7+
};
58

69
use engine_store_ffi::TiFlashEngine;
710

@@ -27,14 +30,10 @@ pub struct ThreadInfoJealloc {
2730

2831
impl ThreadInfoJealloc {
2932
pub fn allocated(&self) -> u64 {
30-
unsafe {
31-
*(self.allocated_ptr as *const u64)
32-
}
33+
unsafe { *(self.allocated_ptr as *const u64) }
3334
}
3435
pub fn deallocated(&self) -> u64 {
35-
unsafe {
36-
*(self.deallocated_ptr as *const u64)
37-
}
36+
unsafe { *(self.deallocated_ptr as *const u64) }
3837
}
3938
pub fn remaining(&self) -> i64 {
4039
self.allocated() as i64 - self.deallocated() as i64
@@ -643,7 +642,12 @@ unsafe extern "C" fn ffi_report_thread_allocate_info(
643642
) {
644643
let store = into_engine_store_server_wrap(arg1);
645644
let tn = std::str::from_utf8(name.to_slice()).unwrap().to_string();
646-
match (*store.engine_store_server).thread_info_map.lock().expect("poisoned").entry(tn) {
645+
match (*store.engine_store_server)
646+
.thread_info_map
647+
.lock()
648+
.expect("poisoned")
649+
.entry(tn)
650+
{
647651
std::collections::hash_map::Entry::Occupied(mut o) => {
648652
if t == 0 {
649653
o.get_mut().allocated_ptr = value;
@@ -655,15 +659,14 @@ unsafe extern "C" fn ffi_report_thread_allocate_info(
655659
if t == 0 {
656660
v.insert(ThreadInfoJealloc {
657661
allocated_ptr: value,
658-
deallocated_ptr: 0
662+
deallocated_ptr: 0,
659663
});
660664
} else {
661665
v.insert(ThreadInfoJealloc {
662666
allocated_ptr: 0,
663-
deallocated_ptr: value
667+
deallocated_ptr: value,
664668
});
665669
}
666670
}
667-
668671
}
669-
}
672+
}

proxy_components/proxy_ffi/src/engine_store_helper_impls.rs

+7-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
// Copyright 2016 TiKV Project Authors. Licensed under Apache-2.0.
2-
use std::pin::Pin;
3-
use std::cell::RefCell;
2+
use std::{cell::RefCell, pin::Pin};
43

54
use kvproto::{kvrpcpb, metapb, raft_cmdpb};
65

@@ -60,18 +59,21 @@ impl EngineStoreServerHelper {
6059
unsafe {
6160
let ptr_alloc: u64 = crate::jemalloc_utils::get_allocatep_on_thread_start();
6261
let ptr_dealloc: u64 = crate::jemalloc_utils::get_deallocatep_on_thread_start();
63-
let thread_name = std::thread::current().name().unwrap_or("<proxy-unknown>").to_string();
62+
let thread_name = std::thread::current()
63+
.name()
64+
.unwrap_or("<proxy-unknown>")
65+
.to_string();
6466
(self.fn_report_thread_allocate_info.into_inner())(
6567
self.inner,
6668
BaseBuffView::from(thread_name.as_bytes()),
6769
0,
68-
ptr_alloc
70+
ptr_alloc,
6971
);
7072
(self.fn_report_thread_allocate_info.into_inner())(
7173
self.inner,
7274
BaseBuffView::from(thread_name.as_bytes()),
7375
1,
74-
ptr_dealloc
76+
ptr_dealloc,
7577
);
7678
}
7779
*(b.borrow_mut()) = true;
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
// Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0.
2+
3+
extern "C" {
4+
// External jemalloc
5+
pub fn mallctl(
6+
name: *const ::std::os::raw::c_char,
7+
oldp: *mut ::std::os::raw::c_void,
8+
oldlenp: *mut u64,
9+
newp: *mut ::std::os::raw::c_void,
10+
newlen: u64,
11+
) -> ::std::os::raw::c_int;
12+
13+
// Embeded jemalloc
14+
pub fn _rjem_mallctl(
15+
name: *const ::std::os::raw::c_char,
16+
oldp: *mut ::std::os::raw::c_void,
17+
oldlenp: *mut u64,
18+
newp: *mut ::std::os::raw::c_void,
19+
newlen: u64,
20+
) -> ::std::os::raw::c_int;
21+
}
22+
23+
pub fn get_allocatep_on_thread_start() -> u64 {
24+
type PtrUnderlying = u64;
25+
let mut ptr: PtrUnderlying = 0;
26+
let mut size = std::mem::size_of::<PtrUnderlying>() as u64;
27+
let c_str = std::ffi::CString::new("thread.allocatedp").unwrap();
28+
let c_ptr: *const ::std::os::raw::c_char = c_str.as_ptr() as *const ::std::os::raw::c_char;
29+
unsafe {
30+
_rjem_mallctl(
31+
c_ptr,
32+
&mut ptr as *mut _ as *mut ::std::os::raw::c_void,
33+
&mut size as *mut u64,
34+
std::ptr::null_mut(),
35+
0,
36+
);
37+
}
38+
return ptr;
39+
}
40+
41+
pub fn get_deallocatep_on_thread_start() -> u64 {
42+
type PtrUnderlying = u64;
43+
let mut ptr: PtrUnderlying = 0;
44+
let mut size = std::mem::size_of::<PtrUnderlying>() as u64;
45+
let c_str = std::ffi::CString::new("thread.deallocatedp").unwrap();
46+
let c_ptr: *const ::std::os::raw::c_char = c_str.as_ptr() as *const ::std::os::raw::c_char;
47+
unsafe {
48+
#[cfg(any(test, feature = "testexport"))]
49+
_rjem_mallctl(
50+
c_ptr,
51+
&mut ptr as *mut _ as *mut ::std::os::raw::c_void,
52+
&mut size as *mut u64,
53+
std::ptr::null_mut(),
54+
0,
55+
);
56+
57+
#[cfg(not(any(test, feature = "testexport")))]
58+
{
59+
#[cfg(feature = "external-jemalloc")]
60+
mallctl(
61+
c_ptr,
62+
&mut ptr as *mut _ as *mut ::std::os::raw::c_void,
63+
&mut size as *mut u64,
64+
std::ptr::null_mut(),
65+
0,
66+
);
67+
}
68+
}
69+
return ptr;
70+
}

proxy_components/proxy_ffi/src/lib.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,9 @@ pub mod raftstore_proxy;
2121
pub mod raftstore_proxy_helper_impls;
2222
pub mod read_index_helper;
2323
// FFI releated with reading from SST/RocksDB files.
24+
pub mod jemalloc_utils;
2425
pub mod snapshot_reader_impls;
2526
pub mod utils;
26-
pub mod jemalloc_utils;
2727

2828
pub use self::{
2929
basic_ffi_impls::*, domain_impls::*, encryption_impls::*, engine_store_helper_impls::*,

proxy_components/proxy_server/src/run.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -622,7 +622,7 @@ impl<ER: RaftEngine, F: KvFormat> TiKvServer<ER, F> {
622622
info!("linked with external jemalloc");
623623
#[cfg(not(feature = "external-jemalloc"))]
624624
info!("linked without external jemalloc");
625-
625+
626626
// Initialize and check config
627627
info!("using proxy config"; "config" => ?proxy_config);
628628
crate::config::address_proxy_config(&mut config, &proxy_config);

proxy_tests/proxy/shared/jemalloc.rs

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
// Copyright 2024 TiKV Project Authors. Licensed under Apache-2.0.
2+
3+
use collections::HashMap;
4+
use mock_engine_store::ThreadInfoJealloc;
5+
use more_asserts::assert_gt;
6+
use proxy_ffi::jemalloc_utils::{get_allocatep_on_thread_start, get_deallocatep_on_thread_start};
7+
8+
use crate::utils::v1::*;
9+
10+
#[test]
11+
fn test_alloc_dealloc() {
12+
let dummy: Vec<u64> = Vec::with_capacity(100000);
13+
let ptr_alloc = get_allocatep_on_thread_start();
14+
let actual_alloc: &u64 = unsafe { &*(ptr_alloc as *const u64) };
15+
let ptr_dealloc = get_deallocatep_on_thread_start();
16+
let actual_dealloc: &u64 = unsafe { &*(ptr_dealloc as *const u64) };
17+
assert_gt!(*actual_alloc, 100000 * std::mem::size_of::<u64>() as u64);
18+
let dummy2: Vec<u64> = Vec::with_capacity(100000);
19+
assert_gt!(
20+
*actual_alloc,
21+
2 * 100000 * std::mem::size_of::<u64>() as u64
22+
);
23+
drop(dummy);
24+
assert_gt!(*actual_dealloc, 100000 * std::mem::size_of::<u64>() as u64);
25+
drop(dummy2);
26+
assert_gt!(
27+
*actual_dealloc,
28+
2 * 100000 * std::mem::size_of::<u64>() as u64
29+
);
30+
}
31+
32+
fn collect_thread_state(
33+
cluster_ext: &ClusterExt,
34+
store_id: u64,
35+
) -> HashMap<String, ThreadInfoJealloc> {
36+
let mut res: HashMap<String, ThreadInfoJealloc> = Default::default();
37+
cluster_ext.iter_ffi_helpers(Some(vec![store_id]), &mut |_, ffi: &mut FFIHelperSet| {
38+
res = (*ffi.engine_store_server.thread_info_map.lock().expect("")).clone();
39+
});
40+
res
41+
}
42+
43+
fn gather(m: &HashMap<String, ThreadInfoJealloc>, pattern: &str) -> i64 {
44+
m.iter()
45+
.filter(|(k, _)| k.contains(pattern))
46+
.fold(0, |acc, e| acc + e.1.remaining())
47+
}
48+
49+
#[test]
50+
fn test_ffi() {
51+
let (mut cluster, _pd_client) = new_mock_cluster(0, 1);
52+
53+
let _ = cluster.run();
54+
55+
let prev = collect_thread_state(&cluster.cluster_ext, 1);
56+
let before_raftstore = gather(&prev, "raftstore");
57+
let before_apply = gather(&prev, "apply");
58+
59+
for i in 0..10 {
60+
let k = format!("k{}", i);
61+
let v = format!("v{}", i);
62+
cluster.must_put(k.as_bytes(), v.as_bytes());
63+
}
64+
65+
for i in 0..10 {
66+
let k = format!("k{}", i);
67+
let v = format!("v{}", i);
68+
check_key(&cluster, k.as_bytes(), v.as_bytes(), Some(true), None, None);
69+
}
70+
71+
let after = collect_thread_state(&cluster.cluster_ext, 1);
72+
let after_raftstore = gather(&after, "raftstore");
73+
let after_apply = gather(&after, "apply");
74+
assert_gt!(after_raftstore, before_raftstore);
75+
assert_gt!(after_apply, before_apply);
76+
77+
cluster.shutdown();
78+
}

proxy_tests/proxy/shared/mod.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ mod engine;
66
mod fast_add_peer;
77
mod ffi;
88
mod ingest;
9+
mod jemalloc;
910
mod mock;
1011
mod normal;
1112
mod region;
@@ -14,4 +15,3 @@ mod server_cluster_test;
1415
mod snapshot;
1516
mod store;
1617
mod write;
17-
mod jemalloc;

0 commit comments

Comments
 (0)