From b706201c133cde7af0165467c8d183f16dbccb98 Mon Sep 17 00:00:00 2001 From: Tomasz Gromadzki Date: Tue, 24 Oct 2023 11:13:12 +0200 Subject: [PATCH] common: reduce stack usage by eliminating complex inline function Signed-off-by: Tomasz Gromadzki --- ChangeLog | 4 ++ src/libpmem2/x86_64/memcpy/memcpy_t_avx512f.c | 6 +-- src/stat/stack-usage-debug.txt | 24 +++++----- src/stat/stack-usage-nondebug.txt | 44 ++++++++++++++----- 4 files changed, 55 insertions(+), 23 deletions(-) diff --git a/ChangeLog b/ChangeLog index b2df03d8f63..f9dfb576643 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,7 +1,11 @@ * Unreleased * This release: +<<<<<<< HEAD - Significantly reduces the libpmem's stack usage. +======= + - Make memmove_mov_* function on-inline to prevent huge stack consumption. +>>>>>>> 91293182e2 (common: reduce stack usage by eliminating complex inline function) Tue Aug 8 2023 Oksana Sałyk diff --git a/src/libpmem2/x86_64/memcpy/memcpy_t_avx512f.c b/src/libpmem2/x86_64/memcpy/memcpy_t_avx512f.c index b1775283ed9..fbfef891b14 100644 --- a/src/libpmem2/x86_64/memcpy/memcpy_t_avx512f.c +++ b/src/libpmem2/x86_64/memcpy/memcpy_t_avx512f.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: BSD-3-Clause -/* Copyright 2017-2020, Intel Corporation */ +/* Copyright 2017-2023, Intel Corporation */ #include #include @@ -254,7 +254,7 @@ memmove_mov1x64b(char *dest, const char *src, flush64b_fn flush64b) flush64b(dest + 0 * 64); } -static force_inline void +static void memmove_mov_avx512f_fw(char *dest, const char *src, size_t len, flush_fn flush, flush64b_fn flush64b) { @@ -319,7 +319,7 @@ memmove_mov_avx512f_fw(char *dest, const char *src, size_t len, memmove_small_avx512f(dest, src, len, flush); } -static force_inline void +static void memmove_mov_avx512f_bw(char *dest, const char *src, size_t len, flush_fn flush, flush64b_fn flush64b) { diff --git a/src/stat/stack-usage-debug.txt b/src/stat/stack-usage-debug.txt index 5b1505963d8..9852c9d8c64 100644 --- a/src/stat/stack-usage-debug.txt +++ b/src/stat/stack-usage-debug.txt @@ -8,16 +8,10 @@ 51344 src/debug/libpmem2/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c:memmove_movnt_avx512f_clflush dynamic,bounded 51344 src/debug/libpmem2/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c:memmove_movnt_avx512f_empty dynamic,bounded 51344 src/debug/libpmem2/memcpy_nt_avx512f.su:memcpy_nt_avx512f.c:memmove_movnt_avx512f_noflush dynamic,bounded -50640 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clwb dynamic,bounded -50640 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflushopt dynamic,bounded -50640 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflush dynamic,bounded -50640 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_empty dynamic,bounded -50640 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_noflush dynamic,bounded -50640 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clwb dynamic,bounded -50640 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflushopt dynamic,bounded -50640 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflush dynamic,bounded -50640 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_empty dynamic,bounded -50640 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_noflush dynamic,bounded +25344 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_bw static +25344 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_fw static +25344 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_bw static +25344 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_fw static 24144 src/debug/libpmem/memcpy_nt_avx.su:memcpy_nt_avx.c:memmove_movnt_avx_clwb_wcbarrier dynamic,bounded 24144 src/debug/libpmem/memcpy_nt_avx.su:memcpy_nt_avx.c:memmove_movnt_avx_clflushopt_wcbarrier dynamic,bounded 24144 src/debug/libpmem/memcpy_nt_avx.su:memcpy_nt_avx.c:memmove_movnt_avx_clflush_wcbarrier dynamic,bounded @@ -955,12 +949,22 @@ 112 src/debug/libpmemobj/alloc_class.su:alloc_class.c:alloc_class_new static 112 src/debug/libpmem/mmap.su:mmap.c:util_range_split dynamic,bounded 112 src/debug/libpmem/mmap.su:mmap.c:util_range_register dynamic,bounded +112 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clwb dynamic,bounded +112 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflushopt dynamic,bounded +112 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflush dynamic,bounded +112 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_empty dynamic,bounded +112 src/debug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_noflush dynamic,bounded 112 src/debug/libpmem/file.su:file.c:util_file_open dynamic,bounded 112 src/debug/libpmem/file.su:file.c:util_file_zero dynamic,bounded 112 src/debug/libpmem2/vm_reservation.su:vm_reservation.c:pmem2_vm_reservation_new static 112 src/debug/libpmem2/usc_ndctl.su:usc_ndctl.c:pmem2_source_device_usc dynamic,bounded 112 src/debug/libpmem2/persist.su:persist.c:pmem2_flush_file_buffers static 112 src/debug/libpmem2/persist.su:persist.c:pmem2_log_flush dynamic,bounded +112 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clwb dynamic,bounded +112 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflushopt dynamic,bounded +112 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflush dynamic,bounded +112 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_empty dynamic,bounded +112 src/debug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_noflush dynamic,bounded 112 src/debug/libpmem2/mcsafe_ops_posix.su:mcsafe_ops_posix.c:devdax_write dynamic,bounded 112 src/debug/libpmem2/mcsafe_ops_posix.su:mcsafe_ops_posix.c:devdax_read dynamic,bounded 112 src/debug/libpmem2/deep_flush.su:deep_flush.c:pmem2_deep_flush dynamic,bounded diff --git a/src/stat/stack-usage-nondebug.txt b/src/stat/stack-usage-nondebug.txt index b7b07db9944..dc60a9ce7b3 100644 --- a/src/stat/stack-usage-nondebug.txt +++ b/src/stat/stack-usage-nondebug.txt @@ -10,8 +10,6 @@ 8224 src/nondebug/libpmemobj/pmem2_utils_linux.su:pmem2_utils_linux.c:pmem2_get_type_from_stat static 8224 src/nondebug/libpmem2/pmem2_utils_linux.su:pmem2_utils_linux.c:pmem2_get_type_from_stat static 8224 src/nondebug/common/pmem2_utils_linux.su:pmem2_utils_linux.c:pmem2_get_type_from_stat static -7424 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_empty static -7424 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_empty static 4352 src/nondebug/libpmempool/pool.su:pool.c:pool_params_parse.isra.2.constprop dynamic,bounded 4272 src/nondebug/libpmem/region_namespace_ndctl.su:region_namespace_ndctl.c:ndctl_match_fsdax static 4272 src/nondebug/libpmem/region_namespace_ndctl.su:region_namespace_ndctl.c:ndctl_match_devdax static @@ -419,10 +417,14 @@ 128 src/nondebug/libpmem/memset_t_avx.su:memset_t_avx.c:memset_mov_avx_clflushopt static 128 src/nondebug/libpmem/memset_t_avx.su:memset_t_avx.c:memset_mov_avx_clflush static 128 src/nondebug/libpmem/memset_t_avx.su:memset_t_avx.c:memset_mov_avx_noflush static +128 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_bw static +128 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_fw static 128 src/nondebug/libpmem2/memset_t_avx.su:memset_t_avx.c:memset_mov_avx_clwb static 128 src/nondebug/libpmem2/memset_t_avx.su:memset_t_avx.c:memset_mov_avx_clflushopt static 128 src/nondebug/libpmem2/memset_t_avx.su:memset_t_avx.c:memset_mov_avx_clflush static 128 src/nondebug/libpmem2/memset_t_avx.su:memset_t_avx.c:memset_mov_avx_noflush static +128 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_bw static +128 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_fw static 128 src/nondebug/libpmem2/map.su:map.c:pmem2_map_from_existing static 128 src/nondebug/libpmem2/badblocks_ndctl.su:badblocks_ndctl.c:pmem2_badblock_context_new static 128 src/nondebug/common/badblocks_ndctl.su:badblocks_ndctl.c:pmem2_badblock_context_new static @@ -715,10 +717,6 @@ 64 src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c:memmove_mov_avx_clflushopt static 64 src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c:memmove_mov_avx_clflush static 64 src/nondebug/libpmem/memcpy_t_avx.su:memcpy_t_avx.c:memmove_mov_avx_noflush static -64 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clwb static -64 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflushopt static -64 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflush static -64 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_noflush static 64 src/nondebug/libpmem/file.su:file.c:util_file_zero dynamic,bounded 64 src/nondebug/libpmem/file.su:file.c:util_file_map_whole dynamic,bounded 64 src/nondebug/libpmem2/usc_ndctl.su:usc_ndctl.c:pmem2_source_device_id static @@ -739,10 +737,6 @@ 64 src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c:memmove_mov_avx_clflushopt static 64 src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c:memmove_mov_avx_clflush static 64 src/nondebug/libpmem2/memcpy_t_avx.su:memcpy_t_avx.c:memmove_mov_avx_noflush static -64 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clwb static -64 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflushopt static -64 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflush static -64 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_noflush static 64 src/nondebug/libpmem2/mcsafe_ops_posix.su:mcsafe_ops_posix.c:pmem2_source_check_op_size dynamic,bounded 64 src/nondebug/libpmem2/map.su:map.c:get_min_granularity static 64 src/nondebug/libpmem2/badblocks_ndctl.su:badblocks_ndctl.c:pmem2_badblock_clear static @@ -1492,6 +1486,11 @@ 16 src/nondebug/libpmem/mmap.su:mmap.c:util_range_rw static 16 src/nondebug/libpmem/mmap.su:mmap.c:util_range_ro static 16 src/nondebug/libpmem/mmap.su:mmap.c:util_unmap static +16 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clwb static +16 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflushopt static +16 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflush static +16 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_empty static +16 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_noflush static 16 src/nondebug/libpmem/libpmem.su:libpmem.c:pmem_check_version static 16 src/nondebug/libpmem/libpmem.su:libpmem.c:libpmem_fini static 16 src/nondebug/libpmem/libpmem.su:libpmem.c:libpmem_init static @@ -1546,6 +1545,11 @@ 16 src/nondebug/libpmem2/out.su:out.c:out_fini static 16 src/nondebug/libpmem2/out.su:out.c:out_init static 16 src/nondebug/libpmem2/os_thread_posix.su:os_thread_posix.c:os_thread_self static +16 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clwb static +16 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflushopt static +16 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_clflush static +16 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_empty static +16 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_t_avx512f.c:memmove_mov_avx512f_noflush static 16 src/nondebug/libpmem2/mcsafe_ops_posix.su:mcsafe_ops_posix.c:signal_handler static 16 src/nondebug/libpmem2/map.su:map.c:pmem2_map_fini static 16 src/nondebug/libpmem2/map.su:map.c:pmem2_map_init static @@ -2145,6 +2149,16 @@ 8 src/nondebug/libpmemobj/alloc_class.su:alloc_class.c:alloc_class_delete static 8 src/nondebug/libpmemobj/alloc_class.su:alloc_class.c:alloc_class_reserve static 8 src/nondebug/libpmemobj/alloc_class.su:alloc_class.c:alloc_class_find_first_free_slot static +8 src/nondebug/libpmem/memcpy_t_avx512f.su:pmem2_arch.h:flush_empty_nolog static +8 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_memset.h:noflush64b static +8 src/nondebug/libpmem/memcpy_t_avx512f.su:memcpy_memset.h:noflush static +8 src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h:flush64b_empty static +8 src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h:flush_clwb_nolog static +8 src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h:flush_clflushopt_nolog static +8 src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h:flush_clflush_nolog static +8 src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h:pmem_clwb static +8 src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h:pmem_clflushopt static +8 src/nondebug/libpmem/memcpy_t_avx512f.su:flush.h:pmem_clflush static 8 src/nondebug/libpmem/libpmem.su:libpmem.c:pmem_errormsg static 8 src/nondebug/libpmem/init.su:init.c:flush_clwb static 8 src/nondebug/libpmem/init.su:init.c:flush_clflushopt static @@ -2258,6 +2272,16 @@ 8 src/nondebug/libpmem2/os_posix.su:os_posix.c:os_chmod static 8 src/nondebug/libpmem2/os_posix.su:os_posix.c:os_fdopen static 8 src/nondebug/libpmem2/os_posix.su:os_posix.c:os_fopen static +8 src/nondebug/libpmem2/memcpy_t_avx512f.su:pmem2_arch.h:flush_empty_nolog static +8 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_memset.h:noflush64b static +8 src/nondebug/libpmem2/memcpy_t_avx512f.su:memcpy_memset.h:noflush static +8 src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h:flush64b_empty static +8 src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h:flush_clwb_nolog static +8 src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h:flush_clflushopt_nolog static +8 src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h:flush_clflush_nolog static +8 src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h:pmem_clwb static +8 src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h:pmem_clflushopt static +8 src/nondebug/libpmem2/memcpy_t_avx512f.su:flush.h:pmem_clflush static 8 src/nondebug/libpmem2/membuf.su:membuf.c:membuf_ptr_user_data static 8 src/nondebug/libpmem2/membuf.su:membuf.c:membuf_free static 8 src/nondebug/libpmem2/mcsafe_ops_posix.su:mcsafe_ops_posix.c:mcsafe_op_devdax_write static