From 60cc1b9d845b40f40043f0413de4dbf333da9594 Mon Sep 17 00:00:00 2001
From: Jiping Yin <jiping@yunshan.net>
Date: Wed, 25 Oct 2023 10:56:13 +0800
Subject: [PATCH] [eBPF] Modify the storage method of Java perf files (#4536)

* [eBPF] Modify the storage method of Java perf files

Previously, we stored Java perf map files in the '/tmp' directory of the target POD container, which caused a few issues:

- The target POD's '/tmp' directory might have size limitations, and storing perf files there could exceed those limits, leading to abnormalities in the target POD.
- Java perf map files would persist indefinitely, potentially being seen as invasive behavior.

We have made the following changes:

- We modified the storage location, moving the perf map files to the '/deepflow' directory of the target POD.
- After generating Java perf map files, we immediately transfer the files to the '/tmp' directory of the 'deepflow-agent' POD.
- No files are left behind in the target POD.

Additionally, you can use the 'deepflow-jattach clean' command within the 'deepflow-agent' POD to remove the previously resident 'perf-PID.log' and 'perf-PID.map' files in the customer's POD.

* [eBPF] Add comments for 'deepflow-ebpfctl cpdbg ...'

* [eBPF] Add java symbols write space limit

'java_syms_space_limit' The maximum space occupied by the Java symbol files
in the target POD. Its valid range is [2, 10], which means it falls within
the interval of 2Mi to 10Mi. If the configuration value is outside this
range, the default value of 10(10Mi), will be used.

The size of the Java perf-PID.map file, excluding the additional space occupied
by 'agent.so' and log files, represents the actual limit of space occupied in the
target POD's root path ('/'). The actual writing limit is controlled by the global
variable 'g_java_syms_write_bytes_max', and the size (in bytes) of the Java perf-PID.map
file will not exceed it.

* [eBPF] Make the Java agent configurable and run successfully on the host

HotSpot JVM does not support agent unloading. However, you
may "attach" the same library multiple times with different
arguments. The library will not be loaded again, but
Agent_OnAttach will still be called multiple times with
different arguments.

We have achieved flexibility and configurability for the Java agent.

* [eBPF] Ensure that the socket tracer has completed before the profiler parses

The profiler's processing depends on probe interfaces provided by the socket tracer, such as process exit events. We want to ensure that everything is ready before the profiler performs address translation.

* [eBPF] Add a configurable option 'java_update_delay'

  @java_update_delay To allow Java to run for an extended period and gather
  more symbol information, we delay symbol retrieval when encountering unknown symbols.
  The recommended range for values is [60, 86400], default valuse is 300.

* [eBPF] Adjust the quota for Java symbol space occupancy

* [eBPF] Delete old perf map files in the customer's POD

* [eBPF] Adjust Java symbols delay update time

* [eBPF] Add configuration options for Java ebpf profiler
---
 agent/src/ebpf/mod.rs                         |  11 +
 .../ebpf/samples/rust/profiler/src/main.rs    |   4 +-
 agent/src/ebpf/user/common.c                  |   5 +-
 agent/src/ebpf/user/config.h                  |  21 +-
 agent/src/ebpf/user/profile/attach.c          |  28 ++-
 agent/src/ebpf/user/profile/attach.h          |   1 +
 agent/src/ebpf/user/profile/java/agent.c      | 131 +++++++----
 agent/src/ebpf/user/profile/java/config.h     |  42 ++++
 agent/src/ebpf/user/profile/java/df_jattach.c | 204 +++++++++++++-----
 agent/src/ebpf/user/profile/java/df_jattach.h |  13 +-
 agent/src/ebpf/user/profile/perf_profiler.c   |  72 +++++--
 agent/src/ebpf/user/profile/perf_profiler.h   |   3 +-
 agent/src/ebpf/user/profile/stringifier.c     |  69 +++---
 agent/src/ebpf/user/socket.c                  |  12 +-
 agent/src/ebpf/user/socket.h                  |   1 +
 agent/src/ebpf/user/symbol.c                  | 170 ++++++++-------
 agent/src/ebpf/user/symbol.h                  |   4 +
 agent/src/ebpf/user/tracer.c                  |   4 +
 agent/src/ebpf_dispatcher/ebpf_dispatcher.rs  |   2 +
 .../model/agent_group_config_example.yaml     |  19 ++
 20 files changed, 585 insertions(+), 231 deletions(-)
 create mode 100644 agent/src/ebpf/user/profile/java/config.h

diff --git a/agent/src/ebpf/mod.rs b/agent/src/ebpf/mod.rs
index 63e4cd1334f..1a3244ff3ff 100644
--- a/agent/src/ebpf/mod.rs
+++ b/agent/src/ebpf/mod.rs
@@ -465,11 +465,22 @@ extern "C" {
     /*
      * start continuous profiler
      * @freq sample frequency, Hertz. (e.g. 99 profile stack traces at 99 Hertz)
+     * @java_syms_space_limit The maximum space occupied by the Java symbol files
+     *   in the '/' directory of the target POD container.The recommended range for
+     *   values is [2, 100], which means it falls within the interval of 2Mi to 100Mi.
+     *   If the configuration value is outside this range, the default value of
+     *   10(10Mi), will be used.
+     * @java_syms_update_delay To allow Java to run for an extended period and gather
+     *   more symbol information, we delay symbol retrieval when encountering unknown
+     *   symbols. The unit of measurement used is seconds.
+     *   The recommended range for values is [5, 3600], default valuse is 60.
      * @callback Profile data processing callback interface
      * @returns 0 on success, < 0 on error
      */
     pub fn start_continuous_profiler(
         freq: c_int,
+        java_syms_space_limit: c_int,
+        java_syms_update_delay: c_int,
         callback: extern "C" fn(_data: *mut stack_profile_data),
     ) -> c_int;
 
diff --git a/agent/src/ebpf/samples/rust/profiler/src/main.rs b/agent/src/ebpf/samples/rust/profiler/src/main.rs
index 15b6569a132..ed54b593e89 100644
--- a/agent/src/ebpf/samples/rust/profiler/src/main.rs
+++ b/agent/src/ebpf/samples/rust/profiler/src/main.rs
@@ -153,7 +153,7 @@ fn main() {
 
         // Used to test our DeepFlow products, written as 97 frequency, so that
         // it will not affect the sampling test of deepflow agent (using 99Hz).
-        if start_continuous_profiler(97, continuous_profiler_callback) != 0 {
+        if start_continuous_profiler(97, 10, 300, continuous_profiler_callback) != 0 {
             println!("start_continuous_profiler() error.");
             ::std::process::exit(1);
         }
@@ -179,7 +179,7 @@ fn main() {
             std::thread::sleep(Duration::from_secs(1));
         }
 
-        thread::sleep(Duration::from_secs(65));
+        thread::sleep(Duration::from_secs(150));
         stop_continuous_profiler();
         print!(
             "====== capture count {}, sum {}\n",
diff --git a/agent/src/ebpf/user/common.c b/agent/src/ebpf/user/common.c
index 53117cfbb3c..1ace0b2e877 100644
--- a/agent/src/ebpf/user/common.c
+++ b/agent/src/ebpf/user/common.c
@@ -42,6 +42,7 @@
 #include "common.h"
 #include "log.h"
 #include "string.h"
+#include "profile/java/config.h"
 
 #define MAXLINE 1024
 
@@ -946,7 +947,7 @@ int exec_command(const char *cmd, const char *args)
 {
 	FILE *fp;
 	int rc = 0;
-	char cmd_buf[64];
+	char cmd_buf[PERF_PATH_SZ * 2];
 	snprintf(cmd_buf, sizeof(cmd_buf), "%s %s", cmd, args);
 	fp = popen(cmd_buf, "r");
 	if (NULL == fp) {
@@ -968,8 +969,6 @@ int exec_command(const char *cmd, const char *args)
 			     cmd_buf, strerror(errno));
 	} else {
 		if (WIFEXITED(rc)) {
-			ebpf_info("'%s' normal termination, exit status %d\n",
-				  cmd_buf, WEXITSTATUS(rc));
 			return WEXITSTATUS(rc);
 		} else if (WIFSIGNALED(rc)) {
 			ebpf_info
diff --git a/agent/src/ebpf/user/config.h b/agent/src/ebpf/user/config.h
index ac9b48244a5..5d5fc6bae05 100644
--- a/agent/src/ebpf/user/config.h
+++ b/agent/src/ebpf/user/config.h
@@ -165,7 +165,9 @@ enum {
  * date the Java symbol table. This is done The purpose is to avoid freque-
  * nt updates of the java symbol table.
  */
-#define JAVA_SYMS_TABLE_UPDATE_PERIOD 300 // 300 seconds
+#define JAVA_SYMS_UPDATE_DELAY_DEF 60 // 60 seconds
+#define JAVA_SYMS_UPDATE_DELAY_MIN 5 // 5 seconds
+#define JAVA_SYMS_UPDATE_DELAY_MAX 3600 // 3600 seconds
 
 /* Profiler - maximum data push interval time (in nanosecond). */
 #define MAX_PUSH_MSG_TIME_INTERVAL 1000000000ULL	/* 1 seconds */ 
@@ -205,4 +207,21 @@ enum {
  * check cycle time (unit is milliseconds).
  */
 #define CHECK_KERN_ADAPT_PERIOD 100 // 100 ticks(1 seconds)
+
+/*
+ * The maximum space occupied by the Java symbol files in the target POD.
+ * Its valid range is [2, 100], which means it falls within the interval
+ * of 2Mi to 100Mi. If the configuration value is outside this range, the
+ * default value of 10(10Mi), will be used.
+ */
+#define JAVA_POD_WRITE_FILES_SPACE_MIN 2097152 // 2Mi
+#define JAVA_POD_WRITE_FILES_SPACE_MAX 104857600 // 100Mi
+#define JAVA_POD_WRITE_FILES_SPACE_DEF 10485760 // 10Mi
+/*
+ * The `df_java_agent_musl.so` and `df_java_agent.so` files will also be
+ * placed in the target POD for loading operations. They occupy less than
+ * 300Ki of space.
+ */
+#define JAVA_POD_EXTRA_SPACE_MMA 307200 // 300Ki
+
 #endif /* DF_EBPF_CONFIG_H */
diff --git a/agent/src/ebpf/user/profile/attach.c b/agent/src/ebpf/user/profile/attach.c
index d850371f361..ca0804354b3 100644
--- a/agent/src/ebpf/user/profile/attach.c
+++ b/agent/src/ebpf/user/profile/attach.c
@@ -21,9 +21,12 @@
 
 #include "../common.h"
 #include "../log.h"
+#include "java/config.h"
 #include "java/df_jattach.h"
 #include "attach.h"
 
+extern int g_java_syms_write_bytes_max;
+
 void gen_java_symbols_file(int pid)
 {
 	int target_ns_pid = get_nspid(pid);
@@ -31,9 +34,28 @@ void gen_java_symbols_file(int pid)
 		return;
 	}
 
-	char args[32];
-	snprintf(args, sizeof(args), "%d", pid);
+	char args[PERF_PATH_SZ * 2];
+	if (!is_same_mntns(pid)) {
+		snprintf(args, sizeof(args), "%d %d,%s,%s", pid,
+			 g_java_syms_write_bytes_max,
+			 PERF_MAP_FILE_FMT, PERF_MAP_LOG_FILE_FMT);
+	} else {
+		snprintf(args, sizeof(args), "%d %d,%s,%s", pid,
+			 g_java_syms_write_bytes_max,
+			 DF_AGENT_LOCAL_PATH_FMT ".map",
+			 DF_AGENT_LOCAL_PATH_FMT ".log");
+	}
+
 	exec_command(DF_JAVA_ATTACH_CMD, args);
+	if (!is_same_mntns(pid)) {
+		if (copy_file_from_target_ns(pid, target_ns_pid, "map") ||
+		    copy_file_from_target_ns(pid, target_ns_pid, "log"))
+			ebpf_warning("Copy pid %d files failed\n", pid);
+		clear_target_ns(pid, target_ns_pid);
+	}
+}
 
-	clear_target_ns_so(pid, target_ns_pid);
+void clean_local_java_symbols_files(int pid)
+{
+	clear_local_perf_files(pid);
 }
diff --git a/agent/src/ebpf/user/profile/attach.h b/agent/src/ebpf/user/profile/attach.h
index c934da7c416..84488126667 100644
--- a/agent/src/ebpf/user/profile/attach.h
+++ b/agent/src/ebpf/user/profile/attach.h
@@ -19,4 +19,5 @@
 
 #define DF_JAVA_ATTACH_CMD "/usr/bin/deepflow-jattach"
 void gen_java_symbols_file(int pid);
+void clean_local_java_symbols_files(int pid);
 #endif /* ATTACH_H */
diff --git a/agent/src/ebpf/user/profile/java/agent.c b/agent/src/ebpf/user/profile/java/agent.c
index 020068b74eb..d4de5b2aeb1 100644
--- a/agent/src/ebpf/user/profile/java/agent.c
+++ b/agent/src/ebpf/user/profile/java/agent.c
@@ -35,17 +35,29 @@
 #include <jvmticmlr.h>
 
 #include "../../config.h"
+#include "config.h"
 
 #define STRING_BUFFER_SIZE 2000
 #define BIG_STRING_BUFFER_SIZE 20000
 
-#define PERF_MAP_FILE_FMT "/tmp/perf-%d.map"
-#define PERF_MAP_LOG_FILE_FMT "/tmp/perf-%d.log"
+/*
+ * HotSpot JVM does not support agent unloading. However, you
+ * may "attach" the same library multiple times with different
+ * arguments. The library will not be loaded again, but
+ * Agent_OnAttach will still be called multiple times with
+ * different arguments.
+ */
+
+pthread_mutex_t g_df_lock;
 
-static pthread_mutex_t g_lock;
+char perf_map_file_path[128];
+char perf_log_file_path[128];
 
-static FILE *perf_map_file_ptr = NULL;
-static FILE *perf_map_log_file_ptr = NULL;
+FILE *perf_map_file_ptr = NULL;
+FILE *perf_map_log_file_ptr = NULL;
+
+int g_perf_map_file_size_limit;
+int g_perf_map_file_size;
 
 #define _(e)                                                                \
 	if (e != JNI_OK) {                                                  \
@@ -54,7 +66,7 @@ static FILE *perf_map_log_file_ptr = NULL;
 		return e;                                                   \
 	}
 
-static void df_log(const char *format, ...)
+void df_log(const char *format, ...)
 {
 	if (perf_map_log_file_ptr) {
 		va_list ap;
@@ -66,7 +78,7 @@ static void df_log(const char *format, ...)
 	}
 }
 
-static jint df_open_file(pid_t pid, const char *fmt, FILE ** ptr)
+jint df_open_file(pid_t pid, const char *fmt, FILE ** ptr)
 {
 	FILE *file_ptr;
 	char filename[50];	// Assuming the filename won't exceed 50 characters
@@ -86,17 +98,50 @@ static jint df_open_file(pid_t pid, const char *fmt, FILE ** ptr)
 	return JNI_OK;
 }
 
-static jint open_perf_map_file(pid_t pid)
+jint open_perf_map_file(pid_t pid)
 {
-	return df_open_file(pid, PERF_MAP_FILE_FMT, &perf_map_file_ptr);
+	return df_open_file(pid, perf_map_file_path, &perf_map_file_ptr);
 }
 
-static jint open_perf_map_log_file(pid_t pid)
+jint open_perf_map_log_file(pid_t pid)
 {
-	return df_open_file(pid, PERF_MAP_LOG_FILE_FMT, &perf_map_log_file_ptr);
+	return df_open_file(pid, perf_log_file_path, &perf_map_log_file_ptr);
 }
 
-static jint close_files(void)
+jint df_agent_config(char *opts)
+{
+	g_perf_map_file_size = 0;
+
+	char buf[300];
+	char *start;
+	start = buf;
+	snprintf(buf, sizeof(buf), "%s", opts);
+
+	/* g_perf_map_file_size_limit */
+	char *p = strchr(start, ',');
+	if (p == NULL)
+		return JNI_ERR;
+	*p = '\0';
+	g_perf_map_file_size_limit = atoi(start);
+
+	/* perf_map_file_path[] */
+	start = ++p;
+	p = strchr(start, ',');
+	if (p == NULL)
+		return JNI_ERR;
+	*p = '\0';
+	snprintf(perf_map_file_path, sizeof(perf_map_file_path), "%s", start);
+
+	/* perf_log_file_path[] */
+	start = ++p;
+	if (start == NULL)
+		return JNI_ERR;
+	snprintf(perf_log_file_path, sizeof(perf_log_file_path), "%s", start);
+
+	return JNI_OK;
+}
+
+jint close_files(void)
 {
 	if (perf_map_file_ptr)
 		fclose(perf_map_file_ptr);
@@ -131,8 +176,8 @@ jint get_jvmti_env(JavaVM * jvm, jvmtiEnv ** jvmti)
 	return JNI_OK;
 }
 
-static void jvmti_err_log(jvmtiEnv * jvmti, const jvmtiError err_num,
-			  const char *help_msg_or_null)
+void jvmti_err_log(jvmtiEnv * jvmti, const jvmtiError err_num,
+		   const char *help_msg_or_null)
 {
 	if (err_num == JVMTI_ERROR_NONE) {
 		return;		// No need to log if there's no error
@@ -152,7 +197,7 @@ static void jvmti_err_log(jvmtiEnv * jvmti, const jvmtiError err_num,
 	df_log("[error][%d] %s: %s.", err_num, err_name_str, help_message);
 }
 
-static jint enable_capabilities(jvmtiEnv * jvmti)
+jint enable_capabilities(jvmtiEnv * jvmti)
 {
 	jvmtiCapabilities capabilities;
 	memset(&capabilities, 0, sizeof(jvmtiCapabilities));
@@ -171,25 +216,34 @@ static jint enable_capabilities(jvmtiEnv * jvmti)
 	return JNI_OK;
 }
 
-static void deallocate(jvmtiEnv * jvmti, void *string)
+void deallocate(jvmtiEnv * jvmti, void *string)
 {
 	if (string != NULL)
 		(*jvmti)->Deallocate(jvmti, (unsigned char *)string);
 }
 
-static void write_symbol(const void *code_addr, unsigned int code_size,
-			 const char *entry)
+void df_write_symbol(const void *code_addr, unsigned int code_size,
+		     const char *entry)
 {
-	pthread_mutex_lock(&g_lock);
-	fprintf(perf_map_file_ptr, "%lx %x %s\n", (unsigned long)code_addr,
-		code_size, entry);
+	char symbol_str[1024];
+	int bytes_all = 0;
+	pthread_mutex_lock(&g_df_lock);
+	snprintf(symbol_str, sizeof(symbol_str), "%lx %x %s\n",
+		 (unsigned long)code_addr, code_size, entry);
+	bytes_all = g_perf_map_file_size + strlen(symbol_str);
+	if (bytes_all >= g_perf_map_file_size_limit) {
+		pthread_mutex_unlock(&g_df_lock);
+		return;
+	}
+	fprintf(perf_map_file_ptr, "%s", symbol_str);
 	fflush(perf_map_file_ptr);
-	pthread_mutex_unlock(&g_lock);
+	g_perf_map_file_size = bytes_all;
+	pthread_mutex_unlock(&g_df_lock);
 }
 
-static void generate_single_entry(jvmtiEnv * jvmti,
-				  jmethodID method, const void *code_addr,
-				  jint code_size)
+void generate_single_entry(jvmtiEnv * jvmti,
+			   jmethodID method, const void *code_addr,
+			   jint code_size)
 {
 	jclass class;
 	char *method_name = NULL;
@@ -225,10 +279,10 @@ static void generate_single_entry(jvmtiEnv * jvmti,
 	deallocate(jvmti, (unsigned char *)method_name);
 	deallocate(jvmti, (unsigned char *)msig);
 
-	write_symbol(code_addr, (unsigned int)code_size, output);
+	df_write_symbol(code_addr, (unsigned int)code_size, output);
 }
 
-static void JNICALL
+void JNICALL
 cbCompiledMethodLoad(jvmtiEnv * jvmti,
 		     jmethodID method,
 		     jint code_size,
@@ -239,22 +293,22 @@ cbCompiledMethodLoad(jvmtiEnv * jvmti,
 	generate_single_entry(jvmti, method, code_addr, code_size);
 }
 
-static void JNICALL
+void JNICALL
 cbDynamicCodeGenerated(jvmtiEnv * jvmti,
 		       const char *name, const void *address, jint length)
 {
 
-	write_symbol(address, (unsigned int)length, name);
+	df_write_symbol(address, (unsigned int)length, name);
 }
 
-static void JNICALL
+void JNICALL
 cbCompiledMethodUnload(jvmtiEnv * jvmti, jmethodID method, const void *address)
 {
 
-	write_symbol(address, 0, "");
+	df_write_symbol(address, 0, "");
 }
 
-static jvmtiError set_callback_funs(jvmtiEnv * jvmti)
+jvmtiError set_callback_funs(jvmtiEnv * jvmti)
 {
 	jvmtiEventCallbacks callbacks;
 
@@ -274,7 +328,7 @@ static jvmtiError set_callback_funs(jvmtiEnv * jvmti)
 	return JNI_OK;
 }
 
-static jint set_notification_modes(jvmtiEnv * jvmti, jvmtiEventMode mode)
+jint set_notification_modes(jvmtiEnv * jvmti, jvmtiEventMode mode)
 {
 	jvmtiError error;
 
@@ -311,7 +365,7 @@ static jint set_notification_modes(jvmtiEnv * jvmti, jvmtiEventMode mode)
 	return JNI_OK;
 }
 
-static jint replay_callbacks(jvmtiEnv * jvmti)
+jint replay_callbacks(jvmtiEnv * jvmti)
 {
 	jvmtiError error;
 	jvmtiPhase phase;
@@ -351,8 +405,13 @@ JNIEXPORT jint JNICALL
 Agent_OnAttach(JavaVM * vm, char *options, void *reserved)
 {
 	jvmtiEnv *jvmti;
-	pthread_mutex_init(&g_lock, NULL);
+	pthread_mutex_init(&g_df_lock, NULL);
+	_(df_agent_config(options));
 	_(open_perf_map_log_file(getpid()));
+	df_log("- JVMTI g_perf_map_file_size_limit: %d, "
+	       "perf_map_file_path: %s perf_log_file_path: %s",
+	       g_perf_map_file_size_limit,
+	       perf_map_file_path, perf_log_file_path);
 	_(open_perf_map_file(getpid()));
 	_(get_jvmti_env(vm, &jvmti));
 	_(enable_capabilities(jvmti));
@@ -361,7 +420,7 @@ Agent_OnAttach(JavaVM * vm, char *options, void *reserved)
 	_(replay_callbacks(jvmti));
 	_(set_notification_modes(jvmti, JVMTI_DISABLE));
 
-	df_log("JVMTI symbolization agent startup sequence complete.");
+	df_log("- JVMTI symbolization agent startup sequence complete.");
 
 	close_files();
 	return 0;
diff --git a/agent/src/ebpf/user/profile/java/config.h b/agent/src/ebpf/user/profile/java/config.h
new file mode 100644
index 00000000000..b07bca29f53
--- /dev/null
+++ b/agent/src/ebpf/user/profile/java/config.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2023 Yunshan Networks
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef DF_JAVA_CONFIG_H
+#define DF_JAVA_CONFIG_H
+
+#define TARGET_NS_STORAGE_PATH "/proc/%d/root/deepflow"
+
+#define AGENT_LIB_NAME "df_java_agent.so"
+#define AGENT_LIB_SRC_PATH "/tmp/" AGENT_LIB_NAME
+#define AGENT_LIB_TARGET_PATH "/deepflow/" AGENT_LIB_NAME
+
+#define AGENT_MUSL_LIB_NAME "df_java_agent_musl.so"
+#define AGENT_MUSL_LIB_SRC_PATH "/tmp/" AGENT_MUSL_LIB_NAME
+#define AGENT_MUSL_LIB_TARGET_PATH "/deepflow/" AGENT_MUSL_LIB_NAME
+
+#define JAVA_LOG_TAG "[JAVA]"
+
+#define PERF_PATH_SZ 256
+#define DF_AGENT_MAP_PATH_FMT "/proc/%d/root/deepflow/df-perf-%d.map"
+#define DF_AGENT_LOG_PATH_FMT "/proc/%d/root/deepflow/df-perf-%d.log"
+
+#define DF_AGENT_PATH_FMT "/proc/%d/root/deepflow/df-perf-%d"
+#define DF_AGENT_LOCAL_PATH_FMT "/tmp/perf-%d"
+
+#define PERF_MAP_FILE_FMT "/deepflow/df-perf-%d.map"
+#define PERF_MAP_LOG_FILE_FMT "/deepflow/df-perf-%d.log"
+
+#endif /* DF_JAVA_CONFIG_H */
diff --git a/agent/src/ebpf/user/profile/java/df_jattach.c b/agent/src/ebpf/user/profile/java/df_jattach.c
index 7f968c5e3c8..116de8c9c51 100644
--- a/agent/src/ebpf/user/profile/java/df_jattach.c
+++ b/agent/src/ebpf/user/profile/java/df_jattach.c
@@ -18,10 +18,12 @@
 #include <unistd.h>
 #include <sys/stat.h>
 #include <dlfcn.h>
+#include <dirent.h>
 
 #include "../../config.h"
 #include "../../common.h"
 #include "../../log.h"
+#include "config.h"
 #include "df_jattach.h"
 
 #define jattach_log(fmt, ...)				\
@@ -67,8 +69,17 @@ static int copy_agent_libs_into_target_ns(pid_t target_pid, int target_uid,
 	int ret;
 	char copy_target_path[MAX_PATH_LENGTH];
 	int len = snprintf(copy_target_path, sizeof(copy_target_path),
-			   "/proc/%d/root/tmp", target_pid);
+			   TARGET_NS_STORAGE_PATH, target_pid);
 	if (access(copy_target_path, F_OK)) {
+		/*
+		 * The purpose of umask(0); is to set the current process's file
+		 * creation mask (umask) to 0, which means that no permission
+		 * bits will be cleared when creating a file or directory. Files
+		 * and directories will have the permission bits specified at the
+		 * time of creation.
+		 */
+		umask(0);
+
 		if (mkdir(copy_target_path, 0777) != 0) {
 			jattach_log(JAVA_LOG_TAG "Fun %s cannot mkdir() '%s'\n",
 				    __func__, copy_target_path);
@@ -166,21 +177,34 @@ static void select_sitable_agent_lib(pid_t pid, bool is_same_mntns)
 	df_enter_ns(pid, "mnt", &mnt_self_fd);
 
 	agent_lib_so_path[0] = '\0';
-	if (test_dl_open(AGENT_LIB_SRC_PATH)) {
-		snprintf(agent_lib_so_path, MAX_PATH_LENGTH, "%s",
+	char test_path[PERF_PATH_SZ];
+	if (!is_same_mntns)
+		snprintf(test_path, sizeof(test_path), "%s",
+			 AGENT_LIB_TARGET_PATH);
+	else
+		snprintf(test_path, sizeof(test_path), "%s",
 			 AGENT_LIB_SRC_PATH);
+
+	if (test_dl_open(test_path)) {
+		snprintf(agent_lib_so_path, MAX_PATH_LENGTH, "%s", test_path);
 		jattach_log(JAVA_LOG_TAG
 			    "Func %s target PID %d test %s, success.\n",
-			    __func__, pid, AGENT_LIB_SRC_PATH);
+			    __func__, pid, test_path);
 		goto found;
 	}
 
-	if (test_dl_open(AGENT_MUSL_LIB_SRC_PATH)) {
-		snprintf(agent_lib_so_path, MAX_PATH_LENGTH, "%s",
+	if (!is_same_mntns)
+		snprintf(test_path, sizeof(test_path), "%s",
+			 AGENT_MUSL_LIB_TARGET_PATH);
+	else
+		snprintf(test_path, sizeof(test_path), "%s",
 			 AGENT_MUSL_LIB_SRC_PATH);
+
+	if (test_dl_open(test_path)) {
+		snprintf(agent_lib_so_path, MAX_PATH_LENGTH, "%s", test_path);
 		jattach_log(JAVA_LOG_TAG
 			    "Func %s target PID %d test %s, success.\n",
-			    __func__, pid, AGENT_MUSL_LIB_SRC_PATH);
+			    __func__, pid, test_path);
 		goto found;
 	}
 
@@ -189,10 +213,10 @@ static void select_sitable_agent_lib(pid_t pid, bool is_same_mntns)
 found:
 
 	if (!is_same_mntns) {
-		if (strcmp(agent_lib_so_path, AGENT_LIB_SRC_PATH) == 0) {
-			clear_target_ns_tmp_file(AGENT_MUSL_LIB_SRC_PATH);
+		if (strcmp(agent_lib_so_path, AGENT_LIB_TARGET_PATH) == 0) {
+			clear_target_ns_tmp_file(AGENT_MUSL_LIB_TARGET_PATH);
 		} else {
-			clear_target_ns_tmp_file(AGENT_LIB_SRC_PATH);
+			clear_target_ns_tmp_file(AGENT_LIB_TARGET_PATH);
 		}
 	}
 
@@ -200,10 +224,11 @@ static void select_sitable_agent_lib(pid_t pid, bool is_same_mntns)
 	df_exit_ns(mnt_self_fd);
 }
 
-static int attach(pid_t pid)
+static int attach(pid_t pid, char *opts)
 {
-	char *argv[] = { "load", agent_lib_so_path, "true" };
+	char *argv[] = { "load", agent_lib_so_path, "true", opts };
 	int argc = sizeof(argv) / sizeof(argv[0]);
+	printf("argc %d opts %s\n", argc, argv[3]);
 	int ret = jattach(pid, argc, (char **)argv);
 	jattach_log(JAVA_LOG_TAG
 		    "jattach pid %d argv: \"load %s true\" return %d\n", pid,
@@ -240,24 +265,58 @@ static inline bool __is_same_ns(int target_pid, const char *tag)
 	return false;
 }
 
-static bool __unused is_same_netns(int target_pid)
+static bool __unused is_same_netns(int pid)
 {
-	return __is_same_ns(target_pid, "net");
+	return __is_same_ns(pid, "net");
 }
 
-static bool is_same_mntns(int target_pid)
+bool is_same_mntns(int pid)
 {
-	return __is_same_ns(target_pid, "mnt");
+	return __is_same_ns(pid, "mnt");
+}
+
+void clear_local_perf_files(int pid)
+{
+	char local_path[MAX_PATH_LENGTH];
+	snprintf(local_path, sizeof(local_path),
+		 DF_AGENT_LOCAL_PATH_FMT ".map", pid);
+	clear_target_ns_tmp_file(local_path);
+
+	snprintf(local_path, sizeof(local_path),
+		 DF_AGENT_LOCAL_PATH_FMT ".log", pid);
+	clear_target_ns_tmp_file(local_path);
+}
+
+void __unused clear_old_target_perf_files(int pid)
+{
+	if (is_same_mntns(pid))
+                return;
+
+	/* if pid == target_ns_pid, run in same namespace */
+	int target_ns_pid = get_nspid(pid);
+	if (target_ns_pid < 0) {
+		return;
+	}
+
+	char path[MAX_PATH_LENGTH];
+	snprintf(path, sizeof(path),
+		 "/proc/%d/root/tmp/perf-%d.log", pid, target_ns_pid);
+	if (access(path, F_OK) == 0) {
+		clear_target_ns_tmp_file(path);
+		snprintf(path, sizeof(path),
+			 "/proc/%d/root/tmp/perf-%d.map", pid, target_ns_pid);
+		clear_target_ns_tmp_file(path);
+	}
 }
 
 void clear_target_ns(int pid, int target_ns_pid)
 {
 	/*
 	 * Delete files:
-	 *  /tmp/perf-<pid>.map
-	 *  /tmp/perf-<pid>.log
-	 *  /tmp/df_java_agent.so
-	 *  /tmp/df_java_agent_musl.so
+	 *  path/df_perf-<pid>.map
+	 *  path/df_perf-<pid>.log
+	 *  path/df_java_agent.so
+	 *  path/df_java_agent_musl.so
 	 */
 
 	if (is_same_mntns(pid))
@@ -265,17 +324,21 @@ void clear_target_ns(int pid, int target_ns_pid)
 
 	char target_path[MAX_PATH_LENGTH];
 	snprintf(target_path, sizeof(target_path),
-		 "/proc/%d/root/tmp/perf-%d.map", pid, target_ns_pid);
+		 DF_AGENT_MAP_PATH_FMT, pid, target_ns_pid);
 	clear_target_ns_tmp_file(target_path);
 	snprintf(target_path, sizeof(target_path),
-		 "/proc/%d/root/tmp/perf-%d.log", pid, target_ns_pid);
+		 DF_AGENT_LOG_PATH_FMT, pid, target_ns_pid);
 	clear_target_ns_tmp_file(target_path);
+
 	snprintf(target_path, sizeof(target_path), "/proc/%d/root%s", pid,
-		 AGENT_MUSL_LIB_SRC_PATH);
+		 AGENT_MUSL_LIB_TARGET_PATH);
 	clear_target_ns_tmp_file(target_path);
 	snprintf(target_path, sizeof(target_path), "/proc/%d/root%s", pid,
-		 AGENT_LIB_SRC_PATH);
+		 AGENT_LIB_TARGET_PATH);
 	clear_target_ns_tmp_file(target_path);
+
+	snprintf(target_path, sizeof(target_path), TARGET_NS_STORAGE_PATH, pid);
+	rmdir(target_path);
 }
 
 void clear_target_ns_so(int pid, int target_ns_pid)
@@ -289,10 +352,10 @@ void clear_target_ns_so(int pid, int target_ns_pid)
 
 	char target_path[MAX_PATH_LENGTH];
 	snprintf(target_path, sizeof(target_path), "/proc/%d/root%s", pid,
-		 AGENT_MUSL_LIB_SRC_PATH);
+		 AGENT_MUSL_LIB_TARGET_PATH);
 	clear_target_ns_tmp_file(target_path);
 	snprintf(target_path, sizeof(target_path), "/proc/%d/root%s", pid,
-		 AGENT_LIB_SRC_PATH);
+		 AGENT_LIB_TARGET_PATH);
 	clear_target_ns_tmp_file(target_path);
 }
 
@@ -329,34 +392,34 @@ static inline void switch_to_root_ns(int root_fd)
 	df_exit_ns(root_fd);
 }
 
-void copy_file_from_target_ns(int pid, int ns_pid, const char *file_type)
+int copy_file_from_target_ns(int pid, int ns_pid, const char *file_type)
 {
-	if (is_same_mntns(pid))
-		return;
-
-	char target_path[128];
-	char src_path[128];
-	snprintf(src_path, sizeof(src_path), "/proc/%d/root/tmp/perf-%d.%s",
+	char target_path[PERF_PATH_SZ];
+	char src_path[PERF_PATH_SZ];
+	snprintf(src_path, sizeof(src_path), DF_AGENT_PATH_FMT ".%s",
 		 pid, ns_pid, file_type);
-	snprintf(target_path, sizeof(target_path), "/tmp/perf-%d.%s", pid,
-		 file_type);
+	snprintf(target_path, sizeof(target_path),
+		 DF_AGENT_LOCAL_PATH_FMT ".%s", pid, file_type);
 
-	if (access(src_path, F_OK) != 0) {
-		return;
+	if (access(src_path, F_OK)) {
+		return -1;
 	}
 
 	if (access(target_path, F_OK) == 0) {
-		if (unlink(target_path) != 0)
-			return;
+		if (unlink(target_path) != 0) {
+			return -1;
+		}
 	}
 
 	if (copy_file(src_path, target_path)) {
 		jattach_log("Copy '%s' to '%s' failed.\n", src_path,
 			    target_path);
 	}
+
+	return 0;
 }
 
-int java_attach(pid_t pid)
+int java_attach(pid_t pid, char *opts)
 {
 #ifdef NS_FILES_COPY_TEST
 	int net_fd, ipc_fd, mnt_fd;
@@ -368,22 +431,31 @@ int java_attach(pid_t pid)
 		return -1;
 	}
 
+	/* if pid == target_ns_pid, run in same namespace */
 	int target_ns_pid = get_nspid(pid);
 	if (target_ns_pid < 0) {
 		return -1;
 	}
 
+	/*
+	 * If the agent is installed directly on the node or host,
+	 * be careful to delete the perf-pid.map and
+	 * perf-pid.log on them.
+	 */
 	bool is_same_mnt = is_same_mntns(pid);
 	if (is_same_mnt) {
-		char path[128];
-		snprintf(path, sizeof(path), "/proc/%d/root/tmp/perf-%d.map",
-			 pid, target_ns_pid);
+		char path[PERF_PATH_SZ];
+		snprintf(path, sizeof(path), DF_AGENT_LOCAL_PATH_FMT ".map",
+			 pid);
 		clear_target_ns_tmp_file(path);
-		snprintf(path, sizeof(path), "/proc/%d/root/tmp/perf-%d.log",
-			 pid, target_ns_pid);
+		snprintf(path, sizeof(path), DF_AGENT_LOCAL_PATH_FMT ".log",
+			 pid);
 		clear_target_ns_tmp_file(path);
-
 	} else {
+		/*
+		 * Delete the files on the target file system if they
+		 * are not on the same mount point.
+		 */
 		clear_target_ns(pid, target_ns_pid);
 	}
 
@@ -434,7 +506,7 @@ int java_attach(pid_t pid)
 		goto failed;
 #endif
 
-	ret = attach(pid);
+	ret = attach(pid, opts);
 
 #ifdef NS_FILES_COPY_TEST
 	/*
@@ -477,15 +549,47 @@ int java_attach(pid_t pid)
 }
 
 #ifdef JAVA_AGENT_ATTACH_TOOL
+static void clean_old_java_perf_files(void)
+{
+	struct dirent *entry = NULL;
+	DIR *fddir = NULL;
+
+	fddir = opendir("/proc/");
+	if (fddir == NULL) {
+		jattach_log("Failed to open '/proc/'\n");
+		return;
+	}
+
+	pid_t pid;
+	while ((entry = readdir(fddir)) != NULL) {
+		pid = atoi(entry->d_name);
+		if (entry->d_type == DT_DIR && pid > 0 && is_process(pid)) {
+			char comm[16];
+			memset(comm, 0, sizeof(comm));
+			get_process_starttime_and_comm(pid, comm, sizeof(comm));
+			if (strcmp(comm, "java") == 0) {
+				clear_old_target_perf_files(pid);
+			}
+		}
+	}
+
+	closedir(fddir);
+}
+
 int main(int argc, char **argv)
 {
-	if (argc != 2) {
-		fprintf(stderr, "Usage: %s <pid>\n", argv[0]);
+	if (strcmp(argv[1], "clean") == 0) {
+		clean_old_java_perf_files();
+		return 0;
+	}
+
+	if (argc != 3) {
+		fprintf(stderr, "Usage: %s <pid> <opts>\n", argv[0]);
 		return -1;
 	}
 
 	log_to_stdout = true;
 	int pid = atoi(argv[1]);
-	return java_attach(pid);
+	return java_attach(pid, argv[2]);
 }
 #endif /* JAVA_AGENT_ATTACH_TEST */
diff --git a/agent/src/ebpf/user/profile/java/df_jattach.h b/agent/src/ebpf/user/profile/java/df_jattach.h
index 5b4f659ce27..62ace1d2d32 100644
--- a/agent/src/ebpf/user/profile/java/df_jattach.h
+++ b/agent/src/ebpf/user/profile/java/df_jattach.h
@@ -17,18 +17,13 @@
 #ifndef DF_JATTACH_H
 #define DF_JATTACH_H
 
-#define AGENT_LIB_NAME "df_java_agent.so"
-#define AGENT_LIB_SRC_PATH "/tmp/" AGENT_LIB_NAME
-
-#define AGENT_MUSL_LIB_NAME "df_java_agent_musl.so"
-#define AGENT_MUSL_LIB_SRC_PATH "/tmp/" AGENT_MUSL_LIB_NAME
-
-#define JAVA_LOG_TAG "[JAVA]"
-
 typedef uint64_t(*agent_test_t) (void);
 
 void clear_target_ns_tmp_file(const char *target_path);
-void copy_file_from_target_ns(int pid, int ns_pid, const char *file_type);
+int copy_file_from_target_ns(int pid, int ns_pid, const char *file_type);
 void clear_target_ns(int pid, int target_ns_pid);
 void clear_target_ns_so(int pid, int target_ns_pid);
+void clear_local_perf_files(int pid);
+bool is_same_mntns(int target_pid);
+void __unused clear_old_target_perf_files(int pid);
 #endif /* DF_JATTACH_H */
diff --git a/agent/src/ebpf/user/profile/perf_profiler.c b/agent/src/ebpf/user/profile/perf_profiler.c
index bba4b960650..86f7334b082 100644
--- a/agent/src/ebpf/user/profile/perf_profiler.c
+++ b/agent/src/ebpf/user/profile/perf_profiler.c
@@ -30,6 +30,7 @@
 #include "../types.h"
 #include "../vec.h"
 #include "../tracer.h"
+#include "../socket.h"
 #include "attach.h"
 #include "perf_profiler.h"
 #include "../elf.h"
@@ -40,6 +41,7 @@
 #include "stringifier.h"
 #include "../table.h"
 #include <regex.h>
+#include "java/config.h"
 #include "java/df_jattach.h"
 
 #include "../perf_profiler_bpf_common.c"
@@ -64,6 +66,9 @@
 #define CP_TRACER_NAME	"continuous_profiler"
 #define CP_PERF_PG_NUM	16
 
+/* The maximum bytes limit for writing the df_perf-PID.map file by agent.so */
+int g_java_syms_write_bytes_max;
+
 extern int major, minor;
 extern char linux_release[128];
 extern __thread uword thread_index;
@@ -352,14 +357,24 @@ static void print_cp_data(stack_trace_msg_t * msg)
 	else
 		cid = (char *)msg->container_id;
 
-	ebpf_info
-	    ("netns_id %lu container_id %s process_name %s pid %u stime %lu "
-	     "u_stack_id %lu k_statck_id %lu cpu %u count %u comm %s tiemstamp"
-	     " %lu datalen %u data %s\n",
-	     msg->netns_id, cid,
-	     msg->process_name, msg->pid, msg->stime, msg->u_stack_id,
-	     msg->k_stack_id, msg->cpu, msg->count, msg->comm, msg->time_stamp,
-	     msg->data_len, msg->data);
+	/*
+	 * TODO(@jiping)
+	 * We didn't use the 'ebpf_info()' interface here to send data to the
+	 * Rust log. The reason is that after 'ebpf_info()' -> 'rust_info_wrapper()',
+	 * we noticed some instability, and occasional segmentation faults occurred.
+	 * This is something that needs to be resolved in the future. 
+	 */
+	fprintf(stdout,
+		"\n-------------------------------\n"
+		"netns_id %lu container_id %s process_name %s pid %u stime %lu "
+		"u_stack_id %u k_statck_id %u cpu %u count %u comm %s tiemstamp"
+		" %lu datalen %u data %s\n",
+		msg->netns_id, cid,
+		msg->process_name, msg->pid, msg->stime, msg->u_stack_id,
+		msg->k_stack_id, msg->cpu, msg->count, msg->comm,
+		msg->time_stamp, msg->data_len, msg->data);
+
+	fflush(stdout);
 }
 
 static void cpdbg_process(stack_trace_msg_t * msg)
@@ -745,9 +760,16 @@ static void cp_reader_work(void *arg)
 
 		/* 
 		 * Waiting for the regular expression to be configured
-		 * and start working. 
+		 * and start working. Ensure the socket tracer is in
+		 * the 'running' state to prevent starting the profiler
+		 * before the socket tracer has completed its attach
+		 * operation. The profiler's processing depends on probe
+		 * interfaces provided by the socket tracer, such as process
+		 * exit events. We want to ensure that everything is ready
+		 * before the profiler performs address translation.
 		 */
-		if (unlikely(!regex_existed)) {
+		if (unlikely(!regex_existed ||
+			     get_socket_tracer_state() != TRACER_RUNNING)) {
 			exec_symbol_cache_update();
 			sleep(1);
 			continue;
@@ -821,6 +843,10 @@ static int create_profiler(struct bpf_tracer *tracer)
 		return ETR_NORESOURCE;
 	}
 
+	/* clear old perf files */
+	exec_command("/usr/bin/rm -rf /tmp/perf-*.map", "");
+	exec_command("/usr/bin/rm -rf /tmp/perf-*.log", "");
+
 	/* syms_cache_hash maps from pid to BCC symbol cache.
 	 * Use of void* is inherited from the BCC library. */
 	create_and_init_symbolizer_caches();
@@ -1018,15 +1044,24 @@ static struct tracer_sockopts cpdbg_sockopts = {
 /*
  * start continuous profiler
  * @freq sample frequency, Hertz. (e.g. 99 profile stack traces at 99 Hertz)
+ * @java_syms_space_limit The maximum space occupied by the Java symbol files
+ *                        in the target POD. 
+ * @java_syms_update_delay To allow Java to run for an extended period and gather
+ *                    more symbol information, we delay symbol retrieval when
+ *                    encountering unknown symbols. The default value is
+ *                    'JAVA_SYMS_UPDATE_DELAY_DEF'.
+ *                    This represents the delay in seconds.
  * @callback Profile data processing callback interface
  * @returns 0 on success, < 0 on error
  */
-int start_continuous_profiler(int freq, tracer_callback_t callback)
+int start_continuous_profiler(int freq, int java_syms_space_limit,
+			      int java_syms_update_delay,
+			      tracer_callback_t callback)
 {
 	char bpf_load_buffer_name[NAME_LEN];
 	void *bpf_bin_buffer;
 	uword buffer_sz;
-
+ 
 	// REQUIRES: Linux 4.9+ (BPF_PROG_TYPE_PERF_EVENT support).
 	if (check_kernel_version(4, 9) != 0) {
 		ebpf_warning
@@ -1048,6 +1083,19 @@ int start_continuous_profiler(int freq, tracer_callback_t callback)
 		return (-1);
 	}
 
+	int java_space_bytes = java_syms_space_limit * 1024 *1024;
+	if ((java_space_bytes < JAVA_POD_WRITE_FILES_SPACE_MIN) ||
+	    (java_space_bytes > JAVA_POD_WRITE_FILES_SPACE_MAX))
+		java_space_bytes = JAVA_POD_WRITE_FILES_SPACE_DEF;
+	g_java_syms_write_bytes_max = java_space_bytes - JAVA_POD_EXTRA_SPACE_MMA;
+	ebpf_info("set java_syms_write_bytes_max : %d\n", g_java_syms_write_bytes_max);
+
+	if ((java_syms_update_delay < JAVA_SYMS_UPDATE_DELAY_MIN) ||
+	    (java_syms_update_delay > JAVA_SYMS_UPDATE_DELAY_MAX))
+		java_syms_update_delay = JAVA_SYMS_UPDATE_DELAY_DEF;
+	set_java_syms_fetch_delay(java_syms_update_delay);
+	ebpf_info("set java_syms_update_delay : %lu\n", java_syms_update_delay);
+
 	atomic64_init(&process_lost_count);
 
 	/*
diff --git a/agent/src/ebpf/user/profile/perf_profiler.h b/agent/src/ebpf/user/profile/perf_profiler.h
index d8c01a90ab5..2687c76e872 100644
--- a/agent/src/ebpf/user/profile/perf_profiler.h
+++ b/agent/src/ebpf/user/profile/perf_profiler.h
@@ -142,7 +142,8 @@ typedef struct {
 } stack_trace_msg_t;
 
 int stop_continuous_profiler(void);
-int start_continuous_profiler(int freq,
+int start_continuous_profiler(int freq, int java_syms_space_limit,
+			      int java_syms_update_delay,
 			      tracer_callback_t callback);
 void process_stack_trace_data_for_flame_graph(stack_trace_msg_t *val);
 void release_flame_graph_hash(void);
diff --git a/agent/src/ebpf/user/profile/stringifier.c b/agent/src/ebpf/user/profile/stringifier.c
index dd2241bff3e..b03cd63234e 100644
--- a/agent/src/ebpf/user/profile/stringifier.c
+++ b/agent/src/ebpf/user/profile/stringifier.c
@@ -56,6 +56,7 @@
 #include "../table.h"
 #include "../bihash_8_8.h"
 #include "../bihash_16_8.h"
+#include "attach.h"
 #include "stringifier.h"
 #include <bcc/bcc_syms.h>
 
@@ -87,11 +88,11 @@ int init_stack_str_hash(stack_str_hash_t * h, const char *name)
 {
 	memset(h, 0, sizeof(*h));
 	u32 nbuckets = STRINGIFIER_STACK_STR_HASH_BUCKETS_NUM;
-	u64 hash_memory_size = STRINGIFIER_STACK_STR_HASH_MEM_SZ; // 1G bytes
+	u64 hash_memory_size = STRINGIFIER_STACK_STR_HASH_MEM_SZ;	// 1G bytes
 	h->private =
-		clib_mem_alloc_aligned("hash_ext_data",
-				       sizeof(struct stack_str_hash_ext_data),
-				       0, NULL);
+	    clib_mem_alloc_aligned("hash_ext_data",
+				   sizeof(struct stack_str_hash_ext_data),
+				   0, NULL);
 	if (h->private == NULL)
 		return ETR_NOMEM;
 
@@ -102,7 +103,7 @@ int init_stack_str_hash(stack_str_hash_t * h, const char *name)
 	return stack_str_hash_init(h, (char *)name, nbuckets, hash_memory_size);
 }
 
-void release_stack_str_hash(stack_str_hash_t *h)
+void release_stack_str_hash(stack_str_hash_t * h)
 {
 	if (h->private) {
 		struct stack_str_hash_ext_data *ext = h->private;
@@ -201,6 +202,20 @@ static char *resolve_addr(pid_t pid, u64 address, bool is_create, void *info_p)
 
 	int ret = symcache_resolve(pid, resolver, address, &sym);
 	if (ret == 0) {
+		if (info_p && pid > 0) {
+			struct symbolizer_proc_info *p = info_p;
+			if (p->new_java_syms_file) {
+				/*
+				 * TODO @jiping
+				 * If you delete the local file perf-PID.map immediately,
+				 * it will leave many symbolic links in /tmp/perf-PID.map.
+				 * We need a better method to delete these map files. Currently,
+				 * they are first saved in the local directory '/tmp'.
+				 */
+				//clean_local_java_symbols_files(pid);
+				p->new_java_syms_file = false;
+			}
+		}
 		ptr = symbol_name_fetch(pid, &sym);
 		if (ptr) {
 			char *p = ptr;
@@ -225,13 +240,12 @@ static char *resolve_addr(pid_t pid, u64 address, bool is_create, void *info_p)
 		 * [/lib64/xxx.so]
 		 */
 		char format_str[4096];
-		snprintf(format_str, sizeof(format_str), "[%s]",
-			 sym.module);
+		snprintf(format_str, sizeof(format_str), "[%s]", sym.module);
 		len = strlen(format_str);
 		ptr = create_symbol_str(len, format_str, "");
 		if (info_p) {
 			struct symbolizer_proc_info *p = info_p;
-			if (p->is_java) {
+			if (p->is_java && strstr(format_str, "perf-")) {
 				p->unknown_syms_found = true;
 			}
 		}
@@ -269,10 +283,9 @@ static char *build_stack_trace_string(struct bpf_tracer *t,
 				      const char *stack_map_name,
 				      pid_t pid,
 				      int stack_id,
-				      stack_str_hash_t *h,
+				      stack_str_hash_t * h,
 				      bool new_cache,
-				      int *ret_val,
-				      void *info_p)
+				      int *ret_val, void *info_p)
 {
 	ASSERT(pid >= 0 && stack_id >= 0);
 
@@ -347,14 +360,12 @@ static char *build_stack_trace_string(struct bpf_tracer *t,
 	return NULL;
 }
 
-static char *
-folded_stack_trace_string(struct bpf_tracer *t,
-			  int stack_id,
-			  pid_t pid,
-			  const char *stack_map_name,
-			  stack_str_hash_t *h,
-			  bool new_cache,
-			  void *info_p)
+static char *folded_stack_trace_string(struct bpf_tracer *t,
+				       int stack_id,
+				       pid_t pid,
+				       const char *stack_map_name,
+				       stack_str_hash_t * h,
+				       bool new_cache, void *info_p)
 {
 	ASSERT(pid >= 0 && stack_id >= 0);
 
@@ -429,13 +440,11 @@ static inline char *alloc_stack_trace_str(int len)
 	return trace_str;
 }
 
-char *
-resolve_and_gen_stack_trace_str(struct bpf_tracer *t,
-				struct stack_trace_key_t *v,
-				const char *stack_map_name,
-				stack_str_hash_t *h,
-				bool new_cache,
-				void *info_p)
+char *resolve_and_gen_stack_trace_str(struct bpf_tracer *t,
+				      struct stack_trace_key_t *v,
+				      const char *stack_map_name,
+				      stack_str_hash_t * h,
+				      bool new_cache, void *info_p)
 {
 	/*
 	 * We need to prepare a hashtable (stack_trace_strs) to record the results
@@ -460,18 +469,14 @@ resolve_and_gen_stack_trace_str(struct bpf_tracer *t,
 	if (v->kernstack >= 0) {
 		k_trace_str = folded_stack_trace_string(t, v->kernstack,
 							0, stack_map_name,
-							h,
-							new_cache,
-							info_p);
+							h, new_cache, info_p);
 	}
 
 	if (v->userstack >= 0) {
 		u_trace_str = folded_stack_trace_string(t, v->userstack,
 							v->tgid,
 							stack_map_name,
-							h,
-							new_cache,
-							info_p);
+							h, new_cache, info_p);
 	}
 
 	/* trace_str = u_stack_str_fn() + ";" + k_stack_str_fn(); */
diff --git a/agent/src/ebpf/user/socket.c b/agent/src/ebpf/user/socket.c
index 236d986a7f5..a562c6565be 100644
--- a/agent/src/ebpf/user/socket.c
+++ b/agent/src/ebpf/user/socket.c
@@ -57,6 +57,7 @@ static volatile uint64_t probes_act;
 
 extern int sys_cpus_count;
 extern bool *cpu_online;
+extern uint32_t attach_failed_count;
 
 static int infer_socktrace_fd;
 static uint32_t conf_max_socket_entries;
@@ -967,7 +968,7 @@ static int check_kern_adapt_and_state_update(void)
 	if (t == NULL)
 		return -1;
 
-	if (is_adapt_success(t)) {
+	if (is_adapt_success(t) && attach_failed_count == 0) {
 		ebpf_info("[eBPF Kernel Adapt] Linux %s adapt success. "
 			  "Set the status to TRACER_RUNNING\n", linux_release);
 		t->state = TRACER_RUNNING;
@@ -1970,6 +1971,15 @@ int socket_tracer_start(void)
 	return 0;
 }
 
+enum tracer_state __unused get_socket_tracer_state(void)
+{
+	struct bpf_tracer *t = find_bpf_tracer(SK_TRACER_NAME);
+	if (t == NULL) 
+		return TRACER_STOP_ERR;
+
+	return t->state;
+}
+
 static bool bpf_stats_map_collect(struct bpf_tracer *tracer,
 				  struct trace_stats *stats_total)
 {
diff --git a/agent/src/ebpf/user/socket.h b/agent/src/ebpf/user/socket.h
index d00d6d5ff8c..f3e47683f4f 100644
--- a/agent/src/ebpf/user/socket.h
+++ b/agent/src/ebpf/user/socket.h
@@ -326,4 +326,5 @@ int running_socket_tracer(tracer_callback_t handle,
 int register_event_handle(uint32_t type, void (*fn)(void *));
 int socket_tracer_stop(void);
 int socket_tracer_start(void);
+enum tracer_state get_socket_tracer_state(void);
 #endif /* DF_USER_SOCKET_H */
diff --git a/agent/src/ebpf/user/symbol.c b/agent/src/ebpf/user/symbol.c
index 9ce88dee2b6..83595181810 100644
--- a/agent/src/ebpf/user/symbol.c
+++ b/agent/src/ebpf/user/symbol.c
@@ -51,12 +51,29 @@
 static u64 add_symcache_count;
 static u64 free_symcache_count;
 
+/*
+ * To allow Java to run for an extended period and gather more symbol
+ * information, we delay symbol retrieval when encountering unknown symbols.
+ * The default value is 'JAVA_SYMS_UPDATE_DELAY_DEF'.
+ */
+static volatile u64 java_syms_fetch_delay; // In seconds.
+   
 /*
  * When a process exits, save the symbol cache pids
  * to be deleted.
  */
 static struct symbol_cache_del_pids cache_del_pids;
 
+void set_java_syms_fetch_delay(int delay_secs)
+{
+	java_syms_fetch_delay = delay_secs;
+}
+
+u64 get_java_syms_fetch_delay(void)
+{
+	return java_syms_fetch_delay;
+}
+
 void free_uprobe_symbol(struct symbol_uprobe *u_sym,
 			struct tracer_probes_conf *conf)
 {
@@ -490,9 +507,8 @@ static inline void free_symbolizer_cache_kvp(struct symbolizer_cache_kvp *kv)
 		if (p->is_java) {
 			/* Delete target ns Java files */
 			int pid = (int)kv->k.pid;
-			int target_ns_pid = get_nspid(pid);
-			if (target_ns_pid > 0) {
-				clear_target_ns(pid, target_ns_pid);
+			if (pid > 0) {
+				clean_local_java_symbols_files(pid);
 			}
 		}
 
@@ -631,29 +647,6 @@ u64 get_pid_stime(pid_t pid)
 	return 0;
 }
 
-static void java_proc_info_config(struct symbolizer_proc_info *p, int pid)
-{
-	if (strcmp(p->comm, "java") == 0)
-		p->is_java = true;
-	else
-		p->is_java = false;
-
-	if ((current_sys_time_secs() - (p->stime / 1000ULL)) >=
-	    PROC_INFO_VERIFY_TIME) {
-		p->verified = true;
-		/*
-		 * Ensure the Java program runs stably for a period
-		 * of time before obtaining the Java address and
-		 * symbol mapping table.
-		 */
-		if (p->is_java)
-			gen_java_symbols_file(pid);
-
-	} else {
-		p->verified = false;
-	}
-}
-
 static struct bcc_symbol_option lazy_opt = {
 	.use_debug_file = false,
 	.check_debug_file_crc = false,
@@ -665,6 +658,7 @@ static int config_symbolizer_proc_info(struct symbolizer_proc_info *p, int pid)
 {
 	memset(p, 0, sizeof(*p));
 	p->unknown_syms_found = false;
+	p->new_java_syms_file = false;
 	p->netns_id = get_netns_id_from_pid(pid);
 	if (p->netns_id == 0)
 		return ETR_INVAL;
@@ -678,6 +672,18 @@ static int config_symbolizer_proc_info(struct symbolizer_proc_info *p, int pid)
 	if (p->stime == 0)
 		return ETR_INVAL;
 
+	if (strcmp(p->comm, "java") == 0)
+		p->is_java = true;
+	else
+		p->is_java = false;
+
+	if ((current_sys_time_secs() - (p->stime / 1000ULL)) >=
+	    PROC_INFO_VERIFY_TIME) {
+		p->verified = true;
+	} else {
+		p->verified = false;
+	}
+
 	return ETR_OK;
 }
 
@@ -717,7 +723,6 @@ void get_process_info_by_pid(pid_t pid, u64 * stime, u64 * netns_id, char *name,
 			return;
 		}
 
-		java_proc_info_config(p, pid);
 		kv.v.proc_info_p = pointer_to_uword(p);
 		kv.v.cache = 0;
 		if (symbol_caches_hash_add_del
@@ -732,46 +737,7 @@ void get_process_info_by_pid(pid_t pid, u64 * stime, u64 * netns_id, char *name,
 	} else {
 		p = (struct symbolizer_proc_info *)kv.v.proc_info_p;
 		u64 curr_time = current_sys_time_secs();
-		if (p->verified) {
-			/*
-			 * If an unknown frame appears during the process of symbolizing
-			 * the address of the Java process, we need to re-obtain the sy-
-			 * mbols table of the Java process after a delay.
-			 */
-			if (p->is_java && p->unknown_syms_found
-			    && p->update_syms_table_time == 0) {
-				p->update_syms_table_time =
-				    curr_time + JAVA_SYMS_TABLE_UPDATE_PERIOD;
-			}
-
-			if (p->update_syms_table_time > 0
-			    && curr_time >= p->update_syms_table_time) {
-				/* Update java symbols table, will be executed during
-				 * the next Java symbolication */
-				gen_java_symbols_file(pid);
-				if (kv.v.cache) {
-					bcc_free_symcache((void *)kv.v.cache,
-							  kv.k.pid);
-					kv.v.cache =
-					    pointer_to_uword(bcc_symcache_new
-							     ((int)kv.k.pid,
-							      &lazy_opt));
-					if (symbol_caches_hash_add_del
-					    (h, (symbol_caches_hash_kv *) & kv,
-					     1 /* is_add */ )) {
-						ebpf_warning
-						    ("symbol_caches_hash_add_del() failed.(pid %d)\n",
-						     (int)kv.k.pid);
-					}
-				}
-
-				p->unknown_syms_found = false;
-				p->update_syms_table_time = 0;
-				ebpf_info
-				    ("Update java symbols table, (Pid %d)\n",
-				     (int)kv.k.pid);
-			}
-		} else {
+		if (!p->verified) {
 			if (((curr_time - (p->stime / 1000ULL)) <
 			     PROC_INFO_VERIFY_TIME)) {
 				goto fetch_proc_info;
@@ -807,15 +773,6 @@ void get_process_info_by_pid(pid_t pid, u64 * stime, u64 * netns_id, char *name,
 			else
 				p->is_java = false;
 
-			/*
-			 * We obtain the symbol table in the Java program after it has been
-			 * running for a while, rather than during the program's startup p-
-			 * rocess, to avoid situations where the symbol table cannot be re-
-			 * trieved.
-			 */
-			if (p->is_java)
-				gen_java_symbols_file(pid);
-
 			p->verified = true;
 		}
 
@@ -852,18 +809,66 @@ void *get_symbol_cache(pid_t pid, bool new_cache)
 		return k_resolver;
 
 	symbol_caches_hash_t *h = &syms_cache_hash;
+	struct symbolizer_proc_info *p;
 	struct symbolizer_cache_kvp kv;
 	kv.k.pid = (u64) pid;
 	kv.v.proc_info_p = 0;
 	kv.v.cache = 0;
 	if (symbol_caches_hash_search(h, (symbol_caches_hash_kv *) & kv,
 				      (symbol_caches_hash_kv *) & kv) == 0) {
-		if (kv.v.cache)
-			return (void *)kv.v.cache;
-
 		if (!new_cache)
 			return NULL;
 
+		p = (struct symbolizer_proc_info *)kv.v.proc_info_p;
+		u64 curr_time = current_sys_time_secs();
+		if (p->verified) {
+			/*
+			 * If an unknown frame appears during the process of symbolizing
+			 * the address of the Java process, we need to re-obtain the sy-
+			 * mbols table of the Java process after a delay.
+			 */
+			if (p->is_java && p->unknown_syms_found
+			    && p->update_syms_table_time == 0) {
+				p->update_syms_table_time =
+				    curr_time + get_java_syms_fetch_delay();
+			}
+
+			if (p->update_syms_table_time > 0
+			    && curr_time >= p->update_syms_table_time) {
+				/* Update java symbols table, will be executed during
+				 * the next Java symbolication */
+				gen_java_symbols_file(pid);
+				p->new_java_syms_file = true;
+
+				if (kv.v.cache) {
+					bcc_free_symcache((void *)kv.v.cache,
+							  kv.k.pid);
+					kv.v.cache =
+					    pointer_to_uword(bcc_symcache_new
+							     ((int)kv.k.pid,
+							      &lazy_opt));
+					if (symbol_caches_hash_add_del
+					    (h, (symbol_caches_hash_kv *) & kv,
+					     1 /* is_add */ )) {
+						ebpf_warning
+						    ("symbol_caches_hash_add_del() failed.(pid %d)\n",
+						     (int)kv.k.pid);
+					}
+				}
+
+				p->unknown_syms_found = false;
+				p->update_syms_table_time = 0;
+			}
+
+			if (p->is_java && (void *)kv.v.cache == NULL) {
+				gen_java_symbols_file(pid);
+				p->new_java_syms_file = true;
+			}
+		}
+
+		if (kv.v.cache)
+			return (void *)kv.v.cache;
+
 		kv.v.cache = pointer_to_uword(bcc_symcache_new(pid, &lazy_opt));
 		if (kv.v.cache > 0)
 			add_symcache_count++;
@@ -891,7 +896,7 @@ int create_and_init_symbolizer_caches(void)
 
 	fddir = opendir("/proc/");
 	if (fddir == NULL) {
-		ebpf_warning("Failed to open %s.\n");
+		ebpf_warning("Failed to open '/proc'\n");
 		return ETR_PROC_FAIL;
 	}
 
@@ -920,7 +925,10 @@ int create_and_init_symbolizer_caches(void)
 				continue;
 			}
 
-			java_proc_info_config(p, pid);
+			if (p->is_java) {
+				clear_old_target_perf_files(pid);
+			}
+
 			sym.v.proc_info_p = pointer_to_uword(p);
 			sym.v.cache = 0;
 			if (symbol_caches_hash_add_del
diff --git a/agent/src/ebpf/user/symbol.h b/agent/src/ebpf/user/symbol.h
index 677fc76ceee..2c98da062a7 100644
--- a/agent/src/ebpf/user/symbol.h
+++ b/agent/src/ebpf/user/symbol.h
@@ -62,6 +62,8 @@ struct symbolizer_proc_info {
 	bool verified;
 	/* To mark whether it is a Java process? */
 	bool is_java;
+	/* Have a new perf map file ? */
+	bool new_java_syms_file;
 	/* Unknown symbols was found, and it is currently mainly used to
 	 * obtain the match of the Java process.*/
 	bool unknown_syms_found;
@@ -181,6 +183,8 @@ int create_and_init_symbolizer_caches(void);
 void release_symbol_caches(void);
 u64 get_pid_stime(pid_t pid);
 void exec_symbol_cache_update(void);
+void set_java_syms_fetch_delay(int delay_secs);
+u64 get_java_syms_fetch_delay(void);
 #endif
 void update_symbol_cache(pid_t pid);
 #endif /* _BPF_SYMBOL_H_ */
diff --git a/agent/src/ebpf/user/tracer.c b/agent/src/ebpf/user/tracer.c
index b59761b1cc6..62963b17fa8 100644
--- a/agent/src/ebpf/user/tracer.c
+++ b/agent/src/ebpf/user/tracer.c
@@ -52,6 +52,8 @@ struct kprobe_port_bitmap bypass_port_bitmap;
 
 uint64_t adapt_kern_uid;	// Indicates the identifier of the adaptation kernel
 
+uint32_t attach_failed_count; // attach failure statistics
+
 /*
  * tracers
  */
@@ -697,6 +699,7 @@ static struct ebpf_link *exec_attach_kprobe(struct ebpf_prog *prog, char *name,
 	if (ret != 0) {
 		ebpf_warning
 		    ("program__attach_kprobe failed, ev_name:%s.\n", ev_name);
+		__sync_fetch_and_add(&attach_failed_count, 1);
 	}
 
 	return link;
@@ -812,6 +815,7 @@ static int tracepoint_attach(struct tracepoint *tp)
 	if (bl == NULL) {
 		ebpf_warning("program__attach_tracepoint() failed, name:%s.\n",
 			     tp->name);
+		__sync_fetch_and_add(&attach_failed_count, 1);
 		return ETR_INVAL;
 	}
 
diff --git a/agent/src/ebpf_dispatcher/ebpf_dispatcher.rs b/agent/src/ebpf_dispatcher/ebpf_dispatcher.rs
index 78bab6b7e5c..a4959fcbad8 100644
--- a/agent/src/ebpf_dispatcher/ebpf_dispatcher.rs
+++ b/agent/src/ebpf_dispatcher/ebpf_dispatcher.rs
@@ -542,6 +542,8 @@ impl EbpfCollector {
             if !on_cpu_profile_config.disabled {
                 if start_continuous_profiler(
                     on_cpu_profile_config.frequency as i32,
+                    10,
+                    60,
                     Self::ebpf_on_cpu_callback,
                 ) != 0
                 {
diff --git a/server/controller/model/agent_group_config_example.yaml b/server/controller/model/agent_group_config_example.yaml
index 80499a9653a..ea33a62d63b 100644
--- a/server/controller/model/agent_group_config_example.yaml
+++ b/server/controller/model/agent_group_config_example.yaml
@@ -1154,6 +1154,25 @@ vtap_group_id: g-xxxxxx
       ## Default: ^deepflow-.*
       #regex: ^deepflow-.*
 
+      ## The maximum limit space in the root directory of each Java process. deepflow-agent will
+      ## automatically create a `/deepflow/` directory to temporarily store the symbol files of the
+      ## Java process. The space occupied by the files written by deepflow-agent will not exceed this
+      ## configured value. All files are stored temporarily and are immediately cleared once the Java
+      ## symbol files are obtained.
+      ## Default: 10. Range: [2, 100]
+      ## Which means it falls within the interval of 2Mi to 100Mi. If the configuration value is outside
+      ## this range, the default value of 10(10Mi), will be used.
+      #java_symbol_file_max_space_limit: 10
+
+
+      ## When deepflow-agent finds that an unresolved function name appears in the function call stack
+      ## of a Java process, it will trigger the regeneration of the symbol file of the process.
+      ## Because Java utilizes the Just-In-Time (JIT) compilation mechanism, to obtain more symbols for
+      ## Java processes, the regeneration will be deferred for a period of time.
+      ## Default: 60. Range: [5, 3600]
+      ## The unit of measurement used is seconds.
+      #java_symbol_file_refresh_defer_interval: 60
+
   ######################################
   ## Agent Running in Standalone Mode ##
   ######################################