From dea06c5d056ed3e0d6944aebe9600e3f6011b37e Mon Sep 17 00:00:00 2001
From: 924060929 <924060929@qq.com>
Date: Fri, 17 May 2024 18:52:54 +0800
Subject: [PATCH] [feature](tools) embed async profiler to help user generate
 flame graph for frontend (#35010)

background: some users want to find the bottleneck of the frontend, but they can not provide their query sql and schema because of privacy, so I embed async profiler to help they generate flame graph for frontend in their environment.

There has a script file `${FE_DEPLOY_PATH}/bin/profile_fe.sh` do the profile:
start profiler example
```shell
$ bin/profile_fe.sh
$ PROFILE_SECONDS=1 bin/profile_fe.sh
```

This is the real example:
```shell
$ PROFILE_SECONDS=1 bin/profile_fe.sh
DORIS_HOME: /Users/lanhuajian/github/doris/output/fe
JAVA: /Users/lanhuajian/Library/Java/JavaVirtualMachines/corretto-17.0.10/Contents/Home/bin/java
DorisFE pid: 29494
Begin profiling 1 seconds and generate flame graph to /Users/lanhuajian/github/doris/output/fe/log/profile_20240517154934.html...

Profiling for 1 seconds
Done
Generated flame graph to /Users/lanhuajian/github/doris/output/fe/log/profile_20240517154934.html
```

You can use web browser to open the generated html file to troubleshooting performance issues.

The flame graph html file would look like this:
<img width="1432" alt="image" src="https://github.com/apache/doris/assets/8806055/032a021e-6cbe-4e82-acf5-34a22dac3546">

**Note**: this profiler use the `async-profiler` tool, so it only support in Linux and MacOS
---
 LICENSE.txt        |  8 ++++++
 bin/profile_fe.sh  | 63 ++++++++++++++++++++++++++++++++++++++++++++++
 fe/fe-core/pom.xml |  9 +++++--
 3 files changed, 78 insertions(+), 2 deletions(-)
 create mode 100755 bin/profile_fe.sh

diff --git a/LICENSE.txt b/LICENSE.txt
index 63c614cda48763..e28911d07a4092 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -718,3 +718,11 @@ LICENSE file in the root directory of this source tree. An additional grant
 of patent rights can be found in the PATENTS file in the same directory.
 
 ----------------------------------------------------------------------------------
+
+me.bechberger.ap-loader-all: Apache 2.0 license
+
+Apache 2.0, Copyright 2023 SAP SE or an SAP affiliate company, Johannes Bechberger and ap-loader contributors
+
+This project is maintained by the SapMachine team at SAP SE
+
+----------------------------------------------------------------------------------
\ No newline at end of file
diff --git a/bin/profile_fe.sh b/bin/profile_fe.sh
new file mode 100755
index 00000000000000..1481cce4069596
--- /dev/null
+++ b/bin/profile_fe.sh
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+curdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+
+DORIS_HOME="$(
+    cd "${curdir}/.." || exit 1
+    pwd
+)"
+export DORIS_HOME
+echo "DORIS_HOME: ${DORIS_HOME}"
+
+if [[ -z "${JAVA_HOME}" ]]; then
+    if ! command -v java &>/dev/null; then
+        JAVA=""
+    else
+        JAVA="$(command -v java)"
+    fi
+else
+    JAVA="${JAVA_HOME}/bin/java"
+fi
+echo "JAVA: ${JAVA}"
+
+if [[ ! -x "${JAVA}" ]]; then
+    echo "The JAVA_HOME environment variable is not set correctly"
+    echo "This environment variable is required to run this program"
+    echo "Note: JAVA_HOME should point to a JDK and not a JRE"
+    echo "You can set JAVA_HOME in the fe.conf configuration file"
+    exit 1
+fi
+
+FE_PID=$(jps | grep DorisFE | awk '{print $1}')
+if [[ -z "${FE_PID}" ]]; then
+    echo "DorisFe not started"
+    exit 1
+fi
+echo "DorisFE pid: ${FE_PID}"
+
+mkdir -p "${DORIS_HOME}/log"
+NOW=$(date +'%Y%m%d%H%M%S')
+PROFILE_OUTPUT="${DORIS_HOME}/log/profile_${NOW}.html"
+if [[ -z "${PROFILE_SECONDS}" ]]; then
+    PROFILE_SECONDS="10"
+fi
+
+echo "Begin profiling ${PROFILE_SECONDS} seconds and generate flame graph to ${PROFILE_OUTPUT}..."
+${JAVA} -jar "${DORIS_HOME}"/lib/ap-loader-all-*.jar profiler -a -n -l -i 200us -d "${PROFILE_SECONDS}" -f "${PROFILE_OUTPUT}" "${FE_PID}"
+echo "Generated flame graph to ${PROFILE_OUTPUT}"
diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index 22b60de41e20b2..fb8d7ba3950b68 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -640,8 +640,8 @@ under the License.
         <dependency>
             <groupId>org.apache.parquet</groupId>
             <artifactId>parquet-avro</artifactId>
-        </dependency>        
-        <dependency> 
+        </dependency>
+        <dependency>
             <groupId>org.mariadb.jdbc</groupId>
             <artifactId>mariadb-java-client</artifactId>
         </dependency>
@@ -760,6 +760,11 @@ under the License.
             <artifactId>hbase-hadoop-compat</artifactId>
             <version>2.5.2-hadoop3</version>
         </dependency>
+        <dependency>
+            <groupId>me.bechberger</groupId>
+            <artifactId>ap-loader-all</artifactId>
+            <version>3.0-8</version>
+        </dependency>
     </dependencies>
     <repositories>
         <!-- for huawei obs sdk -->