[opt](memory) Support Jemalloc heap profile start at runtime and automatically generate dot

be/CMakeLists.txt

@@ -782,6 +782,7 @@ install(DIRECTORY DESTINATION ${OUTPUT_DIR}/conf)
 install(FILES
     ${BASE_DIR}/../bin/start_be.sh
     ${BASE_DIR}/../bin/stop_be.sh
+    ${BASE_DIR}/../tools/jeprof
     PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE
     GROUP_READ GROUP_WRITE GROUP_EXECUTE
     WORLD_READ WORLD_EXECUTE

be/src/http/action/jeprofile_actions.cpp

@@ -18,69 +18,101 @@
 #include "http/action/jeprofile_actions.h"
 
 #include <jemalloc/jemalloc.h>
-#include <stdlib.h>
 #include <unistd.h>
 
-#include <ctime>
-#include <fstream>
-#include <memory>
-#include <mutex>
 #include <string>
 
-#include "common/config.h"
-#include "common/object_pool.h"
 #include "http/ev_http_server.h"
 #include "http/http_channel.h"
 #include "http/http_handler.h"
 #include "http/http_handler_with_auth.h"
-#include "http/http_method.h"
-#include "io/fs/local_file_system.h"
+#include "http/http_headers.h"
+#include "http/http_request.h"
+#include "runtime/memory/heap_profiler.h"
 
 namespace doris {
-class HttpRequest;
 
-static std::mutex kJeprofileActionMutex;
-class JeHeapAction : public HttpHandlerWithAuth {
-public:
-    JeHeapAction(ExecEnv* exec_env) : HttpHandlerWithAuth(exec_env) {}
-    virtual ~JeHeapAction() = default;
+const static std::string HEADER_JSON = "application/json";
 
-    virtual void handle(HttpRequest* req) override;
-};
-
-void JeHeapAction::handle(HttpRequest* req) {
-    std::lock_guard<std::mutex> lock(kJeprofileActionMutex);
-#ifndef USE_JEMALLOC
-    std::string str = "jemalloc heap dump is not available without setting USE_JEMALLOC";
-    HttpChannel::send_reply(req, str);
+static bool compile_check(HttpRequest* req) {
+#if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) || defined(THREAD_SANITIZER)
+    HttpChannel::send_reply(
+            req, HttpStatus::INTERNAL_SERVER_ERROR,
+            "Jemalloc heap dump is not available with ASAN(address sanitizer) builds.\n");
+    return false;
+#elif !defined(USE_JEMALLOC)
+    HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR,
+                            "jemalloc heap dump is not available without setting USE_JEMALLOC.\n");
+    return false;
 #else
-    std::stringstream tmp_jeprof_file_name;
-    std::time_t now = std::time(nullptr);
-    // Build a temporary file name that is hopefully unique.
-    tmp_jeprof_file_name << config::jeprofile_dir << "/jeheap_dump." << now << "." << getpid()
-                         << "." << rand() << ".heap";
-    const std::string& tmp_file_name_str = tmp_jeprof_file_name.str();
-    const char* file_name_ptr = tmp_file_name_str.c_str();
-    int result = jemallctl("prof.dump", nullptr, nullptr, &file_name_ptr, sizeof(const char*));
-    std::stringstream response;
-    if (result == 0) {
-        response << "Jemalloc heap dump success, dump file path: " << tmp_jeprof_file_name.str()
-                 << "\n";
-    } else {
-        response << "Jemalloc heap dump failed, je_mallctl return: " << result << "\n";
-    }
-    HttpChannel::send_reply(req, response.str());
+    return true;
 #endif
 }
 
-Status JeprofileActions::setup(doris::ExecEnv* exec_env, doris::EvHttpServer* http_server,
-                               doris::ObjectPool& pool) {
-    if (!config::jeprofile_dir.empty()) {
-        RETURN_IF_ERROR(io::global_local_filesystem()->create_directory(config::jeprofile_dir));
+void SetJeHeapProfileActiveActions::handle(HttpRequest* req) {
+    req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.c_str());
+    if (compile_check(req)) {
+        if (req->param("prof_value") == "true") {
+            HeapProfiler::instance()->heap_profiler_start();
+            HttpChannel::send_reply(
+                    req, HttpStatus::OK,
+                    "heap profiler started\nJemalloc will only track and sample the memory "
+                    "allocated and freed after the heap profiler started, it cannot analyze the "
+                    "memory allocated and freed before. Therefore, dumping the heap profile "
+                    "immediately after start heap profiler may prompt `No nodes to print`. If you "
+                    "want to analyze the memory that has been allocated in the past, you can only "
+                    "restart the BE process and start heap profiler immediately.\n");
+        } else {
+            HeapProfiler::instance()->heap_profiler_stop();
+            HttpChannel::send_reply(req, HttpStatus::OK, "heap profiler stoped\n");
+        }
+    }
+}
+
+void DumpJeHeapProfileToDotActions::handle(HttpRequest* req) {
+    req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.c_str());
+    if (compile_check(req)) {
+        if (!HeapProfiler::instance()->check_heap_profiler()) {
+            HttpChannel::send_reply(
+                    req, HttpStatus::INTERNAL_SERVER_ERROR,
+                    "`curl http://be_host:be_webport/jeheap/prof/true` to start heap profiler\n");
+        }
+        std::string dot = HeapProfiler::instance()->dump_heap_profile_to_dot();
+        if (dot.empty()) {
+            HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR,
+                                    "dump heap profile to dot failed, see be.INFO\n");
+        } else {
+            dot += "\n-------------------------------------------------------\n";
+            dot += "Copy the text after `digraph` in the above output to "
+                   "http://www.webgraphviz.com to generate a dot graph.\n"
+                   "after start heap profiler, if there is no operation, will print `No nodes to "
+                   "print`."
+                   "If there are many errors: `addr2line: Dwarf Error`,"
+                   "or other FAQ, reference doc: "
+                   "https://doris.apache.org/community/developer-guide/debug-tool/#4-qa\n";
+            HttpChannel::send_reply(req, HttpStatus::OK, dot);
+        }
+    }
+}
+
+void DumpJeHeapProfileActions::handle(HttpRequest* req) {
+    req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.c_str());
+    if (compile_check(req)) {
+        if (!HeapProfiler::instance()->check_heap_profiler()) {
+            HttpChannel::send_reply(
+                    req, HttpStatus::INTERNAL_SERVER_ERROR,
+                    "`curl http://be_host:be_webport/jeheap/prof/true` to start heap profiler\n");
+        }
+        std::string profile_file_name = HeapProfiler::instance()->dump_heap_profile();
+        if (profile_file_name.empty()) {
+            HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR,
+                                    "jemalloc heap dump failed\n");
+        } else {
+            HttpChannel::send_reply(req, HttpStatus::OK,
+                                    fmt::format("jemalloc heap dump success, dump file path: {}\n",
+                                                profile_file_name));
+        }
     }
-    http_server->register_handler(HttpMethod::GET, "/jeheap/dump",
-                                  pool.add(new JeHeapAction(exec_env)));
-    return Status::OK();
 }
 
 } // namespace doris

be/src/http/action/jeprofile_actions.h

@@ -15,17 +15,35 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#ifndef DORIS_JEPROFILE_ACTIONS_H
-#define DORIS_JEPROFILE_ACTIONS_H
-#include "common/status.h"
+#pragma once
+
+#include "http/http_handler.h"
+#include "http/http_handler_with_auth.h"
+
 namespace doris {
-class EvHttpServer;
+
+class HttpRequest;
 class ExecEnv;
-class ObjectPool;
-class JeprofileActions {
+
+class SetJeHeapProfileActiveActions final : public HttpHandlerWithAuth {
+public:
+    SetJeHeapProfileActiveActions(ExecEnv* exec_env) : HttpHandlerWithAuth(exec_env) {}
+    ~SetJeHeapProfileActiveActions() override = default;
+    void handle(HttpRequest* req) override;
+};
+
+class DumpJeHeapProfileToDotActions final : public HttpHandlerWithAuth {
+public:
+    DumpJeHeapProfileToDotActions(ExecEnv* exec_env) : HttpHandlerWithAuth(exec_env) {}
+    ~DumpJeHeapProfileToDotActions() override = default;
+    void handle(HttpRequest* req) override;
+};
+
+class DumpJeHeapProfileActions final : public HttpHandlerWithAuth {
 public:
-    static Status setup(ExecEnv* exec_env, EvHttpServer* http_server, ObjectPool& pool);
+    DumpJeHeapProfileActions(ExecEnv* exec_env) : HttpHandlerWithAuth(exec_env) {}
+    ~DumpJeHeapProfileActions() override = default;
+    void handle(HttpRequest* req) override;
 };
 
 } // namespace doris
-#endif //DORIS_JEPROFILE_ACTIONS_H

be/src/runtime/exec_env.h

@@ -110,6 +110,7 @@ class RowCache;
 class DummyLRUCache;
 class CacheManager;
 class ProcessProfile;
+class HeapProfiler;
 class WalManager;
 class DNSCache;
 
@@ -306,6 +307,7 @@ class ExecEnv {
     RowCache* get_row_cache() { return _row_cache; }
     CacheManager* get_cache_manager() { return _cache_manager; }
     ProcessProfile* get_process_profile() { return _process_profile; }
+    HeapProfiler* get_heap_profiler() { return _heap_profiler; }
     segment_v2::InvertedIndexSearcherCache* get_inverted_index_searcher_cache() {
         return _inverted_index_searcher_cache;
     }
@@ -445,6 +447,7 @@ class ExecEnv {
     RowCache* _row_cache = nullptr;
     CacheManager* _cache_manager = nullptr;
     ProcessProfile* _process_profile = nullptr;
+    HeapProfiler* _heap_profiler = nullptr;
     segment_v2::InvertedIndexSearcherCache* _inverted_index_searcher_cache = nullptr;
     segment_v2::InvertedIndexQueryCache* _inverted_index_query_cache = nullptr;
     QueryCache* _query_cache = nullptr;

be/src/runtime/exec_env_init.cpp

@@ -71,6 +71,7 @@
 #include "runtime/load_path_mgr.h"
 #include "runtime/load_stream_mgr.h"
 #include "runtime/memory/cache_manager.h"
+#include "runtime/memory/heap_profiler.h"
 #include "runtime/memory/mem_tracker.h"
 #include "runtime/memory/mem_tracker_limiter.h"
 #include "runtime/memory/thread_mem_tracker_mgr.h"
@@ -452,6 +453,7 @@ Status ExecEnv::_init_mem_env() {
     std::stringstream ss;
     // 1. init mem tracker
     _process_profile = ProcessProfile::create_global_instance();
+    _heap_profiler = HeapProfiler::create_global_instance();
     init_mem_tracker();
     thread_context()->thread_mem_tracker_mgr->init();
 #if defined(USE_MEM_TRACKER) && !defined(__SANITIZE_ADDRESS__) && !defined(ADDRESS_SANITIZER) && \
@@ -775,6 +777,7 @@ void ExecEnv::destroy() {
     SAFE_DELETE(_dns_cache);
 
     SAFE_DELETE(_process_profile);
+    SAFE_DELETE(_heap_profiler);
 
     _s_tracking_memory = false;
 

be/src/runtime/memory/heap_profiler.cpp

@@ -0,0 +1,130 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "runtime/memory/heap_profiler.h"
+
+#ifdef USE_JEMALLOC
+#include "jemalloc/jemalloc.h"
+#endif
+#include "agent/utils.h"
+#include "common/config.h"
+#include "io/fs/local_file_system.h"
+
+namespace doris {
+
+void HeapProfiler::set_prof_active(bool prof) {
+#ifdef USE_JEMALLOC
+    std::lock_guard guard(_mutex);
+    try {
+        int err = jemallctl("prof.active", nullptr, nullptr, &prof, 1);
+        err |= jemallctl("prof.thread_active_init", nullptr, nullptr, &prof, 1);
+        if (err) {
+            LOG(WARNING) << "jemalloc heap profiling start failed, " << err;
+        } else {
+            LOG(WARNING) << "jemalloc heap profiling started";
+        }
+    } catch (...) {
+        LOG(WARNING) << "jemalloc heap profiling start failed";
+    }
+#endif
+}
+
+bool HeapProfiler::get_prof_dump(const std::string& profile_file_name) {
+#ifdef USE_JEMALLOC
+    std::lock_guard guard(_mutex);
+    const char* file_name_ptr = profile_file_name.c_str();
+    try {
+        int err = jemallctl("prof.dump", nullptr, nullptr, &file_name_ptr, sizeof(const char*));
+        if (err) {
+            LOG(WARNING) << "dump heap profile failed, " << err;
+            return false;
+        } else {
+            LOG(INFO) << "dump heap profile to " << profile_file_name;
+            return true;
+        }
+    } catch (...) {
+        LOG(WARNING) << "dump heap profile failed";
+        return false;
+    }
+#else
+    return false;
+#endif
+}
+
+static std::string jeprof_profile_to_dot(const std::string& profile_file_name) {
+    AgentUtils util;
+    const static std::string jeprof_path = fmt::format("{}/bin/jeprof", std::getenv("DORIS_HOME"));
+    const static std::string binary_path =
+            fmt::format("{}/lib/doris_be", std::getenv("DORIS_HOME"));
+    // https://doris.apache.org/community/developer-guide/debug-tool/#3-jeprof-parses-heap-profile
+    std::string jeprof_cmd =
+            fmt::format("{} --dot {} {}", jeprof_path, binary_path, profile_file_name);
+    std::string msg;
+    bool rc = util.exec_cmd(jeprof_cmd, &msg);
+    if (!rc) {
+        LOG(WARNING) << "jeprof profile to dot failed: " << msg;
+    }
+    return msg;
+}
+
+void HeapProfiler::heap_profiler_start() {
+    set_prof_active(true);
+}
+
+void HeapProfiler::heap_profiler_stop() {
+    set_prof_active(false);
+}
+
+bool HeapProfiler::check_heap_profiler() {
+#ifdef USE_JEMALLOC
+    size_t value = 0;
+    size_t sz = sizeof(value);
+    jemallctl("prof.active", &value, &sz, nullptr, 0);
+    return value;
+#else
+    return false;
+#endif
+}
+
+std::string HeapProfiler::dump_heap_profile() {
+    if (!config::jeprofile_dir.empty()) {
+        auto st = io::global_local_filesystem()->create_directory(config::jeprofile_dir);
+        if (!st.ok()) {
+            LOG(WARNING) << "create jeprofile dir failed.";
+            return "";
+        }
+    }
+    std::string profile_file_name =
+            fmt::format("{}/jeheap_dump.{}.{}.{}.heap", config::jeprofile_dir, std::time(nullptr),
+                        getpid(), rand());
+    if (get_prof_dump(profile_file_name)) {
+        return profile_file_name;
+    } else {
+        return "";
+    }
+}
+
+std::string HeapProfiler::dump_heap_profile_to_dot() {
+    std::string profile_file_name = dump_heap_profile();
+    if (!profile_file_name.empty()) {
+        return jeprof_profile_to_dot(profile_file_name);
+    } else {
+        return "";
+    }
+}
+
+} // namespace doris