[Enhancement] smooth the jemalloc memory fragmentation (StarRocks#53863…

…).

Signed-off-by: Murphy <mofei@starrocks.com>

be/src/common/config.h

@@ -82,7 +82,7 @@ CONF_String(mem_limit, "90%");
 
 // Enable the jemalloc tracker, which is responsible for reserving memory
 CONF_Bool(enable_jemalloc_memory_tracker, "true");
-// Consider part of jemalloc memory as fragmentation: ratio * (RSS-allocated-metadata)
+// Alpha number of jemalloc memory fragmentation ratio, should in range (0, 1)
 CONF_mDouble(jemalloc_fragmentation_ratio, "0.3");
 
 // The port heartbeat service used.

be/src/common/daemon.cpp

@@ -211,28 +211,38 @@ static void retrieve_jemalloc_stats(JemallocStats* stats) {
 // Tracker the memory usage of jemalloc
 void jemalloc_tracker_daemon(void* arg_this) {
     auto* daemon = static_cast<Daemon*>(arg_this);
+    double smoothed_fragmentation = 0;
     while (!daemon->stopped()) {
         JemallocStats stats;
         retrieve_jemalloc_stats(&stats);
 
-        // metadata
+        // Jemalloc metadata
         if (GlobalEnv::GetInstance()->jemalloc_metadata_traker() && stats.metadata > 0) {
             auto tracker = GlobalEnv::GetInstance()->jemalloc_metadata_traker();
             int64_t delta = stats.metadata - tracker->consumption();
             tracker->consume(delta);
         }
 
-        // fragmentation
+        // Fragmentation and dirty memory:
+        // Jemalloc retains some dirty memory and gradually returns it to the OS, which cannot be reused by the application.
+        // Failing to account for this memory in the MemoryTracker may lead to memory allocation failures or even process
+        // termination by the OS; however, retaining excessive memory in the tracker is also wasteful.
+        // To address this, we employ a smoothing formula to track fragmentation and dirty memory, which also mitigates
+        // the impact of sudden memory releases, such as those occurring when a large query is executed:
+        // S_t = \exp\left(\alpha \cdot \log(1 + x_t) + (1 - \alpha) \cdot \log(1 + S_{t-1})\right) - 1
         if (GlobalEnv::GetInstance()->jemalloc_fragmentation_traker()) {
             if (stats.resident > 0 && stats.allocated > 0 && stats.metadata > 0) {
-                int64_t fragmentation = stats.resident - stats.allocated - stats.metadata;
-                fragmentation *= config::jemalloc_fragmentation_ratio;
-
-                // In case that released a lot of memory but not get purged, we would not consider it as fragmentation
-                bool released_a_lot = stats.allocated < (stats.resident * 0.5);
-                if (released_a_lot) {
-                    fragmentation = 0;
-                }
+                double fragmentation = stats.resident - stats.allocated - stats.metadata;
+                if (fragmentation < 0) fragmentation = 0;
+
+                // log transformation
+                double alpha = std::clamp(config::jemalloc_fragmentation_ratio, 0.1, 0.9);
+                fragmentation = std::log1p(fragmentation);
+                // smoothing
+                fragmentation = alpha * fragmentation + smoothed_fragmentation * (1 - alpha);
+                // restore the log value
+                smoothed_fragmentation = fragmentation;
+                fragmentation = std::expm1(fragmentation);
 
                 if (fragmentation >= 0) {
                     auto tracker = GlobalEnv::GetInstance()->jemalloc_fragmentation_traker();