[ISSUE #221][GENERAL] Constant high memory consumption ...

...after the 'heavy' search queries - Introduce CMake option 'DMA_TC_MALLOC_OPTIMIZATION' - Introduce CMake option 'DMA_TC_MALLOC_PROFILING' - Introduce release RAM optimization for cases when tcmalloc is not used - Remove redundant 'tIntRangePtrWrapper' data structure - Change std::shared_ptr<QString> tFoundMatch::pMatchStr field to the QString tFoundMatch::matchStr - Optimise the performance of the console view for a big set of messages - Change the logic of the console view history (the up arrow is now 'next' action, not 'previous' ) - Update README.md Signed-off-by: Vladyslav Goncharuk <vladyslav_goncharuk@epam.com>
svlad-90 · Nov 23, 2024 · 06e687b · 06e687b
1 parent b4a7d23
commit 06e687b
Show file tree

Hide file tree

Showing 16 changed files with 283 additions and 103 deletions.
diff --git a/dltmessageanalyzerplugin/src/CMakeLists.txt b/dltmessageanalyzerplugin/src/CMakeLists.txt
@@ -3,6 +3,31 @@ set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 ################### CPP ( END ) ###########################
 
+########## OPTIONS ############
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    option(DMA_TC_MALLOC_OPTIMIZATION "[
+        This option enables tcmalloc RAM usage optimization.
+        You will need to link dlt-viewer against tcmalloc to use this feature.
+        ]" OFF)
+    option(DMA_TC_MALLOC_PROFILING "[
+        This option enables tcmalloc profiling. Adds 'dump-memory-stats' command to the console view.
+        You will need to link dlt-viewer against tcmalloc to use this feature.
+        ]" OFF)
+
+    if(DMA_TC_MALLOC_OPTIMIZATION)
+        add_compile_definitions(DMA_TC_MALLOC_OPTIMIZATION_ENABLED)
+        message(STATUS "DMA_TC_MALLOC_OPTIMIZATION enabled")
+    else()
+        add_compile_definitions(DMA_GLIBC_MALLOC_OPTIMIZATION_ENABLED)
+    endif()
+
+    if(DMA_TC_MALLOC_PROFILING)
+        add_compile_definitions(DMA_TC_MALLOC_PROFILING_ENABLED)
+        message(STATUS "DMA_TC_MALLOC_PROFILING enabled")
+    endif()
+endif()
+########## OPTIONS ( END ) ####
+
 ################### QT_SPECIFIC ###########################
 if(${QT_PREFIX}Core_VERSION VERSION_LESS "5.15.0")
     message("DMA_QT5_COMPATIBILITY_MODE is set")

diff --git a/dltmessageanalyzerplugin/src/common/CMakeLists.txt b/dltmessageanalyzerplugin/src/common/CMakeLists.txt
@@ -11,5 +11,9 @@ add_library(DMA_common STATIC
 add_subdirectory(PCRE)
 
 ################### QT ####################################
-target_link_libraries(DMA_common qdlt ${QT_PREFIX}::Widgets )
+target_link_libraries(DMA_common PUBLIC qdlt ${QT_PREFIX}::Widgets )
 ################### QT ( END ) ############################
+
+if(DMA_TC_MALLOC_OPTIMIZATION OR DMA_TC_MALLOC_PROFILING)
+    target_link_libraries(DMA_common PRIVATE tcmalloc)
+endif()
diff --git a/dltmessageanalyzerplugin/src/common/Definitions.cpp b/dltmessageanalyzerplugin/src/common/Definitions.cpp
@@ -11,6 +11,12 @@
 #include <atomic>
 #include <random>
 
+#ifdef DMA_TC_MALLOC_OPTIMIZATION_ENABLED
+#include <gperftools/malloc_extension.h>
+#elif DMA_GLIBC_MALLOC_OPTIMIZATION_ENABLED
+#include <malloc.h>
+#endif
+
 #include <QDateTime>
 
 #include <QElapsedTimer>
@@ -471,56 +477,6 @@ bool tHighlightingRange::operator< ( const tHighlightingRange& rVal ) const
     return bResult;
 }
 
-/////////////////////////////tIntRangePtrWrapper/////////////////////////////
-bool tIntRangePtrWrapper::operator< ( const tIntRangePtrWrapper& rVal ) const
-{
-    bool bResult = false;
-
-    if(pRange == nullptr && rVal.pRange != nullptr)
-        bResult = true;
-    else if(pRange != nullptr && rVal.pRange == nullptr)
-        bResult = false;
-    else if(pRange == nullptr && rVal.pRange == nullptr)
-        bResult = true;
-    else
-    {
-        if( pRange->from < rVal.pRange->from )
-        {
-            bResult = true;
-        }
-        else if( pRange->from > rVal.pRange->from )
-        {
-            bResult = false;
-        }
-        else // if from == rVal.from
-        {
-            if( pRange->to < rVal.pRange->to )
-            {
-                bResult = true;
-            }
-            else
-            {
-                bResult = false;
-            }
-        }
-    }
-
-    return bResult;
-}
-
-bool tIntRangePtrWrapper::operator== ( const tIntRangePtrWrapper& rVal ) const
-{
-    if(pRange == nullptr && rVal.pRange != nullptr)
-        return false;
-    else if(pRange != nullptr && rVal.pRange == nullptr)
-        return false;
-    else if(pRange == nullptr && rVal.pRange == nullptr)
-        return true;
-
-    return ( pRange->from == rVal.pRange->from && pRange->to == rVal.pRange->to );
-}
-//////////////////////////////////////////////////////////////////////////
-
 /////////////////////////////tQStringPtrWrapper///////////////////////////
 tQStringPtrWrapper::tQStringPtrWrapper(): pString(nullptr)
 {}
@@ -664,9 +620,8 @@ tTreeItemSharedPtr getMatchesTree( const tFoundMatches& foundMatches )
 
             assert(false == data.empty());
 
-            tIntRangePtrWrapper rangePtrWrapper;
-            rangePtrWrapper.pRange = &match.range;
-            tDataItem rangeVariant( rangePtrWrapper );
+            tIntRange range = match.range;
+            tDataItem rangeVariant( range );
             auto* pAddedChild = pCurrentItem->appendChild(rangeVariant, data);
             pCurrentItem = pAddedChild;
 
@@ -756,7 +711,7 @@ tCalcRangesCoverageMulticolorResult calcRangesCoverageMulticolor( const tTreeIte
         //         .arg(match.idx)
         //         .arg(match.range.from)
         //         .arg(match.range.to)
-        //         .arg(*match.pMatchStr));
+        //         .arg(match.matchStr));
 
         if( ( match.range.from < inputRange.from && match.range.to < inputRange.from ) ||
             ( match.range.from > inputRange.to && match.range.to > inputRange.to ) )
@@ -1024,7 +979,7 @@ tTreeItemSharedPtr tItemMetadata::updateHighlightingInfo( const tFoundMatches& f
 
         for(const auto& match : sortedMatches)
         {
-            if(nullptr != match.second->pMatchStr && false == match.second->pMatchStr->isEmpty())
+            if(false == match.second->matchStr.isEmpty())
             {
                 result.insert(std::make_pair( match.second->idx, gradientColorsCounter % maxGradientColorsSize ));
                 ++gradientColorsCounter;
@@ -1336,16 +1291,17 @@ tItemMetadata::updatePlotViewInfo(const tFoundMatches& foundMatches,
 }
 
 //tFoundMatch
+
 tFoundMatch::tFoundMatch():
-pMatchStr(std::make_shared<QString>()),
+matchStr(),
 range(0,0),
 idx(0)
 {}
 
-tFoundMatch::tFoundMatch( const tQStringPtr& pMatchStr_,
+tFoundMatch::tFoundMatch( const QString& matchStr_,
                           const tIntRange& range_,
                           const int& idx_):
-pMatchStr((nullptr!=pMatchStr_)?pMatchStr_:std::make_shared<QString>()),
+matchStr(matchStr_),
 range(range_),
 idx(idx_)
 {}
@@ -1380,7 +1336,7 @@ tFoundMatchesPackItem::tFoundMatchesPackItem()
 tFoundMatchesPackItem::tFoundMatchesPackItem( tItemMetadata&& itemMetadata_,
                                               tFoundMatches&& foundMatches_ ):
   mItemMetadata(std::move(itemMetadata_)),
-  mFoundMatches(foundMatches_)
+  mFoundMatches(std::move(foundMatches_))
 {
 }
 
@@ -2730,9 +2686,9 @@ QVariant toQVariant(const tDataItem& item)
     {
         result.setValue(item.get<tGroupedViewMetadata>());
     }
-    else if(item.index() == tDataItem::index_of<tIntRangePtrWrapper>())
+    else if(item.index() == tDataItem::index_of<tIntRange>())
     {
-        result.setValue(item.get<tIntRangePtrWrapper>());
+        result.setValue(item.get<tIntRange>());
     }
     else if(item.index() == tDataItem::index_of<tFoundMatch*>())
     {
@@ -3677,6 +3633,43 @@ QColor getChartColor()
     return sColors[sColorsCounter++ % sColorsSize];
 }
 
+void releaseMemoryToOS()
+{
+#ifdef DMA_TC_MALLOC_OPTIMIZATION_ENABLED
+    MallocExtension::instance()->ReleaseFreeMemory();
+#elif DMA_GLIBC_MALLOC_OPTIMIZATION_ENABLED
+    malloc_trim(0);
+#endif
+}
+
+#ifdef DMA_TC_MALLOC_PROFILING_ENABLED
+void dumpMemoryStatistics()
+{
+    SEND_MSG("");
+    SEND_MSG("----------------------------------------------------|");
+    SEND_MSG("---------------TC_MALLOC_OUTPUT_START---------------|");
+    SEND_MSG("----------------------------------------------------|");
+    SEND_MSG("");
+
+    const int bufferSize = 100000;
+    char stats[bufferSize];
+    MallocExtension::instance()->GetStats(stats, bufferSize);
+    QString str(stats);
+    auto strVec = str.split("\n");
+
+    for(const auto& str : strVec)
+    {
+        SEND_MSG(QString("%1").arg(str));
+    }
+
+    SEND_MSG("");
+    SEND_MSG("----------------------------------------------------|");
+    SEND_MSG("----------------TC_MALLOC_OUTPUT_END----------------|");
+    SEND_MSG("----------------------------------------------------|");
+    SEND_MSG("");
+}
+#endif
+
 PUML_PACKAGE_BEGIN(Qt)
     PUML_CLASS_BEGIN(QThread)
     PUML_CLASS_END()

diff --git a/dltmessageanalyzerplugin/src/common/Definitions.hpp b/dltmessageanalyzerplugin/src/common/Definitions.hpp
@@ -155,18 +155,9 @@ struct tRange
     tRangeItem to;
 };
 
-typedef tRange<int> tIntRange;
+typedef tRange<int32_t> tIntRange;
 Q_DECLARE_METATYPE(tIntRange)
 
-struct tIntRangePtrWrapper
-{
-    bool operator== ( const tIntRangePtrWrapper& rVal ) const;
-    bool operator< ( const tIntRangePtrWrapper& rVal ) const;
-    const tIntRange* pRange = nullptr;
-};
-
-Q_DECLARE_METATYPE(tIntRangePtrWrapper)
-
 typedef QVector<tIntRange> tIntRangeList;
 typedef std::set<tIntRange> tIntRangeSet;
 
@@ -340,7 +331,7 @@ typedef QMap<eSearchResultColumn, tIntRange> tFieldRanges;
 struct tFoundMatch
 {
     tFoundMatch();
-    tFoundMatch( const tQStringPtr& pMatchStr_,
+    tFoundMatch( const QString& matchStr_,
                  const tIntRange& range_,
                  const int& idx_ );
 
@@ -352,7 +343,7 @@ struct tFoundMatch
      */
     bool operator< (const tFoundMatch& rhs) const;
 
-    tQStringPtr pMatchStr;
+    QString matchStr;
     tIntRange range;
     int idx;
 };
@@ -377,10 +368,9 @@ typedef nonstd::variant<QString,
                         int,
                         double,
                         tGroupedViewMetadata,
-                        tIntRangePtrWrapper,
+                        tIntRange,
                         const tFoundMatch*,
                         tColorWrapper,
-                        tIntRange,
                         eRegexFiltersRowType> tTreeDataItem;
 typedef tTreeDataItem tDataItem; // just to refactor less code
 QVariant toQVariant(const tDataItem& item);
@@ -829,4 +819,50 @@ enum class eTabIndexes
     CONSOLE_VIEW = 6
 };
 
+/**
+ * @brief Releases unused memory back to the operating system.
+ *
+ * This function ensures that unused memory managed by the allocator is returned
+ * to the operating system to free up resources. It supports different memory
+ * management systems:
+ *
+ * - If `DMA_TC_MALLOC_OPTIMIZATION_ENABLED` is defined, it uses TCMalloc's
+ *   `ReleaseFreeMemory` method to release unused memory.
+ * - If `DMA_GLIBC_MALLOC_OPTIMIZATION_ENABLED` is defined, it uses glibc's
+ *   `malloc_trim(0)` function to achieve the same effect.
+ *
+ * Use this function after memory-intensive operations to optimize memory usage.
+ */
+void releaseMemoryToOS();
+
+#ifdef DMA_TC_MALLOC_PROFILING_ENABLED
+/**
+ * @brief Dumps detailed memory allocation statistics.
+ *
+ * This function retrieves and outputs memory usage statistics when TCMalloc
+ * profiling is enabled (`DMA_TC_MALLOC_PROFILING_ENABLED`). The statistics
+ * provide insights into memory allocation patterns, freelist usage, and
+ * overall memory consumption managed by TCMalloc.
+ *
+ * The output includes a formatted summary of TCMalloc's memory state,
+ * surrounded by clearly marked start and end tags for easier parsing or
+ * debugging.
+ *
+ * **Example Output:**
+ * ```
+ * ----------------------------------------------------|
+ * ---------------TC_MALLOC_OUTPUT_START---------------|
+ * ----------------------------------------------------|
+ * MALLOC: <details>
+ * ----------------------------------------------------|
+ * ----------------TC_MALLOC_OUTPUT_END----------------|
+ * ----------------------------------------------------|
+ * ```
+ *
+ * This method is useful for debugging and profiling memory usage in
+ * applications relying on TCMalloc.
+ */
+void dumpMemoryStatistics();
+#endif
+
 #endif // DEFINITIONS_HPP
diff --git a/dltmessageanalyzerplugin/src/components/analyzer/src/CDLTRegexAnalyzerWorker.cpp b/dltmessageanalyzerplugin/src/components/analyzer/src/CDLTRegexAnalyzerWorker.cpp
@@ -128,7 +128,7 @@ void CDLTRegexAnalyzerWorker::analyzePortion(  const tAnalyzePortionData& analyz
                         if(0 != match.capturedLength(i))
                         {
                             const auto& matchItem = match.captured(i);
-                            foundMatches.foundMatchesVec.push_back( tFoundMatch( std::make_shared<QString>(matchItem),
+                            foundMatches.foundMatchesVec.emplace_back( tFoundMatch( matchItem,
                                                                  tIntRange( match.capturedStart(i), match.capturedEnd(i) - 1 ),
                                                                  i)  );
                         }

diff --git a/dltmessageanalyzerplugin/src/components/filtersView/src/CFiltersModel.cpp b/dltmessageanalyzerplugin/src/components/filtersView/src/CFiltersModel.cpp
@@ -750,7 +750,7 @@ void CFiltersModel::addCompletionData( const tFoundMatches& foundMatches )
 
             if(foundVarGroup != mVarGroupsMap.end())
             {
-                mCompletionCache[foundMatch.idx].insert( tQStringPtrWrapper( foundMatch.pMatchStr ) );
+                mCompletionCache[foundMatch.idx].insert( foundMatch.matchStr );
             }
         }
     }
@@ -784,16 +784,16 @@ QStringList CFiltersModel::getCompletionData( const int& groupIndex,
 
             if(false == getSettingsManager()->getFiltersCompletion_SearchPolicy())
             {
-                bStringFound = completionItem.pString->startsWith(input, caseSensitiveOption);
+                bStringFound = completionItem.startsWith(input, caseSensitiveOption);
             }
             else
             {
-                bStringFound = completionItem.pString->contains(input, caseSensitiveOption);
+                bStringFound = completionItem.contains(input, caseSensitiveOption);
             }
 
-            if(completionItem.pString && bStringFound)
+            if(bStringFound)
             {
-                result.push_back(completionItem.pString->mid(0, maxLengthOfSuggestions));
+                result.push_back(completionItem.mid(0, maxLengthOfSuggestions));
                 ++numberOfSuggestions;
             }
 

diff --git a/dltmessageanalyzerplugin/src/components/filtersView/src/CFiltersModel.hpp b/dltmessageanalyzerplugin/src/components/filtersView/src/CFiltersModel.hpp
@@ -67,8 +67,8 @@ class CFiltersModel : public IFiltersModel,
     CTreeItem::tSortingFunction mSortingHandler;
     QString mFilter;
 
-    typedef std::set<tQStringPtrWrapper> tStringPtrWrapperSet;
-    typedef std::map<int /*group id*/, tStringPtrWrapperSet> tCompletionCache;
+    typedef std::set<QString> tStringSet;
+    typedef std::map<int /*group id*/, tStringSet> tCompletionCache;
     tCompletionCache mCompletionCache;
 
     typedef std::set<int /*group id*/> tVarGroupsMap;

diff --git a/dltmessageanalyzerplugin/src/components/groupedView/src/CGroupedViewModel.cpp b/dltmessageanalyzerplugin/src/components/groupedView/src/CGroupedViewModel.cpp
@@ -494,7 +494,7 @@ void CGroupedViewModel::addMatches( const tGroupedViewIndices& groupedViewIndice
                 {
                     CTreeItem::tData data;
                     data.reserve(9);
-                    data.push_back( tQStringPtrWrapper(match.pMatchStr) ); /*SubString*/
+                    data.push_back(match.matchStr); /*SubString*/
                     data.push_back(tDataItem(1)); /*Messages*/
                     data.push_back(tDataItem(0)); /*MessagesPercantage*/
                     data.push_back(tDataItem(0)); /*MessagesPerSecond*/