split rrdhost/rrdset/rrddim and rrd.h (netdata#19399)

split files - no functional changes

CMakeLists.txt

@@ -1448,8 +1448,8 @@ set(RRD_PLUGIN_FILES
         src/database/rrd.c
         src/database/rrd.h
         src/database/rrdset.c
-        src/database/storage_engine.c
-        src/database/storage_engine.h
+        src/database/storage-engine.c
+        src/database/storage-engine.h
         src/database/ram/rrddim_mem.c
         src/database/ram/rrddim_mem.h
         src/database/sqlite/sqlite_metadata.c
@@ -1486,6 +1486,23 @@ set(RRD_PLUGIN_FILES
         src/database/rrdhost-system-info.c
         src/database/rrdhost-system-info.h
         src/database/contexts/contexts-loading.c
+        src/database/rrdset-index-id.c
+        src/database/rrdset-index-id.h
+        src/database/rrdset-index-name.c
+        src/database/rrdset-index-name.h
+        src/database/rrdset-slots.c
+        src/database/rrdset-slots.h
+        src/database/rrdset-collection.c
+        src/database/rrdset-collection.h
+        src/database/rrdset.h
+        src/database/rrdhost.h
+        src/database/rrdhost-status.c
+        src/database/rrdhost-status.h
+        src/database/rrddim.h
+        src/database/rrddim-backfill.c
+        src/database/rrddim-backfill.h
+        src/database/rrddim-collection.c
+        src/database/rrddim-collection.h
 )
 
 if(ENABLE_DBENGINE)
@@ -1592,8 +1609,6 @@ set(STREAMING_PLUGIN_FILES
         src/streaming/stream-conf.h
         src/streaming/stream-handshake.h
         src/streaming/stream-parents.h
-        src/streaming/rrdhost-status.c
-        src/streaming/rrdhost-status.h
         src/streaming/stream-sender-api.c
         src/streaming/stream-receiver-internals.h
         src/streaming/stream-receiver-api.c

src/daemon/config/netdata-conf-db.c

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include "netdata-conf-db.h"
+#include "daemon/common.h"
 
 int default_rrd_history_entries = RRD_DEFAULT_HISTORY_ENTRIES;
 

src/daemon/config/netdata-conf-directories.c

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include "netdata-conf-directories.h"
+#include "daemon/common.h"
 
 static const char *get_varlib_subdir_from_config(const char *prefix, const char *dir) {
     char filename[FILENAME_MAX + 1];

src/daemon/config/netdata-conf-global.c

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include "netdata-conf-global.h"
+#include "daemon/common.h"
 
 size_t netdata_conf_cpus(void) {
     static size_t processors = 0;

src/daemon/config/netdata-conf-logs.c

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include "netdata-conf-logs.h"
+#include "daemon/common.h"
 
 static void debug_flags_initialize(void) {
     // --------------------------------------------------------------------

src/daemon/config/netdata-conf.c

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include "netdata-conf.h"
+#include "daemon/common.h"
 
 struct config netdata_config = APPCONFIG_INITIALIZER;
 

src/daemon/config/netdata-conf.h

@@ -17,6 +17,4 @@ bool netdata_conf_load(char *filename, char overwrite_used, const char **user);
 #include "netdata-conf-web.h"
 #include "netdata-conf-cloud.h"
 
-#include "daemon/common.h"
-
 #endif //NETDATA_DAEMON_NETDATA_CONF_H

src/database/contexts/internal.h

@@ -8,7 +8,7 @@
 #include "../../aclk/schema-wrappers/context.h"
 #include "../../aclk/aclk_contexts_api.h"
 #include "../../aclk/aclk.h"
-#include "../storage_engine.h"
+#include "../storage-engine.h"
 
 #define MESSAGES_PER_BUNDLE_TO_SEND_TO_HUB_PER_HOST         5000
 #define FULL_RETENTION_SCAN_DELAY_AFTER_DB_ROTATION_SECS    120

src/database/contexts/query_target.c

@@ -210,7 +210,7 @@ typedef struct query_target_locals {
     QUERY_NODE *qn; // temp to pass on callbacks, ignore otherwise - no need to free
 } QUERY_TARGET_LOCALS;
 
-struct storage_engine *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier) {
+struct storage *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier) {
     QUERY_NODE *qn = query_node(qt, qm->link.query_node_id);
     return qn->rrdhost->db[tier].eng;
 }

src/database/contexts/rrdcontext.h

@@ -582,7 +582,7 @@ static inline const char *query_metric_name(QUERY_TARGET *qt, QUERY_METRIC *qm)
     return rrdmetric_acquired_name(qd->rma);
 }
 
-struct storage_engine *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier);
+struct storage *query_metric_storage_engine(QUERY_TARGET *qt, QUERY_METRIC *qm, size_t tier);
 
 STRING *query_instance_id_fqdn(QUERY_INSTANCE *qi, size_t version);
 STRING *query_instance_name_fqdn(QUERY_INSTANCE *qi, size_t version);

src/database/engine/pagecache.c

@@ -1,5 +1,4 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
-#define NETDATA_RRD_INTERNALS
 
 #include "rrdengine.h"
 

src/database/engine/pdc.c

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
-#define NETDATA_RRD_INTERNALS
+
 #include "pdc.h"
 #include "dbengine-compression.h"
 

src/database/engine/rrdengine.c

@@ -1,5 +1,4 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
-#define NETDATA_RRD_INTERNALS
 
 #include "rrdengine.h"
 #include "pdc.h"

src/database/rrd.c

@@ -1,8 +1,7 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
-#define NETDATA_RRD_INTERNALS 1
 
+#define RRDHOST_INTERNALS
 #include "rrd.h"
-#include "storage_engine.h"
 
 // ----------------------------------------------------------------------------
 // globals
@@ -59,50 +58,143 @@ const char *rrd_algorithm_name(RRD_ALGORITHM algorithm) {
     }
 }
 
+// ----------------------------------------------------------------------------
+// RRD - string management
+
+STRING *rrd_string_strdupz(const char *s) {
+    if(unlikely(!s || !*s)) return string_strdupz(s);
+
+    char *tmp = strdupz(s);
+    json_fix_string(tmp);
+    STRING *ret = string_strdupz(tmp);
+    freez(tmp);
+    return ret;
+}
 
 // ----------------------------------------------------------------------------
-// RRD - chart types
 
-inline RRDSET_TYPE rrdset_type_id(const char *name) {
-    if(unlikely(strcmp(name, RRDSET_TYPE_AREA_NAME) == 0))
-        return RRDSET_TYPE_AREA;
+inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) {
+    if(mode == RRD_MEMORY_MODE_DBENGINE) return 0;
+    if(mode == RRD_MEMORY_MODE_NONE) return 5;
 
-    else if(unlikely(strcmp(name, RRDSET_TYPE_STACKED_NAME) == 0))
-        return RRDSET_TYPE_STACKED;
+    if(entries < 5) entries = 5;
+    if(entries > RRD_HISTORY_ENTRIES_MAX) entries = RRD_HISTORY_ENTRIES_MAX;
 
-    else if(unlikely(strcmp(name, RRDSET_TYPE_HEATMAP_NAME) == 0))
-        return RRDSET_TYPE_HEATMAP;
+    if(mode == RRD_MEMORY_MODE_RAM) {
+        long header_size = 0;
+
+        long page = (long)sysconf(_SC_PAGESIZE);
+        long size = (long)(header_size + entries * sizeof(storage_number));
+        if (unlikely(size % page)) {
+            size -= (size % page);
+            size += page;
+
+            long n = (long)((size - header_size) / sizeof(storage_number));
+            return n;
+        }
+    }
 
-    else // if(unlikely(strcmp(name, RRDSET_TYPE_LINE_NAME) == 0))
-        return RRDSET_TYPE_LINE;
+    return entries;
 }
 
-const char *rrdset_type_name(RRDSET_TYPE chart_type) {
-    switch(chart_type) {
-        case RRDSET_TYPE_LINE:
-        default:
-            return RRDSET_TYPE_LINE_NAME;
+void api_v1_management_init(void);
+int rrd_init(const char *hostname, struct rrdhost_system_info *system_info, bool unittest) {
+    rrdhost_init();
 
-        case RRDSET_TYPE_AREA:
-            return RRDSET_TYPE_AREA_NAME;
+    if (unlikely(sql_init_meta_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
+        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+            set_late_analytics_variables(system_info);
+            fatal("Failed to initialize SQLite");
+        }
 
-        case RRDSET_TYPE_STACKED:
-            return RRDSET_TYPE_STACKED_NAME;
+        nd_log(NDLS_DAEMON, NDLP_DEBUG,
+               "Skipping SQLITE metadata initialization since memory mode is not dbengine");
+    }
 
-        case RRDSET_TYPE_HEATMAP:
-            return RRDSET_TYPE_HEATMAP_NAME;
+    if (unlikely(sql_init_context_database(system_info ? 0 : 1))) {
+        error_report("Failed to initialize context metadata database");
     }
-}
 
-// ----------------------------------------------------------------------------
-// RRD - string management
+    if (unlikely(unittest)) {
+        dbengine_enabled = true;
+    }
+    else {
+        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE || stream_conf_receiver_needs_dbengine()) {
+            nd_log(NDLS_DAEMON, NDLP_DEBUG,
+                   "DBENGINE: Initializing ...");
+
+            netdata_conf_dbengine_init(hostname);
+        }
+        else
+            nd_profile.storage_tiers = 1;
+
+        if (!dbengine_enabled) {
+            if (nd_profile.storage_tiers > 1) {
+                nd_log(NDLS_DAEMON, NDLP_WARNING,
+                       "dbengine is not enabled, but %zu tiers have been requested. Resetting tiers to 1",
+                       nd_profile.storage_tiers);
+
+                nd_profile.storage_tiers = 1;
+            }
+
+            if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+                nd_log(NDLS_DAEMON, NDLP_WARNING,
+                       "dbengine is not enabled, but it has been given as the default db mode. "
+                       "Resetting db mode to alloc");
+
+                default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;
+            }
+        }
+    }
 
-STRING *rrd_string_strdupz(const char *s) {
-    if(unlikely(!s || !*s)) return string_strdupz(s);
+    if(!unittest)
+        metadata_sync_init();
+
+    localhost = rrdhost_create(
+        hostname
+        , registry_get_this_machine_hostname()
+            , registry_get_this_machine_guid()
+            , os_type
+        , netdata_configured_timezone
+        , netdata_configured_abbrev_timezone
+        , netdata_configured_utc_offset
+        , program_name
+        , NETDATA_VERSION
+        ,
+        nd_profile.update_every, default_rrd_history_entries
+        , default_rrd_memory_mode
+        , health_plugin_enabled()
+            , stream_send.enabled
+        , stream_send.parents.destination
+        , stream_send.api_key
+        , stream_send.send_charts_matching
+        , stream_receive.replication.enabled
+        , stream_receive.replication.period
+        , stream_receive.replication.step
+        , system_info
+        , 1
+        , 0
+    );
+
+    if (unlikely(!localhost))
+        return 1;
+
+    rrdhost_flag_set(localhost, RRDHOST_FLAG_COLLECTOR_ONLINE);
+    object_state_activate(&localhost->state_id);
+
+    ml_host_start(localhost);
+    dyncfg_host_init(localhost);
+
+    if(!unittest)
+        health_plugin_init();
+
+    global_functions_add();
+
+    if (likely(system_info)) {
+        detect_machine_guid_change(&localhost->host_id.uuid);
+        sql_aclk_sync_init();
+        api_v1_management_init();
+    }
 
-    char *tmp = strdupz(s);
-    json_fix_string(tmp);
-    STRING *ret = string_strdupz(tmp);
-    freez(tmp);
-    return ret;
+    return 0;
 }