Merge pull request syslog-ng#4399 from alltilla/disk-buffer-dir-metrics

disk-buffer: add dir metrics

modules/diskq/CMakeLists.txt

@@ -11,6 +11,8 @@ set(SYSLOG_NG_DISK_BUFFER_SOURCES
     logqueue-disk-reliable.h
     qdisk.h
     qdisk.c
+    diskq-global-metrics.h
+    diskq-global-metrics.c
 )
 
 add_library(syslog-ng-disk-buffer STATIC ${SYSLOG_NG_DISK_BUFFER_SOURCES})

modules/diskq/Makefile.am

@@ -14,7 +14,9 @@ modules_diskq_libsyslog_ng_disk_buffer_la_SOURCES = \
   modules/diskq/logqueue-disk-reliable.c \
   modules/diskq/logqueue-disk-reliable.h \
   modules/diskq/qdisk.h \
-  modules/diskq/qdisk.c
+  modules/diskq/qdisk.c \
+  modules/diskq/diskq-global-metrics.h \
+  modules/diskq/diskq-global-metrics.c
 
 modules_diskq_libsyslog_ng_disk_buffer_la_CPPFLAGS = \
   $(AM_CPPFLAGS) \

modules/diskq/diskq-config.c

@@ -29,6 +29,7 @@
 #define DEFAULT_TRUNCATE_SIZE_RATIO_V3_X (0.1)
 #define DEFAULT_TRUNCATE_SIZE_RATIO_V4_X (1)
 #define DEFAULT_PREALLOC FALSE
+#define DEFAULT_STATS_FREQ (300)
 
 static void
 disk_queue_config_free(ModuleConfig *s)
@@ -43,6 +44,7 @@ disk_queue_config_new(GlobalConfig *cfg)
 
   self->truncate_size_ratio = -1;
   self->prealloc = -1;
+  self->stats_freq = -1;
 
   self->super.free_fn = disk_queue_config_free;
   return self;
@@ -113,3 +115,28 @@ disk_queue_config_is_prealloc_set_explicitly(GlobalConfig *cfg)
   DiskQueueConfig *dqc = disk_queue_config_get(cfg);
   return dqc->prealloc != -1;
 }
+
+void
+disk_queue_config_set_stats_freq(GlobalConfig *cfg, gint stats_freq)
+{
+  DiskQueueConfig *dqc = disk_queue_config_get(cfg);
+  dqc->stats_freq = stats_freq;
+}
+
+gboolean
+disk_queue_config_get_stats_freq(GlobalConfig *cfg)
+{
+  DiskQueueConfig *dqc = disk_queue_config_get(cfg);
+
+  if (!disk_queue_config_is_stats_freq_set_explicitly(cfg))
+    return DEFAULT_STATS_FREQ;
+
+  return dqc->stats_freq;
+}
+
+gboolean
+disk_queue_config_is_stats_freq_set_explicitly(GlobalConfig *cfg)
+{
+  DiskQueueConfig *dqc = disk_queue_config_get(cfg);
+  return dqc->stats_freq != -1;
+}

modules/diskq/diskq-config.h

@@ -30,6 +30,7 @@ typedef struct _DiskQueueConfig
   ModuleConfig super;
   gdouble truncate_size_ratio;
   gboolean prealloc;
+  gint stats_freq;
 } DiskQueueConfig;
 
 DiskQueueConfig *disk_queue_config_get(GlobalConfig *cfg);
@@ -42,5 +43,9 @@ void disk_queue_config_set_prealloc(GlobalConfig *cfg, gboolean prealloc);
 gboolean disk_queue_config_get_prealloc(GlobalConfig *cfg);
 gboolean disk_queue_config_is_prealloc_set_explicitly(GlobalConfig *cfg);
 
+void disk_queue_config_set_stats_freq(GlobalConfig *cfg, gint stats_freq);
+gboolean disk_queue_config_get_stats_freq(GlobalConfig *cfg);
+gboolean disk_queue_config_is_stats_freq_set_explicitly(GlobalConfig *cfg);
+
 #endif
 

modules/diskq/diskq-global-metrics.c

@@ -0,0 +1,197 @@
+/*
+ * Copyright (c) 2023 Attila Szakacs
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ *
+ */
+
+#include "apphook.h"
+#include "cfg.h"
+#include "diskq-config.h"
+#include "diskq-global-metrics.h"
+#include "mainloop.h"
+#include "messages.h"
+#include "stats/stats-registry.h"
+#include "stats/stats-cluster-single.h"
+#include "timeutils/misc.h"
+
+#include <iv.h>
+#include <sys/statvfs.h>
+
+typedef struct DiskQGlobalMetrics_
+{
+  GMutex lock;
+  struct iv_timer dir_watch_timer;
+
+  GHashTable *dirs;
+  gint freq;
+} DiskQGlobalMetrics;
+
+static DiskQGlobalMetrics diskq_global_metrics;
+static gboolean diskq_global_metrics_global_inited;
+
+static void
+_dir_watch_timer_stop(DiskQGlobalMetrics *self)
+{
+  if (iv_timer_registered(&self->dir_watch_timer))
+    iv_timer_unregister(&self->dir_watch_timer);
+}
+
+static void
+_dir_watch_timer_start(DiskQGlobalMetrics *self)
+{
+  _dir_watch_timer_stop(self);
+
+  iv_validate_now();
+  self->dir_watch_timer.expires = iv_now;
+  timespec_add_msec(&self->dir_watch_timer.expires, self->freq * 1000);
+  iv_timer_register(&self->dir_watch_timer);
+}
+
+static gboolean
+_get_available_space_mib_in_dir(const gchar *dir, gint64 *available_space_mib)
+{
+  struct statvfs stat;
+  if (statvfs(dir, &stat) < 0 )
+    {
+      msg_error("disk-buffer: Failed to get filesystem info",
+                evt_tag_str("dir", dir),
+                evt_tag_error("error"));
+      return FALSE;
+    }
+
+  *available_space_mib = stat.f_bsize * stat.f_bavail / (1024 * 1024);
+
+  return TRUE;
+}
+
+static void
+_update_dir_metrics(gpointer key, gpointer value, gpointer user_data)
+{
+  const gchar *dir = (const gchar *) key;
+
+  gint64 available_space_mib;
+  if (!_get_available_space_mib_in_dir(dir, &available_space_mib))
+    return;
+
+  StatsClusterLabel labels[] = { stats_cluster_label("dir", dir), };
+  StatsClusterKey sc_key;
+  stats_cluster_single_key_set(&sc_key, "disk_queue_dir_available_bytes", labels, G_N_ELEMENTS(labels));
+  /* Up to 4096 TiB with 32 bit atomic counters. */
+  stats_cluster_single_key_add_unit(&sc_key, SCU_MIB);
+
+  stats_lock();
+  {
+    StatsCounterItem *counter;
+    StatsCluster *cluster = stats_register_dynamic_counter(STATS_LEVEL1, &sc_key, SC_TYPE_SINGLE_VALUE, &counter);
+    stats_counter_set(counter, available_space_mib);
+    stats_unregister_dynamic_counter(cluster, SC_TYPE_SINGLE_VALUE, &counter);
+  }
+  stats_unlock();
+}
+
+static void
+_update_all_dir_metrics(gpointer s)
+{
+  DiskQGlobalMetrics *self = (DiskQGlobalMetrics *) s;
+
+  g_mutex_lock(&self->lock);
+  {
+    g_hash_table_foreach(self->dirs, _update_dir_metrics, NULL);
+  }
+  g_mutex_unlock(&self->lock);
+
+  _dir_watch_timer_start(self);
+}
+
+static void
+_new(gint type, gpointer c)
+{
+  DiskQGlobalMetrics *self = &diskq_global_metrics;
+
+  IV_TIMER_INIT(&self->dir_watch_timer);
+  self->dir_watch_timer.handler = _update_all_dir_metrics;
+  self->dir_watch_timer.cookie = self;
+
+  self->dirs = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL);
+}
+
+static void
+_init(gint type, gpointer c)
+{
+  GlobalConfig *cfg = main_loop_get_current_config(main_loop_get_instance());
+  if (!cfg)
+    return;
+
+  DiskQGlobalMetrics *self = &diskq_global_metrics;
+
+  self->freq = disk_queue_config_get_stats_freq(cfg);
+  if (self->freq == 0)
+    return;
+
+  _update_all_dir_metrics(self);
+}
+
+static void
+_deinit(gint type, gpointer c)
+{
+  DiskQGlobalMetrics *self = &diskq_global_metrics;
+
+  _dir_watch_timer_stop(self);
+
+  g_mutex_lock(&self->lock);
+  {
+    g_hash_table_remove_all(self->dirs);
+  }
+  g_mutex_unlock(&self->lock);
+}
+
+static void
+_free(gint type, gpointer c)
+{
+  DiskQGlobalMetrics *self = &diskq_global_metrics;
+
+  g_hash_table_destroy(self->dirs);
+}
+
+void
+diskq_global_metrics_init(void)
+{
+  if (diskq_global_metrics_global_inited)
+    return;
+
+  register_application_hook(AH_STARTUP, _new, NULL, AHM_RUN_ONCE);
+  register_application_hook(AH_CONFIG_CHANGED, _init, NULL, AHM_RUN_REPEAT);
+  register_application_hook(AH_CONFIG_STOPPED, _deinit, NULL, AHM_RUN_REPEAT);
+  register_application_hook(AH_SHUTDOWN, _free, NULL, AHM_RUN_ONCE);
+
+  diskq_global_metrics_global_inited = TRUE;
+}
+
+void
+diskq_global_metrics_watch_dir(const gchar *dir)
+{
+  DiskQGlobalMetrics *self = &diskq_global_metrics;
+
+  g_mutex_lock(&self->lock);
+  {
+    if (!g_hash_table_contains(self->dirs, dir))
+      g_hash_table_insert(self->dirs, g_strdup(dir), NULL);
+  }
+  g_mutex_unlock(&self->lock);
+}

modules/diskq/diskq-global-metrics.h

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2023 Attila Szakacs
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ *
+ */
+
+#ifndef DISKQ_GLOBAL_METRICS_H_
+#define DISKQ_GLOBAL_METRICS_H_
+
+#include "syslog-ng.h"
+
+void diskq_global_metrics_init(void);
+void diskq_global_metrics_watch_dir(const gchar *dir);
+
+#endif /* DISKQ_GLOBAL_METRICS_H_ */

modules/diskq/diskq-grammar.ym

@@ -114,6 +114,16 @@ diskq_global_options
 diskq_global_option
         : KW_TRUNCATE_SIZE_RATIO '(' float_between_0_and_1 ')' { disk_queue_config_set_truncate_size_ratio(configuration, $3); }
         | KW_PREALLOC '(' yesno ')'                            { disk_queue_config_set_prealloc(configuration, $3); }
+        | KW_STATS '(' diskq_global_stats_options ')'
+        ;
+
+diskq_global_stats_options
+        : diskq_global_stats_option diskq_global_stats_options
+        |
+        ;
+
+diskq_global_stats_option
+        : KW_FREQ '(' nonnegative_integer ')'                  { disk_queue_config_set_stats_freq(configuration, $3); }
         ;
 
 float_between_0_and_1

modules/diskq/diskq-parser.c

@@ -40,6 +40,8 @@ static CfgLexerKeyword diskq_keywords[] =
   { "dir",               KW_DIR },
   { "truncate_size_ratio", KW_TRUNCATE_SIZE_RATIO },
   { "prealloc",          KW_PREALLOC },
+  { "stats",             KW_STATS },
+  { "freq",              KW_FREQ },
   { NULL }
 };
 

modules/diskq/diskq-plugin.c

@@ -22,6 +22,7 @@
  */
 
 #include "cfg-parser.h"
+#include "diskq-global-metrics.h"
 #include "plugin.h"
 #include "plugin-types.h"
 
@@ -57,5 +58,6 @@ gboolean
 disk_buffer_module_init(PluginContext *context, CfgArgs *args)
 {
   plugin_register(context, diskq_plugins, G_N_ELEMENTS(diskq_plugins));
+  diskq_global_metrics_init();
   return TRUE;
 }

modules/diskq/diskq.c

@@ -25,6 +25,7 @@
 
 #include "diskq.h"
 #include "diskq-config.h"
+#include "diskq-global-metrics.h"
 
 #include "driver.h"
 #include "messages.h"
@@ -300,6 +301,8 @@ _attach(LogDriverPlugin *s, LogDriver *d)
   if (!_set_truncate_size_ratio_and_prealloc(self, dd))
     return FALSE;
 
+  diskq_global_metrics_watch_dir(self->options.dir);
+
   dd->acquire_queue = _acquire_queue;
   dd->release_queue = _release_queue;
   return TRUE;

news/feature-4399.md

@@ -0,0 +1,20 @@
+`disk-buffer`: Added metrics for monitoring the available space in disk-buffer `dir()`s.
+
+Metrics are available from `stats(level(1))`.
+
+By default, the metrics are generated every 5 minutes, but it can be changed in the global options:
+```
+options {
+  disk-buffer(
+    stats(
+      freq(10)
+    )
+  );
+};
+```
+Setting `freq(0)` disabled this feature.
+
+Example metrics:
+```
+syslogng_disk_queue_dir_available_bytes{dir="/var/syslog-ng"} 870109413376
+```

tests/copyright/policy

@@ -160,6 +160,7 @@ modules/add-contextual-data/tests/test_glob_selector\.c$
 modules/affile/tests/test_file_writer\.c$
 modules/afsocket/socket-options-unix\.[ch]$
 modules/basicfuncs/vp-funcs\.[ch]$
+modules/diskq/diskq-global-metrics\.[ch]$
 modules/diskq/tests/test_logqueue_disk\.c$
 modules/metrics-probe/metrics-probe(|-parser|-plugin|-grammar)\.(c|h|ym)$
 modules/metrics-probe/label-template\.(c|h)$