From dc4d7fa2ad23d3d75afadbe60d505b4289a18e0c Mon Sep 17 00:00:00 2001 From: Eric Joseph Jean Carbonneau Date: Tue, 7 Mar 2023 11:18:33 -0800 Subject: [PATCH] Added support for zfs in ldms The following plugins were added 1) zfs_zpool Stastistics on zfs pools. 2) zfs_leafvdevs Statistic on virtual devices leaves only. 3) zfs_topvdevs Statidstics on top virtual devices in the device tree (parent only). --- configure.ac | 14 + ldms/src/sampler/Makefile.am | 6 + ldms/src/sampler/zfs_leafvdevs/Makefile.am | 18 + .../zfs_leafvdevs/Plugin_zfs_leafvdevs.man | 47 ++ .../src/sampler/zfs_leafvdevs/zfs_leafvdevs.c | 503 +++++++++++++++++ ldms/src/sampler/zfs_topvdevs/Makefile.am | 18 + .../zfs_topvdevs/Plugin_zfs_topvdevs.man | 47 ++ ldms/src/sampler/zfs_topvdevs/zfs_topvdevs.c | 485 ++++++++++++++++ ldms/src/sampler/zfs_zpool/Makefile.am | 19 + .../sampler/zfs_zpool/Plugin_zfs_zpool.man | 36 ++ ldms/src/sampler/zfs_zpool/zfs_zpool.c | 516 ++++++++++++++++++ 11 files changed, 1709 insertions(+) create mode 100644 ldms/src/sampler/zfs_leafvdevs/Makefile.am create mode 100644 ldms/src/sampler/zfs_leafvdevs/Plugin_zfs_leafvdevs.man create mode 100644 ldms/src/sampler/zfs_leafvdevs/zfs_leafvdevs.c create mode 100644 ldms/src/sampler/zfs_topvdevs/Makefile.am create mode 100644 ldms/src/sampler/zfs_topvdevs/Plugin_zfs_topvdevs.man create mode 100644 ldms/src/sampler/zfs_topvdevs/zfs_topvdevs.c create mode 100644 ldms/src/sampler/zfs_zpool/Makefile.am create mode 100644 ldms/src/sampler/zfs_zpool/Plugin_zfs_zpool.man create mode 100644 ldms/src/sampler/zfs_zpool/zfs_zpool.c diff --git a/configure.ac b/configure.ac index 3ddde13c7..6e5410b4d 100644 --- a/configure.ac +++ b/configure.ac @@ -894,6 +894,17 @@ AS_IF([test "x$enable_slingshot" = xyes],[ [AC_MSG_ERROR([libcxi or its headers not found])]) ]) +AC_ARG_ENABLE([zfs], + [AS_HELP_STRING([--enable-zfs], [require the zfs related plugins @<:@default=check@:>@])], + [], + [enable_zfs="check"]) +AC_SEARCH_LIBS([zpool_vdev_name], [zfs],[HAVE_ZFS=yes]) +AM_CONDITIONAL([ENABLE_ZFS], [test "x$enable_zfs" != xno -a "x$HAVE_ZFS" = xyes]) +AS_IF([test "x$enable_zfs" = xyes],[ + AS_IF([test "x$HAVE_ZFS" = xno], + [AC_MSG_ERROR([libzfs or its headers not found])]) +]) + # define substitutions for configvars and other sed-generated files. # note carefully the escapes. OVIS_DO_SUBST([LDMS_SUBST_RULE], ["sed \ @@ -1054,6 +1065,9 @@ ldms/src/sampler/json/Makefile ldms/src/sampler/hweventpapi/Makefile \ ldms/src/sampler/rapl/Makefile \ ldms/src/sampler/tsampler/Makefile \ +ldms/src/sampler/zfs_topvdevs/Makefile +ldms/src/sampler/zfs_leafvdevs/Makefile +ldms/src/sampler/zfs_zpool/Makefile ldms/src/contrib/sampler/Makefile ldms/src/contrib/sampler/daos/Makefile ldms/src/contrib/sampler/daos/test/Makefile diff --git a/ldms/src/sampler/Makefile.am b/ldms/src/sampler/Makefile.am index 0bb8005f6..bcdc9f3cf 100644 --- a/ldms/src/sampler/Makefile.am +++ b/ldms/src/sampler/Makefile.am @@ -275,6 +275,12 @@ if ENABLE_BLOB_STREAM SUBDIRS += blob_stream endif +if ENABLE_ZFS +SUBDIRS += zfs_topvdevs +SUBDIRS += zfs_leafvdevs +SUBDIRS += zfs_zpool +endif + if ENABLE_SLINGSHOT if HAVE_LIBCXI SUBDIRS += slingshot_metrics diff --git a/ldms/src/sampler/zfs_leafvdevs/Makefile.am b/ldms/src/sampler/zfs_leafvdevs/Makefile.am new file mode 100644 index 000000000..690330944 --- /dev/null +++ b/ldms/src/sampler/zfs_leafvdevs/Makefile.am @@ -0,0 +1,18 @@ +pkglib_LTLIBRARIES = +dist_man1_MANS = + +AUTOMAKE_OPTIONS = subdir-objects + +AM_CPPFLAGS = @OVIS_INCLUDE_ABS@ \ + -I/usr/include/libspl \ + -I/usr/include/libzfs \ + -Wall + +AM_LDFLAGS = @OVIS_LIB_ABS@ +COMMON_LIBADD = -lsampler_base -lldms -lovis_util -lcoll \ + @LDFLAGS_GETTIME@ + +libzfs_leafvdevs_la_SOURCES = zfs_leafvdevs.c +libzfs_leafvdevs_la_LIBADD = $(COMMON_LIBADD) +pkglib_LTLIBRARIES += libzfs_leafvdevs.la +dist_man7_MANS = Plugin_zfs_leafvdevs.man diff --git a/ldms/src/sampler/zfs_leafvdevs/Plugin_zfs_leafvdevs.man b/ldms/src/sampler/zfs_leafvdevs/Plugin_zfs_leafvdevs.man new file mode 100644 index 000000000..e45e3b7f3 --- /dev/null +++ b/ldms/src/sampler/zfs_leafvdevs/Plugin_zfs_leafvdevs.man @@ -0,0 +1,47 @@ +.\" Manpage for Plugin_zfs_leafvdevs +.\" Contact ovis-help@ca.sandia.gov to correct errors or typos. +.TH man 7 "19 Apr 2023" "v4" "LDMS Plugin zfs_leafvdevs man page" + +.SH NAME +Plugin_zfs_leafvdevs - man page for the LDMS zfs_leafvdevs plugin + +.SH SYNOPSIS +Within ldmsd_controller or a configuration file: +.br +config name=zfs_leafvdevs + +.SH DESCRIPTION +With LDMS (Lightweight Distributed Metric Service), plugins for the ldmsd (ldms +daemon) are configured via ldmsd_controller or a configuration file. The +zfs_leafvdevs plugin uses LDMS_V_LIST and LDMS_V_RECORD to provide zfs leaf +virtual devices using libzfs. + +.SH CONFIGURATION ATTRIBUTE SYNTAX +The zfs_leafvdevs plugin uses the sampler_base base class. This man page covers +only the configuration attributes, or those with default values, specific to the +this plugin; see ldms_sampler_base.man for the attributes of the base class. + +.TP +.BR config +name= +.br +configuration line +.RS +.TP +name= +.br +This MUST be zfs_leafvdevs. +.RE + +.SH EXAMPLES +.PP +Within ldmsd_controller or a configuration file: +.nf +load name=zfs_leafvdevs +config name=zfs_leafvdevs producer=${HOSTNAME} instance=${HOSTNAME}/zfs_leafvdevs +start name=zfs_leafvdevs interval=1000000 offset=0 +.fi + +.SH SEE ALSO +ldmsd(8), ldms_quickstart(7), ldmsd_controller(8), ldms_sampler_base(7), +Plugin_leafvdevs(7) diff --git a/ldms/src/sampler/zfs_leafvdevs/zfs_leafvdevs.c b/ldms/src/sampler/zfs_leafvdevs/zfs_leafvdevs.c new file mode 100644 index 000000000..70f32ee3e --- /dev/null +++ b/ldms/src/sampler/zfs_leafvdevs/zfs_leafvdevs.c @@ -0,0 +1,503 @@ +/* -*- c-basic-offset: 8 -*- */ +/* Copyright 2022 Lawrence Livermore National Security, LLC + * See the top-level COPYING file for details. + * + * SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ldms.h" +#include "ldmsd.h" +#include "config.h" +#include "sampler_base.h" +#include "libzfs.h" +#include + +#define SAMP "zfs_leafvdevs" + +#ifndef ARRAY_LEN +#define ARRAY_LEN(a) (sizeof(a) / sizeof(*a)) +#endif + +/* Function prototypes */ + +static int get_leaf_stats(zpool_handle_t *, void *); +static int vdevs_count(zpool_handle_t *, void *); +typedef int (*stat_printer_f) (nvlist_t *, const char *, zpool_handle_t *); + +static base_data_t sampler_base; +static libzfs_handle_t *g_zfs; + +static struct { + int vdev_list_idx; + int vdev_rec_idx; +} index_store; + +static ldms_mval_t list_handle; + +/* metric templates for a virtual device */ +static struct ldms_metric_template_s zfs_leafvdevs[] = { + {"zpoolname", 0, LDMS_V_CHAR_ARRAY, "", ZFS_MAX_DATASET_NAME_LEN}, + {"state", 0, LDMS_V_CHAR_ARRAY, "", ZFS_MAX_DATASET_NAME_LEN}, + {"vdevname", 0, LDMS_V_CHAR_ARRAY, "", ZFS_MAX_DATASET_NAME_LEN}, + {"vdevpath", 0, LDMS_V_CHAR_ARRAY, "", ZFS_MAX_DATASET_NAME_LEN}, + {"vdevguid", 0, LDMS_V_CHAR_ARRAY, "", ZFS_MAX_DATASET_NAME_LEN}, + {"alloc", 0, LDMS_V_U64, "", 1}, + {"free", 0, LDMS_V_U64, "", 1}, + {"size", 0, LDMS_V_U64, "", 1}, + {"read_bytes", 0, LDMS_V_U64, "", 1}, + {"read_errors", 0, LDMS_V_U64, "", 1}, + {"read_ops", 0, LDMS_V_U64, "", 1}, + {"write_bytes", 0, LDMS_V_U64, "", 1}, + {"write_errors", 0, LDMS_V_U64, "", 1}, + {"write_ops", 0, LDMS_V_U64, "", 1}, + {"checksum_errors", 0, LDMS_V_U64, "", 1}, + {"fragmentation", 0, LDMS_V_U64, "", 1}, + {"init_errors", 0, LDMS_V_U64, "", 1}, + {"trim_errors", 0, LDMS_V_U64, "", 1}, + {"slow_ios", 0, LDMS_V_U64, "", 1}, + {0}, +}; + +#define VDEV_METRICS_LEN (ARRAY_LEN(zfs_leafvdevs) - 1) +static int vdev_metric_ids[VDEV_METRICS_LEN]; +static size_t zpool_vdev_heap_sz; +static uint_t leaf_vdev_count; + +static ovis_log_t mylog; + + +/***************************************************************************** + * Initialize the structure as schema and add them to the base schema. + * Also calculate the size of memory needed per schema and add it to the ldms + * schema list. + ****************************************************************************/ + +static int initialize_ldms_structs() +{ + ldms_record_t zpool_vdev_def; /* a pointer */ + int rc; + + ovis_log(mylog, OVIS_LDEBUG, SAMP " initialize()\n"); + + /* Create the schema */ + base_schema_new(sampler_base); + if (sampler_base->schema == NULL) + goto err1; + + /* create the vdev record */ + zpool_vdev_def = ldms_record_from_template("zfs_leafvdevs_stats", + zfs_leafvdevs, + vdev_metric_ids); + if (zpool_vdev_def == NULL) + goto err2; + + zpool_vdev_heap_sz = ldms_record_heap_size_get(zpool_vdev_def); + rc = ldms_schema_record_add(sampler_base->schema, zpool_vdev_def); + if (rc < 0) + goto err3; + + index_store.vdev_rec_idx = rc; + leaf_vdev_count = 0; + rc = zpool_iter(g_zfs, vdevs_count, NULL); + /* add error for iter in case here */ + ovis_log(mylog, OVIS_LDEBUG, SAMP " leaf_vdev_count : %d\n", leaf_vdev_count); + rc = ldms_schema_metric_list_add(sampler_base->schema, + "zpool_vdev_list", + NULL, + leaf_vdev_count * zpool_vdev_heap_sz); + if (rc < 0) + goto err2; + + index_store.vdev_list_idx = rc; + + /* Create the metric set */ + base_set_new(sampler_base); + if (sampler_base->set == NULL) + goto err2; + + return 0; + + err3: + /* We only manually delete record template when it + * hasn't been added to the schema yet */ + ldms_record_delete(zpool_vdev_def); + err2: + base_schema_delete(sampler_base); + err1: + ovis_log(mylog, OVIS_LERROR, SAMP " initialization failed\n"); + return -1; +} + +/***************************************************************************** + * WHAT: + * 1) Initialize the sampler base schema. + * 2) Initialize all structure and memory. + * 3) initialize the zfslib to sample the zpools stats. + * CALLER: + * ldms daemon. In error the plugin is aborted. + ****************************************************************************/ + +static int config(struct ldmsd_plugin *self, + struct attr_value_list *kwl, struct attr_value_list *avl) +{ + int rc = 0; + + ovis_log(mylog, OVIS_LDEBUG, SAMP " config() called\n"); + + sampler_base = base_config(avl, SAMP, "zfs_leafvdevs", mylog); + if ((g_zfs = libzfs_init()) == NULL) { + rc = errno; + ovis_log(mylog, OVIS_LERROR, + SAMP " : Failed to initialize libzfs: %d\n", errno); + ovis_log(mylog, OVIS_LERROR, + SAMP + " : Is the zfs module loaded or zrepl running?\n"); + } else { + rc = initialize_ldms_structs(); + } + + if (rc < 0) { + base_del(sampler_base); + sampler_base = NULL; + } + + return rc; +} + +/***************************************************************************** + * WHAT: + * reallocate heap size plus 1 zpool struct and one vdev struct + * CALLER: + * self, (plugin) + ****************************************************************************/ +static int resize_metric_set() +{ + size_t previous_heap_size; + size_t new_heap_size; + int rc = 0; + + previous_heap_size = ldms_set_heap_size_get(sampler_base->set); + base_set_delete(sampler_base); + + new_heap_size = previous_heap_size; + new_heap_size += zpool_vdev_heap_sz; + + if (base_set_new_heap(sampler_base, new_heap_size) == NULL) { + rc = errno; + ovis_log(mylog, OVIS_LERROR, + SAMP " : Failed to resize metric set heap: %d\n", + errno); + } else { + ovis_log(mylog, OVIS_LDEBUG, "ldms resize of list successful\n"); + } + return rc; +} + +static int sample(struct ldmsd_sampler *self) +{ + int rc = 0; + + base_sample_begin(sampler_base); + + list_handle = + ldms_metric_get(sampler_base->set, index_store.vdev_list_idx); + ldms_list_purge(sampler_base->set, list_handle); + + rc = zpool_iter(g_zfs, get_leaf_stats, NULL); + if (rc != 0) { + ovis_log(mylog, OVIS_LERROR, + SAMP " sample():zfs_pool print_stat() failed: %d\n", rc); + } + + base_sample_end(sampler_base); + + return rc; +} + +static void term(struct ldmsd_plugin *self) +{ + ovis_log(mylog, OVIS_LDEBUG, SAMP " term() called\n"); + base_set_delete(sampler_base); + base_del(sampler_base); + sampler_base = NULL; +} + +static ldms_set_t get_set(struct ldmsd_sampler *self) +{ + return NULL; +} + +static const char *usage(struct ldmsd_plugin *self) +{ + ovis_log(mylog, OVIS_LDEBUG, SAMP " usage() called\n"); + return "config name=" SAMP " " BASE_CONFIG_SYNOPSIS BASE_CONFIG_DESC; +} + +struct ldmsd_plugin *get_plugin() +{ + int rc; + static struct ldmsd_sampler plugin = { + .base = { + .name = SAMP, + .type = LDMSD_PLUGIN_SAMPLER, + .term = term, + .config = config, + .usage = usage, + }, + .get_set = get_set, + .sample = sample, + }; + + mylog = ovis_log_register("sampler."SAMP, "Message for the " SAMP " plugin"); + if (!mylog) { + rc = errno; + ovis_log(NULL, OVIS_LWARN, "Failed to create the log subsystem " + "of '" SAMP "' plugin. Error %d\n", rc); + } + + return &plugin.base; +} + +/*********** let's count the vdev here *************/ +static int get_leaf_vdevs_count(nvlist_t * nvroot, const char *pool_name) +{ + uint_t children; + nvlist_t **child; + + if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN, + &child, &children) == 0) { + for (int c = 0; c < children; c++) { + get_leaf_vdevs_count(child[c], pool_name); + } + } else { + leaf_vdev_count++; + } + return (0); +} + +static int vdevs_count(zpool_handle_t * zhp, void *data) +{ + int rc = 0; + nvlist_t *config, *nvroot; + char *pool_name; + + if ((config = zpool_get_config(zhp, NULL)) != NULL) { + if ((rc = + nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, + &nvroot)) != 0) { + zpool_close(zhp); + return (rc); + } + + pool_name = (char *)zpool_get_name(zhp); + rc = get_leaf_vdevs_count(nvroot, pool_name); + } else { + ovis_log(mylog, OVIS_LERROR, + SAMP " zpool get config failed in vdevs_count\n"); + } + zpool_close(zhp); + return (rc); +} + +/********************* Done counting vdevs *****************************/ + +/* + * vdev summary stats are a combination of the data shown by + * zpool status` and `zpool list -v + * zpoolname + * state + * vdevname + * alloc + * free + * size + * read_bytes + * read_errors + * read_ops + * write_bytes + * write_errors + * write_ops + * checksum_errors + * fragmentation + * init_errors + */ +static int get_leafvdev_stats(nvlist_t * nvroot, const char *pool_name, + zpool_handle_t * zhp) +{ + uint_t c; + vdev_stat_t *vs; + char *vdev_name = NULL; + char *vdev_path = NULL; + char *vdev_guid = NULL; + ldms_mval_t record_instance; + int rc = 0; /*return code */ + + vdev_name = zpool_vdev_name(g_zfs, zhp, nvroot, 0); + vdev_path = + zpool_vdev_name(g_zfs, zhp, nvroot, + VDEV_NAME_PATH | VDEV_NAME_FOLLOW_LINKS); + vdev_guid = zpool_vdev_name(g_zfs, zhp, nvroot, VDEV_NAME_GUID); + + if (nvlist_lookup_uint64_array(nvroot, + ZPOOL_CONFIG_VDEV_STATS, + (uint64_t **) & vs, &c) != 0) { + rc = 1; + } + + record_instance = ldms_record_alloc(sampler_base->set, + index_store.vdev_rec_idx); + + if (record_instance == NULL) { + ovis_log(mylog, OVIS_LDEBUG, + SAMP + ": ldms_record_alloc() failed, resizing metric set\n"); + resize_metric_set(); + record_instance = ldms_record_alloc(sampler_base->set, + index_store.vdev_rec_idx); + if (record_instance == NULL) + rc = 2; + } + + if (rc == 0) { + rc = ldms_list_append_record(sampler_base->set, list_handle, + record_instance); + + /* zpoolname 0 */ + ldms_record_array_set_str(record_instance, vdev_metric_ids[0], + pool_name); + /* zpool state 1 */ + ldms_record_array_set_str(record_instance, vdev_metric_ids[1], + zpool_state_to_name((vdev_state_t) + vs->vs_state, + (vdev_aux_t) + vs->vs_aux)); + /* vdevname 2 */ + ldms_record_array_set_str(record_instance, vdev_metric_ids[2], + vdev_name); + /* vdevpath 3 */ + ldms_record_array_set_str(record_instance, vdev_metric_ids[3], + vdev_path); + /* vdevguid 4 */ + ldms_record_array_set_str(record_instance, vdev_metric_ids[4], + vdev_guid); + /* alloc 5 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[5], + vs->vs_alloc); + /* free 6 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[6], + vs->vs_space - vs->vs_alloc); + /* size 7 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[7], + vs->vs_space); + /* read_bytes 8 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[8], + vs->vs_bytes[ZIO_TYPE_READ]); + /* iread_errors 9 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[9], + vs->vs_read_errors); + /* read_ops 10 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[10], + vs->vs_ops[ZIO_TYPE_READ]); + /* write_bytes 11 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[11], + vs->vs_bytes[ZIO_TYPE_WRITE]); + /* write_errors 12 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[12], + vs->vs_write_errors); + /* write_ops 13 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[13], + vs->vs_ops[ZIO_TYPE_WRITE]); + /* checksum errors 14 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[14], + vs->vs_checksum_errors); + /* fragmentation 15 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[15], + vs->vs_fragmentation); + /* initialization errors 16 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[16], + vs->vs_initialize_errors); + /* trim errors 17 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[17], + vs->vs_trim_errors); + /* slow ios 18 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[18], + vs->vs_slow_ios); + + } + + free(vdev_name); + free(vdev_path); + free(vdev_guid); + + return (rc); +} + +/* + * recursive stats printer + */ +static int get_recursive_stats(stat_printer_f func, nvlist_t * nvroot, + const char *pool_name, zpool_handle_t * zhp) +{ + uint_t c, children; + nvlist_t **child; + int err; + + if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN, + &child, &children) == 0) { + for (c = 0; c < children; c++) { + get_recursive_stats(func, child[c], pool_name, zhp); + } + } else { + err = func(nvroot, pool_name, zhp); + if (err) + return (err); + } + return (0); +} + +/* + * call-back to print the stats from the pool config + * + * Note: if the pool is broken, this can hang indefinitely and perhaps in an + * unkillable state. + */ + +static int get_leaf_stats(zpool_handle_t * zhp, void *data) +{ + int err = 0; + nvlist_t *config, *nvroot; + char *pool_name; + + if ((config = zpool_get_config(zhp, NULL)) != NULL) { + if ((err = + nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, + &nvroot)) != 0) + zpool_close(zhp); + if (err == 0) { + pool_name = (char *)zpool_get_name(zhp); + err = get_recursive_stats(get_leafvdev_stats, nvroot, + pool_name, zhp); + } + } else { + ovis_log(mylog, OVIS_LERROR, + SAMP " zpool get config failed in get_leaf_stats\n"); + err = 1; + } + zpool_close(zhp); + return (err); +} diff --git a/ldms/src/sampler/zfs_topvdevs/Makefile.am b/ldms/src/sampler/zfs_topvdevs/Makefile.am new file mode 100644 index 000000000..e2e337177 --- /dev/null +++ b/ldms/src/sampler/zfs_topvdevs/Makefile.am @@ -0,0 +1,18 @@ +pkglib_LTLIBRARIES = +dist_man1_MANS = + +AUTOMAKE_OPTIONS = subdir-objects + +AM_CPPFLAGS = @OVIS_INCLUDE_ABS@ \ + -I/usr/include/libspl \ + -I/usr/include/libzfs \ + -Wall + +AM_LDFLAGS = @OVIS_LIB_ABS@ +COMMON_LIBADD = -lsampler_base -lldms -lovis_util -lcoll \ + @LDFLAGS_GETTIME@ + +libzfs_topvdevs_la_SOURCES = zfs_topvdevs.c +libzfs_topvdevs_la_LIBADD = $(COMMON_LIBADD) +pkglib_LTLIBRARIES += libzfs_topvdevs.la +dist_man7_MANS = Plugin_zfs_topvdevs.man diff --git a/ldms/src/sampler/zfs_topvdevs/Plugin_zfs_topvdevs.man b/ldms/src/sampler/zfs_topvdevs/Plugin_zfs_topvdevs.man new file mode 100644 index 000000000..570048359 --- /dev/null +++ b/ldms/src/sampler/zfs_topvdevs/Plugin_zfs_topvdevs.man @@ -0,0 +1,47 @@ +.\" Manpage for Plugin_zfs_topvdevs +.\" Contact ovis-help@ca.sandia.gov to correct errors or typos. +.TH man 7 "19 Apr 2023" "v4" "LDMS Plugin zfs_topvdevs man page" + +.SH NAME +Plugin_zfs_topvdevs - man page for the LDMS zfs_topvdevs plugin + +.SH SYNOPSIS +Within ldmsd_controller or a configuration file: +.br +config name=zfs_topvdevs + +.SH DESCRIPTION +With LDMS (Lightweight Distributed Metric Service), plugins for the ldmsd (ldms +daemon) are configured via ldmsd_controller or a configuration file. The +zfs_topvdevs plugin uses LDMS_V_LIST and LDMS_V_RECORD to provide top level +zfs virtual devices info using libzfs. + +.SH CONFIGURATION ATTRIBUTE SYNTAX +The zfs_topvdevs plugin uses the sampler_base base class. This man page covers +only the configuration attributes, or those with default values, specific to the +this plugin; see ldms_sampler_base.man for the attributes of the base class. + +.TP +.BR config +name= +.br +configuration line +.RS +.TP +name= +.br +This MUST be zfs_topvdevs. +.RE + +.SH EXAMPLES +.PP +Within ldmsd_controller or a configuration file: +.nf +load name=procnetdev +config name=zfs_topvdevs producer=${HOSTNAME} instance=${HOSTNAME}/zfs_topvdevs +start name=zfs_topvdevs interval=1000000 offset=0 +.fi + +.SH SEE ALSO +ldmsd(8), ldms_quickstart(7), ldmsd_controller(8), ldms_sampler_base(7), +Plugin_zfs_topvdevs(7) diff --git a/ldms/src/sampler/zfs_topvdevs/zfs_topvdevs.c b/ldms/src/sampler/zfs_topvdevs/zfs_topvdevs.c new file mode 100644 index 000000000..5e54a05fe --- /dev/null +++ b/ldms/src/sampler/zfs_topvdevs/zfs_topvdevs.c @@ -0,0 +1,485 @@ +/* -*- c-basic-offset: 8 -*- */ +/* Copyright 2022 Lawrence Livermore National Security, LLC + * See the top-level COPYING file for details. + * + * SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) + */ + +#define _GNU_SOURCE +/* Next we include the headers to bring in the zfslib in action */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ldms.h" +#include "ldmsd.h" +#include "config.h" +#include "sampler_base.h" +#include "libzfs.h" +#include + +#define SAMP "zfs_topvdevs" + +#ifndef ARRAY_LEN +#define ARRAY_LEN(a) (sizeof(a) / sizeof(*a)) +#endif + +/* Function prototypes */ + +static int get_stats(zpool_handle_t *, void *); +static int vdevs_count(zpool_handle_t *, void *); +typedef int (*stat_printer_f) (nvlist_t *, const char *, zpool_handle_t *); + +static base_data_t sampler_base; +static libzfs_handle_t *g_zfs; + +static struct { + int vdev_list_idx; + int vdev_rec_idx; +} index_store; + +static ldms_mval_t list_handle; +static uint_t top_vdev_count; + +/* metric templates for a virtual device */ +static struct ldms_metric_template_s zfs_topvdevs[] = { + {"zpoolname", 0, LDMS_V_CHAR_ARRAY, "", ZFS_MAX_DATASET_NAME_LEN}, + {"state", 0, LDMS_V_CHAR_ARRAY, "", ZFS_MAX_DATASET_NAME_LEN}, + {"vdevname", 0, LDMS_V_CHAR_ARRAY, "", ZFS_MAX_DATASET_NAME_LEN}, + {"alloc", 0, LDMS_V_U64, "", 1}, + {"free", 0, LDMS_V_U64, "", 1}, + {"size", 0, LDMS_V_U64, "", 1}, + {"read_bytes", 0, LDMS_V_U64, "", 1}, + {"read_errors", 0, LDMS_V_U64, "", 1}, + {"read_ops", 0, LDMS_V_U64, "", 1}, + {"write_bytes", 0, LDMS_V_U64, "", 1}, + {"write_errors", 0, LDMS_V_U64, "", 1}, + {"write_ops", 0, LDMS_V_U64, "", 1}, + {"checksum_errors", 0, LDMS_V_U64, "", 1}, + {"fragmentation", 0, LDMS_V_U64, "", 1}, + {"init_errors", 0, LDMS_V_U64, "", 1}, + {"trim_errors", 0, LDMS_V_U64, "", 1}, + {"slow_ios", 0, LDMS_V_U64, "", 1}, + {0}, +}; + +/* need to find a better way more intuitive than that + * to manage heap. Like auto resize as a base function */ + +#define VDEV_METRICS_LEN (ARRAY_LEN(zfs_topvdevs) - 1) +static int vdev_metric_ids[VDEV_METRICS_LEN]; +static size_t zpool_vdev_heap_sz; + +static ovis_log_t mylog; + +/***************************************************************************** + * Initialize the structure as schema and add them to the base schema. + * Also calculate the size of memory needed per schema and add it to the ldms + * schema list. + ****************************************************************************/ + +static int initialize_ldms_structs() +{ + ldms_record_t zpool_vdev_def; /* a pointer */ + int rc = 0; + + ovis_log(mylog, OVIS_LDEBUG, SAMP " initialize()\n"); + + /* Create the schema */ + base_schema_new(sampler_base); + if (sampler_base->schema == NULL) + goto err1; + + /* create the vdev record */ + zpool_vdev_def = ldms_record_from_template("zfs_topvdevs_stats", + zfs_topvdevs, + vdev_metric_ids); + if (zpool_vdev_def == NULL) + goto err2; + + zpool_vdev_heap_sz = ldms_record_heap_size_get(zpool_vdev_def); + rc = ldms_schema_record_add(sampler_base->schema, zpool_vdev_def); + if (rc < 0) + goto err3; + + index_store.vdev_rec_idx = rc; + top_vdev_count = 0; + rc = zpool_iter(g_zfs, vdevs_count, NULL); + /* add error for iter in case here */ + rc = ldms_schema_metric_list_add(sampler_base->schema, + "zpool_vdev_list", + NULL, + top_vdev_count * zpool_vdev_heap_sz); + if (rc < 0) + goto err2; + + index_store.vdev_list_idx = rc; + + /* Create the metric set */ + base_set_new(sampler_base); + if (sampler_base->set == NULL) + goto err2; + + /* as rc is used for index we cannot return it + * so we return 0 at this point everything went + * fine + */ + return (0); + + err3: + /* We only manually delete record template when it + * hasn't been added to the schema yet */ + ldms_record_delete(zpool_vdev_def); + err2: + base_schema_delete(sampler_base); + err1: + ovis_log(mylog, OVIS_LERROR, SAMP " initialization failed\n"); + return -1; +} + +/***************************************************************************** + * WHAT: + * 1) Initialize the sampler base schema. + * 2) Initialize all structure and memory. + * 3) initialize the zfslib to sample the zpools stats. + * CALLER: + * ldms daemon. In error the plugin is aborted. + ****************************************************************************/ + +static int config(struct ldmsd_plugin *self, + struct attr_value_list *kwl, struct attr_value_list *avl) +{ + int rc = 0; + + ovis_log(mylog, OVIS_LDEBUG, SAMP " config() called\n"); + + sampler_base = base_config(avl, SAMP, "zfs_topvdevs", mylog); + if ((g_zfs = libzfs_init()) == NULL) { + rc = errno; + ovis_log(mylog, OVIS_LERROR, + SAMP " : Failed to initialize libzfs: %s\n", + libzfs_error_init(rc)); + ovis_log(mylog, OVIS_LERROR, + SAMP + " : Is the zfs module loaded or zrepl running?\n"); + } else { + rc = initialize_ldms_structs(); + } + + if (rc < 0) { + base_del(sampler_base); + sampler_base = NULL; + } + + return rc; +} + +/***************************************************************************** + * WHAT: + * reallocate heap size plus 1 zpool struct and one vdev struct + * CALLER: + * self, (plugin) + ****************************************************************************/ +static int resize_metric_set() +{ + size_t previous_heap_size; + size_t new_heap_size; + int rc = 0; + + previous_heap_size = ldms_set_heap_size_get(sampler_base->set); + base_set_delete(sampler_base); + + new_heap_size = previous_heap_size; + new_heap_size += zpool_vdev_heap_sz; + + if (base_set_new_heap(sampler_base, new_heap_size) == NULL) { + rc = errno; + ovis_log(mylog, OVIS_LERROR, + SAMP " : Failed to resize metric set heap: %d\n", + errno); + } else { + ovis_log(mylog, OVIS_LDEBUG, "ldms resize of list successful\n"); + } + return rc; +} + +static int sample(struct ldmsd_sampler *self) +{ + int rc = 0; + + base_sample_begin(sampler_base); + + list_handle = + ldms_metric_get(sampler_base->set, index_store.vdev_list_idx); + ldms_list_purge(sampler_base->set, list_handle); + + rc = zpool_iter(g_zfs, get_stats, NULL); + if (rc != 0) + ovis_log(mylog, OVIS_LERROR, SAMP " sample():get_stat() failed: %d\n", + rc); + + base_sample_end(sampler_base); + + return rc; +} + +static void term(struct ldmsd_plugin *self) +{ + ovis_log(mylog, OVIS_LDEBUG, SAMP " term() called\n"); + base_set_delete(sampler_base); + base_del(sampler_base); + sampler_base = NULL; + if (mylog) + ovis_log_destroy(mylog); + +} + +static ldms_set_t get_set(struct ldmsd_sampler *self) +{ + return NULL; +} + +static const char *usage(struct ldmsd_plugin *self) +{ + ovis_log(mylog, OVIS_LDEBUG, SAMP " usage() called\n"); + return "config name=" SAMP " " BASE_CONFIG_SYNOPSIS BASE_CONFIG_DESC; +} + +struct ldmsd_plugin *get_plugin() +{ + int rc; + static struct ldmsd_sampler plugin = { + .base = { + .name = SAMP, + .type = LDMSD_PLUGIN_SAMPLER, + .term = term, + .config = config, + .usage = usage, + }, + .get_set = get_set, + .sample = sample, + }; + + mylog = ovis_log_register("sampler."SAMP, "Message for the " SAMP " plugin"); + if (!mylog) { + rc = errno; + ovis_log(NULL, OVIS_LWARN, "Failed to create the log subsystem " + "of '" SAMP "' plugin. Error %d\n", rc); + } + + return &plugin.base; +} + +static int vdevs_count(zpool_handle_t * zhp, void *data) +{ + int rc = 0; + nvlist_t *config, *nvroot; + uint_t children; + nvlist_t **child; + + if ((config = zpool_get_config(zhp, NULL)) != NULL) { + if ((rc = nvlist_lookup_nvlist(config, + ZPOOL_CONFIG_VDEV_TREE, + &nvroot)) != 0) { + zpool_close(zhp); + return (rc); + } + + if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN, + &child, &children) == 0) + top_vdev_count += children; + + top_vdev_count++; /* add one for vdev root in zpool top. */ + } else { + ovis_log(mylog, OVIS_LERROR, + SAMP " Zpool get config failed in vdevs_count\n"); + rc = 1; + } + zpool_close(zhp); + return (rc); +} + +/* + * vdev summary stats are a combination of the data shown by + * zpool status` and `zpool list -v + * zpoolname + * state + * vdevname + * alloc + * free + * size + * read_bytes + * read_errors + * read_ops + * write_bytes + * write_errors + * write_ops + * checksum_errors + * fragmentation + * init_errors + * trim_errors + * slow_ios + */ +static int get_vdev_stats(nvlist_t * nvroot, const char *pool_name, + zpool_handle_t * zhp) +{ + uint_t c; + vdev_stat_t *vs; + char *vdev_name = NULL; + ldms_mval_t record_instance; + int rc = 0; /*return code */ + + vdev_name = zpool_vdev_name(g_zfs, zhp, nvroot, VDEV_NAME_TYPE_ID); + + rc = nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS, + (uint64_t **) & vs, &c); + + record_instance = ldms_record_alloc(sampler_base->set, + index_store.vdev_rec_idx); + + if (record_instance == NULL) { + ovis_log(mylog, OVIS_LDEBUG, + SAMP + ": ldms_record_alloc() failed, resizing metric set\n"); + resize_metric_set(); + record_instance = ldms_record_alloc(sampler_base->set, + index_store.vdev_rec_idx); + if (record_instance == NULL) + rc = errno; + } + + if (rc == 0) { + rc = ldms_list_append_record(sampler_base->set, list_handle, + record_instance); + + /* zpoolname 0 */ + ldms_record_array_set_str(record_instance, vdev_metric_ids[0], + pool_name); + /* zpool state 1 */ + ldms_record_array_set_str(record_instance, vdev_metric_ids[1], + zpool_state_to_name((vdev_state_t) + vs->vs_state, + (vdev_aux_t) + vs->vs_aux)); + /* vdevname 2 */ + ldms_record_array_set_str(record_instance, vdev_metric_ids[2], + vdev_name); + /* alloc 3 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[3], + vs->vs_alloc); + /* free 4 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[4], + vs->vs_space - vs->vs_alloc); + /* size 5 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[5], + vs->vs_space); + /* read_bytes 6 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[6], + vs->vs_bytes[ZIO_TYPE_READ]); + /* read_errors 7 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[7], + vs->vs_read_errors); + /* read_ops 8 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[8], + vs->vs_ops[ZIO_TYPE_READ]); + /* write_bytes 9 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[9], + vs->vs_bytes[ZIO_TYPE_WRITE]); + /* write_errors 10 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[10], + vs->vs_write_errors); + /* write_ops 11 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[11], + vs->vs_ops[ZIO_TYPE_WRITE]); + /* checksum errors 12 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[12], + vs->vs_checksum_errors); + /* fragmentation 13 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[13], + vs->vs_fragmentation); + /* initialization errors 14 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[14], + vs->vs_initialize_errors); + /* trim errors 15 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[15], + vs->vs_trim_errors); + /* slow ios 16 */ + ldms_record_set_u64(record_instance, vdev_metric_ids[16], + vs->vs_slow_ios); + } + + free(vdev_name); + + return (rc); +} + +/* + * recursive stats printer + */ +static int get_recursive_stats(stat_printer_f func, nvlist_t * nvroot, + const char *pool_name, zpool_handle_t * zhp) +{ + uint_t c, children; + nvlist_t **child; + int rc = 0; + + if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN, + &child, &children) == 0) { + for (c = 0; c < children; c++) { + rc = func(child[c], pool_name, zhp); + } + } + return (rc); +} + +/* + * call-back to print the stats from the pool config + * + * Note: if the pool is broken, this can hang indefinitely and perhaps in an + * unkillable state. + */ + +static int get_stats(zpool_handle_t * zhp, void *data) +{ + uint_t c; + int rc = 0; + nvlist_t *config, *nvroot; + vdev_stat_t *vs; + char *pool_name; + + if ((config = zpool_get_config(zhp, NULL)) != NULL) { + if ((rc = + nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, + &nvroot)) != 0) { + zpool_close(zhp); + return (rc); + } + if ((rc = + nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS, + (uint64_t **) & vs, &c)) != 0) { + zpool_close(zhp); + return (rc); + } + + pool_name = (char *)zpool_get_name(zhp); + rc = get_recursive_stats(get_vdev_stats, nvroot, pool_name, + zhp); + } else { + ovis_log(mylog, OVIS_LERROR, + SAMP " Failed to get zpool config in get_stats\n"); + rc = 1; + } + zpool_close(zhp); + return (rc); +} diff --git a/ldms/src/sampler/zfs_zpool/Makefile.am b/ldms/src/sampler/zfs_zpool/Makefile.am new file mode 100644 index 000000000..0c18dc279 --- /dev/null +++ b/ldms/src/sampler/zfs_zpool/Makefile.am @@ -0,0 +1,19 @@ +pkglib_LTLIBRARIES = +dist_man1_MANS = + +AUTOMAKE_OPTIONS = subdir-objects + +AM_CPPFLAGS = @OVIS_INCLUDE_ABS@ \ + -I/usr/include/libspl \ + -I/usr/include/libzfs \ + -Wall + +AM_LDFLAGS = @OVIS_LIB_ABS@ +COMMON_LIBADD = -lsampler_base -lldms -lovis_util -lcoll \ + @LDFLAGS_GETTIME@ + +libzfs_zpool_la_SOURCES = zfs_zpool.c +libzfs_zpool_la_LIBADD = $(COMMON_LIBADD) + +pkglib_LTLIBRARIES += libzfs_zpool.la +dist_man7_MANS = Plugin_zfs_zpool.man diff --git a/ldms/src/sampler/zfs_zpool/Plugin_zfs_zpool.man b/ldms/src/sampler/zfs_zpool/Plugin_zfs_zpool.man new file mode 100644 index 000000000..48bb44c77 --- /dev/null +++ b/ldms/src/sampler/zfs_zpool/Plugin_zfs_zpool.man @@ -0,0 +1,36 @@ +.\" Manpage for Plugin_zfs_zpool +.\" Contact ovis-help@ca.sandia.gov to correct errors or typos. +.TH man 7 "19 Apr 2023" "v4" "LDMS Plugin zfs_zpool man page" + +.SH NAME +Plugin_zfs_zpool - man page for the LDMS zfs_zpool plugin + +.SH SYNOPSIS +Within ldmsd_controller or a configuration file: +.br +config name=zfs_zpool [ = ] + +.SH DESCRIPTION +With LDMS (Lightweight Distributed Metric Service), plugins for the ldmsd (ldms +daemon) are configured via ldmsd_controller or a configuration file. The +zfs_zpool plugin uses LDMS_V_LIST and LDMS_V_RECORD to provide zpool info +using libzfs. + +.SH CONFIGURATION ATTRIBUTE SYNTAX +The zfs_zpool plugin uses the sampler_base base class. This man page covers +only the configuration attributes, or those with default values, specific to the +this plugin; see ldms_sampler_base.man for the attributes of the base class. + +.SH EXAMPLES +.PP +Within ldmsd_controller or a configuration file: +.nf +load name=zfs_zpool +config name=zfs_zpool producer=${HOSTNAME} instance=${HOSTNAME}/zfs_zpool +schema=zpools_stats job_set=${HOSTNAME}/zpools_stats +start name=zfs_zpool interval=10000000 offset=15 +.fi + +.SH SEE ALSO +ldmsd(8), ldms_quickstart(7), ldmsd_controller(8), ldms_sampler_base(7), +Plugin_zfs_zpool(7) diff --git a/ldms/src/sampler/zfs_zpool/zfs_zpool.c b/ldms/src/sampler/zfs_zpool/zfs_zpool.c new file mode 100644 index 000000000..a0a7b3b94 --- /dev/null +++ b/ldms/src/sampler/zfs_zpool/zfs_zpool.c @@ -0,0 +1,516 @@ +/* -*- c-basic-offset: 8 -*- */ +/* Copyright 2022 Lawrence Livermore National Security, LLC + * See the top-level COPYING file for details. + * + * SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) + */ + +#define _GNU_SOURCE +/* Next we include the headers to bring in the zfslib in action */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ldms.h" +#include "ldmsd.h" +#include "config.h" +#include "sampler_base.h" +#include "libzfs.h" +#include + +#define SAMP "zfs_zpool" + +#ifndef ARRAY_LEN +#define ARRAY_LEN(a) (sizeof(a) / sizeof(*a)) +#endif + +/* Function prototypes */ +static int get_zpool_stats(zpool_handle_t * zhp, void *data); +static int get_zpool_count(zpool_handle_t * zhp, void *data); +static int get_pool_scan_status(nvlist_t * nvroot, const char *pool_name, + ldms_mval_t * record_instance); + +#define POOL_MEASUREMENT "zpool_stats" +#define SCAN_MEASUREMENT "zpool_scan_stats" +#define VDEV_MEASUREMENT "zpool_vdev_stats" +#define POOL_LATENCY_MEASUREMENT "zpool_latency" +#define POOL_QUEUE_MEASUREMENT "zpool_vdev_queue" +#define MIN_LAT_INDEX 10 /* minimum latency index 10 = 1024ns */ +#define POOL_IO_SIZE_MEASUREMENT "zpool_io_size" +#define MIN_SIZE_INDEX 9 /* minimum size index 9 = 512 bytes */ + +typedef int (*stat_printer_f) (nvlist_t *, const char *, const char *); + +int complained_about_sync = 0; + +static base_data_t sampler_base; +static libzfs_handle_t *g_zfs; + +static struct { + int vdev_list_idx; + int vdev_rec_idx; +} index_store; + +static ldms_mval_t list_handle; + +typedef enum op_func_type { + FUNC_NOFUNCREQ = 0, + FUNC_SCRUB = 1, + FUNC_RESILVER = 2, + FUNC_REBUILD = 3, + FUNC_SCAN = 4 +} op_func_type_; + +static const char *const operation_types[] = { + [FUNC_NOFUNCREQ] = "none", + [FUNC_SCRUB] = "scrub", + [FUNC_RESILVER] = "resilver", + [FUNC_REBUILD] = "rebuild", + [FUNC_SCAN] = "scan" +}; + +/* metric templates for a zpool and scan status if any */ +static struct ldms_metric_template_s zfs_zpool[] = { + {"pool", 0, LDMS_V_CHAR_ARRAY, "", ZFS_MAX_DATASET_NAME_LEN}, + {"state", 0, LDMS_V_CHAR_ARRAY, "", ZFS_MAX_DATASET_NAME_LEN}, + {"total", 0, LDMS_V_U64, "", 1}, + {"allocated", 0, LDMS_V_U64, "", 1}, + {"free", 0, LDMS_V_U64, "", 1}, + {"used", 0, LDMS_V_U64, "", 1}, + {"scan_func", 0, LDMS_V_CHAR_ARRAY, "", ZFS_MAX_DATASET_NAME_LEN}, + {"scan_status", 0, LDMS_V_CHAR_ARRAY, "", ZFS_MAX_DATASET_NAME_LEN}, + {"scan_repaired", 0, LDMS_V_U64, "", 1}, + {"scan_completed_in", 0, LDMS_V_U64, "", 1}, + {"scan_errors", 0, LDMS_V_U32, "", 1}, + {"scan_completed_on", 0, LDMS_V_U64, "", 1}, + {0} +}; + +#define ZPOOL_METRICS_LEN (ARRAY_LEN(zfs_zpool) - 1) +static int zpool_metric_ids[ZPOOL_METRICS_LEN]; +static size_t zpool_heap_sz; + +static ovis_log_t mylog; + +static int zpool_list_len = 0; /* Aggregated number of vdev per zpool */ + +/***************************************************************************** + * Initialize the structure as schema and add them to the base schema. + * Also calculate the size of memory needed per schema and add it to the ldms + * schema list. + ****************************************************************************/ + +static int initialize_ldms_structs() +{ + /*ldms_record_t zpool_def; a pointer */ + ldms_record_t zpool_vdev_def; /* a pointer */ + int rc; + + ovis_log(mylog, OVIS_LDEBUG, SAMP " initialize()\n"); + + /* Create the schema */ + base_schema_new(sampler_base); + if (sampler_base->schema == NULL) + goto err1; + + /* create the vdev record */ + zpool_vdev_def = ldms_record_from_template("zfs_zpool_stats", + zfs_zpool, zpool_metric_ids); + if (zpool_vdev_def == NULL) + goto err2; + + zpool_heap_sz = ldms_record_heap_size_get(zpool_vdev_def); + rc = ldms_schema_record_add(sampler_base->schema, zpool_vdev_def); + if (rc < 0) + goto err3; + + index_store.vdev_rec_idx = rc; + rc = zpool_iter(g_zfs, get_zpool_count, NULL); + /* add error for iter in case here */ + rc = ldms_schema_metric_list_add(sampler_base->schema, + "zpool_list", + NULL, zpool_list_len * zpool_heap_sz); + if (rc < 0) + goto err2; + + index_store.vdev_list_idx = rc; + + /* Create the metric set */ + base_set_new(sampler_base); + if (sampler_base->set == NULL) + goto err2; + + return 0; + + err3: + /* We only manually delete record template when it + * hasn't been added to the schema yet */ + ldms_record_delete(zpool_vdev_def); + err2: + base_schema_delete(sampler_base); + err1: + ovis_log(mylog, OVIS_LERROR, SAMP " initialization failed\n"); + return -1; +} + +/***************************************************************************** + * WHAT: + * 1) Initialize the sampler base schema. + * 2) Initialize all structure and memory. + * 3) initialize the zfslib to sample the zpools stats. + * CALLER: + * ldms daemon. In error the plugin is aborted. + ****************************************************************************/ + +static int config(struct ldmsd_plugin *self, + struct attr_value_list *kwl, struct attr_value_list *avl) +{ + int rc = 0; + + ovis_log(mylog, OVIS_LDEBUG, SAMP " config() called\n"); + + sampler_base = base_config(avl, SAMP, "zfs_zpool", mylog); + if ((g_zfs = libzfs_init()) == NULL) { + rc = errno; + ovis_log(mylog, OVIS_LERROR, + SAMP " : Failed to initialize libzfs: %d\n", errno); + ovis_log(mylog, OVIS_LERROR, + SAMP + " : Is the zfs module loaded or zrepl running?\n"); + } else { + rc = initialize_ldms_structs(); + } + + if (rc < 0) { + base_del(sampler_base); + sampler_base = NULL; + } + + return rc; +} + +/***************************************************************************** + * WHAT: + * reallocate heap size plus 1 zpool struct and one vdev struct + * CALLER: + * self, (plugin) + ****************************************************************************/ +static int resize_metric_set() +{ + size_t previous_heap_size; + size_t new_heap_size; + int rc = 0; + + previous_heap_size = ldms_set_heap_size_get(sampler_base->set); + base_set_delete(sampler_base); + + new_heap_size = previous_heap_size; + new_heap_size += zpool_heap_sz; + + if (base_set_new_heap(sampler_base, new_heap_size) == NULL) { + rc = errno; + ovis_log(mylog, OVIS_LERROR, + SAMP " : Failed to resize metric set heap: %d\n", + errno); + } else { + ovis_log(mylog, OVIS_LDEBUG, "ldms resize of list successful\n"); + } + return rc; +} + +static int sample(struct ldmsd_sampler *self) +{ + int rc = 0; + + base_sample_begin(sampler_base); + + list_handle = + ldms_metric_get(sampler_base->set, index_store.vdev_list_idx); + ldms_list_purge(sampler_base->set, list_handle); + + rc = zpool_iter(g_zfs, get_zpool_stats, NULL); + if (rc != 0) { + ovis_log(mylog, OVIS_LERROR, + SAMP " sample():zfs_pool print_stat() failed: %d\n", rc); + base_sample_end(sampler_base); + goto err1; + } + /* this is where the rubber meets the pavement */ + + base_sample_end(sampler_base); + + err1: + return rc; +} + +static void term(struct ldmsd_plugin *self) +{ + ovis_log(mylog, OVIS_LDEBUG, SAMP " term() called\n"); + base_set_delete(sampler_base); + base_del(sampler_base); + sampler_base = NULL; +} + +static ldms_set_t get_set(struct ldmsd_sampler *self) +{ + return NULL; +} + +static const char *usage(struct ldmsd_plugin *self) +{ + ovis_log(mylog, OVIS_LDEBUG, SAMP " usage() called\n"); + return "config name=" SAMP " " BASE_CONFIG_SYNOPSIS BASE_CONFIG_DESC; +} + +struct ldmsd_plugin *get_plugin() +{ + int rc; + static struct ldmsd_sampler plugin = { + .base = { + .name = SAMP, + .type = LDMSD_PLUGIN_SAMPLER, + .term = term, + .config = config, + .usage = usage, + }, + .get_set = get_set, + .sample = sample, + }; + mylog = ovis_log_register("sampler."SAMP, "Message for the " SAMP " plugin"); + if (!mylog) { + rc = errno; + ovis_log(NULL, OVIS_LWARN, "Failed to create the log subsystem " + "of '" SAMP "' plugin. Error %d\n", rc); + } + + return &plugin.base; +} + +/* + * top-level vdev stats are at the pool level moving to its own plugin + */ + +static int get_detailed_pool_stats(nvlist_t * nvroot, const char *pool_name) +{ + nvlist_t *nv_ex; + uint64_t cap; + ldms_mval_t record_instance; + uint_t c; + vdev_stat_t *vs; + int rc = 0; + + if (nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS, + (uint64_t **) & vs, &c) != 0) { + return (1); + } + + if (nvlist_lookup_nvlist(nvroot, + ZPOOL_CONFIG_VDEV_STATS_EX, &nv_ex) != 0) { + rc = 6; + } + + if (rc == 0) { + record_instance = ldms_record_alloc(sampler_base->set, + index_store.vdev_rec_idx); + + if (record_instance == NULL) { + ovis_log(mylog, OVIS_LDEBUG, + SAMP + ": ldms_record_alloc() failed, resizing metric set\n"); + resize_metric_set(); + record_instance = ldms_record_alloc(sampler_base->set, + index_store.vdev_rec_idx); + if (record_instance == NULL) + rc = 2; + } + } + + if (rc == 0) { + rc = ldms_list_append_record(sampler_base->set, list_handle, + record_instance); + + /* zpoolname 0 */ + ldms_record_array_set_str(record_instance, zpool_metric_ids[0], + pool_name); + /* zpool state 1 */ + ldms_record_array_set_str(record_instance, zpool_metric_ids[1], + zpool_state_to_name((vdev_state_t) + vs->vs_state, + (vdev_aux_t) + vs->vs_aux)); + /* total 2 */ + ldms_record_set_u64(record_instance, zpool_metric_ids[2], + vs->vs_space); + /* allocated 3 */ + ldms_record_set_u64(record_instance, zpool_metric_ids[3], + vs->vs_alloc); + /* free 4 */ + ldms_record_set_u64(record_instance, zpool_metric_ids[4], + vs->vs_space - vs->vs_alloc); + /* used 5 */ + cap = (vs->vs_space == 0) ? 0 : + (vs->vs_alloc * 100 / vs->vs_space); + ldms_record_set_u64(record_instance, zpool_metric_ids[5], cap); + + /* Here we call the get_pool_scan function to fill the rest of the + * record */ + rc = get_pool_scan_status(nvroot, pool_name, &record_instance); + + } + return (0); +} + +static int get_zpool_count(zpool_handle_t * zhp, void *data) +{ + int rc = 0; + + if (zhp != NULL) { + zpool_list_len++; + zpool_close(zhp); + } else { + rc = 1; + } + return (rc); +} + +static int get_pool_scan_status(nvlist_t * nvroot, const char *pool_name, + ldms_mval_t * record_instance) +{ + uint_t c; + int64_t elapsed; + uint64_t pass_exam, paused_time, rate; + pool_scan_stat_t *ps = NULL; + char *state[DSS_NUM_STATES] = { + "NONE", + "scanning", + "finished", + "canceled" + }; + /* operation_types func; */ + const char *func; + + (void)nvlist_lookup_uint64_array(nvroot, + ZPOOL_CONFIG_SCAN_STATS, + (uint64_t **) & ps, &c); + + /* + * ignore if there are no stats + */ + if (ps == NULL) + return (0); + + /* + * return error if state is bogus + */ + if (ps->pss_state >= DSS_NUM_STATES || ps->pss_func >= POOL_SCAN_FUNCS) { + if (complained_about_sync % 1000 == 0) { + fprintf(stderr, "error: cannot decode scan stats: " + "ZFS is out of sync with compiled zfs_zpool (ldms)"); + complained_about_sync++; + } + return (1); + } + + switch (ps->pss_func) { + + case POOL_SCAN_NONE: + func = operation_types[FUNC_NOFUNCREQ]; + break; + case POOL_SCAN_SCRUB: + func = operation_types[FUNC_SCRUB]; + break; + case POOL_SCAN_RESILVER: + func = operation_types[FUNC_RESILVER]; + break; + default: + func = operation_types[FUNC_SCAN]; + } + + paused_time = ps->pss_pass_scrub_spent_paused; + + /* calculations for this pass */ + if (ps->pss_state == DSS_SCANNING) { + elapsed = (int64_t) time(NULL) - (int64_t) ps->pss_pass_start - + (int64_t) paused_time; + elapsed = (elapsed > 0) ? elapsed : 1; + pass_exam = ps->pss_pass_exam ? ps->pss_pass_exam : 1; + rate = pass_exam / elapsed; + rate = (rate > 0) ? rate : 1; + } else { + elapsed = + (int64_t) ps->pss_end_time - (int64_t) ps->pss_pass_start - + (int64_t) paused_time; + elapsed = (elapsed > 0) ? elapsed : 1; + pass_exam = ps->pss_pass_exam ? ps->pss_pass_exam : 1; + rate = pass_exam / elapsed; + } + rate = rate ? rate : 1; + + /* scan_func 6 */ + ldms_record_array_set_str(*record_instance, zpool_metric_ids[6], func); + /* scan_status 7 */ + ldms_record_array_set_str(*record_instance, zpool_metric_ids[7], + state[ps->pss_state]); + /* scan_repaired 8 */ + ldms_record_set_u64(*record_instance, zpool_metric_ids[8], + ps->pss_processed); + /* scan_completed_in 9 */ + ldms_record_set_u64(*record_instance, zpool_metric_ids[9], + ps->pss_end_time - ps->pss_start_time); + /* scan_errors 10 */ + ldms_record_set_u64(*record_instance, zpool_metric_ids[10], + ps->pss_errors); + /* scan_completed_on 11 */ + ldms_record_set_u64(*record_instance, zpool_metric_ids[11], + ps->pss_end_time); + + return (0); +} + +/* + * call-back to print the stats from the pool config + * + * Note: if the pool is broken, this can hang indefinitely and perhaps in an + * unkillable state. + */ + +static int get_zpool_stats(zpool_handle_t * zhp, void *data) +{ + uint_t c; + int err = 0; + nvlist_t *config, *nvroot; + vdev_stat_t *vs; + char *pool_name; + + if ((config = zpool_get_config(zhp, NULL)) != NULL) { + if (nvlist_lookup_nvlist + (config, ZPOOL_CONFIG_VDEV_TREE, &nvroot) != 0) { + err = 2; + goto terminate; + } + if (nvlist_lookup_uint64_array(nvroot, ZPOOL_CONFIG_VDEV_STATS, + (uint64_t **) & vs, &c) != 0) { + err = 3; + goto terminate; + } + + pool_name = (char *)zpool_get_name(zhp); + + err = get_detailed_pool_stats(nvroot, pool_name); + } + + terminate: + zpool_close(zhp); + return (err); +}