in_podman_metrics: add per-container disk I/O metrics

Add four counters exposing per-container block I/O, read from the cgroups
v2 io.stat file and summed across block devices:

- container_disk_read_bytes_total
- container_disk_write_bytes_total
- container_disk_reads_total
- container_disk_writes_total

This complements the existing CPU, memory and network metrics. The values
are collected in the cgroups v2 path; on cgroups v1 hosts the counters are
reported as invalid and skipped.

Signed-off-by: Stefano Tondo <stondo@gmail.com>

Author: stondo

plugins/in_podman_metrics/podman_metrics.c

@@ -158,6 +158,10 @@ static int add_container_to_list(struct flb_in_metrics *ctx, flb_sds_t id, flb_s
     cnt->rss = UINT64_MAX;
     cnt->cpu_user = UINT64_MAX;
     cnt->cpu = UINT64_MAX;
+    cnt->disk_read_bytes = UINT64_MAX;
+    cnt->disk_write_bytes = UINT64_MAX;
+    cnt->disk_reads = UINT64_MAX;
+    cnt->disk_writes = UINT64_MAX;
 
     mk_list_init(&cnt->net_data);
 
@@ -303,6 +307,10 @@ static int create_gauge(struct flb_in_metrics *ctx, struct cmt_gauge **gauge, fl
  * - container_network_receive_errors_total
  * - container_network_transmit_bytes_total
  * - container_network_transmit_errors_total
+ * - container_disk_read_bytes_total
+ * - container_disk_write_bytes_total
+ * - container_disk_reads_total
+ * - container_disk_writes_total
  */
 static int create_counters(struct flb_in_metrics *ctx)
 {
@@ -328,6 +336,14 @@ static int create_counters(struct flb_in_metrics *ctx)
                        DESCRIPTION_CPU_USER, NULL, cnt->cpu_user);
         create_counter(ctx, &ctx->c_cpu, cnt->id, cnt->name, cnt->image_name, COUNTER_CPU_PREFIX, FIELDS_METRIC, COUNTER_CPU,
                        DESCRIPTION_CPU, NULL, cnt->cpu);
+        create_counter(ctx, &ctx->c_disk_read_bytes, cnt->id, cnt->name, cnt->image_name, COUNTER_DISK_PREFIX, FIELDS_METRIC, COUNTER_DISK_READ_BYTES,
+                       DESCRIPTION_DISK_READ_BYTES, NULL, cnt->disk_read_bytes);
+        create_counter(ctx, &ctx->c_disk_write_bytes, cnt->id, cnt->name, cnt->image_name, COUNTER_DISK_PREFIX, FIELDS_METRIC, COUNTER_DISK_WRITE_BYTES,
+                       DESCRIPTION_DISK_WRITE_BYTES, NULL, cnt->disk_write_bytes);
+        create_counter(ctx, &ctx->c_disk_reads, cnt->id, cnt->name, cnt->image_name, COUNTER_DISK_PREFIX, FIELDS_METRIC, COUNTER_DISK_READS,
+                       DESCRIPTION_DISK_READS, NULL, cnt->disk_reads);
+        create_counter(ctx, &ctx->c_disk_writes, cnt->id, cnt->name, cnt->image_name, COUNTER_DISK_PREFIX, FIELDS_METRIC, COUNTER_DISK_WRITES,
+                       DESCRIPTION_DISK_WRITES, NULL, cnt->disk_writes);
         mk_list_foreach_safe(inner_head, inner_tmp, &cnt->net_data)
         {
             iface = mk_list_entry(inner_head, struct net_iface, _head);
@@ -423,6 +439,10 @@ static int in_metrics_init(struct flb_input_instance *in, struct flb_config *con
     ctx->c_memory_limit = NULL;
     ctx->c_cpu_user = NULL;
     ctx->c_cpu = NULL;
+    ctx->c_disk_read_bytes = NULL;
+    ctx->c_disk_write_bytes = NULL;
+    ctx->c_disk_reads = NULL;
+    ctx->c_disk_writes = NULL;
     ctx->rx_bytes = NULL;
     ctx->rx_errors = NULL;
     ctx->tx_bytes = NULL;

plugins/in_podman_metrics/podman_metrics_config.h

@@ -86,6 +86,12 @@
 #define STAT_KEY_CPU                    "usage_usec"
 #define STAT_KEY_CPU_USER               "user_usec"
 
+/* Field tokens in cgroups v2 io.stat (per block device, summed across devices) */
+#define IO_STAT_KEY_READ_BYTES          "rbytes="
+#define IO_STAT_KEY_WRITE_BYTES         "wbytes="
+#define IO_STAT_KEY_READS               "rios="
+#define IO_STAT_KEY_WRITES              "wios="
+
 /* Static lists of fields in counters or gauges */
 #define FIELDS_METRIC                  (char*[3]){"id", "name", "image" }
 #define FIELDS_METRIC_WITH_IFACE       (char*[4]){"id", "name", "image", "interface" }
@@ -107,6 +113,7 @@
 #define V2_SYSFS_FILE_CPU_STAT         "cpu.stat"
 #define V2_SYSFS_FILE_PIDS             "cgroup.procs"
 #define V2_SYSFS_FILE_PIDS_ALT         "containers/cgroup.procs"
+#define V2_SYSFS_FILE_IO_STAT          "io.stat"
 
 /* Values used to construct counters/gauges names and descriptions */
 #define COUNTER_PREFIX                  "container"
@@ -138,6 +145,16 @@
 #define COUNTER_TX_ERRORS               "transmit_errors_total"
 #define DESCRIPTION_TX_ERRORS           "Network transmitedd errors"
 
+#define COUNTER_DISK_PREFIX             "disk"
+#define COUNTER_DISK_READ_BYTES         "read_bytes_total"
+#define DESCRIPTION_DISK_READ_BYTES     "Container block I/O bytes read"
+#define COUNTER_DISK_WRITE_BYTES        "write_bytes_total"
+#define DESCRIPTION_DISK_WRITE_BYTES    "Container block I/O bytes written"
+#define COUNTER_DISK_READS              "reads_total"
+#define DESCRIPTION_DISK_READS          "Container block I/O reads completed"
+#define COUNTER_DISK_WRITES             "writes_total"
+#define DESCRIPTION_DISK_WRITES         "Container block I/O writes completed"
+
 
 struct net_iface {
     flb_sds_t       name;
@@ -160,6 +177,10 @@ struct container {
     uint64_t        cpu;
     uint64_t        cpu_user;
     uint64_t        rss;
+    uint64_t        disk_read_bytes;
+    uint64_t        disk_write_bytes;
+    uint64_t        disk_reads;
+    uint64_t        disk_writes;
 
     struct mk_list  net_data;
 };
@@ -192,6 +213,10 @@ struct flb_in_metrics {
     struct cmt_counter *rx_errors;
     struct cmt_counter *tx_bytes;
     struct cmt_counter *tx_errors;
+    struct cmt_counter *c_disk_read_bytes;
+    struct cmt_counter *c_disk_write_bytes;
+    struct cmt_counter *c_disk_reads;
+    struct cmt_counter *c_disk_writes;
 
     /* cgroup version used by host */
     int cgroup_version;

plugins/in_podman_metrics/podman_metrics_data.c

@@ -132,6 +132,50 @@ uint64_t get_data_from_sysfs(struct flb_in_metrics *ctx, flb_sds_t dir, flb_sds_
     return data;
 }
 
+/*
+ * Sum a cgroups v2 io.stat field across all block devices. io.stat lines look
+ * like: "8:0 rbytes=1024 wbytes=0 rios=2 wios=0 dbytes=0 dios=0". The key is the
+ * field token including '=', e.g. "rbytes=". Returns the summed value, or
+ * UINT64_MAX if the file cannot be opened (interpreted as an invalid counter).
+ * An existing but empty io.stat (no I/O yet) correctly returns 0.
+ */
+uint64_t sum_io_stat_field(struct flb_in_metrics *ctx, flb_sds_t dir, flb_sds_t name, flb_sds_t key)
+{
+    char path[SYSFS_FILE_PATH_SIZE];
+    uint64_t total = 0;
+    FILE *fp;
+    char *line = NULL;
+    char *pos;
+    size_t len = 0;
+    size_t key_len;
+
+    if (dir == NULL) {
+        return UINT64_MAX;
+    }
+
+    snprintf(path, sizeof(path), "%s/%s", dir, name);
+
+    fp = fopen(path, "r");
+    if (!fp) {
+        flb_plg_warn(ctx->ins, "Failed to read %s", path);
+        return UINT64_MAX;
+    }
+
+    key_len = strlen(key);
+    while (getline(&line, &len, fp) != -1) {
+        pos = line;
+        while ((pos = strstr(pos, key)) != NULL) {
+            pos += key_len;
+            total += strtoull(pos, NULL, 10);
+        }
+    }
+    flb_free(line);
+    fclose(fp);
+
+    flb_plg_debug(ctx->ins, "%s [%s]: %lu", path, key, total);
+    return total;
+}
+
 /*
  * Check if container sysfs data is pressent in previously generated list of sysfs directories.
  * For cgroups v1, use subsystem (directory, for example memory) to search full path.
@@ -367,6 +411,10 @@ int fill_counters_with_sysfs_data_v2(struct flb_in_metrics *ctx)
         cnt->memory_limit = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_MEMORY_LIMIT, NULL);
         cnt->cpu_user = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_CPU_STAT, STAT_KEY_CPU_USER);
         cnt->cpu = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_CPU_STAT, STAT_KEY_CPU);
+        cnt->disk_read_bytes = sum_io_stat_field(ctx, path, V2_SYSFS_FILE_IO_STAT, IO_STAT_KEY_READ_BYTES);
+        cnt->disk_write_bytes = sum_io_stat_field(ctx, path, V2_SYSFS_FILE_IO_STAT, IO_STAT_KEY_WRITE_BYTES);
+        cnt->disk_reads = sum_io_stat_field(ctx, path, V2_SYSFS_FILE_IO_STAT, IO_STAT_KEY_READS);
+        cnt->disk_writes = sum_io_stat_field(ctx, path, V2_SYSFS_FILE_IO_STAT, IO_STAT_KEY_WRITES);
         pid = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_PIDS, NULL);
         if (!pid || pid == UINT64_MAX) {
             pid = get_data_from_sysfs(ctx, path, V2_SYSFS_FILE_PIDS_ALT, NULL);

plugins/in_podman_metrics/podman_metrics_data.h

@@ -37,6 +37,7 @@ int destroy_gauge(struct flb_in_metrics *ctx, struct cmt_gauge **g);
 uint64_t read_from_file(struct flb_in_metrics *ctx, flb_sds_t path);
 uint64_t read_key_value_from_file(struct flb_in_metrics *ctx, flb_sds_t path, flb_sds_t key);
 uint64_t get_data_from_sysfs(struct flb_in_metrics *ctx, flb_sds_t dir, flb_sds_t name, flb_sds_t key);
+uint64_t sum_io_stat_field(struct flb_in_metrics *ctx, flb_sds_t dir, flb_sds_t name, flb_sds_t key);
 
 int get_container_sysfs_subdirectory(struct flb_in_metrics *ctx, flb_sds_t id, flb_sds_t subsystem, flb_sds_t *path);
 int get_net_data_from_proc(struct flb_in_metrics *ctx, struct container *cnt, uint64_t pid);