benchmarking: expand cAdvisor scrape allowlist for kernel PSI, networ…

benchmarking/monitoring.yaml

@@ -155,18 +155,29 @@ data:
             target_label: __metrics_path__
             replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
         metric_relabel_configs:
+          # cAdvisor reports host-level root cgroup metrics (such as Linux kernel
+          # PSI pressure stall information) with id="/" and an empty container
+          # label. Relabel id="/" to container="node" before dropping empty
+          # container labels so host-level pressure and utilization are kept.
+          - source_labels: [id]
+            regex: ^/$
+            target_label: container
+            replacement: node
           # cAdvisor sends a very large number of metrics. Keep only the
-          # metrics that the benchmarks read.
+          # metrics that the benchmarks read: kernel pressure stall (PSI),
+          # container network bandwidth, disk I/O, memory RSS/working-set,
+          # CPU utilization, and CFS throttling.
           - source_labels: [__name__]
             action: keep
-            regex: container_(memory_working_set_bytes|cpu_usage_seconds_total|spec_memory_limit_bytes)
-          # Drop the series that have an empty container label, not the label
-          # itself. cAdvisor gives a rollup for the whole pod cgroup next to
-          # the series for each container. Keeping both counts the memory of
-          # each pod two times.
-          - source_labels: [container]
+            regex: container_((.*pressure.*)|network_(receive|transmit)_bytes_total|fs_(reads|writes)_bytes_total|memory_rss|memory_working_set_bytes|cpu_usage_seconds_total|cpu_cfs_.*|spec_memory_limit_bytes)
+          # Drop the series that have an empty container label only for cumulative
+          # memory working set and CPU usage. cAdvisor gives a rollup for the whole
+          # pod cgroup next to the series for each container; keeping both counts
+          # the memory/CPU of each pod two times. Dropping empty container on other
+          # metrics (such as network or PSI) would drop valid pod/node stats.
+          - source_labels: [__name__, container]
             action: drop
-            regex: ""
+            regex: container_(memory_working_set_bytes|cpu_usage_seconds_total);$
 ---
 apiVersion: apps/v1
 kind: Deployment