Add support for complex counter values

This allows having a single map value to support multiple counters:

    ivan@vm:~$ curl -s http://localhost:9435/metrics | fgrep block_rq
    # HELP ebpf_exporter_block_rq_completed_bytes_total Total number of bytes served by block requests completions
    # TYPE ebpf_exporter_block_rq_completed_bytes_total counter
    ebpf_exporter_block_rq_completed_bytes_total{device="nvme0n1"} 258048
    ebpf_exporter_block_rq_completed_bytes_total{device="nvme1n1"} 966656
    # HELP ebpf_exporter_block_rq_completions_total Total number of block request completions
    # TYPE ebpf_exporter_block_rq_completions_total counter
    ebpf_exporter_block_rq_completions_total{device="nvme0n1"} 35
    ebpf_exporter_block_rq_completions_total{device="nvme1n1"} 103
    ebpf_exporter_ebpf_program_info{config="complex-value",id="72",program="block_rq_complete",tag="34d2100b313409cd"} 1

Author: bobrik

.vscode/config-schema.yaml

@@ -36,6 +36,19 @@ properties:
               type: string
             labels:
               $ref: "#/definitions/labels"
+            values:
+              type: array
+              items:
+                type: object
+                additionalProperties: false
+                required:
+                  - name
+                  - help
+                properties:
+                  name:
+                    type: string
+                  help:
+                    type: string
       histograms:
         type: array
         items:

README.md

@@ -759,10 +759,13 @@ perf_event_array: <whether map is a BPF_MAP_TYPE_PERF_EVENT_ARRAY map: bool>
 flush_interval: <how often should we flush metrics from the perf_event_array: time.Duration>
 labels:
   [ - label ]
+[ values: { name: "...", help: "..."}]
 ```
 
 An example of `perf_map` can be found [here](examples/oomkill.yaml).
 
+An example of `values` can be found [here](examples/complex-value.yaml).
+
 #### `histogram`
 
 See [Histograms](#histograms) section for more details.

config/config.go

@@ -31,6 +31,7 @@ type Counter struct {
 	PerfEventArray bool          `yaml:"perf_event_array"`
 	FlushInterval  time.Duration `yaml:"flush_interval"`
 	Labels         []Label       `yaml:"labels"`
+	Values         []Value       `yaml:"values"`
 }
 
 // Histogram is a metric defining prometheus histogram
@@ -75,6 +76,12 @@ type Decoder struct {
 	AllowUnknown bool              `yaml:"allow_unknown"`
 }
 
+// Value describes a metric in when it's split across multiple u64
+type Value struct {
+	Name string `yaml:"name"`
+	Help string `yaml:"help"`
+}
+
 // HistogramBucketType is an enum to define how to interpret histogram
 type HistogramBucketType string
 

examples/complex-value.bpf.c

@@ -0,0 +1,83 @@
+#include <vmlinux.h>
+#include <bpf/bpf_core_read.h>
+#include <bpf/bpf_tracing.h>
+
+#define MKDEV(ma, mi) ((mi & 0xff) | (ma << 8) | ((mi & ~0xff) << 12))
+
+/**
+ * commit d152c682f03c ("block: add an explicit ->disk backpointer to the
+ * request_queue") and commit f3fa33acca9f ("block: remove the ->rq_disk
+ * field in struct request") make some changes to `struct request` and
+ * `struct request_queue`. Now, to get the `struct gendisk *` field in a CO-RE
+ * way, we need both `struct request` and `struct request_queue`.
+ * see:
+ *     https://github.com/torvalds/linux/commit/d152c682f03c
+ *     https://github.com/torvalds/linux/commit/f3fa33acca9f
+ */
+struct request_queue___x {
+    struct gendisk *disk;
+} __attribute__((preserve_access_index));
+
+struct request___x {
+    struct request_queue___x *q;
+    struct gendisk *rq_disk;
+} __attribute__((preserve_access_index));
+
+struct key_t {
+    u32 dev;
+};
+
+struct value_t {
+    u64 count;
+    u64 bytes;
+};
+
+struct {
+    __uint(type, BPF_MAP_TYPE_HASH);
+    __uint(max_entries, 1024);
+    __type(key, struct key_t);
+    __type(value, struct value_t);
+} block_rq_completions SEC(".maps");
+
+static __always_inline struct gendisk *get_disk(void *request)
+{
+    struct request___x *r = request;
+
+    if (bpf_core_field_exists(r->rq_disk))
+        return r->rq_disk;
+    return r->q->disk;
+}
+
+static struct value_t *get_value(void *map, struct key_t *key)
+{
+    struct value_t *value = bpf_map_lookup_elem(map, key);
+    if (!value) {
+        struct value_t zero = { .count = 0, .bytes = 0 };
+        bpf_map_update_elem(map, key, &zero, BPF_NOEXIST);
+        value = bpf_map_lookup_elem(map, key);
+        if (!value) {
+            return NULL;
+        }
+    }
+
+    return value;
+}
+
+SEC("tp_btf/block_rq_complete")
+int BPF_PROG(block_rq_complete, struct request *rq, blk_status_t error, unsigned int nr_bytes)
+{
+    struct gendisk *disk = get_disk(rq);
+    struct key_t key = { .dev = disk ? MKDEV(disk->major, disk->first_minor) : 0 };
+    struct value_t *value = get_value(&block_rq_completions, &key);
+
+    if (!value) {
+        return 0;
+    }
+
+    __sync_fetch_and_add(&value->count, 1);
+    __sync_fetch_and_add(&value->bytes, nr_bytes);
+
+    return 0;
+}
+
+char LICENSE[] SEC("license") = "GPL";

examples/complex-value.yaml

@@ -0,0 +1,14 @@
+metrics:
+  counters:
+    - name: block_rq_completions
+      help: Block request completions split into count and bytes
+      labels:
+        - name: device
+          size: 4
+          decoders:
+            - name: majorminor
+      values:
+        - name: block_rq_completions_total
+          help: Total number of block request completions
+        - name: block_rq_completed_bytes_total
+          help: Total number of bytes served by block requests completions

exporter/exporter.go

@@ -392,7 +392,13 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
 				e.perfEventArrayCollectors = append(e.perfEventArrayCollectors, perfSink)
 			}
 
-			addDescs(cfg.Name, counter.Name, counter.Help, counter.Labels)
+			if counter.Values != nil {
+				for _, value := range counter.Values {
+					addDescs(cfg.Name, value.Name, value.Help, counter.Labels)
+				}
+			} else {
+				addDescs(cfg.Name, counter.Name, counter.Help, counter.Labels)
+			}
 		}
 
 		for _, histogram := range cfg.Metrics.Histograms {
@@ -477,10 +483,14 @@ func (e *Exporter) collectCounters(ch chan<- prometheus.Metric) {
 
 			aggregatedMapValues := aggregateMapValues(mapValues)
 
-			desc := e.descs[cfg.Name][counter.Name]
-
 			for _, metricValue := range aggregatedMapValues {
-				ch <- prometheus.MustNewConstMetric(desc, prometheus.CounterValue, metricValue.value, metricValue.labels...)
+				if counter.Values != nil {
+					for i, value := range counter.Values {
+						ch <- prometheus.MustNewConstMetric(e.descs[cfg.Name][value.Name], prometheus.CounterValue, metricValue.value[i], metricValue.labels...)
+					}
+				} else {
+					ch <- prometheus.MustNewConstMetric(e.descs[cfg.Name][counter.Name], prometheus.CounterValue, metricValue.value[0], metricValue.labels...)
+				}
 			}
 		}
 	}
@@ -531,7 +541,7 @@ func (e *Exporter) collectHistograms(ch chan<- prometheus.Metric) {
 					break
 				}
 
-				histograms[key].buckets[float64(leUint)] = uint64(metricValue.value)
+				histograms[key].buckets[float64(leUint)] = uint64(metricValue.value[0])
 			}
 
 			if skip {
@@ -673,29 +683,33 @@ func (e *Exporter) MapsHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func validateMaps(module *libbpfgo.Module, cfg config.Config) error {
-	maps := []string{}
+	sizes := map[string]int{}
 
 	for _, counter := range cfg.Metrics.Counters {
 		if counter.Name != "" && !counter.PerfEventArray {
-			maps = append(maps, counter.Name)
+			if counter.Values != nil {
+				sizes[counter.Name] = len(counter.Values) * 8
+			} else {
+				sizes[counter.Name] = 8
+			}
 		}
 	}
 
 	for _, histogram := range cfg.Metrics.Histograms {
 		if histogram.Name != "" {
-			maps = append(maps, histogram.Name)
+			sizes[histogram.Name] = 8
 		}
 	}
 
-	for _, name := range maps {
+	for name, expected := range sizes {
 		m, err := module.GetMap(name)
 		if err != nil {
 			return fmt.Errorf("failed to get map %q: %w", name, err)
 		}
 
 		valueSize := m.ValueSize()
-		if valueSize != 8 {
-			return fmt.Errorf("value size for map %q is not expected 8 bytes (u64), it is %d bytes", name, valueSize)
+		if valueSize != expected {
+			return fmt.Errorf("value size for map %q is not expected %d bytes (8 bytes per u64 value), it is %d bytes", name, expected, valueSize)
 		}
 	}
 
@@ -721,7 +735,9 @@ func aggregateMapValues(values []metricValue) []aggregatedMetricValue {
 				value:  value.value,
 			}
 		} else {
-			existing.value += value.value
+			for i := range existing.value {
+				existing.value[i] += value.value[i]
+			}
 		}
 	}
 
@@ -785,17 +801,18 @@ func readMapValues(m *libbpfgo.BPFMap, labels []config.Label) ([]metricValue, er
 	return metricValues, nil
 }
 
-func mapValue(m *libbpfgo.BPFMap, key []byte) (float64, error) {
+func mapValue(m *libbpfgo.BPFMap, key []byte) ([]float64, error) {
 	v, err := m.GetValue(unsafe.Pointer(&key[0]))
 	if err != nil {
-		return 0.0, err
+		return []float64{0.0}, err
 	}
 
 	return decodeValue(v), nil
 }
 
-func mapValuePerCPU(m *libbpfgo.BPFMap, key []byte) ([]float64, error) {
-	values := []float64{}
+func mapValuePerCPU(m *libbpfgo.BPFMap, key []byte) ([][]float64, error) {
+	values := [][]float64{}
+
 	size := m.ValueSize()
 
 	value, err := m.GetValue(unsafe.Pointer(&key[0]))
@@ -811,8 +828,14 @@ func mapValuePerCPU(m *libbpfgo.BPFMap, key []byte) ([]float64, error) {
 }
 
 // Assuming counter's value type is always u64
-func decodeValue(value []byte) float64 {
-	return float64(util.GetHostByteOrder().Uint64(value))
+func decodeValue(value []byte) []float64 {
+	values := make([]float64, len(value)/8)
+
+	for i := range values {
+		values[i] = float64(util.GetHostByteOrder().Uint64(value[i*8:]))
+	}
+
+	return values
 }
 
 // metricValue is a row in a kernel map
@@ -822,13 +845,13 @@ type metricValue struct {
 	// labels are decoded from the raw key
 	labels []string
 	// value is the kernel map value
-	value float64
+	value []float64
 }
 
 // aggregatedMetricValue is a value after aggregation of equal label sets
 type aggregatedMetricValue struct {
 	// labels are decoded from the raw key
 	labels []string
 	// value is the kernel map value
-	value float64
+	value []float64
 }

exporter/exporter_test.go

@@ -10,32 +10,32 @@ func TestAggregatedMetricValues(t *testing.T) {
 	values := []metricValue{
 		{
 			labels: []string{"foo"},
-			value:  8,
+			value:  []float64{8},
 		},
 		{
 			labels: []string{"bar"},
-			value:  1,
+			value:  []float64{1},
 		},
 		{
 			labels: []string{"foo"},
-			value:  3,
+			value:  []float64{3},
 		},
 	}
 
 	aggregated := aggregateMapValues(values)
 
 	sort.Slice(aggregated, func(i, j int) bool {
-		return aggregated[i].value > aggregated[j].value
+		return aggregated[i].value[0] > aggregated[j].value[0]
 	})
 
 	expected := []aggregatedMetricValue{
 		{
 			labels: []string{"foo"},
-			value:  11,
+			value:  []float64{11},
 		},
 		{
 			labels: []string{"bar"},
-			value:  1,
+			value:  []float64{1},
 		},
 	}