netstacklat: Add option to group by interface

Add the -I/--groupby-interface option to collect and report the data
on a per-interface (or rather ifindex) basis. Note that the network
interfaces are tracked based on their ifindex, so if network namespace
filtering has been disabled and there exists interfaces in different
namespaces with a common ifindex, their data will be merged into the
same histogram.

Always write the interface index rather than the interface name in the
output. While the interface name for the same network namespace as the
user space agent runs in can easily be retrieved with
e.g. if_indextoname(), that will only be valid if the user has
configured netstacklat to only monitor its own network namespace. If a
different network namespace is monitored, or filtration for network
namespaces is disabled, translating to the interface names in the
current namespace might produce misleading results.

An alternative could be to print out the interface names in case the
current network namespace is the one monitored (the default), or the
index if there's a risk that the data might be from a different
namespace. However, in addition to that added complexity, that will
produce somewhat inconsistent output (i.e. you might get interface
names or interface indices depending on how you configure
netstacklat).

Signed-off-by: Simon Sundberg <simon.sundberg@kau.se>

Author: simosund

netstacklat/netstacklat.bpf.c

@@ -19,6 +19,7 @@ volatile const struct netstacklat_bpf_config user_config = {
 	.filter_ifindex = false,
 	.filter_cgroup = false,
 	.filter_nonempty_sockqueue = false,
+	.groupby_ifindex = false,
 };
 
 /*
@@ -36,7 +37,7 @@ struct sk_buff___old {
 
 struct {
 	__uint(type, BPF_MAP_TYPE_PERCPU_HASH);
-	__uint(max_entries, HIST_NBUCKETS * NETSTACKLAT_N_HOOKS);
+	__uint(max_entries, HIST_NBUCKETS * NETSTACKLAT_N_HOOKS * 16);
 	__type(key, struct hist_key);
 	__type(value, u64);
 } netstack_latency_seconds SEC(".maps");
@@ -135,18 +136,17 @@ static ktime_t time_since(ktime_t tstamp)
 	return now - tstamp;
 }
 
-static void record_latency(ktime_t latency, enum netstacklat_hook hook)
+static void record_latency(ktime_t latency, const struct hist_key *key)
 {
-	struct hist_key key = { .hook = hook };
-	increment_exp2_histogram_nosync(&netstack_latency_seconds, key, latency,
+	increment_exp2_histogram_nosync(&netstack_latency_seconds, *key, latency,
 					HIST_MAX_LATENCY_SLOT);
 }
 
-static void record_latency_since(ktime_t tstamp, enum netstacklat_hook hook)
+static void record_latency_since(ktime_t tstamp, const struct hist_key *key)
 {
 	ktime_t latency = time_since(tstamp);
 	if (latency >= 0)
-		record_latency(latency, hook);
+		record_latency(latency, key);
 }
 
 static bool filter_ifindex(u32 ifindex)
@@ -187,6 +187,9 @@ static __u64 get_network_ns(struct sk_buff *skb, struct sock *sk)
 
 static void record_skb_latency(struct sk_buff *skb, struct sock *sk, enum netstacklat_hook hook)
 {
+	struct hist_key key = { .hook = hook };
+	u32 ifindex;
+
 	if (bpf_core_field_exists(skb->tstamp_type)) {
 		/*
 		 * For kernels >= v6.11 the tstamp_type being non-zero
@@ -210,13 +213,17 @@ static void record_skb_latency(struct sk_buff *skb, struct sock *sk, enum netsta
 			return;
 	}
 
-	if (!filter_ifindex(skb->skb_iif))
+	ifindex = skb->skb_iif;
+	if (!filter_ifindex(ifindex))
 		return;
 
 	if (!filter_network_ns(get_network_ns(skb, sk)))
 		return;
 
-	record_latency_since(skb->tstamp, hook);
+	if (user_config.groupby_ifindex)
+		key.ifindex = ifindex;
+
+	record_latency_since(skb->tstamp, &key);
 }
 
 static bool filter_pid(u32 pid)
@@ -286,6 +293,7 @@ static bool filter_nonempty_sockqueue(struct sock *sk)
 static void record_socket_latency(struct sock *sk, struct sk_buff *skb,
 				  ktime_t tstamp, enum netstacklat_hook hook)
 {
+	struct hist_key key = { .hook = hook };
 	u32 ifindex;
 
 	if (!filter_nonempty_sockqueue(sk))
@@ -301,7 +309,10 @@ static void record_socket_latency(struct sock *sk, struct sk_buff *skb,
 	if (!filter_network_ns(get_network_ns(skb, sk)))
 		return;
 
-	record_latency_since(tstamp, hook);
+	if (user_config.groupby_ifindex)
+		key.ifindex = ifindex;
+
+	record_latency_since(tstamp, &key);
 }
 
 SEC("fentry/ip_rcv_core")

netstacklat/netstacklat.c

@@ -97,6 +97,7 @@ static const struct option long_options[] = {
 	{ "network-namespace", required_argument, NULL, 'n' },
 	{ "cgroups",           required_argument, NULL, 'c' },
 	{ "nonempty-queue",    no_argument,       NULL, 'q' },
+	{ "groupby-interface", no_argument,       NULL, 'I' },
 	{ 0, 0, 0, 0 }
 };
 
@@ -562,6 +563,7 @@ static int parse_arguments(int argc, char *argv[],
 	conf->bpf_conf.filter_pid = false;
 	conf->bpf_conf.filter_ifindex = false;
 	conf->bpf_conf.filter_nonempty_sockqueue = false;
+	conf->bpf_conf.groupby_ifindex = false;
 
 	conf->pids = calloc(MAX_PARSED_PIDS, sizeof(*conf->pids));
 	conf->ifindices = calloc(MAX_PARSED_IFACES, sizeof(*conf->ifindices));
@@ -649,9 +651,13 @@ static int parse_arguments(int argc, char *argv[],
 			conf->ncgroups += ret;
 			conf->bpf_conf.filter_cgroup = true;
 			break;
+
 		case 'q': // nonempty-queue
 			conf->bpf_conf.filter_nonempty_sockqueue = true;
 			break;
+		case 'I': // groupby-interface
+			conf->bpf_conf.groupby_ifindex = true;
+			break;
 		case 'h': // help
 			print_usage(stdout, argv[0]);
 			exit(EXIT_SUCCESS);
@@ -822,13 +828,22 @@ static void print_log2hist(FILE *stream, size_t n, const __u64 hist[n],
 static void print_histkey(FILE *stream, const struct hist_key *key)
 {
 	fprintf(stream, "%s", hook_to_str(key->hook));
+
+	if (key->ifindex)
+		fprintf(stream, ", interface=%u", key->ifindex);
 }
 
 static int cmp_histkey(const void *val1, const void *val2)
 {
 	const struct hist_key *key1 = val1, *key2 = val2;
 
-	return key1->hook == key2->hook ? 0 : key1->hook > key2->hook ? 1 : -1;
+	if (key1->hook != key2->hook)
+		return key1->hook > key2->hook ? 1 : -1;
+
+	if (key1->ifindex != key2->ifindex)
+		return key1->ifindex > key2->ifindex ? 1 : -1;
+
+	return 0;
 }
 
 static int cmp_histentry(const void *val1, const void *val2)
@@ -1022,6 +1037,11 @@ static int init_histogram_buffer(struct histogram_buffer *buf,
 			max_hists++;
 	}
 
+	if (conf->bpf_conf.groupby_ifindex)
+		max_hists *= conf->bpf_conf.filter_ifindex ?
+				     min(conf->nifindices, 64) :
+				     32;
+
 	buf->hists = calloc(max_hists, sizeof(*buf->hists));
 	if (!buf->hists)
 		return -errno;

netstacklat/netstacklat.h

@@ -33,6 +33,15 @@
 	})
 #endif
 
+#ifndef min
+#define min(a, b)                   \
+	({                          \
+		typeof(a) _a = (a); \
+		typeof(b) _b = (b); \
+		_a < _b ? _a : _b;  \
+	})
+#endif
+
 enum netstacklat_hook {
 	NETSTACKLAT_HOOK_INVALID = 0,
 	NETSTACKLAT_HOOK_IP_RCV,
@@ -51,6 +60,7 @@ enum netstacklat_hook {
  * member is named "bucket" and is the histogram bucket index.
  */
 struct hist_key {
+	__u32 ifindex;
 	__u16 hook; // need well defined size for ebpf-exporter to decode
 	__u16 bucket; // needs to be last to be compatible with ebpf-exporter
 };
@@ -61,6 +71,7 @@ struct netstacklat_bpf_config {
 	bool filter_ifindex;
 	bool filter_cgroup;
 	bool filter_nonempty_sockqueue;
+	bool groupby_ifindex;
 };
 
 #endif

netstacklat/netstacklat.yaml

@@ -7,6 +7,13 @@ metrics:
       bucket_max: 34
       bucket_multiplier: 0.000000001 # nanoseconds to seconds
       labels:
+        - name: iface
+          size: 4
+          decoders:
+            # If including output from a different network namespace than ebpf-exporter
+            # you probably just want to decode as a uint (ifindex) instead
+            # - name: uint # For the ifname decoder you apparently don't first need a uint decoder like the others
+            - name: ifname
         - name: hook
           size: 2
           decoders: