netstacklat: Add option to groupby cgroup

Add the -C/--groupby-cgroup option to collect and report data on a
per-cgroup basis. Just like the -c/--cgroups option, this will only
apply to probes in the process context, which is currently only
tcp-socket-read and udp-socket-read.

When reporting the data, print out the cgroup ID (inode number)
directly instead of the cgroup path. As far as I can tell, the only
way to resolve the ID into a path is the walk the entire cgroup
mount (e.g. /sys/fs/cgroup) and stat each path to find the matching
inode. Doing this every time the cgroup needs to be printed seems
highly inefficient, and to create an efficient cache the most suitable
data structure seems like a hashmap, which C lacks. Adding support for
printing out the cgroup path would thus be a significant
implementation effort for something we in the end primarily will rely
on ebpf-exporter for anyways.

Signed-off-by: Simon Sundberg <simon.sundberg@kau.se>

Author: simosund

netstacklat/netstacklat.bpf.c

@@ -20,6 +20,7 @@ volatile const struct netstacklat_bpf_config user_config = {
 	.filter_cgroup = false,
 	.filter_nonempty_sockqueue = false,
 	.groupby_ifindex = false,
+	.groupby_cgroup = false,
 };
 
 /*
@@ -37,7 +38,7 @@ struct sk_buff___old {
 
 struct {
 	__uint(type, BPF_MAP_TYPE_PERCPU_HASH);
-	__uint(max_entries, HIST_NBUCKETS * NETSTACKLAT_N_HOOKS * 16);
+	__uint(max_entries, HIST_NBUCKETS * NETSTACKLAT_N_HOOKS * 64);
 	__type(key, struct hist_key);
 	__type(value, u64);
 } netstack_latency_seconds SEC(".maps");
@@ -250,21 +251,18 @@ static bool filter_cgroup(u64 cgroup_id)
 	return bpf_map_lookup_elem(&netstack_cgroupfilter, &cgroup_id) != NULL;
 }
 
-static bool filter_current_task(void)
+static bool filter_current_task(u64 cgroup)
 {
 	bool ok = true;
-	__u64 cgroup;
 	__u32 tgid;
 
 	if (user_config.filter_pid) {
 		tgid = bpf_get_current_pid_tgid() >> 32;
 		ok = ok && filter_pid(tgid);
 	}
 
-	if (user_config.filter_cgroup) {
-		cgroup = bpf_get_current_cgroup_id();
+	if (user_config.filter_cgroup)
 		ok = ok && filter_cgroup(cgroup);
-	}
 
 	return ok;
 }
@@ -294,12 +292,16 @@ static void record_socket_latency(struct sock *sk, struct sk_buff *skb,
 				  ktime_t tstamp, enum netstacklat_hook hook)
 {
 	struct hist_key key = { .hook = hook };
+	u64 cgroup = 0;
 	u32 ifindex;
 
 	if (!filter_nonempty_sockqueue(sk))
 		return;
 
-	if (!filter_current_task())
+	if (user_config.filter_cgroup || user_config.groupby_cgroup)
+		cgroup = bpf_get_current_cgroup_id();
+
+	if (!filter_current_task(cgroup))
 		return;
 
 	ifindex = skb ? skb->skb_iif : sk->sk_rx_dst_ifindex;
@@ -311,6 +313,8 @@ static void record_socket_latency(struct sock *sk, struct sk_buff *skb,
 
 	if (user_config.groupby_ifindex)
 		key.ifindex = ifindex;
+	if (user_config.groupby_cgroup)
+		key.cgroup = cgroup;
 
 	record_latency_since(tstamp, &key);
 }

netstacklat/netstacklat.c

@@ -98,6 +98,7 @@ static const struct option long_options[] = {
 	{ "cgroups",           required_argument, NULL, 'c' },
 	{ "nonempty-queue",    no_argument,       NULL, 'q' },
 	{ "groupby-interface", no_argument,       NULL, 'I' },
+	{ "groupby-cgroup",    no_argument,       NULL, 'C' },
 	{ 0, 0, 0, 0 }
 };
 
@@ -564,6 +565,7 @@ static int parse_arguments(int argc, char *argv[],
 	conf->bpf_conf.filter_ifindex = false;
 	conf->bpf_conf.filter_nonempty_sockqueue = false;
 	conf->bpf_conf.groupby_ifindex = false;
+	conf->bpf_conf.groupby_cgroup = false;
 
 	conf->pids = calloc(MAX_PARSED_PIDS, sizeof(*conf->pids));
 	conf->ifindices = calloc(MAX_PARSED_IFACES, sizeof(*conf->ifindices));
@@ -658,6 +660,9 @@ static int parse_arguments(int argc, char *argv[],
 		case 'I': // groupby-interface
 			conf->bpf_conf.groupby_ifindex = true;
 			break;
+		case 'C': // groupby-cgroup
+			conf->bpf_conf.groupby_cgroup = true;
+			break;
 		case 'h': // help
 			print_usage(stdout, argv[0]);
 			exit(EXIT_SUCCESS);
@@ -831,6 +836,9 @@ static void print_histkey(FILE *stream, const struct hist_key *key)
 
 	if (key->ifindex)
 		fprintf(stream, ", interface=%u", key->ifindex);
+
+	if (key->cgroup)
+		fprintf(stream, ", cgroup=%llu", key->cgroup);
 }
 
 static int cmp_histkey(const void *val1, const void *val2)
@@ -843,6 +851,9 @@ static int cmp_histkey(const void *val1, const void *val2)
 	if (key1->ifindex != key2->ifindex)
 		return key1->ifindex > key2->ifindex ? 1 : -1;
 
+	if (key1->cgroup != key2->cgroup)
+		return key1->cgroup > key2->cgroup ? 1 : -1;
+
 	return 0;
 }
 
@@ -1042,6 +1053,11 @@ static int init_histogram_buffer(struct histogram_buffer *buf,
 				     min(conf->nifindices, 64) :
 				     32;
 
+	if (conf->bpf_conf.groupby_cgroup)
+		max_hists *= conf->bpf_conf.filter_cgroup ?
+				     min(conf->ncgroups, 128) :
+				     64;
+
 	buf->hists = calloc(max_hists, sizeof(*buf->hists));
 	if (!buf->hists)
 		return -errno;

netstacklat/netstacklat.h

@@ -60,6 +60,7 @@ enum netstacklat_hook {
  * member is named "bucket" and is the histogram bucket index.
  */
 struct hist_key {
+	__u64 cgroup;
 	__u32 ifindex;
 	__u16 hook; // need well defined size for ebpf-exporter to decode
 	__u16 bucket; // needs to be last to be compatible with ebpf-exporter
@@ -72,6 +73,7 @@ struct netstacklat_bpf_config {
 	bool filter_cgroup;
 	bool filter_nonempty_sockqueue;
 	bool groupby_ifindex;
+	bool groupby_cgroup;
 };
 
 #endif

netstacklat/netstacklat.yaml

@@ -7,6 +7,11 @@ metrics:
       bucket_max: 34
       bucket_multiplier: 0.000000001 # nanoseconds to seconds
       labels:
+        - name: cgroup
+          size: 8
+          decoders:
+            - name: uint
+            - name: cgroup
         - name: iface
           size: 4
           decoders: