add a negative cache to ebpf exporter

Often, some decoders such as regexp can run repeatedly on the same input and skip them
with regexp filter. An common example is matching cgroup path in a chain like so:

```
- name: cgroup
- name: regexp
  regexps:
  - ^.*(system.slice).*$
```

Anything that is not in system.slice cgroup will be skipped. When only a small
subset of inputs is matched, the overhead of regexp matching can often be noticable.

We add a skip cache here to test for input that would produce ErrSkipLabelSet and
skip regex matching on them to reduce the work done on regexp matching.
The cache size is customizable with the flag `config.skip-cache-size`

Signed-off-by: Daniel Dao <dqminh@cloudflare.com>

Author: dqminh

cmd/ebpf_exporter/main.go

@@ -41,6 +41,7 @@ func main() {
 	metricsPath := kingpin.Flag("web.telemetry-path", "Path under which to expose metrics.").Default("/metrics").String()
 	capabilities := kingpin.Flag("capabilities.keep", "Comma separated list of capabilities to keep (cap_syslog, cap_bpf, etc.), 'all' or 'none'").Default("all").String()
 	btfPath := kingpin.Flag("btf.path", "Optional BTF file path.").Default("").String()
+	skipCacheSize := kingpin.Flag("config.skip-cache-size", "Size of the LRU skip cache").Int()
 	kingpin.Version(version.Print("ebpf_exporter"))
 	kingpin.HelpFlag.Short('h')
 	kingpin.Parse()
@@ -92,7 +93,7 @@ func main() {
 
 	notify("creating exporter...")
 
-	e, err := exporter.New(configs, tracing.NewProvider(processor), *btfPath)
+	e, err := exporter.New(configs, *skipCacheSize, tracing.NewProvider(processor), *btfPath)
 	if err != nil {
 		log.Fatalf("Error creating exporter: %s", err)
 	}

decoder/decoder.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/cloudflare/ebpf_exporter/v2/config"
 	"github.com/cloudflare/ebpf_exporter/v2/kallsyms"
+	lru "github.com/hashicorp/golang-lru/v2"
 )
 
 // ErrSkipLabelSet instructs exporter to skip label set
@@ -21,13 +22,14 @@ type Decoder interface {
 
 // Set is a set of Decoders that may be applied to produce a label
 type Set struct {
-	mu       sync.Mutex
-	decoders map[string]Decoder
-	cache    map[string]map[string][]string
+	mu        sync.Mutex
+	decoders  map[string]Decoder
+	cache     map[string]map[string][]string
+	skipCache *lru.Cache[string, struct{}]
 }
 
 // NewSet creates a Set with all known decoders
-func NewSet() (*Set, error) {
+func NewSet(skipCacheSize int) (*Set, error) {
 	cgroup, err := NewCgroupDecoder()
 	if err != nil {
 		return nil, fmt.Errorf("error creating cgroup decoder: %w", err)
@@ -38,7 +40,7 @@ func NewSet() (*Set, error) {
 		return nil, fmt.Errorf("error creating ksym decoder: %w", err)
 	}
 
-	return &Set{
+	s := &Set{
 		decoders: map[string]Decoder{
 			"cgroup":       cgroup,
 			"dname":        &Dname{},
@@ -60,7 +62,16 @@ func NewSet() (*Set, error) {
 			"uint":         &UInt{},
 		},
 		cache: map[string]map[string][]string{},
-	}, nil
+	}
+
+	if skipCacheSize > 0 {
+		skipCache, err := lru.New[string, struct{}](skipCacheSize)
+		if err != nil {
+			return nil, err
+		}
+		s.skipCache = skipCache
+	}
+	return s, nil
 }
 
 // decode transforms input byte field into a string according to configuration
@@ -75,6 +86,9 @@ func (s *Set) decode(in []byte, label config.Label) ([]byte, error) {
 		decoded, err := s.decoders[decoder.Name].Decode(result, decoder)
 		if err != nil {
 			if errors.Is(err, ErrSkipLabelSet) {
+				if s.skipCache != nil {
+					s.skipCache.Add(string(in), struct{}{})
+				}
 				return decoded, err
 			}
 
@@ -106,6 +120,14 @@ func (s *Set) DecodeLabelsForMetrics(in []byte, name string, labels []config.Lab
 		return cached, nil
 	}
 
+	// Also check the skip cache if the input would have return ErrSkipLabelSet
+	// and return the error early.
+	if s.skipCache != nil {
+		if _, ok := s.skipCache.Get(string(in)); ok {
+			return nil, ErrSkipLabelSet
+		}
+	}
+
 	values, err := s.decodeLabels(in, labels)
 	if err != nil {
 		return nil, err

decoder/decoder_test.go

@@ -1,6 +1,7 @@
 package decoder
 
 import (
+	"errors"
 	"fmt"
 	"sync"
 	"testing"
@@ -148,7 +149,7 @@ func TestDecodeLabels(t *testing.T) {
 	}
 
 	for i, c := range cases {
-		s, err := NewSet()
+		s, err := NewSet(0)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -178,6 +179,118 @@ func TestDecodeLabels(t *testing.T) {
 	}
 }
 
+func TestDecodeSkipLabels(t *testing.T) {
+	cases := []struct {
+		in          []byte
+		skipCacheIn string
+		labels      []config.Label
+		out         []string
+		err         bool
+	}{
+		{
+			in:          append([]byte{0x8, 0x0, 0x0, 0x0}, zeroPaddedString("bananas", 32)...),
+			skipCacheIn: "",
+			labels: []config.Label{
+				{
+					Name: "number",
+					Size: 4,
+					Decoders: []config.Decoder{
+						{
+							Name: "uint",
+						},
+					},
+				},
+				{
+					Name: "fruit",
+					Size: 32,
+					Decoders: []config.Decoder{
+						{
+							Name: "string",
+						},
+						{
+							Name: "regexp",
+							Regexps: []string{
+								"^bananas$",
+								"$is-banana-even-fruit$",
+							},
+						},
+					},
+				},
+			},
+			out: []string{"8", "bananas"},
+			err: false,
+		},
+		{
+			in:          append([]byte{0x8, 0x0, 0x0, 0x0}, zeroPaddedString("bananas", 32)...),
+			skipCacheIn: string(zeroPaddedString("bananas", 32)),
+			labels: []config.Label{
+				{
+					Name: "number",
+					Size: 4,
+					Decoders: []config.Decoder{
+						{
+							Name: "uint",
+						},
+					},
+				},
+				{
+					Name: "fruit",
+					Size: 32,
+					Decoders: []config.Decoder{
+						{
+							Name: "string",
+						},
+						{
+							Name: "regexp",
+							Regexps: []string{
+								"^tomato$",
+							},
+						},
+					},
+				},
+			},
+			out: []string{"8", "bananas"},
+			err: true, // this label should be skipped, only tomatoes allowed
+		},
+	}
+
+	for i, c := range cases {
+		s, err := NewSet(100)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		out, err := s.DecodeLabelsForMetrics(c.in, fmt.Sprintf("test:%d", i), c.labels)
+		if c.err {
+			if err == nil {
+				t.Errorf("Expected error for input %#v and labels %#v, but did not receive it", c.in, c.labels)
+			}
+
+			if errors.Is(err, ErrSkipLabelSet) {
+				if !s.skipCache.Contains(c.skipCacheIn) {
+					t.Errorf("Expected skipCache to have input %#v", c.skipCacheIn)
+				}
+			}
+
+			continue
+		}
+
+		if err != nil {
+			t.Errorf("Error decoding %#v with labels set to %#v: %s", c.in, c.labels, err)
+		}
+
+		if len(c.out) != len(out) {
+			t.Errorf("Expected %d outputs (%v), received %d (%v)", len(c.out), c.out, len(out), out)
+		}
+
+		for i := 0; i < len(c.out) && i < len(out); i++ {
+			if c.out[i] != out[i] {
+				t.Errorf("Output label %d for input %#v is wrong: expected %s, but received %s", i, c.in, c.out[i], out[i])
+			}
+		}
+	}
+}
+
 func TestDecoderSetConcurrency(t *testing.T) {
 	in := append([]byte{0x8, 0x0, 0x0, 0x0}, zeroPaddedString("bananas", 32)...)
 
@@ -209,7 +322,7 @@ func TestDecoderSetConcurrency(t *testing.T) {
 		},
 	}
 
-	s, err := NewSet()
+	s, err := NewSet(0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -274,7 +387,7 @@ func TestDecoderSetCache(t *testing.T) {
 		},
 	}
 
-	s, err := NewSet()
+	s, err := NewSet(0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -345,7 +458,7 @@ func BenchmarkCache(b *testing.B) {
 		},
 	}
 
-	s, err := NewSet()
+	s, err := NewSet(0)
 	if err != nil {
 		b.Fatal(err)
 	}

exporter/exporter.go

@@ -56,7 +56,7 @@ type Exporter struct {
 }
 
 // New creates a new exporter with the provided config
-func New(configs []config.Config, tracingProvider tracing.Provider, btfPath string) (*Exporter, error) {
+func New(configs []config.Config, skipCacheSize int, tracingProvider tracing.Provider, btfPath string) (*Exporter, error) {
 	enabledConfigsDesc := prometheus.NewDesc(
 		prometheus.BuildFQName(prometheusNamespace, "", "enabled_configs"),
 		"The set of enabled configs",
@@ -101,7 +101,7 @@ func New(configs []config.Config, tracingProvider tracing.Provider, btfPath stri
 		decoderErrorCount.WithLabelValues(config.Name).Add(0.0)
 	}
 
-	decoders, err := decoder.NewSet()
+	decoders, err := decoder.NewSet(skipCacheSize)
 	if err != nil {
 		return nil, fmt.Errorf("error creating decoder set: %w", err)
 	}

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/aquasecurity/libbpfgo v0.8.0-libbpf-1.5
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595
+	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/iovisor/gobpf v0.2.0
 	github.com/jaypipes/pcidb v1.0.1
 	github.com/mdlayher/sdnotify v1.0.0

go.sum

@@ -30,6 +30,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI=
+github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
+github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/iovisor/gobpf v0.2.0 h1:34xkQxft+35GagXBk3n23eqhm0v7q0ejeVirb8sqEOQ=
 github.com/iovisor/gobpf v0.2.0/go.mod h1:WSY9Jj5RhdgC3ci1QaacvbFdQ8cbrEjrpiZbLHLt2s4=
 github.com/jaypipes/pcidb v1.0.1 h1:WB2zh27T3nwg8AE8ei81sNRb9yWBii3JGNJtT7K9Oic=

tracing/extract_test.go

@@ -151,7 +151,7 @@ func TestExtractFilled(t *testing.T) {
 		},
 	}
 
-	decoders, err := decoder.NewSet()
+	decoders, err := decoder.NewSet(0)
 	if err != nil {
 		t.Fatalf("Error creating decoders set: %v", err)
 	}