fix(repeat): memory leak (#413)

* test: add repeat non regression test

* fix(profiler): recognize mem keyword

* refactor(process): wip! rewrite processor with go iterators

* refactor(process): wip! impact pipeline api

* refactor(process): wip! dry queue

* refactor(process): wip! deprecate repeater process

* refactor(process): wip! fix failed unit tests

* refactor(process): wip! fix wasm build

* refactor(process): wip! fix caches

* refactor(process): wip! fix conversion with non dict selector

* refactor(process): wip! fix wasm build

* refactor(process): wip! repeat until and while

* refactor(process): wip! refactor until and while

* refactor(process): wip! rollback repeatCondition

* refactor(process): wip! rollback repeater

* refactor(process): wip! fix TempSource

* refactor(process): wip! fix repeat while

* refactor(process): changelog

* refactor(repeat): cleanup unused code

Author: adrienaury

CHANGELOG.md

@@ -14,6 +14,10 @@ Types of changes
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [1.31.1]
+
+- `Fixed` memory leak with `--repeat` option
+
 ## [1.31.0]
 
 - `Added` insecure, proxy, proxy-user and user flags to mock command

cmd/pimo/main.go

@@ -328,7 +328,7 @@ func run(cmd *cobra.Command) {
 
 	if profiling == "cpu" {
 		profiler = profile.Start(profile.CPUProfile, profile.ProfilePath("."), profile.Quiet)
-	} else if profiling == "memory" {
+	} else if profiling == "memory" || profiling == "mem" {
 		profiler = profile.Start(profile.MemProfile, profile.ProfilePath("."), profile.Quiet)
 	}
 
@@ -355,7 +355,7 @@ func run(cmd *cobra.Command) {
 		dumpStats(stats)
 	}
 
-	if profiling == "cpu" || profiling == "memory" {
+	if profiling == "cpu" || profiling == "memory" || profiling == "mem" {
 		profiler.Stop()
 	}
 

internal/app/pimo/mock/process.go

@@ -22,7 +22,7 @@ func (s BlackHoleSink) ProcessDictionary(value model.Entry) error {
 
 type Processor struct {
 	mutex    *sync.Mutex
-	source   *model.CallableMapSource
+	source   *model.MutableSource
 	pipeline model.SinkedPipeline
 }
 
@@ -44,7 +44,7 @@ func NewProcessor(maskingFile string, globalSeed *int64, caches map[string]model
 		}
 	}
 
-	source := model.NewCallableMapSource()
+	source := model.NewMutableSource()
 
 	pipeline, caches, err := model.BuildPipeline(model.NewPipeline(source), pdef, caches, nil, "", "")
 	if err != nil {
@@ -62,7 +62,7 @@ func (p *Processor) Process(dict model.Dictionary) error {
 	p.mutex.Lock()
 	defer p.mutex.Unlock()
 
-	p.source.SetValue(dict)
+	p.source.SetValues(dict)
 
 	err := p.pipeline.Run()
 

internal/app/pimo/pimo.go

@@ -122,13 +122,13 @@ func (ctx *Context) Configure(cfg Config) error {
 	switch {
 	case cfg.XMLCallback:
 		over.MDC().Set("context", "callback-input")
-		ctx.source = model.NewCallableMapSource()
+		ctx.source = model.NewMutableSource()
 	case cfg.EmptyInput:
 		over.MDC().Set("context", "empty-input")
-		ctx.source = model.NewSourceFromSlice([]model.Dictionary{model.NewPackedDictionary()})
+		ctx.source = model.NewSliceSource([]model.Dictionary{model.NewPackedDictionary()})
 	case cfg.SingleInput != nil:
 		over.MDC().Set("context", "single-input")
-		ctx.source = model.NewSourceFromSlice([]model.Dictionary{cfg.SingleInput.Pack()})
+		ctx.source = model.NewSliceSource([]model.Dictionary{cfg.SingleInput.Pack()})
 	case cfg.ParquetInput != "":
 		over.MDC().Set("context", "parquet")
 		source, err := parquet.NewPackedSource(cfg.ParquetInput)
@@ -145,6 +145,10 @@ func (ctx *Context) Configure(cfg Config) error {
 		return fmt.Errorf("Cannot use repeatUntil and repeatWhile options together")
 	}
 
+	if cfg.Iteration > 1 {
+		ctx.source = model.NewCountRepeater(ctx.source, cfg.Iteration)
+	}
+
 	ctx.repeatCondition = cfg.RepeatWhile
 	ctx.repeatConditionMode = "while"
 	if cfg.RepeatUntil != "" {
@@ -153,8 +157,8 @@ func (ctx *Context) Configure(cfg Config) error {
 	}
 
 	ctx.pipeline = model.NewPipeline(ctx.source).
-		Process(model.NewCounterProcessWithCallback("input-line", 0, updateContext)).
-		Process(model.NewRepeaterProcess(cfg.Iteration))
+		Process(model.NewCounterProcessWithCallback("input-line", 0, updateContext))
+
 	over.AddGlobalFields("input-line")
 
 	injectTemplateFuncs()
@@ -399,11 +403,11 @@ func (ctx *Context) ExecuteMap(data map[string]any) (map[string]any, error) {
 	for k, v := range data {
 		input = input.With(k, v)
 	}
-	source, ok := ctx.source.(*model.CallableMapSource)
+	source, ok := ctx.source.(*model.MutableSource)
 	if !ok {
-		return nil, fmt.Errorf("Source is not CallableMapSource")
+		return nil, fmt.Errorf("Source is not MutableSource")
 	}
-	source.SetValue(input)
+	source.SetValues(input)
 	result := []model.Entry{}
 	err := ctx.pipeline.AddSink(model.NewSinkToSlice(&result)).Run()
 	if err != nil {

pkg/jsonline/jsonline.go

@@ -20,6 +20,7 @@ package jsonline
 import (
 	"bufio"
 	"io"
+	"iter"
 
 	over "github.com/adrienaury/zeromdc"
 	"github.com/cgi-fr/pimo/pkg/model"
@@ -54,6 +55,40 @@ func (s *Source) Open() error {
 	return nil
 }
 
+func (s *Source) Close() error {
+	return nil
+}
+
+func (s *Source) Values() iter.Seq2[model.Entry, error] {
+	return func(yield func(model.Entry, error) bool) {
+		for s.fscanner.Scan() {
+			line := s.fscanner.Bytes()
+
+			var dict model.Dictionary
+			var err error
+
+			if s.packed {
+				dict, err = JSONToPackedDictionary(line)
+			} else {
+				dict, err = JSONToDictionary(line)
+			}
+
+			if err != nil {
+				yield(dict, err)
+				return
+			}
+
+			if !yield(dict, nil) {
+				return
+			}
+		}
+
+		if s.fscanner.Err() != nil {
+			yield(model.NewDictionary(), s.fscanner.Err())
+		}
+	}
+}
+
 // Next convert next line to model.Dictionary
 func (s *Source) Next() bool {
 	if !s.fscanner.Scan() {

pkg/model/common.go

@@ -0,0 +1,6 @@
+package model
+
+type Resource interface {
+	Open() error
+	Close() error
+}

pkg/model/iterable.go

@@ -0,0 +1,9 @@
+package model
+
+import "iter"
+
+// Iterable is a resource that can be iterated over to produce values of type T. It extends the Resource interface.
+type Iterable[T any] interface {
+	Resource
+	Values() iter.Seq2[T, error]
+}