Merge pull request #197 from CGI-FR/196-proposal-add-a-command-to-generate-json-dump-file-with-execution-stats

feat(stats): adds a command to generate a stat file

Author: Youen Péron

README.md

@@ -171,6 +171,8 @@ This takes the `data.json` file, masks the data contained inside it and put the
 * `--mask` Declare a simple masking definition in command line (minified YAML format: `--mask "value={fluxUri: 'pimo://nameFR'}"`, or `--mask "value=[{add: ''},{fluxUri: 'pimo://nameFR'}]"` for multiple masks). For advanced use case (e.g. if caches needed) `masking.yml` file definition will be preferred.
 * `--repeat-until <condition>` This flag will make PIMO keep masking every input until the condition is met. Condition format is using [Template](https://pkg.go.dev/text/template). Last output verifies the condition.
 * `--repeat-while <condition>` This flag will make PIMO keep masking every input while the condition is met. Condition format is using [Template](https://pkg.go.dev/text/template).
+* `--stats <filename | url>` This flag either outputs run statistics to the specified file or send them to specified url (has to start with `http` or `https`).
+* `--statsTemplate <string>` This flag will have PIMO use the value as a template to generate statistics. Please use go templating format to include statistics. To include them you have to specify them as `{{ .Stats }}`. (i.e. `{"software":"PIMO","stats":{{ .Stats }}}`)
 
 ### PIMO Play
 

cmd/pimo/main.go

@@ -18,16 +18,20 @@
 package main
 
 import (
+	"bytes"
 	"fmt"
+	netHttp "net/http"
 	"os"
 	"runtime"
 	"strings"
+	"text/template"
 	"time"
 
 	over "github.com/adrienaury/zeromdc"
 	"github.com/cgi-fr/pimo/internal/app/pimo"
 	"github.com/cgi-fr/pimo/pkg/flow"
 	"github.com/cgi-fr/pimo/pkg/model"
+	"github.com/cgi-fr/pimo/pkg/statistics"
 	"github.com/mattn/go-isatty"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
@@ -42,21 +46,23 @@ var (
 	buildDate string
 	builtBy   string
 
-	verbosity        string
-	debug            bool
-	jsonlog          bool
-	colormode        string
-	iteration        int
-	emptyInput       bool
-	maskingFile      string
-	cachesToDump     map[string]string
-	cachesToLoad     map[string]string
-	skipLineOnError  bool
-	skipFieldOnError bool
-	seedValue        int64
-	maskingOneLiner  []string
-	repeatUntil      string
-	repeatWhile      string
+	verbosity             string
+	debug                 bool
+	jsonlog               bool
+	colormode             string
+	iteration             int
+	emptyInput            bool
+	maskingFile           string
+	cachesToDump          map[string]string
+	cachesToLoad          map[string]string
+	skipLineOnError       bool
+	skipFieldOnError      bool
+	seedValue             int64
+	maskingOneLiner       []string
+	repeatUntil           string
+	repeatWhile           string
+	statisticsDestination string
+	statsTemplate         string
 )
 
 func main() {
@@ -89,6 +95,8 @@ There is NO WARRANTY, to the extent permitted by law.`, version, commit, buildDa
 	rootCmd.PersistentFlags().StringArrayVarP(&maskingOneLiner, "mask", "m", []string{}, "one liner masking")
 	rootCmd.PersistentFlags().StringVar(&repeatUntil, "repeat-until", "", "mask each input repeatedly until the given condition is met")
 	rootCmd.PersistentFlags().StringVar(&repeatWhile, "repeat-while", "", "mask each input repeatedly while the given condition is met")
+	rootCmd.PersistentFlags().StringVar(&statisticsDestination, "stats", "", "generate execution statistics in the specified dump file")
+	rootCmd.PersistentFlags().StringVar(&statsTemplate, "statsTemplate", "", "template string to format stats (to include them you have to specify them as `{{ .Stats }}` like `{\"software\":\"PIMO\",\"stats\":{{ .Stats }}}`)")
 
 	rootCmd.AddCommand(&cobra.Command{
 		Use: "jsonschema",
@@ -186,14 +194,18 @@ func run() {
 		os.Exit(1)
 	}
 
+	startTime := time.Now()
+
 	stats, err := ctx.Execute(os.Stdout)
 	if err != nil {
 		log.Err(err).Msg("Cannot execute pipeline")
 		log.Warn().Int("return", stats.GetErrorCode()).Msg("End PIMO")
 		os.Exit(stats.GetErrorCode())
 	}
 
-	log.Info().RawJSON("stats", stats.ToJSON()).Int("return", 0).Msg("End PIMO")
+	duration := time.Since(startTime)
+	statistics.SetDuration(duration)
+	dumpStats(stats)
 	os.Exit(0)
 }
 
@@ -240,3 +252,58 @@ func initLog() {
 	over.MDC().Set("config", maskingFile)
 	over.SetGlobalFields([]string{"config"})
 }
+
+func dumpStats(stats statistics.ExecutionStats) {
+	statsToWrite := stats.ToJSON()
+	if statsTemplate != "" {
+		tmpl, err := template.New("statsTemplate").Parse(statsTemplate)
+		if err != nil {
+			log.Error().Err(err).Msg(("Error parsing statistics template"))
+			os.Exit(1)
+		}
+		var output bytes.Buffer
+		err = tmpl.ExecuteTemplate(&output, "statsTemplate", Stats{Stats: string(stats.ToJSON())})
+		if err != nil {
+			log.Error().Err(err).Msg("Error adding stats to template")
+			os.Exit(1)
+		}
+		statsToWrite = output.Bytes()
+	}
+	if statisticsDestination != "" {
+		if strings.HasPrefix(statisticsDestination, "http") {
+			sendMetrics(statisticsDestination, statsToWrite)
+		} else {
+			writeMetricsToFile(statisticsDestination, statsToWrite)
+		}
+	}
+
+	log.Info().RawJSON("stats", stats.ToJSON()).Int("return", 0).Msg("End PIMO")
+}
+
+func writeMetricsToFile(statsFile string, statsByte []byte) {
+	file, err := os.Create(statsFile)
+	if err != nil {
+		log.Error().Err(err).Msg("Error generating statistics dump file")
+	}
+	defer file.Close()
+
+	_, err = file.Write(statsByte)
+	if err != nil {
+		log.Error().Err(err).Msg("Error writing statistics to dump file")
+	}
+	log.Info().Msgf("Statistics exported to file %s", file.Name())
+}
+
+func sendMetrics(statsDestination string, statsByte []byte) {
+	requestBody := bytes.NewBuffer(statsByte)
+	// nolint: gosec
+	_, err := netHttp.Post(statsDestination, "application/json", requestBody)
+	if err != nil {
+		log.Error().Err(err).Msgf("An error occurred trying to send metrics to %s", statsDestination)
+	}
+	log.Info().Msgf("Statistics sent to %s", statsDestination)
+}
+
+type Stats struct {
+	Stats interface{} `json:"stats"`
+}

pkg/statistics/statistics.go

@@ -2,6 +2,7 @@ package statistics
 
 import (
 	"encoding/json"
+	"time"
 
 	over "github.com/adrienaury/zeromdc"
 	"github.com/rs/zerolog/log"
@@ -14,15 +15,17 @@ type ExecutionStats interface {
 	GetIgnoredPathsCount() int  // counter for path not found in data
 	GetIgnoredLinesCount() int  // counter for line skipped (flag --skip-line-on-error)
 	GetIgnoredFieldsCount() int // counter for field skipped (flag --skip-field-on-error)
+	GetDuration() time.Duration
 
 	ToJSON() []byte
 }
 
 type stats struct {
 	errorCode            int
-	IgnoredPathsCounter  int `json:"ignoredPaths"`
-	IgnoredLinesCounter  int `json:"skippedLines"`
-	IgnoredFieldsCounter int `json:"skippedFields"`
+	IgnoredPathsCounter  int           `json:"ignoredPaths"`
+	IgnoredLinesCounter  int           `json:"skippedLines"`
+	IgnoredFieldsCounter int           `json:"skippedFields"`
+	Duration             time.Duration `json:"duration"`
 }
 
 // Reset all statistics to zero
@@ -101,3 +104,12 @@ func getStats() *stats {
 	log.Warn().Msg("Statistics uncorrectly initialized")
 	return &stats{}
 }
+
+func SetDuration(duration time.Duration) {
+	stats := getStats()
+	stats.Duration = duration
+}
+
+func (s *stats) GetDuration() time.Duration {
+	return s.Duration
+}

test/httpmock/default.json

@@ -0,0 +1,10 @@
+[
+    {
+        "httpRequest": {
+            "path": "/api/v1/stats"
+        },
+        "httpResponse": {
+            "statusCode": 200
+        }
+    }
+]

test/suites/dump-statistics.yml

@@ -0,0 +1,50 @@
+name: dump statistics file features
+testcases:
+- name: dump file
+  steps:
+  - script: rm -f masking.yml
+  - script: rm -f pimo-stats.json
+  - script: |-
+      cat > masking.yml <<EOF
+      version: "1"
+      masking:
+        - selector:
+            jsonpath: "name"
+          mask:
+            add: "Dorothy"
+      EOF
+  - script: |-
+      echo '{ "name": "John" }' | pimo -v 3 --stats pimo-stats.json
+    assertions:
+    - result.code ShouldEqual 0
+    - result.systemerr ShouldContainSubstring Statistics exported to file
+  - script: cat pimo-stats.json
+    assertions:
+    - result.code ShouldEqual 0
+    - result.systemout ShouldContainSubstring "ignoredPaths":0
+    - result.systemout ShouldContainSubstring duration
+
+- name: dump file with template
+  steps:
+  - script: rm -f masking.yml
+  - script: rm -f pimo-stats-template.json
+  - script: |-
+      cat > masking.yml <<EOF
+      version: "1"
+      masking:
+        - selector:
+            jsonpath: "name"
+          mask:
+            add: "Dorothy"
+      EOF
+  - script: |-
+      echo '{ "name": "John" }' | pimo -v 3 --log-json --stats pimo-stats-template.json --statsTemplate '{"software":"PIMO","stats":{{ .Stats }}}'
+    assertions:
+    - result.code ShouldEqual 0
+    - result.systemerr ShouldContainSubstring Statistics exported to file
+  - script: cat pimo-stats-template.json
+    assertions:
+    - result.code ShouldEqual 0
+    - result.systemout ShouldContainSubstring {"software":"PIMO","stats"
+    - result.systemout ShouldContainSubstring ignoredPaths":0
+    - result.systemout ShouldContainSubstring duration