fix: use fromcache after a mask which causes a change in the type of the value (#110)

* test: add bug in venom test

* fix(fromCache): bad number type in cache

* fix: regression on dataParser mask

* test: fix test clean type

* test: fix template notation

* test: fix venom test

* docs: update changelog

Co-authored-by: Marie Giraud <marie.giraud@cgi.com>
Co-authored-by: adrienaury <adrien.aury@cgi.com>

Author: 3 people

CHANGELOG.md

@@ -14,6 +14,10 @@ Types of changes
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [1.12.1]
+
+- `Fixed` use fromcache after a mask which causes a change in the type of the value
+
 ## [1.12.0]
 
 - `Added` markov mask to generate pseudo text based on a sample text

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/labstack/echo/v4 v4.7.2
 	github.com/mattn/go-isatty v0.0.14
 	github.com/rs/zerolog v1.26.1
+	github.com/spf13/cast v1.4.1
 	github.com/spf13/cobra v1.4.0
 	github.com/stretchr/testify v1.7.1
 	github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea
@@ -37,7 +38,6 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/smartystreets/goconvey v1.6.4 // indirect
-	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.1 // indirect

pkg/constant/constant.go

@@ -29,7 +29,7 @@ type MaskEngine struct {
 
 // NewMask return a MaskEngine from a value
 func NewMask(data model.Entry) MaskEngine {
-	return MaskEngine{data}
+	return MaskEngine{model.CleanTypes(data)}
 }
 
 // Mask return a Constant from a MaskEngine

pkg/dateparser/dateparser.go

@@ -18,12 +18,12 @@
 package dateparser
 
 import (
-	"encoding/json"
 	"fmt"
 	"time"
 
 	"github.com/cgi-fr/pimo/pkg/model"
 	"github.com/rs/zerolog/log"
+	"github.com/spf13/cast"
 )
 
 // MaskEngine is a type to change a date format
@@ -47,7 +47,7 @@ func (me MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Ent
 	var err error
 	switch {
 	case me.inputFormat == "unixEpoch":
-		i, err := e.(json.Number).Int64()
+		i, err := cast.ToInt64E(e)
 		if err != nil {
 			return nil, err
 		}

pkg/dateparser/dateparser_test.go

@@ -96,7 +96,7 @@ func TestFactoryShouldCreateAMask(t *testing.T) {
 func TestMaskingShouldReplaceUnixEpochByDateString(t *testing.T) {
 	outputFormat := "02/01/06"
 	dateMask := NewMask("unixEpoch", outputFormat)
-	data := json.Number("1647512434")
+	data := model.CleanTypes(json.Number("1647512434"))
 	resulttime, err := dateMask.Mask(data)
 	assert.Equal(t, nil, err, "error should be nil")
 	assert.Equal(t, "17/03/22", resulttime, "Should return the same time")

pkg/model/model.go

@@ -422,7 +422,7 @@ func (sink *SinkToCache) Open() error {
 }
 
 func (sink *SinkToCache) ProcessDictionary(dictionary Dictionary) error {
-	sink.cache.Put(dictionary.Get("key"), dictionary.Get("value"))
+	sink.cache.Put(CleanTypes(dictionary.Get("key")), CleanTypes(dictionary.Get("value")))
 	return nil
 }
 

pkg/model/ordered_dict.go

@@ -1,6 +1,7 @@
 package model
 
 import (
+	"encoding/json"
 	"fmt"
 
 	"github.com/rs/zerolog/log"
@@ -91,6 +92,23 @@ func CleanTypes(inter interface{}) interface{} {
 		}
 
 		return tab
+
+	case json.Number:
+
+		resFloat64, err := typedInter.Float64()
+		if err == nil {
+			return resFloat64
+		}
+
+		return typedInter.String()
+
+	case uint64:
+		res := float64(typedInter)
+		return res
+
+	case int64:
+		res := float64(typedInter)
+		return res
 	default:
 		return inter
 	}

pkg/rangemask/rangemask.go

@@ -18,7 +18,7 @@
 package rangemask
 
 import (
-	"encoding/json"
+	"fmt"
 	"strconv"
 
 	"github.com/cgi-fr/pimo/pkg/model"
@@ -41,9 +41,10 @@ func (rm MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Ent
 	if e == nil {
 		return e, nil
 	}
-	i, err := e.(json.Number).Int64()
-	if err != nil {
-		return nil, err
+	i, ok := e.(float64)
+
+	if !ok {
+		return e, fmt.Errorf("%v is not a number", e)
 	}
 	scaledValue := int(i) / rm.rangeScale * rm.rangeScale
 	rangedValue := "[" + strconv.Itoa(scaledValue) + ";" + strconv.Itoa(scaledValue+rm.rangeScale-1) + "]"

pkg/rangemask/rangemask_test.go

@@ -19,6 +19,7 @@ package rangemask
 
 import (
 	"encoding/json"
+	"fmt"
 	"testing"
 
 	"github.com/cgi-fr/pimo/pkg/model"
@@ -27,13 +28,19 @@ import (
 
 func TestMaskingShouldReplaceSensitiveValueByRangedValue(t *testing.T) {
 	rangeMask := NewMask(10)
-	result, err := rangeMask.Mask(json.Number("25"))
+	result, err := rangeMask.Mask(model.CleanTypes(json.Number("25")))
 	assert.Equal(t, nil, err, "error should be nil")
 	waited := "[20;29]"
 	assert.NotEqual(t, float64(25), result, "should be masked")
 	assert.Equal(t, waited, result, "should be [20;29]")
 }
 
+func TestMaskingShouldFailwithNotNumberInput(t *testing.T) {
+	rangeMask := NewMask(10)
+	_, err := rangeMask.Mask("not a number")
+	assert.Equal(t, fmt.Errorf("not a number is not a number"), err, "error should not be nil")
+}
+
 func TestFactoryShouldCreateAMask(t *testing.T) {
 	maskingConfig := model.Masking{Mask: model.MaskType{RangeMask: 15}}
 	mask, present, err := Factory(maskingConfig, 0, nil)

test/suites/masking_cache.yml

@@ -515,3 +515,71 @@ testcases:
     - result.systemout ShouldContainSubstring {"key":"Mathieu","value":"FDDO8965"}
     - result.systemout ShouldContainSubstring {"key":"Marcel","value":"FTTO4452"}
     - result.systemout ShouldContainSubstring {"key":"Mickael","value":"FJUS4442"}
+
+- name: bug fromCache with calculated value 1
+  steps:
+  - script: rm -f masking.yml
+  - script: |-
+      cat > masking.yml <<EOF
+      version: "1"
+      seed: 42
+      masking:
+        - selector:
+            jsonpath: "sexe"
+          masks:
+            - constant: 2
+            - fromCache: "cacheSex"
+
+      caches:
+        cacheSex :
+          unique: true
+          reverse:  true
+      EOF
+  - script: |-
+      cat > cacheSex.jsonl <<EOF
+      {"key": "M", "value": 2}
+      {"key": "F", "value": 1}
+      EOF
+  - script: |-
+      pimo --load-cache cacheSex=./cacheSex.jsonl <<EOF
+      {"sexe": 2.4}
+      EOF
+    assertions:
+    - result.code ShouldEqual 0
+    - result.systemout ShouldContainSubstring '{"sexe":"M"}'
+    - result.systemerr ShouldBeEmpty
+
+- name: bug fromCache with calculated value 2
+  steps:
+  - script: rm -f masking.yml
+  - script: |-
+      cat  > masking.yml <<EOF
+      version: "1"
+      seed: 42
+      masking:
+        - selector:
+            jsonpath: "sexe"
+          masks:
+            - template : '[[ round (toString .sexe) 0  ]]'
+            - fromjson: "sexe"
+            - fromCache: "cacheSex"
+      caches:
+        cacheSex :
+          unique: true
+          reverse:  true
+      EOF
+  - script: sed -i  "s/\[\[/\{\{/g"  masking.yml
+  - script: sed -i  "s/\]\]/\}\}/g"  masking.yml
+  - script: |-
+      cat > cacheSex.jsonl <<EOF
+      {"key": "M", "value": 2}
+      {"key": "F", "value": 1}
+      EOF
+  - script: |-
+      pimo --load-cache cacheSex=./cacheSex.jsonl <<EOF
+      {"sexe": 2.4}
+      EOF
+    assertions:
+    - result.code ShouldEqual 0
+    - result.systemout ShouldContainSubstring '{"sexe":"M"}'
+    - result.systemerr ShouldBeEmpty