feat: luhn mask (#62)

* docs: fix typo

* feat(luhn): add venom test

* feat(luhn): add mask skeleton

* feat(luhn): implement mask

* feat(luhn): update json schema

* feat(luhn): add venom test

* docs(luhn): add mask example

* docs(luhn): add mask example

* test(luhn): ok

* docs(luhn): add universe example

Author: adrienaury

README.md

@@ -84,6 +84,7 @@ The following types of masks can be used :
   * [`fluxUri`](#fluxUri) is to replace by a sequence of values defined in an external resource.
   * [`replacement`](#replacement) is to mask a data with another data from the jsonline.
   * [`pipe`](#pipe) is a mask to handle complex nested array structures, it can read an array as an object stream and process it with a sub-pipeline.
+  * [`luhn`](#luhn) can generate valid numbers using the Luhn algorithm (e.g. french SIRET or SIREN).
 
 A full `masking.yml` file example, using every kind of mask, is given with the source code.
 
@@ -256,7 +257,7 @@ This example will mask the `town` field of the input jsonlines with a value from
   - selector:
       jsonpath: "name"
     mask:
-      hashInUri: "pimo://nameFR
+      hashInUri: "pimo://nameFR"
 ```
 
 This example will mask the `name` field of the input jsonlines with a value from the list nameFR contained in pimo, the same way as for `hash` mask. The different URI usable with this selector are : `pimo`, `file` and `http`/`https`.
@@ -513,6 +514,7 @@ This mask will replace an integer value `{"age": 27}` with a range like this `{"
 If the data structure contains arrays of object like in the example below, this mask can pipe the objects into a sub pipeline definition.
 
 **`data.jsonl`**
+
 ```json
 {
     "organizations": [
@@ -550,7 +552,33 @@ If the data structure contains arrays of object like in the example below, this
 }
 ```
 
+### Luhn
+
+The [Luhn](https://en.wikipedia.org/wiki/Luhn_algorithm) algorithm is a simple checksum formula used to validate a variety of identification numbers.
+
+The `luhn` mask can calculate the checksum for any value.
+
+```yaml
+  - selector:
+      jsonpath: "siret"
+    mask:
+      luhn: {}
+```
+
+In this example, the `siret` value will be appended with the correct checksum, to create a valid SIRET number (french business identifier).
+
+The mask can be parametered to use a different universe of valid characters, internally using the [Luhn mod N](https://en.wikipedia.org/wiki/Luhn_mod_N_algorithm) algorithm.
+
+```yaml
+  - selector:
+      jsonpath: "siret"
+    mask:
+      luhn:
+        universe: "abcde"
+```
+
 **`masking.yml`**
+
 ```yaml
 version: "1"
 seed: 42

cmd/pimo/main.go

@@ -39,6 +39,7 @@ import (
 	"github.com/cgi-fr/pimo/pkg/hash"
 	"github.com/cgi-fr/pimo/pkg/increment"
 	"github.com/cgi-fr/pimo/pkg/jsonline"
+	"github.com/cgi-fr/pimo/pkg/luhn"
 	"github.com/cgi-fr/pimo/pkg/model"
 	"github.com/cgi-fr/pimo/pkg/pipe"
 	"github.com/cgi-fr/pimo/pkg/randdate"
@@ -260,6 +261,7 @@ func injectMaskFactories() []model.MaskFactory {
 		randomdecimal.Factory,
 		dateparser.Factory,
 		ff1.Factory,
+		luhn.Factory,
 	}
 }
 

pkg/luhn/luhn.go

@@ -0,0 +1,91 @@
+// Copyright (C) 2021 CGI France
+//
+// This file is part of PIMO.
+//
+// PIMO is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// PIMO is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with PIMO.  If not, see <http://www.gnu.org/licenses/>.
+
+package luhn
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/cgi-fr/pimo/pkg/model"
+	"github.com/rs/zerolog/log"
+)
+
+// MaskEngine is a struct to create incremental int
+type MaskEngine struct {
+	Universe []byte
+}
+
+// NewMask create an Luhn mask
+func NewMask(universe []byte) MaskEngine {
+	return MaskEngine{universe}
+}
+
+// Mask return the value with luhn checksum
+func (l MaskEngine) Mask(e model.Entry, context ...model.Dictionary) (model.Entry, error) {
+	if e == nil {
+		// Cannot use a nil value so we leave it untouched
+		log.Warn().Msg("Mask luhn - ignored null value")
+		return e, nil
+	}
+
+	log.Info().Msg("Mask luhn")
+
+	factor := 2
+	sum := 0
+	n := len(l.Universe)
+	input := e.(string)
+
+	// Starting from the right and working leftwards is easier since
+	// the initial "factor" will always be "2".
+	for i := len(input) - 1; i >= 0; i-- {
+		codePoint := bytes.IndexByte(l.Universe, input[i])
+		addend := factor * codePoint
+
+		// Alternate the "factor" that each "codePoint" is multiplied by
+		if factor == 2 {
+			factor = 1
+		} else {
+			factor = 2
+		}
+
+		// Sum the digits of the "addend" as expressed in base "n"
+		addend = addend/n + (addend % n)
+		sum += addend
+	}
+
+	// Calculate the number that must be added to the "sum"
+	// to make it divisible by "n".
+	remainder := sum % n
+	checkCodePoint := (n - remainder) % n
+
+	return input + string(l.Universe[checkCodePoint]), nil
+}
+
+// Create a mask from a configuration
+func Factory(conf model.Masking, seed int64, caches map[string]model.Cache) (model.MaskEngine, bool, error) {
+	if conf.Mask.Luhn != nil {
+		if conf.Mask.Luhn.Universe != "" {
+			if len(conf.Mask.Luhn.Universe)%2 != 0 {
+				return nil, true, fmt.Errorf("luhn universe size must be divisible by 2")
+			}
+			return NewMask([]byte(conf.Mask.Luhn.Universe)), true, nil
+		}
+		return NewMask([]byte("0123456789")), true, nil
+	}
+	return nil, false, nil
+}

pkg/model/model.go

@@ -122,6 +122,10 @@ type TemplateEachType struct {
 	Template string `yaml:"template,omitempty"`
 }
 
+type LuhnType struct {
+	Universe string `yaml:"universe,omitempty"`
+}
+
 type MaskType struct {
 	Add               Entry                `yaml:"add,omitempty" jsonschema:"oneof_required=Add"`
 	AddTransient      Entry                `yaml:"add-transient,omitempty" jsonschema:"oneof_required=AddTransient"`
@@ -150,6 +154,7 @@ type MaskType struct {
 	FF1               FF1Type              `yaml:"ff1,omitempty" jsonschema:"oneof_required=FF1"`
 	Pipe              PipeType             `yaml:"pipe,omitempty" jsonschema:"oneof_required=Pipe"`
 	FromJSON          string               `yaml:"fromjson,omitempty" jsonschema:"oneof_required=FromJSON"`
+	Luhn              *LuhnType            `yaml:"luhn,omitempty" jsonschema:"oneof_required=Luhn"`
 }
 
 type Masking struct {

schema/v1/pimo.schema.json

@@ -92,6 +92,15 @@
       "additionalProperties": false,
       "type": "object"
     },
+    "LuhnType": {
+      "properties": {
+        "universe": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
     "MaskType": {
       "properties": {
         "add": {
@@ -193,6 +202,10 @@
         },
         "fromjson": {
           "type": "string"
+        },
+        "luhn": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/LuhnType"
         }
       },
       "additionalProperties": false,
@@ -359,6 +372,12 @@
             "fromjson"
           ],
           "title": "FromJSON"
+        },
+        {
+          "required": [
+            "luhn"
+          ],
+          "title": "Luhn"
         }
       ]
     },

test/suites/masking_luhn.yml

@@ -0,0 +1,58 @@
+name: luhn mask
+testcases:
+- name: default universe
+  steps:
+  - script: rm -f masking.yml
+  - script: |-
+      cat > masking.yml <<EOF
+      version: "1"
+      masking:
+        - selector:
+            jsonpath: "siret"
+          mask:
+            luhn: {}
+      EOF
+  - script: |-
+      echo '{"siret": "12345678"}' | pimo
+    assertions:
+    - result.code ShouldEqual 0
+    - result.systemoutjson.siret ShouldEqual 123456782
+    - result.systemerr ShouldBeEmpty
+- name: custom universe
+  steps:
+  - script: rm -f masking.yml
+  - script: |-
+      cat > masking.yml <<EOF
+      version: "1"
+      masking:
+        - selector:
+            jsonpath: "siret"
+          mask:
+            luhn:
+              universe: "abcdef"
+      EOF
+  - script: |-
+      echo '{"siret": "abcdef"}' | pimo
+    assertions:
+    - result.code ShouldEqual 0
+    - result.systemoutjson.siret ShouldEqual abcdefe
+    - result.systemerr ShouldBeEmpty
+
+- name: default universe - additional test
+  steps:
+  - script: rm -f masking.yml
+  - script: |-
+      cat > masking.yml <<EOF
+      version: "1"
+      masking:
+        - selector:
+            jsonpath: "siret"
+          mask:
+            luhn: {}
+      EOF
+  - script: |-
+      echo '{"siret": "1234567821234"}' | pimo
+    assertions:
+    - result.code ShouldEqual 0
+    - result.systemoutjson.siret ShouldEqual 12345678212346
+    - result.systemerr ShouldBeEmpty