Merge pull request #13 from PayU/fix-edge-case-json-parse

feat: add special escaped cases

Author: shyimo

README.md

@@ -2,32 +2,36 @@
 
 [![Known Vulnerabilities](https://snyk.io//test/github/PayU/fluent-plugin-masking/badge.svg?targetFile=Gemfile.lock)](https://snyk.io//test/github/PayU/fluent-plugin-masking?targetFile=Gemfile.lock) [![Build Status](https://travis-ci.com/PayU/fluent-plugin-masking.svg?branch=master)](https://travis-ci.com/PayU/fluent-plugin-masking)
 
-## Overview
+# Overview
 Fluentd filter plugin to mask sensitive or privacy records with `*******` in place of the original value. This data masking plugin protects data such as name, email, phonenumber, address, and any other field you would like to mask.
 
-## Requirements
+# Requirements
 | fluent-plugin-masking    | fluentd    | ruby   |
 | ---------------------    | ---------- | ------ |
 | 1.2.x                    | 	>= v0.14.0 | >= 2.5 |
 
 
-## Installation
+# Installation
 Install with gem:
 
-`gem install fluent-plugin-masking`
+`fluent-gem install fluent-plugin-masking`
 
-## Setup
+# Setup
 In order to setup this plugin, the parameter `fieldsToMaskFilePath` needs to be a valid path to a file containing a list of all the fields to mask. The file should have a unique field on each line. These fields **are** case-sensitive (`Name` != `name`).
 
-In addition, there's an optional parameter called `fieldsToExcludeJSONPaths` which receives as input a comma separated string of JSON fields that should be excluded in the masking procedure. Nested JSON fields are supported by `dot notation` (i.e: `path.to.excluded.field.in.record.nestedExcludedField`)
-The JSON fields that are excluded are comma separated.  
-This can be used for logs of registration services or audit log entries which do not need to be masked.  
-This is configured as shown below:
+### Optional configuration
+ - `fieldsToExcludeJSONPaths` - this field receives as input a comma separated string of JSON fields that should be excluded in the masking procedure. Nested JSON fields are supported by `dot notation` (i.e: `path.to.excluded.field.in.record.nestedExcludedField`) The JSON fields that are excluded are comma separated.  
+This can be used for logs of registration services or audit log entries which do not need to be masked.
+
+- `handleSpecialEscapedJsonCases` - a boolean value that try to fix special escaped json cases. this feature is currently on alpha stage (default: false). for more details about thoose special cases see [Special Json Cases](#Special-escaped-json-cases-handling)
+
+An example with optional configuration parameters:
 ```
 <filter "**">
   @type masking
   fieldsToMaskFilePath "/path/to/fields-to-mask-file"
   fieldsToExcludeJSONPaths "excludedField,exclude.path.nestedExcludedField"
+  handleSpecialEscapedJsonCases true
 </filter>
 ```
 
@@ -38,7 +42,7 @@ email
 phone/i # the '/i' suffix will make sure phone field will be case insensitive
 ```
 
-## Quick Guide
+# Quick Guide
 
 ### Configuration:
 ```
@@ -98,10 +102,12 @@ echo '{ :body => "{\"first_name\":\"mickey\", \"type\":\"puggle\", \"last_name\"
 2019-12-01 14:25:53.385681000 +0300 maskme: {"message":"{ :body => \"{\\\"first_name\\\":\\\"mickey\\\", \\\"type\\\":\\\"puggle\\\", \\\"last_name\\\":\\\"the-dog\\\", \\\"password\\\":\\\"*******\\\"}\"}"}
 ```
 
-
-### Run Unit Tests
+# Run Unit Tests
 ```
 gem install bundler
 bundle install
 ruby -r ./test/*.rb
 ```
+
+# Special escaped json cases handling
+

lib/fluent/plugin/filter_masking.rb

@@ -30,14 +30,20 @@ def maskRecord(record)
         end
         begin
           recordStr = record.to_s
+
+          if @handleSpecialEscapedJsonCases == true
+            @specialEscapedJsonRegexs.each do | regex, replace |
+              recordStr = recordStr.gsub(regex, replace)
+            end
+          end  
+          
           @fields_to_mask_regex.each do | fieldToMaskRegex, fieldToMaskRegexStringReplacement |
             if !(excludedFields.include? @fields_to_mask_keys[fieldToMaskRegex])
               recordStr = recordStr.gsub(fieldToMaskRegex, fieldToMaskRegexStringReplacement) 
             end
           end
-
+          
           maskedRecord = strToHash(recordStr)
-
         rescue Exception => e
           $log.error "Failed to mask record: #{e}"
         end
@@ -51,13 +57,19 @@ def initialize
         @fields_to_mask_regex = {}
         @fields_to_mask_keys = {}
         @fieldsToExcludeJSONPathsArray = []
+
+        @handleSpecialEscapedJsonCases = false
+        @specialEscapedJsonRegexs = {
+          Regexp.new(/,(( *)(\\*)("*)( *)),/m) => "\1,"
+        }
       end
 
       # this method only called ones (on startup time)
       def configure(conf)
         super
         fieldsToMaskFilePath = conf['fieldsToMaskFilePath']
         fieldsToExcludeJSONPaths = conf['fieldsToExcludeJSONPaths']
+        handleSpecialCases = conf['handleSpecialEscapedJsonCases']
 
         if fieldsToExcludeJSONPaths != nil && fieldsToExcludeJSONPaths.size() > 0 
           fieldsToExcludeJSONPaths.split(",").each do | field |
@@ -80,12 +92,12 @@ def configure(conf)
             if value.end_with? "/i"
               # case insensitive
               value = value.delete_suffix('/i')
-              hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/mi)
-              innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+"\s*:\s*(\\+|\{).+?((?=(})|,( *|)(\s|\\+)\"(}*))|(?=}"$)|("}(?!\"|\\)))/mi)
+              hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/mi) # mask element in hash object
+              innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+":\\+.+?((?=(})|,( *|)(\s|\\+)\")|(?=}"$))/mi) # mask element in json string using capture groups that count the level of escaping inside the json string
             else
               # case sensitive
-              hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m)
-              innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+"\s*:\s*(\\+|\{).+?((?=(})|,( *|)(\s|\\+)\"(}*))|(?=}"$)|("}(?!\"|\\)))/m)
+              hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m) # mask element in hash object
+              innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+":\\+.+?((?=(})|,( *|)(\s|\\+)\")|(?=}"$))/m) # mask element in json string using capture groups that count the level of escaping inside the json string
             end
 
             @fields_to_mask.push(value)
@@ -100,6 +112,10 @@ def configure(conf)
           end
         end
 
+        # if true, each record (a json record), will be checked for a special escaped json cases
+        # any found case will be 'gsub' with the right solution 
+        @handleSpecialEscapedJsonCases = handleSpecialCases != nil && handleSpecialCases.casecmp("true") == 0
+
         puts "black list fields:"
         puts @fields_to_mask
       end

test/fields-to-mask

@@ -4,4 +4,4 @@ last_name
 street
 number
 password
-json_str_field
+cookie

test/test_filter_masking.rb

@@ -4,7 +4,6 @@
 require "fluent/test/helpers"
 require "./lib/fluent/plugin/filter_masking.rb"
 
-
 MASK_STRING = "*******"
 
 class YourOwnFilterTest < Test::Unit::TestCase
@@ -28,6 +27,13 @@ def setup
     fieldsToMaskFilePath test/fields-to-mask-insensitive
   ]
 
+  # configuration for special json escaped cases
+  CONFIG_SPECIAL_CASES = %[
+    fieldsToMaskFilePath test/fields-to-mask
+    fieldsToExcludeJSONPaths excludedField,exclude.path.nestedExcludedField
+    handleSpecialEscapedJsonCases true
+  ]
+
   def create_driver(conf = CONFIG)
     Fluent::Test::Driver::Filter.new(Fluent::Plugin::MaskingFilter).configure(conf)
   end
@@ -102,6 +108,7 @@ def filter(config, messages)
       filtered_records = filter(conf, messages)
       assert_equal(expected, filtered_records)
     end
+
     test 'mask field in hash object with exclude' do
       conf = CONFIG
       messages = [
@@ -113,6 +120,7 @@ def filter(config, messages)
       filtered_records = filter(conf, messages)
       assert_equal(expected, filtered_records)
     end
+    
     test 'mask field in hash object with nested exclude' do
       conf = CONFIG
       messages = [
@@ -149,37 +157,6 @@ def filter(config, messages)
       assert_equal(expected, filtered_records)
     end
 
-    test 'mask field which is inner json string field (should mask the whole object)' do
-      conf = CONFIG
-      messages = [
-        {
-          :body => { 
-            :action_name => "some_action", 
-            :action_type => "some type",
-            :request => {
-              :body_str => "{\"str_field\":\"mickey\",\"json_str_field\": {\"id\":\"ed8a8378-3235-4923-b802-7700167d1870\"},\"not_mask\":\"some_value\"}"
-            }
-          },
-          :timestamp => "2020-06-08T16:00:57.341Z"
-        }
-      ]
-
-      expected = [
-        {
-          :body => { 
-            :action_name => "some_action", 
-            :action_type => "some type",
-            :request => {
-              :body_str => "{\"str_field\":\"mickey\",\"json_str_field\":\"*******\",\"not_mask\":\"some_value\"}"
-            }
-          },
-          :timestamp => "2020-06-08T16:00:57.341Z"
-        }
-      ]
-
-      filtered_records = filter(conf, messages)
-      assert_equal(expected, filtered_records)
-    end
   end
 
   sub_test_case 'plugin will mask all fields that need masking - case INSENSITIVE fields' do
@@ -223,7 +200,7 @@ def filter(config, messages)
     test 'mask case insensitive and case sensitive field in nested json escaped string' do
       conf = CONFIG_CASE_INSENSITIVE
       messages = [
-        { :body => "{\"firsT_naMe\":\"mickey\",\"last_name\":\"the-dog\",\"address\":\"{\\\"Street\":\\\"Austin\\\",\\\"number\":\\\"89\\\"}\", \"type\":\"puggle\"}" } 
+        { :body => "{\"firsT_naMe\":\"mickey\",\"last_NAME\":\"the-dog\",\"address\":\"{\\\"Street\":\\\"Austin\\\",\\\"number\":\\\"89\\\"}\", \"type\":\"puggle\"}" } 
       ]
       expected = [
         { :body => "{\"firsT_naMe\":\"mickey\",\"last_name\":\"*******\",\"address\":\"{\\\"street\\\":\\\"*******\\\",\\\"number\\\":\\\"*******\\\"}\", \"type\":\"puggle\"}" }
@@ -234,4 +211,17 @@ def filter(config, messages)
 
   end
 
+  sub_test_case 'plugin will mask all fields that need masking - special json escaped cases' do
+    test 'mask field in nested json escaped string when one of the values ends with "," (the value for "some_custom" field)' do
+      conf = CONFIG_SPECIAL_CASES
+      messages = [
+        { :body => "{\"first_name\":\"mickey\",\"last_name\":\"the-dog\",\"address\":\"{\\\"street\":\\\"Austin\\\",\\\"number\":\\\"89\\\"}\", \"type\":\"puggle\", \"cookie\":\"some_custom=,,live,default,,2097403972,2.22.242.38,\", \"city\":\"new york\"}" } 
+      ]
+      expected = [
+        { :body => "{\"first_name\":\"*******\",\"last_name\":\"*******\",\"address\":\"{\\\"street\\\":\\\"*******\\\",\\\"number\\\":\\\"*******\\\"}\", \"type\":\"puggle\", \"cookie\":\"*******\", \"city\":\"new york\"}" }
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
+  end  
 end