Merge pull request #6 from PayU/exclude

Support for exclude fields

Author: NivLipetz

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-masking (1.0.8)
+    fluent-plugin-masking (1.1.0)
       fluentd (>= 0.14.0)
 
 GEM
@@ -10,7 +10,7 @@ GEM
     concurrent-ruby (1.1.5)
     cool.io (1.5.4)
     dig_rb (1.0.1)
-    fluentd (1.7.2)
+    fluentd (1.7.4)
       cool.io (>= 1.4.5, < 2.0.0)
       dig_rb (~> 1.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
@@ -26,7 +26,7 @@ GEM
     power_assert (1.1.5)
     rake (12.3.3)
     rr (1.2.1)
-    serverengine (2.1.1)
+    serverengine (2.2.0)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
     strptime (0.2.3)

README.md

@@ -19,11 +19,15 @@ Install with gem:
 ## Setup
 In order to setup this plugin, the parameter `fieldsToMaskFilePath` needs to be a valid path to a file containing a list of all the fields to mask. The file should have a unique field on each line. These fields **are** case-sensitive (`Name` != `name`).
 
+In addition, there's an optional parameter called `fieldsToExcludeJSONPaths` which receives as input a comma separated string of JSON fields that should be excluded in the masking procedure. Nested JSON fields are supported by `dot notation` (i.e: `path.to.excluded.field.in.record.nestedExcludedField`)
+The JSON fields that are excluded are comma separated.  
+This can be used for logs of registration services or audit log entries which do not need to be masked.  
 This is configured as shown below:
 ```
 <filter "**">
   @type masking
   fieldsToMaskFilePath "/path/to/fields-to-mask-file"
+  fieldsToExcludeJSONPaths "excludedField,exclude.path.nestedExcludedField"
 </filter>
 ```
 
@@ -52,6 +56,7 @@ phone
 <filter "**">
   @type masking
   fieldsToMaskFilePath "/path/to/fields-to-mask-file"
+  fieldsToExcludeJSONPaths "excludedField,exclude.path.nestedExcludedField"
 </filter>
 
 <match "**">
@@ -83,3 +88,12 @@ This sample result is created from the above configuration file `fluent.conf`. A
 ```
 2019-09-15 16:12:50.359191000 +0300 maskme: {"message":"{ :body => \"{\\\"first_name\\\":\\\"*******\\\", \\\"type\\\":\\\"puggle\\\", \\\"last_name\\\":\\\"*******\\\", \\\"password\\\":\\\"*******\\\"}\"}"}
 ```
+
+A sample with exclude in use:
+fluentd -c fluent.conf
+echo '{ :body => "{\"first_name\":\"mickey\", \"type\":\"puggle\", \"last_name\":\"the-dog\", \"password\":\"d0g43u39\"}", "excludeMaskFields"=>"first_name,last_name"}' > /tmp/test.log
+```
+
+```
+2019-12-01 14:25:53.385681000 +0300 maskme: {"message":"{ :body => \"{\\\"first_name\\\":\\\"mickey\\\", \\\"type\\\":\\\"puggle\\\", \\\"last_name\\\":\\\"the-dog\\\", \\\"password\\\":\\\"*******\\\"}\"}"}
+```

lib/fluent/plugin/filter_masking.rb

@@ -15,11 +15,19 @@ def strToHash(str)
       # error safe method - if any error occurs the original record is returned
       def maskRecord(record)
         maskedRecord = record
-        
-        begin
+        excludedFields = []
+        @fieldsToExcludeJSONPathsArray.each do | field |
+          field_value = record.dig(*field)
+          if  field_value != nil
+            excludedFields = excludedFields + field_value.split(',')
+          end
+        end
+        begin 
           recordStr = record.to_s
           @fields_to_mask_regex.each do | fieldToMaskRegex, fieldToMaskRegexStringReplacement |
-            recordStr = recordStr.gsub(fieldToMaskRegex, fieldToMaskRegexStringReplacement) 
+            if !(excludedFields.include? @fields_to_mask_keys[fieldToMaskRegex])
+              recordStr = recordStr.gsub(fieldToMaskRegex, fieldToMaskRegexStringReplacement) 
+            end
           end
 
           maskedRecord = strToHash(recordStr)
@@ -35,12 +43,27 @@ def initialize
         super
         @fields_to_mask = []
         @fields_to_mask_regex = {}
+        @fields_to_mask_keys = {}
+        @fieldsToExcludeJSONPathsArray = []
       end
 
       # this method only called ones (on startup time)
       def configure(conf)
         super
         fieldsToMaskFilePath = conf['fieldsToMaskFilePath']
+        fieldsToExcludeJSONPaths = conf['fieldsToExcludeJSONPaths']
+
+        if fieldsToExcludeJSONPaths != nil && fieldsToExcludeJSONPaths.size() > 0 
+          fieldsToExcludeJSONPaths.split(",").each do | field |
+            # To save splits we'll save the path as an array
+            splitArray = field.split(".")
+            symArray = []
+            splitArray.each do | pathPortion |
+              symArray.push(pathPortion.to_sym)
+            end
+            @fieldsToExcludeJSONPathsArray.push(symArray)
+          end
+        end
 
         File.open(fieldsToMaskFilePath, "r") do |f|
           f.each_line do |line|
@@ -54,10 +77,12 @@ def configure(conf)
             hashObjectRegex = Regexp.new(/(?::#{value}=>")(.*?)(?:")/m) # mask element in hash object
             hashObjectRegexStringReplacement = ":#{value}=>\"#{MASK_STRING}\""
             @fields_to_mask_regex[hashObjectRegex] = hashObjectRegexStringReplacement
+            @fields_to_mask_keys[hashObjectRegex] = value
 
             innerJSONStringRegex = Regexp.new(/(\\+)"#{value}\\+":\\+.+?((?=(})|,( *|)(\s|\\+)\")|(?=}"$))/m) # mask element in json string using capture groups that count the level of escaping inside the json string
             innerJSONStringRegexStringReplacement = "\\1\"#{value}\\1\":\\1\"#{MASK_STRING}\\1\""
             @fields_to_mask_regex[innerJSONStringRegex] = innerJSONStringRegexStringReplacement
+            @fields_to_mask_keys[innerJSONStringRegex] = value
           end
         end
 

lib/fluent/plugin/version.rb

@@ -1,3 +1,3 @@
 module FilterMasking
-    VERSION = "1.0.8"
+    VERSION = "1.1.0"
 end 

test/test_filter_masking.rb

@@ -17,6 +17,12 @@ def setup
   # default configuration for tests
   CONFIG = %[
     fieldsToMaskFilePath test/fields-to-mask
+    fieldsToExcludeJSONPaths excludedField,exclude.path.nestedExcludedField
+  ]
+
+  # configuration for tests without exclude parameter
+  CONFIG_NO_EXCLUDE = %[
+    fieldsToMaskFilePath test/fields-to-mask
   ]
 
   def create_driver(conf = CONFIG)
@@ -35,7 +41,7 @@ def filter(config, messages)
 
   sub_test_case 'plugin will mask all fields that need masking' do
     test 'mask field in hash object' do
-      conf = CONFIG
+      conf = CONFIG_NO_EXCLUDE
       messages = [
         {:not_masked_field=>"mickey-the-dog", :email=>"mickey-the-dog@zooz.com"}
       ]
@@ -93,5 +99,38 @@ def filter(config, messages)
       filtered_records = filter(conf, messages)
       assert_equal(expected, filtered_records)
     end
+    test 'mask field in hash object with exclude' do
+      conf = CONFIG
+      messages = [
+        {:not_masked_field=>"mickey-the-dog", :email=>"mickey-the-dog@zooz.com", :first_name=>"Micky", :excludedField=>"first_name"}
+      ]
+      expected = [
+        {:not_masked_field=>"mickey-the-dog", :email=>MASK_STRING, :first_name=>"Micky", :excludedField=>"first_name"}
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
+    test 'mask field in hash object with nested exclude' do
+      conf = CONFIG
+      messages = [
+        {:not_masked_field=>"mickey-the-dog", :last_name=>"the dog", :email=>"mickey-the-dog@zooz.com", :first_name=>"Micky",  :exclude=>{:path=>{:nestedExcludedField=>"first_name,last_name"}}}
+      ]
+      expected = [
+        {:not_masked_field=>"mickey-the-dog", :last_name=>"the dog", :email=>MASK_STRING, :first_name=>"Micky", :exclude=>{:path=>{:nestedExcludedField=>"first_name,last_name"}}}
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
+    test 'mask field in hash object with base and nested exclude' do
+      conf = CONFIG
+      messages = [
+        {:not_masked_field=>"mickey-the-dog", :last_name=>"the dog", :email=>"mickey-the-dog@zooz.com", :first_name=>"Micky", :excludedField=>"first_name", :exclude=>{:path=>{:nestedExcludedField=>"last_name"}}}
+      ]
+      expected = [
+        {:not_masked_field=>"mickey-the-dog", :last_name=>"the dog", :email=>MASK_STRING, :first_name=>"Micky", :excludedField=>"first_name", :exclude=>{:path=>{:nestedExcludedField=>"last_name"}}}
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
   end
 end