Skip to content

Commit b8f99d3

Browse files
kohendNivLipetz
authored andcommitted
Exclude fields support fixes (#7)
1 parent 7e2f254 commit b8f99d3

File tree

7 files changed

+88
-12
lines changed

7 files changed

+88
-12
lines changed

Gemfile.lock

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
PATH
22
remote: .
33
specs:
4-
fluent-plugin-masking (1.1.0)
4+
fluent-plugin-masking (1.1.1)
55
fluentd (>= 0.14.0)
66

77
GEM
@@ -10,15 +10,15 @@ GEM
1010
concurrent-ruby (1.1.5)
1111
cool.io (1.5.4)
1212
dig_rb (1.0.1)
13-
fluentd (1.7.4)
13+
fluentd (1.8.0)
1414
cool.io (>= 1.4.5, < 2.0.0)
1515
dig_rb (~> 1.0.0)
1616
http_parser.rb (>= 0.5.1, < 0.7.0)
1717
msgpack (>= 1.2.0, < 2.0.0)
1818
serverengine (>= 2.0.4, < 3.0.0)
1919
sigdump (~> 0.2.2)
2020
strptime (>= 0.2.2, < 1.0.0)
21-
tzinfo (~> 2.0)
21+
tzinfo (>= 1.0, < 3.0)
2222
tzinfo-data (~> 1.0)
2323
yajl-ruby (~> 1.0)
2424
http_parser.rb (0.6.0)
@@ -45,11 +45,11 @@ PLATFORMS
4545
ruby
4646

4747
DEPENDENCIES
48-
bundler
48+
bundler (= 1.17.3)
4949
fluent-plugin-masking!
5050
rake (~> 12.0)
5151
test-unit (>= 3.1.0)
5252
test-unit-rr
5353

5454
BUNDLED WITH
55-
2.0.2
55+
1.17.3

fluent-plugin-masking.gemspec

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
2121
spec.required_ruby_version = '>= 2.1'
2222

2323
spec.add_runtime_dependency "fluentd", ">= 0.14.0"
24-
spec.add_development_dependency "bundler"
24+
spec.add_development_dependency "bundler", "1.17.3"
2525
spec.add_development_dependency "rake", "~> 12.0"
2626
spec.add_development_dependency "test-unit", ">= 3.1.0"
2727
spec.add_development_dependency "test-unit-rr"

lib/fluent/plugin/filter_masking.rb

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
require 'fluent/filter'
2+
require './lib/fluent/plugin/helpers.rb'
23

34
module Fluent
45
module Plugin
56
class MaskingFilter < Filter
7+
include Helpers
68
Fluent::Plugin.register_filter("masking", self) # for "@type masking" in configuration
79

810
MASK_STRING = "*******"
@@ -16,13 +18,17 @@ def strToHash(str)
1618
def maskRecord(record)
1719
maskedRecord = record
1820
excludedFields = []
19-
@fieldsToExcludeJSONPathsArray.each do | field |
20-
field_value = record.dig(*field)
21-
if field_value != nil
22-
excludedFields = excludedFields + field_value.split(',')
21+
begin
22+
@fieldsToExcludeJSONPathsArray.each do | field |
23+
field_value = myDig(record, field)
24+
if field_value != nil
25+
excludedFields = excludedFields + field_value.split(',')
26+
end
2327
end
28+
rescue Exception => e
29+
$log.error "Failed to find mask exclude record: #{e}"
2430
end
25-
begin
31+
begin
2632
recordStr = record.to_s
2733
@fields_to_mask_regex.each do | fieldToMaskRegex, fieldToMaskRegexStringReplacement |
2834
if !(excludedFields.include? @fields_to_mask_keys[fieldToMaskRegex])

lib/fluent/plugin/helpers.rb

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
module Helpers
2+
def myDig(input, path)
3+
curr = input
4+
for segment in path do
5+
if curr != nil && curr.is_a?(Hash)
6+
if curr[segment] == nil # segment is not a symbol
7+
curr = curr[segment.to_s] # segment as string
8+
else
9+
curr = curr[segment] # segment as symbol
10+
end
11+
else
12+
return nil
13+
end
14+
end
15+
curr
16+
end
17+
end

lib/fluent/plugin/version.rb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
module FilterMasking
2-
VERSION = "1.1.0"
2+
VERSION = "1.1.1"
33
end

test/test_filter_masking.rb

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,5 +132,16 @@ def filter(config, messages)
132132
filtered_records = filter(conf, messages)
133133
assert_equal(expected, filtered_records)
134134
end
135+
test 'mask field in json string with exclude' do
136+
conf = CONFIG
137+
messages = [
138+
{ :body => "{\"first_name\":\"mickey\",\"last_name\":\"the-dog\", \"type\":\"puggle\"}", :excludedField=>"first_name" }
139+
]
140+
expected = [
141+
{ :body => "{\"first_name\":\"mickey\",\"last_name\":\"*******\", \"type\":\"puggle\"}", :excludedField=>"first_name" }
142+
]
143+
filtered_records = filter(conf, messages)
144+
assert_equal(expected, filtered_records)
145+
end
135146
end
136147
end

test/test_helpers.rb

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
require "test/unit"
2+
require "./lib/fluent/plugin/helpers.rb"
3+
4+
class HelpersTest < Test::Unit::TestCase
5+
m = Class.new do
6+
include Helpers
7+
end.new
8+
sub_test_case "myDig function" do
9+
test "Call function with nil" do
10+
t = m.myDig(nil ,[:a])
11+
assert_equal(t, nil)
12+
end
13+
test "Not found" do
14+
t = m.myDig({:b => "t"},[:a])
15+
assert_equal(t, nil)
16+
end
17+
test "Found symbol" do
18+
t = m.myDig({:a => "t"},[:a])
19+
assert_equal(t, "t")
20+
end
21+
test "Found string when given symbol" do
22+
t = m.myDig({"a" => "t"},[:a])
23+
assert_equal(t, "t")
24+
end
25+
test "Found symbol nested" do
26+
t = m.myDig({:a => {:b => "t"}},[:a, :b])
27+
assert_equal(t, "t")
28+
end
29+
test "Found string when given symbol nested" do
30+
t = m.myDig({"a" => {"b" => "t"}},[:a, :b])
31+
assert_equal(t, "t")
32+
end
33+
test "Found hybrid string/symbol when given symbol nested" do
34+
t = m.myDig({"a" => {:b => "t"}},[:a, :b])
35+
assert_equal(t, "t")
36+
end
37+
test "Does not dig in string" do
38+
t = m.myDig({"a" => {:b => "t"}},[:a, :b, :c])
39+
assert_equal(t, nil)
40+
end
41+
end
42+
end

0 commit comments

Comments
 (0)