Skip to content

Commit 9c914c1

Browse files
TheTeaCatTheTeaCat
authored
Merge pull request #43 from FireTail-io/strip-values
Strip values
2 parents 45f4dbb + bd7aad2 commit 9c914c1

7 files changed

+13051
-24
lines changed

‎logging/batch_logger.go

+74-13
Original file line numberDiff line numberDiff line change
@@ -7,28 +7,34 @@ import (
77

88
// A batchLogger receives log entries via its Enqueue method & arranges them into batches that it then passes to its batchHandler
99
type batchLogger struct {
10-
queue chan *LogEntry // A channel down which LogEntrys will be queued to be sent to Firetail
11-
maxBatchSize int // The maximum size of a batch in bytes
12-
maxLogAge time.Duration // The maximum age of a log item to hold onto
13-
batchCallback func([][]byte) // A handler that takes a batch of log entries as a slice of slices of bytes & sends them to Firetail
10+
queue chan *LogEntry // A channel down which LogEntrys will be queued to be sent to Firetail
11+
maxBatchSize int // The maximum size of a batch in bytes
12+
maxLogAge time.Duration // The maximum age of a log item to hold onto
13+
batchCallback func([][]byte) // A handler that takes a batch of log entries as a slice of slices of bytes & sends them to Firetail
14+
redactRequestBodies bool // Whether or not to redact request bodies
15+
redactResponseBodies bool // Whether or not to redact response bodies
1416
}
1517

1618
// BatchLoggerOptions is an options struct used by the NewBatchLogger constructor
1719
type BatchLoggerOptions struct {
18-
MaxBatchSize int // The maximum size of a batch in bytes
19-
MaxLogAge time.Duration // The maximum age of a log item in a batch - once an item is older than this, the batch is passed to the callback
20-
LogApiKey string // The API key used by the default BatchCallback used to send logs to the Firetail logging API
21-
LogApiUrl string // The URL of the Firetail logging API endpoint to send log entries to
22-
BatchCallback func([][]byte) // An optional callback to which batches will be passed; the default callback sends logs to the Firetail logging API
20+
MaxBatchSize int // The maximum size of a batch in bytes
21+
MaxLogAge time.Duration // The maximum age of a log item in a batch - once an item is older than this, the batch is passed to the callback
22+
LogApiKey string // The API key used by the default BatchCallback used to send logs to the Firetail logging API
23+
LogApiUrl string // The URL of the Firetail logging API endpoint to send log entries to
24+
RedactJSONRequestBodies bool // Whether or not values in the request payloads should be redacted, assuming they're JSON
25+
RedactJSONResponseBodies bool // Whether or not values in the response payloads should be redacted, assuming they're JSON
26+
BatchCallback func([][]byte) // An optional callback to which batches will be passed; the default callback sends logs to the Firetail logging API
2327
}
2428

2529
// NewBatchLogger creates a new batchLogger with the provided options
2630
func NewBatchLogger(options BatchLoggerOptions) *batchLogger {
2731
newLogger := &batchLogger{
28-
queue: make(chan *LogEntry),
29-
maxBatchSize: options.MaxBatchSize,
30-
maxLogAge: options.MaxLogAge,
31-
batchCallback: options.BatchCallback,
32+
queue: make(chan *LogEntry),
33+
maxBatchSize: options.MaxBatchSize,
34+
maxLogAge: options.MaxLogAge,
35+
batchCallback: options.BatchCallback,
36+
redactRequestBodies: options.RedactJSONRequestBodies,
37+
redactResponseBodies: options.RedactJSONResponseBodies,
3238
}
3339

3440
if options.BatchCallback == nil {
@@ -61,6 +67,24 @@ func (l *batchLogger) worker() {
6167
// Read a new entry from the queue if there's one available
6268
select {
6369
case newEntry := <-l.queue:
70+
// Redact request/response bodies if this feature is enabled
71+
if l.redactRequestBodies {
72+
redactedRequestBody, err := redactJson([]byte(newEntry.Request.Body))
73+
if err != nil {
74+
newEntry.Request.Body = ""
75+
} else {
76+
newEntry.Request.Body = string(redactedRequestBody)
77+
}
78+
}
79+
if l.redactResponseBodies {
80+
redactedResponseBody, err := redactJson([]byte(newEntry.Response.Body))
81+
if err != nil {
82+
newEntry.Response.Body = ""
83+
} else {
84+
newEntry.Response.Body = string(redactedResponseBody)
85+
}
86+
}
87+
6488
// Marshal the entry to bytes...
6589
entryBytes, err := json.Marshal(newEntry)
6690
if err != nil {
@@ -126,3 +150,40 @@ func (l *batchLogger) worker() {
126150
time.Sleep(1)
127151
}
128152
}
153+
154+
func redactJson(v []byte) ([]byte, error) {
155+
var unmarshalledJSON interface{}
156+
err := json.Unmarshal(v, &unmarshalledJSON)
157+
if err != nil {
158+
return nil, err
159+
}
160+
var redact func(interface{}) interface{}
161+
redact = func(v interface{}) interface{} {
162+
switch u := v.(type) {
163+
case map[string]interface{}:
164+
for k, w := range u {
165+
u[k] = redact(w)
166+
}
167+
return u
168+
case []interface{}:
169+
for i, w := range u {
170+
u[i] = redact(w)
171+
}
172+
return u
173+
case string:
174+
return ""
175+
case float64:
176+
return 0
177+
case bool:
178+
return true
179+
default:
180+
return nil
181+
}
182+
}
183+
redactedJSON := redact(unmarshalledJSON)
184+
redactedBytes, err := json.Marshal(redactedJSON)
185+
if err != nil {
186+
return nil, err
187+
}
188+
return redactedBytes, nil
189+
}

‎logging/batch_logger_test.go

+155-6
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package logging
33
import (
44
"encoding/json"
55
"math/rand"
6+
"os"
67
"strings"
78
"testing"
89
"time"
@@ -11,10 +12,12 @@ import (
1112
"github.com/stretchr/testify/require"
1213
)
1314

14-
func SetupLogger(batchChannel chan *[][]byte, maxBatchSize int, maxLogAge time.Duration) *batchLogger {
15+
func SetupLogger(batchChannel chan *[][]byte, maxBatchSize int, maxLogAge time.Duration, redactRequests bool, redactResponses bool) *batchLogger {
1516
batchLogger := NewBatchLogger(BatchLoggerOptions{
16-
MaxBatchSize: maxBatchSize,
17-
MaxLogAge: maxLogAge,
17+
MaxBatchSize: maxBatchSize,
18+
MaxLogAge: maxLogAge,
19+
RedactJSONRequestBodies: redactRequests,
20+
RedactJSONResponseBodies: redactResponses,
1821
})
1922

2023
// Replace the batchHandler with a custom one to throw the batches into a queue that we can receive from for testing
@@ -29,7 +32,7 @@ func TestOldLogIsSentImmediately(t *testing.T) {
2932
const MaxLogAge = time.Minute
3033

3134
batchChannel := make(chan *[][]byte, 2)
32-
batchLogger := SetupLogger(batchChannel, 1024*512, MaxLogAge)
35+
batchLogger := SetupLogger(batchChannel, 1024*512, MaxLogAge, false, false)
3336

3437
// Create a test log entry & enqueue it
3538
testLogEntry := LogEntry{
@@ -58,7 +61,7 @@ func TestBatchesDoNotExceedMaxSize(t *testing.T) {
5861

5962
// Buffer our batchChannel with TestLogEntryCount spaces (worst case, each entry ends up in its own batch)
6063
batchChannel := make(chan *[][]byte, TestLogEntryCount)
61-
batchLogger := SetupLogger(batchChannel, MaxBatchSize, time.Second)
64+
batchLogger := SetupLogger(batchChannel, MaxBatchSize, time.Second, false, false)
6265

6366
// Create a bunch of test entries
6467
testLogEntries := []*LogEntry{}
@@ -124,7 +127,7 @@ func TestOldLogTriggersBatch(t *testing.T) {
124127
const MaxLogAge = time.Minute
125128

126129
batchChannel := make(chan *[][]byte, 2)
127-
batchLogger := SetupLogger(batchChannel, 1024*512, MaxLogAge)
130+
batchLogger := SetupLogger(batchChannel, 1024*512, MaxLogAge, false, false)
128131

129132
// Create ExpectedLogEntryCount-1 test log entries (the last one will trigger a batch)
130133
testLogEntries := []*LogEntry{}
@@ -172,3 +175,149 @@ func TestOldLogTriggersBatch(t *testing.T) {
172175
// Assert that the batch has all the same byte slices as the expected batch
173176
require.ElementsMatch(t, expectedBatch, *batch)
174177
}
178+
179+
func TestRedaction(t *testing.T) {
180+
testCases := []struct {
181+
enableRequestRedaction bool
182+
enableResponseRedaction bool
183+
}{
184+
{false, false},
185+
{false, true},
186+
{true, false},
187+
{true, true},
188+
}
189+
for _, testCase := range testCases {
190+
const MaxLogAge = time.Minute
191+
192+
batchChannel := make(chan *[][]byte, 2)
193+
batchLogger := SetupLogger(batchChannel, 1024*512, MaxLogAge, testCase.enableRequestRedaction, testCase.enableResponseRedaction)
194+
195+
const testRequestBody = "{\"foo\":\"bar\"}"
196+
const expectedRedactedRequestBody = "{\"foo\":\"\"}"
197+
const testResponseBody = "{\"baz\":\"qux\"}"
198+
const expectedRedactedResponseBody = "{\"baz\":\"\"}"
199+
200+
// Create a test log entry & enqueue it
201+
testLogEntry := LogEntry{
202+
Request: Request{
203+
Body: testRequestBody,
204+
},
205+
Response: Response{
206+
Body: testResponseBody,
207+
},
208+
DateCreated: time.Now().UnixMilli() - MaxLogAge.Milliseconds()*2,
209+
}
210+
211+
// Enqueue the test log entry
212+
batchLogger.Enqueue(&testLogEntry)
213+
214+
// There should then be a batch in the channel for us to receive
215+
batch := <-batchChannel
216+
217+
// Channel should be empty now, as it should only have had one batch in it
218+
assert.Equal(t, 0, len(batchChannel))
219+
220+
// Assert the batch had one log entry in it
221+
require.Equal(t, 1, len(*batch))
222+
223+
// Unmarshal the log and assert its request/response bodies were redacted as expected
224+
var logEntry LogEntry
225+
err := json.Unmarshal((*batch)[0], &logEntry)
226+
require.Nil(t, err)
227+
if testCase.enableRequestRedaction {
228+
assert.Equal(t, expectedRedactedRequestBody, logEntry.Request.Body)
229+
} else {
230+
assert.Equal(t, testRequestBody, logEntry.Request.Body)
231+
}
232+
if testCase.enableResponseRedaction {
233+
assert.Equal(t, expectedRedactedResponseBody, logEntry.Response.Body)
234+
} else {
235+
assert.Equal(t, testResponseBody, logEntry.Response.Body)
236+
}
237+
}
238+
}
239+
240+
func BenchmarkRedactFiretailJSLibPackageLockJson1536B(b *testing.B) {
241+
benchmarkRedactJson(b, "testdata/package-lock-firetail-js-lib-1536B.json")
242+
}
243+
244+
func BenchmarkRedactFiretailJSLibPackageLockJson278KiB(b *testing.B) {
245+
benchmarkRedactJson(b, "testdata/package-lock-firetail-js-lib-478KiB.json")
246+
}
247+
248+
func benchmarkRedactJson(b *testing.B, testFile string) {
249+
testJson, err := os.ReadFile(testFile)
250+
if err != nil {
251+
b.Error(err.Error())
252+
}
253+
b.ResetTimer()
254+
b.StopTimer()
255+
for n := 0; n < b.N; n++ {
256+
// redactJson mutates the input, so we need to copy it each time
257+
testJsonCopy := make([]byte, len(testJson))
258+
copy(testJsonCopy, testJson)
259+
b.StartTimer()
260+
_, err = redactJson(testJsonCopy)
261+
b.StopTimer()
262+
if err != nil {
263+
b.Error(err.Error())
264+
}
265+
}
266+
}
267+
268+
func TestRedactJson(t *testing.T) {
269+
testCases := []struct {
270+
testJson string
271+
expectedJson string
272+
}{
273+
{"\"foo\"", "\"\""},
274+
{"3.14159", "0"},
275+
{"true", "true"},
276+
{"false", "true"},
277+
{"null", "null"},
278+
{"{}", "{}"},
279+
{"[]", "[]"},
280+
// Objects with values of different types
281+
{"{\"foo\":\"bar\"}", "{\"foo\":\"\"}"},
282+
{"{\"foo\":3.14159}", "{\"foo\":0}"},
283+
{"{\"foo\":true}", "{\"foo\":true}"},
284+
{"{\"foo\":false}", "{\"foo\":true}"},
285+
{"{\"foo\":null}", "{\"foo\":null}"},
286+
// Lists with values of different types
287+
{"[\"foo\"]", "[\"\"]"},
288+
{"[3.14159]", "[0]"},
289+
{"[false]", "[true]"},
290+
{"[true]", "[true]"},
291+
{"[null]", "[null]"},
292+
// Lists with two values of the same type
293+
{"[\"foo\",\"bar\"]", "[\"\",\"\"]"},
294+
{"[3.14159,2.71828]", "[0,0]"},
295+
{"[false,false]", "[true,true]"},
296+
{"[false,true]", "[true,true]"},
297+
{"[true,false]", "[true,true]"},
298+
{"[true,true]", "[true,true]"},
299+
{"[null,null]", "[null,null]"},
300+
// Lists with three values of the same type
301+
{"[\"foo\",\"bar\",\"baz\"]", "[\"\",\"\",\"\"]"},
302+
{"[3.14159,2.71828,1.61803]", "[0,0,0]"},
303+
{"[false,false,false]", "[true,true,true]"},
304+
{"[false,false,true]", "[true,true,true]"},
305+
{"[false,true,false]", "[true,true,true]"},
306+
{"[false,true,true]", "[true,true,true]"},
307+
{"[true,false,false]", "[true,true,true]"},
308+
{"[true,false,true]", "[true,true,true]"},
309+
{"[true,true,false]", "[true,true,true]"},
310+
{"[true,true,true]", "[true,true,true]"},
311+
{"[null,null,null]", "[null,null,null]"},
312+
// Lists with mixed types
313+
{"[\"foo\",3.14159,true,false,null]", "[\"\",0,true,true,null]"},
314+
// Some more complex examples
315+
{"{\"foo\":[\"bar\",3.14159,true,false,null]}", "{\"foo\":[\"\",0,true,true,null]}"},
316+
{"[{\"foo\":\"bar\"},{\"baz\":3.14159},{\"qux\":true},{\"quux\":false},{\"corge\":null}]", "[{\"foo\":\"\"},{\"baz\":0},{\"qux\":true},{\"quux\":true},{\"corge\":null}]"},
317+
}
318+
for _, testCase := range testCases {
319+
result, err := redactJson([]byte(testCase.testJson))
320+
require.Nil(t, err)
321+
assert.Equal(t, testCase.expectedJson, string(result))
322+
}
323+
}

‎logging/testdata/package-lock-firetail-js-lib-1536B.json

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
{
2+
"name": "@public.firetail.io/firetail-api",
3+
"version": "0.5.2",
4+
"lockfileVersion": 2,
5+
"requires": true,
6+
"packages": {
7+
"": {
8+
"name": "@public.firetail.io/firetail-api",
9+
"version": "0.5.2",
10+
"license": "LGPL-3.0",
11+
"dependencies": {
12+
"@apidevtools/swagger-parser": "9.1.0",
13+
"ajv": "8.12.0",
14+
"lodash": "4.17.21",
15+
"node-fetch": "2.6.9",
16+
"openapi-request-validator": "12.1.0",
17+
"openapi-response-validator": "12.1.0",
18+
"openapi-security-handler": "12.1.0",
19+
"type-is": "1.6.18"
20+
},
21+
"devDependencies": {
22+
"@types/aws-lambda": "8.10.109",
23+
"@types/express-serve-static-core": "4.17.33",
24+
"@types/http-server": "0.12.1",
25+
"@types/jest": "29.4.0",
26+
"@types/node": "18.11.18",
27+
"@typescript-eslint/eslint-plugin": "5.51.0",
28+
"@typescript-eslint/parser": "5.51.0",
29+
"eslint": "8.34.0",
30+
"eslint-config-standard": "17.0.0",
31+
"eslint-plugin-import": "2.27.5"
32+
},
33+
"engines": {
34+
"node": ">=14.0.0"
35+
}
36+
}
37+
},
38+
"dependencies": {
39+
"@ampproject/remapping": {
40+
"version": "2.2.0",
41+
"resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz",
42+
"integrity": "sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==",
43+
"dev": true,
44+
"requires": {
45+
"@jridgewell/gen-mapping": "^0.1.0",
46+
"@jridgewell/trace-mapping": "^0.3.9"
47+
}
48+
}
49+
}
50+
}

0 commit comments

Comments
 (0)