POA-2742 Incorporate dynamic redaction configuration (#64)

Implements
https://postmanlabs.atlassian.net/wiki/spaces/PO/pages/5818057613/Plan+API+support+for+Fields+to+redact+setting#Agent-changes.

Author: liujed

apidump/apidump.go

@@ -23,6 +23,7 @@ import (
 	"github.com/postmanlabs/postman-insights-agent/apispec"
 	"github.com/postmanlabs/postman-insights-agent/architecture"
 	"github.com/postmanlabs/postman-insights-agent/ci"
+	"github.com/postmanlabs/postman-insights-agent/data_masks"
 	"github.com/postmanlabs/postman-insights-agent/deployment"
 	"github.com/postmanlabs/postman-insights-agent/env"
 	"github.com/postmanlabs/postman-insights-agent/location"
@@ -654,6 +655,11 @@ func (a *apidump) Run() error {
 		go a.TelemetryWorker(stop)
 	}
 
+	redactor, err := data_masks.NewRedactor(a.backendSvc, a.learnClient)
+	if err != nil {
+		return errors.Wrapf(err, "unable to instantiate redactor for %s", a.backendSvc)
+	}
+
 	// Start collecting -- set up one or two collectors per interface, depending on whether filters are in use
 	numCollectors := 0
 	for _, filterState := range []filterState{matchedFilter, notMatchedFilter} {
@@ -690,7 +696,17 @@ func (a *apidump) Run() error {
 			} else {
 				var backendCollector trace.Collector
 				if args.Out.AkitaURI != nil {
-					backendCollector = trace.NewBackendCollector(a.backendSvc, backendLrn, a.learnClient, optionals.Some(a.MaxWitnessSize_bytes), summary, args.ReproMode, args.Plugins)
+					backendCollector = trace.NewBackendCollector(
+						a.backendSvc,
+						backendLrn,
+						a.learnClient,
+						redactor,
+						optionals.Some(a.MaxWitnessSize_bytes),
+						summary,
+						args.ReproMode,
+						args.Plugins,
+					)
+
 					collector = backendCollector
 				} else {
 					return errors.Errorf("invalid output location")

trace/obfuscation_config.go renamed to data_masks/redaction_config.go

@@ -1,4 +1,4 @@
-package trace
+package data_masks
 
 import (
 	"embed"
@@ -11,20 +11,20 @@ import (
 )
 
 var (
-	//go:embed obfuscation_config.yaml
-	obfucationFileFS embed.FS
+	//go:embed redaction_config.yaml
+	redactionFileFS embed.FS
 
 	onceConfigLoad sync.Once
-	config         obfuscationConfig
+	config         redactionConfig
 	configErr      error = nil
 )
 
-type obfuscationConfig struct {
+type redactionConfig struct {
 	SensitiveKeys         []string `yaml:"sensitive_keys"`
 	SensitiveValueRegexes []string `yaml:"sensitive_value_regexes"`
 }
 
-func (c *obfuscationConfig) sanitizeConfigData() bool {
+func (c *redactionConfig) sanitizeConfigData() bool {
 
 	// Convert all the keys to lower case and remove duplicates
 	keys := sets.NewSet[string]()
@@ -36,9 +36,9 @@ func (c *obfuscationConfig) sanitizeConfigData() bool {
 	return false
 }
 
-func loadConfigFromFile() (*obfuscationConfig, error) {
+func loadConfigFromFile() (*redactionConfig, error) {
 	onceConfigLoad.Do(func() {
-		data, err := obfucationFileFS.Open("obfuscation_config.yaml")
+		data, err := redactionFileFS.Open("redaction_config.yaml")
 		if err != nil {
 			configErr = errors.Wrap(err, "failed to open config file")
 		}

trace/obfuscation_config.yaml renamed to data_masks/redaction_config.yaml

@@ -0,0 +1,131 @@
+package data_masks
+
+import (
+	"path/filepath"
+	"regexp"
+	"testing"
+
+	"github.com/akitasoftware/akita-ir/go/api_spec"
+	"github.com/akitasoftware/akita-libs/akid"
+	kgxapi "github.com/akitasoftware/akita-libs/api_schema"
+	"github.com/akitasoftware/akita-libs/test"
+	"github.com/akitasoftware/go-utils/optionals"
+	"github.com/golang/mock/gomock"
+	"github.com/golang/protobuf/proto"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	mockrest "github.com/postmanlabs/postman-insights-agent/rest/mock"
+	"github.com/stretchr/testify/assert"
+)
+
+var cmpOptions = []cmp.Option{
+	cmp.Comparer(proto.Equal),
+	cmpopts.SortSlices(sortMethod),
+	cmpopts.SortSlices(sortWitness),
+}
+
+func sortMethod(m1, m2 *api_spec.Method) bool {
+	return proto.MarshalTextString(m1) < proto.MarshalTextString(m2)
+}
+
+func sortWitness(m1, m2 *api_spec.Witness) bool {
+	return proto.MarshalTextString(m1) < proto.MarshalTextString(m2)
+}
+
+func TestRedaction(t *testing.T) {
+	testCases := map[string]struct {
+		agentConfig  optionals.Optional[*kgxapi.FieldRedactionConfig]
+		inputFile    string
+		expectedFile string
+	}{
+		// Expect values with 16-character identifiers to remain unchanged.
+		"16-character identifier": {
+			inputFile:    "002-witness.pb.txt",
+			expectedFile: "002-witness.pb.txt",
+		},
+
+		"default redaction rules": {
+			inputFile:    "003-witness.pb.txt",
+			expectedFile: "003-expected-default-redaction.pb.txt",
+		},
+
+		"agent config: redact by name": {
+			agentConfig: optionals.Some(&kgxapi.FieldRedactionConfig{
+				FieldNames: []string{"by-name"},
+			}),
+			inputFile:    "003-witness.pb.txt",
+			expectedFile: "003-expected-redact-by-name.pb.txt",
+		},
+
+		"agent config: redact by name regexp": {
+			agentConfig: optionals.Some(&kgxapi.FieldRedactionConfig{
+				FieldNameRegexps: []*regexp.Regexp{regexp.MustCompile("nam.*xp$")},
+			}),
+			inputFile:    "003-witness.pb.txt",
+			expectedFile: "003-expected-redact-by-name-regexp.pb.txt",
+		},
+
+		"agent config: redact by name and by name regexp": {
+			agentConfig: optionals.Some(&kgxapi.FieldRedactionConfig{
+				FieldNames:       []string{"by-name"},
+				FieldNameRegexps: []*regexp.Regexp{regexp.MustCompile("nam.*xp$")},
+			}),
+			inputFile:    "003-witness.pb.txt",
+			expectedFile: "003-expected-redact-by-name-and-by-name-regexp.pb.txt",
+		},
+	}
+
+	for testName, testCase := range testCases {
+		func() {
+			ctrl := gomock.NewController(t)
+			mockClient := mockrest.NewMockLearnClient(ctrl)
+			defer ctrl.Finish()
+
+			agentConfig := kgxapi.NewServiceAgentConfig()
+			if fieldsToRedact, exists := testCase.agentConfig.Get(); exists {
+				agentConfig.FieldsToRedact = fieldsToRedact
+			}
+
+			mockClient.
+				EXPECT().
+				GetDynamicAgentConfigForService(gomock.Any(), gomock.Any()).
+				AnyTimes().
+				Return(agentConfig, nil)
+
+			o, err := NewRedactor(akid.GenerateServiceID(), mockClient)
+			assert.NoError(t, err)
+
+			testWitness := test.LoadWitnessFromFileOrDie(filepath.Join("testdata", testCase.inputFile))
+			expectedWitness := test.LoadWitnessFromFileOrDie(filepath.Join("testdata", testCase.expectedFile))
+
+			o.RedactSensitiveData(testWitness.Method)
+
+			if diff := cmp.Diff(expectedWitness, testWitness, cmpOptions...); diff != "" {
+				t.Errorf("found unexpected diff in test case %q:\n%s", testName, diff)
+			}
+		}()
+	}
+}
+
+func BenchmarkRedaction(b *testing.B) {
+	ctrl := gomock.NewController(b)
+	mockClient := mockrest.NewMockLearnClient(ctrl)
+	defer ctrl.Finish()
+
+	mockClient.
+		EXPECT().
+		GetDynamicAgentConfigForService(gomock.Any(), gomock.Any()).
+		AnyTimes().
+		Return(kgxapi.NewServiceAgentConfig(), nil)
+
+	o, err := NewRedactor(akid.GenerateServiceID(), mockClient)
+	assert.NoError(b, err)
+
+	testWitness := test.LoadWitnessFromFileOrDie(filepath.Join("testdata", "001-witness.pb.txt"))
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		o.RedactSensitiveData(testWitness.Method)
+	}
+}