Fix concurrency issues for parser

Signed-off-by: Quoc Trung Hoang <quoc-trung.hoang@etu.utc.fr>

Author: ichbinfrog

go.sum

@@ -257,7 +257,6 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/model/parser_ctx_kv.go

@@ -26,7 +26,6 @@ type pair struct {
 // 		<equal>: '='
 // 		<value>: \S*
 type kvParser struct {
-	pairs      map[string]pair
 	keyBag     *[]string
 	threshold  float32
 	matcher    *regexp.Regexp
@@ -39,12 +38,12 @@ func newKVParser(keyRegexp, equal, valRegexp, varRegexp string, bag *[]string) *
 		matcher:    regexp.MustCompile(keyRegexp + equal + valRegexp),
 		varMatcher: regexp.MustCompile(varRegexp),
 		keyBag:     bag,
-		pairs:      make(map[string]pair),
 	}
 }
 
 // Parse reads file line by line to scan the KV file
 func (k *kvParser) Parse(reader io.Reader, leakChan chan Leak, file string, rule *CtxParserRule) {
+	pairs := make(map[string]pair)
 	lineNum := 0
 	buf := bufio.NewScanner(reader)
 	for buf.Scan() {
@@ -58,29 +57,41 @@ func (k *kvParser) Parse(reader io.Reader, leakChan chan Leak, file string, rule
 		if len(value) == 0 {
 			continue
 		}
-		k.pairs[string(match[1])] = pair{
+		pairs[string(match[1])] = pair{
 			value:    match[2],
 			line:     lineNum,
 			affected: line,
 		}
 	}
-	for key, pair := range k.pairs {
+	for key, pair := range pairs {
 		for _, keyword := range *k.keyBag {
 			if strings.Contains(
 				strings.ToLower(key),
 				keyword,
 			) {
 				npair := &pair
-				npair.threats = k.parseVariable(pair.value)
+				matches := k.varMatcher.FindAllSubmatch(pair.value, -1)
+				innerCalls := 0
+				for _, match := range matches {
+					if len(match) >= 2 {
+						for key := range pairs {
+							if strings.Compare(key, string(match[2])) == 0 {
+								innerCalls++
+								break
+							}
+						}
+					}
+				}
+				npair.threats = innerCalls
 				if npair.threats == 0 {
 					npair.threats = 1
 				}
-				k.pairs[key] = *npair
+				pairs[key] = *npair
 				break
 			}
 		}
 	}
-	for _, pair := range k.pairs {
+	for _, pair := range pairs {
 		if pair.threats != 0 {
 			leakChan <- FileLeak{
 				File:          file,
@@ -93,22 +104,6 @@ func (k *kvParser) Parse(reader io.Reader, leakChan chan Leak, file string, rule
 	}
 }
 
-func (k *kvParser) parseVariable(value []byte) int {
-	matches := k.varMatcher.FindAllSubmatch(value, -1)
-	innerCalls := 0
-	for _, match := range matches {
-		if len(match) >= 2 {
-			for key := range k.pairs {
-				if strings.Compare(key, string(match[2])) == 0 {
-					innerCalls++
-					break
-				}
-			}
-		}
-	}
-	return innerCalls
-}
-
 // NewEnvParser returns a new environ file leak parser
 func newEnvParser(keyBag *[]string) *kvParser {
 	return newKVParser(

pkg/model/parser_yaml.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"strconv"
 	"strings"
 
 	"github.com/rs/zerolog/log"
@@ -17,21 +18,32 @@ type yamlParser struct {
 func (y *yamlParser) Parse(reader io.Reader, leakChan chan Leak, file string, rule *CtxParserRule) {
 	buf := &bytes.Buffer{}
 	buf.ReadFrom(reader)
-	var root map[string]interface{}
+	var data interface{}
 
-	if err := yaml.Unmarshal(buf.Bytes(), &root); err != nil {
-		log.Error().
+	if err := yaml.Unmarshal(buf.Bytes(), &data); err != nil {
+		log.Trace().
 			Err(err).
 			Str("file", file).
 			Msg("Failed to unmarshal")
+		return
 	}
 
-	// flatten yaml to a single map to search for potential leaks
-	// this method does not allow for identifying line numbers
 	flattened := map[string]string{}
-	for k, v := range root {
-		flatten(k, v, flattened)
+	switch root := data.(type) {
+	// !!map document node type
+	case map[string]interface{}:
+		for k, v := range root {
+			flattenMap(k, v, flattened)
+		}
+		break
+	// !!seq document node type
+	case []interface{}:
+		for k, v := range root {
+			flattenMap(strconv.Itoa(k), v, flattened)
+		}
+		break
 	}
+
 	for k, v := range flattened {
 		for _, key := range *y.keyBag {
 			last := k
@@ -54,21 +66,21 @@ func (y *yamlParser) Parse(reader io.Reader, leakChan chan Leak, file string, ru
 	}
 }
 
-func flatten(prefix string, value interface{}, res map[string]string) {
+func flattenMap(prefix string, value interface{}, res map[string]string) {
 	switch submap := value.(type) {
 	case map[interface{}]interface{}:
 		for k, v := range submap {
-			flatten(fmt.Sprintf("%s.%v", prefix, k), v, res)
+			flattenMap(fmt.Sprintf("%s.%v", prefix, k), v, res)
 		}
 		return
 	case []interface{}:
 		for i, v := range submap {
-			flatten(fmt.Sprintf("%s[%d]", prefix, i), v, res)
+			flattenMap(fmt.Sprintf("%s[%d]", prefix, i), v, res)
 		}
 		return
 	case map[string]interface{}:
 		for k, v := range submap {
-			flatten(prefix+"."+k, v, res)
+			flattenMap(prefix+"."+k, v, res)
 		}
 		return
 	default:

pkg/model/ruleset.go

@@ -150,10 +150,11 @@ func (r *RuleSet) ParsePatch(patch *object.Patch, commit *object.Commit, repo *R
 func (r *RuleSet) ParseFile(file string, leakChan chan Leak) {
 	fd, err := os.Open(file)
 	if err != nil {
-		log.Error().
+		log.Trace().
 			Str("file", file).
 			Err(err).
 			Msg("Failed to read")
+		return
 	}
 	defer fd.Close()
 

pkg/scan/fs_test.go

@@ -1,10 +1,22 @@
 package scan
 
 import (
+	"fmt"
 	"testing"
 )
 
 func TestFsClient(t *testing.T) {
 	f := NewFsScanner(".", "../../resources/rules.yaml", &HTMLReport{}, false)
 	f.Scan(5)
 }
+func BenchmarkFSScan(b *testing.B) {
+	conccurrent := []int{50}
+	for _, i := range conccurrent {
+		b.Run(fmt.Sprintf("fsscan_%d", i), func(b *testing.B) {
+			b.StartTimer()
+			g := NewFsScanner("../..", "../../resources/rules.yaml", &HTMLReport{}, false)
+			g.Scan(i)
+			b.ResetTimer()
+		})
+	}
+}