From 4d94833b951b59354934d00f9b8cebe6297ee89e Mon Sep 17 00:00:00 2001 From: Abhinav Dangeti Date: Mon, 16 Dec 2024 19:52:22 -0700 Subject: [PATCH] Fix interpreting scorch config: "fieldTFRCacheThreshold" + Firstly, refactor FieldTFRCacheThreshold to fieldTFRCacheThreshold for _some_ naming consistency here. + This threshold can be used to toggle recycling of TermFieldReaders on/off. + Couchbase users will have the ability to provide this setting within a JSON payload, which when interpreted into a map[string]interface{} will need to be interpreted as a `float64`. + Should library users set it as `int` we'll honor that setting as well. --- index/scorch/snapshot_index.go | 36 +++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/index/scorch/snapshot_index.go b/index/scorch/snapshot_index.go index 79840a41f..3e26930f5 100644 --- a/index/scorch/snapshot_index.go +++ b/index/scorch/snapshot_index.go @@ -52,15 +52,6 @@ type asynchSegmentResult struct { var reflectStaticSizeIndexSnapshot int -// DefaultFieldTFRCacheThreshold limits the number of TermFieldReaders(TFR) for -// a field in an index snapshot. Without this limit, when recycling TFRs, it is -// possible that a very large number of TFRs may be added to the recycle -// cache, which could eventually lead to significant memory consumption. -// This threshold can be overwritten by users at the library level by changing the -// exported variable, or at the index level by setting the FieldTFRCacheThreshold -// in the kvConfig. -var DefaultFieldTFRCacheThreshold uint64 = 10 - func init() { var is interface{} = IndexSnapshot{} reflectStaticSizeIndexSnapshot = int(reflect.TypeOf(is).Size()) @@ -640,13 +631,30 @@ func (is *IndexSnapshot) allocTermFieldReaderDicts(field string) (tfr *IndexSnap } } -func (is *IndexSnapshot) getFieldTFRCacheThreshold() uint64 { +// defaultFieldTFRCacheThreshold limits the number of TermFieldReaders(TFR) for +// a field in an index snapshot. Without this limit, when recycling TFRs, it is +// possible that a very large number of TFRs may be added to the recycle +// cache, which could eventually lead to significant memory consumption. +// This threshold can be overwritten by users at the library level by changing the +// exported variable, or at the index level by setting the "fieldTFRCacheThreshold" +// in the kvConfig. +const defaultFieldTFRCacheThreshold int = 10 + +func (is *IndexSnapshot) getFieldTFRCacheThreshold() int { if is.parent.config != nil { - if _, ok := is.parent.config["FieldTFRCacheThreshold"]; ok { - return is.parent.config["FieldTFRCacheThreshold"].(uint64) + if _, exists := is.parent.config["fieldTFRCacheThreshold"]; exists { + val := is.parent.config["fieldTFRCacheThreshold"] + if x, ok := val.(float64); ok { + // JSON unmarshal-ed into a map[string]interface{} will default + // to float64 for numbers, so we need to check for float64 first. + return int(x) + } else if x, ok := val.(int); ok { + // If library users provided an int in the config, we'll honor it. + return x + } } } - return DefaultFieldTFRCacheThreshold + return defaultFieldTFRCacheThreshold } func (is *IndexSnapshot) recycleTermFieldReader(tfr *IndexSnapshotTermFieldReader) { @@ -670,7 +678,7 @@ func (is *IndexSnapshot) recycleTermFieldReader(tfr *IndexSnapshotTermFieldReade if is.fieldTFRs == nil { is.fieldTFRs = map[string][]*IndexSnapshotTermFieldReader{} } - if uint64(len(is.fieldTFRs[tfr.field])) < is.getFieldTFRCacheThreshold() { + if len(is.fieldTFRs[tfr.field]) < is.getFieldTFRCacheThreshold() { tfr.bytesRead = 0 is.fieldTFRs[tfr.field] = append(is.fieldTFRs[tfr.field], tfr) }