Skip to content

Commit

Permalink
Fix interpreting scorch config: "fieldTFRCacheThreshold"
Browse files Browse the repository at this point in the history
+ Firstly, refactor FieldTFRCacheThreshold to fieldTFRCacheThreshold
  for _some_ naming consistency here.
+ This threshold can be used to toggle recycling of TermFieldReaders
  on/off.
+ Couchbase users will have the ability to provide this setting within
  a JSON payload, which when interpreted into a map[string]interface{}
  will need to be interpreted as a `float64`.
+ Should library users set it as `int` we'll honor that setting as well.
  • Loading branch information
abhinavdangeti committed Dec 17, 2024
1 parent 7d627b9 commit 4d94833
Showing 1 changed file with 22 additions and 14 deletions.
36 changes: 22 additions & 14 deletions index/scorch/snapshot_index.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,15 +52,6 @@ type asynchSegmentResult struct {

var reflectStaticSizeIndexSnapshot int

// DefaultFieldTFRCacheThreshold limits the number of TermFieldReaders(TFR) for
// a field in an index snapshot. Without this limit, when recycling TFRs, it is
// possible that a very large number of TFRs may be added to the recycle
// cache, which could eventually lead to significant memory consumption.
// This threshold can be overwritten by users at the library level by changing the
// exported variable, or at the index level by setting the FieldTFRCacheThreshold
// in the kvConfig.
var DefaultFieldTFRCacheThreshold uint64 = 10

func init() {
var is interface{} = IndexSnapshot{}
reflectStaticSizeIndexSnapshot = int(reflect.TypeOf(is).Size())
Expand Down Expand Up @@ -640,13 +631,30 @@ func (is *IndexSnapshot) allocTermFieldReaderDicts(field string) (tfr *IndexSnap
}
}

func (is *IndexSnapshot) getFieldTFRCacheThreshold() uint64 {
// defaultFieldTFRCacheThreshold limits the number of TermFieldReaders(TFR) for
// a field in an index snapshot. Without this limit, when recycling TFRs, it is
// possible that a very large number of TFRs may be added to the recycle
// cache, which could eventually lead to significant memory consumption.
// This threshold can be overwritten by users at the library level by changing the
// exported variable, or at the index level by setting the "fieldTFRCacheThreshold"
// in the kvConfig.
const defaultFieldTFRCacheThreshold int = 10

func (is *IndexSnapshot) getFieldTFRCacheThreshold() int {
if is.parent.config != nil {
if _, ok := is.parent.config["FieldTFRCacheThreshold"]; ok {
return is.parent.config["FieldTFRCacheThreshold"].(uint64)
if _, exists := is.parent.config["fieldTFRCacheThreshold"]; exists {
val := is.parent.config["fieldTFRCacheThreshold"]
if x, ok := val.(float64); ok {
// JSON unmarshal-ed into a map[string]interface{} will default
// to float64 for numbers, so we need to check for float64 first.
return int(x)
} else if x, ok := val.(int); ok {
// If library users provided an int in the config, we'll honor it.
return x
}
}
}
return DefaultFieldTFRCacheThreshold
return defaultFieldTFRCacheThreshold
}

func (is *IndexSnapshot) recycleTermFieldReader(tfr *IndexSnapshotTermFieldReader) {
Expand All @@ -670,7 +678,7 @@ func (is *IndexSnapshot) recycleTermFieldReader(tfr *IndexSnapshotTermFieldReade
if is.fieldTFRs == nil {
is.fieldTFRs = map[string][]*IndexSnapshotTermFieldReader{}
}
if uint64(len(is.fieldTFRs[tfr.field])) < is.getFieldTFRCacheThreshold() {
if len(is.fieldTFRs[tfr.field]) < is.getFieldTFRCacheThreshold() {
tfr.bytesRead = 0
is.fieldTFRs[tfr.field] = append(is.fieldTFRs[tfr.field], tfr)
}
Expand Down

0 comments on commit 4d94833

Please sign in to comment.