Fix Size Calculation (#2052)

- Fix Size() calculation to account for dynamic entries of the struct,
to ensure that accurate information is passed on to the memory throttler

document/field_boolean.go

@@ -43,10 +43,15 @@ type BooleanField struct {
 }
 
 func (b *BooleanField) Size() int {
+	var freqSize int
+	if b.frequencies != nil {
+		freqSize = b.frequencies.Size()
+	}
 	return reflectStaticSizeBooleanField + size.SizeOfPtr +
 		len(b.name) +
 		len(b.arrayPositions)*size.SizeOfUint64 +
-		len(b.value)
+		len(b.value) +
+		freqSize
 }
 
 func (b *BooleanField) Name() string {

document/field_composite.go

@@ -68,13 +68,16 @@ func (c *CompositeField) Size() int {
 	sizeInBytes := reflectStaticSizeCompositeField + size.SizeOfPtr +
 		len(c.name)
 
-	for k, _ := range c.includedFields {
+	for k := range c.includedFields {
 		sizeInBytes += size.SizeOfString + len(k) + size.SizeOfBool
 	}
 
-	for k, _ := range c.excludedFields {
+	for k := range c.excludedFields {
 		sizeInBytes += size.SizeOfString + len(k) + size.SizeOfBool
 	}
+	if c.compositeFrequencies != nil {
+		sizeInBytes += c.compositeFrequencies.Size()
+	}
 
 	return sizeInBytes
 }

document/field_datetime.go

@@ -53,9 +53,15 @@ type DateTimeField struct {
 }
 
 func (n *DateTimeField) Size() int {
+	var freqSize int
+	if n.frequencies != nil {
+		freqSize = n.frequencies.Size()
+	}
 	return reflectStaticSizeDateTimeField + size.SizeOfPtr +
 		len(n.name) +
-		len(n.arrayPositions)*size.SizeOfUint64
+		len(n.arrayPositions)*size.SizeOfUint64 +
+		len(n.value) +
+		freqSize
 }
 
 func (n *DateTimeField) Name() string {

document/field_geopoint.go

@@ -47,9 +47,15 @@ type GeoPointField struct {
 }
 
 func (n *GeoPointField) Size() int {
+	var freqSize int
+	if n.frequencies != nil {
+		freqSize = n.frequencies.Size()
+	}
 	return reflectStaticSizeGeoPointField + size.SizeOfPtr +
 		len(n.name) +
-		len(n.arrayPositions)*size.SizeOfUint64
+		len(n.arrayPositions)*size.SizeOfUint64 +
+		len(n.value) +
+		freqSize
 }
 
 func (n *GeoPointField) Name() string {

document/field_geoshape.go

@@ -48,9 +48,16 @@ type GeoShapeField struct {
 }
 
 func (n *GeoShapeField) Size() int {
+	var freqSize int
+	if n.frequencies != nil {
+		freqSize = n.frequencies.Size()
+	}
 	return reflectStaticSizeGeoShapeField + size.SizeOfPtr +
 		len(n.name) +
-		len(n.arrayPositions)*size.SizeOfUint64
+		len(n.arrayPositions)*size.SizeOfUint64 +
+		len(n.encodedValue) +
+		len(n.value) +
+		freqSize
 }
 
 func (n *GeoShapeField) Name() string {

document/field_ip.go

@@ -44,10 +44,15 @@ type IPField struct {
 }
 
 func (b *IPField) Size() int {
+	var freqSize int
+	if b.frequencies != nil {
+		freqSize = b.frequencies.Size()
+	}
 	return reflectStaticSizeIPField + size.SizeOfPtr +
 		len(b.name) +
 		len(b.arrayPositions)*size.SizeOfUint64 +
-		len(b.value)
+		len(b.value) +
+		freqSize
 }
 
 func (b *IPField) Name() string {

document/field_numeric.go

@@ -46,9 +46,15 @@ type NumericField struct {
 }
 
 func (n *NumericField) Size() int {
+	var freqSize int
+	if n.frequencies != nil {
+		freqSize = n.frequencies.Size()
+	}
 	return reflectStaticSizeNumericField + size.SizeOfPtr +
 		len(n.name) +
-		len(n.arrayPositions)*size.SizeOfPtr
+		len(n.arrayPositions)*size.SizeOfUint64 +
+		len(n.value) +
+		freqSize
 }
 
 func (n *NumericField) Name() string {

document/field_text.go

@@ -44,10 +44,15 @@ type TextField struct {
 }
 
 func (t *TextField) Size() int {
+	var freqSize int
+	if t.frequencies != nil {
+		freqSize = t.frequencies.Size()
+	}
 	return reflectStaticSizeTextField + size.SizeOfPtr +
 		len(t.name) +
 		len(t.arrayPositions)*size.SizeOfUint64 +
-		len(t.value)
+		len(t.value) +
+		freqSize
 }
 
 func (t *TextField) Name() string {

document/field_vector.go

@@ -47,6 +47,8 @@ type VectorField struct {
 func (n *VectorField) Size() int {
 	return reflectStaticSizeVectorField + size.SizeOfPtr +
 		len(n.name) +
+		len(n.similarity) +
+		len(n.vectorIndexOptimizedFor) +
 		int(numBytesFloat32s(n.value))
 }
 

document/field_vector_base64.go

@@ -22,19 +22,33 @@ import (
 	"encoding/binary"
 	"fmt"
 	"math"
+	"reflect"
 
 	"github.com/blevesearch/bleve/v2/size"
 	"github.com/blevesearch/bleve/v2/util"
 	index "github.com/blevesearch/bleve_index_api"
 )
 
+var reflectStaticSizeVectorBase64Field int
+
+func init() {
+	var f VectorBase64Field
+	reflectStaticSizeVectorBase64Field = int(reflect.TypeOf(f).Size())
+}
+
 type VectorBase64Field struct {
 	vectorField    *VectorField
 	base64Encoding string
 }
 
 func (n *VectorBase64Field) Size() int {
-	return n.vectorField.Size()
+	var vecFieldSize int
+	if n.vectorField != nil {
+		vecFieldSize = n.vectorField.Size()
+	}
+	return reflectStaticSizeVectorBase64Field + size.SizeOfPtr +
+		len(n.base64Encoding) +
+		vecFieldSize
 }
 
 func (n *VectorBase64Field) Name() string {

index/scorch/snapshot_index_vr.go

@@ -52,7 +52,9 @@ type IndexSnapshotVectorReader struct {
 
 func (i *IndexSnapshotVectorReader) Size() int {
 	sizeInBytes := reflectStaticSizeIndexSnapshotVectorReader + size.SizeOfPtr +
-		len(i.vector) + len(i.field) + len(i.currID)
+		len(i.vector)*size.SizeOfFloat32 +
+		len(i.field) +
+		len(i.currID)
 
 	for _, entry := range i.postings {
 		sizeInBytes += entry.Size()

search/scorer/scorer_knn.go

@@ -47,7 +47,8 @@ type KNNQueryScorer struct {
 
 func (s *KNNQueryScorer) Size() int {
 	sizeInBytes := reflectStaticSizeKNNQueryScorer + size.SizeOfPtr +
-		(len(s.queryVector) * size.SizeOfFloat32) + len(s.queryField)
+		(len(s.queryVector) * size.SizeOfFloat32) + len(s.queryField) +
+		len(s.similarityMetric)
 
 	if s.queryWeightExplanation != nil {
 		sizeInBytes += s.queryWeightExplanation.Size()