MB-63831: Toy: Limiting number of documents per in memory merge

blevesearch · Nov 7, 2024 · 6436a32 · 6436a32
1 parent bed244c
commit 6436a32
Showing 1 changed file with 10 additions and 3 deletions.
diff --git a/index/scorch/persister.go b/index/scorch/persister.go
@@ -371,11 +371,18 @@ func (s *Scorch) persistSnapshotMaybeMerge(snapshot *IndexSnapshot) (
 	var sbsDrops []*roaring.Bitmap
 	var sbsIndexes []int
 
+	count := 0
+	requiredCount := 1000000
 	for i, segmentSnapshot := range snapshot.segment {
 		if _, ok := segmentSnapshot.segment.(segment.PersistedSegment); !ok {
-			sbs = append(sbs, segmentSnapshot.segment)
-			sbsDrops = append(sbsDrops, segmentSnapshot.deleted)
-			sbsIndexes = append(sbsIndexes, i)
+			if count < requiredCount {
+				count = count + int(segmentSnapshot.segment.Count())
+				sbs = append(sbs, segmentSnapshot.segment)
+				sbsDrops = append(sbsDrops, segmentSnapshot.deleted)
+				sbsIndexes = append(sbsIndexes, i)
+			} else {
+				break
+			}
 		}
 	}