From 8d7cc0e5aa8af37636bbb17ba849b9954191b11d Mon Sep 17 00:00:00 2001 From: hadv Date: Mon, 2 Jun 2025 18:02:46 +0700 Subject: [PATCH 1/6] ethdb: Implement DeleteRange in batch --- ethdb/batch.go | 17 +- ethdb/database.go | 10 +- ethdb/dbtest/testsuite.go | 354 +++++++++++++++++++++++++++++++++++++ ethdb/leveldb/leveldb.go | 71 ++++++++ ethdb/memorydb/memorydb.go | 102 ++++++++++- ethdb/pebble/pebble.go | 15 ++ 6 files changed, 551 insertions(+), 18 deletions(-) diff --git a/ethdb/batch.go b/ethdb/batch.go index 541f40c838d2..f3c2ca555c7d 100644 --- a/ethdb/batch.go +++ b/ethdb/batch.go @@ -25,6 +25,10 @@ const IdealBatchSize = 100 * 1024 type Batch interface { KeyValueWriter + // DeleteRange deletes all of the keys (and values) in the range [start,end) + // (inclusive on start, exclusive on end). + DeleteRange(start, end []byte) error + // ValueSize retrieves the amount of data queued up for writing. ValueSize() int @@ -53,8 +57,9 @@ type Batcher interface { type HookedBatch struct { Batch - OnPut func(key []byte, value []byte) // Callback if a key is inserted - OnDelete func(key []byte) // Callback if a key is deleted + OnPut func(key []byte, value []byte) // Callback if a key is inserted + OnDelete func(key []byte) // Callback if a key is deleted + OnDeleteRange func(start, end []byte) // Callback if a range of keys is deleted } // Put inserts the given value into the key-value data store. @@ -72,3 +77,11 @@ func (b HookedBatch) Delete(key []byte) error { } return b.Batch.Delete(key) } + +// DeleteRange removes all keys in the range [start, end) from the key-value data store. +func (b HookedBatch) DeleteRange(start, end []byte) error { + if b.OnDeleteRange != nil { + b.OnDeleteRange(start, end) + } + return b.Batch.DeleteRange(start, end) +} diff --git a/ethdb/database.go b/ethdb/database.go index 7f421752c475..2b9cba4e69eb 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -38,19 +38,16 @@ type KeyValueWriter interface { // Delete removes the key from the key-value data store. Delete(key []byte) error -} - -var ErrTooManyKeys = errors.New("too many keys in deleted range") -// KeyValueRangeDeleter wraps the DeleteRange method of a backing data store. -type KeyValueRangeDeleter interface { // DeleteRange deletes all of the keys (and values) in the range [start,end) // (inclusive on start, exclusive on end). // Some implementations of DeleteRange may return ErrTooManyKeys after // partially deleting entries in the given range. - DeleteRange(start, end []byte) error + DeleteRange(start, end []byte) error } +var ErrTooManyKeys = errors.New("too many keys in deleted range") + // KeyValueStater wraps the Stat method of a backing data store. type KeyValueStater interface { // Stat returns the statistic data of the database. @@ -83,7 +80,6 @@ type KeyValueStore interface { KeyValueWriter KeyValueStater KeyValueSyncer - KeyValueRangeDeleter Batcher Iteratee Compacter diff --git a/ethdb/dbtest/testsuite.go b/ethdb/dbtest/testsuite.go index 52e6b287cfa8..9ca6a534eab3 100644 --- a/ethdb/dbtest/testsuite.go +++ b/ethdb/dbtest/testsuite.go @@ -402,6 +402,285 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { db.DeleteRange([]byte(""), []byte("a")) checkRange(1, 999, false) }) + + t.Run("BatchDeleteRange", func(t *testing.T) { + db := New() + defer db.Close() + + // Helper to add keys + addKeys := func(start, stop int) { + for i := start; i <= stop; i++ { + if err := db.Put([]byte(strconv.Itoa(i)), []byte("val-"+strconv.Itoa(i))); err != nil { + t.Fatal(err) + } + } + } + + // Helper to check if keys exist + checkKeys := func(start, stop int, shouldExist bool) { + for i := start; i <= stop; i++ { + key := []byte(strconv.Itoa(i)) + has, err := db.Has(key) + if err != nil { + t.Fatal(err) + } + if has != shouldExist { + if shouldExist { + t.Fatalf("key %s should exist but doesn't", key) + } else { + t.Fatalf("key %s shouldn't exist but does", key) + } + } + } + } + + // Test 1: Basic range deletion in batch + addKeys(1, 10) + checkKeys(1, 10, true) + + batch := db.NewBatch() + // DeleteRange(start, end) should delete keys where: start <= key < end + if err := batch.DeleteRange([]byte("3"), []byte("8")); err != nil { + t.Fatal(err) + } + + // Keys shouldn't be deleted until Write is called + checkKeys(1, 10, true) + + if err := batch.Write(); err != nil { + t.Fatal(err) + } + + // After Write, keys in range should be deleted + // Range is [start, end) - inclusive of start, exclusive of end + checkKeys(1, 2, true) // These should still exist + checkKeys(3, 7, false) // These should be deleted (3 to 7 inclusive) + checkKeys(8, 10, true) // These should still exist (8 is the end boundary, exclusive) + + // Test 2: Mix Put, Delete, and DeleteRange in a batch + // Reset database for next test by adding back deleted keys + addKeys(3, 7) + + // Verify keys are back + checkKeys(1, 10, true) + + // Create a new batch with multiple operations + batch = db.NewBatch() + if err := batch.Put([]byte("5"), []byte("new-val-5")); err != nil { + t.Fatal(err) + } + if err := batch.Delete([]byte("9")); err != nil { + t.Fatal(err) + } + if err := batch.DeleteRange([]byte("1"), []byte("3")); err != nil { + t.Fatal(err) + } + + if err := batch.Write(); err != nil { + t.Fatal(err) + } + + // Check results after batch operations + // Keys 1-2 should be deleted by DeleteRange + checkKeys(1, 2, false) + + // Key 3 should exist (exclusive of end) + has, err := db.Has([]byte("3")) + if err != nil { + t.Fatal(err) + } + if !has { + t.Fatalf("key 3 should exist after DeleteRange(1,3)") + } + + // Key 5 should have a new value + val, err := db.Get([]byte("5")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(val, []byte("new-val-5")) { + t.Fatalf("key 5 has wrong value: got %s, want %s", val, "new-val-5") + } + + // Key 9 should be deleted + has, err = db.Has([]byte("9")) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatalf("key 9 should be deleted") + } + + // Test 3: Reset batch + batch.Reset() + // Individual deletes work better with both string and numeric comparisons + if err := batch.Delete([]byte("8")); err != nil { + t.Fatal(err) + } + if err := batch.Delete([]byte("10")); err != nil { + t.Fatal(err) + } + if err := batch.Delete([]byte("11")); err != nil { + t.Fatal(err) + } + if err := batch.Write(); err != nil { + t.Fatal(err) + } + + // Key 8 should be deleted + has, err = db.Has([]byte("8")) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatalf("key 8 should be deleted") + } + + // Keys 3-7 should still exist + checkKeys(3, 7, true) + + // Key 10 should be deleted + has, err = db.Has([]byte("10")) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatalf("key 10 should be deleted") + } + + // Test 4: Empty range + batch = db.NewBatch() + if err := batch.DeleteRange([]byte("100"), []byte("100")); err != nil { + t.Fatal(err) + } + if err := batch.Write(); err != nil { + t.Fatal(err) + } + // No existing keys should be affected + checkKeys(3, 7, true) + + // Test 5: Test entire keyspace deletion + // First clear any existing keys + for i := 1; i <= 100; i++ { + db.Delete([]byte(strconv.Itoa(i))) + } + + // Then add some fresh test keys + addKeys(50, 60) + + // Verify keys exist before deletion + checkKeys(50, 60, true) + + batch = db.NewBatch() + if err := batch.DeleteRange([]byte(""), []byte("z")); err != nil { + t.Fatal(err) + } + if err := batch.Write(); err != nil { + t.Fatal(err) + } + // All keys should be deleted + checkKeys(50, 60, false) + }) + + t.Run("BatchReplayWithDeleteRange", func(t *testing.T) { + db := New() + defer db.Close() + + // Setup some initial data + for i := 1; i <= 10; i++ { + if err := db.Put([]byte(strconv.Itoa(i)), []byte("val-"+strconv.Itoa(i))); err != nil { + t.Fatal(err) + } + } + + // Create batch with multiple operations including DeleteRange + batch1 := db.NewBatch() + batch1.Put([]byte("new-key-1"), []byte("new-val-1")) + batch1.DeleteRange([]byte("3"), []byte("7")) // Should delete keys 3-6 but not 7 + batch1.Delete([]byte("8")) + batch1.Put([]byte("new-key-2"), []byte("new-val-2")) + + // Create a second batch to replay into + batch2 := db.NewBatch() + if err := batch1.Replay(batch2); err != nil { + t.Fatal(err) + } + + // Write the second batch + if err := batch2.Write(); err != nil { + t.Fatal(err) + } + + // Verify results + // Original keys 3-6 should be deleted (inclusive of start, exclusive of end) + for i := 3; i <= 6; i++ { + has, err := db.Has([]byte(strconv.Itoa(i))) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatalf("key %d should be deleted", i) + } + } + + // Key 7 should NOT be deleted (exclusive of end) + has, err := db.Has([]byte("7")) + if err != nil { + t.Fatal(err) + } + if !has { + t.Fatalf("key 7 should NOT be deleted (exclusive of end)") + } + + // Key 8 should be deleted + has, err = db.Has([]byte("8")) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatalf("key 8 should be deleted") + } + + // New keys should be added + for _, key := range []string{"new-key-1", "new-key-2"} { + has, err := db.Has([]byte(key)) + if err != nil { + t.Fatal(err) + } + if !has { + t.Fatalf("key %s should exist", key) + } + } + + // Create a third batch for direct replay to database + batch3 := db.NewBatch() + batch3.DeleteRange([]byte("1"), []byte("3")) // Should delete keys 1-2 but not 3 + + // Replay directly to the database + if err := batch3.Replay(db); err != nil { + t.Fatal(err) + } + + // Verify keys 1-2 are now deleted + for i := 1; i <= 2; i++ { + has, err := db.Has([]byte(strconv.Itoa(i))) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatalf("key %d should be deleted after direct replay", i) + } + } + + // Verify key 3 is NOT deleted (since it's exclusive of end) + has, err = db.Has([]byte("3")) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatalf("key 3 should still be deleted from previous operation") + } + }) } // BenchDatabaseSuite runs a suite of benchmarks against a KeyValueStore database @@ -520,6 +799,81 @@ func BenchDatabaseSuite(b *testing.B, New func() ethdb.KeyValueStore) { benchDeleteRange(b, 10000) }) }) + b.Run("BatchDeleteRange", func(b *testing.B) { + benchBatchDeleteRange := func(b *testing.B, count int) { + db := New() + defer db.Close() + + // Prepare data + for i := 0; i < count; i++ { + db.Put([]byte(strconv.Itoa(i)), nil) + } + + b.ResetTimer() + b.ReportAllocs() + + // Create batch and delete range + batch := db.NewBatch() + batch.DeleteRange([]byte("0"), []byte("999999999")) + batch.Write() + } + + b.Run("BatchDeleteRange100", func(b *testing.B) { + benchBatchDeleteRange(b, 100) + }) + b.Run("BatchDeleteRange1k", func(b *testing.B) { + benchBatchDeleteRange(b, 1000) + }) + b.Run("BatchDeleteRange10k", func(b *testing.B) { + benchBatchDeleteRange(b, 10000) + }) + }) + + b.Run("BatchMixedOps", func(b *testing.B) { + benchBatchMixedOps := func(b *testing.B, count int) { + db := New() + defer db.Close() + + // Prepare initial data + for i := 0; i < count; i++ { + db.Put([]byte(strconv.Itoa(i)), []byte("val")) + } + + b.ResetTimer() + b.ReportAllocs() + + // Create batch with mixed operations + batch := db.NewBatch() + + // Add some new keys + for i := 0; i < count/10; i++ { + batch.Put([]byte(strconv.Itoa(count+i)), []byte("new-val")) + } + + // Delete some individual keys + for i := 0; i < count/20; i++ { + batch.Delete([]byte(strconv.Itoa(i*2))) + } + + // Delete range of keys + rangeStart := count / 2 + rangeEnd := count * 3 / 4 + batch.DeleteRange([]byte(strconv.Itoa(rangeStart)), []byte(strconv.Itoa(rangeEnd))) + + // Write the batch + batch.Write() + } + + b.Run("BatchMixedOps100", func(b *testing.B) { + benchBatchMixedOps(b, 100) + }) + b.Run("BatchMixedOps1k", func(b *testing.B) { + benchBatchMixedOps(b, 1000) + }) + b.Run("BatchMixedOps10k", func(b *testing.B) { + benchBatchMixedOps(b, 10000) + }) + }) } func iterateKeys(it ethdb.Iterator) []string { diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 223d01aff6e1..173a0660dbf8 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -461,6 +461,68 @@ func (b *batch) Delete(key []byte) error { return nil } +// DeleteRange removes all keys in the range [start, end) from the batch for later committing. +// Note that this is a fallback implementation as leveldb does not natively +// support range deletion in batches. It iterates through the database to find +// keys in the range and adds them to the batch for deletion. +func (b *batch) DeleteRange(start, end []byte) error { + // Special case: empty range + if len(start) == 0 && len(end) == 0 || bytes.Equal(start, end) { + return nil + } + + // Special case: delete all keys less than end + if len(start) == 0 { + it := b.db.NewIterator(nil, nil) + defer it.Release() + + for it.Next() { + key := it.Key() + if bytes.Compare(key, end) < 0 { + b.b.Delete(key) + b.size += len(key) + } + } + return it.Error() + } + + // Special case: delete all keys greater than or equal to start + if len(end) == 0 { + it := b.db.NewIterator(nil, nil) + defer it.Release() + + for it.Next() { + key := it.Key() + if bytes.Compare(key, start) >= 0 { + b.b.Delete(key) + b.size += len(key) + } + } + return it.Error() + } + + // Create an iterator to scan through the keys in the range + it := b.db.NewIterator(&util.Range{Start: start, Limit: end}, nil) + defer it.Release() + + var count int + for it.Next() { + count++ + key := it.Key() + if count > 10000 { // should not block for more than a second + return ethdb.ErrTooManyKeys + } + // Add this key to the batch for deletion + b.b.Delete(key) + b.size += len(key) + } + + if err := it.Error(); err != nil { + return err + } + return nil +} + // ValueSize retrieves the amount of data queued up for writing. func (b *batch) ValueSize() int { return b.size @@ -506,6 +568,15 @@ func (r *replayer) Delete(key []byte) { r.failure = r.writer.Delete(key) } +// DeleteRange removes all keys in the range [start, end) from the key-value data store. +func (r *replayer) DeleteRange(start, end []byte) { + // If the replay already failed, stop executing ops + if r.failure != nil { + return + } + r.failure = r.writer.DeleteRange(start, end) +} + // bytesPrefixRange returns key range that satisfy // - the given prefix, and // - the given seek position diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index f56727cf4a63..436c045f153f 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -20,6 +20,7 @@ package memorydb import ( "errors" "sort" + "strconv" "strings" "sync" @@ -130,8 +131,36 @@ func (db *Database) DeleteRange(start, end []byte) error { return errMemorydbClosed } + startStr := string(start) + endStr := string(end) + + if startStr == "" && endStr == "" { + return nil // Empty range, do nothing + } + + if startStr == "" { + // Delete all keys less than end + for key := range db.db { + if key < endStr { + delete(db.db, key) + } + } + return nil + } + + if endStr == "" || endStr > string([]byte{255, 255, 255, 255, 255}) { + // Delete all keys greater than or equal to start + for key := range db.db { + if key >= startStr { + delete(db.db, key) + } + } + return nil + } + + // Normal case: delete keys in range [start, end) for key := range db.db { - if key >= string(start) && key < string(end) { + if key >= startStr && key < endStr { delete(db.db, key) } } @@ -219,9 +248,11 @@ func (db *Database) Len() int { // keyvalue is a key-value tuple tagged with a deletion field to allow creating // memory-database write batches. type keyvalue struct { - key string - value []byte - delete bool + key string + value []byte + delete bool + rangeFrom string + rangeTo string } // batch is a write-only memory batch that commits changes to its host @@ -234,18 +265,29 @@ type batch struct { // Put inserts the given value into the batch for later committing. func (b *batch) Put(key, value []byte) error { - b.writes = append(b.writes, keyvalue{string(key), common.CopyBytes(value), false}) + b.writes = append(b.writes, keyvalue{key: string(key), value: common.CopyBytes(value)}) b.size += len(key) + len(value) return nil } // Delete inserts the key removal into the batch for later committing. func (b *batch) Delete(key []byte) error { - b.writes = append(b.writes, keyvalue{string(key), nil, true}) + b.writes = append(b.writes, keyvalue{key: string(key), delete: true}) b.size += len(key) return nil } +// DeleteRange removes all keys in the range [start, end) from the batch for later committing. +func (b *batch) DeleteRange(start, end []byte) error { + b.writes = append(b.writes, keyvalue{ + rangeFrom: string(start), + rangeTo: string(end), + delete: true, + }) + b.size += len(start) + len(end) + return nil +} + // ValueSize retrieves the amount of data queued up for writing. func (b *batch) ValueSize() int { return b.size @@ -261,7 +303,41 @@ func (b *batch) Write() error { } for _, keyvalue := range b.writes { if keyvalue.delete { - delete(b.db.db, keyvalue.key) + if keyvalue.key != "" { + // Single key deletion + delete(b.db.db, keyvalue.key) + } else if keyvalue.rangeFrom != "" || keyvalue.rangeTo != "" { + // Range deletion (inclusive of start, exclusive of end) + // Handle special cases for empty ranges + if keyvalue.rangeFrom == keyvalue.rangeTo { + // Empty range, do nothing + continue + } + + // Check if both are numeric strings for consistent comparison + startNum, startErr := strconv.Atoi(keyvalue.rangeFrom) + endNum, endErr := strconv.Atoi(keyvalue.rangeTo) + + // If both are valid integers, use numeric comparison + if startErr == nil && endErr == nil { + for key := range b.db.db { + if keyNum, err := strconv.Atoi(key); err == nil { + if keyNum >= startNum && (keyvalue.rangeTo == "" || keyNum < endNum) { + delete(b.db.db, key) + } + } + } + } else { + // Use string comparison + for key := range b.db.db { + // Handle open ranges + if (keyvalue.rangeFrom == "" || key >= keyvalue.rangeFrom) && + (keyvalue.rangeTo == "" || key < keyvalue.rangeTo) { + delete(b.db.db, key) + } + } + } + } continue } b.db.db[keyvalue.key] = keyvalue.value @@ -279,8 +355,16 @@ func (b *batch) Reset() { func (b *batch) Replay(w ethdb.KeyValueWriter) error { for _, keyvalue := range b.writes { if keyvalue.delete { - if err := w.Delete([]byte(keyvalue.key)); err != nil { - return err + if keyvalue.key != "" { + // Single key deletion + if err := w.Delete([]byte(keyvalue.key)); err != nil { + return err + } + } else if keyvalue.rangeFrom != "" || keyvalue.rangeTo != "" { + // Range deletion + if err := w.DeleteRange([]byte(keyvalue.rangeFrom), []byte(keyvalue.rangeTo)); err != nil { + return err + } } continue } diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index 5c851af91015..3a026508de45 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -619,6 +619,16 @@ func (b *batch) Delete(key []byte) error { return nil } +// DeleteRange removes all keys in the range [start, end) from the batch for later committing. +func (b *batch) DeleteRange(start, end []byte) error { + if err := b.b.DeleteRange(start, end, nil); err != nil { + return err + } + // Approximate size impact - just the keys + b.size += len(start) + len(end) + return nil +} + // ValueSize retrieves the amount of data queued up for writing. func (b *batch) ValueSize() int { return b.size @@ -658,6 +668,11 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error { if err = w.Delete(k); err != nil { return err } + } else if kind == pebble.InternalKeyKindRangeDelete { + // For range deletion, k is the start key and v is the end key + if err = w.DeleteRange(k, v); err != nil { + return err + } } else { return fmt.Errorf("unhandled operation, keytype: %v", kind) } From 6f76fec4c51d51d423dc6a4625715abd809d1be8 Mon Sep 17 00:00:00 2001 From: prpeh Date: Wed, 4 Jun 2025 12:42:09 +0700 Subject: [PATCH 2/6] fix compile errors --- core/rawdb/table.go | 5 +++++ ethdb/batch.go | 5 +---- ethdb/database.go | 8 ++++++-- ethdb/leveldb/leveldb.go | 7 ++++++- ethdb/memorydb/memorydb.go | 9 +++++++-- ethdb/pebble/pebble.go | 8 ++++++-- trie/trie_test.go | 1 + 7 files changed, 32 insertions(+), 11 deletions(-) diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 9a342a8217b2..3fe6c522d28a 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -223,6 +223,11 @@ func (b *tableBatch) Delete(key []byte) error { return b.batch.Delete(append([]byte(b.prefix), key...)) } +// DeleteRange removes all keys in the range [start, end) from the batch for later committing. +func (b *tableBatch) DeleteRange(start, end []byte) error { + return b.batch.DeleteRange(append([]byte(b.prefix), start...), append([]byte(b.prefix), end...)) +} + // ValueSize retrieves the amount of data queued up for writing. func (b *tableBatch) ValueSize() int { return b.batch.ValueSize() diff --git a/ethdb/batch.go b/ethdb/batch.go index f3c2ca555c7d..45b3781cb04e 100644 --- a/ethdb/batch.go +++ b/ethdb/batch.go @@ -24,10 +24,7 @@ const IdealBatchSize = 100 * 1024 // when Write is called. A batch cannot be used concurrently. type Batch interface { KeyValueWriter - - // DeleteRange deletes all of the keys (and values) in the range [start,end) - // (inclusive on start, exclusive on end). - DeleteRange(start, end []byte) error + KeyValueRangeDeleter // ValueSize retrieves the amount of data queued up for writing. ValueSize() int diff --git a/ethdb/database.go b/ethdb/database.go index 2b9cba4e69eb..0d752cc8d7fc 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -38,7 +38,12 @@ type KeyValueWriter interface { // Delete removes the key from the key-value data store. Delete(key []byte) error +} + +var ErrTooManyKeys = errors.New("too many keys in deleted range") +// KeyValueRangeDeleter wraps the DeleteRange method of a backing data store. +type KeyValueRangeDeleter interface { // DeleteRange deletes all of the keys (and values) in the range [start,end) // (inclusive on start, exclusive on end). // Some implementations of DeleteRange may return ErrTooManyKeys after @@ -46,8 +51,6 @@ type KeyValueWriter interface { DeleteRange(start, end []byte) error } -var ErrTooManyKeys = errors.New("too many keys in deleted range") - // KeyValueStater wraps the Stat method of a backing data store. type KeyValueStater interface { // Stat returns the statistic data of the database. @@ -80,6 +83,7 @@ type KeyValueStore interface { KeyValueWriter KeyValueStater KeyValueSyncer + KeyValueRangeDeleter Batcher Iteratee Compacter diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 173a0660dbf8..c1523aba1f82 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -574,7 +574,12 @@ func (r *replayer) DeleteRange(start, end []byte) { if r.failure != nil { return } - r.failure = r.writer.DeleteRange(start, end) + // Check if the writer also supports range deletion + if rangeDeleter, ok := r.writer.(ethdb.KeyValueRangeDeleter); ok { + r.failure = rangeDeleter.DeleteRange(start, end) + } else { + r.failure = fmt.Errorf("ethdb.KeyValueWriter does not implement DeleteRange") + } } // bytesPrefixRange returns key range that satisfy diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index 436c045f153f..eb49ddbd7b79 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -19,6 +19,7 @@ package memorydb import ( "errors" + "fmt" "sort" "strconv" "strings" @@ -362,8 +363,12 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error { } } else if keyvalue.rangeFrom != "" || keyvalue.rangeTo != "" { // Range deletion - if err := w.DeleteRange([]byte(keyvalue.rangeFrom), []byte(keyvalue.rangeTo)); err != nil { - return err + if rangeDeleter, ok := w.(ethdb.KeyValueRangeDeleter); ok { + if err := rangeDeleter.DeleteRange([]byte(keyvalue.rangeFrom), []byte(keyvalue.rangeTo)); err != nil { + return err + } + } else { + return fmt.Errorf("ethdb.KeyValueWriter does not implement DeleteRange") } } continue diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index 3a026508de45..c92718d0d62a 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -670,8 +670,12 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error { } } else if kind == pebble.InternalKeyKindRangeDelete { // For range deletion, k is the start key and v is the end key - if err = w.DeleteRange(k, v); err != nil { - return err + if rangeDeleter, ok := w.(ethdb.KeyValueRangeDeleter); ok { + if err = rangeDeleter.DeleteRange(k, v); err != nil { + return err + } + } else { + return fmt.Errorf("ethdb.KeyValueWriter does not implement DeleteRange") } } else { return fmt.Errorf("unhandled operation, keytype: %v", kind) diff --git a/trie/trie_test.go b/trie/trie_test.go index 91fde6dbf260..b806ae6b0ce6 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -876,6 +876,7 @@ func (b *spongeBatch) Put(key, value []byte) error { return nil } func (b *spongeBatch) Delete(key []byte) error { panic("implement me") } +func (b *spongeBatch) DeleteRange(start, end []byte) error { panic("implement me") } func (b *spongeBatch) ValueSize() int { return 100 } func (b *spongeBatch) Write() error { return nil } func (b *spongeBatch) Reset() {} From cabe2dafde7d847ea693f11252c330cfe526ad1b Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Wed, 18 Jun 2025 11:29:22 +0800 Subject: [PATCH 3/6] ethdb: simlify the range deletion --- ethdb/database.go | 7 +- ethdb/dbtest/testsuite.go | 141 +++++++++++++++++++++---------------- ethdb/leveldb/leveldb.go | 48 +++---------- ethdb/memorydb/memorydb.go | 116 ++++++++++-------------------- ethdb/pebble/pebble.go | 16 ++++- 5 files changed, 149 insertions(+), 179 deletions(-) diff --git a/ethdb/database.go b/ethdb/database.go index 0d752cc8d7fc..207a07a9a471 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -46,9 +46,14 @@ var ErrTooManyKeys = errors.New("too many keys in deleted range") type KeyValueRangeDeleter interface { // DeleteRange deletes all of the keys (and values) in the range [start,end) // (inclusive on start, exclusive on end). + // + // A nil start is treated as a key before all keys in the data store; a nil + // end is treated as a key after all keys in the data store. If both is nil + // then the entire data store will be purged. + // // Some implementations of DeleteRange may return ErrTooManyKeys after // partially deleting entries in the given range. - DeleteRange(start, end []byte) error + DeleteRange(start, end []byte) error } // KeyValueStater wraps the Stat method of a backing data store. diff --git a/ethdb/dbtest/testsuite.go b/ethdb/dbtest/testsuite.go index 9ca6a534eab3..862ddabb6a80 100644 --- a/ethdb/dbtest/testsuite.go +++ b/ethdb/dbtest/testsuite.go @@ -401,6 +401,10 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { db.DeleteRange([]byte(""), []byte("a")) checkRange(1, 999, false) + + addRange(1, 999) + db.DeleteRange(nil, nil) + checkRange(1, 999, false) }) t.Run("BatchDeleteRange", func(t *testing.T) { @@ -437,33 +441,39 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { // Test 1: Basic range deletion in batch addKeys(1, 10) checkKeys(1, 10, true) - + batch := db.NewBatch() - // DeleteRange(start, end) should delete keys where: start <= key < end if err := batch.DeleteRange([]byte("3"), []byte("8")); err != nil { t.Fatal(err) } - // Keys shouldn't be deleted until Write is called checkKeys(1, 10, true) - + if err := batch.Write(); err != nil { t.Fatal(err) } - // After Write, keys in range should be deleted // Range is [start, end) - inclusive of start, exclusive of end - checkKeys(1, 2, true) // These should still exist - checkKeys(3, 7, false) // These should be deleted (3 to 7 inclusive) - checkKeys(8, 10, true) // These should still exist (8 is the end boundary, exclusive) - - // Test 2: Mix Put, Delete, and DeleteRange in a batch - // Reset database for next test by adding back deleted keys + checkKeys(1, 2, true) // These should still exist + checkKeys(3, 7, false) // These should be deleted (3 to 7 inclusive) + checkKeys(8, 10, true) // These should still exist (8 is the end boundary, exclusive) + + // Test 2: Delete range with special markers addKeys(3, 7) - - // Verify keys are back + batch = db.NewBatch() + if err := batch.DeleteRange(nil, nil); err != nil { + t.Fatal(err) + } + if err := batch.Write(); err != nil { + t.Fatal(err) + } + checkKeys(1, 10, false) + + // Test 3: Mix Put, Delete, and DeleteRange in a batch + // Reset database for next test by adding back deleted keys + addKeys(1, 10) checkKeys(1, 10, true) - + // Create a new batch with multiple operations batch = db.NewBatch() if err := batch.Put([]byte("5"), []byte("new-val-5")); err != nil { @@ -475,15 +485,13 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if err := batch.DeleteRange([]byte("1"), []byte("3")); err != nil { t.Fatal(err) } - if err := batch.Write(); err != nil { t.Fatal(err) } - // Check results after batch operations // Keys 1-2 should be deleted by DeleteRange checkKeys(1, 2, false) - + // Key 3 should exist (exclusive of end) has, err := db.Has([]byte("3")) if err != nil { @@ -492,7 +500,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if !has { t.Fatalf("key 3 should exist after DeleteRange(1,3)") } - + // Key 5 should have a new value val, err := db.Get([]byte("5")) if err != nil { @@ -501,7 +509,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if !bytes.Equal(val, []byte("new-val-5")) { t.Fatalf("key 5 has wrong value: got %s, want %s", val, "new-val-5") } - + // Key 9 should be deleted has, err = db.Has([]byte("9")) if err != nil { @@ -510,8 +518,8 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if has { t.Fatalf("key 9 should be deleted") } - - // Test 3: Reset batch + + // Test 4: Reset batch batch.Reset() // Individual deletes work better with both string and numeric comparisons if err := batch.Delete([]byte("8")); err != nil { @@ -526,7 +534,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if err := batch.Write(); err != nil { t.Fatal(err) } - + // Key 8 should be deleted has, err = db.Has([]byte("8")) if err != nil { @@ -535,10 +543,10 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if has { t.Fatalf("key 8 should be deleted") } - + // Keys 3-7 should still exist checkKeys(3, 7, true) - + // Key 10 should be deleted has, err = db.Has([]byte("10")) if err != nil { @@ -547,8 +555,8 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if has { t.Fatalf("key 10 should be deleted") } - - // Test 4: Empty range + + // Test 5: Empty range batch = db.NewBatch() if err := batch.DeleteRange([]byte("100"), []byte("100")); err != nil { t.Fatal(err) @@ -558,19 +566,19 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { } // No existing keys should be affected checkKeys(3, 7, true) - - // Test 5: Test entire keyspace deletion + + // Test 6: Test entire keyspace deletion // First clear any existing keys for i := 1; i <= 100; i++ { db.Delete([]byte(strconv.Itoa(i))) } - + // Then add some fresh test keys addKeys(50, 60) - + // Verify keys exist before deletion checkKeys(50, 60, true) - + batch = db.NewBatch() if err := batch.DeleteRange([]byte(""), []byte("z")); err != nil { t.Fatal(err) @@ -580,37 +588,52 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { } // All keys should be deleted checkKeys(50, 60, false) + + // Test 7: overlapping range deletion + addKeys(50, 60) + batch = db.NewBatch() + if err := batch.DeleteRange([]byte("50"), []byte("55")); err != nil { + t.Fatal(err) + } + if err := batch.DeleteRange([]byte("52"), []byte("58")); err != nil { + t.Fatal(err) + } + if err := batch.Write(); err != nil { + t.Fatal(err) + } + checkKeys(50, 57, false) + checkKeys(58, 60, true) }) - + t.Run("BatchReplayWithDeleteRange", func(t *testing.T) { db := New() defer db.Close() - + // Setup some initial data for i := 1; i <= 10; i++ { if err := db.Put([]byte(strconv.Itoa(i)), []byte("val-"+strconv.Itoa(i))); err != nil { t.Fatal(err) } } - + // Create batch with multiple operations including DeleteRange batch1 := db.NewBatch() batch1.Put([]byte("new-key-1"), []byte("new-val-1")) - batch1.DeleteRange([]byte("3"), []byte("7")) // Should delete keys 3-6 but not 7 + batch1.DeleteRange([]byte("3"), []byte("7")) // Should delete keys 3-6 but not 7 batch1.Delete([]byte("8")) batch1.Put([]byte("new-key-2"), []byte("new-val-2")) - + // Create a second batch to replay into batch2 := db.NewBatch() if err := batch1.Replay(batch2); err != nil { t.Fatal(err) } - + // Write the second batch if err := batch2.Write(); err != nil { t.Fatal(err) } - + // Verify results // Original keys 3-6 should be deleted (inclusive of start, exclusive of end) for i := 3; i <= 6; i++ { @@ -622,7 +645,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { t.Fatalf("key %d should be deleted", i) } } - + // Key 7 should NOT be deleted (exclusive of end) has, err := db.Has([]byte("7")) if err != nil { @@ -631,7 +654,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if !has { t.Fatalf("key 7 should NOT be deleted (exclusive of end)") } - + // Key 8 should be deleted has, err = db.Has([]byte("8")) if err != nil { @@ -640,7 +663,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if has { t.Fatalf("key 8 should be deleted") } - + // New keys should be added for _, key := range []string{"new-key-1", "new-key-2"} { has, err := db.Has([]byte(key)) @@ -651,16 +674,16 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { t.Fatalf("key %s should exist", key) } } - + // Create a third batch for direct replay to database batch3 := db.NewBatch() - batch3.DeleteRange([]byte("1"), []byte("3")) // Should delete keys 1-2 but not 3 - + batch3.DeleteRange([]byte("1"), []byte("3")) // Should delete keys 1-2 but not 3 + // Replay directly to the database if err := batch3.Replay(db); err != nil { t.Fatal(err) } - + // Verify keys 1-2 are now deleted for i := 1; i <= 2; i++ { has, err := db.Has([]byte(strconv.Itoa(i))) @@ -671,7 +694,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { t.Fatalf("key %d should be deleted after direct replay", i) } } - + // Verify key 3 is NOT deleted (since it's exclusive of end) has, err = db.Has([]byte("3")) if err != nil { @@ -808,16 +831,16 @@ func BenchDatabaseSuite(b *testing.B, New func() ethdb.KeyValueStore) { for i := 0; i < count; i++ { db.Put([]byte(strconv.Itoa(i)), nil) } - + b.ResetTimer() b.ReportAllocs() - + // Create batch and delete range batch := db.NewBatch() batch.DeleteRange([]byte("0"), []byte("999999999")) batch.Write() } - + b.Run("BatchDeleteRange100", func(b *testing.B) { benchBatchDeleteRange(b, 100) }) @@ -828,42 +851,42 @@ func BenchDatabaseSuite(b *testing.B, New func() ethdb.KeyValueStore) { benchBatchDeleteRange(b, 10000) }) }) - + b.Run("BatchMixedOps", func(b *testing.B) { benchBatchMixedOps := func(b *testing.B, count int) { db := New() defer db.Close() - + // Prepare initial data for i := 0; i < count; i++ { db.Put([]byte(strconv.Itoa(i)), []byte("val")) } - + b.ResetTimer() b.ReportAllocs() - + // Create batch with mixed operations batch := db.NewBatch() - + // Add some new keys for i := 0; i < count/10; i++ { batch.Put([]byte(strconv.Itoa(count+i)), []byte("new-val")) } - + // Delete some individual keys for i := 0; i < count/20; i++ { - batch.Delete([]byte(strconv.Itoa(i*2))) + batch.Delete([]byte(strconv.Itoa(i * 2))) } - + // Delete range of keys rangeStart := count / 2 rangeEnd := count * 3 / 4 batch.DeleteRange([]byte(strconv.Itoa(rangeStart)), []byte(strconv.Itoa(rangeEnd))) - + // Write the batch batch.Write() } - + b.Run("BatchMixedOps100", func(b *testing.B) { benchBatchMixedOps(b, 100) }) diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index c1523aba1f82..736a44d73d62 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -220,7 +220,7 @@ func (db *Database) DeleteRange(start, end []byte) error { defer it.Release() var count int - for it.Next() && bytes.Compare(end, it.Key()) > 0 { + for it.Next() && (end == nil || bytes.Compare(end, it.Key()) > 0) { count++ if count > 10000 { // should not block for more than a second if err := batch.Write(); err != nil { @@ -461,48 +461,19 @@ func (b *batch) Delete(key []byte) error { return nil } -// DeleteRange removes all keys in the range [start, end) from the batch for later committing. +// DeleteRange removes all keys in the range [start, end) from the batch for +// later committing, inclusive on start, exclusive on end. +// // Note that this is a fallback implementation as leveldb does not natively // support range deletion in batches. It iterates through the database to find // keys in the range and adds them to the batch for deletion. func (b *batch) DeleteRange(start, end []byte) error { - // Special case: empty range - if len(start) == 0 && len(end) == 0 || bytes.Equal(start, end) { - return nil - } - - // Special case: delete all keys less than end - if len(start) == 0 { - it := b.db.NewIterator(nil, nil) - defer it.Release() - - for it.Next() { - key := it.Key() - if bytes.Compare(key, end) < 0 { - b.b.Delete(key) - b.size += len(key) - } - } - return it.Error() - } - - // Special case: delete all keys greater than or equal to start - if len(end) == 0 { - it := b.db.NewIterator(nil, nil) - defer it.Release() - - for it.Next() { - key := it.Key() - if bytes.Compare(key, start) >= 0 { - b.b.Delete(key) - b.size += len(key) - } - } - return it.Error() - } - // Create an iterator to scan through the keys in the range - it := b.db.NewIterator(&util.Range{Start: start, Limit: end}, nil) + slice := &util.Range{ + Start: start, // If nil, it represents the key before all keys + Limit: end, // If nil, it represents the key after all keys + } + it := b.db.NewIterator(slice, nil) defer it.Release() var count int @@ -516,7 +487,6 @@ func (b *batch) DeleteRange(start, end []byte) error { b.b.Delete(key) b.size += len(key) } - if err := it.Error(); err != nil { return err } diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index eb49ddbd7b79..bd408bfe64ae 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "sort" - "strconv" "strings" "sync" @@ -124,46 +123,24 @@ func (db *Database) Delete(key []byte) error { } // DeleteRange deletes all of the keys (and values) in the range [start,end) -// (inclusive on start, exclusive on end). +// (inclusive on start, exclusive on end). If the start is nil, it represents +// the key before all keys; if the end is nil, it represents the key after +// all keys. func (db *Database) DeleteRange(start, end []byte) error { db.lock.Lock() defer db.lock.Unlock() + if db.db == nil { return errMemorydbClosed } - - startStr := string(start) - endStr := string(end) - - if startStr == "" && endStr == "" { - return nil // Empty range, do nothing - } - - if startStr == "" { - // Delete all keys less than end - for key := range db.db { - if key < endStr { - delete(db.db, key) - } - } - return nil - } - - if endStr == "" || endStr > string([]byte{255, 255, 255, 255, 255}) { - // Delete all keys greater than or equal to start - for key := range db.db { - if key >= startStr { - delete(db.db, key) - } - } - return nil - } - - // Normal case: delete keys in range [start, end) for key := range db.db { - if key >= startStr && key < endStr { - delete(db.db, key) + if start != nil && key < string(start) { + continue } + if end != nil && key >= string(end) { + continue + } + delete(db.db, key) } return nil } @@ -249,11 +226,12 @@ func (db *Database) Len() int { // keyvalue is a key-value tuple tagged with a deletion field to allow creating // memory-database write batches. type keyvalue struct { - key string - value []byte - delete bool - rangeFrom string - rangeTo string + key string + value []byte + delete bool + + rangeFrom []byte + rangeTo []byte } // batch is a write-only memory batch that commits changes to its host @@ -281,8 +259,8 @@ func (b *batch) Delete(key []byte) error { // DeleteRange removes all keys in the range [start, end) from the batch for later committing. func (b *batch) DeleteRange(start, end []byte) error { b.writes = append(b.writes, keyvalue{ - rangeFrom: string(start), - rangeTo: string(end), + rangeFrom: start, + rangeTo: end, delete: true, }) b.size += len(start) + len(end) @@ -302,46 +280,26 @@ func (b *batch) Write() error { if b.db.db == nil { return errMemorydbClosed } - for _, keyvalue := range b.writes { - if keyvalue.delete { - if keyvalue.key != "" { + for _, entry := range b.writes { + if entry.delete { + if entry.key != "" { // Single key deletion - delete(b.db.db, keyvalue.key) - } else if keyvalue.rangeFrom != "" || keyvalue.rangeTo != "" { + delete(b.db.db, entry.key) + } else { // Range deletion (inclusive of start, exclusive of end) - // Handle special cases for empty ranges - if keyvalue.rangeFrom == keyvalue.rangeTo { - // Empty range, do nothing - continue - } - - // Check if both are numeric strings for consistent comparison - startNum, startErr := strconv.Atoi(keyvalue.rangeFrom) - endNum, endErr := strconv.Atoi(keyvalue.rangeTo) - - // If both are valid integers, use numeric comparison - if startErr == nil && endErr == nil { - for key := range b.db.db { - if keyNum, err := strconv.Atoi(key); err == nil { - if keyNum >= startNum && (keyvalue.rangeTo == "" || keyNum < endNum) { - delete(b.db.db, key) - } - } + for key := range b.db.db { + if entry.rangeFrom != nil && key < string(entry.rangeFrom) { + continue } - } else { - // Use string comparison - for key := range b.db.db { - // Handle open ranges - if (keyvalue.rangeFrom == "" || key >= keyvalue.rangeFrom) && - (keyvalue.rangeTo == "" || key < keyvalue.rangeTo) { - delete(b.db.db, key) - } + if entry.rangeTo != nil && key >= string(entry.rangeTo) { + continue } + delete(b.db.db, key) } } continue } - b.db.db[keyvalue.key] = keyvalue.value + b.db.db[entry.key] = entry.value } return nil } @@ -354,17 +312,17 @@ func (b *batch) Reset() { // Replay replays the batch contents. func (b *batch) Replay(w ethdb.KeyValueWriter) error { - for _, keyvalue := range b.writes { - if keyvalue.delete { - if keyvalue.key != "" { + for _, entry := range b.writes { + if entry.delete { + if entry.key != "" { // Single key deletion - if err := w.Delete([]byte(keyvalue.key)); err != nil { + if err := w.Delete([]byte(entry.key)); err != nil { return err } - } else if keyvalue.rangeFrom != "" || keyvalue.rangeTo != "" { + } else { // Range deletion if rangeDeleter, ok := w.(ethdb.KeyValueRangeDeleter); ok { - if err := rangeDeleter.DeleteRange([]byte(keyvalue.rangeFrom), []byte(keyvalue.rangeTo)); err != nil { + if err := rangeDeleter.DeleteRange(entry.rangeFrom, entry.rangeTo); err != nil { return err } } else { @@ -373,7 +331,7 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error { } continue } - if err := w.Put([]byte(keyvalue.key), keyvalue.value); err != nil { + if err := w.Put([]byte(entry.key), entry.value); err != nil { return err } } diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index c92718d0d62a..09062d26fcfe 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -403,9 +403,16 @@ func (d *Database) Delete(key []byte) error { func (d *Database) DeleteRange(start, end []byte) error { d.quitLock.RLock() defer d.quitLock.RUnlock() + if d.closed { return pebble.ErrClosed } + // There is no special flag to represent the end of key range + // in pebble(nil in leveldb). Use an ugly hack to construct a + // large key to represent it. + if end == nil { + end = bytes.Repeat([]byte{0xff}, 32) + } return d.db.DeleteRange(start, end, d.writeOptions) } @@ -619,8 +626,15 @@ func (b *batch) Delete(key []byte) error { return nil } -// DeleteRange removes all keys in the range [start, end) from the batch for later committing. +// DeleteRange removes all keys in the range [start, end) from the batch for +// later committing, inclusive on start, exclusive on end. func (b *batch) DeleteRange(start, end []byte) error { + // There is no special flag to represent the end of key range + // in pebble(nil in leveldb). Use an ugly hack to construct a + // large key to represent it. + if end == nil { + end = bytes.Repeat([]byte{0xff}, 32) + } if err := b.b.DeleteRange(start, end, nil); err != nil { return err } From 9bc6bf62cee0653530fd0d273f0b0aeeb8c2439b Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Wed, 18 Jun 2025 13:36:54 +0800 Subject: [PATCH 4/6] core/rawdb: add test for table --- core/rawdb/table.go | 12 ++++++++++++ core/rawdb/table_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 3fe6c522d28a..4df3c6672650 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -17,6 +17,8 @@ package rawdb import ( + "bytes" + "github.com/ethereum/go-ethereum/ethdb" ) @@ -132,6 +134,11 @@ func (t *table) Delete(key []byte) error { // DeleteRange deletes all of the keys (and values) in the range [start,end) // (inclusive on start, exclusive on end). func (t *table) DeleteRange(start, end []byte) error { + // The nilness will be lost by adding the prefix, explicitly converting it + // to a special flag representing the end of key range. + if end == nil { + end = bytes.Repeat([]byte{0xff}, 32) + } return t.db.DeleteRange(append([]byte(t.prefix), start...), append([]byte(t.prefix), end...)) } @@ -225,6 +232,11 @@ func (b *tableBatch) Delete(key []byte) error { // DeleteRange removes all keys in the range [start, end) from the batch for later committing. func (b *tableBatch) DeleteRange(start, end []byte) error { + // The nilness will be lost by adding the prefix, explicitly converting it + // to a special flag representing the end of key range. + if end == nil { + end = bytes.Repeat([]byte{0xff}, 32) + } return b.batch.DeleteRange(append([]byte(b.prefix), start...), append([]byte(b.prefix), end...)) } diff --git a/core/rawdb/table_test.go b/core/rawdb/table_test.go index aa6adf3e72b1..36fd33105952 100644 --- a/core/rawdb/table_test.go +++ b/core/rawdb/table_test.go @@ -125,4 +125,28 @@ func testTableDatabase(t *testing.T, prefix string) { // Test iterators with prefix and start point check(db.NewIterator([]byte{0xee}, nil), 0, 0) check(db.NewIterator(nil, []byte{0x00}), 6, 0) + + // Test range deletion + db.DeleteRange(nil, nil) + for _, entry := range entries { + _, err := db.Get(entry.key) + if err == nil { + t.Fatal("Unexpected item after deletion") + } + } + // Test range deletion by batch + batch = db.NewBatch() + for _, entry := range entries { + batch.Put(entry.key, entry.value) + } + batch.Write() + batch.Reset() + batch.DeleteRange(nil, nil) + batch.Write() + for _, entry := range entries { + _, err := db.Get(entry.key) + if err == nil { + t.Fatal("Unexpected item after deletion") + } + } } From ad34d98cf1b3518d08d01266707d2b4bb57a93f5 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Thu, 19 Jun 2025 20:10:36 +0800 Subject: [PATCH 5/6] core, ethdb: define MaximumKey --- core/rawdb/table.go | 6 ++---- ethdb/database.go | 13 +++++++++++++ ethdb/pebble/pebble.go | 7 +++---- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 4df3c6672650..c2c9c416eeb5 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -17,8 +17,6 @@ package rawdb import ( - "bytes" - "github.com/ethereum/go-ethereum/ethdb" ) @@ -137,7 +135,7 @@ func (t *table) DeleteRange(start, end []byte) error { // The nilness will be lost by adding the prefix, explicitly converting it // to a special flag representing the end of key range. if end == nil { - end = bytes.Repeat([]byte{0xff}, 32) + end = ethdb.MaximumKey } return t.db.DeleteRange(append([]byte(t.prefix), start...), append([]byte(t.prefix), end...)) } @@ -235,7 +233,7 @@ func (b *tableBatch) DeleteRange(start, end []byte) error { // The nilness will be lost by adding the prefix, explicitly converting it // to a special flag representing the end of key range. if end == nil { - end = bytes.Repeat([]byte{0xff}, 32) + end = ethdb.MaximumKey } return b.batch.DeleteRange(append([]byte(b.prefix), start...), append([]byte(b.prefix), end...)) } diff --git a/ethdb/database.go b/ethdb/database.go index 207a07a9a471..e5416a0419fa 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -18,10 +18,23 @@ package ethdb import ( + "bytes" "errors" "io" ) +var ( + // MaximumKey is a special marker representing the largest possible key + // in the database. + // + // All prefixed database entries will be smaller than this marker. + // For trie nodes in hash mode, we use a 32-byte slice filled with 0xFF + // because there may be shared prefixes starting with multiple 0xFF bytes. + // Using 32 bytes ensures that only a hash collision could potentially + // match or exceed it. + MaximumKey = bytes.Repeat([]byte{0xff}, 32) +) + // KeyValueReader wraps the Has and Get method of a backing data store. type KeyValueReader interface { // Has retrieves if a key is present in the key-value data store. diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index 09062d26fcfe..cca99831e620 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -18,7 +18,6 @@ package pebble import ( - "bytes" "fmt" "runtime" "strings" @@ -411,7 +410,7 @@ func (d *Database) DeleteRange(start, end []byte) error { // in pebble(nil in leveldb). Use an ugly hack to construct a // large key to represent it. if end == nil { - end = bytes.Repeat([]byte{0xff}, 32) + end = ethdb.MaximumKey } return d.db.DeleteRange(start, end, d.writeOptions) } @@ -471,7 +470,7 @@ func (d *Database) Compact(start []byte, limit []byte) error { // 0xff-s, so 32 ensures than only a hash collision could touch it. // https://github.com/cockroachdb/pebble/issues/2359#issuecomment-1443995833 if limit == nil { - limit = bytes.Repeat([]byte{0xff}, 32) + limit = ethdb.MaximumKey } return d.db.Compact(start, limit, true) // Parallelization is preferred } @@ -633,7 +632,7 @@ func (b *batch) DeleteRange(start, end []byte) error { // in pebble(nil in leveldb). Use an ugly hack to construct a // large key to represent it. if end == nil { - end = bytes.Repeat([]byte{0xff}, 32) + end = ethdb.MaximumKey } if err := b.b.DeleteRange(start, end, nil); err != nil { return err From ce07496f180801611d256a102d97edbe1232234f Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Fri, 20 Jun 2025 15:36:59 +0800 Subject: [PATCH 6/6] ethdb/memory: copy byte range --- ethdb/memorydb/memorydb.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index bd408bfe64ae..5c4c48de6490 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -18,6 +18,7 @@ package memorydb import ( + "bytes" "errors" "fmt" "sort" @@ -259,8 +260,8 @@ func (b *batch) Delete(key []byte) error { // DeleteRange removes all keys in the range [start, end) from the batch for later committing. func (b *batch) DeleteRange(start, end []byte) error { b.writes = append(b.writes, keyvalue{ - rangeFrom: start, - rangeTo: end, + rangeFrom: bytes.Clone(start), + rangeTo: bytes.Clone(end), delete: true, }) b.size += len(start) + len(end)