Merge branch 'find-skipped'

fastnear · Mar 14, 2024 · 126e459 · 126e459
2 parents af985ad + f4ea378
commit 126e459
Show file tree

Hide file tree

Showing 2 changed files with 41 additions and 0 deletions.
diff --git a/scripts/load-from-near-lake.js b/scripts/load-from-near-lake.js
@@ -194,6 +194,11 @@ if (require.main === module) {
                     .option('dump-changes', {
                         describe: 'Dump state changes into storage. Use FAST_NEAR_STORAGE_TYPE to specify storage type. Defaults to `redis`.',
                         boolean: true
+                    })
+                    .option('print-skipped', {
+                        describe: 'Print skipped blocks',
+                        boolean: true,
+                        default: false
                     }),
                 async argv => {
 
@@ -209,8 +214,14 @@ if (require.main === module) {
                 include,
                 exclude,
                 dumpChanges,
+                printSkipped
             } = argv;
 
+            if (printSkipped) {
+                await storage.cachedStorage._storage.findSkippedBlockHeights(startBlockHeight, startBlockHeight + limit);
+                return;
+            }
+
             let blocksProcessed = 0;
 
             for await (let streamerMessage of stream({

diff --git a/storage/lmdb-embedded.js b/storage/lmdb-embedded.js
@@ -96,6 +96,36 @@ class LMDBStorage {
         return this.db.put(`t:${blockHeight}`, timestamp);
     }
 
+    async findSkippedBlockHeights(startBlockHeight, endBlockHeight) {
+        console.log('Finding skipped block heights', startBlockHeight, endBlockHeight);
+        let offset = 0;
+        let keys
+        let lastKey
+        do {
+            // TODO: Is this guaranteed to be sorted?
+            // TODO: Won't range get messed up if different number of digits?
+            keys = this.db.getKeys({
+                start: `t:${startBlockHeight}`,
+                end: `t:${endBlockHeight}`,
+                offset,
+                limit: 1000,
+            }).flatMap(key => parseInt(key.slice(2))).asArray;
+
+            for (let key of keys) {
+                if (lastKey && lastKey + 1 < key) {
+                    for (let i = lastKey + 1; i < key; i++) {
+                        console.log('Skipped block height', i);
+                        // yield i;
+                    }
+                }
+                lastKey = key;
+            }
+
+            offset += keys.length;
+        } while (keys.length > 0);
+    }
+
+
     async getLatestDataBlockHeight(compKey, blockHeight) {
         const key = truncatedKey(compKey);
         const [latest] = this.db.getKeys({