diff --git a/cmd/rpcdaemon/commands/zkevm_api.go b/cmd/rpcdaemon/commands/zkevm_api.go index e8bbe3bc813..d505cc89c47 100644 --- a/cmd/rpcdaemon/commands/zkevm_api.go +++ b/cmd/rpcdaemon/commands/zkevm_api.go @@ -236,6 +236,31 @@ func (api *ZkEvmAPIImpl) GetBatchDataByNumbers(ctx context.Context, batchNumbers hermezDb := hermez_db.NewHermezDbReader(tx) + // use inbuilt rpc.BlockNumber type to implement the 'latest' behaviour + // highest block/batch tied to last block synced + // unless the node is still syncing - in which case 'current block' is used + // this is the batch number of stage progress of the Finish stage + + highestBlock, err := rawdb.ReadLastBlockSynced(tx) + if err != nil { + return nil, err + } + + highestBatchNo, err := hermezDb.GetBatchNoByL2Block(highestBlock.NumberU64()) + if err != nil { + return nil, err + } + + // check sync status of node + syncing, err := api.ethApi.Syncing(ctx) + if err != nil { + return nil, err + } + if syncing != nil && syncing != false { + bn := syncing.(map[string]interface{})["currentBlock"] + highestBatchNo, err = hermezDb.GetBatchNoByL2Block(uint64(bn.(hexutil.Uint64))) + } + bds := make([]*types.BatchDataSlim, 0, len(batchNumbers)) for _, batchNumber := range batchNumbers { @@ -244,15 +269,6 @@ func (api *ZkEvmAPIImpl) GetBatchDataByNumbers(ctx context.Context, batchNumbers Empty: false, } - highestBlock, err := rawdb.ReadLastBlockSynced(tx) - if err != nil { - return nil, err - } - highestBatchNo, err := hermezDb.GetBatchNoByL2Block(highestBlock.NumberU64()) - if err != nil { - return nil, err - } - // return null if we're not at this block height yet if batchNumber > rpc.BlockNumber(highestBatchNo) { bd.Empty = true @@ -363,16 +379,30 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B defer tx.Rollback() hermezDb := hermez_db.NewHermezDbReader(tx) + // use inbuilt rpc.BlockNumber type to implement the 'latest' behaviour + // highest block/batch tied to last block synced + // unless the node is still syncing - in which case 'current block' is used + // this is the batch number of stage progress of the Finish stage + highestBlock, err := rawdb.ReadLastBlockSynced(tx) if err != nil { return nil, err } + highestBatchNo, err := hermezDb.GetBatchNoByL2Block(highestBlock.NumberU64()) if err != nil { return nil, err } - // return null if we're not at this block height yet + // check sync status of node + syncing, err := api.ethApi.Syncing(ctx) + if err != nil { + return nil, err + } + if syncing != nil && syncing != false { + bn := syncing.(map[string]interface{})["currentBlock"] + highestBatchNo, err = hermezDb.GetBatchNoByL2Block(uint64(bn.(hexutil.Uint64))) + } if batchNumber > rpc.BlockNumber(highestBatchNo) { return nil, nil } @@ -411,6 +441,8 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B // last block in batch data batch.Coinbase = block.Coinbase() batch.StateRoot = block.Root() + + // TODO: this logic is wrong it is the L1 verification timestamp we need batch.Timestamp = types.ArgUint64(block.Time()) // block numbers in batch @@ -516,40 +548,12 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B } // global exit root of batch - batchGer, foundBatchGerNumber, err := hermezDb.GetLastBatchGlobalExitRoot(batchNo) - if err != nil { - return nil, err - } - - // get last block in batch - lastBlockInbatch, err := hermezDb.GetHighestBlockInBatch(batchNo) - if err != nil { - return nil, err - } - - // get latest found ger by block - latestBlockHer, blockNum, err := hermezDb.GetLastBlockGlobalExitRoot(lastBlockInbatch) - if err != nil { - return nil, err - } - - //get latest block ger batch number - latestBlockGerBatchNumber, err := hermezDb.GetBatchNoByL2Block(blockNum) + batchGer, _, err := hermezDb.GetLastBlockGlobalExitRoot(blockNo) if err != nil { return nil, err } - var ger *common.Hash - if batchGer != nil { - ger = &batchGer.GlobalExitRoot - } - if foundBatchGerNumber < latestBlockGerBatchNumber { - ger = &latestBlockHer - } - - if ger != nil { - batch.GlobalExitRoot = *ger - } + batch.GlobalExitRoot = batchGer // sequence seq, err := hermezDb.GetSequenceByBatchNo(batchNo) @@ -576,7 +580,7 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B } // exit roots (MainnetExitRoot, RollupExitRoot) - infoTreeUpdate, err := hermezDb.GetL1InfoTreeUpdateByGer(batch.GlobalExitRoot) + infoTreeUpdate, err := hermezDb.GetL1InfoTreeUpdateByGer(batchGer) if err != nil { return nil, err } @@ -644,12 +648,16 @@ func (api *ZkEvmAPIImpl) GetBatchByNumber(ctx context.Context, batchNumber rpc.B if forkId < 12 { // get the previous batches exit roots prevBatchNo := batchNo - 1 - prevBatchGer, _, err := hermezDb.GetLastBatchGlobalExitRoot(prevBatchNo) + prevBatchHighestBlock, err := hermezDb.GetHighestBlockInBatch(prevBatchNo) + if err != nil { + return nil, err + } + prevBatchGer, _, err := hermezDb.GetLastBlockGlobalExitRoot(prevBatchHighestBlock) if err != nil { return nil, err } - itu, err := hermezDb.GetL1InfoTreeUpdateByGer(prevBatchGer.GlobalExitRoot) + itu, err := hermezDb.GetL1InfoTreeUpdateByGer(prevBatchGer) if err != nil { return nil, err } diff --git a/zk/debug_tools/rpc-batch-compare/main.go b/zk/debug_tools/rpc-batch-compare/main.go new file mode 100644 index 00000000000..3fed11248b0 --- /dev/null +++ b/zk/debug_tools/rpc-batch-compare/main.go @@ -0,0 +1,181 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "log" + "math/big" + "net/http" + "os" + + "github.com/google/go-cmp/cmp" + "io" +) + +func getBatchNumber(url string) (*big.Int, error) { + requestBody, _ := json.Marshal(map[string]interface{}{ + "jsonrpc": "2.0", + "method": "zkevm_batchNumber", + "params": []interface{}{}, + "id": 1, + }) + + resp, err := http.Post(url, "application/json", bytes.NewBuffer(requestBody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %v", err) + } + + var result map[string]interface{} + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %v", err) + } + + if errorField, ok := result["error"]; ok { + return nil, fmt.Errorf("node error: %v", errorField) + } + + batchNumberHex, ok := result["result"].(string) + if !ok { + return nil, fmt.Errorf("invalid response format") + } + + batchNumber := new(big.Int) + if _, ok := batchNumber.SetString(batchNumberHex[2:], 16); !ok { + return nil, fmt.Errorf("failed to convert batch number to big.Int") + } + return batchNumber, nil +} + +func getBatchByNumber(url string, number *big.Int) (map[string]interface{}, error) { + requestBody, _ := json.Marshal(map[string]interface{}{ + "jsonrpc": "2.0", + "method": "zkevm_getBatchByNumber", + "params": []interface{}{number.String(), true}, + "id": 1, + }) + + resp, err := http.Post(url, "application/json", bytes.NewBuffer(requestBody)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %v", err) + } + + var result map[string]interface{} + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("failed to unmarshal response: %v", err) + } + + if errorField, ok := result["error"]; ok { + return nil, fmt.Errorf("node error: %v", errorField) + } + + batchData, ok := result["result"].(map[string]interface{}) + if !ok || batchData == nil { + return nil, fmt.Errorf("batch not found") + } + + return batchData, nil +} + +func compareBatches(erigonURL, legacyURL string, batchNumber *big.Int) (string, error) { + batch1, err := getBatchByNumber(erigonURL, batchNumber) + if err != nil { + return "", fmt.Errorf("Error getting batch %d from Erigon node: %v", batchNumber, err) + } + + batch2, err := getBatchByNumber(legacyURL, batchNumber) + if err != nil { + return "", fmt.Errorf("Error getting batch %d from Legacy node: %v", batchNumber, err) + } + + // ignore list + il := []string{ + "timestamp", + "verifyBatchTxHash", + "sendSequencesTxHash", + "accInputHash", + "globalExitRoot", + "mainnetExitRoot", + "rollupExitRoot", + } + for _, i := range il { + delete(batch1, i) + delete(batch2, i) + } + + if !cmp.Equal(batch1, batch2) { + return fmt.Sprintf("Mismatch at batch %d:\nErigon vs Legacy:\n%s", + batchNumber, cmp.Diff(batch1, batch2)), nil + } + return "", nil +} + +func main() { + erigonURL := flag.String("erigon", "http://localhost:8545", "RPC URL of the Erigon node") + legacyURL := flag.String("legacy", "http://localhost:8546", "RPC URL of the Legacy node") + skip := flag.Int("skip", 1, "Number of batches to skip between each check") + numBatches := flag.Int("batches", 1000, "Number of batches to check") + startOffset := flag.Int("offset", 0, "Offset from highest getBatchNumber") + overrideStartAt := flag.Int("override", 0, "Override start batch number") + flag.Parse() + + erigonLatestBatch, err := getBatchNumber(*erigonURL) + if err != nil { + log.Fatalf("Failed to get latest batch number from Erigon node: %v", err) + } + log.Println("Erigon latest batch number: ", erigonLatestBatch) + + legacyLatestBatch, err := getBatchNumber(*legacyURL) + if err != nil { + log.Fatalf("Failed to get latest batch number from Legacy node: %v", err) + } + log.Println("Legacy latest batch number: ", legacyLatestBatch) + + startBatch := legacyLatestBatch + if erigonLatestBatch.Cmp(startBatch) < 0 { + startBatch = erigonLatestBatch + } + + // offset start batch + startBatch = new(big.Int).Sub(startBatch, big.NewInt(int64(*startOffset))) + + if *overrideStartAt != 0 { + startBatch = big.NewInt(int64(*overrideStartAt)) + log.Println("Overriding start batch to", startBatch) + } + + log.Printf("Checking %d batches\n", *numBatches) + log.Printf("Starting from batch %d\n", startBatch) + log.Printf("Skipping %d batches\n", *skip) + + for i := 0; i < *numBatches; i++ { + log.Println("Checking batch", i+1, "of", *numBatches) + batchNumber := new(big.Int).Sub(startBatch, big.NewInt(int64(i**skip))) + diff, err := compareBatches(*erigonURL, *legacyURL, batchNumber) + if err != nil { + log.Println(err) + os.Exit(1) + } + if diff != "" { + log.Println(diff) + os.Exit(1) + } + log.Println("Batch", batchNumber, "matches") + } + + log.Println("No differences found") + os.Exit(0) +} diff --git a/zk/hermez_db/db.go b/zk/hermez_db/db.go index fa73b4000e1..453b202463e 100644 --- a/zk/hermez_db/db.go +++ b/zk/hermez_db/db.go @@ -748,6 +748,7 @@ func (db *HermezDb) WriteBatchGlobalExitRoot(batchNumber uint64, ger dstypes.Ger return db.tx.Put(GLOBAL_EXIT_ROOTS_BATCHES, Uint64ToBytes(batchNumber), ger.EncodeToBytes()) } +// deprecated: post etrog this will not work func (db *HermezDbReader) GetBatchGlobalExitRoots(fromBatchNum, toBatchNum uint64) (*[]dstypes.GerUpdate, error) { c, err := db.tx.Cursor(GLOBAL_EXIT_ROOTS_BATCHES) if err != nil { @@ -775,6 +776,7 @@ func (db *HermezDbReader) GetBatchGlobalExitRoots(fromBatchNum, toBatchNum uint6 return &gers, err } +// GetLastBatchGlobalExitRoot deprecated: post etrog this will not work func (db *HermezDbReader) GetLastBatchGlobalExitRoot(batchNum uint64) (*dstypes.GerUpdate, uint64, error) { c, err := db.tx.Cursor(GLOBAL_EXIT_ROOTS_BATCHES) if err != nil { @@ -820,6 +822,7 @@ func (db *HermezDbReader) GetBatchGlobalExitRootsProto(fromBatchNum, toBatchNum return gersProto, nil } +// GetBatchGlobalExitRoot deprecated: post etrog this will not work func (db *HermezDbReader) GetBatchGlobalExitRoot(batchNum uint64) (*dstypes.GerUpdate, error) { gerUpdateBytes, err := db.tx.GetOne(GLOBAL_EXIT_ROOTS_BATCHES, Uint64ToBytes(batchNum)) if err != nil { diff --git a/zk/stages/stage_batches.go b/zk/stages/stage_batches.go index 3f50e8aae4f..636d1e4fc90 100644 --- a/zk/stages/stage_batches.go +++ b/zk/stages/stage_batches.go @@ -31,7 +31,8 @@ const ( preForkId7BlockGasLimit = 30_000_000 forkId7BlockGasLimit = 18446744073709551615 // 0xffffffffffffffff forkId8BlockGasLimit = 1125899906842624 // 0x4000000000000 - HIGHEST_KNOWN_FORK = 9 + HIGHEST_KNOWN_FORK = 11 + STAGE_PROGRESS_SAVE = 3000000 ) type ErigonDb interface { diff --git a/zk/stages/stage_l1_info_tree.go b/zk/stages/stage_l1_info_tree.go index 3af89c8d678..70498a86b11 100644 --- a/zk/stages/stage_l1_info_tree.go +++ b/zk/stages/stage_l1_info_tree.go @@ -1,20 +1,21 @@ package stages import ( + "context" + "fmt" + "sort" + "time" + + "github.com/gateway-fm/cdk-erigon-lib/common" "github.com/gateway-fm/cdk-erigon-lib/kv" + "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/eth/stagedsync" - "fmt" - "github.com/ledgerwatch/log/v3" - "context" - "github.com/ledgerwatch/erigon/zk/hermez_db" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" - "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/zk/contracts" - "sort" - "time" + "github.com/ledgerwatch/erigon/zk/hermez_db" "github.com/ledgerwatch/erigon/zk/l1infotree" - "github.com/gateway-fm/cdk-erigon-lib/common" + "github.com/ledgerwatch/log/v3" ) type L1InfoTreeCfg struct { @@ -93,7 +94,16 @@ LOOP: // sort the logs by block number - it is important that we process them in order to get the index correct sort.Slice(allLogs, func(i, j int) bool { - return allLogs[i].BlockNumber < allLogs[j].BlockNumber + l1 := allLogs[i] + l2 := allLogs[j] + // first sort by block number and if equal then by tx index + if l1.BlockNumber != l2.BlockNumber { + return l1.BlockNumber < l2.BlockNumber + } + if l1.TxIndex != l2.TxIndex { + return l1.TxIndex < l2.TxIndex + } + return l1.Index < l2.Index }) // chunk the logs into batches, so we don't overload the RPC endpoints too much at once @@ -152,6 +162,14 @@ LOOP: if err != nil { return err } + log.Debug("New L1 Index", + "index", latestUpdate.Index, + "root", newRoot.String(), + "mainnet", latestUpdate.MainnetExitRoot.String(), + "rollup", latestUpdate.RollupExitRoot.String(), + "ger", latestUpdate.GER.String(), + "parent", latestUpdate.ParentHash.String(), + ) err = hermezDb.WriteL1InfoTreeRoot(common.BytesToHash(newRoot[:]), latestUpdate.Index) if err != nil { diff --git a/zk/tx/tx.go b/zk/tx/tx.go index 67a9e75f566..b7124e4c6fd 100644 --- a/zk/tx/tx.go +++ b/zk/tx/tx.go @@ -322,7 +322,7 @@ func TransactionToL2Data(tx types.Transaction, forkId uint16, efficiencyPercenta removeLeadingZeroesFromBytes(gas), to, // don't remove leading 0s from addr removeLeadingZeroesFromBytes(valueBytes), - removeLeadingZeroesFromBytes(tx.GetData()), + tx.GetData(), } if !tx.GetChainID().Eq(uint256.NewInt(0)) || !(v.Eq(uint256.NewInt(27)) || v.Eq(uint256.NewInt(28))) { @@ -336,10 +336,6 @@ func TransactionToL2Data(tx types.Transaction, forkId uint16, efficiencyPercenta return nil, err } - if strings.Contains(hex.EncodeToString(encoded), "937c4") { - log.Debug("break me!") - } - // reverse the eip-155 changes for the V value for transport v = GetDecodedV(tx, v) txV := new(big.Int).SetBytes(v.Bytes()) @@ -357,11 +353,6 @@ func TransactionToL2Data(tx types.Transaction, forkId uint16, efficiencyPercenta encoded = append(encoded, ep...) } - // break if encoded contains 937c4 - if strings.Contains(hex.EncodeToString(encoded), "937c4") { - log.Debug("break me!") - } - return encoded, nil }