From 42828ea3153db16fa453db39d34d743278091164 Mon Sep 17 00:00:00 2001 From: Mihajlo Pavlovic Date: Tue, 26 Dec 2023 14:10:30 +0100 Subject: [PATCH 1/2] Fix getGas price in set-ask, set-stake and specify which blockchain is used when getting gasPrice in submit commands (#2832) Co-authored-by: djordjekovac --- scripts/set-ask.js | 4 ++-- scripts/set-stake.js | 4 ++-- src/commands/protocols/common/submit-commit-command.js | 2 +- src/commands/protocols/common/submit-proofs-command.js | 2 +- .../protocols/update/receiver/submit-update-commit-command.js | 2 +- .../blockchain/implementation/gnosis/gnosis-service.js | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/set-ask.js b/scripts/set-ask.js index 2ce056f599..9c1c3deed4 100644 --- a/scripts/set-ask.js +++ b/scripts/set-ask.js @@ -31,11 +31,11 @@ async function getGasPrice(gasPriceOracleLink) { if ( gasPriceOracleLink === 'https://api.gnosisscan.io/api?module=proxy&action=eth_gasPrice' ) { - gasPrice = Number(response.result, 10); + gasPrice = Number(response.data.result, 10); } else if ( gasPriceOracleLink === 'https://blockscout.chiadochain.net/api/v1/gas-price-oracle' ) { - gasPrice = Math.round(response.average * 1e9); + gasPrice = Math.round(response.data.average * 1e9); } else { gasPrice = Math.round(response.result * 1e9); } diff --git a/scripts/set-stake.js b/scripts/set-stake.js index 64938814e1..4db4be7fb0 100644 --- a/scripts/set-stake.js +++ b/scripts/set-stake.js @@ -38,11 +38,11 @@ async function getGasPrice(gasPriceOracleLink) { if ( gasPriceOracleLink === 'https://api.gnosisscan.io/api?module=proxy&action=eth_gasPrice' ) { - gasPrice = Number(response.result, 10); + gasPrice = Number(response.data.result, 10); } else if ( gasPriceOracleLink === 'https://blockscout.chiadochain.net/api/v1/gas-price-oracle' ) { - gasPrice = Math.round(response.average * 1e9); + gasPrice = Math.round(response.data.average * 1e9); } else { gasPrice = Math.round(response.result * 1e9); } diff --git a/src/commands/protocols/common/submit-commit-command.js b/src/commands/protocols/common/submit-commit-command.js index 124e985231..a0484d1c6c 100644 --- a/src/commands/protocols/common/submit-commit-command.js +++ b/src/commands/protocols/common/submit-commit-command.js @@ -84,7 +84,7 @@ class SubmitCommitCommand extends Command { return Command.empty(); } - const txGasPrice = gasPrice ?? (await this.blockchainModuleManager.getGasPrice()); + const txGasPrice = gasPrice ?? (await this.blockchainModuleManager.getGasPrice(blockchain)); const transactionCompletePromise = new Promise((resolve, reject) => { this.blockchainModuleManager.submitCommit( diff --git a/src/commands/protocols/common/submit-proofs-command.js b/src/commands/protocols/common/submit-proofs-command.js index 462d51fb27..f3c7e6d9ca 100644 --- a/src/commands/protocols/common/submit-proofs-command.js +++ b/src/commands/protocols/common/submit-proofs-command.js @@ -134,7 +134,7 @@ class SubmitProofsCommand extends Command { return Command.empty(); } - const txGasPrice = gasPrice ?? (await this.blockchainModuleManager.getGasPrice()); + const txGasPrice = gasPrice ?? (await this.blockchainModuleManager.getGasPrice(blockchain)); const transactionCompletePromise = new Promise((resolve, reject) => { this.blockchainModuleManager.sendProof( diff --git a/src/commands/protocols/update/receiver/submit-update-commit-command.js b/src/commands/protocols/update/receiver/submit-update-commit-command.js index e253372f87..d8c9b27297 100644 --- a/src/commands/protocols/update/receiver/submit-update-commit-command.js +++ b/src/commands/protocols/update/receiver/submit-update-commit-command.js @@ -68,7 +68,7 @@ class SubmitUpdateCommitCommand extends Command { return Command.empty(); } - const txGasPrice = gasPrice ?? (await this.blockchainModuleManager.getGasPrice()); + const txGasPrice = gasPrice ?? (await this.blockchainModuleManager.getGasPrice(blockchain)); const transactionCompletePromise = new Promise((resolve, reject) => { this.blockchainModuleManager.submitUpdateCommit( diff --git a/src/modules/blockchain/implementation/gnosis/gnosis-service.js b/src/modules/blockchain/implementation/gnosis/gnosis-service.js index 4c74dbb03b..a265283c0a 100644 --- a/src/modules/blockchain/implementation/gnosis/gnosis-service.js +++ b/src/modules/blockchain/implementation/gnosis/gnosis-service.js @@ -19,9 +19,9 @@ class GnosisService extends Web3Service { const response = await axios.get(this.config.gasPriceOracleLink); let gasPrice; if (this.config.name.split(':')[1] === '100') { - gasPrice = Number(response.result, 10); + gasPrice = Number(response.data.result, 10); } else if (this.config.name.split(':')[1] === '10200') { - gasPrice = Math.round(response.average * 1e9); + gasPrice = Math.round(response.data.average * 1e9); } this.logger.debug(`Gas price on Gnosis: ${gasPrice}`); return gasPrice; From 6b02de4387b9dc97d9e880079b00b56ce9079095 Mon Sep 17 00:00:00 2001 From: Nikola Todorovic Date: Tue, 26 Dec 2023 14:29:21 +0100 Subject: [PATCH 2/2] Migration for events synchronization & events chunking for events insertion (#2855) * Upp sharding table migration enabled * Update event insert in chunks * version bump * Add migration for makring staking events as processed * Updated migration executor added mark staking events as processed migration * Updated migration added await on get latest block * Updated migration for all contracts * Migration updated * Bump version to 6.1.2 --------- Co-authored-by: Djordje Kovacevic --- ot-node.js | 12 ++++++ package-lock.json | 4 +- package.json | 2 +- ...k-staking-events-as-processed-migration.js | 25 ++++++++++++ src/migration/migration-executor.js | 28 +++++++++++++- .../blockchain-event-repository.js | 38 ++++++++++++------- 6 files changed, 91 insertions(+), 18 deletions(-) create mode 100644 src/migration/mark-staking-events-as-processed-migration.js diff --git a/ot-node.js b/ot-node.js index e2e59b0224..6bdcf4fa9b 100644 --- a/ot-node.js +++ b/ot-node.js @@ -54,6 +54,12 @@ class OTNode { await this.initializeCommandExecutor(); await this.initializeShardingTableService(); + await MigrationExecutor.executeMarkStakingEventsAsProcessedMigration( + this.container, + this.logger, + this.config, + ); + MigrationExecutor.executeUalExtensionTripleStoreMigration( this.container, this.logger, @@ -62,6 +68,12 @@ class OTNode { await this.initializeBlockchainEventListenerService(); }); + await MigrationExecutor.executePullShardingTableMigration( + this.container, + this.logger, + this.config, + ); + await this.initializeRouters(); await this.startNetworkModule(); this.startTelemetryModule(); diff --git a/package-lock.json b/package-lock.json index 43c41e49d3..71744602bc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "origintrail_node", - "version": "6.1.1", + "version": "6.1.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "origintrail_node", - "version": "6.1.1", + "version": "6.1.2", "license": "ISC", "dependencies": { "@comunica/query-sparql": "^2.4.3", diff --git a/package.json b/package.json index 7ddcc1b845..36d67b1463 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "origintrail_node", - "version": "6.1.1", + "version": "6.1.2", "description": "OTNode V6", "main": "index.js", "type": "module", diff --git a/src/migration/mark-staking-events-as-processed-migration.js b/src/migration/mark-staking-events-as-processed-migration.js new file mode 100644 index 0000000000..2fe63ac39d --- /dev/null +++ b/src/migration/mark-staking-events-as-processed-migration.js @@ -0,0 +1,25 @@ +/* eslint-disable no-await-in-loop */ +import BaseMigration from './base-migration.js'; + +class MarkStakingEventsAsProcessedMigration extends BaseMigration { + constructor(migrationName, logger, config, repositoryModuleManager, blockchainModuleManager) { + super(migrationName, logger, config); + this.repositoryModuleManager = repositoryModuleManager; + this.blockchainModuleManager = blockchainModuleManager; + } + + async executeMigration() { + this.logger.info('Marking old blockchain events as processed'); + for (const blockchain of this.blockchainModuleManager.getImplementationNames()) { + const timestamp = Date.now(); + const block = await this.blockchainModuleManager.getLatestBlock(blockchain); + const query = `update blockchain + set last_checked_block = ${block.number}, + last_checked_timestamp = ${timestamp} + where blockchain_id = 'otp:2043'`; + await this.repositoryModuleManager.query(query); + } + } +} + +export default MarkStakingEventsAsProcessedMigration; diff --git a/src/migration/migration-executor.js b/src/migration/migration-executor.js index c639c10b47..847a9308cf 100644 --- a/src/migration/migration-executor.js +++ b/src/migration/migration-executor.js @@ -14,6 +14,7 @@ import ServiceAgreementsDataInspector from './service-agreements-data-inspector. import ServiceAgreementsInvalidDataMigration from './service-agreements-invalid-data-migration.js'; import UalExtensionUserConfigurationMigration from './ual-extension-user-configuration-migration.js'; import UalExtensionTripleStoreMigration from './ual-extension-triple-store-migration.js'; +import MarkStakingEventsAsProcessedMigration from './mark-staking-events-as-processed-migration.js'; class MigrationExecutor { static async executePullShardingTableMigration(container, logger, config) { @@ -28,7 +29,7 @@ class MigrationExecutor { const validationModuleManager = container.resolve('validationModuleManager'); const migration = new PullBlockchainShardingTableMigration( - 'pullShardingTableMigrationV613', + 'pullShardingTableMigrationV611', logger, config, repositoryModuleManager, @@ -338,6 +339,31 @@ class MigrationExecutor { } } + static async executeMarkStakingEventsAsProcessedMigration(container, logger, config) { + if (process.env.NODE_ENV !== NODE_ENVIRONMENTS.MAINNET) return; + + const repositoryModuleManager = container.resolve('repositoryModuleManager'); + const blockchainModuleManager = container.resolve('blockchainModuleManager'); + + const migration = new MarkStakingEventsAsProcessedMigration( + 'markStakingEventsAsProcessedMigration', + logger, + config, + repositoryModuleManager, + blockchainModuleManager, + ); + if (!(await migration.migrationAlreadyExecuted())) { + try { + await migration.migrate(); + } catch (error) { + logger.error( + `Unable to execute mark staking events as processed migration. Error: ${error.message}`, + ); + this.exitNode(1); + } + } + } + static exitNode(code = 0) { process.exit(code); } diff --git a/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js b/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js index c03087cadb..f19fe1530f 100644 --- a/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js +++ b/src/modules/repository/implementation/sequelize/repositories/blockchain-event-repository.js @@ -7,20 +7,30 @@ class BlockchainEventRepository { } async insertBlockchainEvents(events) { - const inserted = await this.model.bulkCreate( - events.map((event) => ({ - contract: event.contract, - event: event.event, - data: event.data, - block: event.block, - blockchainId: event.blockchainId, - processed: false, - })), - { - ignoreDuplicates: true, - }, - ); - return inserted.map((event) => event.dataValues); + const chunkSize = 10000; + let insertedEvents = []; + + for (let i = 0; i < events.length; i += chunkSize) { + const chunk = events.slice(i, i + chunkSize); + // eslint-disable-next-line no-await-in-loop + const insertedChunk = await this.model.bulkCreate( + chunk.map((event) => ({ + contract: event.contract, + event: event.event, + data: event.data, + block: event.block, + blockchainId: event.blockchainId, + processed: false, + })), + { + ignoreDuplicates: true, + }, + ); + + insertedEvents = insertedEvents.concat(insertedChunk.map((event) => event.dataValues)); + } + + return insertedEvents; } async getAllUnprocessedBlockchainEvents(eventNames, blockchainId) {