From cfd815801bd097531f70b4d86e3f50c368577bb9 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 15 Jan 2024 19:12:28 +0000 Subject: [PATCH 01/14] skale-admin#832 add change ip functionality --- core/node.py | 42 +----- core/nodes.py | 126 ++++++++++++++++++ core/schains/checks.py | 17 ++- core/schains/external_config.py | 12 +- core/schains/monitor/action.py | 37 +++-- core/schains/monitor/config_monitor.py | 6 +- core/schains/monitor/main.py | 4 + core/schains/monitor/skaled_monitor.py | 45 +++++-- hardhat-node | 2 +- requirements-dev.txt | 2 +- tests/conftest.py | 1 + tests/nodes_test.py | 68 ++++++++++ .../monitor/action/config_action_test.py | 5 + tests/schains/monitor/config_monitor_test.py | 44 +++++- tests/schains/monitor/skaled_monitor_test.py | 71 +++++++++- tools/configs/__init__.py | 2 + 16 files changed, 411 insertions(+), 73 deletions(-) create mode 100644 core/nodes.py create mode 100644 tests/nodes_test.py diff --git a/core/node.py b/core/node.py index ba4419bbf..3ba103737 100644 --- a/core/node.py +++ b/core/node.py @@ -20,7 +20,6 @@ import json import time import os -import socket import psutil import logging import platform @@ -43,7 +42,7 @@ from core.filebeat import update_filebeat_service -from tools.configs import CHECK_REPORT_PATH, META_FILEPATH, WATCHDOG_PORT +from tools.configs import CHECK_REPORT_PATH, META_FILEPATH from tools.helper import read_json from tools.str_formatters import arguments_list_string from tools.wallet_utils import check_required_balance @@ -345,45 +344,10 @@ def get_btrfs_info() -> dict: } -def is_port_open(ip, port): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(1) - try: - s.connect((ip, int(port))) - s.shutdown(1) - return True - except Exception: - return False - - -def check_validator_nodes(skale, node_id): - try: - node = skale.nodes.get(node_id) - node_ids = skale.nodes.get_validator_node_indices(node['validator_id']) - - try: - node_ids.remove(node_id) - except ValueError: - logger.warning( - f'node_id: {node_id} was not found in validator nodes: {node_ids}') - - res = [] - for node_id in node_ids: - if str(skale.nodes.get_node_status(node_id)) == str(NodeStatus.ACTIVE.value): - ip_bytes = skale.nodes.contract.functions.getNodeIP( - node_id).call() - ip = ip_from_bytes(ip_bytes) - res.append([node_id, ip, is_port_open(ip, WATCHDOG_PORT)]) - logger.info(f'validator_nodes check - node_id: {node_id}, res: {res}') - except Exception as err: - return {'status': 1, 'errors': [err]} - return {'status': 0, 'data': res} - - def get_check_report(report_path: str = CHECK_REPORT_PATH) -> Dict: - if not os.path.isfile(CHECK_REPORT_PATH): + if not os.path.isfile(report_path): return {} - with open(CHECK_REPORT_PATH) as report_file: + with open(report_path) as report_file: return json.load(report_file) diff --git a/core/nodes.py b/core/nodes.py new file mode 100644 index 000000000..82fe94ea9 --- /dev/null +++ b/core/nodes.py @@ -0,0 +1,126 @@ +# -*- coding: utf-8 -*- +# +# This file is part of SKALE Admin +# +# Copyright (C) 2024 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import socket +import logging +from typing import List, TypedDict + +from skale import Skale +from skale.utils.helper import ip_from_bytes +from skale.schain_config.generator import get_nodes_for_schain + +from core.node import NodeStatus +from tools.configs import WATCHDOG_PORT, CHANGE_IP_DELAY + +logger = logging.getLogger(__name__) + + +class ManagerNodeInfo(TypedDict): + name: str + ip: str + publicIP: str + port: int + start_block: int + last_reward_date: int + finish_time: int + status: int + validator_id: int + publicKey: str + domain_name: str + + +class ExtendedManagerNodeInfo(ManagerNodeInfo): + ip_change_ts: int + + +def get_current_nodes(skale: Skale, name: str) -> List[ExtendedManagerNodeInfo]: + current_nodes: ManagerNodeInfo = get_nodes_for_schain(skale, name) + for node in current_nodes: + node['ip_change_ts'] = skale.nodes.get_last_change_ip_time(node['id']) + node['ip'] = ip_from_bytes(node['ip']) + node['publicIP'] = ip_from_bytes(node['publicIP']) + return current_nodes + + +def get_current_ips(current_nodes: List[ExtendedManagerNodeInfo]) -> List[str]: + return [node['ip'] for node in current_nodes] + + +def get_max_ip_change_ts(current_nodes: List[ExtendedManagerNodeInfo]) -> int | None: + max_ip_change_ts = max(current_nodes, key=lambda node: node['ip_change_ts'])['ip_change_ts'] + return None if max_ip_change_ts == 0 else max_ip_change_ts + + +def calc_reload_ts(current_nodes: List[ExtendedManagerNodeInfo], node_index: int) -> int: + max_ip_change_ts = get_max_ip_change_ts(current_nodes) + if max_ip_change_ts is None: + return + return max_ip_change_ts + get_node_delay(node_index) + + +def get_node_delay(node_index: int) -> int: + """ + Returns delay for node in seconds. + Example: for node with index 3 and delay 300 it will be 1200 seconds + """ + return CHANGE_IP_DELAY * (node_index + 1) + + +def get_node_index_in_group(skale: Skale, schain_name: str, node_id: int) -> int | None: + """Returns node index in group or None if node is not in group""" + try: + node_ids = skale.schains_internal.get_node_ids_for_schain(schain_name) + return node_ids.index(node_id) + except ValueError: + return None + + +def is_port_open(ip, port): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(1) + try: + s.connect((ip, int(port))) + s.shutdown(1) + return True + except Exception: + return False + + +def check_validator_nodes(skale, node_id): + try: + node = skale.nodes.get(node_id) + node_ids = skale.nodes.get_validator_node_indices(node['validator_id']) + + try: + node_ids.remove(node_id) + except ValueError: + logger.warning( + f'node_id: {node_id} was not found in validator nodes: {node_ids}') + + res = [] + for node_id in node_ids: + if str(skale.nodes.get_node_status(node_id)) == str(NodeStatus.ACTIVE.value): + ip_bytes = skale.nodes.contract.functions.getNodeIP( + node_id).call() + ip = ip_from_bytes(ip_bytes) + res.append([node_id, ip, is_port_open(ip, WATCHDOG_PORT)]) + logger.info(f'validator_nodes check - node_id: {node_id}, res: {res}') + except Exception as err: + return {'status': 1, 'errors': [err]} + return {'status': 0, 'data': res} diff --git a/core/schains/checks.py b/core/schains/checks.py index 704c10fa8..ef44556a8 100644 --- a/core/schains/checks.py +++ b/core/schains/checks.py @@ -22,6 +22,7 @@ import time from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional +from core.nodes import get_current_ips from core.schains.config.directory import get_schain_check_filepath from core.schains.config.file_manager import ConfigFileManager @@ -141,6 +142,7 @@ def __init__(self, schain_record: SChainRecord, rotation_id: int, stream_version: str, + current_nodes, estate: ExternalState, econfig: Optional[ExternalConfig] = None ) -> None: @@ -149,6 +151,7 @@ def __init__(self, self.schain_record = schain_record self.rotation_id = rotation_id self.stream_version = stream_version + self.current_nodes = current_nodes self.estate = estate self.econfig = econfig or ExternalConfig(schain_name) self.cfm: ConfigFileManager = ConfigFileManager( @@ -173,11 +176,21 @@ def dkg(self) -> CheckRes: ) return CheckRes(os.path.isfile(secret_key_share_filepath)) + @property + def node_ips(self) -> CheckRes: + """Checks that IP list on the skale-manager is the same as in the config""" + res = False + if self.cfm.upstream_exist_for_rotation_id(self.rotation_id): + conf = self.cfm.skaled_config + node_ips = get_node_ips_from_config(conf) + current_ips = get_current_ips(self.current_nodes) + res = set(node_ips) == set(current_ips) + return CheckRes(res) + @property def upstream_config(self) -> CheckRes: """Checks that config exists for rotation id and stream""" exists = self.cfm.upstream_exist_for_rotation_id(self.rotation_id) - logger.debug('Upstream configs status for %s: %s', self.name, exists) return CheckRes( exists and @@ -203,7 +216,7 @@ def __init__( rule_controller: IRuleController, *, econfig: Optional[ExternalConfig] = None, - dutils: DockerUtils = None + dutils: DockerUtils | None = None ): self.name = schain_name self.schain_record = schain_record diff --git a/core/schains/external_config.py b/core/schains/external_config.py index 4975c8426..8bdd63c88 100644 --- a/core/schains/external_config.py +++ b/core/schains/external_config.py @@ -13,12 +13,14 @@ class ExternalState: chain_id: int ranges: field(default_factory=list) ima_linked: bool = False + reload_ts: Optional[int] = None def to_dict(self): return { 'chain_id': self.chain_id, 'ima_linked': self.ima_linked, - 'ranges': list(map(list, self.ranges)) + 'ranges': list(map(list, self.ranges)), + 'reload_ts': self.reload_ts } @@ -38,6 +40,10 @@ def ima_linked(self) -> bool: def chain_id(self) -> Optional[int]: return self.read().get('chain_id', None) + @property + def reload_ts(self) -> Optional[int]: + return self.read().get('reload_ts', None) + @property def ranges(self) -> List[IpRange]: plain_ranges = self.read().get('ranges', []) @@ -49,8 +55,8 @@ def get(self) -> Optional[ExternalState]: return ExternalState( chain_id=plain['chain_id'], ima_linked=plain['ima_linked'], - ranges=list(sorted(map(lambda r: IpRange(*r), plain['ranges']))) - + ranges=list(sorted(map(lambda r: IpRange(*r), plain['ranges']))), + reload_ts=plain['reload_ts'] ) return None diff --git a/core/schains/monitor/action.py b/core/schains/monitor/action.py index a7f04d110..bd05ead4d 100644 --- a/core/schains/monitor/action.py +++ b/core/schains/monitor/action.py @@ -21,11 +21,12 @@ import time from datetime import datetime from functools import wraps -from typing import Dict, Optional +from typing import Dict, Optional, List from skale import Skale from core.node_config import NodeConfig +from core.nodes import ExtendedManagerNodeInfo, calc_reload_ts, get_node_index_in_group from core.schains.checks import ConfigChecks, SkaledChecks from core.schains.dkg import ( DkgError, @@ -142,6 +143,7 @@ def __init__( stream_version: str, checks: ConfigChecks, estate: ExternalState, + current_nodes: List[ExtendedManagerNodeInfo], econfig: Optional[ExternalConfig] = None ): self.skale = skale @@ -150,6 +152,7 @@ def __init__( self.node_config = node_config self.checks = checks self.stream_version = stream_version + self.current_nodes = current_nodes self.rotation_data = rotation_data self.rotation_id = rotation_data['rotation_id'] @@ -245,6 +248,25 @@ def external_state(self) -> bool: self.econfig.update(self.estate) return True + @BaseActionManager.monitor_block + def set_reload_ts(self) -> bool: + logger.info('Setting reload_ts') + node_index_in_group = get_node_index_in_group(self.skale, self.name, self.node_config.id) + if node_index_in_group is None: + logger.warning(f'node {self.node_config.id} is not in chain {self.name}') + return False + self.estate.reload_ts = calc_reload_ts(self.current_nodes, node_index_in_group) + logger.info(f'Setting reload_ts to {self.estate.reload_ts}') + self.econfig.update(self.estate) + return True + + @BaseActionManager.monitor_block + def reset_reload_ts(self) -> bool: + logger.info('Resetting reload_ts') + self.estate.reload_ts = None + self.econfig.update(self.estate) + return True + class SkaledActionManager(BaseActionManager): def __init__( @@ -285,12 +307,12 @@ def volume(self) -> bool: return initial_status @BaseActionManager.monitor_block - def firewall_rules(self) -> bool: + def firewall_rules(self, upstream: bool = False) -> bool: initial_status = self.checks.firewall_rules.status if not initial_status: logger.info('Configuring firewall rules') - conf = self.cfm.skaled_config + conf = self.cfm.latest_upstream_config if upstream else self.cfm.skaled_config base_port = get_base_port_from_config(conf) node_ips = get_node_ips_from_config(conf) own_ip = get_own_ip_from_config(conf) @@ -449,14 +471,13 @@ def update_config(self) -> bool: return self.cfm.sync_skaled_config_with_upstream() @BaseActionManager.monitor_block - def schedule_skaled_exit(self) -> None: + def schedule_skaled_exit(self, exit_ts: int) -> None: if self.skaled_status.exit_time_reached or self.esfm.exists(): logger.info('Exit time has been already set') return - finish_ts = self.upstream_finish_ts - if finish_ts is not None: - logger.info('Scheduling skaled exit time %d', finish_ts) - self.esfm.exit_ts = finish_ts + if exit_ts is not None: + logger.info('Scheduling skaled exit time %d', exit_ts) + self.esfm.exit_ts = exit_ts @BaseActionManager.monitor_block def reset_exit_schedule(self) -> None: diff --git a/core/schains/monitor/config_monitor.py b/core/schains/monitor/config_monitor.py index a5d8be3af..a22181bac 100644 --- a/core/schains/monitor/config_monitor.py +++ b/core/schains/monitor/config_monitor.py @@ -61,6 +61,10 @@ def execute(self) -> None: self.am.dkg() if not self.checks.external_state: self.am.external_state() - if not self.checks.upstream_config: + if not self.checks.node_ips: + self.am.set_reload_ts() + else: + self.am.reset_reload_ts() + if not self.checks.upstream_config or not self.checks.node_ips: self.am.upstream_config() self.am.reset_config_record() diff --git a/core/schains/monitor/main.py b/core/schains/monitor/main.py index af4b971ec..46aa3e9c1 100644 --- a/core/schains/monitor/main.py +++ b/core/schains/monitor/main.py @@ -49,6 +49,7 @@ from core.schains.task import keep_tasks_running, Task from core.schains.config.static_params import get_automatic_repair_option from core.schains.skaled_status import get_skaled_status +from core.nodes import get_current_nodes from tools.docker_utils import DockerUtils from tools.configs.ima import DISABLE_IMA from tools.notifications.messages import notify_checks @@ -77,6 +78,7 @@ def run_config_pipeline( rotation_data = skale.node_rotation.get_rotation(name) allowed_ranges = get_sync_agent_ranges(skale) ima_linked = not DISABLE_IMA and skale_ima.linker.has_schain(name) + current_nodes = get_current_nodes(skale, name) estate = ExternalState( ima_linked=ima_linked, @@ -90,6 +92,7 @@ def run_config_pipeline( schain_record=schain_record, stream_version=stream_version, rotation_id=rotation_data['rotation_id'], + current_nodes=current_nodes, econfig=econfig, estate=estate ) @@ -101,6 +104,7 @@ def run_config_pipeline( rotation_data=rotation_data, stream_version=stream_version, checks=config_checks, + current_nodes=current_nodes, estate=estate, econfig=econfig ) diff --git a/core/schains/monitor/skaled_monitor.py b/core/schains/monitor/skaled_monitor.py index 773108b54..fe94ca4cc 100644 --- a/core/schains/monitor/skaled_monitor.py +++ b/core/schains/monitor/skaled_monitor.py @@ -151,9 +151,9 @@ def execute(self) -> None: self.am.recreated_schain_containers(abort_on_exit=False) -class NewConfigSkaledMonitor(BaseSkaledMonitor): +class ReloadGroupSkaledMonitor(BaseSkaledMonitor): """ - When config is outdated request setExitTime with latest finish_ts from config + When config is outdated set exit time to the latest finish_ts from schain config """ def execute(self): @@ -169,7 +169,28 @@ def execute(self): self.am.skaled_rpc() if not self.checks.ima_container: self.am.ima_container() - self.am.schedule_skaled_exit() + self.am.schedule_skaled_exit(self.am.upstream_finish_ts) + + +class ReloadIpSkaledMonitor(BaseSkaledMonitor): + """ + When config is outdated set exit time to reload_ts from external config + """ + + def execute(self): + if not self.checks.firewall_rules: + self.am.firewall_rules(upstream=True) + if not self.checks.volume: + self.am.volume() + if not self.checks.skaled_container: + self.am.skaled_container() + else: + self.am.reset_restart_counter() + if not self.checks.rpc: + self.am.skaled_rpc() + if not self.checks.ima_container: + self.am.ima_container() + self.am.schedule_skaled_exit(self.am.econfig.reload_ts) class NoConfigSkaledMonitor(BaseSkaledMonitor): @@ -224,16 +245,19 @@ def is_repair_mode( return automatic_repair and is_skaled_repair_status(status, skaled_status) -def is_new_config_mode( - status: Dict, - finish_ts: Optional[int] -) -> bool: +def is_reload_group_mode(status: Dict, finish_ts: Optional[int]) -> bool: ts = int(time.time()) if finish_ts is None: return False return finish_ts > ts and status['config'] and not status['config_updated'] +def is_reload_ip_mode(status: Dict, reload_ts: Optional[int]) -> bool: + if reload_ts is None: + return False + return status['config'] and not status['config_updated'] + + def is_config_update_time( status: Dict, skaled_status: Optional[SkaledStatus] @@ -291,7 +315,8 @@ def get_skaled_monitor( mon_type = NewNodeSkaledMonitor elif is_config_update_time(status, skaled_status): mon_type = UpdateConfigSkaledMonitor - elif is_new_config_mode(status, action_manager.upstream_finish_ts): - mon_type = NewConfigSkaledMonitor - + elif is_reload_group_mode(status, action_manager.upstream_finish_ts): + mon_type = ReloadGroupSkaledMonitor + elif is_reload_ip_mode(status, action_manager.econfig.reload_ts): + mon_type = ReloadIpSkaledMonitor return mon_type diff --git a/hardhat-node b/hardhat-node index a7cfb2977..8a4b03fd1 160000 --- a/hardhat-node +++ b/hardhat-node @@ -1 +1 @@ -Subproject commit a7cfb29778c90553cc7ae6fa42b7dd4df0fe6519 +Subproject commit 8a4b03fd1051960a3e0182280bf4bfdc43129997 diff --git a/requirements-dev.txt b/requirements-dev.txt index 6510ec576..3b706ce51 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ pytest==7.1.3 flake8==5.0.4 freezegun==0.3.15 mock==4.0.2 -blinker==1.4 +blinker==1.6.2 pytest-cov==2.9.0 codecov==2.1.13 diff --git a/tests/conftest.py b/tests/conftest.py index badd06a18..0ff1d17bc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -151,6 +151,7 @@ def node_skales(skale, node_wallets): @pytest.fixture def nodes(skale, node_skales, validator): + cleanup_nodes(skale, skale.nodes.get_active_node_ids()) link_nodes_to_validator(skale, validator, node_skales) ids = create_nodes(node_skales) try: diff --git a/tests/nodes_test.py b/tests/nodes_test.py new file mode 100644 index 000000000..d403579a1 --- /dev/null +++ b/tests/nodes_test.py @@ -0,0 +1,68 @@ +from skale.utils.helper import ip_to_bytes +from core.nodes import ( + get_current_nodes, + get_current_ips, + get_max_ip_change_ts, + calc_reload_ts, + get_node_index_in_group, + get_node_delay +) +from tests.utils import generate_random_ip +from tests.conftest import NUMBER_OF_NODES + + +def test_get_current_nodes(skale, schain_on_contracts): + current_nodes = get_current_nodes(skale, schain_on_contracts) + assert len(current_nodes) == NUMBER_OF_NODES + + +def test_get_current_ips(skale, schain_on_contracts): + current_nodes = get_current_nodes(skale, schain_on_contracts) + current_ips = get_current_ips(current_nodes) + assert len(current_ips) == NUMBER_OF_NODES + assert current_ips[0] == current_nodes[0]['ip'] + + +def test_get_max_ip_change_ts(skale, schain_on_contracts): + current_nodes = get_current_nodes(skale, schain_on_contracts) + max_ip_change_ts = get_max_ip_change_ts(current_nodes) + assert max_ip_change_ts is None + new_ip = generate_random_ip() + skale.nodes.change_ip(current_nodes[0]['id'], ip_to_bytes(new_ip), ip_to_bytes(new_ip)) + current_nodes = get_current_nodes(skale, schain_on_contracts) + max_ip_change_ts = get_max_ip_change_ts(current_nodes) + assert max_ip_change_ts is not None + assert max_ip_change_ts > 0 + + +def test_calc_reload_ts(skale, schain_on_contracts): + current_nodes = get_current_nodes(skale, schain_on_contracts) + reload_ts = calc_reload_ts(current_nodes, 4) + assert reload_ts is None + new_ip = generate_random_ip() + skale.nodes.change_ip(current_nodes[0]['id'], ip_to_bytes(new_ip), ip_to_bytes(new_ip)) + current_nodes = get_current_nodes(skale, schain_on_contracts) + max_ip_change_ts = get_max_ip_change_ts(current_nodes) + + reload_ts = calc_reload_ts(current_nodes, 4) + assert max_ip_change_ts < reload_ts + + reload_ts = calc_reload_ts(current_nodes, 0) + assert reload_ts == max_ip_change_ts + 300 + + reload_ts = calc_reload_ts([{'ip_change_ts': 0}, {'ip_change_ts': 100}, {'ip_change_ts': 0}], 2) + assert reload_ts == 1000 + + +def test_get_node_index_in_group(skale, schain_on_contracts): + current_nodes = get_current_nodes(skale, schain_on_contracts) + node_index = get_node_index_in_group(skale, schain_on_contracts, current_nodes[1]['id']) + assert node_index == 1 + node_index = get_node_index_in_group(skale, schain_on_contracts, 99999999) + assert node_index is None + + +def test_get_node_delay(): + assert get_node_delay(3) == 1200 + assert get_node_delay(0) == 300 + assert get_node_delay(16) == 5100 diff --git a/tests/schains/monitor/action/config_action_test.py b/tests/schains/monitor/action/config_action_test.py index 4bfaa780d..00dd4c919 100644 --- a/tests/schains/monitor/action/config_action_test.py +++ b/tests/schains/monitor/action/config_action_test.py @@ -1,6 +1,7 @@ import shutil import pytest +from core.nodes import get_current_nodes from core.schains.checks import ConfigChecks from core.schains.config.directory import schain_config_dir @@ -28,12 +29,14 @@ def config_checks( ): name = schain_db schain_record = SChainRecord.get_by_name(name) + current_nodes = get_current_nodes(skale, name) return ConfigChecks( schain_name=name, node_id=node_config.id, schain_record=schain_record, rotation_id=rotation_data['rotation_id'], stream_version=CONFIG_STREAM, + current_nodes=current_nodes, estate=estate ) @@ -52,6 +55,7 @@ def config_am( name = schain_db rotation_data = skale.node_rotation.get_rotation(name) schain = skale.schains.get_by_name(name) + current_nodes = get_current_nodes(skale, name) return ConfigActionManager( skale=skale, schain=schain, @@ -59,6 +63,7 @@ def config_am( rotation_data=rotation_data, checks=config_checks, stream_version=CONFIG_STREAM, + current_nodes=current_nodes, estate=estate ) diff --git a/tests/schains/monitor/config_monitor_test.py b/tests/schains/monitor/config_monitor_test.py index 5fa5a823c..e49133ea7 100644 --- a/tests/schains/monitor/config_monitor_test.py +++ b/tests/schains/monitor/config_monitor_test.py @@ -2,16 +2,20 @@ import os import pytest +from skale.utils.helper import ip_to_bytes + +from core.nodes import get_current_nodes from core.schains.checks import ConfigChecks from core.schains.config.directory import schain_config_dir from core.schains.monitor.action import ConfigActionManager from core.schains.monitor.config_monitor import RegularConfigMonitor +from core.schains.external_config import ExternalConfig from web.models.schain import SChainRecord -from tests.utils import CONFIG_STREAM +from tests.utils import CONFIG_STREAM, generate_random_ip @pytest.fixture @@ -30,12 +34,14 @@ def config_checks( ): name = schain_db schain_record = SChainRecord.get_by_name(name) + current_nodes = get_current_nodes(skale, name) return ConfigChecks( schain_name=name, node_id=node_config.id, schain_record=schain_record, rotation_id=rotation_data['rotation_id'], stream_version=CONFIG_STREAM, + current_nodes=current_nodes, estate=estate ) @@ -54,6 +60,7 @@ def config_am( name = schain_db rotation_data = skale.node_rotation.get_rotation(name) schain = skale.schains.get_by_name(name) + current_nodes = get_current_nodes(skale, name) am = ConfigActionManager( skale=skale, @@ -62,6 +69,7 @@ def config_am( rotation_data=rotation_data, stream_version=CONFIG_STREAM, checks=config_checks, + current_nodes=current_nodes, estate=estate ) am.dkg = lambda s: True @@ -88,3 +96,37 @@ def test_regular_config_monitor(schain_db, regular_config_monitor, rotation_data ) filenames = glob.glob(pattern) assert os.path.isfile(filenames[0]) + + +def test_regular_config_monitor_change_ip( + skale, + schain_db, + regular_config_monitor, + rotation_data +): + name = schain_db + econfig = ExternalConfig(name=name) + assert econfig.reload_ts is None + + regular_config_monitor.run() + assert econfig.reload_ts is None + + current_nodes = get_current_nodes(skale, name) + new_ip = generate_random_ip() + skale.nodes.change_ip(current_nodes[0]['id'], ip_to_bytes(new_ip), ip_to_bytes(new_ip)) + + current_nodes = get_current_nodes(skale, name) + regular_config_monitor.am.current_nodes = current_nodes + regular_config_monitor.checks.current_nodes = current_nodes + + regular_config_monitor.run() + assert econfig.reload_ts is not None + assert econfig.reload_ts > 0 + + current_nodes = get_current_nodes(skale, name) + regular_config_monitor.am.current_nodes = current_nodes + regular_config_monitor.checks.current_nodes = current_nodes + + regular_config_monitor.am.cfm.sync_skaled_config_with_upstream() + regular_config_monitor.run() + assert econfig.reload_ts is None diff --git a/tests/schains/monitor/skaled_monitor_test.py b/tests/schains/monitor/skaled_monitor_test.py index 38a658956..bded111d5 100644 --- a/tests/schains/monitor/skaled_monitor_test.py +++ b/tests/schains/monitor/skaled_monitor_test.py @@ -12,7 +12,8 @@ from core.schains.monitor.skaled_monitor import ( BackupSkaledMonitor, get_skaled_monitor, - NewConfigSkaledMonitor, + ReloadGroupSkaledMonitor, + ReloadIpSkaledMonitor, NewNodeSkaledMonitor, NoConfigSkaledMonitor, RecreateSkaledMonitor, @@ -20,6 +21,7 @@ RepairSkaledMonitor, UpdateConfigSkaledMonitor ) +from core.schains.external_config import ExternalConfig from core.schains.exit_scheduler import ExitScheduleFileManager from core.schains.runner import get_container_info from tools.configs.containers import SCHAIN_CONTAINER, IMA_CONTAINER @@ -288,7 +290,7 @@ def skaled_checks_new_config( @freezegun.freeze_time(CURRENT_DATETIME) -def test_get_skaled_monitor_new_config( +def test_get_skaled_monitor_reload_group( skale, skaled_am, skaled_checks_new_config, @@ -345,7 +347,62 @@ def test_get_skaled_monitor_new_config( schain_record, skaled_status ) - assert mon == NewConfigSkaledMonitor + assert mon == ReloadGroupSkaledMonitor + + +@freezegun.freeze_time(CURRENT_DATETIME) +def test_get_skaled_monitor_reload_ip( + skale, + skaled_am, + skaled_checks_new_config, + schain_db, + skaled_status, + node_config, + rule_controller, + schain_on_contracts, + predeployed_ima, + rotation_data, + secret_keys, + ssl_folder, + skaled_checks, + dutils +): + name = schain_db + schain_record = SChainRecord.get_by_name(name) + + state = skaled_checks_new_config.get_all() + state['rotation_id_updated'] = False + + schain = skale.schains.get_by_name(name) + + econfig = ExternalConfig(name) + + skaled_am = SkaledActionManager( + schain=schain, + rule_controller=rule_controller, + node_config=node_config, + checks=skaled_checks, + dutils=dutils + ) + mon = get_skaled_monitor( + skaled_am, + state, + schain_record, + skaled_status + ) + assert mon == RegularSkaledMonitor + + estate = econfig.read() + estate['reload_ts'] = CURRENT_TIMESTAMP + 10 + econfig.write(estate) + + mon = get_skaled_monitor( + skaled_am, + state, + schain_record, + skaled_status + ) + assert mon == ReloadIpSkaledMonitor @freezegun.freeze_time(CURRENT_DATETIME) @@ -483,8 +540,8 @@ def test_repair_skaled_monitor(skaled_am, skaled_checks, clean_docker, dutils): assert not dutils.safe_get_container(f'skale_ima_{skaled_am.name}') -def test_new_config_skaled_monitor(skaled_am, skaled_checks, clean_docker, dutils): - mon = NewConfigSkaledMonitor(skaled_am, skaled_checks) +def test_group_reload_skaled_monitor(skaled_am, skaled_checks, clean_docker, dutils): + mon = ReloadGroupSkaledMonitor(skaled_am, skaled_checks) ts = time.time() esfm = ExitScheduleFileManager(mon.am.name) with mock.patch('core.schains.monitor.action.get_finish_ts_from_latest_upstream', @@ -498,8 +555,8 @@ def test_new_config_skaled_monitor(skaled_am, skaled_checks, clean_docker, dutil @pytest.mark.skip -def test_new_config_skaled_monitor_failed_skaled(skaled_am, skaled_checks, clean_docker, dutils): - mon = NewConfigSkaledMonitor(skaled_am, skaled_checks) +def test_group_reload_skaled_monitor_failed_skaled(skaled_am, skaled_checks, clean_docker, dutils): + mon = ReloadGroupSkaledMonitor(skaled_am, skaled_checks) with mock.patch('core.schains.monitor.containers.run_schain_container') \ as run_skaled_container_mock: mon.run() diff --git a/tools/configs/__init__.py b/tools/configs/__init__.py index da5c6c63c..8ec237b62 100644 --- a/tools/configs/__init__.py +++ b/tools/configs/__init__.py @@ -88,3 +88,5 @@ NODE_OPTIONS_FILEPATH = os.path.join(NODE_DATA_PATH, 'node_options.json') PULL_CONFIG_FOR_SCHAIN = os.getenv('PULL_CONFIG_FOR_SCHAIN') + +CHANGE_IP_DELAY = 300 From e28519240c8c68182c57de24e90708e063dbb8c0 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 15 Jan 2024 19:24:36 +0000 Subject: [PATCH 02/14] skale-admin#832 update check_validator_nodes function import --- web/routes/node.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/routes/node.py b/web/routes/node.py index 4a2f6dc16..91d9309cc 100644 --- a/web/routes/node.py +++ b/web/routes/node.py @@ -27,9 +27,9 @@ from core.node import Node, NodeStatus from tools.helper import get_endpoint_call_speed -from core.node import ( - get_meta_info, get_node_hardware_info, get_btrfs_info, check_validator_nodes, get_abi_hash -) +from core.node import get_meta_info, get_node_hardware_info, get_btrfs_info, get_abi_hash +from core.nodes import check_validator_nodes + from tools.configs.web3 import ABI_FILEPATH, ENDPOINT, UNTRUSTED_PROVIDERS from tools.configs.ima import MAINNET_IMA_ABI_FILEPATH From 16360e304805e886a00d4549d3a7f95cd1454a17 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 15 Jan 2024 19:38:09 +0000 Subject: [PATCH 03/14] skale-admin#832 update external config - get reload_ts --- core/schains/external_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/schains/external_config.py b/core/schains/external_config.py index 8bdd63c88..e8a7d54e9 100644 --- a/core/schains/external_config.py +++ b/core/schains/external_config.py @@ -56,7 +56,7 @@ def get(self) -> Optional[ExternalState]: chain_id=plain['chain_id'], ima_linked=plain['ima_linked'], ranges=list(sorted(map(lambda r: IpRange(*r), plain['ranges']))), - reload_ts=plain['reload_ts'] + reload_ts=plain.get('reload_ts') ) return None From 334b7f8f82fb9a63d4771b78d61ae351a6aac266 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 15 Jan 2024 20:29:10 +0000 Subject: [PATCH 04/14] skale-admin#832 update config monitor logic --- core/schains/checks.py | 2 +- core/schains/monitor/action.py | 14 ++++++-------- core/schains/monitor/config_monitor.py | 8 +++----- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/core/schains/checks.py b/core/schains/checks.py index ef44556a8..12ddca80d 100644 --- a/core/schains/checks.py +++ b/core/schains/checks.py @@ -181,7 +181,7 @@ def node_ips(self) -> CheckRes: """Checks that IP list on the skale-manager is the same as in the config""" res = False if self.cfm.upstream_exist_for_rotation_id(self.rotation_id): - conf = self.cfm.skaled_config + conf = self.cfm.latest_upstream_config node_ips = get_node_ips_from_config(conf) current_ips = get_current_ips(self.current_nodes) res = set(node_ips) == set(current_ips) diff --git a/core/schains/monitor/action.py b/core/schains/monitor/action.py index bd05ead4d..6c6b8293b 100644 --- a/core/schains/monitor/action.py +++ b/core/schains/monitor/action.py @@ -249,8 +249,13 @@ def external_state(self) -> bool: return True @BaseActionManager.monitor_block - def set_reload_ts(self) -> bool: + def set_reload_ts(self, ip_changed: bool) -> bool: logger.info('Setting reload_ts') + if not ip_changed: + logger.info('Resetting reload_ts') + self.estate.reload_ts = None + self.econfig.update(self.estate) + return True node_index_in_group = get_node_index_in_group(self.skale, self.name, self.node_config.id) if node_index_in_group is None: logger.warning(f'node {self.node_config.id} is not in chain {self.name}') @@ -260,13 +265,6 @@ def set_reload_ts(self) -> bool: self.econfig.update(self.estate) return True - @BaseActionManager.monitor_block - def reset_reload_ts(self) -> bool: - logger.info('Resetting reload_ts') - self.estate.reload_ts = None - self.econfig.update(self.estate) - return True - class SkaledActionManager(BaseActionManager): def __init__( diff --git a/core/schains/monitor/config_monitor.py b/core/schains/monitor/config_monitor.py index a22181bac..bdae03297 100644 --- a/core/schains/monitor/config_monitor.py +++ b/core/schains/monitor/config_monitor.py @@ -55,16 +55,14 @@ def run(self): class RegularConfigMonitor(BaseConfigMonitor): def execute(self) -> None: + ip_changed = not self.checks.node_ips if not self.checks.config_dir: self.am.config_dir() if not self.checks.dkg: self.am.dkg() if not self.checks.external_state: self.am.external_state() - if not self.checks.node_ips: - self.am.set_reload_ts() - else: - self.am.reset_reload_ts() - if not self.checks.upstream_config or not self.checks.node_ips: + if not self.checks.upstream_config or ip_changed: self.am.upstream_config() + self.am.set_reload_ts(ip_changed) self.am.reset_config_record() From a39fb1c2f93490dad274818275ba863d4d345df9 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 15 Jan 2024 21:13:43 +0000 Subject: [PATCH 05/14] skale-admin#832 update config monitor logic --- core/schains/checks.py | 15 +++++++++++++-- core/schains/monitor/action.py | 4 ++-- core/schains/monitor/config_monitor.py | 5 ++--- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/core/schains/checks.py b/core/schains/checks.py index 12ddca80d..945b4e145 100644 --- a/core/schains/checks.py +++ b/core/schains/checks.py @@ -177,8 +177,8 @@ def dkg(self) -> CheckRes: return CheckRes(os.path.isfile(secret_key_share_filepath)) @property - def node_ips(self) -> CheckRes: - """Checks that IP list on the skale-manager is the same as in the config""" + def upstream_node_ips(self) -> CheckRes: + """Checks that IP list on the skale-manager is the same as in the upstream config""" res = False if self.cfm.upstream_exist_for_rotation_id(self.rotation_id): conf = self.cfm.latest_upstream_config @@ -187,6 +187,17 @@ def node_ips(self) -> CheckRes: res = set(node_ips) == set(current_ips) return CheckRes(res) + @property + def skaled_node_ips(self) -> CheckRes: + """Checks that IP list on the skale-manager is the same as in the skaled config""" + res = False + if self.cfm.skaled_config_exists(): + conf = self.cfm.skaled_config + node_ips = get_node_ips_from_config(conf) + current_ips = get_current_ips(self.current_nodes) + res = set(node_ips) == set(current_ips) + return CheckRes(res) + @property def upstream_config(self) -> CheckRes: """Checks that config exists for rotation id and stream""" diff --git a/core/schains/monitor/action.py b/core/schains/monitor/action.py index 6c6b8293b..4dac548c3 100644 --- a/core/schains/monitor/action.py +++ b/core/schains/monitor/action.py @@ -249,9 +249,9 @@ def external_state(self) -> bool: return True @BaseActionManager.monitor_block - def set_reload_ts(self, ip_changed: bool) -> bool: + def set_reload_ts(self, ip_matched: bool) -> bool: logger.info('Setting reload_ts') - if not ip_changed: + if ip_matched: logger.info('Resetting reload_ts') self.estate.reload_ts = None self.econfig.update(self.estate) diff --git a/core/schains/monitor/config_monitor.py b/core/schains/monitor/config_monitor.py index bdae03297..c952d77de 100644 --- a/core/schains/monitor/config_monitor.py +++ b/core/schains/monitor/config_monitor.py @@ -55,14 +55,13 @@ def run(self): class RegularConfigMonitor(BaseConfigMonitor): def execute(self) -> None: - ip_changed = not self.checks.node_ips if not self.checks.config_dir: self.am.config_dir() if not self.checks.dkg: self.am.dkg() if not self.checks.external_state: self.am.external_state() - if not self.checks.upstream_config or ip_changed: + if not self.checks.upstream_config or not self.checks.upstream_node_ips: self.am.upstream_config() - self.am.set_reload_ts(ip_changed) + self.am.set_reload_ts(self.checks.skaled_node_ips) self.am.reset_config_record() From 9655cb1e5fe94130f958777d9524c52894ff7738 Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 16 Jan 2024 13:40:42 +0000 Subject: [PATCH 06/14] Fix checks tests --- core/nodes.py | 8 +++--- core/schains/checks.py | 8 +++--- tests/conftest.py | 10 ++++++- tests/schains/checks_test.py | 26 ++++++++++++++----- .../monitor/action/config_action_test.py | 3 ++- web/routes/health.py | 3 +++ 6 files changed, 42 insertions(+), 16 deletions(-) diff --git a/core/nodes.py b/core/nodes.py index 82fe94ea9..b5aea771e 100644 --- a/core/nodes.py +++ b/core/nodes.py @@ -19,7 +19,7 @@ import socket import logging -from typing import List, TypedDict +from typing import List, Optional, TypedDict from skale import Skale from skale.utils.helper import ip_from_bytes @@ -58,11 +58,11 @@ def get_current_nodes(skale: Skale, name: str) -> List[ExtendedManagerNodeInfo]: return current_nodes -def get_current_ips(current_nodes: List[ExtendedManagerNodeInfo]) -> List[str]: +def get_current_ips(current_nodes: List[ExtendedManagerNodeInfo]) -> list[str]: return [node['ip'] for node in current_nodes] -def get_max_ip_change_ts(current_nodes: List[ExtendedManagerNodeInfo]) -> int | None: +def get_max_ip_change_ts(current_nodes: List[ExtendedManagerNodeInfo]) -> Optional[int]: max_ip_change_ts = max(current_nodes, key=lambda node: node['ip_change_ts'])['ip_change_ts'] return None if max_ip_change_ts == 0 else max_ip_change_ts @@ -82,7 +82,7 @@ def get_node_delay(node_index: int) -> int: return CHANGE_IP_DELAY * (node_index + 1) -def get_node_index_in_group(skale: Skale, schain_name: str, node_id: int) -> int | None: +def get_node_index_in_group(skale: Skale, schain_name: str, node_id: int) -> Optional[int]: """Returns node index in group or None if node is not in group""" try: node_ids = skale.schains_internal.get_node_ids_for_schain(schain_name) diff --git a/core/schains/checks.py b/core/schains/checks.py index 945b4e145..16a8c4f98 100644 --- a/core/schains/checks.py +++ b/core/schains/checks.py @@ -22,7 +22,7 @@ import time from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional -from core.nodes import get_current_ips +from core.nodes import ExtendedManagerNodeInfo, get_current_ips from core.schains.config.directory import get_schain_check_filepath from core.schains.config.file_manager import ConfigFileManager @@ -142,7 +142,7 @@ def __init__(self, schain_record: SChainRecord, rotation_id: int, stream_version: str, - current_nodes, + current_nodes: list[ExtendedManagerNodeInfo], estate: ExternalState, econfig: Optional[ExternalConfig] = None ) -> None: @@ -227,7 +227,7 @@ def __init__( rule_controller: IRuleController, *, econfig: Optional[ExternalConfig] = None, - dutils: DockerUtils | None = None + dutils: Optional[DockerUtils] = None ): self.name = schain_name self.schain_record = schain_record @@ -385,6 +385,7 @@ def __init__( rule_controller: IRuleController, stream_version: str, estate: ExternalState, + current_nodes: list[ExtendedManagerNodeInfo], rotation_id: int = 0, *, econfig: Optional[ExternalConfig] = None, @@ -397,6 +398,7 @@ def __init__( schain_record=schain_record, rotation_id=rotation_id, stream_version=stream_version, + current_nodes=current_nodes, estate=estate, econfig=econfig ), diff --git a/tests/conftest.py b/tests/conftest.py index 0ff1d17bc..f3a77b220 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,6 +31,7 @@ from skale.utils.web3_utils import init_web3 from core.ima.schain import update_predeployed_ima +from core.nodes import get_current_nodes from core.node_config import NodeConfig from core.schains.checks import SChainChecks from core.schains.config.helper import ( @@ -675,7 +676,7 @@ def node_config(skale, nodes): @pytest.fixture -def schain_checks(schain_config, schain_db, rule_controller, estate, dutils): +def schain_checks(schain_config, schain_db, current_nodes, rule_controller, estate, dutils): schain_name = schain_config['skaleConfig']['sChain']['schainName'] schain_record = SChainRecord.get_by_name(schain_name) node_id = schain_config['skaleConfig']['sChain']['nodes'][0]['nodeID'] @@ -685,6 +686,7 @@ def schain_checks(schain_config, schain_db, rule_controller, estate, dutils): schain_record=schain_record, rule_controller=rule_controller, stream_version=CONFIG_STREAM, + current_nodes=current_nodes, estate=estate, dutils=dutils ) @@ -766,6 +768,12 @@ def econfig(schain_db, estate): return ec +@pytest.fixture +def current_nodes(skale, schain_db, schain_on_contracts): + name = schain_db + return get_current_nodes(skale, name) + + @pytest.fixture def upstreams(schain_db, schain_config): name = schain_db diff --git a/tests/schains/checks_test.py b/tests/schains/checks_test.py index 85a5c14a7..1c5554049 100644 --- a/tests/schains/checks_test.py +++ b/tests/schains/checks_test.py @@ -77,7 +77,7 @@ def firewall_rules(self) -> CheckRes: @pytest.fixture -def sample_false_checks(schain_config, schain_db, rule_controller, estate, dutils): +def sample_false_checks(schain_config, schain_db, rule_controller, current_nodes, estate, dutils): schain_name = schain_config['skaleConfig']['sChain']['schainName'] schain_record = SChainRecord.get_by_name(schain_name) return SChainChecks( @@ -86,6 +86,7 @@ def sample_false_checks(schain_config, schain_db, rule_controller, estate, dutil schain_record=schain_record, rule_controller=rule_controller, stream_version=CONFIG_STREAM, + current_nodes=current_nodes, estate=estate, dutils=dutils ) @@ -96,6 +97,7 @@ def rules_unsynced_checks( schain_config, uninited_rule_controller, schain_db, + current_nodes, estate, dutils ): @@ -107,6 +109,7 @@ def rules_unsynced_checks( schain_record=schain_record, rule_controller=uninited_rule_controller, stream_version=CONFIG_STREAM, + current_nodes=current_nodes, estate=estate, dutils=dutils ) @@ -256,7 +259,7 @@ def test_blocks_check(schain_checks): assert not schain_checks.blocks -def test_init_checks(skale, schain_db, uninited_rule_controller, estate, dutils): +def test_init_checks(skale, schain_db, current_nodes, uninited_rule_controller, estate, dutils): schain_name = schain_db schain_record = SChainRecord.get_by_name(schain_name) checks = SChainChecks( @@ -265,6 +268,7 @@ def test_init_checks(skale, schain_db, uninited_rule_controller, estate, dutils) schain_record=schain_record, rule_controller=uninited_rule_controller, stream_version=CONFIG_STREAM, + current_nodes=current_nodes, estate=estate, dutils=dutils ) @@ -272,7 +276,7 @@ def test_init_checks(skale, schain_db, uninited_rule_controller, estate, dutils) assert checks.node_id == TEST_NODE_ID -def test_exit_code(skale, rule_controller, schain_db, estate, dutils): +def test_exit_code(skale, rule_controller, schain_db, current_nodes, estate, dutils): test_schain_name = schain_db image_name, container_name, _, _ = get_container_info( SCHAIN_CONTAINER, test_schain_name) @@ -292,6 +296,7 @@ def test_exit_code(skale, rule_controller, schain_db, estate, dutils): schain_record=schain_record, rule_controller=rule_controller, stream_version=CONFIG_STREAM, + current_nodes=current_nodes, estate=estate, dutils=dutils ) @@ -302,7 +307,7 @@ def test_exit_code(skale, rule_controller, schain_db, estate, dutils): dutils.safe_rm(container_name) -def test_process(skale, rule_controller, schain_db, estate, dutils): +def test_process(skale, rule_controller, schain_db, current_nodes, estate, dutils): schain_record = SChainRecord.get_by_name(schain_db) checks = SChainChecks( schain_db, @@ -310,6 +315,7 @@ def test_process(skale, rule_controller, schain_db, estate, dutils): schain_record=schain_record, rule_controller=rule_controller, stream_version=CONFIG_STREAM, + current_nodes=current_nodes, estate=estate, dutils=dutils ) @@ -323,7 +329,7 @@ def test_process(skale, rule_controller, schain_db, estate, dutils): assert not checks.process.status -def test_get_all(schain_config, rule_controller, dutils, schain_db, estate): +def test_get_all(schain_config, rule_controller, dutils, current_nodes, schain_db, estate): schain_name = schain_config['skaleConfig']['sChain']['schainName'] schain_record = SChainRecord.get_by_name(schain_name) node_id = schain_config['skaleConfig']['sChain']['nodes'][0]['nodeID'] @@ -333,6 +339,7 @@ def test_get_all(schain_config, rule_controller, dutils, schain_db, estate): schain_record=schain_record, rule_controller=rule_controller, stream_version=CONFIG_STREAM, + current_nodes=current_nodes, estate=estate, dutils=dutils ) @@ -354,6 +361,7 @@ def test_get_all(schain_config, rule_controller, dutils, schain_db, estate): schain_record=schain_record, rule_controller=rule_controller, stream_version=CONFIG_STREAM, + current_nodes=current_nodes, estate=estate, dutils=dutils ) @@ -372,7 +380,7 @@ def test_get_all(schain_config, rule_controller, dutils, schain_db, estate): assert len(filtered_checks) == 0 -def test_get_all_with_save(node_config, rule_controller, dutils, schain_db, estate): +def test_get_all_with_save(node_config, rule_controller, current_nodes, dutils, schain_db, estate): schain_record = upsert_schain_record(schain_db) checks = SChainChecksMock( schain_db, @@ -380,6 +388,7 @@ def test_get_all_with_save(node_config, rule_controller, dutils, schain_db, esta schain_record=schain_record, rule_controller=rule_controller, stream_version=CONFIG_STREAM, + current_nodes=current_nodes, estate=estate, dutils=dutils ) @@ -391,7 +400,7 @@ def test_get_all_with_save(node_config, rule_controller, dutils, schain_db, esta assert schain_checks == checks_from_file['checks'] -def test_config_updated(skale, rule_controller, schain_db, estate, dutils): +def test_config_updated(skale, rule_controller, schain_db, current_nodes, estate, dutils): name = schain_db folder = schain_config_dir(name) @@ -403,6 +412,7 @@ def test_config_updated(skale, rule_controller, schain_db, estate, dutils): schain_record=schain_record, rule_controller=rule_controller, stream_version=CONFIG_STREAM, + current_nodes=current_nodes, estate=estate, dutils=dutils ) @@ -423,6 +433,7 @@ def test_config_updated(skale, rule_controller, schain_db, estate, dutils): schain_record=schain_record, rule_controller=rule_controller, stream_version=CONFIG_STREAM, + current_nodes=current_nodes, estate=estate, dutils=dutils ) @@ -435,6 +446,7 @@ def test_config_updated(skale, rule_controller, schain_db, estate, dutils): schain_record=schain_record, rule_controller=rule_controller, stream_version=CONFIG_STREAM, + current_nodes=current_nodes, estate=estate, dutils=dutils ) diff --git a/tests/schains/monitor/action/config_action_test.py b/tests/schains/monitor/action/config_action_test.py index 00dd4c919..12d265038 100644 --- a/tests/schains/monitor/action/config_action_test.py +++ b/tests/schains/monitor/action/config_action_test.py @@ -109,6 +109,7 @@ def test_external_state_config_actions(config_am, config_checks, empty_econfig): assert econfig_data == { 'ima_linked': True, 'chain_id': config_am.skale.web3.eth.chain_id, - 'ranges': [['1.1.1.1', '2.2.2.2'], ['3.3.3.3', '4.4.4.4']] + 'ranges': [['1.1.1.1', '2.2.2.2'], ['3.3.3.3', '4.4.4.4']], + 'reload_ts': None } assert config_checks.external_state diff --git a/web/routes/health.py b/web/routes/health.py index d4306659c..c29f36d05 100644 --- a/web/routes/health.py +++ b/web/routes/health.py @@ -29,6 +29,7 @@ from urllib.parse import urlparse from core.node import get_check_report, get_skale_node_version +from core.nodes import get_current_nodes from core.schains.checks import SChainChecks from core.schains.firewall.utils import ( get_default_rule_controller, @@ -101,6 +102,7 @@ def schains_checks(): name=schain['name'], sync_agent_ranges=sync_agent_ranges ) + current_nodes = get_current_nodes(g.skale, schain['name']) schain_record = SChainRecord.get_by_name(schain['name']) schain_checks = SChainChecks( schain['name'], @@ -109,6 +111,7 @@ def schains_checks(): rule_controller=rc, rotation_id=rotation_id, stream_version=stream_version, + current_nodes=current_nodes, estate=estate ).get_all(needed=checks_filter) checks.append({ From 1e997c9995088e6e624292c3c503b338fbca80e9 Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 16 Jan 2024 15:20:42 +0000 Subject: [PATCH 07/14] Fix schain health test --- tests/routes/health_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/routes/health_test.py b/tests/routes/health_test.py index c0b9d61b4..da2758371 100644 --- a/tests/routes/health_test.py +++ b/tests/routes/health_test.py @@ -87,7 +87,7 @@ def test_containers_all(skale_bp, dutils, schain_db, cleanup_schain_containers): assert data == expected -def test_schains_checks(skale_bp, skale, schain_db, dutils): +def test_schains_checks(skale_bp, skale, schain_on_contracts, schain_db, dutils): schain_name = schain_db class SChainChecksMock(SChainChecks): From e021da710344715be9ef1372b0b7138c2598b4c0 Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 16 Jan 2024 17:56:51 +0000 Subject: [PATCH 08/14] Merge core.node and core.nodes --- core/node.py | 127 ++++++++++++++++-- core/schains/checks.py | 2 +- core/schains/monitor/action.py | 7 +- core/schains/monitor/main.py | 2 +- tests/conftest.py | 2 +- tests/nodes_test.py | 2 +- .../monitor/action/config_action_test.py | 2 +- tests/schains/monitor/config_monitor_test.py | 2 +- web/routes/health.py | 2 +- web/routes/node.py | 2 +- 10 files changed, 126 insertions(+), 24 deletions(-) diff --git a/core/node.py b/core/node.py index 3ba103737..849f9ce5a 100644 --- a/core/node.py +++ b/core/node.py @@ -18,37 +18,41 @@ # along with this program. If not, see . import json -import time -import os -import psutil +import hashlib import logging +import os import platform -import hashlib - -import requests - +import psutil +import socket +import time from enum import Enum -from typing import Dict +from typing import Dict, List, Optional, TypedDict -try: - from sh import lsmod -except ImportError: - logging.warning('Could not import lsmod from sh package') +import requests +from skale import Skale +from skale.schain_config.generator import get_nodes_for_schain from skale.transactions.exceptions import TransactionLogicError from skale.utils.exceptions import InvalidNodeIdError from skale.utils.helper import ip_from_bytes from skale.utils.web3_utils import public_key_to_address, to_checksum_address from core.filebeat import update_filebeat_service - -from tools.configs import CHECK_REPORT_PATH, META_FILEPATH +from tools.configs import WATCHDOG_PORT, CHANGE_IP_DELAY, CHECK_REPORT_PATH, META_FILEPATH from tools.helper import read_json from tools.str_formatters import arguments_list_string from tools.wallet_utils import check_required_balance logger = logging.getLogger(__name__) +try: + from sh import lsmod +except ImportError: + logging.warning('Could not import lsmod from sh package') + + +logger = logging.getLogger(__name__) + class NodeStatus(Enum): """This class contains possible node statuses""" @@ -355,3 +359,98 @@ def get_abi_hash(file_path): with open(file_path, 'rb') as file: abi_hash = hashlib.sha256(file.read()).hexdigest() return abi_hash + + +class ManagerNodeInfo(TypedDict): + name: str + ip: str + publicIP: str + port: int + start_block: int + last_reward_date: int + finish_time: int + status: int + validator_id: int + publicKey: str + domain_name: str + + +class ExtendedManagerNodeInfo(ManagerNodeInfo): + ip_change_ts: int + + +def get_current_nodes(skale: Skale, name: str) -> List[ExtendedManagerNodeInfo]: + current_nodes: ManagerNodeInfo = get_nodes_for_schain(skale, name) + for node in current_nodes: + node['ip_change_ts'] = skale.nodes.get_last_change_ip_time(node['id']) + node['ip'] = ip_from_bytes(node['ip']) + node['publicIP'] = ip_from_bytes(node['publicIP']) + return current_nodes + + +def get_current_ips(current_nodes: List[ExtendedManagerNodeInfo]) -> list[str]: + return [node['ip'] for node in current_nodes] + + +def get_max_ip_change_ts(current_nodes: List[ExtendedManagerNodeInfo]) -> Optional[int]: + max_ip_change_ts = max(current_nodes, key=lambda node: node['ip_change_ts'])['ip_change_ts'] + return None if max_ip_change_ts == 0 else max_ip_change_ts + + +def calc_reload_ts(current_nodes: List[ExtendedManagerNodeInfo], node_index: int) -> int: + max_ip_change_ts = get_max_ip_change_ts(current_nodes) + if max_ip_change_ts is None: + return + return max_ip_change_ts + get_node_delay(node_index) + + +def get_node_delay(node_index: int) -> int: + """ + Returns delay for node in seconds. + Example: for node with index 3 and delay 300 it will be 1200 seconds + """ + return CHANGE_IP_DELAY * (node_index + 1) + + +def get_node_index_in_group(skale: Skale, schain_name: str, node_id: int) -> Optional[int]: + """Returns node index in group or None if node is not in group""" + try: + node_ids = skale.schains_internal.get_node_ids_for_schain(schain_name) + return node_ids.index(node_id) + except ValueError: + return None + + +def is_port_open(ip, port): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(1) + try: + s.connect((ip, int(port))) + s.shutdown(1) + return True + except Exception: + return False + + +def check_validator_nodes(skale, node_id): + try: + node = skale.nodes.get(node_id) + node_ids = skale.nodes.get_validator_node_indices(node['validator_id']) + + try: + node_ids.remove(node_id) + except ValueError: + logger.warning( + f'node_id: {node_id} was not found in validator nodes: {node_ids}') + + res = [] + for node_id in node_ids: + if str(skale.nodes.get_node_status(node_id)) == str(NodeStatus.ACTIVE.value): + ip_bytes = skale.nodes.contract.functions.getNodeIP( + node_id).call() + ip = ip_from_bytes(ip_bytes) + res.append([node_id, ip, is_port_open(ip, WATCHDOG_PORT)]) + logger.info(f'validator_nodes check - node_id: {node_id}, res: {res}') + except Exception as err: + return {'status': 1, 'errors': [err]} + return {'status': 0, 'data': res} diff --git a/core/schains/checks.py b/core/schains/checks.py index 16a8c4f98..a024bacac 100644 --- a/core/schains/checks.py +++ b/core/schains/checks.py @@ -22,7 +22,7 @@ import time from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional -from core.nodes import ExtendedManagerNodeInfo, get_current_ips +from core.node import ExtendedManagerNodeInfo, get_current_ips from core.schains.config.directory import get_schain_check_filepath from core.schains.config.file_manager import ConfigFileManager diff --git a/core/schains/monitor/action.py b/core/schains/monitor/action.py index 4dac548c3..11e9610c9 100644 --- a/core/schains/monitor/action.py +++ b/core/schains/monitor/action.py @@ -26,7 +26,7 @@ from skale import Skale from core.node_config import NodeConfig -from core.nodes import ExtendedManagerNodeInfo, calc_reload_ts, get_node_index_in_group +from core.node import ExtendedManagerNodeInfo, calc_reload_ts, get_node_index_in_group from core.schains.checks import ConfigChecks, SkaledChecks from core.schains.dkg import ( DkgError, @@ -224,7 +224,10 @@ def upstream_config(self) -> bool: if not self.cfm.upstream_config_exists() or new_config != self.cfm.latest_upstream_config: rotation_id = self.rotation_data['rotation_id'] logger.info( - 'Saving new upstream config rotation_id: %d', rotation_id) + 'Saving new upstream config rotation_id: %d, ips: %s', + rotation_id, + self.current_nodes + ) self.cfm.save_new_upstream(rotation_id, new_config) result = True else: diff --git a/core/schains/monitor/main.py b/core/schains/monitor/main.py index 46aa3e9c1..417c56e96 100644 --- a/core/schains/monitor/main.py +++ b/core/schains/monitor/main.py @@ -49,7 +49,7 @@ from core.schains.task import keep_tasks_running, Task from core.schains.config.static_params import get_automatic_repair_option from core.schains.skaled_status import get_skaled_status -from core.nodes import get_current_nodes +from core.node import get_current_nodes from tools.docker_utils import DockerUtils from tools.configs.ima import DISABLE_IMA from tools.notifications.messages import notify_checks diff --git a/tests/conftest.py b/tests/conftest.py index f3a77b220..31dc73b38 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,7 +31,7 @@ from skale.utils.web3_utils import init_web3 from core.ima.schain import update_predeployed_ima -from core.nodes import get_current_nodes +from core.node import get_current_nodes from core.node_config import NodeConfig from core.schains.checks import SChainChecks from core.schains.config.helper import ( diff --git a/tests/nodes_test.py b/tests/nodes_test.py index d403579a1..f641abf4f 100644 --- a/tests/nodes_test.py +++ b/tests/nodes_test.py @@ -1,5 +1,5 @@ from skale.utils.helper import ip_to_bytes -from core.nodes import ( +from core.node import ( get_current_nodes, get_current_ips, get_max_ip_change_ts, diff --git a/tests/schains/monitor/action/config_action_test.py b/tests/schains/monitor/action/config_action_test.py index 12d265038..19033b9a5 100644 --- a/tests/schains/monitor/action/config_action_test.py +++ b/tests/schains/monitor/action/config_action_test.py @@ -1,7 +1,7 @@ import shutil import pytest -from core.nodes import get_current_nodes +from core.node import get_current_nodes from core.schains.checks import ConfigChecks from core.schains.config.directory import schain_config_dir diff --git a/tests/schains/monitor/config_monitor_test.py b/tests/schains/monitor/config_monitor_test.py index e49133ea7..d7c211f65 100644 --- a/tests/schains/monitor/config_monitor_test.py +++ b/tests/schains/monitor/config_monitor_test.py @@ -4,7 +4,7 @@ import pytest from skale.utils.helper import ip_to_bytes -from core.nodes import get_current_nodes +from core.node import get_current_nodes from core.schains.checks import ConfigChecks from core.schains.config.directory import schain_config_dir diff --git a/web/routes/health.py b/web/routes/health.py index c29f36d05..d2c8d1725 100644 --- a/web/routes/health.py +++ b/web/routes/health.py @@ -29,7 +29,7 @@ from urllib.parse import urlparse from core.node import get_check_report, get_skale_node_version -from core.nodes import get_current_nodes +from core.node import get_current_nodes from core.schains.checks import SChainChecks from core.schains.firewall.utils import ( get_default_rule_controller, diff --git a/web/routes/node.py b/web/routes/node.py index 91d9309cc..373603383 100644 --- a/web/routes/node.py +++ b/web/routes/node.py @@ -28,7 +28,7 @@ from tools.helper import get_endpoint_call_speed from core.node import get_meta_info, get_node_hardware_info, get_btrfs_info, get_abi_hash -from core.nodes import check_validator_nodes +from core.node import check_validator_nodes from tools.configs.web3 import ABI_FILEPATH, ENDPOINT, UNTRUSTED_PROVIDERS From 3d7fd783f7eb278681c97d4ae8fbf252791faaf0 Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 16 Jan 2024 19:06:43 +0000 Subject: [PATCH 09/14] Fix cleaner --- core/schains/cleaner.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/schains/cleaner.py b/core/schains/cleaner.py index 2bf5c3a07..257bdf438 100644 --- a/core/schains/cleaner.py +++ b/core/schains/cleaner.py @@ -24,7 +24,7 @@ from sgx import SgxClient -from core.node import get_skale_node_version +from core.node import get_current_nodes, get_skale_node_version from core.schains.checks import SChainChecks from core.schains.config.file_manager import ConfigFileManager from core.schains.config.directory import schain_config_dir @@ -207,11 +207,13 @@ def remove_schain(skale, node_id, schain_name, msg, dutils=None) -> None: rotation_data = skale.node_rotation.get_rotation(schain_name) rotation_id = rotation_data['rotation_id'] estate = ExternalConfig(name=schain_name).get() + current_nodes = get_current_nodes(skale, schain_name) cleanup_schain( node_id, schain_name, sync_agent_ranges, rotation_id=rotation_id, + current_nodes=current_nodes, estate=estate, dutils=dutils ) @@ -222,6 +224,7 @@ def cleanup_schain( schain_name, sync_agent_ranges, rotation_id, + current_nodes, estate, dutils=None ) -> None: @@ -239,6 +242,7 @@ def cleanup_schain( rule_controller=rc, stream_version=stream_version, schain_record=schain_record, + current_nodes=current_nodes, rotation_id=rotation_id, estate=estate ) From 3e716d96a39063b533af7b741aabc90bd521462a Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 16 Jan 2024 19:28:28 +0000 Subject: [PATCH 10/14] Make get_current_nodes return [] if no such chain --- core/node.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/node.py b/core/node.py index 849f9ce5a..b6979557c 100644 --- a/core/node.py +++ b/core/node.py @@ -380,6 +380,8 @@ class ExtendedManagerNodeInfo(ManagerNodeInfo): def get_current_nodes(skale: Skale, name: str) -> List[ExtendedManagerNodeInfo]: + if not skale.schains_internal.is_schain_exist(name): + return [] current_nodes: ManagerNodeInfo = get_nodes_for_schain(skale, name) for node in current_nodes: node['ip_change_ts'] = skale.nodes.get_last_change_ip_time(node['id']) From 7001bac024cd2a36aac128a14971c5e01f47af74 Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 17 Jan 2024 15:47:17 +0000 Subject: [PATCH 11/14] Add cleanup_schain test --- core/schains/cleaner.py | 4 +++- tests/schains/cleaner_test.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/core/schains/cleaner.py b/core/schains/cleaner.py index 257bdf438..974240155 100644 --- a/core/schains/cleaner.py +++ b/core/schains/cleaner.py @@ -228,6 +228,7 @@ def cleanup_schain( estate, dutils=None ) -> None: + breakpoint() dutils = dutils or DockerUtils() schain_record = upsert_schain_record(schain_name) @@ -244,7 +245,8 @@ def cleanup_schain( schain_record=schain_record, current_nodes=current_nodes, rotation_id=rotation_id, - estate=estate + estate=estate, + dutils=dutils ) status = checks.get_all() if status['skaled_container'] or is_exited( diff --git a/tests/schains/cleaner_test.py b/tests/schains/cleaner_test.py index a342ebc51..e8319474c 100644 --- a/tests/schains/cleaner_test.py +++ b/tests/schains/cleaner_test.py @@ -11,6 +11,7 @@ from skale.skale_manager import spawn_skale_manager_lib from core.schains.cleaner import ( + cleanup_schain, delete_bls_keys, monitor, get_schains_on_node, @@ -235,3 +236,33 @@ def test_get_schains_on_node(schain_dirs_for_monitor, TEST_SCHAIN_NAME_1, TEST_SCHAIN_NAME_2, PHANTOM_SCHAIN_NAME, schain_name ]).issubset(set(result)) + + +def test_cleanup_schain( + schain_db, + node_config, + schain_on_contracts, + current_nodes, + estate, + dutils, + secret_key +): + schain_name = schain_db + schain_dir_path = os.path.join(SCHAINS_DIR_PATH, schain_name) + assert os.path.isdir(schain_dir_path) + cleanup_schain( + node_config.id, + schain_name, + current_nodes=current_nodes, + sync_agent_ranges=[], + rotation_id=0, + estate=estate, + dutils=dutils + ) + + container_name = SCHAIN_CONTAINER_NAME_TEMPLATE.format(schain_name) + assert not is_container_running(dutils, container_name) + schain_dir_path = os.path.join(SCHAINS_DIR_PATH, schain_name) + assert not os.path.isdir(schain_dir_path) + record = SChainRecord.get_by_name(schain_name) + assert record.is_deleted is True From 41bb068ba0d663b496f1dd34415f616143ddd44d Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 17 Jan 2024 16:54:31 +0000 Subject: [PATCH 12/14] Remove breakpoint --- core/schains/cleaner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/schains/cleaner.py b/core/schains/cleaner.py index 974240155..62c8ca1ab 100644 --- a/core/schains/cleaner.py +++ b/core/schains/cleaner.py @@ -228,7 +228,6 @@ def cleanup_schain( estate, dutils=None ) -> None: - breakpoint() dutils = dutils or DockerUtils() schain_record = upsert_schain_record(schain_name) From 6f0ff581bc3690cdf02aba00ce63507dbe91afaa Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 17 Jan 2024 18:26:11 +0000 Subject: [PATCH 13/14] Do not use upstream_node_ips check --- core/schains/checks.py | 39 ++-- core/schains/monitor/action.py | 2 +- core/schains/monitor/config_monitor.py | 4 +- tests/conftest.py | 178 +----------------- tests/schains/checks_test.py | 26 ++- .../monitor/action/config_action_test.py | 13 +- tests/utils.py | 177 +++++++++++++++++ 7 files changed, 239 insertions(+), 200 deletions(-) diff --git a/core/schains/checks.py b/core/schains/checks.py index a024bacac..ec93d4b31 100644 --- a/core/schains/checks.py +++ b/core/schains/checks.py @@ -176,17 +176,6 @@ def dkg(self) -> CheckRes: ) return CheckRes(os.path.isfile(secret_key_share_filepath)) - @property - def upstream_node_ips(self) -> CheckRes: - """Checks that IP list on the skale-manager is the same as in the upstream config""" - res = False - if self.cfm.upstream_exist_for_rotation_id(self.rotation_id): - conf = self.cfm.latest_upstream_config - node_ips = get_node_ips_from_config(conf) - current_ips = get_current_ips(self.current_nodes) - res = set(node_ips) == set(current_ips) - return CheckRes(res) - @property def skaled_node_ips(self) -> CheckRes: """Checks that IP list on the skale-manager is the same as in the skaled config""" @@ -200,14 +189,32 @@ def skaled_node_ips(self) -> CheckRes: @property def upstream_config(self) -> CheckRes: - """Checks that config exists for rotation id and stream""" + """ + Returns True if config exists for current rotation id, + node ip addresses and stream version are up to date + and config regeneration was not triggered manually. + Returns False otherwise. + """ exists = self.cfm.upstream_exist_for_rotation_id(self.rotation_id) logger.debug('Upstream configs status for %s: %s', self.name, exists) - return CheckRes( - exists and - self.schain_record.config_version == self.stream_version and - not self.schain_record.sync_config_run + stream_updated = self.schain_record.config_version == self.stream_version + node_ips_updated = True + triggered = self.schain_record.sync_config_run + if exists: + conf = self.cfm.latest_upstream_config + upstream_node_ips = get_node_ips_from_config(conf) + current_ips = get_current_ips(self.current_nodes) + node_ips_updated = set(upstream_node_ips) == set(current_ips) + + logger.info( + 'Upstream config status, rotation_id %s: exist: %s, ips: %s, stream: %s, triggered: %s', + self.rotation_id, + exists, + node_ips_updated, + stream_updated, + triggered ) + return CheckRes(exists and node_ips_updated and stream_updated and not triggered) @property def external_state(self) -> CheckRes: diff --git a/core/schains/monitor/action.py b/core/schains/monitor/action.py index 11e9610c9..3a9438916 100644 --- a/core/schains/monitor/action.py +++ b/core/schains/monitor/action.py @@ -252,7 +252,7 @@ def external_state(self) -> bool: return True @BaseActionManager.monitor_block - def set_reload_ts(self, ip_matched: bool) -> bool: + def update_reload_ts(self, ip_matched: bool) -> bool: logger.info('Setting reload_ts') if ip_matched: logger.info('Resetting reload_ts') diff --git a/core/schains/monitor/config_monitor.py b/core/schains/monitor/config_monitor.py index c952d77de..3bd285ae3 100644 --- a/core/schains/monitor/config_monitor.py +++ b/core/schains/monitor/config_monitor.py @@ -61,7 +61,7 @@ def execute(self) -> None: self.am.dkg() if not self.checks.external_state: self.am.external_state() - if not self.checks.upstream_config or not self.checks.upstream_node_ips: + if not self.checks.upstream_config: self.am.upstream_config() - self.am.set_reload_ts(self.checks.skaled_node_ips) + self.am.update_reload_ts(self.checks.skaled_node_ips) self.am.reset_config_record() diff --git a/tests/conftest.py b/tests/conftest.py index 31dc73b38..41685e421 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -63,6 +63,7 @@ ETH_AMOUNT_PER_NODE, ETH_PRIVATE_KEY, generate_cert, + generate_schain_config, get_test_rule_controller, init_skale_from_wallet, init_skale_ima, @@ -217,183 +218,6 @@ def get_skaled_status_dict( } -def generate_schain_config(schain_name): - return { - "sealEngine": "Ethash", - "params": { - "accountStartNonce": "0x00", - "homesteadForkBlock": "0x0", - "daoHardforkBlock": "0x0", - "EIP150ForkBlock": "0x00", - "EIP158ForkBlock": "0x00", - "byzantiumForkBlock": "0x0", - "constantinopleForkBlock": "0x0", - "networkID": "12313219", - "chainID": "0x01", - "maximumExtraDataSize": "0x20", - "tieBreakingGas": False, - "minGasLimit": "0xFFFFFFF", - "maxGasLimit": "7fffffffffffffff", - "gasLimitBoundDivisor": "0x0400", - "minimumDifficulty": "0x020000", - "difficultyBoundDivisor": "0x0800", - "durationLimit": "0x0d", - "blockReward": "0x4563918244F40000", - "skaleDisableChainIdCheck": True - }, - "genesis": { - "nonce": "0x0000000000000042", - "difficulty": "0x020000", - "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "author": "0x0000000000000000000000000000000000000000", - "timestamp": "0x00", - "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", - "extraData": "0x11bbe8db4e347b4e8c937c1c8370e4b5ed33adb3db69cbdb7a38e1e50b1b82fa", - "gasLimit": "0xFFFFFFF" - }, - "accounts": { - }, - "skaleConfig": { - "nodeInfo": { - "nodeID": 0, - "nodeName": "test-node1", - "basePort": 10000, - "httpRpcPort": 10003, - "httpsRpcPort": 10008, - "wsRpcPort": 10002, - "wssRpcPort": 10007, - "infoHttpRpcPort": 10008, - "bindIP": "0.0.0.0", - "ecdsaKeyName": "NEK:518", - "imaMonitoringPort": 10006, - "wallets": { - "ima": { - "keyShareName": "bls_key:schain_id:33333333333333333333333333333333333333333333333333333333333333333333333333333:node_id:0:dkg_id:0", # noqa - "t": 11, - "n": 16, - "certfile": "sgx.crt", - "keyfile": "sgx.key", - "commonBlsPublicKey0": "11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "commonBlsPublicKey1": "1111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "commonBlsPublicKey2": "1111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "commonBlsPublicKey3": "11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "blsPublicKey0": "1111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "blsPublicKey1": "1111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "blsPublicKey2": "1111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa - "blsPublicKey3": "11111111111111111111111111111111111111111111111111111111111111111111111111111" # noqa - } - }, - }, - "sChain": { - "schainID": 1, - "schainName": schain_name, - "schainOwner": "0x3483A10F7d6fDeE0b0C1E9ad39cbCE13BD094b12", - - - "nodeGroups": { - "1": { - "rotation": None, - "nodes": { - "2": [ - 0, - 2, - "0xc21d242070e84fe5f8e80f14b8867856b714cf7d1984eaa9eb3f83c2a0a0e291b9b05754d071fbe89a91d4811b9b182d350f706dea6e91205905b86b4764ef9a" # noqa - ], - "5": [ - 1, - 5, - "0xc37b6db727683379d305a4e38532ddeb58c014ebb151662635839edf3f20042bcdaa8e4b1938e8304512c730671aedf310da76315e329be0814709279a45222a" # noqa - ], - "4": [ - 2, - 4, - "0x8b335f65ecf0845d93bc65a340cc2f4b8c49896f5023ecdff7db6f04bc39f9044239f541702ca7ad98c97aa6a7807aa7c41e394262cca0a32847e3c7c187baf5" # noqa - ], - "3": [ - 3, - 3, - "0xf3496966c7fd4a82967d32809267abec49bf5c4cc6d88737cee9b1a436366324d4847127a1220575f4ea6a7661723cd5861c9f8de221405b260511b998a0bbc8" # noqa - ] - }, - "finish_ts": None, - "bls_public_key": { - "blsPublicKey0": "8609115311055863404517113391175862520685049234001839865086978176708009850942", # noqa - "blsPublicKey1": "12596903066793884087763787291339131389612748572700005223043813683790087081", # noqa - "blsPublicKey2": "20949401227653007081557504259342598891084201308661070577835940778932311075846", # noqa - "blsPublicKey3": "5476329286206272760147989277520100256618500160343291262709092037265666120930" # noqa - } - }, - "0": { - "rotation": { - "leaving_node_id": 1, - "new_node_id": 5 - }, - "nodes": { - "2": [ - 0, - 2, - "0xc21d242070e84fe5f8e80f14b8867856b714cf7d1984eaa9eb3f83c2a0a0e291b9b05754d071fbe89a91d4811b9b182d350f706dea6e91205905b86b4764ef9a" # noqa - ], - "4": [ - 2, - 4, - "0x8b335f65ecf0845d93bc65a340cc2f4b8c49896f5023ecdff7db6f04bc39f9044239f541702ca7ad98c97aa6a7807aa7c41e394262cca0a32847e3c7c187baf5" # noqa - ], - "3": [ - 3, - 3, - "0xf3496966c7fd4a82967d32809267abec49bf5c4cc6d88737cee9b1a436366324d4847127a1220575f4ea6a7661723cd5861c9f8de221405b260511b998a0bbc8" # noqa - ], - "1": [ - 1, - 1, - "0x1a857aa4a982ba242c2386febf1eb72dcd1f9669b4237a17878eb836086618af6cda473afa2dfb37c0d2786887397d39bec9601234d933d4384fe38a39b399df" # noqa - ] - }, - "finish_ts": 1687180291, - "bls_public_key": { - "blsPublicKey0": "12452613198400495171048259986807077228209876295033433688114313813034253740478", # noqa - "blsPublicKey1": "10490413552821776191285904316985887024952448646239144269897585941191848882433", # noqa - "blsPublicKey2": "892041650350974543318836112385472656918171041007469041098688469382831828315", # noqa - "blsPublicKey3": "14699659615059580586774988732364564692366017113631037780839594032948908579205" # noqa - } - } - }, - "nodes": [ - { - "nodeID": 0, - "nodeName": "test-node0", - "basePort": 10000, - "httpRpcPort": 100003, - "httpsRpcPort": 10008, - "wsRpcPort": 10002, - "wssRpcPort": 10007, - "infoHttpRpcPort": 10008, - "schainIndex": 1, - "ip": "127.0.0.1", - "owner": "0x41", - "publicIP": "127.0.0.1" - }, - { - "nodeID": 1, - "nodeName": "test-node1", - "basePort": 10010, - "httpRpcPort": 10013, - "httpsRpcPort": 10017, - "wsRpcPort": 10012, - "wssRpcPort": 10018, - "infoHttpRpcPort": 10019, - "schainIndex": 1, - "ip": "127.0.0.2", - "owner": "0x42", - "publicIP": "127.0.0.2" - } - ] - } - } - } - - SECRET_KEY = { "common_public_key": [ 11111111111111111111111111111111111111111111111111111111111111111111111111111, diff --git a/tests/schains/checks_test.py b/tests/schains/checks_test.py index 1c5554049..0c09d7df4 100644 --- a/tests/schains/checks_test.py +++ b/tests/schains/checks_test.py @@ -10,12 +10,16 @@ import docker import pytest +from skale.schain_config.generator import get_schain_nodes_with_schains + + from core.schains.checks import SChainChecks, CheckRes from core.schains.config.file_manager import UpstreamConfigFilename from core.schains.config.directory import ( get_schain_check_filepath, schain_config_dir ) +from core.schains.config.schain_node import generate_schain_nodes from core.schains.skaled_exit_codes import SkaledExitCodes from core.schains.runner import get_container_info, get_image_name, run_ima_container # from core.schains.cleaner import remove_ima_container @@ -25,7 +29,13 @@ from web.models.schain import upsert_schain_record, SChainRecord -from tests.utils import CONFIG_STREAM, get_schain_contracts_data, response_mock, request_mock +from tests.utils import ( + CONFIG_STREAM, + generate_schain_config, + get_schain_contracts_data, + response_mock, + request_mock +) NOT_EXISTS_SCHAIN_NAME = 'qwerty123' @@ -125,7 +135,7 @@ def test_dkg_check(schain_checks, sample_false_checks): assert not sample_false_checks.dkg.status -def test_upstream_config_check(schain_checks): +def test_upstream_config_check(skale, schain_checks): assert not schain_checks.upstream_config ts = int(time.time()) name, rotation_id = schain_checks.name, schain_checks.rotation_id @@ -135,8 +145,18 @@ def test_upstream_config_check(schain_checks): f'schain_{name}_{rotation_id}_{ts}.json' ) + schain_nodes_with_schains = get_schain_nodes_with_schains(skale, name) + nodes = generate_schain_nodes( + schain_nodes_with_schains=schain_nodes_with_schains, + schain_name=name, + rotation_id=rotation_id + ) + + config = generate_schain_config(name) + config['skaleConfig']['sChain']['nodes'] = nodes + with open(upstream_path, 'w') as upstream_file: - json.dump({'config': 'upstream'}, upstream_file) + json.dump(config, upstream_file) assert schain_checks.upstream_config diff --git a/tests/schains/monitor/action/config_action_test.py b/tests/schains/monitor/action/config_action_test.py index 19033b9a5..771769727 100644 --- a/tests/schains/monitor/action/config_action_test.py +++ b/tests/schains/monitor/action/config_action_test.py @@ -1,8 +1,9 @@ import shutil +from copy import deepcopy import pytest -from core.node import get_current_nodes +from core.node import get_current_nodes from core.schains.checks import ConfigChecks from core.schains.config.directory import schain_config_dir from core.schains.monitor.action import ConfigActionManager @@ -94,6 +95,16 @@ def test_upstream_config_actions(config_am, config_checks): config_am.upstream_config() assert config_checks.upstream_config + # Modify node ips to and test that check fails + nodes = config_checks.current_nodes + new_nodes = deepcopy(config_checks.current_nodes) + try: + new_nodes[0]['ip'] = new_nodes[1]['ip'] + config_checks.current_nodes = new_nodes + assert not config_checks.upstream_config + finally: + config_checks.current_nodes = nodes + @pytest.fixture def empty_econfig(schain_db): diff --git a/tests/utils.py b/tests/utils.py index 8185ccb09..fab6628ff 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -283,3 +283,180 @@ def set_automine(w3: Web3, value: bool) -> None: data = {'jsonrpc': '2.0', 'method': 'evm_setAutomine', 'params': [value], "id": 102} r = requests.post(endpoint, json=data) assert r.status_code == 200 and 'error' not in r.json() + + +def generate_schain_config(schain_name): + return { + "sealEngine": "Ethash", + "params": { + "accountStartNonce": "0x00", + "homesteadForkBlock": "0x0", + "daoHardforkBlock": "0x0", + "EIP150ForkBlock": "0x00", + "EIP158ForkBlock": "0x00", + "byzantiumForkBlock": "0x0", + "constantinopleForkBlock": "0x0", + "networkID": "12313219", + "chainID": "0x01", + "maximumExtraDataSize": "0x20", + "tieBreakingGas": False, + "minGasLimit": "0xFFFFFFF", + "maxGasLimit": "7fffffffffffffff", + "gasLimitBoundDivisor": "0x0400", + "minimumDifficulty": "0x020000", + "difficultyBoundDivisor": "0x0800", + "durationLimit": "0x0d", + "blockReward": "0x4563918244F40000", + "skaleDisableChainIdCheck": True + }, + "genesis": { + "nonce": "0x0000000000000042", + "difficulty": "0x020000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "author": "0x0000000000000000000000000000000000000000", + "timestamp": "0x00", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData": "0x11bbe8db4e347b4e8c937c1c8370e4b5ed33adb3db69cbdb7a38e1e50b1b82fa", + "gasLimit": "0xFFFFFFF" + }, + "accounts": { + }, + "skaleConfig": { + "nodeInfo": { + "nodeID": 0, + "nodeName": "test-node1", + "basePort": 10000, + "httpRpcPort": 10003, + "httpsRpcPort": 10008, + "wsRpcPort": 10002, + "wssRpcPort": 10007, + "infoHttpRpcPort": 10008, + "bindIP": "0.0.0.0", + "ecdsaKeyName": "NEK:518", + "imaMonitoringPort": 10006, + "wallets": { + "ima": { + "keyShareName": "bls_key:schain_id:33333333333333333333333333333333333333333333333333333333333333333333333333333:node_id:0:dkg_id:0", # noqa + "t": 11, + "n": 16, + "certfile": "sgx.crt", + "keyfile": "sgx.key", + "commonBlsPublicKey0": "11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa + "commonBlsPublicKey1": "1111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa + "commonBlsPublicKey2": "1111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa + "commonBlsPublicKey3": "11111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa + "blsPublicKey0": "1111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa + "blsPublicKey1": "1111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa + "blsPublicKey2": "1111111111111111111111111111111111111111111111111111111111111111111111111111", # noqa + "blsPublicKey3": "11111111111111111111111111111111111111111111111111111111111111111111111111111" # noqa + } + }, + }, + "sChain": { + "schainID": 1, + "schainName": schain_name, + "schainOwner": "0x3483A10F7d6fDeE0b0C1E9ad39cbCE13BD094b12", + + + "nodeGroups": { + "1": { + "rotation": None, + "nodes": { + "2": [ + 0, + 2, + "0xc21d242070e84fe5f8e80f14b8867856b714cf7d1984eaa9eb3f83c2a0a0e291b9b05754d071fbe89a91d4811b9b182d350f706dea6e91205905b86b4764ef9a" # noqa + ], + "5": [ + 1, + 5, + "0xc37b6db727683379d305a4e38532ddeb58c014ebb151662635839edf3f20042bcdaa8e4b1938e8304512c730671aedf310da76315e329be0814709279a45222a" # noqa + ], + "4": [ + 2, + 4, + "0x8b335f65ecf0845d93bc65a340cc2f4b8c49896f5023ecdff7db6f04bc39f9044239f541702ca7ad98c97aa6a7807aa7c41e394262cca0a32847e3c7c187baf5" # noqa + ], + "3": [ + 3, + 3, + "0xf3496966c7fd4a82967d32809267abec49bf5c4cc6d88737cee9b1a436366324d4847127a1220575f4ea6a7661723cd5861c9f8de221405b260511b998a0bbc8" # noqa + ] + }, + "finish_ts": None, + "bls_public_key": { + "blsPublicKey0": "8609115311055863404517113391175862520685049234001839865086978176708009850942", # noqa + "blsPublicKey1": "12596903066793884087763787291339131389612748572700005223043813683790087081", # noqa + "blsPublicKey2": "20949401227653007081557504259342598891084201308661070577835940778932311075846", # noqa + "blsPublicKey3": "5476329286206272760147989277520100256618500160343291262709092037265666120930" # noqa + } + }, + "0": { + "rotation": { + "leaving_node_id": 1, + "new_node_id": 5 + }, + "nodes": { + "2": [ + 0, + 2, + "0xc21d242070e84fe5f8e80f14b8867856b714cf7d1984eaa9eb3f83c2a0a0e291b9b05754d071fbe89a91d4811b9b182d350f706dea6e91205905b86b4764ef9a" # noqa + ], + "4": [ + 2, + 4, + "0x8b335f65ecf0845d93bc65a340cc2f4b8c49896f5023ecdff7db6f04bc39f9044239f541702ca7ad98c97aa6a7807aa7c41e394262cca0a32847e3c7c187baf5" # noqa + ], + "3": [ + 3, + 3, + "0xf3496966c7fd4a82967d32809267abec49bf5c4cc6d88737cee9b1a436366324d4847127a1220575f4ea6a7661723cd5861c9f8de221405b260511b998a0bbc8" # noqa + ], + "1": [ + 1, + 1, + "0x1a857aa4a982ba242c2386febf1eb72dcd1f9669b4237a17878eb836086618af6cda473afa2dfb37c0d2786887397d39bec9601234d933d4384fe38a39b399df" # noqa + ] + }, + "finish_ts": 1687180291, + "bls_public_key": { + "blsPublicKey0": "12452613198400495171048259986807077228209876295033433688114313813034253740478", # noqa + "blsPublicKey1": "10490413552821776191285904316985887024952448646239144269897585941191848882433", # noqa + "blsPublicKey2": "892041650350974543318836112385472656918171041007469041098688469382831828315", # noqa + "blsPublicKey3": "14699659615059580586774988732364564692366017113631037780839594032948908579205" # noqa + } + } + }, + "nodes": [ + { + "nodeID": 0, + "nodeName": "test-node0", + "basePort": 10000, + "httpRpcPort": 100003, + "httpsRpcPort": 10008, + "wsRpcPort": 10002, + "wssRpcPort": 10007, + "infoHttpRpcPort": 10008, + "schainIndex": 1, + "ip": "127.0.0.1", + "owner": "0x41", + "publicIP": "127.0.0.1" + }, + { + "nodeID": 1, + "nodeName": "test-node1", + "basePort": 10010, + "httpRpcPort": 10013, + "httpsRpcPort": 10017, + "wsRpcPort": 10012, + "wssRpcPort": 10018, + "infoHttpRpcPort": 10019, + "schainIndex": 1, + "ip": "127.0.0.2", + "owner": "0x42", + "publicIP": "127.0.0.2" + } + ] + } + } + } From 280b4a89cc0d31c1120415df85666f0c851b80fe Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 17 Jan 2024 20:01:25 +0000 Subject: [PATCH 14/14] Remove nodes.py --- core/nodes.py | 126 -------------------------------------------------- 1 file changed, 126 deletions(-) delete mode 100644 core/nodes.py diff --git a/core/nodes.py b/core/nodes.py deleted file mode 100644 index b5aea771e..000000000 --- a/core/nodes.py +++ /dev/null @@ -1,126 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of SKALE Admin -# -# Copyright (C) 2024 SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import socket -import logging -from typing import List, Optional, TypedDict - -from skale import Skale -from skale.utils.helper import ip_from_bytes -from skale.schain_config.generator import get_nodes_for_schain - -from core.node import NodeStatus -from tools.configs import WATCHDOG_PORT, CHANGE_IP_DELAY - -logger = logging.getLogger(__name__) - - -class ManagerNodeInfo(TypedDict): - name: str - ip: str - publicIP: str - port: int - start_block: int - last_reward_date: int - finish_time: int - status: int - validator_id: int - publicKey: str - domain_name: str - - -class ExtendedManagerNodeInfo(ManagerNodeInfo): - ip_change_ts: int - - -def get_current_nodes(skale: Skale, name: str) -> List[ExtendedManagerNodeInfo]: - current_nodes: ManagerNodeInfo = get_nodes_for_schain(skale, name) - for node in current_nodes: - node['ip_change_ts'] = skale.nodes.get_last_change_ip_time(node['id']) - node['ip'] = ip_from_bytes(node['ip']) - node['publicIP'] = ip_from_bytes(node['publicIP']) - return current_nodes - - -def get_current_ips(current_nodes: List[ExtendedManagerNodeInfo]) -> list[str]: - return [node['ip'] for node in current_nodes] - - -def get_max_ip_change_ts(current_nodes: List[ExtendedManagerNodeInfo]) -> Optional[int]: - max_ip_change_ts = max(current_nodes, key=lambda node: node['ip_change_ts'])['ip_change_ts'] - return None if max_ip_change_ts == 0 else max_ip_change_ts - - -def calc_reload_ts(current_nodes: List[ExtendedManagerNodeInfo], node_index: int) -> int: - max_ip_change_ts = get_max_ip_change_ts(current_nodes) - if max_ip_change_ts is None: - return - return max_ip_change_ts + get_node_delay(node_index) - - -def get_node_delay(node_index: int) -> int: - """ - Returns delay for node in seconds. - Example: for node with index 3 and delay 300 it will be 1200 seconds - """ - return CHANGE_IP_DELAY * (node_index + 1) - - -def get_node_index_in_group(skale: Skale, schain_name: str, node_id: int) -> Optional[int]: - """Returns node index in group or None if node is not in group""" - try: - node_ids = skale.schains_internal.get_node_ids_for_schain(schain_name) - return node_ids.index(node_id) - except ValueError: - return None - - -def is_port_open(ip, port): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(1) - try: - s.connect((ip, int(port))) - s.shutdown(1) - return True - except Exception: - return False - - -def check_validator_nodes(skale, node_id): - try: - node = skale.nodes.get(node_id) - node_ids = skale.nodes.get_validator_node_indices(node['validator_id']) - - try: - node_ids.remove(node_id) - except ValueError: - logger.warning( - f'node_id: {node_id} was not found in validator nodes: {node_ids}') - - res = [] - for node_id in node_ids: - if str(skale.nodes.get_node_status(node_id)) == str(NodeStatus.ACTIVE.value): - ip_bytes = skale.nodes.contract.functions.getNodeIP( - node_id).call() - ip = ip_from_bytes(ip_bytes) - res.append([node_id, ip, is_port_open(ip, WATCHDOG_PORT)]) - logger.info(f'validator_nodes check - node_id: {node_id}, res: {res}') - except Exception as err: - return {'status': 1, 'errors': [err]} - return {'status': 0, 'data': res}