Merge pull request #1045 from skalenetwork/merge-sync-node-to-develop

Merge sync node to develop, update monitors logic

.github/workflows/test.yml

@@ -15,7 +15,6 @@ jobs:
       ETH_PRIVATE_KEY: ${{ secrets.ETH_PRIVATE_KEY }}
       ENDPOINT: http://127.0.0.1:8545
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-      IMA_ENDPOINT: ${{ secrets.IMA_ENDPOINT }}
       SCHAIN_TYPE: ${{ secrets.SCHAIN_TYPE }}
     steps:
       - uses: actions/checkout@v2

.gitignore

@@ -63,6 +63,8 @@ schain_status_sample.json
 *.lock
 
 skale-manager-*
+manager.json
+meta.json
 
 tests/skale-data/contracts_info/ima.json
 tests/skale-data/contracts_info/schain_ima_abi.json

VERSION

@@ -1 +1 @@
-2.6.0
+2.7.0

core/node_config.py

@@ -17,80 +17,46 @@
 #   You should have received a copy of the GNU Affero General Public License
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-import functools
 import logging
-from filelock import FileLock
 
-from tools.helper import read_json, write_json, init_file
-from tools.configs import NODE_CONFIG_FILEPATH, NODE_CONFIG_LOCK_PATH
+from tools.configs import NODE_CONFIG_FILEPATH
+from tools.json_object import JsonObject
 
 logger = logging.getLogger(__name__)
 
 
-def config_setter(func):
-    @functools.wraps(func)
-    def wrapper_decorator(*args, **kwargs):
-        field_name, field_value = func(*args, **kwargs)
-        lock = FileLock(NODE_CONFIG_LOCK_PATH)
-        with lock:
-            config = read_json(NODE_CONFIG_FILEPATH)
-            config[field_name] = field_value
-            write_json(NODE_CONFIG_FILEPATH, config)
-    return wrapper_decorator
-
-
-def config_getter(func):
-    @functools.wraps(func)
-    def wrapper_decorator(*args, **kwargs):
-        field_name = func(*args, **kwargs)
-        config = read_json(NODE_CONFIG_FILEPATH)
-        return config.get(field_name)
-    return wrapper_decorator
-
-
-class NodeConfig:
+class NodeConfig(JsonObject):
     def __init__(self, filepath: str = NODE_CONFIG_FILEPATH):
-        init_file(filepath, {})
+        super().__init__(filepath=filepath)
 
     @property
-    @config_getter
     def id(self) -> int:
-        return 'node_id'
+        return self._get('node_id')
 
     @id.setter
-    @config_setter
     def id(self, node_id: int) -> None:
-        return 'node_id', node_id
+        return self._set('node_id', node_id)
 
     @property
-    @config_getter
     def ip(self) -> str:
-        return 'node_ip'
+        return self._get('node_ip')
 
     @ip.setter
-    @config_setter
     def ip(self, ip: str) -> None:
-        return 'node_ip', ip
+        return self._set('node_ip', ip)
 
     @property
-    @config_getter
     def name(self) -> str:
-        return 'name'
+        return self._get('name')
 
     @name.setter
-    @config_setter
     def name(self, node_name: str) -> None:
-        return 'name', node_name
+        return self._set('name', node_name)
 
     @property
-    @config_getter
     def sgx_key_name(self) -> int:
-        return 'sgx_key_name'
+        return self._get('sgx_key_name')
 
     @sgx_key_name.setter
-    @config_setter
     def sgx_key_name(self, sgx_key_name: int) -> None:
-        return 'sgx_key_name', sgx_key_name
-
-    def all(self) -> dict:
-        return read_json(NODE_CONFIG_FILEPATH)
+        return self._set('sgx_key_name', sgx_key_name)

core/schains/checks.py

@@ -48,6 +48,7 @@
 from core.schains.external_config import ExternalConfig, ExternalState
 from core.schains.runner import get_container_name, get_image_name, is_new_image_pulled
 from core.schains.skaled_exit_codes import SkaledExitCodes
+from core.schains.volume import is_volume_exists
 
 from tools.configs.containers import IMA_CONTAINER, SCHAIN_CONTAINER
 from tools.docker_utils import DockerUtils
@@ -144,6 +145,7 @@ def __init__(self,
                  stream_version: str,
                  current_nodes: list[ExtendedManagerNodeInfo],
                  estate: ExternalState,
+                 sync_node: bool = False,
                  econfig: Optional[ExternalConfig] = None
                  ) -> None:
         self.name = schain_name
@@ -153,6 +155,7 @@ def __init__(self,
         self.stream_version = stream_version
         self.current_nodes = current_nodes
         self.estate = estate
+        self.sync_node = sync_node
         self.econfig = econfig or ExternalConfig(schain_name)
         self.cfm: ConfigFileManager = ConfigFileManager(
             schain_name=schain_name
@@ -234,13 +237,15 @@ def __init__(
         rule_controller: IRuleController,
         *,
         econfig: Optional[ExternalConfig] = None,
-        dutils: Optional[DockerUtils] = None
+        dutils: Optional[DockerUtils] = None,
+        sync_node: bool = False
     ):
         self.name = schain_name
         self.schain_record = schain_record
         self.dutils = dutils or DockerUtils()
         self.container_name = get_container_name(SCHAIN_CONTAINER, self.name)
         self.econfig = econfig or ExternalConfig(name=schain_name)
+        self.sync_node = sync_node
         self.rc = rule_controller
         self.cfm: ConfigFileManager = ConfigFileManager(
             schain_name=schain_name
@@ -280,7 +285,13 @@ def config(self) -> CheckRes:
     @property
     def volume(self) -> CheckRes:
         """Checks that sChain volume exists"""
-        return CheckRes(self.dutils.is_data_volume_exists(self.name))
+
+        return CheckRes(
+            is_volume_exists(
+                self.name,
+                sync_node=self.sync_node,
+                dutils=self.dutils)
+            )
 
     @property
     def firewall_rules(self) -> CheckRes:
@@ -321,8 +332,7 @@ def ima_container(self) -> CheckRes:
         if not self.econfig.ima_linked:
             return CheckRes(True)
         container_name = get_container_name(IMA_CONTAINER, self.name)
-        new_image_pulled = is_new_image_pulled(
-            type=IMA_CONTAINER, dutils=self.dutils)
+        new_image_pulled = is_new_image_pulled(image_type=IMA_CONTAINER, dutils=self.dutils)
 
         migration_ts = get_ima_migration_ts(self.name)
         new = time.time() > migration_ts
@@ -331,7 +341,7 @@ def ima_container(self) -> CheckRes:
 
         updated_image = False
         if container_running:
-            expected_image = get_image_name(type=IMA_CONTAINER, new=new)
+            expected_image = get_image_name(image_type=IMA_CONTAINER, new=new)
             image = self.dutils.get_container_image_name(container_name)
             updated_image = image == expected_image
 
@@ -396,7 +406,8 @@ def __init__(
         rotation_id: int = 0,
         *,
         econfig: Optional[ExternalConfig] = None,
-        dutils: DockerUtils = None
+        dutils: DockerUtils = None,
+        sync_node: bool = False
     ):
         self._subjects = [
             ConfigChecks(
@@ -407,14 +418,16 @@ def __init__(
                 stream_version=stream_version,
                 current_nodes=current_nodes,
                 estate=estate,
-                econfig=econfig
+                econfig=econfig,
+                sync_node=sync_node
             ),
             SkaledChecks(
                 schain_name=schain_name,
                 schain_record=schain_record,
                 rule_controller=rule_controller,
                 econfig=econfig,
-                dutils=dutils
+                dutils=dutils,
+                sync_node=sync_node
             )
         ]
 

core/schains/cleaner.py

@@ -41,7 +41,7 @@
 from core.schains.types import ContainerType
 from core.schains.firewall.utils import get_sync_agent_ranges
 
-from tools.configs import SGX_CERTIFICATES_FOLDER
+from tools.configs import SGX_CERTIFICATES_FOLDER, SYNC_NODE
 from tools.configs.schains import SCHAINS_DIR_PATH
 from tools.configs.containers import (
     SCHAIN_CONTAINER, IMA_CONTAINER, SCHAIN_STOP_TIMEOUT
@@ -245,7 +245,8 @@ def cleanup_schain(
         current_nodes=current_nodes,
         rotation_id=rotation_id,
         estate=estate,
-        dutils=dutils
+        dutils=dutils,
+        sync_node=SYNC_NODE
     )
     status = checks.get_all()
     if status['skaled_container'] or is_exited(

core/schains/cmd.py

@@ -25,18 +25,19 @@
 
 from tools.configs import SGX_SERVER_URL
 from tools.configs.containers import DATA_DIR_CONTAINER_PATH, SHARED_SPACE_CONTAINER_PATH
-from tools.configs.ima import IMA_ENDPOINT
+from tools.configs.web3 import ENDPOINT
 
 
 def get_schain_container_cmd(
     schain_name: str,
     start_ts: int = None,
     download_snapshot: bool = False,
     enable_ssl: bool = True,
+    sync_node: bool = False,
     snapshot_from: str = ''
 ) -> str:
     """Returns parameters that will be passed to skaled binary in the sChain container"""
-    opts = get_schain_container_base_opts(schain_name, enable_ssl=enable_ssl)
+    opts = get_schain_container_base_opts(schain_name, enable_ssl=enable_ssl, sync_node=sync_node)
     if snapshot_from:
         opts.extend(['--no-snapshot-majority', snapshot_from])
     if download_snapshot:
@@ -54,13 +55,17 @@ def get_schain_container_sync_opts(start_ts: int = None) -> list:
     return sync_opts
 
 
-def get_schain_container_base_opts(schain_name: str,
-                                   enable_ssl: bool = True) -> list:
+def get_schain_container_base_opts(
+    schain_name: str,
+    enable_ssl: bool = True,
+    sync_node: bool = False
+) -> list:
     config_filepath = get_skaled_container_config_path(schain_name)
     ssl_key, ssl_cert = get_ssl_filepath()
     config = ConfigFileManager(schain_name=schain_name).skaled_config
     ports = get_schain_ports_from_config(config)
     static_schain_cmd = get_static_schain_cmd()
+
     cmd = [
         f'--config {config_filepath}',
         f'-d {DATA_DIR_CONTAINER_PATH}',
@@ -69,11 +74,15 @@ def get_schain_container_base_opts(schain_name: str,
         f'--https-port {ports["https"]}',
         f'--ws-port {ports["ws"]}',
         f'--wss-port {ports["wss"]}',
-        f'--sgx-url {SGX_SERVER_URL}',
-        f'--shared-space-path {SHARED_SPACE_CONTAINER_PATH}/data',
-        f'--main-net-url {IMA_ENDPOINT}'
+        f'--main-net-url {ENDPOINT}'
     ]
 
+    if not sync_node:
+        cmd.extend([
+            f'--sgx-url {SGX_SERVER_URL}',
+            f'--shared-space-path {SHARED_SPACE_CONTAINER_PATH}/data'
+        ])
+
     if static_schain_cmd:
         cmd.extend(static_schain_cmd)