Skip to content

Commit 06d836a

Browse files
Dev: migration: run checks on remote nodes (jsc#PED-8252)
1 parent 097abeb commit 06d836a

File tree

2 files changed

+193
-27
lines changed

2 files changed

+193
-27
lines changed

crmsh/migration.py

Lines changed: 188 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
1+
import argparse
2+
import json
13
import logging
24
import os
35
import re
46
import shutil
7+
import sys
8+
import threading
59
import tempfile
610
import typing
711

@@ -14,6 +18,7 @@
1418
from crmsh import service_manager
1519
from crmsh import sh
1620
from crmsh import utils
21+
from crmsh.prun import prun
1722

1823
logger = logging.getLogger(__name__)
1924

@@ -22,6 +27,60 @@ class MigrationFailure(Exception):
2227
pass
2328

2429

30+
class CheckResultHandler:
31+
def log_info(self, fmt: str, *args):
32+
raise NotImplementedError
33+
34+
def handle_problem(self, is_fatal: bool, title: str, detail: typing.Iterable[str]):
35+
raise NotImplementedError
36+
37+
38+
class CheckResultJsonHandler(CheckResultHandler):
39+
def __init__(self):
40+
self.json_result = {
41+
"pass": True,
42+
"problems": [],
43+
}
44+
def log_info(self, fmt: str, *args):
45+
logger.debug(fmt, *args)
46+
47+
def handle_problem(self, is_fatal: bool, title: str, detail: typing.Iterable[str]):
48+
self.json_result["pass"] = False
49+
self.json_result["problems"].append({
50+
"is_fatal": is_fatal,
51+
"title": title,
52+
"descriptions": detail if isinstance(detail, list) else list(detail),
53+
})
54+
55+
56+
class CheckResultInteractiveHandler(CheckResultHandler):
57+
def __init__(self):
58+
self.has_problems = False
59+
60+
def log_info(self, fmt: str, *args):
61+
self.write_in_color(sys.stdout, constants.GREEN, '[INFO] ')
62+
print(fmt % args)
63+
64+
def handle_problem(self, is_fatal: bool, title: str, details: typing.Iterable[str]):
65+
self.has_problems = True
66+
self.write_in_color(sys.stdout, constants.YELLOW, '[FAIL] ')
67+
print(title)
68+
for line in details:
69+
sys.stdout.write(' ')
70+
print(line)
71+
if is_fatal:
72+
raise MigrationFailure('Unable to start migration.')
73+
74+
@staticmethod
75+
def write_in_color(f, color: str, text: str):
76+
if f.isatty():
77+
f.write(color)
78+
f.write(text)
79+
f.write(constants.END)
80+
else:
81+
f.write(text)
82+
83+
2584
def migrate():
2685
try:
2786
check()
@@ -34,46 +93,151 @@ def migrate():
3493
return 1
3594

3695

37-
def check():
38-
has_problems = [False]
39-
def problem_handler(is_fatal: bool, title: str, detail: typing.Iterable[str]):
40-
has_problems[0] = True
41-
logger.error('%s', title)
42-
for line in detail:
43-
logger.info(' %s', line)
44-
if is_fatal:
45-
raise MigrationFailure('Unable to start migration.')
46-
check_dependency_version(problem_handler)
47-
check_service_status(problem_handler)
48-
check_unsupported_corosync_features(problem_handler)
49-
# TODO: run checks on all cluster nodes
50-
if has_problems[0]:
51-
raise MigrationFailure('Unable to start migration.')
52-
53-
54-
def check_dependency_version(handler: typing.Callable[[bool, str, typing.Iterable[str]], None]):
55-
logger.info('Checking dependency version...')
96+
def check(args: typing.Sequence[str]) -> int:
97+
parser = argparse.ArgumentParser()
98+
parser.add_argument('--json', default='no', choices=['no', 'oneline', 'pretty'])
99+
parser.add_argument('--local', action='store_true')
100+
parsed_args = parser.parse_args(args)
101+
ret = 0
102+
if not parsed_args.local:
103+
check_remote_yield = check_remote()
104+
next(check_remote_yield)
105+
print('------ localhost ------')
106+
else:
107+
check_remote_yield = [0]
108+
match parsed_args.json:
109+
case 'no':
110+
handler = CheckResultInteractiveHandler()
111+
case 'oneline' | 'pretty':
112+
handler = CheckResultJsonHandler()
113+
case _:
114+
assert False
115+
check_local(handler)
116+
match handler:
117+
case CheckResultJsonHandler():
118+
json.dump(
119+
handler.json_result,
120+
sys.stdout,
121+
ensure_ascii=False,
122+
indent=2 if parsed_args.json == 'pretty' else None,
123+
)
124+
sys.stdout.write('\n')
125+
return handler.json_result["pass"]
126+
case CheckResultInteractiveHandler():
127+
if handler.has_problems:
128+
ret = 1
129+
else:
130+
handler.write_in_color(
131+
sys.stdout, constants.GREEN, '[PASS]\n'
132+
)
133+
if check_remote_yield:
134+
remote_ret = next(check_remote_yield)
135+
if remote_ret > ret:
136+
ret = remote_ret
137+
return ret
138+
139+
140+
def check_local(handler: CheckResultHandler):
141+
check_dependency_version(handler)
142+
check_service_status(handler)
143+
check_unsupported_corosync_features(handler)
144+
145+
146+
def check_remote():
147+
handler = CheckResultInteractiveHandler()
148+
class CheckRemoteThread(threading.Thread):
149+
def run(self):
150+
self.result = prun.prun({
151+
node: 'crm cluster health sles16 --local --json=oneline'
152+
for node in utils.list_cluster_nodes_except_me()
153+
})
154+
prun_thread = CheckRemoteThread()
155+
prun_thread.start()
156+
yield
157+
prun_thread.join()
158+
ret = 0
159+
for host, result in prun_thread.result.items():
160+
match result:
161+
case prun.SSHError() as e:
162+
handler.write_in_color(
163+
sys.stdout, constants.YELLOW,
164+
f'\n------ {host} ------\n',
165+
)
166+
handler.write_in_color(
167+
sys.stdout, constants.YELLOW,
168+
str(e)
169+
)
170+
sys.stdout.write('\n')
171+
ret = 255
172+
case prun.ProcessResult() as result:
173+
if result.returncode > 1:
174+
handler.write_in_color(
175+
sys.stdout, constants.YELLOW,
176+
f'\n------ {host} ------\n',
177+
)
178+
print(result.stdout.decode('utf-8', 'backslashreplace'))
179+
handler.write_in_color(
180+
sys.stdout, constants.YELLOW,
181+
result.stderr.decode('utf-8', 'backslashreplace')
182+
)
183+
sys.stdout.write('\n')
184+
ret = result.returncode
185+
else:
186+
try:
187+
result = json.loads(result.stdout.decode('utf-8'))
188+
except (UnicodeDecodeError, json.JSONDecodeError):
189+
handler.write_in_color(
190+
sys.stdout, constants.YELLOW,
191+
f'\n------ {host} ------\n',
192+
)
193+
print(result.stdout.decode('utf-8', 'backslashreplace'))
194+
handler.write_in_color(
195+
sys.stdout, constants.YELLOW,
196+
result.stdout.decode('utf-8', 'backslashreplace')
197+
)
198+
sys.stdout.write('\n')
199+
ret = result.returncode
200+
else:
201+
passed = result.get("pass", False)
202+
handler.write_in_color(
203+
sys.stdout, constants.GREEN if passed else constants.YELLOW,
204+
f'\n------ {host} ------\n',
205+
)
206+
handler = CheckResultInteractiveHandler()
207+
for problem in result.get("problems", list()):
208+
handler.handle_problem(False, problem.get("title", ""), problem.get("descriptions"))
209+
if passed:
210+
handler.write_in_color(
211+
sys.stdout, constants.GREEN, '[PASS]\n'
212+
)
213+
else:
214+
ret = 1
215+
yield ret
216+
217+
218+
def check_dependency_version(handler: CheckResultHandler):
219+
handler.log_info('Checking dependency version...')
56220
shell = sh.LocalShell()
57221
out = shell.get_stdout_or_raise_error(None, 'corosync -v')
58222
match = re.search(r"version\s+'((\d+)(?:\.\d+)*)'", out)
59223
if not match or match.group(2) != '3':
60-
handler(
224+
handler.handle_problem(
61225
False, 'Corosync version not supported', [
62226
'Supported version: corosync >= 3',
63227
f'Actual version: corosync == {match.group(1)}',
64228
],
65229
)
66230

67231

68-
def check_service_status(handler: typing.Callable[[bool, str, typing.Iterable[str]], None]):
69-
logger.info('Checking service status...')
232+
def check_service_status(handler: CheckResultHandler):
233+
handler.log_info('Checking service status...')
70234
manager = service_manager.ServiceManager()
71235
active_services = [x for x in ['corosync', 'pacemaker'] if manager.service_is_active(x)]
72236
if active_services:
73-
handler(False, 'Cluster services are running', (f'* {x}' for x in active_services))
237+
handler.handle_problem(False, 'Cluster services are running', (f'* {x}' for x in active_services))
74238

75239

76-
def check_unsupported_corosync_features(handler: typing.Callable[[bool, str, str], None]):
240+
def check_unsupported_corosync_features(handler: CheckResultHandler):
77241
pass
78242

79243

crmsh/ui_cluster.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -753,9 +753,12 @@ def do_health(self, context, *args):
753753
parser = argparse.ArgumentParser()
754754
parser.add_argument('component', choices=['hawk2', 'sles16'])
755755
parser.add_argument('-f', '--fix', action='store_true')
756-
parsed_args = parser.parse_args(args)
756+
parsed_args, remaining_args = parser.parse_known_args(args)
757757
match parsed_args.component:
758758
case 'hawk2':
759+
if remaining_args:
760+
logger.error('Known arguments: %s', ' '.join(remaining_args))
761+
return False
759762
nodes = utils.list_cluster_nodes()
760763
if parsed_args.fix:
761764
if not healthcheck.feature_full_check(healthcheck.PasswordlessPrimaryUserAuthenticationFeature(), nodes):
@@ -788,8 +791,7 @@ def do_health(self, context, *args):
788791
if parsed_args.fix:
789792
migration.migrate()
790793
else:
791-
migration.check()
792-
return True
794+
return 0 == migration.check(remaining_args)
793795
except migration.MigrationFailure as e:
794796
logger.error('%s', e)
795797
return False

0 commit comments

Comments
 (0)