Skip to content

Commit

Permalink
provide a traceback to our timeout errors
Browse files Browse the repository at this point in the history
  • Loading branch information
yrro committed Mar 13, 2024
1 parent ab79e14 commit 35a813a
Showing 1 changed file with 10 additions and 7 deletions.
17 changes: 10 additions & 7 deletions src/ipahealthcheck/core/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,25 +46,26 @@ def find_plugins(name, registry):

def run_plugin(plugin, available=(), timeout=constants.DEFAULT_TIMEOUT):
plugin_name = F"{plugin.__class__.__module__}:{plugin.__class__.__name__}"
timed_out = []
timed_out = {}

def signal_handler(signum, frame):
# Our exception will likely be caught by code called by the plugin,
# which may not log the traceback, or may even catch the exception
# itself! So while we throw the exception in order to interrupt the
# plugin code, we also stash a copy of it so that we can log it after
# the plugin returns.
timed_out.append(TimeoutError(
timed_out["exception"] = TimeoutError(
f"Health check {plugin_name} cancelled after {timeout} sec"
))
logger.error("--- %s ---", timed_out[0])
)
logger.error("--- %s ---", timed_out["exception"])
traceback.print_stack()
timed_out["stack"] = traceback.format_stack()
logger.error(
"--- The following messages were logged by the plugin after it"
" was cancelled. They may not indicate the reason why the plugin"
" took too long to run. ---"
)
raise timed_out[0]
raise timed_out["exception"]

# manually calculate duration when we create results of our own
start = datetime.now(tz=timezone.utc)
Expand Down Expand Up @@ -110,8 +111,10 @@ def signal_handler(signum, frame):
# case let's err on the side of caution and return an additional
# result.
# TODO: suggest that timeouts are CRITICAL?
yield Result(plugin, constants.ERROR, exception=str(timed_out[0]),
key="healthcheck_timeout", start=start)
yield Result(plugin, constants.ERROR,
exception=str(timed_out["exception"]),
key="healthcheck_timeout", start=start,
traceback=timed_out["stack"])
else:
logging.debug(
"Plugin %s complete after %s sec",
Expand Down

0 comments on commit 35a813a

Please sign in to comment.