Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add evaluator info #154

Merged
merged 3 commits into from
Jan 19, 2024
Merged

feat: add evaluator info #154

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6461356
fix: modify page title
iwatake2222 Jan 17, 2024
f33d79d
feat: display information from evaluator
iwatake2222 Jan 17, 2024
5a1cd1e
chore(find_valid_duration): use default value
iwatake2222 Jan 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 12 additions & 9 deletions report/analyze_node/make_report_analyze_node.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,31 +24,32 @@
app = flask.Flask(__name__)


def render_index_page(stats, report_name, component_name, destination_path, template_path):
def render_index_page(stats, title, sub_title, destination_path, template_path):
"""Render html page"""
with app.app_context():
with open(template_path, 'r', encoding='utf-8') as f_html:
template_string = f_html.read()
rendered = flask.render_template_string(
template_string,
title=f'{component_name}: {report_name}',
title=title,
sub_title=sub_title,
stats=stats,
)

with open(destination_path, 'w', encoding='utf-8') as f_html:
f_html.write(rendered)


def render_detail_page(node_info, report_name, node_name, destination_path, template_path):
def render_detail_page(node_info, title, sub_title, destination_path, template_path):
"""Render html page"""
with app.app_context():
with open(template_path, 'r', encoding='utf-8') as f_html:
template_string = f_html.read()
rendered = flask.render_template_string(
template_string,
title=f'Node detail: {report_name}',
title=title,
sub_title=sub_title,
node_info=node_info,
node_name=node_name,
metrics_list=['Frequency', 'Period', 'Latency'],
metrics_unit=['[Hz]', '[ms]', '[ms]']
)
Expand All @@ -64,19 +65,21 @@ def make_report(stats_path: str):
stats = yaml.safe_load(f_yaml)

stats_dir = Path(stats_path).resolve().parent
report_name = stats_path.split('/')[-4]
component_name = stats_path.split('/')[-2]
destination_path = f'{stats_dir}/index.html'
template_path = f'{Path(__file__).resolve().parent}/template_node_index.html'
render_index_page(stats, report_name, component_name, destination_path, template_path)
title = f"Component: {stats_path.split('/')[-2]}"
sub_title = stats_path.split('/')[-4]
render_index_page(stats, title, sub_title, destination_path, template_path)

for node_name, node_info in stats.items():
if node_info is None:
continue
filename = node_name.replace('/', '_')[1:]
destination_path = f'{stats_dir}/index_{filename}.html'
template_path = f'{Path(__file__).resolve().parent}/template_node_detail.html'
render_detail_page(node_info, report_name, node_name, destination_path, template_path)
title = f'Node: {node_name}'
sub_title = stats_path.split('/')[-4]
render_detail_page(node_info, title, sub_title, destination_path, template_path)



Expand Down
6 changes: 2 additions & 4 deletions report/analyze_node/template_node_detail.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,17 @@
integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC"
crossorigin="anonymous"
/>
<title>{{ title }}</title>
<title>{{ title }}, {{ sub_title }}</title>
</head>

<body>
<div class="container">
<h1>{{ title }}</h1>
<h2>{{ sub_title }}</h2>

<p><a href="../../index.html">Back to Top</a>, <a href="./index.html">Back to Node List</a></p>
<hr />

<h2 id="{{ node_name }}">Node Name: {{ node_name }}</h2>
<hr />

{% for metrics in metrics_list %}
<h3>{{ metrics }}</h3>

Expand Down
3 changes: 2 additions & 1 deletion report/analyze_node/template_node_index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,13 @@
integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC"
crossorigin="anonymous"
/>
<title>{{ title }}</title>
<title>{{ title }}, {{ sub_title }}</title>
</head>

<body>
<div class="container">
<h1>{{ title }}</h1>
<h2>{{ sub_title }}</h2>

<p><a href="../../index.html">Back to Top </a></p>
<hr />
Expand Down
15 changes: 9 additions & 6 deletions report/analyze_path/make_report_analyze_path.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,15 @@
app = flask.Flask(__name__)


def render_page(stats, report_name, destination_path, template_path):
def render_page(stats, title, sub_title, destination_path, template_path):
"""Render html page"""
with app.app_context():
with open(template_path, 'r', encoding='utf-8') as f_html:
template_string = f_html.read()
rendered = flask.render_template_string(
template_string,
title=f'Path Analysis: {report_name}',
title=title,
sub_title=sub_title,
stats=stats,
)

Expand All @@ -46,17 +47,19 @@ def make_report(stats_path: str):
stats = yaml.safe_load(f_yaml)

stats_dir = Path(stats_path).resolve().parent
report_name = stats_path.split('/')[-3]

destination_path = f'{stats_dir}/index.html'
template_path = f'{Path(__file__).resolve().parent}/template_path_index.html'
render_page(stats, report_name, destination_path, template_path)
title = 'Path List'
sub_title = stats_path.split('/')[-3]
render_page(stats, title, sub_title, destination_path, template_path)

for path_info in stats:
target_path_name = path_info['target_path_name']
destination_path = f'{stats_dir}/{target_path_name}.html'
template_path = f'{Path(__file__).resolve().parent}/template_path_detail.html'
render_page(path_info, target_path_name, destination_path, template_path)
title = f'Path: {target_path_name}'
sub_title = stats_path.split('/')[-3]
render_page(path_info, title, sub_title, destination_path, template_path)


def parse_arg():
Expand Down
3 changes: 2 additions & 1 deletion report/analyze_path/template_path_detail.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,13 @@
integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC"
crossorigin="anonymous"
/>
<title>{{ title }}</title>
<title>{{ title }}, {{ sub_title }}</title>
</head>

<body>
<div class="container">
<h1>{{ title }}</h1>
<h2>{{ sub_title }}</h2>
<p><a href="../index.html">Back to Top </a>, <a href="./index.html">Back to Path List</a></p>
<hr />

Expand Down
3 changes: 2 additions & 1 deletion report/analyze_path/template_path_index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,13 @@
integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC"
crossorigin="anonymous"
/>
<title>{{ title }}</title>
<title>{{ title }}, {{ sub_title }}</title>
</head>

<body>
<div class="container">
<h1>{{ title }}</h1>
<h2>{{ sub_title }}</h2>
<p><a href="../index.html">Back to Top </a></p>
<hr />

Expand Down
31 changes: 17 additions & 14 deletions report/common/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@


class ComponentManager:
def __new__(cls, *args, **kargs):

Check warning on line 175 in report/common/utils.py

View workflow job for this annotation

GitHub Actions / spell-check-differential 

Unknown word (kargs)
if not hasattr(cls, "_instance"):
cls._instance = super(ComponentManager, cls).__new__(cls)
cls.component_dict = {} # pairs of component name and regular_exp
Expand Down Expand Up @@ -283,7 +283,6 @@
trace_datetime = datetime.datetime.strptime(trace_datetime.group(), '%Y%m%d-%H%M%S')
else:
trace_datetime = 'unknown'
note_text_top += f'<li>trace_data: {trace_data_name}</li>\n'
note_text_top += f'<li>trace_start_datetime: {trace_datetime}</li>\n'

# version info
Expand All @@ -294,35 +293,39 @@
note_text_top += f'<li>caret_analysis_version: {caret_analysis_version}</li>\n'

try:
repo_url_lines = subprocess.run(['git', 'remote', '-v'], text=True, stdout=subprocess.PIPE).stdout
repo_url_line = repo_url_lines.split('\n')[0]
repo_name = 'unknown'
for repo_url in repo_url_line.split():
if ('https://' in repo_url or 'git@' in repo_url) and '.git' in repo_url:
repo_url = repo_url.split('/')[-1]
repo_name = repo_url.split('.git')[0]
repo_url_lines = subprocess.run(['git', 'config', '--get', 'remote.origin.url'], text=True, stdout=subprocess.PIPE).stdout
repo_url = repo_url_lines.split('\n')[0]
report_setting_version = subprocess.run(['git', 'log', '-n 1', '--format=%H'], text=True, stdout=subprocess.PIPE).stdout
except:
repo_name = 'unknown'
report_setting_version = 'unknown'
note_text_top += f'<li>report_setting_repo: {repo_name}</li>\n'
note_text_top += f'<li>report_setting_repo: {repo_url}</li>\n'
note_text_top += f'<li>report_setting_version: {report_setting_version}</li>\n'
note_text_top += '</ul>\n'

# overwrite note_text_top if caret_record_info.yaml exists
# for info from evaluator, display only limited info
caret_record_info_path = f'{dest_dir}/caret_record_info.yaml'
if os.path.exists(caret_record_info_path):
with open(caret_record_info_path, encoding='UTF-8') as f_yaml:
caret_record_info = yaml.safe_load(f_yaml)
note_text_top = '<ul>\n'
note_text_top += f'<li>trace_data: {trace_data_name}</li>\n'
note_text_top += f'<li>trace_start_datetime: {trace_datetime}</li>\n'
caret_record_info = yaml.safe_load(f_yaml)
new_caret_record_info = {}
if 'caret_version' in caret_record_info:
caret_record_info['caret_version'] = caret_record_info.pop('caret_version')
caret_record_info['caret_version'] = caret_record_info.pop('caret_version') # move to the last
for key, value in caret_record_info.items():
note_text_top += f'<li>{key}: {value}</li>\n'
if 'evaluator_' in key:
display_key_list = ['evaluator_job_description', 'evaluator_git_commit_url', 'evaluator_report_url', 'evaluator_scenario_display_name']
if not any(display_key in key for display_key in display_key_list):
continue
key = key.replace('evaluator_', '')
if 'http' in value:
note_text_top += f'<li>{key}: <a href={value}>{value}</a></li>\n'
else:
note_text_top += f'<li>{key}: {value}</li>\n'
note_text_top += f'<li>caret_analysis_version: {caret_analysis_version}</li>\n'
note_text_top += f'<li>report_setting_repo: {repo_name}</li>\n'
note_text_top += f'<li>report_setting_repo: {repo_url}</li>\n'
note_text_top += f'<li>report_setting_version: {report_setting_version}</li>\n'
note_text_top += '</ul>\n'

Expand Down
2 changes: 1 addition & 1 deletion report/find_valid_duration/find_valid_duration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ def parse_arg():
description='Script to analyze path')
parser.add_argument('trace_data', nargs=1, type=str)
parser.add_argument('--architecture_file_path', type=str, default='architecture_path.yaml')
parser.add_argument('--duration', type=float, default=60.0,
parser.add_argument('--duration', type=float, default=1800.0,
help='Duration [sec] to load trace data')
parser.add_argument('--load_duration', type=float, default=120.0,
help='Duration [sec] to load trace data to find valid duration')
Expand Down
2 changes: 1 addition & 1 deletion report/find_valid_duration/find_valid_duration.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@ trace_data_name=$(basename "${trace_data}")

# Path analysis
python3 "${script_path}"/analyze_path/add_path_to_architecture.py "${trace_data}" "${target_path_json}" --architecture_file_path=architecture_path.yaml --max_node_depth="${max_node_depth}" --timeout="${timeout}" -v
python3 "${script_path}"/find_valid_duration/find_valid_duration.py "${trace_data}" --architecture_file_path=architecture_path.yaml --duration "${duration}" -v
python3 "${script_path}"/find_valid_duration/find_valid_duration.py "${trace_data}" --architecture_file_path=architecture_path.yaml -v
8 changes: 5 additions & 3 deletions report/report_analysis/make_report_analysis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from pathlib import Path
import yaml
import flask
from markupsafe import Markup

Check warning on line 24 in report/report_analysis/make_report_analysis.py

View workflow job for this annotation

GitHub Actions / spell-check-differential 

Unknown word (markupsafe)
sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/..')
from common.utils import ComponentManager
from common.utils import read_note_text
Expand All @@ -29,16 +29,16 @@
app = flask.Flask(__name__)


def render_page(destination_path, template_path, component_list, stats_node_dict,
def render_page(title, sub_title, destination_path, template_path, component_list, stats_node_dict,
stats_path, note_text_top, note_text_bottom, num_back):
"""Render html page"""
title = f'Analysis report'
with app.app_context():
with open(template_path, 'r', encoding='utf-8') as f_html:
template_string = f_html.read()
rendered = flask.render_template_string(
template_string,
title=title,
sub_title=sub_title,
component_list=component_list,
stats_node_dict=stats_node_dict,
stats_path=stats_path,
Expand Down Expand Up @@ -80,7 +80,7 @@
return stats


def find_latency_topk(component_name, stats_node, numk=20) -> None:

Check warning on line 83 in report/report_analysis/make_report_analysis.py

View workflow job for this annotation

GitHub Actions / spell-check-differential 

Unknown word (numk)
"""Find callback functions whose latency time is the longest(top5), and add this information into stats"""
callback_latency_list = []
for node_name, node_info in stats_node.items():
Expand All @@ -89,7 +89,7 @@
trigger = callback_info['subscribe_topic_name'] if callback_info['period_ns'] == -1 else f'{float(callback_info["period_ns"]) / 1e6} [ms]'
callback_latency_list.append({
'link': f'analyze_node/{component_name}/index{node_name.replace("/", "_")}.html',
'displayname': Markup(node_name + '<br>' + callback_info['callback_legend'] + ': ' + trigger),

Check warning on line 92 in report/report_analysis/make_report_analysis.py

View workflow job for this annotation

GitHub Actions / spell-check-differential 

Unknown word (displayname)
'avg': callback_info['Latency']['avg'] if isinstance(callback_info['Latency']['avg'], (int, float)) else 0,
'min': callback_info['Latency']['min'] if isinstance(callback_info['Latency']['min'], (int, float)) else 0,
'max': callback_info['Latency']['max'] if isinstance(callback_info['Latency']['max'], (int, float)) else 0,
Expand All @@ -99,7 +99,7 @@
})

callback_latency_list = sorted(callback_latency_list, reverse=True, key=lambda x: x['p50'])
callback_latency_list = callback_latency_list[:numk]

Check warning on line 102 in report/report_analysis/make_report_analysis.py

View workflow job for this annotation

GitHub Actions / spell-check-differential 

Unknown word (numk)
stats_node['latency_topk'] = callback_latency_list


Expand All @@ -120,7 +120,9 @@

destination_path = f'{dest_dir}/{index_filename}.html'
template_path = f'{Path(__file__).resolve().parent}/template_report_analysis.html'
render_page(destination_path, template_path, component_list, stats_node_dict,
title = 'Analysis report'
sub_title = dest_dir.split('/')[-1]
render_page(title, sub_title, destination_path, template_path, component_list, stats_node_dict,
stats_path, note_text_top, note_text_bottom, args.num_back)


Expand Down
3 changes: 2 additions & 1 deletion report/report_analysis/template_report_analysis.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,14 @@
integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC"
crossorigin="anonymous"
/>
<title>{{ title }}</title>
<title>{{ title }}, {{ sub_title }}</title>
</head>

<body>
<div class="container">
<!-- <div> -->
<h1>{{ title }}</h1>
<h2>{{ sub_title }}</h2>
{% if link_back != "" %}
<p><a href={{ link_back }}>Back</a></p>
{% endif %}
Expand Down
5 changes: 3 additions & 2 deletions report/report_validation/make_report_validation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@


app = flask.Flask(__name__)
app.jinja_env.add_extension('jinja2.ext.loopcontrols')

Check warning on line 33 in report/report_validation/make_report_validation.py

View workflow job for this annotation

GitHub Actions / spell-check-differential 

Unknown word (loopcontrols)


def make_report(report_dir: str, component_list_json: str, note_text_top, note_text_bottom, num_back):
Expand Down Expand Up @@ -62,16 +62,17 @@
summary_topic_dict['cnt_pass'] += summary[Metrics.FREQUENCY.name]['cnt_pass']
summary_topic_dict['cnt_failed'] += summary[Metrics.FREQUENCY.name]['cnt_failed']

title = f'Validation report'
destination_path = f'{report_dir}/index.html'
template_path = f'{Path(__file__).resolve().parent}/template_report_validation.html'

title = 'Validation report'
sub_title = report_dir.split('/')[-1]
with app.app_context():
with open(template_path, 'r', encoding='utf-8') as f_html:
template_string = f_html.read()
rendered = flask.render_template_string(
template_string,
title=title,
sub_title=sub_title,
summary_callback_dict_component_metrics=summary_callback_dict_component_metrics,
summary_topic_dict_component_pair_metrics=summary_topic_dict_component_pair_metrics,
summary_callback_dict=summary_callback_dict,
Expand Down
4 changes: 3 additions & 1 deletion report/report_validation/template_report_validation.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,20 @@
rel="stylesheet"
href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.8.0/font/bootstrap-icons.css"
/>
<title>{{ title }}</title>
<title>{{ title }}, {{ sub_title }}</title>
</head>

<body>
<div class="container">
<h1>{{ title }}</h1>
<h2>{{ sub_title }}</h2>
{% if link_back != "" %}
<p><a href={{ link_back }}>Back</a></p>
{% endif %}

<hr />
{% if note_text_top %} {{ note_text_top | safe }} {% endif %}
<hr />

<ul>
<li>Validation Result</li>
Expand Down
45 changes: 27 additions & 18 deletions report/track_path/make_report_track_path.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
return f'{target_path_name.replace("/", "_")}.html'


def render_page(reportpath_version_dict, stats_path_dict, filename_path_dict, destination_path, template_path):

Check warning on line 56 in report/track_path/make_report_track_path.py

View workflow job for this annotation

GitHub Actions / spell-check-differential 

Unknown word (reportpath)
"""Render html page"""
with app.app_context():
with open(template_path, 'r', encoding='utf-8') as f_html:
Expand All @@ -61,7 +61,7 @@
rendered = flask.render_template_string(
template_string,
title='Response Time Tracking',
reportpath_version_dict=reportpath_version_dict,

Check warning on line 64 in report/track_path/make_report_track_path.py

View workflow job for this annotation

GitHub Actions / spell-check-differential 

Unknown word (reportpath)

Check warning on line 64 in report/track_path/make_report_track_path.py

View workflow job for this annotation

GitHub Actions / spell-check-differential 

Unknown word (reportpath)
stats_path_dict=stats_path_dict,
filename_path_dict=filename_path_dict,
)
Expand Down Expand Up @@ -91,21 +91,35 @@
return str(trace_datetime)


def get_trace_data_info_from_yaml(yaml_dir: str):
caret_record_info_path = Path(yaml_dir).joinpath('caret_record_info.yaml')
autoware_version, env, route = '', '', ''
def get_version_str_from_yaml(report_dir: str):
caret_record_info = {}
caret_record_info_path = Path(report_dir).joinpath('caret_record_info.yaml')
if os.path.exists(caret_record_info_path):
with open(caret_record_info_path, encoding='UTF-8') as f_yaml:
caret_record_info = yaml.safe_load(f_yaml)
if 'autoware_version' in caret_record_info:
autoware_version = str(caret_record_info['autoware_version']).split('/')[-1]
if 'env' in caret_record_info:
env = caret_record_info['env']
elif 'map' in caret_record_info:
env = caret_record_info['map']
if 'route' in caret_record_info:
route = caret_record_info['route']
return autoware_version, env, route
autoware_version = caret_record_info.get('autoware_version', '')
pilot_auto_version = caret_record_info.get('pilot_auto_version', '')
env = caret_record_info.get('env', '')
env = caret_record_info.get('map', '') if env == '' else env
route = caret_record_info.get('route', '')
job_description = caret_record_info.get('evaluator_job_description', '') # info from evaluator

datetime = get_datetime_from_report_name(report_dir)
version = datetime
if autoware_version and pilot_auto_version:
version = version + ', ' + f'{pilot_auto_version}({autoware_version})'
elif autoware_version and pilot_auto_version == '':
version = version + ', ' + autoware_version
elif autoware_version == '' and pilot_auto_version:
version = version + ', ' + pilot_auto_version
if job_description:
version = version + ', ' + job_description
if env:
version = version + ', ' + env
if route:
version = version + ', ' + route

return version


def make_stats_file_dict(dest_dir: str, report_store_dir: str) -> list[tuple[str,str]]:
Expand All @@ -126,12 +140,7 @@

for report_dir in report_list:
report_dir = report_dir.rstrip('/')
datetime = get_datetime_from_report_name(report_dir)
autoware_version, env, route = get_trace_data_info_from_yaml(report_dir)
if autoware_version != '':
version = datetime + ', ' + autoware_version + ', ' + env + ',' + route
else:
version = datetime
version = get_version_str_from_yaml(report_dir)
stats_path = f'{report_dir}/analyze_path/stats_path.yaml'
if os.path.isfile(stats_path):
if version in stats_file_dict:
Expand All @@ -157,7 +166,7 @@
"""Make stats"""
stats_file_dict = make_stats_file_dict(dest_dir, report_store_dir)

reportpath_version_dict = {} # key: version, value: path to report

Check warning on line 169 in report/track_path/make_report_track_path.py

View workflow job for this annotation

GitHub Actions / spell-check-differential 

Unknown word (reportpath)
for version, stats_file in stats_file_dict.items():
path_report_file = Path(stats_file).parent.joinpath('index.html').resolve()
top_report_file = Path(stats_file).parent.parent.joinpath('index.html').resolve()
Expand Down
Loading
Loading