From f729c0742a0403bbd7da6c05fca8dac78632b540 Mon Sep 17 00:00:00 2001 From: "Myan V." Date: Fri, 25 Jul 2025 00:36:55 +1200 Subject: [PATCH 1/7] Add agent subsection parsing --- report/parse_logs.py | 116 +++++++++++++++++------ report/templates/sample/sample.html | 139 ++++++++++++++++++++++++++-- 2 files changed, 221 insertions(+), 34 deletions(-) diff --git a/report/parse_logs.py b/report/parse_logs.py index 60104bca1..df9537900 100644 --- a/report/parse_logs.py +++ b/report/parse_logs.py @@ -35,6 +35,64 @@ class LogsParser: def __init__(self, logs: list[LogPart]): self._logs = logs + def _parse_steps_from_logs(self, agent_logs: list[LogPart]) -> list[dict]: + """Parse steps from agent logs, grouping by step number.""" + step_pattern = re.compile(r"Step #(\d+) - \"(.+?)\":") + simple_step_pattern = re.compile(r"Step #(\d+)") + + steps_dict = {} + current_step_number = None + current_step_name = None + + for log_part in agent_logs: + content = log_part.content.strip() + if not content: + continue + + lines = content.split('\n') + + step_header_found = False + for line in lines: + step_match = step_pattern.search(line) + if not step_match: + simple_match = simple_step_pattern.search(line) + if simple_match: + step_match = simple_match + + if step_match: + step_header_found = True + current_step_number = step_match.group(1) + if len(step_match.groups()) > 1: + current_step_name = step_match.group(2).strip() + else: + current_step_name = "agent-step" + + if current_step_number not in steps_dict: + steps_dict[current_step_number] = { + 'number': current_step_number, + 'name': current_step_name, + 'type': 'Step', + 'log_parts': [] + } + break + + if not step_header_found and current_step_number: + steps_dict[current_step_number]['log_parts'].append(log_part) + elif not step_header_found and not current_step_number and not steps_dict: + steps_dict['0'] = { + 'number': None, + 'name': None, + 'type': 'Content', + 'log_parts': [log_part] + } + + steps = [] + for step_num in sorted(steps_dict.keys(), + key=lambda x: int(x) if x.isdigit() else 999): + steps.append(steps_dict[step_num]) + + return steps + def get_agent_sections(self) -> dict[str, list[LogPart]]: """Get the agent sections from the logs.""" @@ -91,16 +149,22 @@ def get_agent_cycles(self) -> list[dict]: cycles_dict = {} for agent_name, agent_logs in agent_sections.items(): + # Parse steps for this agent + steps = self._parse_steps_from_logs(agent_logs) + cycle_match = re.search(r'\(Cycle (\d+)\)', agent_name) if cycle_match: cycle_number = int(cycle_match.group(1)) if cycle_number not in cycles_dict: cycles_dict[cycle_number] = {} - cycles_dict[cycle_number][agent_name] = agent_logs + cycles_dict[cycle_number][agent_name] = { + 'logs': agent_logs, + 'steps': steps + } else: if 0 not in cycles_dict: cycles_dict[0] = {} - cycles_dict[0][agent_name] = agent_logs + cycles_dict[0][agent_name] = {'logs': agent_logs, 'steps': steps} return [cycles_dict[cycle] for cycle in sorted(cycles_dict.keys())] @@ -178,30 +242,30 @@ def get_formatted_stack_traces(self, function_name = in_match.group(1) path = in_match.group(2) - if '/src/' in path and 'llvm-project' not in path and self._benchmark_id and self._sample_id: - path_parts = path.split(':') - file_path = path_parts[0] # Just the file path without line numbers - line_number = path_parts[1] if len(path_parts) > 1 else None - - relative_path = file_path.lstrip('/') - - # If coverage_report_path is set, it's a local run - # Otherwise it's cloud - if self._coverage_report_path: - url = f'{self._coverage_report_path}{relative_path}.html' - url_with_line_number = f'{url}#L{line_number}' if line_number else url - else: - url = ( - f'{base_url}/results/{self._benchmark_id}/code-coverage-reports/' - f'{self._sample_id}.fuzz_target/report/linux/' - f'{relative_path}.html') - url_with_line_number = f'{url}#L{line_number}' if line_number else url - stack_traces[frame_num] = { - "url": url_with_line_number, - "path": path, - "function": function_name, - "memory_address": memory_addr - } + if '/src/' in path and 'llvm-project' not in path: + if self._benchmark_id and self._sample_id: + path_parts = path.split(':') + file_path = path_parts[0] + line_number = path_parts[1] if len(path_parts) > 1 else None + + relative_path = file_path.lstrip('/') + + # If coverage_report_path is set, it's a local run + # Otherwise it's cloud + if self._coverage_report_path: + url = f'{self._coverage_report_path}{relative_path}.html' + url_line_number = f'{url}#L{line_number}' if line_number else url + else: + url = (f'{base_url}/results/{self._benchmark_id}/' + f'code-coverage-reports/{self._sample_id}.fuzz_target/' + f'report/linux/{relative_path}.html') + url_line_number = f'{url}#L{line_number}' if line_number else url + stack_traces[frame_num] = { + "url": url_line_number, + "path": path, + "function": function_name, + "memory_address": memory_addr + } return stack_traces diff --git a/report/templates/sample/sample.html b/report/templates/sample/sample.html index 67f10c909..ce9c62c59 100644 --- a/report/templates/sample/sample.html +++ b/report/templates/sample/sample.html @@ -283,7 +283,7 @@

Build Script

{% for cycle_data in agent_cycles %} {% if 'standalone' in cycle_data %} - {% for agent_name, agent_logs in cycle_data.standalone.items() %} + {% for agent_name, agent_data in cycle_data.standalone.items() %}
- {% for log_part in agent_logs %} - 
{{ log_part.content }}
- {% endfor %} + {% if agent_data.steps %} + {% for step in agent_data.steps %} +
+ +
+ 
{{ step.content }}
+
+
+ {% endfor %} + {% else %} + {% for log_part in agent_data.logs %} + 
{{ log_part.content }}
+ {% endfor %} + {% endif %}
{% endfor %} @@ -314,7 +333,7 @@

Build Script

Cycle {{ loop.index }}
- {% for agent_name, agent_logs in cycle_data.items() %} + {% for agent_name, agent_data in cycle_data.items() %}
- {% for log_part in agent_logs %} - 
{{ log_part.content }}
- {% endfor %} + {% if agent_data.steps %} + {% if agent_data.steps|length == 1 %} + + {% set step = agent_data.steps[0] %} + {% if step.number and step.name %} +
+
Step {{ step.number }} - {{ step.name }}
+ {% if step.log_parts %} + {% for log_part in step.log_parts %} +
+ +
+ 
{{ log_part.content }}
+
+
+ {% endfor %} + {% else %} + 
{{ step.content }}
+ {% endif %} +
+ {% else %} + {% if step.log_parts %} + {% for log_part in step.log_parts %} + 
{{ log_part.content }}
+ {% endfor %} + {% else %} + 
{{ step.content }}
+ {% endif %} + {% endif %} + {% else %} + + {% for step in agent_data.steps %} +
+ +
+ {% if step.log_parts %} + {% for log_part in step.log_parts %} +
+ +
+ 
{{ log_part.content }}
+
+
+ {% endfor %} + {% else %} + 
{{ step.content }}
+ {% endif %} +
+
+ {% endfor %} + {% endif %} + {% else %} + {% for log_part in agent_data.logs %} + 
{{ log_part.content }}
+ {% endfor %} + {% endif %}
{% endfor %} From 761ad74c6664e11b2443e39d9aa57af7018c2eb7 Mon Sep 17 00:00:00 2001 From: "Myan V." Date: Tue, 29 Jul 2025 11:56:03 +1200 Subject: [PATCH 2/7] Remove the added layer of collapse for chat prompt/response, add tools in the title --- report/parse_logs.py | 37 ++++++++++++++++---- report/templates/sample/sample.html | 52 +++-------------------------- 2 files changed, 35 insertions(+), 54 deletions(-) diff --git a/report/parse_logs.py b/report/parse_logs.py index df9537900..10b302155 100644 --- a/report/parse_logs.py +++ b/report/parse_logs.py @@ -35,6 +35,30 @@ class LogsParser: def __init__(self, logs: list[LogPart]): self._logs = logs + def _extract_tool_names(self, content: str) -> list[str]: + """Extract tool names from content.""" + tool_counts = {} + lines = content.split('\n') + + for i, line in enumerate(lines): + line = line.strip() + if (line in ['', '', '', '', ''] and + not line.startswith('': + if i + 1 < len(lines) and lines[i + 1].strip(): + tool_counts['Stderr'] = tool_counts.get('Stderr', 0) + 1 + + tool_names = [] + for tool_name, count in tool_counts.items(): + if count == 1: + tool_names.append(tool_name) + else: + tool_names.append(f"{tool_name} (x{count})") + + return tool_names + def _parse_steps_from_logs(self, agent_logs: list[LogPart]) -> list[dict]: """Parse steps from agent logs, grouping by step number.""" step_pattern = re.compile(r"Step #(\d+) - \"(.+?)\":") @@ -62,15 +86,10 @@ def _parse_steps_from_logs(self, agent_logs: list[LogPart]) -> list[dict]: if step_match: step_header_found = True current_step_number = step_match.group(1) - if len(step_match.groups()) > 1: - current_step_name = step_match.group(2).strip() - else: - current_step_name = "agent-step" if current_step_number not in steps_dict: steps_dict[current_step_number] = { 'number': current_step_number, - 'name': current_step_name, 'type': 'Step', 'log_parts': [] } @@ -81,11 +100,17 @@ def _parse_steps_from_logs(self, agent_logs: list[LogPart]) -> list[dict]: elif not step_header_found and not current_step_number and not steps_dict: steps_dict['0'] = { 'number': None, - 'name': None, 'type': 'Content', 'log_parts': [log_part] } + for step_num, step_data in steps_dict.items(): + if step_data['log_parts']: + all_content = '\n'.join([part.content for part in step_data['log_parts']]) + tool_names = self._extract_tool_names(all_content) + if tool_names: + step_data['name'] = f"{', '.join(tool_names)}" + steps = [] for step_num in sorted(steps_dict.keys(), key=lambda x: int(x) if x.isdigit() else 999): diff --git a/report/templates/sample/sample.html b/report/templates/sample/sample.html index ce9c62c59..f93f77b0d 100644 --- a/report/templates/sample/sample.html +++ b/report/templates/sample/sample.html @@ -351,32 +351,10 @@

Build Script

Step {{ step.number }} - {{ step.name }}
{% if step.log_parts %} {% for log_part in step.log_parts %} -
- -
- 
{{ log_part.content }}
-
-
+ 
{{ log_part.content }}
{% endfor %} {% else %} - 
{{ step.content }}
+ 
{{ step.content }}
{% endif %} {% else %} @@ -413,32 +391,10 @@

Build Script

{% if step.log_parts %} {% for log_part in step.log_parts %} -
- -
- 
{{ log_part.content }}
-
-
+ 
{{ log_part.content }}
{% endfor %} {% else %} - 
{{ step.content }}
+ 
{{ step.content }}
{% endif %}
From a1292bc2a19d9916c7a7030705725a6794ba37ef Mon Sep 17 00:00:00 2001 From: "Myan V." Date: Tue, 29 Jul 2025 12:17:50 +1200 Subject: [PATCH 3/7] Add bash tool preview as subtitle heading --- report/parse_logs.py | 93 +++++++++++++++++++++++++++-- report/templates/sample/sample.html | 54 +++++++++++++---- 2 files changed, 131 insertions(+), 16 deletions(-) diff --git a/report/parse_logs.py b/report/parse_logs.py index 10b302155..713b26e77 100644 --- a/report/parse_logs.py +++ b/report/parse_logs.py @@ -35,6 +35,78 @@ class LogsParser: def __init__(self, logs: list[LogPart]): self._logs = logs + def _extract_bash_commands(self, content: str) -> list[str]: + """Extract and parse bash commands from content.""" + commands = [] + lines = content.split('\n') + + for i, line in enumerate(lines): + line = line.strip() + if line == '': + # Look for the next closing tag + for j in range(i + 1, len(lines)): + if lines[j].strip() == '': + # Extract bash content between tags + bash_content = '\n'.join(lines[i+1:j]).strip() + if bash_content: + # Parse the first line as the main command + first_line = bash_content.split('\n')[0].strip() + if first_line: + # Skip comments and placeholder text + if (first_line.startswith('#') or + first_line.startswith('[The command') or + first_line.startswith('No bash') or + 'No bash' in first_line or + len(first_line) < 3): + continue + + # Extract command and key arguments + parts = first_line.split() + if parts: + cmd = parts[0] + + # Special handling for grep commands + if cmd == 'grep': + # Extract the search term (usually the first quoted argument) + import re + quoted_match = re.search(r"'([^']+)'", first_line) + if quoted_match: + search_term = quoted_match.group(1) + command_summary = f"grep '{search_term}'" + else: + # Fallback to regular parsing + key_args = [] + for part in parts[1:]: + if not part.startswith('-') and len(part) > 1: + if len(part) > 20: + part = part[:17] + '...' + key_args.append(part) + if len(key_args) >= 1: # Limit to 1 arg for grep + break + command_summary = f"{cmd} {' '.join(key_args)}".strip() + else: + # Regular command parsing + key_args = [] + for part in parts[1:]: + if not part.startswith('-') and len(part) > 1: + if len(part) > 20: + part = part[:17] + '...' + key_args.append(part) + if len(key_args) >= 2: # Limit to 2 key args + break + + command_summary = f"{cmd} {' '.join(key_args)}".strip() + + if len(command_summary) > 40: + command_summary = command_summary[:37] + '...' + + # Only add if it's not already in the list + if command_summary not in commands: + commands.append(command_summary) + break + + return commands + def _extract_tool_names(self, content: str) -> list[str]: """Extract tool names from content.""" tool_counts = {} @@ -52,10 +124,7 @@ def _extract_tool_names(self, content: str) -> list[str]: tool_names = [] for tool_name, count in tool_counts.items(): - if count == 1: - tool_names.append(tool_name) - else: - tool_names.append(f"{tool_name} (x{count})") + tool_names.append(tool_name) return tool_names @@ -106,10 +175,24 @@ def _parse_steps_from_logs(self, agent_logs: list[LogPart]) -> list[dict]: for step_num, step_data in steps_dict.items(): if step_data['log_parts']: - all_content = '\n'.join([part.content for part in step_data['log_parts']]) + # For the first step, exclude the first chat prompt (instruction prompt) + if step_num == '1' and len(step_data['log_parts']) > 1: + # Skip the first log part if it's a chat prompt + first_part = step_data['log_parts'][0] + if first_part.chat_prompt: + content_parts = step_data['log_parts'][1:] + else: + content_parts = step_data['log_parts'] + else: + content_parts = step_data['log_parts'] + + all_content = '\n'.join([part.content for part in content_parts]) tool_names = self._extract_tool_names(all_content) + bash_commands = self._extract_bash_commands(all_content) if tool_names: step_data['name'] = f"{', '.join(tool_names)}" + if bash_commands: + step_data['bash_commands'] = bash_commands steps = [] for step_num in sorted(steps_dict.keys(), diff --git a/report/templates/sample/sample.html b/report/templates/sample/sample.html index f93f77b0d..e52b374d5 100644 --- a/report/templates/sample/sample.html +++ b/report/templates/sample/sample.html @@ -348,7 +348,21 @@

Build Script

{% set step = agent_data.steps[0] %} {% if step.number and step.name %}
-
Step {{ step.number }} - {{ step.name }}
+
Step {{ step.number }} - + {% if 'Stderr' in step.name %} + {{ step.name.replace('Stderr', 'Stderr') | safe }} + {% else %} + {{ step.name }} + {% endif %} +
+ {% if step.bash_commands %} +
+ Commands: + {% for cmd in step.bash_commands %} + {{ cmd }} + {% endfor %} +
+ {% endif %} {% if step.log_parts %} {% for log_part in step.log_parts %} 
{{ log_part.content }}
@@ -371,19 +385,37 @@

Build Script

{% for step in agent_data.steps %}
- 
{{ step.content }}
+ {% for log_part in step.log_parts %} + 
{{ log_part.content|syntax_highlight|safe }}
+ {% endfor %}
{% endfor %} {% else %} {% for log_part in agent_data.logs %} - 
{{ log_part.content }}
+ 
{{ log_part.content|syntax_highlight|safe }}
{% endfor %} {% endif %}
@@ -348,7 +350,7 @@

Build Script

{% set step = agent_data.steps[0] %} {% if step.number and step.name %}
-
Step {{ step.number }} - +
Step {{ step.number }} - {% if 'Stderr' in step.name %} {{ step.name.replace('Stderr', 'Stderr') | safe }} {% else %} @@ -365,19 +367,19 @@

Build Script

{% endif %} {% if step.log_parts %} {% for log_part in step.log_parts %} - 
{{ log_part.content }}
+ 
{{ log_part.content|syntax_highlight|safe }}
{% endfor %} {% else %} - 
{{ step.content }}
+ 
{{ step.content|syntax_highlight|safe }}
{% endif %}
{% else %} {% if step.log_parts %} {% for log_part in step.log_parts %} - 
{{ log_part.content }}
+ 
{{ log_part.content|syntax_highlight|safe }}
{% endfor %} {% else %} - 
{{ step.content }}
+ 
{{ step.content|syntax_highlight|safe }}
{% endif %} {% endif %} {% else %} @@ -389,7 +391,7 @@

Build Script

{% if step.number %} {% if step.name %} - Step {{ step.number }} - + Step {{ step.number }} - {% if 'Stderr' in step.name %} {{ step.name.replace('Stderr', 'Stderr') | safe }} {% else %} @@ -423,10 +425,10 @@

Build Script

{% if step.log_parts %} {% for log_part in step.log_parts %} - 
{{ log_part.content }}
+ 
{{ log_part.content|syntax_highlight|safe }}
{% endfor %} {% else %} - 
{{ step.content }}
+ 
{{ step.content|syntax_highlight|safe }}
{% endif %}
@@ -434,7 +436,7 @@

Build Script

{% endif %} {% else %} {% for log_part in agent_data.logs %} - 
{{ log_part.content }}
+ 
{{ log_part.content|syntax_highlight|safe }}
{% endfor %} {% endif %}
@@ -460,7 +462,7 @@

Build Script

{% for part in logs %} - 
{{ part.content }}
+ 
{{ part.content|syntax_highlight|safe }}
{% endfor %}
@@ -514,6 +516,7 @@

Build Script

+ + {% if unified_data %} @@ -909,16 +913,51 @@

}); - - - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + + + + \ No newline at end of file diff --git a/report/templates/index/index.html b/report/templates/index/index.html index 8c222333a..55b36f1ed 100644 --- a/report/templates/index/index.html +++ b/report/templates/index/index.html @@ -35,6 +35,7 @@ languageOpen: true, crashesFoundOpen: true, ofgMetricsOpen: true, + chartsOpen: true, }" class="space-y-2">
@@ -489,6 +490,27 @@
+
+
+ +
+
+
+
+
+
+
+
{% for log_part in step.log_parts %} - 
{{ log_part.content|syntax_highlight|safe }}
+
{{ log_part.content|syntax_highlight|safe }}
{% endfor %}
{% endfor %} {% else %} {% for log_part in agent_data.logs %} - 
{{ log_part.content|syntax_highlight|safe }}
+
{{ log_part.content|syntax_highlight(default_lang)|safe }}
{% endfor %} {% endif %} @@ -367,19 +367,19 @@

Build Script

{% endif %} {% if step.log_parts %} {% for log_part in step.log_parts %} - 
{{ log_part.content|syntax_highlight|safe }}
+
{{ log_part.content|syntax_highlight|safe }}
{% endfor %} {% else %} - 
{{ step.content|syntax_highlight|safe }}
+
{{ step.content|syntax_highlight|safe }}
{% endif %} {% else %} {% if step.log_parts %} {% for log_part in step.log_parts %} - 
{{ log_part.content|syntax_highlight|safe }}
+
{{ log_part.content|syntax_highlight|safe }}
{% endfor %} {% else %} - 
{{ step.content|syntax_highlight|safe }}
+
{{ step.content|syntax_highlight|safe }}
{% endif %} {% endif %} {% else %} @@ -425,10 +425,10 @@

Build Script

{% if step.log_parts %} {% for log_part in step.log_parts %} - 
{{ log_part.content|syntax_highlight|safe }}
+
{{ log_part.content|syntax_highlight(default_lang)|safe }}
{% endfor %} {% else %} - 
{{ step.content|syntax_highlight|safe }}
+
{{ step.content|syntax_highlight|safe }}
{% endif %}
@@ -436,7 +436,7 @@

Build Script

{% endif %} {% else %} {% for log_part in agent_data.logs %} - 
{{ log_part.content|syntax_highlight|safe }}
+
{{ log_part.content|syntax_highlight(default_lang)|safe }}
{% endfor %} {% endif %} @@ -462,7 +462,7 @@

Build Script

{% for part in logs %} - 
{{ part.content|syntax_highlight|safe }}
+
{{ part.content|syntax_highlight(default_lang)|safe }}
{% endfor %}
diff --git a/report/web.py b/report/web.py index aa052882a..bf729efc8 100644 --- a/report/web.py +++ b/report/web.py @@ -191,6 +191,25 @@ def _copy_and_set_coverage_report(self, benchmark, sample): sample.result.coverage_report_path = \ f'/sample/{benchmark.id}/coverage/{sample.id}/linux/' + def _copy_plot_library(self): + """Copies the Plot.js library to the output directory.""" + d3_js_path = os.path.join(self._jinja.get_template_search_path()[0] + '/../trends_report_web/', + 'd3.min.js') + plot_js_path = os.path.join(self._jinja.get_template_search_path()[0] + '/../trends_report_web/', + 'plot.min.js') + if os.path.exists(d3_js_path): + os.makedirs(self._output_dir, exist_ok=True) + shutil.copy(d3_js_path, os.path.join(self._output_dir, 'd3.min.js')) + logging.info('Copied d3.min.js to %s', os.path.join(self._output_dir, 'd3.min.js')) + else: + logging.warning('d3.min.js not found at %s', d3_js_path) + if os.path.exists(plot_js_path): + os.makedirs(self._output_dir, exist_ok=True) + shutil.copy(plot_js_path, os.path.join(self._output_dir, 'plot.min.js')) + logging.info('Copied plot.min.js to %s', os.path.join(self._output_dir, 'plot.min.js')) + else: + logging.warning('Plot.js not found at %s', plot_js_path) + def _read_static_file(self, file_path_in_templates_subdir: str) -> str: """Reads a static file from the templates directory.""" @@ -218,6 +237,8 @@ def _read_static_file(self, file_path_in_templates_subdir: str) -> str: def generate(self): """Generate and write every report file.""" + self._copy_plot_library() + benchmarks = [] samples_with_bugs = [] # First pass: collect benchmarks and samples @@ -377,6 +398,7 @@ def _write_benchmark_sample(self, benchmark: Benchmark, sample: Sample, agent_cycles=agent_cycles, logs=logs, logs_parser=logs_parser, + default_lang=(benchmark.language.lower() if getattr(benchmark, 'language', '') else ''), triage=triage, targets=sample_targets, sample_css_content=sample_css_content, From 812b6c8fb3990700362b49e311a008c05ba377a0 Mon Sep 17 00:00:00 2001 From: "Myan V." Date: Thu, 14 Aug 2025 23:06:42 +1200 Subject: [PATCH 6/7] Add basic charts on the index page --- report/parse_logs.py | 85 +++-- report/templates/index/index.html | 5 +- report/templates/index/index.js | 584 +++++++++++++++++------------- report/web.py | 66 ++-- 4 files changed, 419 insertions(+), 321 deletions(-) diff --git a/report/parse_logs.py b/report/parse_logs.py index d45725095..ba0f55c8f 100644 --- a/report/parse_logs.py +++ b/report/parse_logs.py @@ -210,7 +210,9 @@ def _parse_steps_by_chat_pairs(self, agent_logs: list[LogPart]) -> list[dict]: return steps - def _syntax_highlight_content(self, content: str, default_language: str = "") -> str: + def _syntax_highlight_content(self, + content: str, + default_language: str = "") -> str: """Syntax highlights content while preserving visible tags.""" # Escape everything first so raw logs are safe to render in HTML @@ -240,45 +242,48 @@ def _normalize_lang(lang: str) -> str: lang_key = _normalize_lang(default_language) - escaped = _sub(r'<conclusion>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</conclusion>', - r'<conclusion>' - r'
\1
' - r'</conclusion>', escaped) - escaped = _sub(r'<reason>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</reason>', - r'<reason>' - r'
\1
' - r'</reason>', escaped) - - escaped = _sub(r'<bash>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</bash>', - r'<bash>' - r'
\1
' - r'</bash>', - escaped) - escaped = _sub(r'<build_script>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</build_script>', - r'<build_script>' - r'
\1
' - r'</build_script>', - escaped) - escaped = _sub(r'<fuzz target>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</fuzz target>', - rf'<fuzz target>' - rf'
\1
' - rf'</fuzz target>', - escaped) - - escaped = _sub(r'<stdout>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</stdout>', - r'<stdout>' - r'
\1
' - r'</stdout>', - escaped) - escaped = _sub(r'<stderr>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</stderr>', - r'<stderr>' - r'
\1
' - r'</stderr>', - escaped) - escaped = _sub(r'<return_code>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</return_code>', - r'<return_code>' - r'
\1
' - r'</return_code>', escaped) + escaped = _sub( + r'<conclusion>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</conclusion>', + r'<conclusion>' + r'
\1
' + r'</conclusion>', escaped) + escaped = _sub( + r'<reason>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</reason>', + r'<reason>' + r'
\1
' + r'</reason>', escaped) + + escaped = _sub( + r'<bash>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</bash>', + r'<bash>' + r'
\1
' + r'</bash>', escaped) + escaped = _sub( + r'<build_script>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</build_script>', + r'<build_script>' + r'
\1
' + r'</build_script>', escaped) + escaped = _sub( + r'<fuzz target>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</fuzz target>', + rf'<fuzz target>' + rf'
\1
' + rf'</fuzz target>', escaped) + + escaped = _sub( + r'<stdout>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</stdout>', + r'<stdout>' + r'
\1
' + r'</stdout>', escaped) + escaped = _sub( + r'<stderr>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</stderr>', + r'<stderr>' + r'
\1
' + r'</stderr>', escaped) + escaped = _sub( + r'<return_code>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</return_code>', + r'<return_code>' + r'
\1
' + r'</return_code>', escaped) return escaped diff --git a/report/templates/index/index.html b/report/templates/index/index.html index 55b36f1ed..079ce8803 100644 --- a/report/templates/index/index.html +++ b/report/templates/index/index.html @@ -506,7 +506,10 @@
-
+
+
+
+
diff --git a/report/templates/index/index.js b/report/templates/index/index.js index 99cacef5b..6a431976f 100644 --- a/report/templates/index/index.js +++ b/report/templates/index/index.js @@ -1,277 +1,363 @@ document.addEventListener('DOMContentLoaded', function() { - function waitForPlot() { - if (typeof Plot !== 'undefined') { - setTimeout(initializeCharts, 100); - } else { - setTimeout(waitForPlot, 100); - } - } + function waitForPlot() { + if (typeof Plot !== 'undefined') { + setTimeout(initializeCharts, 100); + } else { + setTimeout(waitForPlot, 100); + } + } - function getBarY() { - if (Plot.barY) return Plot.barY; - if (Plot.BarY) return Plot.BarY; - if (Plot.rectY) { - return (data, opts) => { - const { x, y, fill, title } = opts || {}; - return Plot.rectY(data, { x, y2: y, y1: 0, fill, title }); - }; - } - return null; - } + function getBarY() { + if (Plot.barY) return Plot.barY; + if (Plot.BarY) return Plot.BarY; + if (Plot.rectY) { + return (data, opts) => { + const { x, y, fill, title } = opts || {}; + return Plot.rectY(data, { x, y2: y, y1: 0, fill, title }); + }; + } + return null; + } - function initializeCharts() { - const BarY = getBarY(); - if (!BarY) return; + function readUnifiedData() { + const el = document.getElementById('unified-data'); + if (!el) return null; + try { return JSON.parse(el.textContent); } catch (_) { return null; } + } - const projectData = Array.from(document.querySelectorAll('#project-summary-table tbody tr.project-data-row')).map(row => { - const cells = row.querySelectorAll('td'); - if (cells.length >= 9) { - return { - project: cells[1].dataset.sortValue, - new_lines: parseInt(cells[7].dataset.sortValue) || 0, - existing_lines: parseInt(cells[8].dataset.sortValue) || 0 - }; - } - return null; - }).filter(Boolean); + function containerSize(el, fallbackW = 800, fallbackH = 300) { + if (!el) return { width: fallbackW, height: fallbackH }; + const rect = el.getBoundingClientRect(); + const width = Math.max(300, Math.floor(rect.width - 20)); + const height = Math.max(220, Math.floor(rect.height - 20)); + return { width, height }; + } - if (projectData.length > 0) { - try { - const coveragePlot = Plot.plot({ - title: "New vs. Existing Code Coverage by Project", - x: { label: "Project", domain: projectData.map(d => d.project) }, - y: { label: "Lines of Code" }, - marks: [ - BarY(projectData, { x: "project", y: "existing_lines", fill: "#94a3b8", title: "Existing Coverage" }), - BarY(projectData, { x: "project", y: "new_lines", fill: "#3b82f6", title: "New Coverage" }) - ], - width: 800, - height: 400 - }); - const el = document.getElementById('coverage-chart'); - if (el) { el.innerHTML = ''; el.appendChild(coveragePlot); } - } catch (error) { - const el = document.getElementById('coverage-chart'); - if (el) el.innerHTML = '

' + error.message + '

'; - } - } + function appendTitle(el, text) { + const title = document.createElement('div'); + title.textContent = text; + title.style.fontWeight = '600'; + title.style.marginBottom = '8px'; + el.appendChild(title); + } - const bugsData = {}; - projectData.forEach(project => { bugsData[project.project] = 0; }); - const bugRows = document.querySelectorAll('#project-summary-table tbody tr.project-data-row'); - bugRows.forEach(row => { - const cells = row.querySelectorAll('td'); - if (cells.length >= 6) { - const project = cells[1].dataset.sortValue; - const bugs = parseInt(cells[5].dataset.sortValue) || 0; - if (project in bugsData) bugsData[project] += bugs; - } - }); + function renderLanguagePie(langData) { + const el = document.getElementById('language-coverage-chart'); + if (!el || typeof d3 === 'undefined') return; + el.innerHTML = ''; + appendTitle(el, 'Language Coverage (Experiment new lines)'); + const { width, height } = containerSize(el); + const reserved = 60; + const svgHeight = Math.max(220, height - reserved); + const legend = d3.select(el).append('div').style('display','flex').style('flexWrap','wrap').style('gap','10px').style('justifyContent','center').style('marginBottom','8px'); + const color = (d3.schemeTableau10 || d3.schemeCategory10 || []).length ? d3.scaleOrdinal((d3.schemeTableau10 || d3.schemeCategory10)) : d3.scaleOrdinal().range(['#3b82f6','#22c55e','#ef4444','#f59e0b','#8b5cf6','#06b6d4','#84cc16','#e11d48','#64748b','#a855f7']); + color.domain(langData.map(d=>d.language)); + langData.forEach(d => { + const item = legend.append('div').style('display','flex').style('alignItems','center').style('gap','6px'); + item.append('span').style('display','inline-block').style('width','12px').style('height','12px').style('background', color(d.language)); + item.append('span').text(`${d.language}: ${d.experiment_new}`); + }); + const values = langData.map(d => d.experiment_new || 0); + const sum = values.reduce((a,b)=>a+b,0); + if (sum <= 0) { el.innerHTML = '

No language coverage data

'; return; } + const radius = Math.min(width, svgHeight) / 2 - 8; + const svg = d3.select(el).append('svg').attr('width', width).attr('height', svgHeight) + .append('g').attr('transform', `translate(${width/2},${svgHeight/2})`); + const pie = d3.pie().sort(null).value(d => d.experiment_new)(langData); + const arc = d3.arc().outerRadius(radius).innerRadius(radius*0.5); + svg.selectAll('path').data(pie).enter().append('path') + .attr('d', arc) + .attr('fill', d => color(d.data.language)) + .append('title').text(d => `${d.data.language}: ${d.data.experiment_new}`); + } - const bugsChartData = Object.entries(bugsData).map(([project, bugs]) => ({ project, bugs })); - if (bugsChartData.length > 0) { - try { - const bugsPlot = Plot.plot({ - title: "Bugs Found by Project", - x: { label: "Project", domain: bugsChartData.map(d => d.project) }, - y: { label: "Number of Bugs" }, - marks: [ - BarY(bugsChartData, { x: "project", y: "bugs", fill: "#ef4444", title: "Bugs Found" }) - ], - width: 800, - height: 400 - }); - const el = document.getElementById('bugs-chart'); - if (el) { el.innerHTML = ''; el.appendChild(bugsPlot); } - } catch (error) { - const el = document.getElementById('bugs-chart'); - if (el) el.innerHTML = '

' + error.message + '

'; - } - } - } + function initializeCharts() { + const BarY = getBarY(); + if (!BarY) return; - waitForPlot(); + const projectData = Array.from(document.querySelectorAll('#project-summary-table tbody tr.project-data-row')).map(row => { + const cells = row.querySelectorAll('td'); + if (cells.length >= 9) { + return { + project: cells[1].dataset.sortValue, + new_lines: parseInt(cells[7].dataset.sortValue) || 0, + existing_lines: parseInt(cells[8].dataset.sortValue) || 0 + }; + } + return null; + }).filter(Boolean); - // Project summary table expand/collapse buttons - const projectSummaryExpandAllButton = document.getElementById('project-summary-expand-all'); - if (projectSummaryExpandAllButton) { - projectSummaryExpandAllButton.addEventListener('click', () => { - document.querySelectorAll('[x-ref^="benchmarks_"]').forEach(el => { - el.classList.remove('hidden'); - }); - }); - } + const coverageEl = document.getElementById('coverage-chart'); + if (projectData.length > 0 && coverageEl) { + try { + coverageEl.innerHTML = ''; + appendTitle(coverageEl, 'New vs Existing Code Coverage by Project'); + const legendDiv = document.createElement('div'); + legendDiv.style.display = 'flex'; + legendDiv.style.gap = '16px'; + legendDiv.style.alignItems = 'center'; + legendDiv.style.fontSize = '14px'; + legendDiv.style.marginBottom = '6px'; + legendDiv.innerHTML = 'Existing CoverageNew Coverage'; + coverageEl.appendChild(legendDiv); + const { width, height } = containerSize(coverageEl); + const plot = Plot.plot({ + title: null, + x: { label: 'Project', domain: projectData.map(d => d.project) }, + y: { label: 'Lines of Code' }, + marks: [ + BarY(projectData, { x: 'project', y: 'existing_lines', fill: '#94a3b8', title: 'Existing Coverage' }), + BarY(projectData, { x: 'project', y: 'new_lines', fill: '#3b82f6', title: 'New Coverage' }) + ], + width, + height: Math.max(240, height - 56) + }); + coverageEl.appendChild(plot); + } catch (error) { + coverageEl.innerHTML = '

' + error.message + '

'; + } + } - const projectSummaryCollapseAllButton = document.getElementById('project-summary-collapse-all'); - if (projectSummaryCollapseAllButton) { - projectSummaryCollapseAllButton.addEventListener('click', () => { - document.querySelectorAll('[x-ref^="benchmarks_"]').forEach(el => { - el.classList.add('hidden'); - }); - }); - } + const langRows = document.querySelectorAll('#language-coverage-gain tbody tr'); + const langData = Array.from(langRows).map(row => { + const cells = row.querySelectorAll('td'); + if (cells.length >= 6) { + return { + language: cells[0].dataset.sortValue, + ossfuzz_covered: parseInt(cells[2].dataset.sortValue) || 0, + experiment_new: parseInt(cells[3].dataset.sortValue) || 0 + }; + } + return null; + }).filter(Boolean); + if (langData.length > 0) { + try { renderLanguagePie(langData); } catch (_) {} + } - const crashesExpandAllButton = document.getElementById('crashes-expand-all'); - if (crashesExpandAllButton) { - crashesExpandAllButton.addEventListener('click', () => { - document.querySelectorAll('[x-ref^="project_"]').forEach(el => { - el.classList.remove('hidden'); - }); - document.querySelectorAll('[x-ref^="samples_"]').forEach(el => { - el.classList.remove('hidden'); - }); - }); - } + const unified = readUnifiedData(); + if (unified) { + const crashReasons = {}; + for (const projectName in unified) { + const project = unified[projectName]; + if (project.benchmarks) { + for (const benchId in project.benchmarks) { + const bench = project.benchmarks[benchId]; + if (bench.samples) { + bench.samples.forEach(s => { + const reason = (s.crash_reason || '').trim() || 'N/A'; + crashReasons[reason] = (crashReasons[reason] || 0) + (s.crashes ? 1 : 0); + }); + } + } + } + } + const crEl = document.getElementById('crash-reasons-chart'); + const crashReasonData = Object.entries(crashReasons).map(([reason, count]) => ({ reason, count })).sort((a,b) => b.count - a.count); + if (crashReasonData.length > 0 && crEl) { + try { + crEl.innerHTML = ''; + appendTitle(crEl, 'Crash Reasons'); + const { width, height } = containerSize(crEl); + const crPlot = Plot.plot({ + title: null, + x: { label: 'Reason', domain: crashReasonData.map(d => d.reason) }, + y: { label: 'Count' }, + marks: [ + BarY(crashReasonData, { x: 'reason', y: 'count', fill: '#f59e0b' }) + ], + width, + height: Math.max(240, height - 28) + }); + crEl.appendChild(crPlot); + } catch (error) { + crEl.innerHTML = '

' + error.message + '

'; + } + } + } + } - const crashesCollapseAllButton = document.getElementById('crashes-collapse-all'); - if (crashesCollapseAllButton) { - crashesCollapseAllButton.addEventListener('click', () => { - document.querySelectorAll('[x-ref^="project_"]').forEach(el => { - el.classList.add('hidden'); - }); - document.querySelectorAll('[x-ref^="samples_"]').forEach(el => { - el.classList.add('hidden'); - }); - }); - } + waitForPlot(); - // Project-level expand/collapse buttons - document.querySelectorAll('[id^="project-expand-all-"]').forEach(button => { - button.addEventListener('click', () => { - const projectIndex = button.id.split('-').pop(); - document.querySelectorAll(`[x-ref^="samples_"][x-ref$="_${projectIndex}"]`).forEach(el => { - el.classList.remove('hidden'); - }); - }); - }); + // Project summary table expand/collapse buttons + const projectSummaryExpandAllButton = document.getElementById('project-summary-expand-all'); + if (projectSummaryExpandAllButton) { + projectSummaryExpandAllButton.addEventListener('click', () => { + document.querySelectorAll('[x-ref^="benchmarks_"]').forEach(el => { + el.classList.remove('hidden'); + }); + }); + } - document.querySelectorAll('[id^="project-collapse-all-"]').forEach(button => { - button.addEventListener('click', () => { - const projectIndex = button.id.split('-').pop(); - document.querySelectorAll(`[x-ref^="samples_"][x-ref$="_${projectIndex}"]`).forEach(el => { - el.classList.add('hidden'); - }); - }); - }); + const projectSummaryCollapseAllButton = document.getElementById('project-summary-collapse-all'); + if (projectSummaryCollapseAllButton) { + projectSummaryCollapseAllButton.addEventListener('click', () => { + document.querySelectorAll('[x-ref^="benchmarks_"]').forEach(el => { + el.classList.add('hidden'); + }); + }); + } - function compareTableCells(cellA, cellB, sortNumber, sortAsc) { - if (!cellA || !cellB) return 0; + const crashesExpandAllButton = document.getElementById('crashes-expand-all'); + if (crashesExpandAllButton) { + crashesExpandAllButton.addEventListener('click', () => { + document.querySelectorAll('[x-ref^="project_"]').forEach(el => { + el.classList.remove('hidden'); + }); + document.querySelectorAll('[x-ref^="samples_"]').forEach(el => { + el.classList.remove('hidden'); + }); + }); + } - let valueA_str = cellA.dataset.sortValue; - let valueB_str = cellB.dataset.sortValue; - let comparison = 0; + const crashesCollapseAllButton = document.getElementById('crashes-collapse-all'); + if (crashesCollapseAllButton) { + crashesCollapseAllButton.addEventListener('click', () => { + document.querySelectorAll('[x-ref^="project_"]').forEach(el => { + el.classList.add('hidden'); + }); + document.querySelectorAll('[x-ref^="samples_"]').forEach(el => { + el.classList.add('hidden'); + }); + }); + } - if (sortNumber) { - let numA = parseFloat(valueA_str); - let numB = parseFloat(valueB_str); + // Project-level expand/collapse buttons + document.querySelectorAll('[id^="project-expand-all-"]').forEach(button => { + button.addEventListener('click', () => { + const projectIndex = button.id.split('-').pop(); + document.querySelectorAll(`[x-ref^="samples_"][x-ref$="_${projectIndex}"]`).forEach(el => { + el.classList.remove('hidden'); + }); + }); + }); - if (isNaN(numA) && isNaN(numB)) { - comparison = 0; - } else if (isNaN(numA)) { - comparison = 1; - } else if (isNaN(numB)) { - comparison = -1; - } else { - comparison = numA - numB; - } - } else { - const strA = (valueA_str === undefined || valueA_str === null) ? "" : String(valueA_str); - const strB = (valueB_str === undefined || valueB_str === null) ? "" : String(valueB_str); - comparison = strA.localeCompare(strB); - } - return sortAsc ? comparison : -comparison; - } + document.querySelectorAll('[id^="project-collapse-all-"]').forEach(button => { + button.addEventListener('click', () => { + const projectIndex = button.id.split('-').pop(); + document.querySelectorAll(`[x-ref^="samples_"][x-ref$="_${projectIndex}"]`).forEach(el => { + el.classList.add('hidden'); + }); + }); + }); - const tables = Array.from(document.querySelectorAll('table.sortable-table')); - tables.forEach(table_element => { - const headers = Array.from(table_element.querySelectorAll('th')); - headers.forEach((th, colindex) => { - if (th.innerText.trim() === '' && colindex === 0) { - return; - } + function compareTableCells(cellA, cellB, sortNumber, sortAsc) { + if (!cellA || !cellB) return 0; - th.addEventListener('click', () => { - const sortAsc = th.dataset.sorted !== "asc"; - const sortNumber = th.hasAttribute('data-sort-number'); + let valueA_str = cellA.dataset.sortValue; + let valueB_str = cellB.dataset.sortValue; + let comparison = 0; - const currentTableHeaders = Array.from(table_element.querySelectorAll('th')); - currentTableHeaders.forEach(innerTH => delete innerTH.dataset.sorted); - th.dataset.sorted = sortAsc ? "asc" : "desc"; + if (sortNumber) { + let numA = parseFloat(valueA_str); + let numB = parseFloat(valueB_str); - const tbody = table_element.querySelector('tbody'); - if (!tbody) return; + if (isNaN(numA) && isNaN(numB)) { + comparison = 0; + } else if (isNaN(numA)) { + comparison = 1; + } else if (isNaN(numB)) { + comparison = -1; + } else { + comparison = numA - numB; + } + } else { + const strA = (valueA_str === undefined || valueA_str === null) ? "" : String(valueA_str); + const strB = (valueB_str === undefined || valueB_str === null) ? "" : String(valueB_str); + comparison = strA.localeCompare(strB); + } + return sortAsc ? comparison : -comparison; + } - let allRowsInBody = Array.from(tbody.children); - let sortableUnits = []; - let appendedRows = []; + const tables = Array.from(document.querySelectorAll('table.sortable-table')); + tables.forEach(table_element => { + const headers = Array.from(table_element.querySelectorAll('th')); + headers.forEach((th, colindex) => { + if (th.innerText.trim() === '' && colindex === 0) { + return; + } - if (table_element.id === 'project-summary-table') { - for (let i = 0; i < allRowsInBody.length; i += 2) { - if (allRowsInBody[i] && allRowsInBody[i+1] && - allRowsInBody[i].classList.contains('project-data-row') && - allRowsInBody[i+1].classList.contains('project-benchmarks-container-row')) { - sortableUnits.push({ - representativeRow: allRowsInBody[i], - actualRows: [allRowsInBody[i], allRowsInBody[i+1]] - }); - } else { - appendedRows.push(...allRowsInBody.slice(i)); - break; - } - } - } else if (table_element.id === 'crashes-table') { - for (let i = 0; i < allRowsInBody.length; i += 2) { - if (allRowsInBody[i] && allRowsInBody[i+1]) { - sortableUnits.push({ - representativeRow: allRowsInBody[i], - actualRows: [allRowsInBody[i], allRowsInBody[i+1]] - }); - } - } - } else if (table_element.closest('[x-ref^="project_"]')) { - for (let i = 0; i < allRowsInBody.length; i += 2) { - if (allRowsInBody[i] && allRowsInBody[i+1]) { - sortableUnits.push({ - representativeRow: allRowsInBody[i], - actualRows: [allRowsInBody[i], allRowsInBody[i+1]] - }); - } - } - } else { - if (table_element.id && table_element.id.startsWith('benchmarks-table-')) { - const averageRowIndex = allRowsInBody.findIndex(row => row.cells.length > 0 && row.cells[0].innerText.trim() === 'Average'); - if (averageRowIndex !== -1) { - appendedRows.push(allRowsInBody.splice(averageRowIndex, 1)[0]); - } - } - allRowsInBody.forEach(row => { - sortableUnits.push({ representativeRow: row, actualRows: [row] }); - }); - } + th.addEventListener('click', () => { + const sortAsc = th.dataset.sorted !== "asc"; + const sortNumber = th.hasAttribute('data-sort-number'); - sortableUnits.sort((unitA, unitB) => { - const cellA = unitA.representativeRow.children[colindex]; - const cellB = unitB.representativeRow.children[colindex]; - return compareTableCells(cellA, cellB, sortNumber, sortAsc); - }); + const currentTableHeaders = Array.from(table_element.querySelectorAll('th')); + currentTableHeaders.forEach(innerTH => delete innerTH.dataset.sorted); + th.dataset.sorted = sortAsc ? "asc" : "desc"; - tbody.innerHTML = ''; - sortableUnits.forEach(unit => { - unit.actualRows.forEach(row => tbody.appendChild(row)); - }); - appendedRows.forEach(row => tbody.appendChild(row)); + const tbody = table_element.querySelector('tbody'); + if (!tbody) return; - let visualIndex = 1; - Array.from(tbody.children).forEach(r => { - if (appendedRows.includes(r)) { - return; - } - const firstCell = r.children[0]; - if (firstCell && firstCell.classList.contains('table-index') && !firstCell.querySelector('button')) { - firstCell.innerText = visualIndex++; - } - }); - }); - }); - }); + let allRowsInBody = Array.from(tbody.children); + let sortableUnits = []; + let appendedRows = []; + + if (table_element.id === 'project-summary-table') { + for (let i = 0; i < allRowsInBody.length; i += 2) { + if (allRowsInBody[i] && allRowsInBody[i+1] && + allRowsInBody[i].classList.contains('project-data-row') && + allRowsInBody[i+1].classList.contains('project-benchmarks-container-row')) { + sortableUnits.push({ + representativeRow: allRowsInBody[i], + actualRows: [allRowsInBody[i], allRowsInBody[i+1]] + }); + } else { + appendedRows.push(...allRowsInBody.slice(i)); + break; + } + } + } else if (table_element.id === 'crashes-table') { + for (let i = 0; i < allRowsInBody.length; i += 2) { + if (allRowsInBody[i] && allRowsInBody[i+1]) { + sortableUnits.push({ + representativeRow: allRowsInBody[i], + actualRows: [allRowsInBody[i], allRowsInBody[i+1]] + }); + } + } + } else if (table_element.closest('[x-ref^="project_"]')) { + for (let i = 0; i < allRowsInBody.length; i += 2) { + if (allRowsInBody[i] && allRowsInBody[i+1]) { + sortableUnits.push({ + representativeRow: allRowsInBody[i], + actualRows: [allRowsInBody[i], allRowsInBody[i+1]] + }); + } + } + } else { + if (table_element.id && table_element.id.startsWith('benchmarks-table-')) { + const averageRowIndex = allRowsInBody.findIndex(row => row.cells.length > 0 && row.cells[0].innerText.trim() === 'Average'); + if (averageRowIndex !== -1) { + appendedRows.push(allRowsInBody.splice(averageRowIndex, 1)[0]); + } + } + allRowsInBody.forEach(row => { + sortableUnits.push({ representativeRow: row, actualRows: [row] }); + }); + } + + sortableUnits.sort((unitA, unitB) => { + const cellA = unitA.representativeRow.children[colindex]; + const cellB = unitB.representativeRow.children[colindex]; + return compareTableCells(cellA, cellB, sortNumber, sortAsc); + }); + + tbody.innerHTML = ''; + sortableUnits.forEach(unit => { + unit.actualRows.forEach(row => tbody.appendChild(row)); + }); + appendedRows.forEach(row => tbody.appendChild(row)); + + let visualIndex = 1; + Array.from(tbody.children).forEach(r => { + if (appendedRows.includes(r)) { + return; + } + const firstCell = r.children[0]; + if (firstCell && firstCell.classList.contains('table-index') && !firstCell.querySelector('button')) { + firstCell.innerText = visualIndex++; + } + }); + }); + }); + }); }); diff --git a/report/web.py b/report/web.py index bf729efc8..ec5ee3881 100644 --- a/report/web.py +++ b/report/web.py @@ -193,22 +193,24 @@ def _copy_and_set_coverage_report(self, benchmark, sample): def _copy_plot_library(self): """Copies the Plot.js library to the output directory.""" - d3_js_path = os.path.join(self._jinja.get_template_search_path()[0] + '/../trends_report_web/', - 'd3.min.js') - plot_js_path = os.path.join(self._jinja.get_template_search_path()[0] + '/../trends_report_web/', - 'plot.min.js') - if os.path.exists(d3_js_path): - os.makedirs(self._output_dir, exist_ok=True) - shutil.copy(d3_js_path, os.path.join(self._output_dir, 'd3.min.js')) - logging.info('Copied d3.min.js to %s', os.path.join(self._output_dir, 'd3.min.js')) - else: - logging.warning('d3.min.js not found at %s', d3_js_path) - if os.path.exists(plot_js_path): - os.makedirs(self._output_dir, exist_ok=True) - shutil.copy(plot_js_path, os.path.join(self._output_dir, 'plot.min.js')) - logging.info('Copied plot.min.js to %s', os.path.join(self._output_dir, 'plot.min.js')) - else: - logging.warning('Plot.js not found at %s', plot_js_path) + search_path = self._jinja.get_template_search_path() + templates_dir = search_path[0] if search_path else 'report/templates' + libs_dir = os.path.abspath( + os.path.join(templates_dir, '..', 'trends_report_web')) + + os.makedirs(self._output_dir, exist_ok=True) + + for lib_name in ['plot.min.js', 'd3.min.js']: + lib_src = os.path.join(libs_dir, lib_name) + lib_dst = os.path.join(self._output_dir, lib_name) + if os.path.exists(lib_src): + try: + shutil.copy(lib_src, lib_dst) + logging.info('Copied %s to %s', lib_name, lib_dst) + except Exception as e: + logging.warning('Failed to copy %s: %s', lib_name, e) + else: + logging.warning('%s not found at %s', lib_name, lib_src) def _read_static_file(self, file_path_in_templates_subdir: str) -> str: """Reads a static file from the templates directory.""" @@ -238,7 +240,7 @@ def _read_static_file(self, file_path_in_templates_subdir: str) -> str: def generate(self): """Generate and write every report file.""" self._copy_plot_library() - + benchmarks = [] samples_with_bugs = [] # First pass: collect benchmarks and samples @@ -391,20 +393,22 @@ def _write_benchmark_sample(self, benchmark: Benchmark, sample: Sample, agent_sections = logs_parser.get_agent_sections() agent_cycles = logs_parser.get_agent_cycles() - rendered = self._jinja.render('sample/sample.html', - benchmark=benchmark, - sample=sample, - agent_sections=agent_sections, - agent_cycles=agent_cycles, - logs=logs, - logs_parser=logs_parser, - default_lang=(benchmark.language.lower() if getattr(benchmark, 'language', '') else ''), - triage=triage, - targets=sample_targets, - sample_css_content=sample_css_content, - sample_js_content=sample_js_content, - crash_info=crash_info, - **common_data) + rendered = self._jinja.render( + 'sample/sample.html', + benchmark=benchmark, + sample=sample, + agent_sections=agent_sections, + agent_cycles=agent_cycles, + logs=logs, + logs_parser=logs_parser, + default_lang=(benchmark.language.lower() if getattr( + benchmark, 'language', '') else ''), + triage=triage, + targets=sample_targets, + sample_css_content=sample_css_content, + sample_js_content=sample_js_content, + crash_info=crash_info, + **common_data) self._write(f'sample/{benchmark.id}/{sample.id}.html', rendered) except Exception as e: From c49c9e8cccdc5438dcfeea85bae4eb7f52429d40 Mon Sep 17 00:00:00 2001 From: "Myan V." Date: Fri, 29 Aug 2025 19:27:01 +1200 Subject: [PATCH 7/7] Add syntax highlighting inside execution stages --- report/parse_logs.py | 326 +++++++++++++++++++++------- report/templates/base.html | 31 ++- report/templates/sample/sample.html | 6 +- report/web.py | 12 +- 4 files changed, 276 insertions(+), 99 deletions(-) diff --git a/report/parse_logs.py b/report/parse_logs.py index ba0f55c8f..16e0c52b7 100644 --- a/report/parse_logs.py +++ b/report/parse_logs.py @@ -44,77 +44,91 @@ def _extract_bash_commands(self, content: str) -> list[str]: for i, line in enumerate(lines): line = line.strip() if line == '': - for j in range(i + 1, len(lines)): - if lines[j].strip() == '': - bash_content = '\n'.join(lines[i + 1:j]).strip() - if bash_content: - first_line = bash_content.split('\n')[0].strip() - if first_line: - # skip comments and placeholder text - if (first_line.startswith('#') or - first_line.startswith('[The command') or - first_line.startswith('No bash') or - 'No bash' in first_line or len(first_line) < 3): - continue - - parts = first_line.split() - if parts: - cmd = parts[0] - - if cmd == 'grep': - # Extract the search term (usually the first quoted argument) - import re - quoted_match = re.search(r"'([^']+)'", first_line) - if quoted_match: - search_term = quoted_match.group(1) - command_summary = f"grep '{search_term}'" - else: - key_args = [] - for part in parts[1:]: - if not part.startswith('-') and len(part) > 1: - if len(part) > 20: - part = part[:17] + '...' - key_args.append(part) - if len(key_args) >= 1: # Limit to 1 arg for grep - break - command_summary = f"{cmd} {' '.join(key_args)}".strip() - else: - key_args = [] - for part in parts[1:]: - if not part.startswith('-') and len(part) > 1: - if len(part) > 20: - part = part[:17] + '...' - key_args.append(part) - if len(key_args) >= 2: # Limit to 2 key args - break - - command_summary = f"{cmd} {' '.join(key_args)}".strip() - - if len(command_summary) > 40: - command_summary = command_summary[:37] + '...' - - if command_summary not in commands: - commands.append(command_summary) - break + command = self._process_bash_block(lines, i) + if command and command not in commands: + commands.append(command) return commands + def _process_bash_block(self, lines: list[str], start_idx: int) -> str: + """Process a single bash block and extract command summary.""" + for j in range(start_idx + 1, len(lines)): + if lines[j].strip() == '': + bash_content = '\n'.join(lines[start_idx + 1:j]).strip() + if bash_content: + return self._extract_command_from_content(bash_content) + break + return "" + + def _extract_command_from_content(self, bash_content: str) -> str: + """Extract command summary from bash content.""" + first_line = bash_content.split('\n', 1)[0].strip() + if not first_line: + return "" + + # Skip comments and placeholder text + if (first_line.startswith('#') or first_line.startswith('[The command') or + first_line.startswith('No bash') or 'No bash' in first_line or + len(first_line) < 3): + return "" + + parts = first_line.split() + if not parts: + return "" + + cmd = parts[0] + command_summary = self._build_command_summary(cmd, parts, first_line) + + if len(command_summary) > 40: + command_summary = command_summary[:37] + '...' + + return command_summary + + def _build_command_summary(self, cmd: str, parts: list[str], + first_line: str) -> str: + """Build command summary based on command type.""" + if cmd == 'grep': + quoted_match = re.search(r"'([^']+)'", first_line) + if quoted_match: + search_term = quoted_match.group(1) + return f"grep '{search_term}'" + return self._extract_key_args(cmd, parts[1:], 1) + return self._extract_key_args(cmd, parts[1:], 2) + + def _extract_key_args(self, cmd: str, parts: list[str], max_args: int) -> str: + """Extract key arguments from command parts.""" + key_args = [] + for part in parts: + if not part.startswith('-') and len(part) > 1: + if len(part) > 20: + part = part[:17] + '...' + key_args.append(part) + if len(key_args) >= max_args: + break + return f"{cmd} {' '.join(key_args)}".strip() + def _extract_tool_names(self, content: str) -> list[str]: """Extract tool names from content.""" tool_counts = {} lines = content.split('\n') + # For step titles + relevant_tool_tags = [ + '', '', '', '', '', + '', '', '', '' + ] + for i, line in enumerate(lines): line = line.strip() - if (line in ['', ''] and not line.startswith('': if i + 1 < len(lines) and lines[i + 1].strip(): tool_counts['Stderr'] = tool_counts.get('Stderr', 0) + 1 tool_names = [] - for tool_name, count in tool_counts.items(): + for tool_name in tool_counts: tool_names.append(tool_name) return tool_names @@ -126,7 +140,6 @@ def _parse_steps_from_logs(self, agent_logs: list[LogPart]) -> list[dict]: steps_dict = {} current_step_number = None - current_step_name = None for log_part in agent_logs: content = log_part.content.strip() @@ -167,6 +180,7 @@ def _parse_steps_from_logs(self, agent_logs: list[LogPart]) -> list[dict]: return self._parse_steps_by_chat_pairs(agent_logs) def _parse_steps_by_chat_pairs(self, agent_logs: list[LogPart]) -> list[dict]: + """Parse steps from agent logs by grouping chat prompt/response pairs.""" steps = [] first_prompt_idx = -1 @@ -210,15 +224,44 @@ def _parse_steps_by_chat_pairs(self, agent_logs: list[LogPart]) -> list[dict]: return steps - def _syntax_highlight_content(self, - content: str, - default_language: str = "") -> str: + def _convert_newlines_outside_tags(self, content: str) -> str: + """Convert \\n to
tags when they appear outside XML tags.""" + tag_pattern = r'</?[^&]*?>' + + tag_matches = list(re.finditer(tag_pattern, content)) + + if not tag_matches: + return content.replace('\\n', '
') + + result = [] + last_end = 0 + + for match in tag_matches: + # Process text before this tag + before_tag = content[last_end:match.start()] + result.append(before_tag.replace('\\n', '
')) + + # Add the tag itself (unchanged) + result.append(match.group()) + + last_end = match.end() + + remaining = content[last_end:] + result.append(remaining.replace('\\n', '
')) + + return ''.join(result) + + def syntax_highlight_content(self, + content: str, + default_language: str = "", + agent_name: str = "") -> str: """Syntax highlights content while preserving visible tags.""" # Escape everything first so raw logs are safe to render in HTML escaped = html.escape(content) - # Helper to simplify substitutions + escaped = self._convert_newlines_outside_tags(escaped) + def _sub(pattern: str, repl: str, text: str) -> str: return re.sub(pattern, repl, text, flags=re.DOTALL) @@ -243,50 +286,171 @@ def _normalize_lang(lang: str) -> str: lang_key = _normalize_lang(default_language) escaped = _sub( - r'<conclusion>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</conclusion>', + r'<conclusion>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)' + r'</conclusion>', r'<conclusion>' - r'
\1
' + r'
\1
' r'</conclusion>', escaped) escaped = _sub( - r'<reason>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</reason>', - r'<reason>' - r'
\1
' + r'<reason>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)' + r'</reason>', r'<reason>' + r'
\1
' r'</reason>', escaped) escaped = _sub( - r'<bash>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</bash>', - r'<bash>' - r'
\1
' + r'<bash>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)' + r'</bash>', r'<bash>' + r'
'
+        r'\1
' r'</bash>', escaped) escaped = _sub( - r'<build_script>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</build_script>', + r'<build_script>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)' + r'</build_script>', r'<build_script>' - r'
\1
' + r'
'
+        r'\1
' r'</build_script>', escaped) escaped = _sub( - r'<fuzz target>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</fuzz target>', + r'<fuzz target>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)' + r'</fuzz target>', rf'<fuzz target>' - rf'
\1
' + rf'
'
+        rf'\1
' rf'</fuzz target>', escaped) escaped = _sub( - r'<stdout>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</stdout>', - r'<stdout>' - r'
\1
' + r'<stdout>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)' + r'</stdout>', r'<stdout>' + r'
'
+        r'\1
' r'</stdout>', escaped) escaped = _sub( - r'<stderr>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</stderr>', - r'<stderr>' - r'
\1
' + r'<stderr>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)' + r'</stderr>', r'<stderr>' + r'
'
+        r'\1
' r'</stderr>', escaped) escaped = _sub( - r'<return_code>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</return_code>', + r'<return_code>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)' + r'</return_code>', r'<return_code>' - r'
\1
' + r'
'
+        r'\1
' r'</return_code>', escaped) + escaped = _sub( + r'<build script>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)' + r'</build script>', + r'<build script>' + r'
'
+        r'\1
' + r'</build script>', escaped) + + escaped = _sub( + r'<gcb>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</gcb>', + r'<gcb>' + r'
'
+        r'\1
' + r'</gcb>', escaped) + + escaped = _sub( + r'<gdb>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</gdb>', + r'<gdb>' + r'
'
+        r'\1
' + r'</gdb>', escaped) + + escaped = _sub( + r'<gdb command>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)' + r'</gdb command>', + r'<gdb command>' + r'
'
+        r'\1
' + r'</gdb command>', escaped) + + escaped = _sub( + r'<gdb output>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)' + r'</gdb output>', + r'<gdb output>' + r'
'
+        r'\1
' + r'</gdb output>', escaped) + + escaped = _sub( + r'<code>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</code>', + r'<code>' + r'
'
+        rf'\1
' + r'</code>', escaped) + + escaped = _sub( + r'<solution>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)' + r'</solution>', r'<solution>' + r'
'
+        rf'\1
' + r'</solution>', escaped) + + def process_system_content(match): + content = match.group(1) + return (r'<system>' + r'
' + content + + r'
</system>') + + escaped = re.sub( + r'<system>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)' + r'</system>', + process_system_content, + escaped, + flags=re.DOTALL) + + # Handle steps tag (usually opening only, no closing tag) + escaped = _sub(r'<steps>', + r'<steps>', escaped) + + # Generic fallback for any remaining XML tags not explicitly handled above + # This ensures all XML tags get the log-tag styling + escaped = _sub(r'<([^/&][^&]*?)>', + r'<\1>', escaped) + escaped = _sub(r'<(/[^&]*?)>', + r'<\1>', escaped) + + # Handle ExecutionStage-specific highlighting for fuzz target source + if "ExecutionStage" in agent_name: + escaped = self._highlight_execution_stage_content(escaped, lang_key) + return escaped + def _highlight_execution_stage_content(self, content: str, + lang_key: str) -> str: + """Add syntax highlighting for ExecutionStage-specific content patterns.""" + + # Pattern to match "Fuzz target source:" followed by code until + # "Build script source:" + fuzz_target_pattern = (r'(Fuzz target source:)\s*\n' + r'(.*?)' + r'(?=Build script source:|$)') + + def replace_fuzz_target(match): + header = match.group(1) + code_content = match.group(2).strip() + + if code_content: + return ( + f'
{header}
' + '
'
+            f'{code_content}
') + return f'
{header}
' + + content = re.sub(fuzz_target_pattern, + replace_fuzz_target, + content, + flags=re.DOTALL) + + return content + def _create_step_data(self, step_number: int, log_parts: list[LogPart]) -> dict: """Create step data from log parts.""" @@ -310,7 +474,7 @@ def _create_step_data(self, step_number: int, def get_agent_sections(self) -> dict[str, list[LogPart]]: """Get the agent sections from the logs.""" - pattern = re.compile(r"\*{24}(.+?)\*{24}") + pattern = re.compile(r"\*{20,}([^*]+?)\*{20,}") agent_sections = {} current_agent = None agent_counters = {} diff --git a/report/templates/base.html b/report/templates/base.html index d0d43b0b8..4fd070e36 100644 --- a/report/templates/base.html +++ b/report/templates/base.html @@ -282,6 +282,24 @@ .dark-mode .toc-item { color: #9ca3af; } + +.log-tag { + color: #8b7355; +} + +.dark-mode .log-tag { + color: #a08968; +} + +.chat_prompt .log-tag { + color: #5a6b8a; +} + +.dark-mode .chat_prompt .log-tag { + color: #7a8ba8; +} + + - +