diff --git a/report/parse_logs.py b/report/parse_logs.py index 60104bca1..ba0f55c8f 100644 --- a/report/parse_logs.py +++ b/report/parse_logs.py @@ -15,6 +15,7 @@ information such as the crash details, crash symptoms, stack traces, etc. to be rendered in the report.""" +import html import re from report.common import LogPart @@ -35,6 +36,277 @@ class LogsParser: def __init__(self, logs: list[LogPart]): self._logs = logs + def _extract_bash_commands(self, content: str) -> list[str]: + """Extract and parse bash commands from content.""" + commands = [] + lines = content.split('\n') + + for i, line in enumerate(lines): + line = line.strip() + if line == '': + for j in range(i + 1, len(lines)): + if lines[j].strip() == '': + bash_content = '\n'.join(lines[i + 1:j]).strip() + if bash_content: + first_line = bash_content.split('\n')[0].strip() + if first_line: + # skip comments and placeholder text + if (first_line.startswith('#') or + first_line.startswith('[The command') or + first_line.startswith('No bash') or + 'No bash' in first_line or len(first_line) < 3): + continue + + parts = first_line.split() + if parts: + cmd = parts[0] + + if cmd == 'grep': + # Extract the search term (usually the first quoted argument) + import re + quoted_match = re.search(r"'([^']+)'", first_line) + if quoted_match: + search_term = quoted_match.group(1) + command_summary = f"grep '{search_term}'" + else: + key_args = [] + for part in parts[1:]: + if not part.startswith('-') and len(part) > 1: + if len(part) > 20: + part = part[:17] + '...' + key_args.append(part) + if len(key_args) >= 1: # Limit to 1 arg for grep + break + command_summary = f"{cmd} {' '.join(key_args)}".strip() + else: + key_args = [] + for part in parts[1:]: + if not part.startswith('-') and len(part) > 1: + if len(part) > 20: + part = part[:17] + '...' + key_args.append(part) + if len(key_args) >= 2: # Limit to 2 key args + break + + command_summary = f"{cmd} {' '.join(key_args)}".strip() + + if len(command_summary) > 40: + command_summary = command_summary[:37] + '...' + + if command_summary not in commands: + commands.append(command_summary) + break + + return commands + + def _extract_tool_names(self, content: str) -> list[str]: + """Extract tool names from content.""" + tool_counts = {} + lines = content.split('\n') + + for i, line in enumerate(lines): + line = line.strip() + if (line in ['', ''] and not line.startswith('': + if i + 1 < len(lines) and lines[i + 1].strip(): + tool_counts['Stderr'] = tool_counts.get('Stderr', 0) + 1 + + tool_names = [] + for tool_name, count in tool_counts.items(): + tool_names.append(tool_name) + + return tool_names + + def _parse_steps_from_logs(self, agent_logs: list[LogPart]) -> list[dict]: + """Parse steps from agent logs, grouping by chat prompt/response pairs.""" + step_pattern = re.compile(r"Step #(\d+) - \"(.+?)\":") + simple_step_pattern = re.compile(r"Step #(\d+)") + + steps_dict = {} + current_step_number = None + current_step_name = None + + for log_part in agent_logs: + content = log_part.content.strip() + if not content: + continue + + lines = content.split('\n') + + step_header_found = False + for line in lines: + step_match = step_pattern.search(line) + if not step_match: + simple_match = simple_step_pattern.search(line) + if simple_match: + step_match = simple_match + + if step_match: + step_header_found = True + current_step_number = step_match.group(1) + + if current_step_number not in steps_dict: + steps_dict[current_step_number] = { + 'number': current_step_number, + 'type': 'Step', + 'log_parts': [] + } + break + + if not step_header_found and current_step_number: + steps_dict[current_step_number]['log_parts'].append(log_part) + elif not step_header_found and not current_step_number and not steps_dict: + steps_dict['0'] = { + 'number': None, + 'type': 'Content', + 'log_parts': [log_part] + } + + return self._parse_steps_by_chat_pairs(agent_logs) + + def _parse_steps_by_chat_pairs(self, agent_logs: list[LogPart]) -> list[dict]: + steps = [] + + first_prompt_idx = -1 + for i, log_part in enumerate(agent_logs): + if log_part.chat_prompt: + first_prompt_idx = i + break + + if first_prompt_idx == -1: + return [] + + steps.append({ + 'number': '0 - System Instructions', + 'type': 'System Instructions', + 'log_parts': [agent_logs[first_prompt_idx]] + }) + + # Process logs after the system prompt to group into steps. + logs_to_process = agent_logs[first_prompt_idx + 1:] + step_counter = 1 + current_step_parts = [] + + for log_part in logs_to_process: + if "agent-step" in log_part.content or "Trial ID:" in log_part.content: + continue + + # A chat_response marks the beginning of a new step. + if log_part.chat_response: + if current_step_parts: + step_data = self._create_step_data(step_counter, current_step_parts) + steps.append(step_data) + step_counter += 1 + current_step_parts = [log_part] + else: + current_step_parts.append(log_part) + + # Append the last step. + if current_step_parts: + step_data = self._create_step_data(step_counter, current_step_parts) + steps.append(step_data) + + return steps + + def _syntax_highlight_content(self, + content: str, + default_language: str = "") -> str: + """Syntax highlights content while preserving visible tags.""" + + # Escape everything first so raw logs are safe to render in HTML + escaped = html.escape(content) + + # Helper to simplify substitutions + def _sub(pattern: str, repl: str, text: str) -> str: + return re.sub(pattern, repl, text, flags=re.DOTALL) + + def _normalize_lang(lang: str) -> str: + if not lang: + return 'cpp' + lang = lang.strip().lower() + if lang in ['c++', 'cpp', 'cxx']: + return 'cpp' + if lang in ['c']: + return 'c' + if lang in ['python', 'py']: + return 'python' + if lang in ['java']: + return 'java' + if lang in ['rust', 'rs']: + return 'rust' + if lang in ['go', 'golang']: + return 'go' + return 'cpp' + + lang_key = _normalize_lang(default_language) + + escaped = _sub( + r'<conclusion>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</conclusion>', + r'<conclusion>' + r'
\1
' + r'</conclusion>', escaped) + escaped = _sub( + r'<reason>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</reason>', + r'<reason>' + r'
\1
' + r'</reason>', escaped) + + escaped = _sub( + r'<bash>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</bash>', + r'<bash>' + r'
\1
' + r'</bash>', escaped) + escaped = _sub( + r'<build_script>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</build_script>', + r'<build_script>' + r'
\1
' + r'</build_script>', escaped) + escaped = _sub( + r'<fuzz target>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</fuzz target>', + rf'<fuzz target>' + rf'
\1
' + rf'</fuzz target>', escaped) + + escaped = _sub( + r'<stdout>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</stdout>', + r'<stdout>' + r'
\1
' + r'</stdout>', escaped) + escaped = _sub( + r'<stderr>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</stderr>', + r'<stderr>' + r'
\1
' + r'</stderr>', escaped) + escaped = _sub( + r'<return_code>(\s*[^\s].*?[^\s]\s*|(?:\s*[^\s].*?)?)</return_code>', + r'<return_code>' + r'
\1
' + r'</return_code>', escaped) + + return escaped + + def _create_step_data(self, step_number: int, + log_parts: list[LogPart]) -> dict: + """Create step data from log parts.""" + step_data = { + 'number': str(step_number), + 'type': 'Step', + 'log_parts': log_parts + } + + all_content = '\n'.join([part.content for part in log_parts]) + tool_names = self._extract_tool_names(all_content) + bash_commands = self._extract_bash_commands(all_content) + + if tool_names: + step_data['name'] = f"{', '.join(tool_names)}" + if bash_commands: + step_data['bash_commands'] = bash_commands + + return step_data + def get_agent_sections(self) -> dict[str, list[LogPart]]: """Get the agent sections from the logs.""" @@ -91,16 +363,22 @@ def get_agent_cycles(self) -> list[dict]: cycles_dict = {} for agent_name, agent_logs in agent_sections.items(): + # Parse steps for this agent + steps = self._parse_steps_from_logs(agent_logs) + cycle_match = re.search(r'\(Cycle (\d+)\)', agent_name) if cycle_match: cycle_number = int(cycle_match.group(1)) if cycle_number not in cycles_dict: cycles_dict[cycle_number] = {} - cycles_dict[cycle_number][agent_name] = agent_logs + cycles_dict[cycle_number][agent_name] = { + 'logs': agent_logs, + 'steps': steps + } else: if 0 not in cycles_dict: cycles_dict[0] = {} - cycles_dict[0][agent_name] = agent_logs + cycles_dict[0][agent_name] = {'logs': agent_logs, 'steps': steps} return [cycles_dict[cycle] for cycle in sorted(cycles_dict.keys())] @@ -178,30 +456,30 @@ def get_formatted_stack_traces(self, function_name = in_match.group(1) path = in_match.group(2) - if '/src/' in path and 'llvm-project' not in path and self._benchmark_id and self._sample_id: - path_parts = path.split(':') - file_path = path_parts[0] # Just the file path without line numbers - line_number = path_parts[1] if len(path_parts) > 1 else None - - relative_path = file_path.lstrip('/') - - # If coverage_report_path is set, it's a local run - # Otherwise it's cloud - if self._coverage_report_path: - url = f'{self._coverage_report_path}{relative_path}.html' - url_with_line_number = f'{url}#L{line_number}' if line_number else url - else: - url = ( - f'{base_url}/results/{self._benchmark_id}/code-coverage-reports/' - f'{self._sample_id}.fuzz_target/report/linux/' - f'{relative_path}.html') - url_with_line_number = f'{url}#L{line_number}' if line_number else url - stack_traces[frame_num] = { - "url": url_with_line_number, - "path": path, - "function": function_name, - "memory_address": memory_addr - } + if '/src/' in path and 'llvm-project' not in path: + if self._benchmark_id and self._sample_id: + path_parts = path.split(':') + file_path = path_parts[0] + line_number = path_parts[1] if len(path_parts) > 1 else None + + relative_path = file_path.lstrip('/') + + # If coverage_report_path is set, it's a local run + # Otherwise it's cloud + if self._coverage_report_path: + url = f'{self._coverage_report_path}{relative_path}.html' + url_line_number = f'{url}#L{line_number}' if line_number else url + else: + url = (f'{base_url}/results/{self._benchmark_id}/' + f'code-coverage-reports/{self._sample_id}.fuzz_target/' + f'report/linux/{relative_path}.html') + url_line_number = f'{url}#L{line_number}' if line_number else url + stack_traces[frame_num] = { + "url": url_line_number, + "path": path, + "function": function_name, + "memory_address": memory_addr + } return stack_traces diff --git a/report/templates/base.html b/report/templates/base.html index 9a4bcd0be..d0d43b0b8 100644 --- a/report/templates/base.html +++ b/report/templates/base.html @@ -87,12 +87,14 @@ background-color: #333333; } .chat_prompt { - background-color: #fff7f2; + background-color: #e2e8f0; + border: 1px solid #a0aec0; max-width: 50%; overflow: scroll; } .chat_response { - background-color: #fcfff2; + background-color: #ececec; + border: 1px solid #718096; max-width: 50%; overflow: scroll; margin-left: auto; @@ -507,6 +509,8 @@

Search Results

+ + {% if unified_data %} @@ -746,8 +750,8 @@

} document.addEventListener('DOMContentLoaded', (event) => { - document.querySelectorAll('pre code.syntax-highlight').forEach((block) => { - hljs.highlightElement(block); + document.querySelectorAll('.prettify-benchmark-name').forEach(el => { + el.textContent = prettifyBenchmarkName(el.textContent); }); const searchInput = document.getElementById('searchInput'); @@ -800,10 +804,6 @@

}, 300); }); - document.querySelectorAll('.prettify-benchmark-name').forEach(el => { - el.textContent = prettifyBenchmarkName(el.textContent); - }); - const tocTree = document.getElementById('toc-tree'); const sections = document.querySelectorAll('.toc-section'); sections.forEach((section, index) => { @@ -913,16 +913,51 @@

}); - - - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + + + + \ No newline at end of file diff --git a/report/templates/index/index.html b/report/templates/index/index.html index 8c222333a..079ce8803 100644 --- a/report/templates/index/index.html +++ b/report/templates/index/index.html @@ -35,6 +35,7 @@ languageOpen: true, crashesFoundOpen: true, ofgMetricsOpen: true, + chartsOpen: true, }" class="space-y-2">
@@ -489,6 +490,30 @@
+
+
+ +
+
+
+
+
+
+
+
+
+
+
- {% for log_part in agent_logs %} -
{{ log_part.content }}
- {% endfor %} + {% if agent_data.steps %} + {% for step in agent_data.steps %} +
+ +
+ {% for log_part in step.log_parts %} +
{{ log_part.content|syntax_highlight|safe }}
+ {% endfor %} +
+
+ {% endfor %} + {% else %} + {% for log_part in agent_data.logs %} +
{{ log_part.content|syntax_highlight(default_lang)|safe }}
+ {% endfor %} + {% endif %}
{% endfor %} @@ -314,7 +335,7 @@

Build Script

Cycle {{ loop.index }}
- {% for agent_name, agent_logs in cycle_data.items() %} + {% for agent_name, agent_data in cycle_data.items() %}
- {% for log_part in agent_logs %} -
{{ log_part.content }}
- {% endfor %} + {% if agent_data.steps %} + {% if agent_data.steps|length == 1 %} + + {% set step = agent_data.steps[0] %} + {% if step.number and step.name %} +
+
Step {{ step.number }} - + {% if 'Stderr' in step.name %} + {{ step.name.replace('Stderr', 'Stderr') | safe }} + {% else %} + {{ step.name }} + {% endif %} +
+ {% if step.bash_commands %} +
+ Commands: + {% for cmd in step.bash_commands %} + {{ cmd }} + {% endfor %} +
+ {% endif %} + {% if step.log_parts %} + {% for log_part in step.log_parts %} +
{{ log_part.content|syntax_highlight|safe }}
+ {% endfor %} + {% else %} +
{{ step.content|syntax_highlight|safe }}
+ {% endif %} +
+ {% else %} + {% if step.log_parts %} + {% for log_part in step.log_parts %} +
{{ log_part.content|syntax_highlight|safe }}
+ {% endfor %} + {% else %} +
{{ step.content|syntax_highlight|safe }}
+ {% endif %} + {% endif %} + {% else %} + + {% for step in agent_data.steps %} +
+ +
+ {% if step.log_parts %} + {% for log_part in step.log_parts %} +
{{ log_part.content|syntax_highlight(default_lang)|safe }}
+ {% endfor %} + {% else %} +
{{ step.content|syntax_highlight|safe }}
+ {% endif %} +
+
+ {% endfor %} + {% endif %} + {% else %} + {% for log_part in agent_data.logs %} +
{{ log_part.content|syntax_highlight(default_lang)|safe }}
+ {% endfor %} + {% endif %}
{% endfor %} @@ -349,7 +462,7 @@

Build Script

{% for part in logs %} -
{{ part.content }}
+
{{ part.content|syntax_highlight(default_lang)|safe }}
{% endfor %}
@@ -403,6 +516,7 @@

Build Script