Skip to content

Add artifact paths to UI display for crashed samples #1180

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Add artifact paths to UI display for crashed samples #1180

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 23 additions & 6 deletions report/export.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,15 @@
class BaseExporter:
"""Base class for exporters."""

def __init__(self, results: Results, output_dir: str, base_url: str = ''):
def __init__(self,
results: Results,
output_dir: str,
base_url: str = '',
gcs_dir: str = ''):
self._results = results
self._output_dir = output_dir
self._base_url = base_url.rstrip('/')
self._gcs_dir = gcs_dir
self._headers = [
"Project", "Function Signature", "Sample", "Crash Type", "Compiles",
"Crashes", "Coverage", "Line Coverage Diff", "Reproducer Path"
Expand All @@ -51,6 +56,21 @@ def get_url_path(self) -> str:
class CSVExporter(BaseExporter):
"""Export a report to CSV."""

def _get_reproducer_url(self, benchmark_id: str,
crash_reproduction_path: str) -> str:
"""Get the reproducer URL, using GCS bucket URL for cloud builds."""
if not crash_reproduction_path:
return ""

if self._gcs_dir:
return (f"https://console.cloud.google.com/storage/browser/"
f"oss-fuzz-gcb-experiment-run-logs/Result-reports/"
f"{self._gcs_dir}/results/{benchmark_id}/artifacts/"
f"{crash_reproduction_path}")
else:
return self._get_full_url(
f'results/{benchmark_id}/artifacts/{crash_reproduction_path}')

def generate(self):
"""Generate a CSV file with the results."""
csv_path = os.path.join(self._output_dir, 'crashes.csv')
Expand All @@ -70,18 +90,15 @@ def generate(self):

project_name = benchmark_id.split("-")[1]

project_name = benchmark_id.split("-")[1]

for sample in samples:
run_logs = self._results.get_run_logs(benchmark_id, sample.id) or ""
parser = RunLogsParser(run_logs, benchmark_id, sample.id)
crash_reproduction_path = parser.get_crash_reproduction_path()

report_url = self._get_full_url(
f"sample/{benchmark_id}/{sample.id}.html")
reproducer_path = self._get_full_url(
f'results/{benchmark_id}/artifacts/{sample.id}.fuzz_target-F0-01/'
f'{crash_reproduction_path}') if crash_reproduction_path else ""
reproducer_path = self._get_reproducer_url(benchmark_id,
crash_reproduction_path)

writer.writerow({
"Project":
Expand Down
22 changes: 22 additions & 0 deletions report/templates/sample/sample.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,28 @@ <h1 class="text-2xl font-bold">
</span>
</div>
<div class="flex flex-col gap-2 mt-8">
{% if crash_info is defined and crash_info.artifact_folder %}
<div class="border rounded-lg p-4 shadow-sm mb-4">
<div class="flex items-center gap-3">
<div class="flex-1">
<p class="text-sm font-medium mb-2">Artifact Folder</p>
<div class="space-y-1">
<div>
<span class="text-xs">Console:</span>
<a href="{{ crash_info.artifact_folder.console_url }}" target="_blank" rel="noopener noreferrer"
class="text-sm hover:underline break-all ml-1">
{{ crash_info.artifact_folder.console_url }}
</a>
</div>
<div>
<span class="text-xs">GCS:</span>
<code class="text-sm break-all ml-1">{{ crash_info.artifact_folder.gs_url }}</code>
</div>
</div>
</div>
</div>
</div>
{% endif %}

{% if sample.result.semantic_error or (crash_info is defined and crash_info) %}
<div class="border rounded-lg p-6 shadow-sm">
Expand Down
2 changes: 1 addition & 1 deletion report/upload_report.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ update_report() {
# Generate the report
if [[ $GCS_DIR != '' ]]; then
CLOUD_BASE_URL="https://llm-exp.oss-fuzz.com/Result-reports/${GCS_DIR}"
$PYTHON -m report.web -r "${RESULTS_DIR:?}" -b "${BENCHMARK_SET:?}" -m "$MODEL" -o results-report --base-url "$CLOUD_BASE_URL" $REPORT_ADDITIONAL_ARGS
$PYTHON -m report.web -r "${RESULTS_DIR:?}" -b "${BENCHMARK_SET:?}" -m "$MODEL" -o results-report --base-url "$CLOUD_BASE_URL" --gcs-dir "${GCS_DIR}" $REPORT_ADDITIONAL_ARGS
else
$PYTHON -m report.web -r "${RESULTS_DIR:?}" -b "${BENCHMARK_SET:?}" -m "$MODEL" -o results-report $REPORT_ADDITIONAL_ARGS
fi
Expand Down
32 changes: 28 additions & 4 deletions report/web.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,14 +148,16 @@ def __init__(self,
jinja_env: JinjaEnv,
results_dir: str,
output_dir: str = 'results-report',
base_url: str = ''):
base_url: str = '',
gcs_dir: str = ''):
self._results = results
self._output_dir = output_dir
self._jinja = jinja_env
self.results_dir = results_dir
# If cloud, this will be
# `llm-exp.oss-fuzz.com/Result-reports/ofg-pr/experiment-name`
self._base_url = base_url
self._gcs_dir = gcs_dir

def read_timings(self):
with open(os.path.join(self.results_dir, 'report.json'), 'r') as f:
Expand Down Expand Up @@ -602,6 +604,21 @@ def _build_unified_data(self, benchmarks: List[Benchmark],

return unified_data

def _get_artifact_folder_url(self, benchmark_id: str, sample: Sample) -> dict:
"""Get the artifact folder URLs for cloud builds."""
if not sample.result.crashes:
return {}
if not self._gcs_dir:
return {}

gs_url = (f"gs://oss-fuzz-gcb-experiment-run-logs/Result-reports/"
f"{self._gcs_dir}/results/{benchmark_id}/artifacts")
console_url = (f"https://console.cloud.google.com/storage/browser/"
f"oss-fuzz-gcb-experiment-run-logs/Result-reports/"
f"{self._gcs_dir}/results/{benchmark_id}/artifacts")

return {"gs_url": gs_url, "console_url": console_url}

def _get_crash_info_from_run_logs(self, benchmark_id: str,
sample: Sample) -> dict:
"""Get the crash info from the run logs."""
Expand All @@ -614,11 +631,13 @@ def _get_crash_info_from_run_logs(self, benchmark_id: str,
crash_symptom = parser.get_crash_symptom()
stack_traces = parser.get_formatted_stack_traces(self._base_url)
execution_stats = parser.get_execution_stats()
artifact_folder = self._get_artifact_folder_url(benchmark_id, sample)
return {
"crash_details": crash_details,
"crash_symptom": crash_symptom,
"stack_traces": stack_traces,
"execution_stats": execution_stats
"execution_stats": execution_stats,
"artifact_folder": artifact_folder
}


Expand All @@ -637,7 +656,8 @@ def generate_report(args: argparse.Namespace) -> None:
if args.with_csv:
csv_reporter = CSVExporter(results=results,
output_dir=args.output_dir,
base_url=base_url)
base_url=base_url,
gcs_dir=args.gcs_dir)
csv_reporter.generate()
# Temporarily commented out because locally this is just a relative path
# template_globals['csv_url_path'] = csv_reporter.get_url_path()
Expand All @@ -652,7 +672,8 @@ def generate_report(args: argparse.Namespace) -> None:
jinja_env=jinja_env,
results_dir=args.results_dir,
output_dir=args.output_dir,
base_url=base_url)
base_url=base_url,
gcs_dir=args.gcs_dir)
gr.generate()


Expand Down Expand Up @@ -709,6 +730,9 @@ def _parse_arguments() -> argparse.Namespace:
'--base-url',
help='Base URL for the report (used in cloud environments).',
default='')
parser.add_argument('--gcs-dir',
help='GCS directory for experiment.',
default='')

return parser.parse_args()

Expand Down