Skip to content

Commit 9344b7e

Browse files
jialinli98jialinli98
authored
chore: add benchmarks (#55)
1 parent d0fdc8d commit 9344b7e

File tree

6 files changed

+256
-0
lines changed

6 files changed

+256
-0
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
name: Benchmarks
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
9+
jobs:
10+
test:
11+
name: Benchmark library
12+
runs-on: ubuntu-latest
13+
steps:
14+
- name: Checkout sources
15+
uses: actions/checkout@v4
16+
17+
- name: Install Nargo
18+
uses: noir-lang/[email protected]
19+
with:
20+
toolchain: 1.0.0-beta.6
21+
22+
- name: Install bb
23+
run: |
24+
curl -L https://bbup.aztec.network | bash
25+
~/.bb/bbup -nv 1.0.0-beta.7
26+
sudo apt install libc++-dev
27+
28+
- name: Build Noir benchmark programs
29+
run: nargo export
30+
31+
- name: Generate gates report
32+
run: ./scripts/build-gates-report.sh
33+
env:
34+
BACKEND: /home/runner/.bb/bb
35+
36+
- name: Store ACIR opcode benchmark result
37+
uses: benchmark-action/github-action-benchmark@v1
38+
with:
39+
name: "ACIR Opcodes"
40+
tool: "customSmallerIsBetter"
41+
output-file-path: "benchmark-opcodes.json"
42+
gh-pages-branch: "gh-pages"
43+
benchmark-data-dir-path: "dev/bench"
44+
github-token: ${{ secrets.GITHUB_TOKEN }}
45+
auto-push: ${{ github.ref == 'refs/heads/main' }}
46+
comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
47+
comment-on-alert: true
48+
alert-threshold: "101%"
49+
fail-on-alert: false
50+
max-items-in-chart: 50
51+
52+
- name: Store gates benchmark result
53+
uses: benchmark-action/github-action-benchmark@v1
54+
with:
55+
name: "Circuit Size"
56+
tool: "customSmallerIsBetter"
57+
output-file-path: "benchmark-circuit.json"
58+
gh-pages-branch: "gh-pages"
59+
benchmark-data-dir-path: "dev/bench"
60+
github-token: ${{ secrets.GITHUB_TOKEN }}
61+
auto-push: ${{ github.ref == 'refs/heads/main' }}
62+
comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
63+
comment-on-alert: true
64+
alert-threshold: "101%"
65+
fail-on-alert: false
66+
max-items-in-chart: 50
67+
skip-fetch-gh-pages: true
68+
69+
# Delete the export files
70+
- name: Delete export files
71+
run: rm -rf export
72+
# Run nargo export again with force-brillig flag
73+
- name: Build Brillig benchmark programs
74+
run: nargo export --force-brillig
75+
76+
- name: Generate brillig report
77+
run: ./scripts/build-brillig-report.sh
78+
79+
- name: Store brillig benchmark result
80+
uses: benchmark-action/github-action-benchmark@v1
81+
with:
82+
name: "Brillig Bytecode Size"
83+
tool: "customSmallerIsBetter"
84+
output-file-path: "benchmark-brillig.json"
85+
gh-pages-branch: "gh-pages"
86+
benchmark-data-dir-path: "dev/bench"
87+
github-token: ${{ secrets.GITHUB_TOKEN }}
88+
auto-push: ${{ github.ref == 'refs/heads/main' }}
89+
comment-always: ${{ contains( github.event.pull_request.labels.*.name, 'bench-show') }}
90+
comment-on-alert: true
91+
alert-threshold: "101%"
92+
fail-on-alert: false
93+
max-items-in-chart: 50
94+
skip-fetch-gh-pages: true
95+

‎scripts/build-brillig-report.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
INSPECTOR=${INSPECTOR:-noir-inspector}
5+
cd $(dirname "$0")/../
6+
7+
artifacts_path="./export"
8+
artifacts=$(ls $artifacts_path)
9+
10+
# Start the JSON array
11+
REPORTS=$(jq --null-input '[]')
12+
13+
for artifact in $artifacts; do
14+
# Get and format the opcode info
15+
OP_CODE_INFO=$($INSPECTOR info --json "$artifacts_path/$artifact")
16+
17+
# Simplified jq expression to output only package_name and opcodes from unconstrained_functions
18+
REPORTS=$(echo $OP_CODE_INFO | jq -c '.programs[0] | del(.functions)' | jq -c --argjson old_reports $REPORTS '$old_reports + [.]')
19+
done
20+
21+
echo $REPORTS | jq '{ programs: . }' > brillig_report.json
22+
23+
# Convert brillig report to benchmark format
24+
output_file_brillig="benchmark-brillig.json"
25+
jq -r '[.programs[] | .unconstrained_functions[] | {
26+
"name": (if (.package_name // "") == "" then .name else "\(.package_name | sub("^null/"; ""))/\(.name)" end),
27+
"unit": "opcodes",
28+
"value": (.opcodes // 0)
29+
}]' brillig_report.json > $output_file_brillig

‎scripts/build-gates-report.sh‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/usr/bin/env bash
2+
set -e
3+
4+
BACKEND=${BACKEND:-bb}
5+
6+
cd $(dirname "$0")/../
7+
8+
artifacts_path="./export"
9+
artifacts=$(ls $artifacts_path)
10+
11+
echo "{\"programs\": [" > gates_report.json
12+
13+
# Bound for checking where to place last parentheses
14+
NUM_ARTIFACTS=$(ls -1q "$artifacts_path" | wc -l)
15+
16+
ITER="1"
17+
for artifact in $artifacts; do
18+
ARTIFACT_NAME=$(basename "$artifact")
19+
20+
GATES_INFO=$($BACKEND gates -b "$artifacts_path/$artifact")
21+
MAIN_FUNCTION_INFO=$(echo $GATES_INFO | jq -r '.functions[0] | .name = "main"')
22+
echo "{\"package_name\": \"$ARTIFACT_NAME\", \"functions\": [$MAIN_FUNCTION_INFO]" >> gates_report.json
23+
24+
if (($ITER == $NUM_ARTIFACTS)); then
25+
echo "}" >> gates_report.json
26+
else
27+
echo "}, " >> gates_report.json
28+
fi
29+
30+
ITER=$(( $ITER + 1 ))
31+
done
32+
33+
echo "]}" >> gates_report.json
34+
35+
# Convert the gates report into separate benchmark files
36+
output_file_opcodes="benchmark-opcodes.json"
37+
output_file_circuit="benchmark-circuit.json"
38+
39+
# Convert gates report - opcodes
40+
jq -r '[.programs[] | {
41+
"name": "\(.package_name)/main",
42+
"unit": "acir_opcodes",
43+
"value": (.functions[0].acir_opcodes // 0)
44+
}]' gates_report.json > $output_file_opcodes
45+
46+
# Convert gates report - circuit size
47+
jq -r '[.programs[] | {
48+
"name": "\(.package_name)/main",
49+
"unit": "circuit_size",
50+
"value": (.functions[0].circuit_size // 0)
51+
}]' gates_report.json > $output_file_circuit

‎src/benchmarks/benchmarks.nr‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
use crate::{JSON16kb, JSON512b, json::JSON};
2+
3+
comptime fn make_bench(m: Module, params: Quoted) -> Quoted {
4+
let module_name = m.name();
5+
let parse_json_from_string = f"parse_json_from_string_{module_name}".quoted_contents();
6+
let parse_json = f"parse_json_{module_name}".quoted_contents();
7+
let get_array = f"get_array_{module_name}".quoted_contents();
8+
let get_object = f"get_object_{module_name}".quoted_contents();
9+
let get_literal = f"get_literal_{module_name}".quoted_contents();
10+
let get_number = f"get_number_{module_name}".quoted_contents();
11+
let get_string = f"get_string_{module_name}".quoted_contents();
12+
let get_value = f"get_value_{module_name}".quoted_contents();
13+
let key_exists = f"key_exists_{module_name}".quoted_contents();
14+
let get_keys_at_root = f"get_keys_at_root_{module_name}".quoted_contents();
15+
let JSON = quote { crate::json::JSON };
16+
let JSONLiteral = quote { crate::get_literal::JSONLiteral };
17+
let JSONValue = quote { crate::json::JSONValue<1024> };
18+
let typ = params.as_type();
19+
20+
quote {
21+
#[export]
22+
fn $parse_json_from_string(s: str<1024>) -> $typ{
23+
$JSON::parse_json_from_string(s)
24+
}
25+
26+
#[export]
27+
fn $parse_json(s: [u8; 1024]) -> $typ{
28+
$JSON::parse_json(s)
29+
}
30+
31+
#[export]
32+
fn $get_array(s: $typ, key: [u8; 1024]) -> Option<$typ>{
33+
$JSON::get_array(s,key)
34+
}
35+
36+
#[export]
37+
fn $get_object(s: $typ, key: [u8; 1024]) -> Option<$typ>{
38+
$JSON::get_object(s,key)
39+
}
40+
41+
#[export]
42+
fn $get_literal(s: $typ, key: [u8; 1024]) -> Option<$JSONLiteral>{
43+
$JSON::get_literal(s,key)
44+
}
45+
46+
#[export]
47+
fn $get_number(s: $typ, key: [u8; 1024]) -> Option<u64>{
48+
$JSON::get_number(s,key)
49+
}
50+
51+
#[export]
52+
fn $get_string(s: $typ, key: [u8; 1024]) -> Option<BoundedVec<u8, 1024>>{
53+
$JSON::get_string(s,key)
54+
}
55+
56+
#[export]
57+
fn $get_value(s: $typ, key: [u8; 1024]) -> Option<$JSONValue>{
58+
$JSON::get_value(s,key)
59+
}
60+
61+
#[export]
62+
fn $key_exists(s: $typ, key: BoundedVec<u8, 1024>) -> bool{
63+
$JSON::key_exists(s, key)
64+
}
65+
66+
#[export]
67+
fn $get_keys_at_root(s: $typ) -> BoundedVec<BoundedVec<u8, 1024>, 1024>{
68+
$JSON::get_keys_at_root(s)
69+
}
70+
71+
72+
}
73+
}
74+
75+
#[make_bench(quote { JSON512b })]
76+
mod JSON512b_Bench {}
77+
78+
#[make_bench(quote { JSON16kb })]
79+
mod JSON16kb_Bench {}

‎src/benchmarks/mod.nr‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
mod benchmarks;

‎src/lib.nr‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ mod get_number;
1515
mod get_literal;
1616
mod get_object;
1717
mod get_array;
18+
mod benchmarks;
1819

1920
use get_literal::JSONLiteral;
2021
use json::{JSON, JSONValue};

0 commit comments

Comments
 (0)