diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a478d5a9..85a72ed7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,6 +70,11 @@ jobs: nixdo nice cargo --version nixdo nice cargo clippy --version nixdo nice cargo clippy --locked -- -D warnings + + - name: cargo-clippy (mozakvm) + working-directory: sdk + run: | + nixdo nice cargo clippy --locked --all-features -- -D warnings - name: cargo-clippy (native, examples) working-directory: examples @@ -77,11 +82,6 @@ jobs: ARCH_TRIPLE="$(nixdo rustc --verbose --version | grep host | awk '{ print $2; }')" nixdo nice cargo clippy --locked --all-features --target ${ARCH_TRIPLE} -- -D warnings - - name: cargo-clippy (mozakvm) - working-directory: sdk - run: | - nixdo nice cargo clippy --locked --all-features -- -D warnings - - name: cargo-clippy (native) working-directory: sdk run: | diff --git a/.gitignore b/.gitignore index 709b92e1c..c7a0d9a48 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ riscv-testdata/testdata/*.i # Temporary scripts *tmp.sh examples/*/out/ + +# self program ids +*self_prog_id.txt diff --git a/Cargo.lock b/Cargo.lock index ab2637316..fa525dc0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -199,6 +199,10 @@ dependencies = [ "generic-array", ] +[[package]] +name = "build_scripts" +version = "0.1.0" + [[package]] name = "bumpalo" version = "3.16.0" @@ -935,6 +939,7 @@ dependencies = [ "clap-verbosity-flag", "clap_derive", "clio", + "criterion", "env_logger", "itertools 0.13.0", "log", diff --git a/Cargo.toml b/Cargo.toml index da1b4bea9..675078bc0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ [workspace] exclude = ["sdk"] members = [ + "build_scripts", "circuits", "cli", "examples-builder", diff --git a/build_scripts/Cargo.toml b/build_scripts/Cargo.toml new file mode 100644 index 000000000..fed0705ab --- /dev/null +++ b/build_scripts/Cargo.toml @@ -0,0 +1,10 @@ +[package] +categories = ["cryptography"] +description = "Build scripts for mozak examples" +edition = "2021" +keywords = ["crypto", "zero-knowledge", "vm"] +license = "All rights reserved" +name = "build_scripts" +readme = "README.md" +repository = "https://github.com/0xmozak/mozak-vm" +version = "0.1.0" diff --git a/build_scripts/src/lib.rs b/build_scripts/src/lib.rs new file mode 100644 index 000000000..42c8820cb --- /dev/null +++ b/build_scripts/src/lib.rs @@ -0,0 +1 @@ +pub mod self_prog_id; diff --git a/build_scripts/src/self_prog_id.rs b/build_scripts/src/self_prog_id.rs new file mode 100644 index 000000000..7161df111 --- /dev/null +++ b/build_scripts/src/self_prog_id.rs @@ -0,0 +1,46 @@ +use std::fs::File; +use std::io; +use std::io::Write; +use std::path::Path; +use std::process::Command; + +const CARGO_MANIFEST_DIR: &str = env!("CARGO_MANIFEST_DIR"); +/// dumps the `ProgramIdentifier` of the input ELF +/// in a file `self_prog_id.txt` lying in same directory. +/// Note that the `ProgramIdentifier` is represented as +/// a string, for example, +/// `MZK-8279ede6d0459962efe306b82d6036c08d6b9e66ad8d89d9e297c5222b3f4572` +pub fn dump_self_prog_id(out_elf_name: &str) { + let cargo_manifest_dir_path = Path::new(CARGO_MANIFEST_DIR); + let out_dir = + cargo_manifest_dir_path.join("../examples/target/riscv32im-mozak-mozakvm-elf/release/"); + let out_elf_path = out_dir.join(out_elf_name); + let out_elf_path_str = out_elf_path.to_str().unwrap(); + let cli_dir = cargo_manifest_dir_path.join("../cli"); + + let args = vec!["run", "--release", "--", "self-prog-id", out_elf_path_str]; + + // execute the cli command `self-prog-id` on the ELF + let output = Command::new("cargo") + .args(args) + .current_dir(cli_dir) + .env_clear() + .envs(std::env::vars().filter(|x| !x.0.starts_with("CARGO_"))) + .output() + .expect("can't find mozak-cli. Please run cargo build --release --bin mozak-cli from project root"); + + if !output.status.success() { + io::stdout().write_all(&output.stdout).unwrap(); + io::stderr().write_all(&output.stderr).unwrap(); + panic!("cargo run -- self-prog-id {out_elf_path_str} failed"); + } + + let self_prog_id = String::from_utf8(output.stdout).unwrap(); + let self_prog_id = self_prog_id.trim(); + + // write `self_prog_id` to a file + let mut self_prog_id_file = + File::create("self_prog_id.txt").expect("failed to create self_prog_id.txt"); + write!(self_prog_id_file, "{}", self_prog_id).expect("failed to write self_prog_id.txt"); + println!("cargo:rerun-if-changed={}", out_elf_path_str); +} diff --git a/cli/Cargo.toml b/cli/Cargo.toml index a0fe3028e..571dcbe4d 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -29,7 +29,7 @@ clio = { version = "0.3", features = ["clap-parse"] } env_logger = "0.11" itertools = "0.13" log = "0.4" -mozak-examples = { path = "../examples-builder", features = ["mozak-sort"] } +mozak-examples = { path = "../examples-builder", features = ["mozak-sort"], optional = true } plonky2 = { workspace = true, default-features = false } plonky2_maybe_rayon = { workspace = true, default-features = false } rkyv = { version = "=0.8.0-alpha.1", default-features = false, features = ["pointer_width_32", "alloc"] } @@ -40,10 +40,13 @@ starky = { workspace = true, default-features = false } tempfile = "3" [dev-dependencies] +# Only as a workaround to be able to control criterion's rayon feature and optional dependency. +criterion = { workspace = true, default-features = false } mozak-circuits = { path = "../circuits", features = ["test"] } mozak-runner = { path = "../runner", features = ["test"] } proptest = "1.4" [features] +bench = ["mozak-examples"] default = [] -parallel = ["plonky2/parallel", "starky/parallel", "mozak-circuits/parallel"] +parallel = ["plonky2/parallel", "starky/parallel", "mozak-circuits/parallel", "criterion/rayon"] diff --git a/cli/src/lib.rs b/cli/src/lib.rs index e9c8fbe43..7828ba3c6 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -1,3 +1,4 @@ +#[cfg(feature = "bench")] pub mod cli_benches; pub mod runner; #[cfg(test)] diff --git a/cli/src/main.rs b/cli/src/main.rs index 3d1237aca..428c36a7c 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -26,6 +26,7 @@ use mozak_circuits::stark::utils::trace_rows_to_poly_values; use mozak_circuits::stark::verifier::verify_proof; use mozak_circuits::storage_device::generation::generate_call_tape_trace; use mozak_circuits::test_utils::{prove_and_verify_mozak_stark, C, D, F, S}; +#[cfg(feature = "bench")] use mozak_cli::cli_benches::benches::BenchArgs; use mozak_cli::runner::{ deserialize_system_tape, get_self_prog_id, load_program, raw_tapes_from_system_tape, @@ -60,8 +61,6 @@ pub struct RunArgs { elf: Input, #[arg(long)] system_tape: Option, - #[arg(long)] - self_prog_id: String, } #[derive(Clone, Debug, Args)] @@ -72,8 +71,6 @@ pub struct ProveArgs { batch_proof: Option, #[arg(long)] system_tape: Option, - #[arg(long)] - self_prog_id: String, recursive_proof: Option, } @@ -107,6 +104,7 @@ enum Command { MemoryInitHash { elf: Input }, /// Compute the Self Program Id of the given ELF, SelfProgId { elf: Input }, + #[cfg(feature = "bench")] /// Bench the function with given parameters Bench(BenchArgs), } @@ -125,25 +123,17 @@ fn main() -> Result<()> { let program = load_program(elf)?; debug!("{program:?}"); } - Command::Run(RunArgs { - elf, - system_tape, - self_prog_id, - }) => { - let raw_tapes = raw_tapes_from_system_tape(system_tape, self_prog_id.into()); + Command::Run(RunArgs { elf, system_tape }) => { let program = load_program(elf).unwrap(); + let self_prog_id = get_self_prog_id::(&program, &config); + let raw_tapes = raw_tapes_from_system_tape(system_tape, self_prog_id); let state: State = State::new(program.clone(), raw_tapes); step(&program, state)?; } - Command::ProveAndVerify(RunArgs { - elf, - system_tape, - self_prog_id, - }) => { + Command::ProveAndVerify(RunArgs { elf, system_tape }) => { let program = load_program(elf).unwrap(); - - let raw_tapes = raw_tapes_from_system_tape(system_tape, self_prog_id.into()); - + let self_prog_id = get_self_prog_id::(&program, &config); + let raw_tapes = raw_tapes_from_system_tape(system_tape, self_prog_id); let state = State::new(program.clone(), raw_tapes); let record = step(&program, state)?; prove_and_verify_mozak_stark(&program, &record, &config)?; @@ -151,14 +141,13 @@ fn main() -> Result<()> { Command::Prove(ProveArgs { elf, system_tape, - self_prog_id, mut proof, recursive_proof, batch_proof, }) => { - let raw_tapes = raw_tapes_from_system_tape(system_tape, self_prog_id.clone().into()); - let self_program_id: ProgramIdentifier = self_prog_id.into(); let program = load_program(elf).unwrap(); + let self_prog_id = get_self_prog_id::(&program, &config); + let raw_tapes = raw_tapes_from_system_tape(system_tape, self_prog_id); let state = State::new(program.clone(), raw_tapes); let record = step(&program, state)?; let stark = if cli.debug { @@ -216,7 +205,7 @@ fn main() -> Result<()> { let public_inputs: VMRecursiveProofPublicInputs = public_inputs_array.into(); assert_eq!( public_inputs.program_hash_as_bytes.to_vec(), - self_program_id + self_prog_id .inner() .into_iter() .map(F::from_canonical_u8) @@ -457,6 +446,7 @@ fn main() -> Result<()> { let self_prog_id = get_self_prog_id::(&program, &config); println!("{self_prog_id:?}"); } + #[cfg(feature = "bench")] Command::Bench(bench) => { let time_taken = bench.bench()?.as_secs_f64(); println!("{time_taken}"); diff --git a/cli/src/tests/integration_test.rs b/cli/src/tests/integration_test.rs index d552cdd1b..8b83e4449 100644 --- a/cli/src/tests/integration_test.rs +++ b/cli/src/tests/integration_test.rs @@ -20,14 +20,6 @@ fn test_prove_and_verify_recursive_proof_command() { // Create mock IO tape files fs::write(system_tape, b"").expect("Failed to create system tape file"); - // Get self_prog_id - let output = Command::new("cargo") - .args(["run", "--", "self-prog-id", elf_file]) - .output() - .expect("Failed to execute self-prog-id command"); - let mut self_prog_id = String::from_utf8(output.stdout).unwrap(); - self_prog_id = self_prog_id.trim().to_string(); - // Execute the `prove` command let output = Command::new("cargo") .args([ @@ -36,8 +28,6 @@ fn test_prove_and_verify_recursive_proof_command() { "prove", elf_file, &proof_file.to_string_lossy(), - "--self-prog-id", - &self_prog_id, &recursive_proof_file.to_string_lossy(), ]) .output() diff --git a/examples/Cargo.lock b/examples/Cargo.lock index 7302d4a65..bedaafac1 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -70,6 +70,10 @@ dependencies = [ "mozak-sdk", ] +[[package]] +name = "build_scripts" +version = "0.1.0" + [[package]] name = "bumpalo" version = "3.16.0" @@ -213,6 +217,7 @@ checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" name = "inputtape" version = "0.1.0" dependencies = [ + "build_scripts", "hex", "mozak-sdk", "rkyv", @@ -713,6 +718,7 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" name = "token" version = "0.1.0" dependencies = [ + "build_scripts", "hex", "mozak-sdk", "rkyv", @@ -751,6 +757,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" name = "wallet" version = "0.1.0" dependencies = [ + "build_scripts", "hex", "mozak-sdk", "rand", diff --git a/examples/inputtape/Cargo.toml b/examples/inputtape/Cargo.toml index 7f7f0bee6..bc9d5126e 100644 --- a/examples/inputtape/Cargo.toml +++ b/examples/inputtape/Cargo.toml @@ -12,6 +12,9 @@ mozak-sdk = { path = "../../sdk" } rkyv = { version = "=0.8.0-alpha.1", default-features = false, features = ["pointer_width_32", "alloc"] } rkyv_derive = "=0.8.0-alpha.1" +[target.'cfg(not(target_os="mozakvm"))'.build-dependencies] +build_scripts = { path = "../../build_scripts" } + [target.'cfg(not(target_os="mozakvm"))'.dependencies.hex] version = "0.4" @@ -30,4 +33,3 @@ path = "core_logic.rs" # The following is read by `run_examples.py` [package.metadata.mozak] -example_program_id = "MZK-5de9bc1bb4cb7a9f9cf991a32230d014b44274ea5464a352f089a8cccc486915" diff --git a/examples/inputtape/build.rs b/examples/inputtape/build.rs new file mode 100644 index 000000000..d8f8aaa44 --- /dev/null +++ b/examples/inputtape/build.rs @@ -0,0 +1,7 @@ +fn main() { + #[cfg(feature = "native")] + { + extern crate build_scripts; + build_scripts::self_prog_id::dump_self_prog_id("inputtapebin"); + } +} diff --git a/examples/inputtape/main_native.rs b/examples/inputtape/main_native.rs index afd4fcd2c..fde9ae896 100644 --- a/examples/inputtape/main_native.rs +++ b/examples/inputtape/main_native.rs @@ -6,7 +6,8 @@ use mozak_sdk::common::types::{Poseidon2Hash, ProgramIdentifier}; use crate::core_logic::{dispatch, MethodArgs}; fn main() { - let token_program = ProgramIdentifier::new_from_rand_seed(3); + let token_program: ProgramIdentifier = + std::fs::read_to_string("self_prog_id.txt").unwrap().into(); let buf1 = Poseidon2Hash::new_from_rand_seed(2).inner(); let buf2 = buf1.iter().map(|x| x.wrapping_add(1)).collect::>(); diff --git a/examples/token/Cargo.toml b/examples/token/Cargo.toml index 9d1140c64..5648daa49 100644 --- a/examples/token/Cargo.toml +++ b/examples/token/Cargo.toml @@ -13,6 +13,9 @@ rkyv = { version = "=0.8.0-alpha.1", default-features = false, features = ["poin rkyv_derive = "=0.8.0-alpha.1" wallet = { path = "../wallet" } +[build-dependencies] +build_scripts = { path = "../../build_scripts" } + [target.'cfg(not(target_os="mozakvm"))'.dependencies.hex] version = "0.4" @@ -33,4 +36,3 @@ path = "core_logic.rs" # The following is read by `run_examples.py` [package.metadata.mozak] example_dependents = ["wallet"] -example_program_id = "MZK-b10da48cea4c09676b8e0efcd806941465060736032bb898420d0863dca72538" diff --git a/examples/token/build.rs b/examples/token/build.rs new file mode 100644 index 000000000..8fae10da8 --- /dev/null +++ b/examples/token/build.rs @@ -0,0 +1,7 @@ +fn main() { + #[cfg(feature = "native")] + { + extern crate build_scripts; + build_scripts::self_prog_id::dump_self_prog_id("tokenbin"); + } +} diff --git a/examples/token/main_native.rs b/examples/token/main_native.rs index ef744af07..a41061e57 100644 --- a/examples/token/main_native.rs +++ b/examples/token/main_native.rs @@ -5,7 +5,8 @@ use rkyv::rancor::Panic; use token::{dispatch, MethodArgs}; fn main() { - let token_program = ProgramIdentifier::new_from_rand_seed(1); + let token_program: ProgramIdentifier = + std::fs::read_to_string("self_prog_id.txt").unwrap().into(); // We assume both wallet are the same program for now let remitter_program = ProgramIdentifier::new_from_rand_seed(2); diff --git a/examples/wallet/Cargo.toml b/examples/wallet/Cargo.toml index 811ffa730..b070a4e77 100644 --- a/examples/wallet/Cargo.toml +++ b/examples/wallet/Cargo.toml @@ -12,6 +12,9 @@ mozak-sdk = { path = "../../sdk" } rkyv = { version = "=0.8.0-alpha.1", default-features = false, features = ["pointer_width_32", "alloc"] } rkyv_derive = "=0.8.0-alpha.1" +[build-dependencies] +build_scripts = { path = "../../build_scripts" } + [target.'cfg(not(target_os="mozakvm"))'.dependencies] hex = "0.4" rand = "0.8" @@ -32,4 +35,3 @@ path = "core_logic.rs" # The following is read by `run_examples.py` [package.metadata.mozak] -example_program_id = "MZK-c51b8a31c98b9fe13065b485c9f8658c194c430843570ccac2720a3b30b47adb" diff --git a/examples/wallet/build.rs b/examples/wallet/build.rs new file mode 100644 index 000000000..8e856cb5f --- /dev/null +++ b/examples/wallet/build.rs @@ -0,0 +1,7 @@ +fn main() { + #[cfg(feature = "native")] + { + extern crate build_scripts; + build_scripts::self_prog_id::dump_self_prog_id("walletbin"); + } +} diff --git a/examples/wallet/main_native.rs b/examples/wallet/main_native.rs index d267bfee4..70cc74fcc 100644 --- a/examples/wallet/main_native.rs +++ b/examples/wallet/main_native.rs @@ -7,7 +7,8 @@ use mozak_sdk::common::types::ProgramIdentifier; use crate::core_logic::{dispatch, BlackBox, MethodArgs, PublicKey, TokenObject}; fn main() { - let wallet_program = ProgramIdentifier::new_from_rand_seed(2); + let wallet_program: ProgramIdentifier = + std::fs::read_to_string("self_prog_id.txt").unwrap().into(); let remitter_program = ProgramIdentifier::new_from_rand_seed(20); let remittee_program = ProgramIdentifier::new_from_rand_seed(21); let public_key = PublicKey::new_from_rand_seed(4); diff --git a/run_examples.py b/run_examples.py index 7524fd942..b704cf577 100644 --- a/run_examples.py +++ b/run_examples.py @@ -79,9 +79,6 @@ def test_core_only_examples(self): capabilities of the `sdk` i.e. alloc, heap, panic etc """ prove_and_verify_exceptions = {"panic"} # TODO: check why `panic` doesn't work - dummy_prog_id = ( - "MZK-0000000000000000000000000000000000000000000000000000000000000001" - ) for folder in set(list_cargo_projects("examples")): if not has_sdk_dependency_beyond_core_features( @@ -110,7 +107,7 @@ def test_core_only_examples(self): else: prove_and_verify_command = f"""cargo run --bin mozak-cli -- prove-and-verify -vvv \ examples/target/riscv32im-mozak-mozakvm-elf/release/{folder} \ - --self-prog-id {dummy_prog_id}""" + """ print( f"ZK prove and verify: {Fore.BLUE}{prove_and_verify_command}{Style.RESET_ALL}" ) @@ -184,8 +181,6 @@ def test_full_featured_examples(self): dependents = [] if "example_dependents" in extra_info.keys(): dependents = extra_info["example_dependents"] - # We assume this to be different from all dependents - prog_id = extra_info["example_program_id"] system_tape_generation_command = f"""cargo run --release --features="native,std" --bin {folder}-native --target {arch_triple}""" print( @@ -206,28 +201,19 @@ def test_full_featured_examples(self): system_tape = f"examples/{folder}/out/tape.json" programs_to_run = [ - ( - f"examples/target/riscv32im-mozak-mozakvm-elf/release/{folder}bin", - prog_id, - ) + f"examples/target/riscv32im-mozak-mozakvm-elf/release/{folder}bin", ] for dependent in dependents: - dependent_prog_id = read_toml_file( - f"examples/{dependent}/Cargo.toml" - )["package"]["metadata"]["mozak"]["example_program_id"] programs_to_run.append( - ( - f"examples/target/riscv32im-mozak-mozakvm-elf/release/{dependent}bin", - dependent_prog_id, - ) + f"examples/target/riscv32im-mozak-mozakvm-elf/release/{dependent}bin" ) - for elf, id_ in programs_to_run: + for elf in programs_to_run: print( - f"ZK prove and verify for {Style.BRIGHT}{Fore.BLUE}{folder}{Style.RESET_ALL} requires execution of {elf} with ID: {id_}", + f"ZK prove and verify for {Style.BRIGHT}{Fore.BLUE}{folder}{Style.RESET_ALL} requires execution of {elf}", ) - execution_command = f"""cargo run --bin mozak-cli -- prove-and-verify -vvv {elf} --system-tape {system_tape} --self-prog-id {id_}""" + execution_command = f"""cargo run --bin mozak-cli -- prove-and-verify -vvv {elf} --system-tape {system_tape} """ print( f"ZK prove and verify (sub-proof): {Fore.BLUE}{execution_command}{Style.RESET_ALL}", ) diff --git a/scripts/test_bundle_tx.sh b/scripts/test_bundle_tx.sh index dedd55916..61f697f7f 100755 --- a/scripts/test_bundle_tx.sh +++ b/scripts/test_bundle_tx.sh @@ -4,17 +4,17 @@ set -euo pipefail # Run native executions and build mozakvm binaries -cd examples/token && cargo run --release \ +cd examples/token && cargo build --bin tokenbin --release && + cargo run --release \ --features="native" \ --bin token-native \ - --target "$(rustc --verbose --version | grep host | awk '{ print $2; }')" && - cargo build --bin tokenbin --release + --target "$(rustc --verbose --version | grep host | awk '{ print $2; }')" -cd ../wallet && cargo run --release \ +cd ../wallet && cargo build --bin walletbin --release && + cargo run --release \ --features="native" \ --bin wallet-native \ --target "$(rustc --verbose --version | grep host | awk '{ print $2; }')" && - cargo build --bin walletbin --release # Run CLI cd ../../