(WIP) feat: add build scripts to automatically dump correct self prog id of ELF

.github/workflows/ci.yml

@@ -70,18 +70,18 @@ jobs:
           nixdo nice cargo --version
           nixdo nice cargo clippy --version
           nixdo nice cargo clippy --locked -- -D warnings
+
+      - name: cargo-clippy (mozakvm)
+        working-directory: sdk
+        run: |
+          nixdo nice cargo clippy --locked --all-features -- -D warnings
 
       - name: cargo-clippy (native, examples)
         working-directory: examples
         run: |
           ARCH_TRIPLE="$(nixdo rustc --verbose --version | grep host | awk '{ print $2; }')"
           nixdo nice cargo clippy --locked --all-features --target ${ARCH_TRIPLE} -- -D warnings
 
-      - name: cargo-clippy (mozakvm)
-        working-directory: sdk
-        run: |
-          nixdo nice cargo clippy --locked --all-features -- -D warnings
-
       - name: cargo-clippy (native)
         working-directory: sdk
         run: |

.gitignore

@@ -36,3 +36,6 @@ riscv-testdata/testdata/*.i
 # Temporary scripts
 *tmp.sh
 examples/*/out/
+
+# self program ids
+*self_prog_id.txt

Cargo.toml

@@ -17,6 +17,7 @@
 [workspace]
 exclude = ["sdk"]
 members = [
+  "build_scripts",
   "circuits",
   "cli",
   "examples-builder",

build_scripts/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+categories = ["cryptography"]
+description = "Build scripts for mozak examples"
+edition = "2021"
+keywords = ["crypto", "zero-knowledge", "vm"]
+license = "All rights reserved"
+name = "build_scripts"
+readme = "README.md"
+repository = "https://github.com/0xmozak/mozak-vm"
+version = "0.1.0"

build_scripts/src/lib.rs

@@ -0,0 +1 @@
+pub mod self_prog_id;

build_scripts/src/self_prog_id.rs

@@ -0,0 +1,46 @@
+use std::fs::File;
+use std::io;
+use std::io::Write;
+use std::path::Path;
+use std::process::Command;
+
+const CARGO_MANIFEST_DIR: &str = env!("CARGO_MANIFEST_DIR");
+/// dumps the `ProgramIdentifier` of the input ELF
+/// in a file `self_prog_id.txt` lying in same directory.
+/// Note that the `ProgramIdentifier` is represented as
+/// a string, for example,
+/// `MZK-8279ede6d0459962efe306b82d6036c08d6b9e66ad8d89d9e297c5222b3f4572`
+pub fn dump_self_prog_id(out_elf_name: &str) {
+    let cargo_manifest_dir_path = Path::new(CARGO_MANIFEST_DIR);
+    let out_dir =
+        cargo_manifest_dir_path.join("../examples/target/riscv32im-mozak-mozakvm-elf/release/");
+    let out_elf_path = out_dir.join(out_elf_name);
+    let out_elf_path_str = out_elf_path.to_str().unwrap();
+    let cli_dir = cargo_manifest_dir_path.join("../cli");
+
+    let args = vec!["run", "--release", "--", "self-prog-id", out_elf_path_str];
+
+    // execute the cli command `self-prog-id` on the ELF
+    let output = Command::new("cargo")
+        .args(args)
+        .current_dir(cli_dir)
+        .env_clear()
+        .envs(std::env::vars().filter(|x| !x.0.starts_with("CARGO_")))
+        .output()
+        .expect("can't find mozak-cli. Please run cargo build --release --bin mozak-cli from project root");
+
+    if !output.status.success() {
+        io::stdout().write_all(&output.stdout).unwrap();
+        io::stderr().write_all(&output.stderr).unwrap();
+        panic!("cargo run -- self-prog-id {out_elf_path_str} failed");
+    }
+
+    let self_prog_id = String::from_utf8(output.stdout).unwrap();
+    let self_prog_id = self_prog_id.trim();
+
+    // write `self_prog_id` to a file
+    let mut self_prog_id_file =
+        File::create("self_prog_id.txt").expect("failed to create self_prog_id.txt");
+    write!(self_prog_id_file, "{}", self_prog_id).expect("failed to write self_prog_id.txt");
+    println!("cargo:rerun-if-changed={}", out_elf_path_str);
+}

cli/Cargo.toml

@@ -29,7 +29,7 @@ clio = { version = "0.3", features = ["clap-parse"] }
 env_logger = "0.11"
 itertools = "0.13"
 log = "0.4"
-mozak-examples = { path = "../examples-builder", features = ["mozak-sort"] }
+mozak-examples = { path = "../examples-builder", features = ["mozak-sort"], optional = true }
 plonky2 = { workspace = true, default-features = false }
 plonky2_maybe_rayon = { workspace = true, default-features = false }
 rkyv = { version = "=0.8.0-alpha.1", default-features = false, features = ["pointer_width_32", "alloc"] }
@@ -40,10 +40,13 @@ starky = { workspace = true, default-features = false }
 tempfile = "3"
 
 [dev-dependencies]
+# Only as a workaround to be able to control criterion's rayon feature and optional dependency.
+criterion = { workspace = true, default-features = false }
 mozak-circuits = { path = "../circuits", features = ["test"] }
 mozak-runner = { path = "../runner", features = ["test"] }
 proptest = "1.4"
 
 [features]
+bench = ["mozak-examples"]
 default = []
-parallel = ["plonky2/parallel", "starky/parallel", "mozak-circuits/parallel"]
+parallel = ["plonky2/parallel", "starky/parallel", "mozak-circuits/parallel", "criterion/rayon"]

cli/src/lib.rs

@@ -1,3 +1,4 @@
+#[cfg(feature = "bench")]
 pub mod cli_benches;
 pub mod runner;
 #[cfg(test)]

cli/src/main.rs

@@ -26,6 +26,7 @@ use mozak_circuits::stark::utils::trace_rows_to_poly_values;
 use mozak_circuits::stark::verifier::verify_proof;
 use mozak_circuits::storage_device::generation::generate_call_tape_trace;
 use mozak_circuits::test_utils::{prove_and_verify_mozak_stark, C, D, F, S};
+#[cfg(feature = "bench")]
 use mozak_cli::cli_benches::benches::BenchArgs;
 use mozak_cli::runner::{
     deserialize_system_tape, get_self_prog_id, load_program, raw_tapes_from_system_tape,
@@ -60,8 +61,6 @@ pub struct RunArgs {
     elf: Input,
     #[arg(long)]
     system_tape: Option<Input>,
-    #[arg(long)]
-    self_prog_id: String,
 }
 
 #[derive(Clone, Debug, Args)]
@@ -72,8 +71,6 @@ pub struct ProveArgs {
     batch_proof: Option<Output>,
     #[arg(long)]
     system_tape: Option<Input>,
-    #[arg(long)]
-    self_prog_id: String,
     recursive_proof: Option<Output>,
 }
 
@@ -107,6 +104,7 @@ enum Command {
     MemoryInitHash { elf: Input },
     /// Compute the Self Program Id of the given ELF,
     SelfProgId { elf: Input },
+    #[cfg(feature = "bench")]
     /// Bench the function with given parameters
     Bench(BenchArgs),
 }
@@ -125,40 +123,31 @@ fn main() -> Result<()> {
             let program = load_program(elf)?;
             debug!("{program:?}");
         }
-        Command::Run(RunArgs {
-            elf,
-            system_tape,
-            self_prog_id,
-        }) => {
-            let raw_tapes = raw_tapes_from_system_tape(system_tape, self_prog_id.into());
+        Command::Run(RunArgs { elf, system_tape }) => {
             let program = load_program(elf).unwrap();
+            let self_prog_id = get_self_prog_id::<F, C, D>(&program, &config);
+            let raw_tapes = raw_tapes_from_system_tape(system_tape, self_prog_id);
             let state: State<F> = State::new(program.clone(), raw_tapes);
             step(&program, state)?;
         }
-        Command::ProveAndVerify(RunArgs {
-            elf,
-            system_tape,
-            self_prog_id,
-        }) => {
+        Command::ProveAndVerify(RunArgs { elf, system_tape }) => {
             let program = load_program(elf).unwrap();
-
-            let raw_tapes = raw_tapes_from_system_tape(system_tape, self_prog_id.into());
-
+            let self_prog_id = get_self_prog_id::<F, C, D>(&program, &config);
+            let raw_tapes = raw_tapes_from_system_tape(system_tape, self_prog_id);
             let state = State::new(program.clone(), raw_tapes);
             let record = step(&program, state)?;
             prove_and_verify_mozak_stark(&program, &record, &config)?;
         }
         Command::Prove(ProveArgs {
             elf,
             system_tape,
-            self_prog_id,
             mut proof,
             recursive_proof,
             batch_proof,
         }) => {
-            let raw_tapes = raw_tapes_from_system_tape(system_tape, self_prog_id.clone().into());
-            let self_program_id: ProgramIdentifier = self_prog_id.into();
             let program = load_program(elf).unwrap();
+            let self_prog_id = get_self_prog_id::<F, C, D>(&program, &config);
+            let raw_tapes = raw_tapes_from_system_tape(system_tape, self_prog_id);
             let state = State::new(program.clone(), raw_tapes);
             let record = step(&program, state)?;
             let stark = if cli.debug {
@@ -216,7 +205,7 @@ fn main() -> Result<()> {
                 let public_inputs: VMRecursiveProofPublicInputs<F> = public_inputs_array.into();
                 assert_eq!(
                     public_inputs.program_hash_as_bytes.to_vec(),
-                    self_program_id
+                    self_prog_id
                         .inner()
                         .into_iter()
                         .map(F::from_canonical_u8)
@@ -457,6 +446,7 @@ fn main() -> Result<()> {
             let self_prog_id = get_self_prog_id::<F, C, D>(&program, &config);
             println!("{self_prog_id:?}");
         }
+        #[cfg(feature = "bench")]
         Command::Bench(bench) => {
             let time_taken = bench.bench()?.as_secs_f64();
             println!("{time_taken}");

cli/src/tests/integration_test.rs

@@ -20,14 +20,6 @@ fn test_prove_and_verify_recursive_proof_command() {
     // Create mock IO tape files
     fs::write(system_tape, b"").expect("Failed to create system tape file");
 
-    // Get self_prog_id
-    let output = Command::new("cargo")
-        .args(["run", "--", "self-prog-id", elf_file])
-        .output()
-        .expect("Failed to execute self-prog-id command");
-    let mut self_prog_id = String::from_utf8(output.stdout).unwrap();
-    self_prog_id = self_prog_id.trim().to_string();
-
     // Execute the `prove` command
     let output = Command::new("cargo")
         .args([
@@ -36,8 +28,6 @@ fn test_prove_and_verify_recursive_proof_command() {
             "prove",
             elf_file,
             &proof_file.to_string_lossy(),
-            "--self-prog-id",
-            &self_prog_id,
             &recursive_proof_file.to_string_lossy(),
         ])
         .output()

examples/inputtape/Cargo.toml

@@ -12,6 +12,9 @@ mozak-sdk = { path = "../../sdk" }
 rkyv = { version = "=0.8.0-alpha.1", default-features = false, features = ["pointer_width_32", "alloc"] }
 rkyv_derive = "=0.8.0-alpha.1"
 
+[target.'cfg(not(target_os="mozakvm"))'.build-dependencies]
+build_scripts = { path = "../../build_scripts" }
+
 [target.'cfg(not(target_os="mozakvm"))'.dependencies.hex]
 version = "0.4"
 
@@ -30,4 +33,3 @@ path = "core_logic.rs"
 
 # The following is read by `run_examples.py`
 [package.metadata.mozak]
-example_program_id = "MZK-5de9bc1bb4cb7a9f9cf991a32230d014b44274ea5464a352f089a8cccc486915"

examples/inputtape/build.rs

@@ -0,0 +1,7 @@
+fn main() {
+    #[cfg(feature = "native")]
+    {
+        extern crate build_scripts;
+        build_scripts::self_prog_id::dump_self_prog_id("inputtapebin");
+    }
+}

examples/inputtape/main_native.rs

@@ -6,7 +6,8 @@ use mozak_sdk::common::types::{Poseidon2Hash, ProgramIdentifier};
 use crate::core_logic::{dispatch, MethodArgs};
 
 fn main() {
-    let token_program = ProgramIdentifier::new_from_rand_seed(3);
+    let token_program: ProgramIdentifier =
+        std::fs::read_to_string("self_prog_id.txt").unwrap().into();
 
     let buf1 = Poseidon2Hash::new_from_rand_seed(2).inner();
     let buf2 = buf1.iter().map(|x| x.wrapping_add(1)).collect::<Vec<u8>>();

examples/token/Cargo.toml

@@ -13,6 +13,9 @@ rkyv = { version = "=0.8.0-alpha.1", default-features = false, features = ["poin
 rkyv_derive = "=0.8.0-alpha.1"
 wallet = { path = "../wallet" }
 
+[build-dependencies]
+build_scripts = { path = "../../build_scripts" }
+
 [target.'cfg(not(target_os="mozakvm"))'.dependencies.hex]
 version = "0.4"
 
@@ -33,4 +36,3 @@ path = "core_logic.rs"
 # The following is read by `run_examples.py`
 [package.metadata.mozak]
 example_dependents = ["wallet"]
-example_program_id = "MZK-b10da48cea4c09676b8e0efcd806941465060736032bb898420d0863dca72538"

examples/token/build.rs

@@ -0,0 +1,7 @@
+fn main() {
+    #[cfg(feature = "native")]
+    {
+        extern crate build_scripts;
+        build_scripts::self_prog_id::dump_self_prog_id("tokenbin");
+    }
+}

examples/token/main_native.rs

@@ -5,7 +5,8 @@ use rkyv::rancor::Panic;
 use token::{dispatch, MethodArgs};
 
 fn main() {
-    let token_program = ProgramIdentifier::new_from_rand_seed(1);
+    let token_program: ProgramIdentifier =
+        std::fs::read_to_string("self_prog_id.txt").unwrap().into();
 
     // We assume both wallet are the same program for now
     let remitter_program = ProgramIdentifier::new_from_rand_seed(2);

examples/wallet/Cargo.toml

@@ -12,6 +12,9 @@ mozak-sdk = { path = "../../sdk" }
 rkyv = { version = "=0.8.0-alpha.1", default-features = false, features = ["pointer_width_32", "alloc"] }
 rkyv_derive = "=0.8.0-alpha.1"
 
+[build-dependencies]
+build_scripts = { path = "../../build_scripts" }
+
 [target.'cfg(not(target_os="mozakvm"))'.dependencies]
 hex = "0.4"
 rand = "0.8"
@@ -32,4 +35,3 @@ path = "core_logic.rs"
 
 # The following is read by `run_examples.py`
 [package.metadata.mozak]
-example_program_id = "MZK-c51b8a31c98b9fe13065b485c9f8658c194c430843570ccac2720a3b30b47adb"

examples/wallet/build.rs

@@ -0,0 +1,7 @@
+fn main() {
+    #[cfg(feature = "native")]
+    {
+        extern crate build_scripts;
+        build_scripts::self_prog_id::dump_self_prog_id("walletbin");
+    }
+}