diff --git a/doc/src/design/ir.md b/doc/src/design/ir.md index 6815d145..5d3ca96f 100644 --- a/doc/src/design/ir.md +++ b/doc/src/design/ir.md @@ -204,7 +204,6 @@ containing the correctly serialized transactions `v15` and `v30`. | `AddWitness` | Adds an item to the witness stack. | | `EndWitnessStack`| Finishes building the witness stack. | | **Compact Block** | **Construct a compact block.**| -| `BuildCompactBlock` | Builds a compact block. | | **Blocktxn building** | **Construct a BIP152 blocktxn message**| | `BeginBuildBlockTxn` | Begins building a blocktxn message after sending a compact block. | | `AddTxToBlockTxn` | Adds a transaction to the blocktxn message. | diff --git a/fuzzamoto-ir/src/builder.rs b/fuzzamoto-ir/src/builder.rs index b77e3db8..c2f41b42 100644 --- a/fuzzamoto-ir/src/builder.rs +++ b/fuzzamoto-ir/src/builder.rs @@ -292,6 +292,7 @@ impl ProgramBuilder { Ok(()) } + pub fn append_program_without_threshold( &mut self, program: Program, @@ -354,6 +355,68 @@ impl ProgramBuilder { } } + /// Given a Block variable index, return the `ConstBlockTransactions` variable index and the + /// list of `ConstTx` variable indices that were added to that block. Both are guaranteed to + /// belong to the same block, making them safe to pass to `AddPrefillTx`. + #[must_use] + pub fn get_block_vars(&self, block_var_index: usize) -> Option<(usize, Vec)> { + // Walk instructions tracking the running variable index to find the BuildBlock + // instruction whose Block output sits at block_var_index. + // BuildBlock outputs: Header (offset 0), Block (offset 1), CoinbaseTx (offset 2). + let mut var_count = 0; + for instruction in &self.instructions { + if matches!(instruction.operation, Operation::BuildBlock) { + // offset 1 is the Block variable + if var_count + 1 == block_var_index { + // inputs[4] is the ConstBlockTransactions variable + let block_transactions_idx = instruction.inputs[4]; + + // Find the MutBlockTransactions that was closed into block_transactions_idx + // by tracing the EndBlockTransactions instruction. + let tx_var_indices = + self.get_tx_var_indices_for_block_transactions(block_transactions_idx); + + return Some((block_transactions_idx, tx_var_indices)); + } + } + var_count += + instruction.operation.num_outputs() + instruction.operation.num_inner_outputs(); + } + None + } + + /// Given a `ConstBlockTransactions` variable index, return the `ConstTx` variable indices that + /// were added to it via `AddTx` instructions. + #[must_use] + fn get_tx_var_indices_for_block_transactions( + &self, + block_transactions_idx: usize, + ) -> Vec { + // Find the EndBlockTransactions instruction whose output is block_transactions_idx, + // then collect all AddTx instructions that fed into the same MutBlockTransactions. + let mut var_count = 0; + for instruction in &self.instructions { + if matches!(instruction.operation, Operation::EndBlockTransactions) + && var_count == block_transactions_idx + { + // inputs[0] is the MutBlockTransactions being closed + let mut_block_transactions_idx = instruction.inputs[0]; + return self + .instructions + .iter() + .filter(|i| { + matches!(i.operation, Operation::AddTx) + && i.inputs[0] == mut_block_transactions_idx + }) + .map(|i| i.inputs[1]) // inputs[1] is the ConstTx var index + .collect(); + } + var_count += + instruction.operation.num_outputs() + instruction.operation.num_inner_outputs(); + } + Vec::new() + } + /// Get the nearest (searched in reverse) available (in the current scope) variable of a given /// type #[must_use] diff --git a/fuzzamoto-ir/src/compiler.rs b/fuzzamoto-ir/src/compiler.rs index ef673713..c22e428a 100644 --- a/fuzzamoto-ir/src/compiler.rs +++ b/fuzzamoto-ir/src/compiler.rs @@ -82,8 +82,8 @@ pub type ConnectionId = usize; #[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] pub struct CompiledMetadata { - // Map from blockhash to (block variable index, list of transaction variable indices) - block_tx_var_map: HashMap)>, + // Map from blockhash to (header_var, block_var, block_transactions_var, tx_var_indices) + block_tx_var_map: HashMap)>, // Map from connection ids to connection variable indices. connection_map: HashMap, // List of instruction indices that correspond to actions in the compiled program (does not include probe operation) @@ -112,15 +112,28 @@ impl CompiledMetadata { } } - // Get the block variable index and list of transaction variable indices for a given block hash + /// Get the header var index, block var index, `block_transactions` var index, and list of + /// transaction variable indices for a given block hash. #[must_use] pub fn block_variables( &self, block_hash: &bitcoin::BlockHash, - ) -> Option<(usize, usize, &[usize])> { + ) -> Option<(usize, usize, usize, &[usize])> { + self.block_tx_var_map.get(block_hash).map( + |(header_var, block_var, block_txs_var, tx_vars)| { + (*header_var, *block_var, *block_txs_var, tx_vars.as_slice()) + }, + ) + } + + /// Look up a block by its block variable index, returning the `block_transactions` var index + /// and the list of transaction variable indices that belong to it. + #[must_use] + pub fn block_vars_by_block_index(&self, block_var_index: usize) -> Option<(usize, &[usize])> { self.block_tx_var_map - .get(block_hash) - .map(|(header_var, block_var, tx_vars)| (*header_var, *block_var, tx_vars.as_slice())) + .values() + .find(|(_, block_var, _, _)| *block_var == block_var_index) + .map(|(_, _, block_txs_var, tx_vars)| (*block_txs_var, tx_vars.as_slice())) } // Get the list of instruction indices that correspond to actions in the compiled program @@ -276,6 +289,11 @@ struct BlockTransactions { var_indices: Vec, } +#[derive(Clone, Debug)] +struct PrefillTransactions { + indices: Vec, +} + #[derive(Clone, Debug)] struct AddrList { entries: Vec<(u32, Address)>, @@ -364,10 +382,6 @@ impl Compiler { self.handle_build_taproot_tree(instruction)?; } - Operation::BuildCompactBlock => { - self.handle_compact_block_building_operations(instruction)?; - } - Operation::BeginBlockTransactions | Operation::AddTx | Operation::EndBlockTransactions @@ -459,6 +473,13 @@ impl Compiler { self.handle_new_connection_operations(instruction)?; } + Operation::BeginPrefillTransactions + | Operation::AddPrefillTx + | Operation::EndPrefillTransactions + | Operation::BuildCompactBlockWithPrefill => { + self.handle_prefill_building_operations(instruction)?; + } + Operation::SendRawMessage | Operation::SendTxNoWit | Operation::SendTx @@ -729,11 +750,7 @@ impl Compiler { let octets: [u8; 16] = payload.as_slice().try_into().expect("length checked"); AddrV2::Cjdns(Ipv6Addr::from(octets)) } - AddrNetwork::Yggdrasil => { - // `bitcoin::p2p::address::AddrV2` has no Yggdrasil variant; preserve the - // network ID via the generic `Unknown` encoding. - AddrV2::Unknown(AddrNetwork::Yggdrasil.id(), payload.clone()) - } + AddrNetwork::Yggdrasil => AddrV2::Unknown(AddrNetwork::Yggdrasil.id(), payload.clone()), AddrNetwork::Unknown(id) => AddrV2::Unknown(id, payload.clone()), }; @@ -837,30 +854,6 @@ impl Compiler { Ok(()) } - fn handle_compact_block_building_operations( - &mut self, - instruction: &Instruction, - ) -> Result<(), CompilerError> { - match &instruction.operation { - Operation::BuildCompactBlock => { - let block = self.get_input::(&instruction.inputs, 0)?; - let nonce = self.get_input::(&instruction.inputs, 1)?; - - // TODO: put other txs than coinbase tx - let prefill = &[]; - let header_and_shortids = HeaderAndShortIds::from_block(block, *nonce, 2, prefill) - .expect("from_block should never fail"); - self.append_variable(CmpctBlock { - compact_block: header_and_shortids, - }); - } - _ => unreachable!( - "Non-compactblock-building operation passed to handle_compact_block_building_operations" - ), - } - Ok(()) - } - fn handle_taproot_conversions( &mut self, instruction: &Instruction, @@ -1785,6 +1778,65 @@ impl Compiler { } } + fn handle_prefill_building_operations( + &mut self, + instruction: &Instruction, + ) -> Result<(), CompilerError> { + match &instruction.operation { + Operation::BeginPrefillTransactions => { + self.append_variable(PrefillTransactions { + indices: Vec::new(), + }); + } + Operation::AddPrefillTx => { + let block_txs = self + .get_input::(&instruction.inputs, 1)? + .clone(); + let tx_to_prefill = self.get_input::(&instruction.inputs, 2)?; + let tx_to_prefill_id = tx_to_prefill.id; + + // Find the position of this tx within the block's transaction list. + // If the tx is not part of this block, skip it silently. + let Some(position) = block_txs.txs.iter().position(|t| t.id == tx_to_prefill_id) + else { + return Ok(()); + }; + + let prefill_var = + self.get_input_mut::(&instruction.inputs, 0)?; + if !prefill_var.indices.contains(&position) { + prefill_var.indices.push(position); + } + } + Operation::EndPrefillTransactions => { + let prefill_var = self + .get_input::(&instruction.inputs, 0)? + .clone(); + self.append_variable(prefill_var); + } + Operation::BuildCompactBlockWithPrefill => { + let block = self.get_input::(&instruction.inputs, 0)?; + let nonce = self.get_input::(&instruction.inputs, 1)?; + let prefill_var = self.get_input::(&instruction.inputs, 2)?; + + let mut prefill = prefill_var.indices.clone(); + // Filter out-of-bounds indices defensively, same as existing BuildCompactBlock. + prefill.retain(|&i| i > 0 && i < block.txdata.len()); + prefill.sort_unstable(); + prefill.dedup(); + + let header_and_shortids = HeaderAndShortIds::from_block(block, *nonce, 2, &prefill) + .expect("from_block should never fail"); + + self.append_variable(CmpctBlock { + compact_block: header_and_shortids, + }); + } + _ => unreachable!("Non-prefill operation passed to handle_prefill_building_operations"), + } + Ok(()) + } + fn handle_new_connection_operations( &mut self, instruction: &Instruction, @@ -1919,6 +1971,10 @@ impl Compiler { .get_input::(&instruction.inputs, 4)? .clone(); + // Capture the ConstBlockTransactions variable index before we start + // appending new variables, so we can store it in the metadata. + let block_transactions_var_index = instruction.inputs[4]; + coinbase_tx_var.tx.tx.input[0].script_sig = ScriptBuf::builder() .push_int(i64::from(header_var.height + 1)) .push_int(0xFFFF_FFFF) @@ -1991,7 +2047,9 @@ impl Compiler { version: block.header.version.to_consensus(), }); - // Record the block variable index and transaction variable indices in metadata + // Record the block variable index, block_transactions variable index, and + // transaction variable indices in metadata so generators can look them up + // by block variable index later. let block_hash = block.header.block_hash(); let block_var_index = self.variables.len(); self.append_variable(block); @@ -2002,6 +2060,7 @@ impl Compiler { .or_insert(( header_var_index, block_var_index, + block_transactions_var_index, block_transactions_var.var_indices.clone(), )); diff --git a/fuzzamoto-ir/src/generators/compact_block.rs b/fuzzamoto-ir/src/generators/compact_block.rs index 6dae6279..1c8a0084 100644 --- a/fuzzamoto-ir/src/generators/compact_block.rs +++ b/fuzzamoto-ir/src/generators/compact_block.rs @@ -3,7 +3,7 @@ use crate::{ Instruction, Operation, PerTestcaseMetadata, Variable, generators::{Generator, ProgramBuilder}, }; -use rand::{Rng, RngCore}; +use rand::{Rng, RngCore, seq::SliceRandom}; /// `CompactBlockGenerator` generates a new `cmpctblock` message. #[derive(Debug, Default)] @@ -16,31 +16,79 @@ impl Generator for CompactBlockGenerator { rng: &mut R, _meta: Option<&PerTestcaseMetadata>, ) -> GeneratorResult { - // choose a block upon which we build the compact block + // Choose a block upon which we build the compact block let Some(block) = builder.get_random_variable(rng, &Variable::Block) else { return Err(GeneratorError::MissingVariables); }; + let Some((block_transactions_idx, tx_var_indices)) = builder.get_block_vars(block.index) + else { + return Err(GeneratorError::MissingVariables); + }; + + // Collect into an owned vec before the loop since we need to call + // builder.append() which takes &mut self. + let tx_var_indices: Vec = tx_var_indices.clone(); + let num_block_txs = tx_var_indices.len(); + let connection_var = builder.get_or_create_random_connection(rng); - let nonce = rng.gen_range(0..u64::MAX); let nonce_var = builder .append(Instruction { inputs: vec![], - operation: Operation::LoadNonce(nonce), + operation: Operation::LoadNonce(rng.gen_range(0..u64::MAX)), + }) + .expect("LoadNonce should always succeed") + .pop() + .expect("LoadNonce should always produce a var"); + + // Build the prefill list dynamically from actual transactions in the block. + let prefill_list = builder + .append(Instruction { + inputs: vec![], + operation: Operation::BeginPrefillTransactions, + }) + .expect("BeginPrefillTransactions should always succeed") + .pop() + .expect("BeginPrefillTransactions should always produce a var"); + + if num_block_txs > 0 { + // Pick a random number of transactions to prefill, bounded by the + // actual number of transactions in the block. + let num_prefill = rng.gen_range(0..=num_block_txs); + + // Shuffle a copy of the known-good tx var indices and take the + // first num_prefill — every tx is guaranteed to belong to this block. + let mut shuffled = tx_var_indices.clone(); + shuffled.shuffle(rng); + + for tx_idx in shuffled.into_iter().take(num_prefill) { + builder + .append(Instruction { + inputs: vec![prefill_list.index, block_transactions_idx, tx_idx], + operation: Operation::AddPrefillTx, + }) + .expect("AddPrefillTx should always succeed"); + } + } + + let const_prefill = builder + .append(Instruction { + inputs: vec![prefill_list.index], + operation: Operation::EndPrefillTransactions, }) - .expect("Inserting BeginBuildCompactBlock should always succeed") + .expect("EndPrefillTransactions should always succeed") .pop() - .expect("BeginBuildCompactBlock should always produce a var"); + .expect("EndPrefillTransactions should always produce a var"); let cmpct_block = builder .append(Instruction { - inputs: vec![block.index, nonce_var.index], - operation: Operation::BuildCompactBlock, + inputs: vec![block.index, nonce_var.index, const_prefill.index], + operation: Operation::BuildCompactBlockWithPrefill, }) - .expect("Inserting BuildCompactBlock should always succeed") + .expect("BuildCompactBlockWithPrefill should always succeed") .pop() - .expect("BuildCompactBlock should always produce a var"); + .expect("BuildCompactBlockWithPrefill should always produce a var"); builder .append(Instruction { diff --git a/fuzzamoto-ir/src/instruction.rs b/fuzzamoto-ir/src/instruction.rs index 6c975a56..709b3cde 100644 --- a/fuzzamoto-ir/src/instruction.rs +++ b/fuzzamoto-ir/src/instruction.rs @@ -93,6 +93,10 @@ impl Instruction { | Operation::AddConnection | Operation::AddConnectionWithHandshake { .. } | Operation::LoadHandshakeOpts { .. } + | Operation::BeginPrefillTransactions + | Operation::AddPrefillTx + | Operation::EndPrefillTransactions + | Operation::BuildCompactBlockWithPrefill | Operation::BuildPayToWitnessScriptHash | Operation::BuildPayToScriptHash | Operation::BuildRawScripts @@ -180,7 +184,6 @@ impl Instruction { | Operation::BeginBlockTransactions | Operation::BeginBuildFilterLoad | Operation::EndBuildFilterLoad - | Operation::BuildCompactBlock | Operation::BeginBuildCoinbaseTx | Operation::EndBuildCoinbaseTx | Operation::BeginBuildCoinbaseTxOutputs diff --git a/fuzzamoto-ir/src/operation.rs b/fuzzamoto-ir/src/operation.rs index 8f3b4e0a..477a2e6e 100644 --- a/fuzzamoto-ir/src/operation.rs +++ b/fuzzamoto-ir/src/operation.rs @@ -115,8 +115,11 @@ pub enum Operation { BuildPayToAnchor, BuildPayToTaproot, - // cmpctblock building operations - BuildCompactBlock, + // Dynamic prefill building operations for compact blocks + BeginPrefillTransactions, + AddPrefillTx, + EndPrefillTransactions, + BuildCompactBlockWithPrefill, // filterload building operations BeginBuildFilterLoad, @@ -368,7 +371,10 @@ impl fmt::Display for Operation { Operation::EndWitnessStack => write!(f, "EndWitnessStack"), Operation::AddWitness => write!(f, "AddWitness"), - Operation::BuildCompactBlock => write!(f, "BuildCompactBlock"), + Operation::BeginPrefillTransactions => write!(f, "BeginPrefillTransactions"), + Operation::AddPrefillTx => write!(f, "AddPrefillTx"), + Operation::EndPrefillTransactions => write!(f, "EndPrefillTransactions"), + Operation::BuildCompactBlockWithPrefill => write!(f, "BuildCompactBlockWithPrefill"), Operation::BeginBuildCoinbaseTx => write!(f, "BeginBuildCoinbaseTx"), Operation::EndBuildCoinbaseTx => write!(f, "EndBuildCoinbaseTx"), @@ -468,6 +474,7 @@ impl Operation { | Operation::AddTx | Operation::AddAddr | Operation::AddAddrV2 + | Operation::AddPrefillTx if index == 0) } @@ -485,7 +492,8 @@ impl Operation { | Operation::BeginBuildFilterLoad | Operation::BeginBuildCoinbaseTx | Operation::BeginBuildBlockTxn - | Operation::BeginBuildCoinbaseTxOutputs => true, + | Operation::BeginBuildCoinbaseTxOutputs + | Operation::BeginPrefillTransactions => true, // Exhaustive match to fail when new ops are added Operation::Nop { .. } | Operation::LoadBytes(_) @@ -531,7 +539,9 @@ impl Operation { | Operation::AddTxoToFilter | Operation::BuildFilterAddFromTx | Operation::BuildFilterAddFromTxo - | Operation::BuildCompactBlock + | Operation::AddPrefillTx + | Operation::EndPrefillTransactions + | Operation::BuildCompactBlockWithPrefill | Operation::LoadNonce(..) | Operation::AddTxToBlockTxn | Operation::EndBuildBlockTxn @@ -627,6 +637,10 @@ impl Operation { Operation::EndBuildCoinbaseTxOutputs ) | (Operation::BeginBuildBlockTxn, Operation::EndBuildBlockTxn) + | ( + Operation::BeginPrefillTransactions, + Operation::EndPrefillTransactions + ) ) } @@ -644,7 +658,8 @@ impl Operation { | Operation::EndBuildFilterLoad | Operation::EndBuildCoinbaseTx | Operation::EndBuildBlockTxn - | Operation::EndBuildCoinbaseTxOutputs => true, + | Operation::EndBuildCoinbaseTxOutputs + | Operation::EndPrefillTransactions => true, // Exhaustive match to fail when new ops are added Operation::Nop { .. } | Operation::LoadBytes(_) @@ -733,7 +748,9 @@ impl Operation { | Operation::AddTxoToFilter | Operation::BuildFilterAddFromTx | Operation::BuildFilterAddFromTxo - | Operation::BuildCompactBlock + | Operation::BeginPrefillTransactions + | Operation::AddPrefillTx + | Operation::BuildCompactBlockWithPrefill | Operation::SendFilterLoad | Operation::SendFilterAdd | Operation::SendFilterClear @@ -853,7 +870,10 @@ impl Operation { Operation::AddTxoToFilter => vec![], Operation::EndBuildFilterLoad => vec![Variable::ConstFilterLoad], - Operation::BuildCompactBlock => vec![Variable::CompactBlock], + Operation::BeginPrefillTransactions => vec![], + Operation::AddPrefillTx => vec![], + Operation::EndPrefillTransactions => vec![Variable::ConstPrefillTransactions], + Operation::BuildCompactBlockWithPrefill => vec![Variable::CompactBlock], Operation::BuildFilterAddFromTx => vec![Variable::FilterAdd], Operation::BuildFilterAddFromTxo => vec![Variable::FilterAdd], @@ -1050,7 +1070,17 @@ impl Operation { Operation::BuildFilterAddFromTx => vec![Variable::ConstTx], Operation::BuildFilterAddFromTxo => vec![Variable::Txo], - Operation::BuildCompactBlock => vec![Variable::Block, Variable::Nonce], + Operation::AddPrefillTx => vec![ + Variable::MutPrefillTransactions, + Variable::ConstBlockTransactions, + Variable::ConstTx, + ], + Operation::EndPrefillTransactions => vec![Variable::MutPrefillTransactions], + Operation::BuildCompactBlockWithPrefill => vec![ + Variable::Block, + Variable::Nonce, + Variable::ConstPrefillTransactions, + ], Operation::SendFilterLoad => vec![Variable::Connection, Variable::ConstFilterLoad], Operation::SendFilterAdd => vec![Variable::Connection, Variable::FilterAdd], @@ -1094,6 +1124,7 @@ impl Operation { | Operation::BeginBuildAddrListV2 | Operation::BeginBlockTransactions | Operation::BeginWitnessStack + | Operation::BeginPrefillTransactions | Operation::BuildPayToAnchor | Operation::Probe => vec![], } @@ -1115,6 +1146,7 @@ impl Operation { Operation::BeginBuildCoinbaseTx => vec![Variable::MutTx], Operation::BeginBuildCoinbaseTxOutputs => vec![Variable::MutTxOutputs], Operation::BeginBuildBlockTxn => vec![Variable::MutBlockTxn], + Operation::BeginPrefillTransactions => vec![Variable::MutPrefillTransactions], Operation::Nop { outputs: _, inner_outputs, @@ -1165,7 +1197,9 @@ impl Operation { | Operation::LoadFilterLoad { .. } | Operation::LoadFilterAdd { .. } | Operation::LoadNonce(..) - | Operation::BuildCompactBlock + | Operation::AddPrefillTx + | Operation::EndPrefillTransactions + | Operation::BuildCompactBlockWithPrefill | Operation::TaprootScriptsUseAnnex | Operation::TaprootTxoUseAnnex | Operation::EndBuildTx diff --git a/fuzzamoto-ir/src/variable.rs b/fuzzamoto-ir/src/variable.rs index 0800ecf6..98f7dc5b 100644 --- a/fuzzamoto-ir/src/variable.rs +++ b/fuzzamoto-ir/src/variable.rs @@ -52,6 +52,9 @@ pub enum Variable { Block, Header, + ConstPrefillTransactions, + MutPrefillTransactions, + BlockVersion, MutFilterLoad, // Mutable filter (under construction) diff --git a/fuzzamoto-scenarios/bin/ir.rs b/fuzzamoto-scenarios/bin/ir.rs index 85689536..7fa6311c 100644 --- a/fuzzamoto-scenarios/bin/ir.rs +++ b/fuzzamoto-scenarios/bin/ir.rs @@ -95,7 +95,7 @@ fn probe_result_mapper( }; }; - let Some((_, block_var, tx_vars)) = metadata.block_variables(&request.block_hash) + let Some((_, block_var, _, tx_vars)) = metadata.block_variables(&request.block_hash) else { return ProbeResult::Failure { command: s.clone(), @@ -468,7 +468,7 @@ pub fn probe_recent_block_hashes( let mut result = Vec::new(); for (height, hash) in &hashes { - if let Some((header, _, _)) = meta.block_variables(hash) + if let Some((header, _, _, _)) = meta.block_variables(hash) && let Some(inst) = meta.variable_indices().get(header) { result.push(RecentBlock {