diff --git a/Cargo.lock b/Cargo.lock index a5e7f0bd5..88bcb56c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -680,6 +680,18 @@ dependencies = [ "typenum", ] +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "blake3" version = "1.8.2" @@ -836,6 +848,28 @@ dependencies = [ "serde", ] +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta 0.1.4", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "bytemuck" version = "1.23.1" @@ -1823,7 +1857,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6503af7917fea18ffef8f7e8553fb8dff89e2e6837e94e09dd7fb069c82d62c" dependencies = [ "bytes", - "rkyv", + "rkyv 0.8.11", "serde", "simdutf8", ] @@ -1983,6 +2017,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.1.31" @@ -3734,7 +3774,7 @@ dependencies = [ "serde", "solana-keypair", "solana-pubkey", - "strum", + "strum 0.24.1", "thiserror 1.0.69", "toml 0.8.23", "url 2.5.4", @@ -3782,8 +3822,8 @@ dependencies = [ [[package]] name = "magicblock-delegation-program" -version = "1.1.0" -source = "git+https://github.com/magicblock-labs/delegation-program.git?rev=aa1de56d90c#aa1de56d90c8a242377accd59899f272f0131f8c" +version = "1.1.2" +source = "git+https://github.com/magicblock-labs/delegation-program.git?rev=e8d03936#e8d039369ac1149e899ea94f31e0f9cc4e600a38" dependencies = [ "bincode", "borsh 1.5.7", @@ -3794,9 +3834,12 @@ dependencies = [ "pinocchio-log", "pinocchio-pubkey", "pinocchio-system", + "rkyv 0.7.45", "solana-curve25519", "solana-program", "solana-security-txt", + "static_assertions", + "strum 0.27.2", "thiserror 1.0.69", ] @@ -5057,13 +5100,33 @@ dependencies = [ "autotools", ] +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive 0.1.4", +] + [[package]] name = "ptr_meta" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe9e76f66d3f9606f44e45598d155cb13ecf09f4a28199e48daf8c8fc937ea90" dependencies = [ - "ptr_meta_derive", + "ptr_meta_derive 0.3.0", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -5190,13 +5253,19 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rancor" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caf5f7161924b9d1cea0e4cabc97c372cea92b5f927fc13c6bca67157a0ad947" dependencies = [ - "ptr_meta", + "ptr_meta 0.3.0", ] [[package]] @@ -5466,6 +5535,15 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] + [[package]] name = "rend" version = "0.5.2" @@ -5548,6 +5626,24 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.7.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "ptr_meta 0.1.4", + "rend 0.4.2", + "rkyv_derive 0.7.45", + "seahash", + "tinyvec", + "uuid", +] + [[package]] name = "rkyv" version = "0.8.11" @@ -5558,14 +5654,25 @@ dependencies = [ "hashbrown 0.15.4", "indexmap 2.10.0", "munge", - "ptr_meta", + "ptr_meta 0.3.0", "rancor", - "rend", - "rkyv_derive", + "rend 0.5.2", + "rkyv_derive 0.8.11", "tinyvec", "uuid", ] +[[package]] +name = "rkyv_derive" +version = "0.7.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "rkyv_derive" version = "0.8.11" @@ -5871,6 +5978,12 @@ version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "security-framework" version = "2.11.1" @@ -7503,8 +7616,8 @@ dependencies = [ "spl-token", "spl-token-2022 7.0.0", "static_assertions", - "strum", - "strum_macros", + "strum 0.24.1", + "strum_macros 0.24.3", "tar", "tempfile", "thiserror 2.0.12", @@ -8501,8 +8614,8 @@ dependencies = [ "solana-vote", "solana-vote-program", "static_assertions", - "strum", - "strum_macros", + "strum 0.24.1", + "strum_macros 0.24.3", "symlink", "tar", "tempfile", @@ -10166,7 +10279,16 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" dependencies = [ - "strum_macros", + "strum_macros 0.24.3", +] + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", ] [[package]] @@ -10182,6 +10304,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "subtle" version = "2.6.1" @@ -10275,6 +10409,12 @@ dependencies = [ "unicode-width 0.1.14", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tar" version = "0.4.44" @@ -11816,6 +11956,15 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "x509-parser" version = "0.14.0" diff --git a/Cargo.toml b/Cargo.toml index b5e080e81..60fb7a778 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,10 +108,10 @@ magicblock-config = { path = "./magicblock-config" } magicblock-config-helpers = { path = "./magicblock-config-helpers" } magicblock-config-macro = { path = "./magicblock-config-macro" } magicblock-core = { path = "./magicblock-core" } -magicblock-delegation-program = { git = "https://github.com/magicblock-labs/delegation-program.git", rev = "aa1de56d90c", features = [ +magicblock-aperture = { path = "./magicblock-aperture" } +magicblock-delegation-program = { git = "https://github.com/magicblock-labs/delegation-program.git", rev = "e8d03936", features = [ "no-entrypoint", ] } -magicblock-aperture = { path = "./magicblock-aperture" } magicblock-geyser-plugin = { path = "./magicblock-geyser-plugin" } magicblock-ledger = { path = "./magicblock-ledger" } magicblock-metrics = { path = "./magicblock-metrics" } diff --git a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs index 6fce85908..432320c67 100644 --- a/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs +++ b/magicblock-committor-service/src/intent_execution_manager/intent_execution_engine.rs @@ -344,6 +344,8 @@ mod tests { use async_trait::async_trait; use magicblock_program::magic_scheduled_base_intent::ScheduledBaseIntent; + use magicblock_rpc_client::MagicBlockRpcClientResult; + use solana_account::Account; use solana_pubkey::{pubkey, Pubkey}; use solana_sdk::{signature::Signature, signer::SignerError}; use tokio::{sync::mpsc, time::sleep}; @@ -798,5 +800,12 @@ mod tests { } fn reset(&self, _: ResetType) {} + + async fn get_base_account( + &self, + _pubkey: &Pubkey, + ) -> MagicBlockRpcClientResult> { + Ok(None) // AccountNotFound + } } } diff --git a/magicblock-committor-service/src/intent_executor/mod.rs b/magicblock-committor-service/src/intent_executor/mod.rs index fa85cfee1..0f7a59fea 100644 --- a/magicblock-committor-service/src/intent_executor/mod.rs +++ b/magicblock-committor-service/src/intent_executor/mod.rs @@ -4,6 +4,9 @@ pub mod single_stage_executor; pub mod task_info_fetcher; pub mod two_stage_executor; +#[cfg(any(test, feature = "dev-context-only-utils"))] +mod null_task_info_fetcher; + use std::{ops::ControlFlow, sync::Arc, time::Duration}; use async_trait::async_trait; @@ -22,6 +25,8 @@ use magicblock_rpc_client::{ MagicBlockRpcClientError, MagicBlockSendTransactionConfig, MagicBlockSendTransactionOutcome, MagicblockRpcClient, }; +#[cfg(any(test, feature = "dev-context-only-utils"))] +pub use null_task_info_fetcher::*; use solana_pubkey::Pubkey; use solana_rpc_client_api::config::RpcTransactionConfig; use solana_sdk::{ @@ -778,6 +783,8 @@ where mod tests { use std::{collections::HashMap, sync::Arc}; + use magicblock_rpc_client::MagicBlockRpcClientResult; + use solana_account::Account; use solana_pubkey::Pubkey; use crate::{ @@ -815,6 +822,13 @@ mod tests { } fn reset(&self, _: ResetType) {} + + async fn get_base_account( + &self, + _pubkey: &Pubkey, + ) -> MagicBlockRpcClientResult> { + Ok(None) // AccountNotFound + } } #[tokio::test] diff --git a/magicblock-committor-service/src/intent_executor/null_task_info_fetcher.rs b/magicblock-committor-service/src/intent_executor/null_task_info_fetcher.rs new file mode 100644 index 000000000..f00c77a17 --- /dev/null +++ b/magicblock-committor-service/src/intent_executor/null_task_info_fetcher.rs @@ -0,0 +1,42 @@ +use std::collections::HashMap; + +use async_trait::async_trait; +use magicblock_rpc_client::MagicBlockRpcClientResult; +use solana_account::Account; +use solana_pubkey::Pubkey; + +use super::task_info_fetcher::{ + ResetType, TaskInfoFetcher, TaskInfoFetcherResult, +}; + +pub struct NullTaskInfoFetcher; + +#[async_trait] +impl TaskInfoFetcher for NullTaskInfoFetcher { + async fn fetch_next_commit_ids( + &self, + _pubkeys: &[Pubkey], + ) -> TaskInfoFetcherResult> { + Ok(Default::default()) + } + + async fn fetch_rent_reimbursements( + &self, + _pubkeys: &[Pubkey], + ) -> TaskInfoFetcherResult> { + Ok(Default::default()) + } + + fn peek_commit_id(&self, _pubkey: &Pubkey) -> Option { + None + } + + fn reset(&self, _: ResetType) {} + + async fn get_base_account( + &self, + _pubkey: &Pubkey, + ) -> MagicBlockRpcClientResult> { + Ok(None) // AccountNotFound + } +} diff --git a/magicblock-committor-service/src/intent_executor/task_info_fetcher.rs b/magicblock-committor-service/src/intent_executor/task_info_fetcher.rs index 2928d2f33..15e68a8bf 100644 --- a/magicblock-committor-service/src/intent_executor/task_info_fetcher.rs +++ b/magicblock-committor-service/src/intent_executor/task_info_fetcher.rs @@ -9,7 +9,10 @@ use dlp::{ use log::{error, warn}; use lru::LruCache; use magicblock_metrics::metrics; -use magicblock_rpc_client::{MagicBlockRpcClientError, MagicblockRpcClient}; +use magicblock_rpc_client::{ + MagicBlockRpcClientError, MagicBlockRpcClientResult, MagicblockRpcClient, +}; +use solana_account::Account; use solana_pubkey::Pubkey; const NUM_FETCH_RETRIES: NonZeroUsize = @@ -36,6 +39,11 @@ pub trait TaskInfoFetcher: Send + Sync + 'static { /// Resets cache for some or all accounts fn reset(&self, reset_type: ResetType); + + async fn get_base_account( + &self, + _pubkey: &Pubkey, + ) -> MagicBlockRpcClientResult>; } pub enum ResetType<'a> { @@ -264,6 +272,13 @@ impl TaskInfoFetcher for CacheTaskInfoFetcher { } } } + + async fn get_base_account( + &self, + pubkey: &Pubkey, + ) -> MagicBlockRpcClientResult> { + self.rpc_client.get_account(pubkey).await + } } #[derive(thiserror::Error, Debug)] diff --git a/magicblock-committor-service/src/tasks/args_task.rs b/magicblock-committor-service/src/tasks/args_task.rs index 301db3b63..aacade3e6 100644 --- a/magicblock-committor-service/src/tasks/args_task.rs +++ b/magicblock-committor-service/src/tasks/args_task.rs @@ -1,4 +1,4 @@ -use dlp::args::{CallHandlerArgs, CommitStateArgs}; +use dlp::args::CallHandlerArgs; use solana_pubkey::Pubkey; use solana_sdk::instruction::{AccountMeta, Instruction}; @@ -44,20 +44,7 @@ impl ArgsTask { impl BaseTask for ArgsTask { fn instruction(&self, validator: &Pubkey) -> Instruction { match &self.task_type { - ArgsTaskType::Commit(value) => { - let args = CommitStateArgs { - nonce: value.commit_id, - lamports: value.committed_account.account.lamports, - data: value.committed_account.account.data.clone(), - allow_undelegation: value.allow_undelegation, - }; - dlp::instruction_builder::commit_state( - *validator, - value.committed_account.pubkey, - value.committed_account.account.owner, - args, - ) - } + ArgsTaskType::Commit(value) => value.create_commit_ix(validator), ArgsTaskType::Finalize(value) => { dlp::instruction_builder::finalize( *validator, @@ -101,6 +88,18 @@ impl BaseTask for ArgsTask { self: Box, ) -> Result, Box> { match self.task_type { + ArgsTaskType::Commit(mut value) if value.is_commit_diff() => { + // TODO (snawaz): Currently, we do not support executing CommitDiff + // as BufferTask, which is why we're forcing CommitTask to use CommitState + // before converting this task into BufferTask. Once CommitDiff is supported + // by BufferTask, we do not have to force_commit_state and we can remove + // force_commit_state stuff, as it's essentially a downgrade. + + value.force_commit_state(); + Ok(Box::new(BufferTask::new_preparation_required( + BufferTaskType::Commit(value), + ))) + } ArgsTaskType::Commit(value) => { Ok(Box::new(BufferTask::new_preparation_required( BufferTaskType::Commit(value), diff --git a/magicblock-committor-service/src/tasks/mod.rs b/magicblock-committor-service/src/tasks/mod.rs index a31e52c4a..8cb58fabf 100644 --- a/magicblock-committor-service/src/tasks/mod.rs +++ b/magicblock-committor-service/src/tasks/mod.rs @@ -1,3 +1,9 @@ +use std::sync::Arc; + +use dlp::{ + args::{CommitDiffArgs, CommitStateArgs}, + compute_diff, +}; use dyn_clone::DynClone; use magicblock_committor_program::{ instruction_builder::{ @@ -13,11 +19,15 @@ use magicblock_committor_program::{ use magicblock_program::magic_scheduled_base_intent::{ BaseAction, CommittedAccount, }; +use solana_account::{Account, ReadableAccount}; use solana_pubkey::Pubkey; use solana_sdk::instruction::Instruction; use thiserror::Error; -use crate::tasks::visitor::Visitor; +use crate::{ + intent_executor::task_info_fetcher::TaskInfoFetcher, + tasks::visitor::Visitor, +}; pub mod args_task; pub mod buffer_task; @@ -53,6 +63,8 @@ pub enum TaskStrategy { pub trait BaseTask: Send + Sync + DynClone { /// Gets all pubkeys that involved in Task's instruction fn involved_accounts(&self, validator: &Pubkey) -> Vec { + // TODO (snawaz): rewrite it. + // currently it is slow as it discards heavy computations and memory allocations. self.instruction(validator) .accounts .iter() @@ -97,11 +109,125 @@ pub trait BaseTask: Send + Sync + DynClone { dyn_clone::clone_trait_object!(BaseTask); +pub struct CommitTaskBuilder; + +impl CommitTaskBuilder { + // Accounts larger than COMMIT_STATE_SIZE_THRESHOLD, use CommitDiff to + // reduce instruction size. Below this, commit is sent as CommitState. + // Chose 256 as thresold seems good enough as it could hold 8 u32 fields + // or 4 u64 fields. + const COMMIT_STATE_SIZE_THRESHOLD: usize = 256; + + pub async fn create_commit_task( + commit_id: u64, + allow_undelegation: bool, + committed_account: CommittedAccount, + task_info_fetcher: &Arc, + ) -> CommitTask { + let base_account = if committed_account.account.data.len() + > CommitTaskBuilder::COMMIT_STATE_SIZE_THRESHOLD + { + match task_info_fetcher + .get_base_account(&committed_account.pubkey) + .await + { + Ok(Some(account)) => Some(account), + Ok(None) => { + log::warn!("AccountNotFound for commit_diff, pubkey: {}, commit_id: {}, Falling back to commit_state.", + committed_account.pubkey, commit_id); + None + } + Err(e) => { + log::warn!("Failed to fetch base account for commit diff, pubkey: {}, commit_id: {}, error: {}. Falling back to commit_state.", + committed_account.pubkey, commit_id, e); + None + } + } + } else { + None + }; + + CommitTask { + commit_id, + allow_undelegation, + committed_account, + base_account, + force_commit_state: false, + } + } +} + #[derive(Clone)] pub struct CommitTask { pub commit_id: u64, pub allow_undelegation: bool, pub committed_account: CommittedAccount, + base_account: Option, + force_commit_state: bool, +} + +impl CommitTask { + pub fn is_commit_diff(&self) -> bool { + !self.force_commit_state + && self.committed_account.account.data.len() + > CommitTaskBuilder::COMMIT_STATE_SIZE_THRESHOLD + && self.base_account.is_some() + } + + pub fn force_commit_state(&mut self) { + self.force_commit_state = true; + } + + pub fn create_commit_ix(&self, validator: &Pubkey) -> Instruction { + if let Some(fetched_account) = self.base_account.as_ref() { + self.create_commit_diff_ix(validator, fetched_account) + } else { + self.create_commit_state_ix(validator) + } + } + + fn create_commit_state_ix(&self, validator: &Pubkey) -> Instruction { + let args = CommitStateArgs { + nonce: self.commit_id, + lamports: self.committed_account.account.lamports, + data: self.committed_account.account.data.clone(), + allow_undelegation: self.allow_undelegation, + }; + dlp::instruction_builder::commit_state( + *validator, + self.committed_account.pubkey, + self.committed_account.account.owner, + args, + ) + } + + fn create_commit_diff_ix( + &self, + validator: &Pubkey, + fetched_account: &Account, + ) -> Instruction { + if self.force_commit_state { + return self.create_commit_state_ix(validator); + } + + let args = CommitDiffArgs { + nonce: self.commit_id, + lamports: self.committed_account.account.lamports, + diff: compute_diff( + fetched_account.data(), + self.committed_account.account.data(), + ) + .to_vec(), + allow_undelegation: self.allow_undelegation, + }; + + dlp::instruction_builder::commit_diff( + *validator, + self.committed_account.pubkey, + self.committed_account.account.owner, + args, + ) + } } #[derive(Clone)] @@ -296,32 +422,39 @@ mod serialization_safety_test { }; use solana_account::Account; - use crate::tasks::{ - args_task::{ArgsTask, ArgsTaskType}, - buffer_task::{BufferTask, BufferTaskType}, - *, + use crate::{ + intent_executor::NullTaskInfoFetcher, + tasks::{ + args_task::{ArgsTask, ArgsTaskType}, + buffer_task::{BufferTask, BufferTaskType}, + *, + }, }; // Test all ArgsTask variants - #[test] - fn test_args_task_instruction_serialization() { + #[tokio::test] + async fn test_args_task_instruction_serialization() { let validator = Pubkey::new_unique(); // Test Commit variant - let commit_task: ArgsTask = ArgsTaskType::Commit(CommitTask { - commit_id: 123, - allow_undelegation: true, - committed_account: CommittedAccount { - pubkey: Pubkey::new_unique(), - account: Account { - lamports: 1000, - data: vec![1, 2, 3], - owner: Pubkey::new_unique(), - executable: false, - rent_epoch: 0, + let commit_task: ArgsTask = ArgsTaskType::Commit( + CommitTaskBuilder::create_commit_task( + 123, + true, + CommittedAccount { + pubkey: Pubkey::new_unique(), + account: Account { + lamports: 1000, + data: vec![1, 2, 3], + owner: Pubkey::new_unique(), + executable: false, + rent_epoch: 0, + }, }, - }, - }) + &Arc::new(NullTaskInfoFetcher), + ) + .await, + ) .into(); assert_serializable(&commit_task.instruction(&validator)); @@ -363,51 +496,57 @@ mod serialization_safety_test { } // Test BufferTask variants - #[test] - fn test_buffer_task_instruction_serialization() { + #[tokio::test] + async fn test_buffer_task_instruction_serialization() { let validator = Pubkey::new_unique(); - let buffer_task = BufferTask::new_preparation_required( - BufferTaskType::Commit(CommitTask { - commit_id: 456, - allow_undelegation: false, - committed_account: CommittedAccount { - pubkey: Pubkey::new_unique(), - account: Account { - lamports: 2000, - data: vec![7, 8, 9], - owner: Pubkey::new_unique(), - executable: false, - rent_epoch: 0, + let buffer_task = + BufferTask::new_preparation_required(BufferTaskType::Commit( + CommitTaskBuilder::create_commit_task( + 456, + false, + CommittedAccount { + pubkey: Pubkey::new_unique(), + account: Account { + lamports: 2000, + data: vec![7, 8, 9], + owner: Pubkey::new_unique(), + executable: false, + rent_epoch: 0, + }, }, - }, - }), - ); + &Arc::new(NullTaskInfoFetcher), + ) + .await, + )); assert_serializable(&buffer_task.instruction(&validator)); } // Test preparation instructions - #[test] - fn test_preparation_instructions_serialization() { + #[tokio::test] + async fn test_preparation_instructions_serialization() { let authority = Pubkey::new_unique(); // Test BufferTask preparation - let buffer_task = BufferTask::new_preparation_required( - BufferTaskType::Commit(CommitTask { - commit_id: 789, - allow_undelegation: true, - committed_account: CommittedAccount { - pubkey: Pubkey::new_unique(), - account: Account { - lamports: 3000, - data: vec![0; 1024], // Larger data to test chunking - owner: Pubkey::new_unique(), - executable: false, - rent_epoch: 0, + let buffer_task = + BufferTask::new_preparation_required(BufferTaskType::Commit( + CommitTaskBuilder::create_commit_task( + 789, + true, + CommittedAccount { + pubkey: Pubkey::new_unique(), + account: Account { + lamports: 3000, + data: vec![0; 1024], // Larger data to test chunking + owner: Pubkey::new_unique(), + executable: false, + rent_epoch: 0, + }, }, - }, - }), - ); + &Arc::new(NullTaskInfoFetcher), + ) + .await, + )); let PreparationState::Required(preparation_task) = buffer_task.preparation_state() diff --git a/magicblock-committor-service/src/tasks/task_builder.rs b/magicblock-committor-service/src/tasks/task_builder.rs index 36c7315eb..95b11b030 100644 --- a/magicblock-committor-service/src/tasks/task_builder.rs +++ b/magicblock-committor-service/src/tasks/task_builder.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use async_trait::async_trait; +use futures_util::future::join_all; use log::error; use magicblock_program::magic_scheduled_base_intent::{ CommitType, CommittedAccount, MagicBaseIntent, ScheduledBaseIntent, @@ -15,7 +16,8 @@ use crate::{ persist::IntentPersister, tasks::{ args_task::{ArgsTask, ArgsTaskType}, - BaseActionTask, BaseTask, CommitTask, FinalizeTask, UndelegateTask, + BaseActionTask, BaseTask, CommitTaskBuilder, FinalizeTask, + UndelegateTask, }, }; @@ -23,14 +25,14 @@ use crate::{ pub trait TasksBuilder { // Creates tasks for commit stage async fn commit_tasks( - commit_id_fetcher: &Arc, + task_info_fetcher: &Arc, base_intent: &ScheduledBaseIntent, persister: &Option

, ) -> TaskBuilderResult>>; // Create tasks for finalize stage async fn finalize_tasks( - info_fetcher: &Arc, + task_info_fetcher: &Arc, base_intent: &ScheduledBaseIntent, ) -> TaskBuilderResult>>; } @@ -43,7 +45,7 @@ pub struct TaskBuilderImpl; impl TasksBuilder for TaskBuilderImpl { /// Returns [`Task`]s for Commit stage async fn commit_tasks( - commit_id_fetcher: &Arc, + task_info_fetcher: &Arc, base_intent: &ScheduledBaseIntent, persister: &Option

, ) -> TaskBuilderResult>> { @@ -71,7 +73,7 @@ impl TasksBuilder for TaskBuilderImpl { .iter() .map(|account| account.pubkey) .collect::>(); - let commit_ids = commit_id_fetcher + let commit_ids = task_info_fetcher .fetch_next_commit_ids(&committed_pubkeys) .await .map_err(TaskBuilderError::CommitTasksBuildError)?; @@ -85,26 +87,26 @@ impl TasksBuilder for TaskBuilderImpl { } }); - let tasks = accounts + let tasks = join_all(accounts .iter() - .map(|account| { + .map(|account| async { let commit_id = *commit_ids.get(&account.pubkey).expect("CommitIdFetcher provide commit ids for all listed pubkeys, or errors!"); - let task = ArgsTaskType::Commit(CommitTask { + let task = ArgsTaskType::Commit(CommitTaskBuilder::create_commit_task( commit_id, allow_undelegation, - committed_account: account.clone(), - }); + account.clone(), + task_info_fetcher, + ).await); Box::new(ArgsTask::new(task)) as Box - }) - .collect(); + })).await; Ok(tasks) } /// Returns [`Task`]s for Finalize stage async fn finalize_tasks( - info_fetcher: &Arc, + task_info_fetcher: &Arc, base_intent: &ScheduledBaseIntent, ) -> TaskBuilderResult>> { // Helper to create a finalize task @@ -167,7 +169,7 @@ impl TasksBuilder for TaskBuilderImpl { .iter() .map(|account| account.pubkey) .collect::>(); - let rent_reimbursements = info_fetcher + let rent_reimbursements = task_info_fetcher .fetch_rent_reimbursements(&pubkeys) .await .map_err(TaskBuilderError::FinalizedTasksBuildError)?; diff --git a/magicblock-committor-service/src/tasks/task_strategist.rs b/magicblock-committor-service/src/tasks/task_strategist.rs index 406ba1a9d..2b7556225 100644 --- a/magicblock-committor-service/src/tasks/task_strategist.rs +++ b/magicblock-committor-service/src/tasks/task_strategist.rs @@ -157,6 +157,7 @@ impl TaskStrategist { ) -> Result { // Get initial transaction size let calculate_tx_length = |tasks: &[Box]| { + // TODO (snawaz): we seem to discard lots of heavy computations here match TransactionUtils::assemble_tasks_tx( &Keypair::new(), // placeholder tasks, @@ -249,6 +250,9 @@ pub type TaskStrategistResult = Result; #[cfg(test)] mod tests { + use std::sync::Arc; + + use futures_util::future::join_all; use magicblock_program::magic_scheduled_base_intent::{ BaseAction, CommittedAccount, ProgramArgs, }; @@ -257,26 +261,36 @@ mod tests { use super::*; use crate::{ + intent_executor::NullTaskInfoFetcher, persist::IntentPersisterImpl, - tasks::{BaseActionTask, CommitTask, TaskStrategy, UndelegateTask}, + tasks::{ + BaseActionTask, CommitTaskBuilder, TaskStrategy, UndelegateTask, + }, }; // Helper to create a simple commit task - fn create_test_commit_task(commit_id: u64, data_size: usize) -> ArgsTask { - ArgsTask::new(ArgsTaskType::Commit(CommitTask { - commit_id, - allow_undelegation: false, - committed_account: CommittedAccount { - pubkey: Pubkey::new_unique(), - account: Account { - lamports: 1000, - data: vec![1; data_size], - owner: system_program::id(), - executable: false, - rent_epoch: 0, + async fn create_test_commit_task( + commit_id: u64, + data_size: usize, + ) -> ArgsTask { + ArgsTask::new(ArgsTaskType::Commit( + CommitTaskBuilder::create_commit_task( + commit_id, + false, + CommittedAccount { + pubkey: Pubkey::new_unique(), + account: Account { + lamports: 1000, + data: vec![1; data_size], + owner: system_program::id(), + executable: false, + rent_epoch: 0, + }, }, - }, - })) + &Arc::new(NullTaskInfoFetcher), + ) + .await, + )) } // Helper to create a Base action task @@ -311,10 +325,10 @@ mod tests { })) } - #[test] - fn test_build_strategy_with_single_small_task() { + #[tokio::test] + async fn test_build_strategy_with_single_small_task() { let validator = Pubkey::new_unique(); - let task = create_test_commit_task(1, 100); + let task = create_test_commit_task(1, 100).await; let tasks = vec![Box::new(task) as Box]; let strategy = TaskStrategist::build_strategy( @@ -328,11 +342,11 @@ mod tests { assert!(strategy.lookup_tables_keys.is_empty()); } - #[test] - fn test_build_strategy_optimizes_to_buffer_when_needed() { + #[tokio::test] + async fn test_build_strategy_optimizes_to_buffer_when_needed() { let validator = Pubkey::new_unique(); - let task = create_test_commit_task(1, 1000); // Large task + let task = create_test_commit_task(1, 1000).await; // Large task let tasks = vec![Box::new(task) as Box]; let strategy = TaskStrategist::build_strategy( @@ -349,11 +363,11 @@ mod tests { )); } - #[test] - fn test_build_strategy_optimizes_to_buffer_u16_exceeded() { + #[tokio::test] + async fn test_build_strategy_optimizes_to_buffer_u16_exceeded() { let validator = Pubkey::new_unique(); - let task = create_test_commit_task(1, 66_000); // Large task + let task = create_test_commit_task(1, 66_000).await; // Large task let tasks = vec![Box::new(task) as Box]; let strategy = TaskStrategist::build_strategy( @@ -370,19 +384,18 @@ mod tests { )); } - #[test] - fn test_build_strategy_creates_multiple_buffers() { + #[tokio::test] + async fn test_build_strategy_creates_multiple_buffers() { // TODO: ALSO MAX NUM WITH PURE BUFFER commits, no alts const NUM_COMMITS: u64 = 3; let validator = Pubkey::new_unique(); - let tasks = (0..NUM_COMMITS) - .map(|i| { - let task = create_test_commit_task(i, 500); // Large task - Box::new(task) as Box - }) - .collect(); + let tasks = join_all((0..NUM_COMMITS).map(|i| async move { + let task = create_test_commit_task(i, 500).await; // Large task + Box::new(task) as Box + })) + .await; let strategy = TaskStrategist::build_strategy( tasks, @@ -397,20 +410,19 @@ mod tests { assert!(strategy.lookup_tables_keys.is_empty()); } - #[test] - fn test_build_strategy_with_lookup_tables_when_needed() { + #[tokio::test] + async fn test_build_strategy_with_lookup_tables_when_needed() { // Also max number of committed accounts fit with ALTs! const NUM_COMMITS: u64 = 22; let validator = Pubkey::new_unique(); - let tasks = (0..NUM_COMMITS) - .map(|i| { - // Large task - let task = create_test_commit_task(i, 10000); - Box::new(task) as Box - }) - .collect(); + let tasks = join_all((0..NUM_COMMITS).map(|i| async move { + // Large task + let task = create_test_commit_task(i, 10000).await; + Box::new(task) as Box + })) + .await; let strategy = TaskStrategist::build_strategy( tasks, @@ -425,19 +437,18 @@ mod tests { assert!(!strategy.lookup_tables_keys.is_empty()); } - #[test] - fn test_build_strategy_fails_when_cant_fit() { + #[tokio::test] + async fn test_build_strategy_fails_when_cant_fit() { const NUM_COMMITS: u64 = 23; let validator = Pubkey::new_unique(); - let tasks = (0..NUM_COMMITS) - .map(|i| { - // Large task - let task = create_test_commit_task(i, 1000); - Box::new(task) as Box - }) - .collect(); + let tasks = join_all((0..NUM_COMMITS).map(|i| async move { + // Large task + let task = create_test_commit_task(i, 1000).await; + Box::new(task) as Box + })) + .await; let result = TaskStrategist::build_strategy( tasks, @@ -447,12 +458,15 @@ mod tests { assert!(matches!(result, Err(TaskStrategistError::FailedToFitError))); } - #[test] - fn test_optimize_strategy_prioritizes_largest_tasks() { + #[tokio::test] + async fn test_optimize_strategy_prioritizes_largest_tasks() { let mut tasks = [ - Box::new(create_test_commit_task(1, 100)) as Box, - Box::new(create_test_commit_task(2, 1000)) as Box, // Larger task - Box::new(create_test_commit_task(3, 1000)) as Box, // Larger task + Box::new(create_test_commit_task(1, 100).await) + as Box, + Box::new(create_test_commit_task(2, 1000).await) + as Box, // Larger task + Box::new(create_test_commit_task(3, 1000).await) + as Box, // Larger task ]; let _ = TaskStrategist::optimize_strategy(&mut tasks); @@ -461,11 +475,12 @@ mod tests { assert!(matches!(tasks[1].strategy(), TaskStrategy::Buffer)); } - #[test] - fn test_mixed_task_types_with_optimization() { + #[tokio::test] + async fn test_mixed_task_types_with_optimization() { let validator = Pubkey::new_unique(); let tasks = vec![ - Box::new(create_test_commit_task(1, 1000)) as Box, + Box::new(create_test_commit_task(1, 1000).await) + as Box, Box::new(create_test_finalize_task()) as Box, Box::new(create_test_base_action_task(500)) as Box, Box::new(create_test_undelegate_task()) as Box, diff --git a/magicblock-processor/src/executor/processing.rs b/magicblock-processor/src/executor/processing.rs index 0bb638c3c..c58c4c200 100644 --- a/magicblock-processor/src/executor/processing.rs +++ b/magicblock-processor/src/executor/processing.rs @@ -373,7 +373,6 @@ impl super::TransactionExecutor { .get_or_insert_default(); let msg = "Feepayer balance has been modified illegally".into(); logs.push(msg); - return; } } } diff --git a/programs/magicblock/src/magic_scheduled_base_intent.rs b/programs/magicblock/src/magic_scheduled_base_intent.rs index 1fb050188..09bead405 100644 --- a/programs/magicblock/src/magic_scheduled_base_intent.rs +++ b/programs/magicblock/src/magic_scheduled_base_intent.rs @@ -308,6 +308,7 @@ impl BaseAction { } type CommittedAccountRef<'a> = (Pubkey, &'a RefCell); + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct CommittedAccount { pub pubkey: Pubkey, diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index 7c76ccce9..d17495e42 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -680,6 +680,18 @@ dependencies = [ "typenum", ] +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "blake3" version = "1.8.2" @@ -836,6 +848,28 @@ dependencies = [ "serde", ] +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta 0.1.4", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "bytemuck" version = "1.23.1" @@ -1726,7 +1760,7 @@ dependencies = [ "ephemeral-rollups-sdk-attribute-commit", "ephemeral-rollups-sdk-attribute-delegate", "ephemeral-rollups-sdk-attribute-ephemeral", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.0", "magicblock-magic-program-api 0.2.1", "solana-program", ] @@ -1844,12 +1878,12 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "faststr" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6503af7917fea18ffef8f7e8553fb8dff89e2e6837e94e09dd7fb069c82d62c" +checksum = "baec6a0289d7f1fe5665586ef7340af82e3037207bef60f5785e57569776f0c8" dependencies = [ "bytes", - "rkyv", + "rkyv 0.8.12", "serde", "simdutf8", ] @@ -2019,6 +2053,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.1.31" @@ -2654,9 +2694,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" +checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" dependencies = [ "bytes", "futures-core", @@ -2924,7 +2964,7 @@ dependencies = [ "log", "magicblock-config", "magicblock-core", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.2", "random-port", "rayon", "serde", @@ -3486,7 +3526,7 @@ dependencies = [ "magicblock-chainlink", "magicblock-committor-service", "magicblock-core", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.2", "magicblock-ledger", "magicblock-magic-program-api 0.2.3", "magicblock-metrics", @@ -3588,7 +3628,7 @@ dependencies = [ "magicblock-committor-service", "magicblock-config", "magicblock-core", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.2", "magicblock-ledger", "magicblock-magic-program-api 0.2.3", "magicblock-metrics", @@ -3623,7 +3663,7 @@ dependencies = [ "log", "lru 0.16.0", "magicblock-core", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.2", "magicblock-magic-program-api 0.2.3", "magicblock-metrics", "serde_json", @@ -3673,7 +3713,7 @@ dependencies = [ "log", "lru 0.16.0", "magicblock-committor-program", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.2", "magicblock-metrics", "magicblock-program", "magicblock-rpc-client", @@ -3705,7 +3745,7 @@ dependencies = [ "serde", "solana-keypair", "solana-pubkey", - "strum", + "strum 0.24.1", "thiserror 1.0.69", "toml 0.8.23", "url 2.5.4", @@ -3768,6 +3808,29 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "magicblock-delegation-program" +version = "1.1.2" +source = "git+https://github.com/magicblock-labs/delegation-program.git?rev=e8d03936#e8d039369ac1149e899ea94f31e0f9cc4e600a38" +dependencies = [ + "bincode", + "borsh 1.5.7", + "bytemuck", + "num_enum", + "paste", + "pinocchio", + "pinocchio-log", + "pinocchio-pubkey", + "pinocchio-system", + "rkyv 0.7.45", + "solana-curve25519", + "solana-program", + "solana-security-txt", + "static_assertions", + "strum 0.27.2", + "thiserror 1.0.69", +] + [[package]] name = "magicblock-ledger" version = "0.2.3" @@ -3949,7 +4012,7 @@ dependencies = [ "anyhow", "log", "magicblock-config", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.2", "magicblock-program", "magicblock-rpc-client", "solana-rpc-client", @@ -4128,18 +4191,18 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "munge" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7feb0b48aa0a25f9fe0899482c6e1379ee7a11b24a53073eacdecb9adb6dc60" +checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" dependencies = [ "munge_macro", ] [[package]] name = "munge_macro" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2e3795a5d2da581a8b252fec6022eee01aea10161a4d1bf237d4cbe47f7e988" +checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" dependencies = [ "proc-macro2", "quote", @@ -4874,8 +4937,10 @@ version = "0.0.0" dependencies = [ "borsh 1.5.7", "ephemeral-rollups-sdk", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.2", + "rkyv 0.7.45", "solana-program", + "static_assertions", ] [[package]] @@ -4994,18 +5059,38 @@ dependencies = [ [[package]] name = "ptr_meta" -version = "0.3.0" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9e76f66d3f9606f44e45598d155cb13ecf09f4a28199e48daf8c8fc937ea90" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" dependencies = [ - "ptr_meta_derive", + "ptr_meta_derive 0.1.4", +] + +[[package]] +name = "ptr_meta" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" +dependencies = [ + "ptr_meta_derive 0.3.1", ] [[package]] name = "ptr_meta_derive" -version = "0.3.0" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca414edb151b4c8d125c12566ab0d74dc9cdba36fb80eb7b848c15f495fd32d1" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", @@ -5125,13 +5210,19 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rancor" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf5f7161924b9d1cea0e4cabc97c372cea92b5f927fc13c6bca67157a0ad947" +checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" dependencies = [ - "ptr_meta", + "ptr_meta 0.3.1", ] [[package]] @@ -5338,18 +5429,18 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", @@ -5399,9 +5490,18 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "rend" -version = "0.5.2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] + +[[package]] +name = "rend" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a35e8a6bf28cd121053a66aa2e6a2e3eaffad4a60012179f0e864aa5ffeff215" +checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" [[package]] name = "reqwest" @@ -5481,27 +5581,56 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.8.11" +version = "0.7.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f5c3e5da784cd8c69d32cdc84673f3204536ca56e1fa01be31a74b92c932ac" +checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "ptr_meta 0.1.4", + "rend 0.4.2", + "rkyv_derive 0.7.45", + "seahash", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a640b26f007713818e9a9b65d34da1cf58538207b052916a83d80e43f3ffa4" dependencies = [ "bytes", "hashbrown 0.15.4", "indexmap 2.10.0", "munge", - "ptr_meta", + "ptr_meta 0.3.1", "rancor", - "rend", - "rkyv_derive", + "rend 0.5.3", + "rkyv_derive 0.8.12", "tinyvec", "uuid", ] [[package]] name = "rkyv_derive" -version = "0.8.11" +version = "0.7.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "rkyv_derive" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4270433626cffc9c4c1d3707dd681f2a2718d3d7b09ad754bec137acecda8d22" +checksum = "bd83f5f173ff41e00337d97f6572e416d022ef8a19f371817259ae960324c482" dependencies = [ "proc-macro2", "quote", @@ -5790,7 +5919,7 @@ dependencies = [ "integration-test-tools", "log", "magicblock-core", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.2", "program-schedulecommit", "solana-program", "solana-rpc-client", @@ -5808,7 +5937,7 @@ dependencies = [ "log", "magicblock-committor-program", "magicblock-committor-service", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.2", "magicblock-program", "magicblock-rpc-client", "magicblock-table-mania", @@ -5827,12 +5956,14 @@ dependencies = [ name = "schedulecommit-test-scenarios" version = "0.0.0" dependencies = [ + "borsh 1.5.7", "ephemeral-rollups-sdk", "integration-test-tools", "log", "magicblock-core", "magicblock-magic-program-api 0.2.3", "program-schedulecommit", + "rand 0.8.5", "schedulecommit-client", "solana-program", "solana-rpc-client", @@ -5877,6 +6008,12 @@ version = "3.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "584e070911c7017da6cb2eb0788d09f43d789029b5877d3e5ecc8acf86ceee21" +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "security-framework" version = "2.11.1" @@ -7526,8 +7663,8 @@ dependencies = [ "spl-token", "spl-token-2022 7.0.0", "static_assertions", - "strum", - "strum_macros", + "strum 0.24.1", + "strum_macros 0.24.3", "tar", "tempfile", "thiserror 2.0.12", @@ -8538,8 +8675,8 @@ dependencies = [ "solana-vote", "solana-vote-program", "static_assertions", - "strum", - "strum_macros", + "strum 0.24.1", + "strum_macros 0.24.3", "symlink", "tar", "tempfile", @@ -9778,9 +9915,9 @@ dependencies = [ [[package]] name = "sonic-rs" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd1adc42def3cb101f3ebef3cd2d642f9a21072bbcd4ec9423343ccaa6afa596" +checksum = "22540d56ba14521e4878ad436d498518c59698c39a89d5905c694932f0bf7134" dependencies = [ "ahash 0.8.12", "bumpalo", @@ -10233,7 +10370,16 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" dependencies = [ - "strum_macros", + "strum_macros 0.24.3", +] + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", ] [[package]] @@ -10249,6 +10395,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "subtle" version = "2.6.1" @@ -10340,6 +10498,12 @@ dependencies = [ "solana-program", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tar" version = "0.4.44" @@ -10438,7 +10602,7 @@ dependencies = [ "integration-test-tools", "log", "magicblock-chainlink", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.2", "program-flexi-counter", "program-mini", "solana-account", @@ -10520,7 +10684,7 @@ dependencies = [ "log", "magicblock-accounts-db", "magicblock-config", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.2", "program-flexi-counter", "solana-rpc-client", "solana-sdk", @@ -10541,7 +10705,7 @@ dependencies = [ "magic-domain-program", "magicblock-api", "magicblock-config", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.2", "magicblock-program", "magicblock-validator-admin", "solana-rpc-client", @@ -10581,7 +10745,7 @@ version = "0.0.0" dependencies = [ "integration-test-tools", "log", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.2", "program-flexi-counter", "solana-rpc-client-api", "solana-sdk", @@ -11273,9 +11437,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.18.0" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ "js-sys", "wasm-bindgen", @@ -11999,6 +12163,15 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "x509-parser" version = "0.14.0" diff --git a/test-integration/Cargo.toml b/test-integration/Cargo.toml index d177a4c06..7f89b65dd 100644 --- a/test-integration/Cargo.toml +++ b/test-integration/Cargo.toml @@ -36,8 +36,8 @@ chrono = "0.4" cleanass = "0.0.1" color-backtrace = { version = "0.7" } ctrlc = "3.4.7" -ephemeral-rollups-sdk = { git = "https://github.com/magicblock-labs/ephemeral-rollups-sdk.git", rev = "2d0f16b" } futures = "0.3.31" +ephemeral-rollups-sdk = { git = "https://github.com/magicblock-labs/ephemeral-rollups-sdk.git", rev = "2d0f16b" } integration-test-tools = { path = "test-tools" } isocountry = "0.3.2" lazy_static = "1.4.0" @@ -57,7 +57,7 @@ magicblock-config = { path = "../magicblock-config" } magicblock-core = { path = "../magicblock-core" } magic-domain-program = { git = "https://github.com/magicblock-labs/magic-domain-program.git", rev = "ea04d46", default-features = false } magicblock_magic_program_api = { package = "magicblock-magic-program-api", path = "../magicblock-magic-program-api" } -magicblock-delegation-program = { git = "https://github.com/magicblock-labs/delegation-program.git", rev = "aa1de56d90c", features = [ +magicblock-delegation-program = { git = "https://github.com/magicblock-labs/delegation-program.git", rev = "e8d03936", features = [ "no-entrypoint", ] } magicblock-program = { path = "../programs/magicblock" } @@ -71,6 +71,7 @@ program-schedulecommit-security = { path = "programs/schedulecommit-security" } rand = "0.8.5" random-port = "0.1.1" rayon = "1.10.0" +rkyv = "0.7.45" schedulecommit-client = { path = "schedulecommit/client" } serde = "1.0.217" serial_test = "3.2.0" @@ -89,6 +90,7 @@ solana-sdk-ids = { version = "2.2" } solana-system-interface = "1.0" solana-transaction-status = "2.2" spl-memo-interface = "1.0" +static_assertions = "1.1.0" teepee = "0.0.1" tempfile = "3.10.1" test-chainlink = { path = "./test-chainlink" } diff --git a/test-integration/programs/schedulecommit/Cargo.toml b/test-integration/programs/schedulecommit/Cargo.toml index 59c45c378..d46da613d 100644 --- a/test-integration/programs/schedulecommit/Cargo.toml +++ b/test-integration/programs/schedulecommit/Cargo.toml @@ -8,6 +8,8 @@ borsh = { workspace = true } ephemeral-rollups-sdk = { workspace = true } solana-program = { workspace = true } magicblock-delegation-program = { workspace = true } +rkyv = { workspace = true } +static_assertions = { workspace = true } [lib] crate-type = ["cdylib", "lib"] diff --git a/test-integration/programs/schedulecommit/src/api.rs b/test-integration/programs/schedulecommit/src/api.rs index 57475abed..ec0316f74 100644 --- a/test-integration/programs/schedulecommit/src/api.rs +++ b/test-integration/programs/schedulecommit/src/api.rs @@ -9,7 +9,8 @@ use solana_program::{ }; use crate::{ - DelegateCpiArgs, ScheduleCommitCpiArgs, ScheduleCommitInstruction, + BookUpdate, DelegateCpiArgs, DelegateOrderBookArgs, ScheduleCommitCpiArgs, + ScheduleCommitInstruction, }; pub fn init_account_instruction( @@ -32,6 +33,47 @@ pub fn init_account_instruction( ) } +pub fn init_order_book_instruction( + payer: Pubkey, + book_manager: Pubkey, + order_book: Pubkey, +) -> Instruction { + let program_id = crate::id(); + let account_metas = vec![ + AccountMeta::new(payer, true), + AccountMeta::new_readonly(book_manager, true), + AccountMeta::new(order_book, false), + AccountMeta::new_readonly(system_program::id(), false), + ]; + + Instruction::new_with_borsh( + program_id, + &ScheduleCommitInstruction::InitOrderBook, + account_metas, + ) +} + +pub fn grow_order_book_instruction( + payer: Pubkey, + book_manager: Pubkey, + order_book: Pubkey, + additional_space: u64, +) -> Instruction { + let program_id = crate::id(); + let account_metas = vec![ + AccountMeta::new(payer, true), + AccountMeta::new_readonly(book_manager, false), + AccountMeta::new(order_book, false), + AccountMeta::new_readonly(system_program::id(), false), + ]; + + Instruction::new_with_borsh( + program_id, + &ScheduleCommitInstruction::GrowOrderBook(additional_space), + account_metas, + ) +} + pub fn init_payer_escrow(payer: Pubkey) -> [Instruction; 2] { let top_up_ix = dlp::instruction_builder::top_up_ephemeral_balance( payer, @@ -58,17 +100,14 @@ pub fn init_payer_escrow(payer: Pubkey) -> [Instruction; 2] { pub fn delegate_account_cpi_instruction( payer: Pubkey, validator: Option, - player: Pubkey, + player_or_book_manager: Pubkey, + user_seed: &[u8], ) -> Instruction { let program_id = crate::id(); - let (pda, _) = pda_and_bump(&player); - - let args = DelegateCpiArgs { - valid_until: i64::MAX, - commit_frequency_ms: 1_000_000_000, - validator, - player, - }; + let (pda, _) = Pubkey::find_program_address( + &[user_seed, player_or_book_manager.as_ref()], + &crate::ID, + ); let delegate_accounts = DelegateAccounts::new(pda, program_id); let delegate_metas = DelegateAccountMetas::from(delegate_accounts); @@ -85,7 +124,22 @@ pub fn delegate_account_cpi_instruction( Instruction::new_with_borsh( program_id, - &ScheduleCommitInstruction::DelegateCpi(args), + &if user_seed == b"magic_schedule_commit" { + ScheduleCommitInstruction::DelegateCpi(DelegateCpiArgs { + valid_until: i64::MAX, + commit_frequency_ms: 1_000_000_000, + player: player_or_book_manager, + validator, + }) + } else { + ScheduleCommitInstruction::DelegateOrderBook( + DelegateOrderBookArgs { + commit_frequency_ms: 1_000_000_000, + book_manager: player_or_book_manager, + validator, + }, + ) + }, account_metas, ) } @@ -121,6 +175,45 @@ pub fn schedule_commit_cpi_instruction( ) } +pub fn update_order_book_instruction( + payer: Pubkey, + order_book: Pubkey, + update: BookUpdate, +) -> Instruction { + let program_id = crate::id(); + let account_metas = vec![ + AccountMeta::new(payer, true), + AccountMeta::new(order_book, false), + ]; + + Instruction::new_with_borsh( + program_id, + &ScheduleCommitInstruction::UpdateOrderBook(update), + account_metas, + ) +} + +pub fn schedule_commit_diff_instruction_for_order_book( + payer: Pubkey, + order_book: Pubkey, + magic_program_id: Pubkey, + magic_context_id: Pubkey, +) -> Instruction { + let program_id = crate::id(); + let account_metas = vec![ + AccountMeta::new(payer, true), + AccountMeta::new(order_book, false), + AccountMeta::new(magic_context_id, false), + AccountMeta::new_readonly(magic_program_id, false), + ]; + + Instruction::new_with_borsh( + program_id, + &ScheduleCommitInstruction::ScheduleCommitForOrderBook, + account_metas, + ) +} + pub fn schedule_commit_with_payer_cpi_instruction( payer: Pubkey, magic_program_id: Pubkey, diff --git a/test-integration/programs/schedulecommit/src/lib.rs b/test-integration/programs/schedulecommit/src/lib.rs index a90da8c26..44dfcb031 100644 --- a/test-integration/programs/schedulecommit/src/lib.rs +++ b/test-integration/programs/schedulecommit/src/lib.rs @@ -11,8 +11,12 @@ use solana_program::{ declare_id, entrypoint::{self, ProgramResult}, msg, + program::invoke, program_error::ProgramError, pubkey::Pubkey, + rent::Rent, + system_instruction, + sysvar::Sysvar, }; use crate::{ @@ -24,8 +28,12 @@ use crate::{ }; pub mod api; pub mod magicblock_program; +mod order_book; mod utils; +use order_book::*; +pub use order_book::{BookUpdate, OrderBookOwned, OrderLevel}; + declare_id!("9hgprgZiRWmy8KkfvUuaVkDGrqo9GzeXMohwq6BazgUY"); #[cfg(not(feature = "no-entrypoint"))] @@ -39,6 +47,13 @@ pub struct DelegateCpiArgs { validator: Option, } +#[derive(BorshSerialize, BorshDeserialize, Debug, Clone)] +pub struct DelegateOrderBookArgs { + commit_frequency_ms: u32, + book_manager: Pubkey, + validator: Option, +} + #[derive(BorshSerialize, BorshDeserialize, Debug, Clone)] pub struct ScheduleCommitCpiArgs { /// Pubkeys of players from which PDAs were derived @@ -105,6 +120,19 @@ pub enum ScheduleCommitInstruction { // // It is not part of this enum as it has a custom discriminator // Undelegate, + /// Initialize an OrderBook + InitOrderBook, + + GrowOrderBook(u64), // additional_space + + /// Delegate order book to ER nodes + DelegateOrderBook(DelegateOrderBookArgs), + + /// Update order book + UpdateOrderBook(BookUpdate), + + /// ScheduleCommitDiffCpi + ScheduleCommitForOrderBook, } pub fn process_instruction<'a>( @@ -128,6 +156,7 @@ pub fn process_instruction<'a>( msg!("ERROR: failed to parse instruction data {:?}", err); ProgramError::InvalidArgument })?; + use ScheduleCommitInstruction::*; match ix { Init => process_init(program_id, accounts), @@ -147,6 +176,15 @@ pub fn process_instruction<'a>( ) } IncreaseCount => process_increase_count(accounts), + InitOrderBook => process_init_order_book(accounts), + GrowOrderBook(additional_space) => { + process_grow_order_book(accounts, additional_space) + } + DelegateOrderBook(args) => process_delegate_order_book(accounts, args), + UpdateOrderBook(args) => process_update_order_book(accounts, args), + ScheduleCommitForOrderBook => { + process_schedulecommit_for_orderbook(accounts) + } } } @@ -160,7 +198,7 @@ pub struct MainAccount { } impl MainAccount { - pub const SIZE: usize = std::mem::size_of::(); + pub const SIZE: u64 = std::mem::size_of::() as u64; pub fn try_decode(data: &[u8]) -> std::io::Result { Self::try_from_slice(data) @@ -234,6 +272,170 @@ fn process_init<'a>( Ok(()) } +// ----------------- +// InitOrderBook +// ----------------- +fn process_init_order_book<'a>( + accounts: &'a [AccountInfo<'a>], +) -> entrypoint::ProgramResult { + msg!("Init OrderBook account"); + let [payer, book_manager, order_book, _system_program] = accounts else { + return Err(ProgramError::NotEnoughAccountKeys); + }; + + assert_is_signer(payer, "payer")?; + + let (pda, bump) = Pubkey::find_program_address( + &[b"order_book", book_manager.key.as_ref()], + &crate::ID, + ); + + assert_keys_equal(order_book.key, &pda, || { + format!( + "PDA for the account ('{}') and for book_manager ('{}') is incorrect", + order_book.key, book_manager.key + ) + })?; + + allocate_account_and_assign_owner(AllocateAndAssignAccountArgs { + payer_info: payer, + account_info: order_book, + owner: &crate::ID, + signer_seeds: &[b"order_book", book_manager.key.as_ref(), &[bump]], + size: 10 * 1024, + })?; + + Ok(()) +} + +fn process_grow_order_book<'a>( + accounts: &'a [AccountInfo<'a>], + additional_space: u64, +) -> entrypoint::ProgramResult { + msg!("Grow OrderBook account"); + let [payer, book_manager, order_book, system_program] = accounts else { + return Err(ProgramError::NotEnoughAccountKeys); + }; + + assert_is_signer(payer, "payer")?; + + let (pda, _bump) = Pubkey::find_program_address( + &[b"order_book", book_manager.key.as_ref()], + &crate::ID, + ); + + assert_keys_equal(order_book.key, &pda, || { + format!( + "PDA for the account ('{}') and for book_manager ('{}') is incorrect", + order_book.key, payer.key + ) + })?; + + let new_size = order_book.data_len() + additional_space as usize; + + // Ideally, we should transfer some lamports from payer to order_book + // so that realloc could use it + + let rent = Rent::get()?; + let required = rent.minimum_balance(new_size); + let current = order_book.lamports(); + if current < required { + let diff = required - current; + invoke( + &system_instruction::transfer(payer.key, order_book.key, diff), + &[payer.clone(), order_book.clone(), system_program.clone()], + )?; + } + + order_book.realloc(new_size, true)?; + + Ok(()) +} + +// ----------------- +// Delegate OrderBook +// ----------------- +pub fn process_delegate_order_book( + accounts: &[AccountInfo], + args: DelegateOrderBookArgs, +) -> Result<(), ProgramError> { + msg!("Processing delegate_order_book instruction"); + + let [payer, order_book, owner_program, buffer, delegation_record, delegation_metadata, delegation_program, system_program] = + accounts + else { + return Err(ProgramError::NotEnoughAccountKeys); + }; + + let seeds_no_bump = [b"order_book", args.book_manager.as_ref()]; + + delegate_account( + DelegateAccounts { + payer, + pda: order_book, + buffer, + delegation_record, + delegation_metadata, + owner_program, + delegation_program, + system_program, + }, + &seeds_no_bump, + DelegateConfig { + commit_frequency_ms: args.commit_frequency_ms, + validator: args.validator, + }, + )?; + + Ok(()) +} + +// ----------------- +// UpdateOrderBook +// ----------------- +fn process_update_order_book<'a>( + accounts: &'a [AccountInfo<'a>], + updates: BookUpdate, +) -> entrypoint::ProgramResult { + msg!("Update orderbook"); + let account_info_iter = &mut accounts.iter(); + let payer_info = next_account_info(account_info_iter)?; + let order_book_account = next_account_info(account_info_iter)?; + + assert_is_signer(payer_info, "payer")?; + + let mut book_raw = order_book_account.try_borrow_mut_data()?; + + OrderBook::new(&mut book_raw).update_from(updates); + + Ok(()) +} + +// ----------------- +// Schedule Commit +// ----------------- +pub fn process_schedulecommit_for_orderbook( + accounts: &[AccountInfo], +) -> Result<(), ProgramError> { + msg!("Processing schedulecommit (for orderbook) instruction"); + + let [payer, order_book_account, magic_context, magic_program] = accounts + else { + return Err(ProgramError::NotEnoughAccountKeys); + }; + + assert_is_signer(payer, "payer")?; + + commit_and_undelegate_accounts( + payer, + vec![order_book_account], + magic_context, + magic_program, + )?; + + Ok(()) +} + // ----------------- // Delegate // ----------------- diff --git a/test-integration/programs/schedulecommit/src/order_book.rs b/test-integration/programs/schedulecommit/src/order_book.rs new file mode 100644 index 000000000..12d655080 --- /dev/null +++ b/test-integration/programs/schedulecommit/src/order_book.rs @@ -0,0 +1,227 @@ +use std::{ + mem::{align_of, size_of}, + slice, +}; + +use borsh::{BorshDeserialize, BorshSerialize}; +use static_assertions::const_assert; + +#[repr(C)] +#[derive( + BorshSerialize, BorshDeserialize, Debug, Clone, Copy, Default, PartialEq, Eq, +)] +pub struct OrderLevel { + pub price: u64, // ideally both fields could be some decimal value + pub size: u64, +} + +const_assert!(align_of::() == align_of::()); +const_assert!(size_of::() == 16); + +#[derive(BorshSerialize, BorshDeserialize, Debug, Clone, Default)] +pub struct BookUpdate { + pub bids: Vec, + pub asks: Vec, +} + +#[repr(C)] +pub struct OrderBookHeader { + pub bids_len: u32, + pub asks_len: u32, +} + +const_assert!(align_of::() == align_of::()); +const_assert!(size_of::() == 8); + +const ORDER_LEVEL_SIZE: usize = std::mem::size_of::(); +const HEADER_SIZE: usize = std::mem::size_of::(); + +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct OrderBookOwned { + pub bids: Vec, + pub asks: Vec, +} + +impl From<&OrderBook<'_>> for OrderBookOwned { + fn from(book_ref: &OrderBook) -> Self { + let mut book = Self::default(); + book.bids.extend_from_slice(book_ref.bids()); + book.asks.extend(book_ref.asks_reversed().iter().rev()); + book + } +} + +impl borsh::de::BorshDeserialize for OrderBookOwned { + fn deserialize(buf: &mut &[u8]) -> Result { + let (book_bytes, rest) = buf.split_at(buf.len()); + *buf = rest; // rest is actually empty + + // I make a copy so that I can get mutable bytes in the unsafe block below. + // I could take mutable bytes from &[u8] as well and unsafe block will not + // stop me, but that would break aliasing rules and therefore would invoke UB. + // It's a test code, so copying should be OK. + let book_bytes = { + let mut aligned = rkyv::AlignedVec::with_capacity(book_bytes.len()); + aligned.extend_from_slice(book_bytes); + aligned + }; + + Ok(Self::from(&OrderBook::new(unsafe { + slice::from_raw_parts_mut( + book_bytes.as_ptr() as *mut u8, + book_bytes.len(), + ) + }))) + } + fn deserialize_reader( + _reader: &mut R, + ) -> ::core::result::Result { + unimplemented!("deserialize_reader() not implemented. Please use buffer version as it needs to know size of the buffer") + } +} +pub struct OrderBook<'a> { + header: &'a mut OrderBookHeader, + capacity: usize, + levels: *mut OrderLevel, +} + +impl<'a> OrderBook<'a> { + // + // ========= Zero-Copy Order Book ========== + // + // ----------------------------------------- + // | account data | + // ----------------------------------------- + // | header | levels | + // ----------------------------------------- + // | asks grows -> <- bids grows | + // ----------------------------------------- + // + // Note: + // + // - asks grows towards right + // - bids grows towards left + // + pub fn new(data: &'a mut [u8]) -> Self { + let (header_bytes, levels_bytes) = data.split_at_mut(HEADER_SIZE); + + assert!( + header_bytes + .as_ptr() + .align_offset(align_of::()) + == 0 + && levels_bytes.as_ptr().align_offset(align_of::()) + == 0, + "data is not properly aligned for OrderBook to be constructed" + ); + + let capacity = levels_bytes.len() / ORDER_LEVEL_SIZE; + let header = + unsafe { &mut *(header_bytes.as_ptr() as *mut OrderBookHeader) }; + + // Ensure header lengths never exceed backing storage; if they do, treat it as corrupted + // data and panic rather than creating out-of-bounds slices in `bids()`/`asks_reversed()`. + let used = header.bids_len as usize + header.asks_len as usize; + + assert!( + used <= capacity, + "OrderBook header lengths (bids_len + asks_len) exceed capacity" + ); + + Self { + header, + capacity, + levels: levels_bytes.as_mut_ptr() as *mut OrderLevel, + } + } + + pub fn update_from(&mut self, updates: BookUpdate) { + self.add_bids(&updates.bids); + self.add_asks(&updates.asks); + } + + pub fn add_bids( + &mut self, + bids: &[OrderLevel], + ) -> Option<&'a [OrderLevel]> { + if self.remaining_capacity() < bids.len() { + return None; + } + let new_bids_len = self.bids_len() + bids.len(); + let bids_space = + unsafe { self.bids_with_uninitialized_slots(new_bids_len) }; + + bids_space[self.bids_len()..].copy_from_slice(bids); + self.header.bids_len = new_bids_len as u32; + + Some(bids_space) + } + + pub fn add_asks( + &mut self, + asks: &[OrderLevel], + ) -> Option<&'a [OrderLevel]> { + if self.remaining_capacity() < asks.len() { + return None; + } + let new_asks_len = self.asks_len() + asks.len(); + let asks_space = + unsafe { self.asks_with_uninitialized_slots(new_asks_len) }; + + // copy in the reverse order + for (dst, src) in + asks_space[..asks.len()].iter_mut().zip(asks.iter().rev()) + { + *dst = *src; + } + self.header.asks_len = new_asks_len as u32; + + Some(asks_space) + } + + pub fn bids(&self) -> &'a [OrderLevel] { + unsafe { slice::from_raw_parts(self.levels, self.bids_len()) } + } + + /// Note that the returned slice is in reverse order, means the first entry is the latest + /// entry and the last entry is the oldest entry. + pub fn asks_reversed(&self) -> &'a [OrderLevel] { + unsafe { + slice::from_raw_parts( + self.levels.add(self.capacity - self.asks_len()), + self.asks_len(), + ) + } + } + + pub fn bids_len(&self) -> usize { + self.header.bids_len as usize + } + + pub fn asks_len(&self) -> usize { + self.header.asks_len as usize + } + + unsafe fn bids_with_uninitialized_slots( + &mut self, + bids_len: usize, + ) -> &'a mut [OrderLevel] { + slice::from_raw_parts_mut(self.levels, bids_len) + } + + unsafe fn asks_with_uninitialized_slots( + &mut self, + asks_len: usize, + ) -> &'a mut [OrderLevel] { + slice::from_raw_parts_mut( + self.levels.add(self.capacity - asks_len), + asks_len, + ) + } + + fn remaining_capacity(&self) -> usize { + self.capacity + .checked_sub((self.header.bids_len + self.header.asks_len) as usize) + .expect("remaining_capacity must exist") + } +} diff --git a/test-integration/programs/schedulecommit/src/utils/mod.rs b/test-integration/programs/schedulecommit/src/utils/mod.rs index 7de5b1250..bb1bd9086 100644 --- a/test-integration/programs/schedulecommit/src/utils/mod.rs +++ b/test-integration/programs/schedulecommit/src/utils/mod.rs @@ -50,7 +50,7 @@ pub struct AllocateAndAssignAccountArgs<'a, 'b> { pub payer_info: &'a AccountInfo<'a>, pub account_info: &'a AccountInfo<'a>, pub owner: &'a Pubkey, - pub size: usize, + pub size: u64, pub signer_seeds: &'b [&'b [u8]], } @@ -68,10 +68,16 @@ pub fn allocate_account_and_assign_owner( } = args; let required_lamports = rent - .minimum_balance(size) + .minimum_balance(size as usize) .max(1) .saturating_sub(account_info.lamports()); + msg!( + "required_lamports: {}, payer has {}", + required_lamports, + payer_info.lamports() + ); + // 1. Transfer the required rent to the account if required_lamports > 0 { transfer_lamports(payer_info, account_info, required_lamports)?; @@ -81,10 +87,7 @@ pub fn allocate_account_and_assign_owner( // At this point the account is still owned by the system program msg!(" create_account() allocate space"); invoke_signed( - &system_instruction::allocate( - account_info.key, - size.try_into().unwrap(), - ), + &system_instruction::allocate(account_info.key, size), // 0. `[WRITE, SIGNER]` New account &[account_info.clone()], &[signer_seeds], diff --git a/test-integration/schedulecommit/client/src/schedule_commit_context.rs b/test-integration/schedulecommit/client/src/schedule_commit_context.rs index de5383e8b..25dbbbe7f 100644 --- a/test-integration/schedulecommit/client/src/schedule_commit_context.rs +++ b/test-integration/schedulecommit/client/src/schedule_commit_context.rs @@ -5,7 +5,7 @@ use integration_test_tools::IntegrationTestContext; use log::*; use program_schedulecommit::api::{ delegate_account_cpi_instruction, init_account_instruction, - init_payer_escrow, pda_and_bump, + init_order_book_instruction, init_payer_escrow, }; use solana_rpc_client::rpc_client::{RpcClient, SerializableTransaction}; use solana_rpc_client_api::config::RpcSendTransactionConfig; @@ -13,6 +13,7 @@ use solana_rpc_client_api::config::RpcSendTransactionConfig; use solana_sdk::signer::SeedDerivable; use solana_sdk::{ commitment_config::CommitmentConfig, + compute_budget::ComputeBudgetInstruction, hash::Hash, native_token::LAMPORTS_PER_SOL, pubkey::Pubkey, @@ -30,6 +31,7 @@ pub struct ScheduleCommitTestContext { pub payer_ephem: Keypair, // The Payer keypairs along with its PDA pubkey which we'll commit pub committees: Vec<(Keypair, Pubkey)>, + user_seed: Vec, common_ctx: IntegrationTestContext, } @@ -61,14 +63,21 @@ impl ScheduleCommitTestContext { // ----------------- // Init // ----------------- - pub fn try_new_random_keys(ncommittees: usize) -> Result { - Self::try_new_internal(ncommittees, true) + pub fn try_new_random_keys( + ncommittees: usize, + user_seed: &[u8], + ) -> Result { + Self::try_new_internal(ncommittees, true, user_seed) } - pub fn try_new(ncommittees: usize) -> Result { - Self::try_new_internal(ncommittees, false) + pub fn try_new(ncommittees: usize, user_seed: &[u8]) -> Result { + Self::try_new_internal(ncommittees, false, user_seed) } - fn try_new_internal(ncommittees: usize, random_keys: bool) -> Result { + fn try_new_internal( + ncommittees: usize, + random_keys: bool, + user_seed: &[u8], + ) -> Result { let ictx = IntegrationTestContext::try_new()?; let payer_chain = if random_keys { @@ -103,7 +112,10 @@ impl ScheduleCommitTestContext { lamports, ) .unwrap(); - let (pda, _) = pda_and_bump(&payer_ephem.pubkey()); + let (pda, _bump) = Pubkey::find_program_address( + &[user_seed, payer_ephem.pubkey().as_ref()], + &program_schedulecommit::ID, + ); (payer_ephem, pda) }) .collect::>(); @@ -143,6 +155,7 @@ impl ScheduleCommitTestContext { payer_ephem, committees, common_ctx: ictx, + user_seed: user_seed.to_vec(), }) } @@ -150,17 +163,52 @@ impl ScheduleCommitTestContext { // Schedule Commit specific Transactions // ----------------- pub fn init_committees(&self) -> Result { - let ixs = self - .committees - .iter() - .map(|(player, committee)| { - init_account_instruction( - self.payer_chain.pubkey(), - player.pubkey(), - *committee, - ) - }) - .collect::>(); + let mut ixs = vec![ + ComputeBudgetInstruction::set_compute_unit_limit(1_400_000), + ComputeBudgetInstruction::set_compute_unit_price(10_000), + ]; + match self.user_seed.as_slice() { + b"magic_schedule_commit" => { + ixs.extend(self.committees.iter().map( + |(player, committee)| { + init_account_instruction( + self.payer_chain.pubkey(), + player.pubkey(), + *committee, + ) + }, + )); + } + b"order_book" => { + ixs.extend(self.committees.iter().map( + |(book_manager, committee)| { + init_order_book_instruction( + self.payer_chain.pubkey(), + book_manager.pubkey(), + *committee, + ) + }, + )); + + //// TODO (snawaz): currently the size of delegatable-account cannot be + //// more than 10K, else delegation will fail. So Let's revisit this when + //// we relax the limit on the account size, then we can use larger + //// account, say even 10 MB, and execute CommitDiff. + // + // ixs.extend(self.committees.iter().flat_map( + // |(payer, committee)| { + // [grow_order_book_instruction( + // payer.pubkey(), + // *committee, + // 10 * 1024 + // )] + // }, + // )); + } + _ => { + return Err(anyhow::anyhow!("Unsupported user_seed: {:?} ; expected b\"magic_schedule_commit\" or b\"order_book\"", self.user_seed)); + } + }; let mut signers = self .committees @@ -224,6 +272,7 @@ impl ScheduleCommitTestContext { self.payer_chain.pubkey(), self.ephem_validator_identity, player.pubkey(), + &self.user_seed, ); ixs.push(ix); } diff --git a/test-integration/schedulecommit/client/src/verify.rs b/test-integration/schedulecommit/client/src/verify.rs index 11098f9c9..d383cbc2f 100644 --- a/test-integration/schedulecommit/client/src/verify.rs +++ b/test-integration/schedulecommit/client/src/verify.rs @@ -1,5 +1,5 @@ use integration_test_tools::scheduled_commits::ScheduledCommitResult; -use program_schedulecommit::MainAccount; +use program_schedulecommit::{MainAccount, OrderBookOwned}; use solana_sdk::signature::Signature; use crate::ScheduleCommitTestContext; @@ -12,3 +12,12 @@ pub fn fetch_and_verify_commit_result_from_logs( res.confirm_commit_transactions_on_chain(ctx).unwrap(); res } + +pub fn fetch_and_verify_order_book_commit_result_from_logs( + ctx: &ScheduleCommitTestContext, + sig: Signature, +) -> ScheduledCommitResult { + let res = ctx.fetch_schedule_commit_result(sig).unwrap(); + res.confirm_commit_transactions_on_chain(ctx).unwrap(); + res +} diff --git a/test-integration/schedulecommit/elfs/dlp.so b/test-integration/schedulecommit/elfs/dlp.so index f07df31f3..decfd0f00 100755 Binary files a/test-integration/schedulecommit/elfs/dlp.so and b/test-integration/schedulecommit/elfs/dlp.so differ diff --git a/test-integration/schedulecommit/test-scenarios/Cargo.toml b/test-integration/schedulecommit/test-scenarios/Cargo.toml index 93d93863a..3cda55a8f 100644 --- a/test-integration/schedulecommit/test-scenarios/Cargo.toml +++ b/test-integration/schedulecommit/test-scenarios/Cargo.toml @@ -16,3 +16,5 @@ solana-rpc-client = { workspace = true } solana-rpc-client-api = { workspace = true } solana-sdk = { workspace = true } test-kit = { workspace = true } +rand = { workspace = true } +borsh = { workspace = true } diff --git a/test-integration/schedulecommit/test-scenarios/tests/01_commits.rs b/test-integration/schedulecommit/test-scenarios/tests/01_commits.rs index 8a4015a95..264b4511a 100644 --- a/test-integration/schedulecommit/test-scenarios/tests/01_commits.rs +++ b/test-integration/schedulecommit/test-scenarios/tests/01_commits.rs @@ -43,7 +43,8 @@ mod utils; #[test] fn test_committing_one_account() { run_test!({ - let ctx = get_context_with_delegated_committees(1); + let ctx = + get_context_with_delegated_committees(1, b"magic_schedule_commit"); let ScheduleCommitTestContextFields { payer_ephem: payer, @@ -96,7 +97,8 @@ fn test_committing_one_account() { #[test] fn test_committing_two_accounts() { run_test!({ - let ctx = get_context_with_delegated_committees(2); + let ctx = + get_context_with_delegated_committees(2, b"magic_schedule_commit"); let ScheduleCommitTestContextFields { payer_ephem: payer, @@ -227,6 +229,7 @@ fn init_and_delegate_player( payer.pubkey(), validator, player.pubkey(), + b"magic_schedule_commit", ); // Send transaction diff --git a/test-integration/schedulecommit/test-scenarios/tests/02_commit_and_undelegate.rs b/test-integration/schedulecommit/test-scenarios/tests/02_commit_and_undelegate.rs index 120650a70..59d6b3065 100644 --- a/test-integration/schedulecommit/test-scenarios/tests/02_commit_and_undelegate.rs +++ b/test-integration/schedulecommit/test-scenarios/tests/02_commit_and_undelegate.rs @@ -4,10 +4,17 @@ use integration_test_tools::{ transactions::send_and_confirm_instructions_with_payer, }; use log::*; -use program_schedulecommit::api::{ - increase_count_instruction, schedule_commit_and_undelegate_cpi_instruction, - schedule_commit_and_undelegate_cpi_with_mod_after_instruction, +use program_schedulecommit::{ + api::{ + increase_count_instruction, + schedule_commit_and_undelegate_cpi_instruction, + schedule_commit_and_undelegate_cpi_with_mod_after_instruction, + schedule_commit_diff_instruction_for_order_book, + update_order_book_instruction, + }, + BookUpdate, OrderLevel, }; +use rand::{RngCore, SeedableRng}; use schedulecommit_client::{ verify, ScheduleCommitTestContext, ScheduleCommitTestContextFields, }; @@ -45,7 +52,8 @@ fn commit_and_undelegate_one_account( Signature, Result, ) { - let ctx = get_context_with_delegated_committees(1); + let ctx = + get_context_with_delegated_committees(1, b"magic_schedule_commit"); let ScheduleCommitTestContextFields { payer_ephem: payer, committees, @@ -99,6 +107,62 @@ fn commit_and_undelegate_one_account( (ctx, *sig, tx_res) } +fn commit_and_undelegate_order_book_account( + update: BookUpdate, +) -> ( + ScheduleCommitTestContext, + Signature, + Result, +) { + let ctx = get_context_with_delegated_committees(1, b"order_book"); + let ScheduleCommitTestContextFields { + payer_ephem, + committees, + commitment, + ephem_client, + .. + } = ctx.fields(); + + assert_eq!(committees.len(), 1); + + let ixs = [ + update_order_book_instruction( + payer_ephem.pubkey(), + committees[0].1, + update, + ), + schedule_commit_diff_instruction_for_order_book( + payer_ephem.pubkey(), + committees[0].1, + magicblock_magic_program_api::id(), + magicblock_magic_program_api::MAGIC_CONTEXT_PUBKEY, + ), + ]; + + let ephem_blockhash = ephem_client.get_latest_blockhash().unwrap(); + let tx = Transaction::new_signed_with_payer( + &ixs, + Some(&payer_ephem.pubkey()), + &[&payer_ephem], + ephem_blockhash, + ); + + let sig = tx.get_signature(); + let tx_res = ephem_client + .send_and_confirm_transaction_with_spinner_and_config( + &tx, + *commitment, + RpcSendTransactionConfig { + skip_preflight: true, + ..Default::default() + }, + ); + println!("txhash (scheduled_commit): {:?}", tx_res); + + debug!("Commit and Undelegate Transaction result: '{:?}'", tx_res); + (ctx, *sig, tx_res) +} + fn commit_and_undelegate_two_accounts( modify_after: bool, ) -> ( @@ -106,7 +170,8 @@ fn commit_and_undelegate_two_accounts( Signature, Result, ) { - let ctx = get_context_with_delegated_committees(2); + let ctx = + get_context_with_delegated_committees(2, b"magic_schedule_commit"); let ScheduleCommitTestContextFields { payer_ephem: payer, committees, @@ -176,6 +241,76 @@ fn test_committing_and_undelegating_one_account() { }); } +#[test] +fn test_committing_and_undelegating_huge_order_book_account() { + run_test!({ + let (rng_seed, update) = { + use rand::{ + rngs::{OsRng, StdRng}, + Rng, + }; + let rng_seed = OsRng.next_u64(); + println!("Important: use {rng_seed} as seed to regenerate the random inputs in case of test failure"); + let mut random = StdRng::seed_from_u64(rng_seed); + let mut update = BookUpdate::default(); + update.bids.extend((0..random.gen_range(5..10)).map(|_| { + OrderLevel { + price: random.gen_range(75000..90000), + size: random.gen_range(1..10), + } + })); + update.asks.extend((0..random.gen_range(5..10)).map(|_| { + OrderLevel { + price: random.gen_range(125000..150000), + size: random.gen_range(1..10), + } + })); + println!( + "BookUpdate: total = {}, bids = {}, asks = {}", + update.bids.len() + update.asks.len(), + update.bids.len(), + update.asks.len() + ); + (rng_seed, update) + }; + let (ctx, sig, tx_res) = + commit_and_undelegate_order_book_account(update.clone()); + info!("'{}' {:?}", sig, tx_res); + + let res = verify::fetch_and_verify_order_book_commit_result_from_logs( + &ctx, sig, + ); + + let book = res + .included + .values() + .next() + .expect("one order-book must exist"); + + assert_eq!( + book.bids.len(), + update.bids.len(), + "Use {rng_seed} to generate the input and investigate" + ); + assert_eq!( + book.asks.len(), + update.asks.len(), + "Use {rng_seed} to generate the input and investigate" + ); + assert_eq!( + book.bids, update.bids, + "Use {rng_seed} to generate the input and investigate" + ); + assert_eq!( + book.asks, update.asks, + "Use {rng_seed} to generate the input and investigate" + ); + + assert_one_committee_was_committed(&ctx, &res, true); + assert_one_committee_account_was_undelegated_on_chain(&ctx); + }); +} + #[test] fn test_committing_and_undelegating_two_accounts_success() { run_test!({ diff --git a/test-integration/schedulecommit/test-scenarios/tests/utils/mod.rs b/test-integration/schedulecommit/test-scenarios/tests/utils/mod.rs index eb8f80dfb..9d8952c9a 100644 --- a/test-integration/schedulecommit/test-scenarios/tests/utils/mod.rs +++ b/test-integration/schedulecommit/test-scenarios/tests/utils/mod.rs @@ -15,16 +15,22 @@ use solana_sdk::{ // ----------------- pub fn get_context_with_delegated_committees( ncommittees: usize, + user_seed: &[u8], ) -> ScheduleCommitTestContext { let ctx = if std::env::var("FIXED_KP").is_ok() { - ScheduleCommitTestContext::try_new(ncommittees) + ScheduleCommitTestContext::try_new(ncommittees, user_seed) } else { - ScheduleCommitTestContext::try_new_random_keys(ncommittees) + ScheduleCommitTestContext::try_new_random_keys(ncommittees, user_seed) } .unwrap(); + println!("get_context_with_delegated_committees inside"); + + let txhash = ctx.init_committees().unwrap(); + println!("txhash (init_committees): {}", txhash); + + let txhash = ctx.delegate_committees().unwrap(); + println!("txhash (delegate_committees): {}", txhash); - ctx.init_committees().unwrap(); - ctx.delegate_committees().unwrap(); ctx } @@ -32,11 +38,13 @@ pub fn get_context_with_delegated_committees( // Asserts // ----------------- #[allow(dead_code)] // used in 02_commit_and_undelegate.rs -pub fn assert_one_committee_was_committed( +pub fn assert_one_committee_was_committed( ctx: &ScheduleCommitTestContext, - res: &ScheduledCommitResult, + res: &ScheduledCommitResult, is_single_stage: bool, -) { +) where + T: std::fmt::Debug + borsh::BorshDeserialize + PartialEq + Eq, +{ let pda = ctx.committees[0].1; assert_eq!(res.included.len(), 1, "includes 1 pda"); diff --git a/test-integration/schedulecommit/test-security/tests/01_invocations.rs b/test-integration/schedulecommit/test-security/tests/01_invocations.rs index b05168035..068c65193 100644 --- a/test-integration/schedulecommit/test-security/tests/01_invocations.rs +++ b/test-integration/schedulecommit/test-security/tests/01_invocations.rs @@ -27,9 +27,12 @@ const NEEDS_TO_BE_OWNED_BY_INVOKING_PROGRAM: &str = fn prepare_ctx_with_account_to_commit() -> ScheduleCommitTestContext { let ctx = if std::env::var("FIXED_KP").is_ok() { - ScheduleCommitTestContext::try_new(2) + ScheduleCommitTestContext::try_new(2, b"magic_schedule_commit") } else { - ScheduleCommitTestContext::try_new_random_keys(2) + ScheduleCommitTestContext::try_new_random_keys( + 2, + b"magic_schedule_commit", + ) } .unwrap(); ctx.init_committees().unwrap(); diff --git a/test-integration/test-committor-service/tests/common.rs b/test-integration/test-committor-service/tests/common.rs index 9966ff56a..ecef5d3e2 100644 --- a/test-integration/test-committor-service/tests/common.rs +++ b/test-integration/test-committor-service/tests/common.rs @@ -12,16 +12,16 @@ use magicblock_committor_service::{ task_info_fetcher::{ ResetType, TaskInfoFetcher, TaskInfoFetcherResult, }, - IntentExecutorImpl, + IntentExecutorImpl, NullTaskInfoFetcher, }, - tasks::CommitTask, + tasks::{CommitTask, CommitTaskBuilder}, transaction_preparator::{ delivery_preparator::DeliveryPreparator, TransactionPreparatorImpl, }, ComputeBudgetConfig, }; use magicblock_program::magic_scheduled_base_intent::CommittedAccount; -use magicblock_rpc_client::MagicblockRpcClient; +use magicblock_rpc_client::{MagicBlockRpcClientResult, MagicblockRpcClient}; use magicblock_table_mania::{GarbageCollectorConfig, TableMania}; use solana_account::Account; use solana_pubkey::Pubkey; @@ -104,17 +104,22 @@ impl TestFixture { ) -> IntentExecutorImpl { let transaction_preparator = self.create_transaction_preparator(); - let task_info_fetcher = Arc::new(MockTaskInfoFetcher); IntentExecutorImpl::new( self.rpc_client.clone(), transaction_preparator, - task_info_fetcher, + self.create_task_info_fetcher(), ) } + + #[allow(dead_code)] + pub fn create_task_info_fetcher(&self) -> Arc { + Arc::new(MockTaskInfoFetcher(self.rpc_client.clone())) + } } -pub struct MockTaskInfoFetcher; +pub struct MockTaskInfoFetcher(MagicblockRpcClient); + #[async_trait] impl TaskInfoFetcher for MockTaskInfoFetcher { async fn fetch_next_commit_ids( @@ -136,6 +141,13 @@ impl TaskInfoFetcher for MockTaskInfoFetcher { } fn reset(&self, _: ResetType) {} + + async fn get_base_account( + &self, + pubkey: &Pubkey, + ) -> MagicBlockRpcClientResult> { + self.0.get_account(pubkey).await + } } #[allow(dead_code)] @@ -147,12 +159,12 @@ pub fn generate_random_bytes(length: usize) -> Vec { } #[allow(dead_code)] -pub fn create_commit_task(data: &[u8]) -> CommitTask { +pub async fn create_commit_task(data: &[u8]) -> CommitTask { static COMMIT_ID: AtomicU64 = AtomicU64::new(0); - CommitTask { - commit_id: COMMIT_ID.fetch_add(1, Ordering::Relaxed), - allow_undelegation: false, - committed_account: CommittedAccount { + CommitTaskBuilder::create_commit_task( + COMMIT_ID.fetch_add(1, Ordering::Relaxed), + false, + CommittedAccount { pubkey: Pubkey::new_unique(), account: Account { lamports: 1000, @@ -162,7 +174,9 @@ pub fn create_commit_task(data: &[u8]) -> CommitTask { rent_epoch: 0, }, }, - } + &Arc::new(NullTaskInfoFetcher), + ) + .await } #[allow(dead_code)] diff --git a/test-integration/test-committor-service/tests/test_delivery_preparator.rs b/test-integration/test-committor-service/tests/test_delivery_preparator.rs index 5c48fa22b..35f9de4b3 100644 --- a/test-integration/test-committor-service/tests/test_delivery_preparator.rs +++ b/test-integration/test-committor-service/tests/test_delivery_preparator.rs @@ -1,4 +1,5 @@ use borsh::BorshDeserialize; +use futures::future::join_all; use magicblock_committor_program::Chunks; use magicblock_committor_service::{ persist::IntentPersisterImpl, @@ -21,7 +22,7 @@ async fn test_prepare_10kb_buffer() { let preparator = fixture.create_delivery_preparator(); let data = generate_random_bytes(10 * 1024); - let buffer_task = BufferTaskType::Commit(create_commit_task(&data)); + let buffer_task = BufferTaskType::Commit(create_commit_task(&data).await); let mut strategy = TransactionStrategy { optimized_tasks: vec![Box::new(BufferTask::new_preparation_required( buffer_task, @@ -86,15 +87,13 @@ async fn test_prepare_multiple_buffers() { generate_random_bytes(10), generate_random_bytes(500 * 1024), ]; - let buffer_tasks = datas - .iter() - .map(|data| { - let task = - BufferTaskType::Commit(create_commit_task(data.as_slice())); - Box::new(BufferTask::new_preparation_required(task)) - as Box - }) - .collect(); + let buffer_tasks = join_all(datas.iter().map(|data| async { + let task = + BufferTaskType::Commit(create_commit_task(data.as_slice()).await); + Box::new(BufferTask::new_preparation_required(task)) + as Box + })) + .await; let mut strategy = TransactionStrategy { optimized_tasks: buffer_tasks, lookup_tables_keys: vec![], @@ -165,14 +164,12 @@ async fn test_lookup_tables() { generate_random_bytes(20), generate_random_bytes(30), ]; - let tasks = datas - .iter() - .map(|data| { - let task = - ArgsTaskType::Commit(create_commit_task(data.as_slice())); - Box::::new(task.into()) as Box - }) - .collect::>(); + let tasks = join_all(datas.iter().map(|data| async { + let task = + ArgsTaskType::Commit(create_commit_task(data.as_slice()).await); + Box::::new(task.into()) as Box + })) + .await; let lookup_tables_keys = TaskStrategist::collect_lookup_table_keys( &fixture.authority.pubkey(), @@ -212,7 +209,7 @@ async fn test_already_initialized_error_handled() { let preparator = fixture.create_delivery_preparator(); let data = generate_random_bytes(10 * 1024); - let mut task = create_commit_task(&data); + let mut task = create_commit_task(&data).await; let buffer_task = BufferTaskType::Commit(task.clone()); let mut strategy = TransactionStrategy { optimized_tasks: vec![Box::new(BufferTask::new_preparation_required( @@ -301,9 +298,9 @@ async fn test_prepare_cleanup_and_reprepare_mixed_tasks() { let buf_b_data = generate_random_bytes(64 * 1024 + 3); // Keep these around to modify data later (same commit IDs, different data) - let mut commit_args = create_commit_task(&args_data); - let mut commit_a = create_commit_task(&buf_a_data); - let mut commit_b = create_commit_task(&buf_b_data); + let mut commit_args = create_commit_task(&args_data).await; + let mut commit_a = create_commit_task(&buf_a_data).await; + let mut commit_b = create_commit_task(&buf_b_data).await; let mut strategy = TransactionStrategy { optimized_tasks: vec![ diff --git a/test-integration/test-committor-service/tests/test_ix_commit_local.rs b/test-integration/test-committor-service/tests/test_ix_commit_local.rs index 4281d3149..68746d1c0 100644 --- a/test-integration/test-committor-service/tests/test_ix_commit_local.rs +++ b/test-integration/test-committor-service/tests/test_ix_commit_local.rs @@ -61,6 +61,7 @@ fn expect_strategies( // ----------------- // Single Account Commits // ----------------- + #[tokio::test] async fn test_ix_commit_single_account_100_bytes() { commit_single_account(100, CommitStrategy::Args, false).await; @@ -147,12 +148,13 @@ async fn commit_single_account( .await; } -// TODO(thlorenz): once delegation program supports larger commits +// TODO(thlorenz/snawaz): once delegation program supports larger commits // add 1MB and 10MB tests // ----------------- // Multiple Account Commits // ----------------- + #[tokio::test] async fn test_ix_commit_two_accounts_1kb_2kb() { init_logger!(); @@ -160,7 +162,7 @@ async fn test_ix_commit_two_accounts_1kb_2kb() { &[1024, 2048], 1, false, - expect_strategies(&[(CommitStrategy::FromBuffer, 2)]), + expect_strategies(&[(CommitStrategy::Args, 2)]), ) .await; } @@ -208,25 +210,22 @@ async fn test_ix_commit_four_accounts_1kb_2kb_5kb_10kb_single_bundle() { &[1024, 2 * 1024, 5 * 1024, 10 * 1024], 1, false, - expect_strategies(&[(CommitStrategy::FromBuffer, 4)]), + expect_strategies(&[(CommitStrategy::Args, 4)]), ) .await; } #[tokio::test] async fn test_commit_20_accounts_1kb_bundle_size_2() { - commit_20_accounts_1kb( - 2, - expect_strategies(&[(CommitStrategy::FromBuffer, 20)]), - ) - .await; + commit_20_accounts_1kb(2, expect_strategies(&[(CommitStrategy::Args, 20)])) + .await; } #[tokio::test] async fn test_commit_5_accounts_1kb_bundle_size_3() { commit_5_accounts_1kb( 3, - expect_strategies(&[(CommitStrategy::FromBuffer, 5)]), + expect_strategies(&[(CommitStrategy::Args, 5)]), false, ) .await; @@ -239,7 +238,7 @@ async fn test_commit_5_accounts_1kb_bundle_size_3_undelegate_all() { expect_strategies(&[ // Intent fits in 1 TX only with ALT, see IntentExecutorImpl::try_unite_tasks (CommitStrategy::FromBufferWithLookupTable, 3), - (CommitStrategy::FromBuffer, 2), + (CommitStrategy::Args, 2), ]), true, ) @@ -251,7 +250,7 @@ async fn test_commit_5_accounts_1kb_bundle_size_4() { commit_5_accounts_1kb( 4, expect_strategies(&[ - (CommitStrategy::FromBuffer, 1), + (CommitStrategy::Args, 1), (CommitStrategy::FromBufferWithLookupTable, 4), ]), false, @@ -264,7 +263,7 @@ async fn test_commit_5_accounts_1kb_bundle_size_4_undelegate_all() { commit_5_accounts_1kb( 4, expect_strategies(&[ - (CommitStrategy::FromBuffer, 1), + (CommitStrategy::Args, 1), (CommitStrategy::FromBufferWithLookupTable, 4), ]), true, @@ -284,11 +283,8 @@ async fn test_commit_5_accounts_1kb_bundle_size_5_undelegate_all() { #[tokio::test] async fn test_commit_20_accounts_1kb_bundle_size_3() { - commit_20_accounts_1kb( - 3, - expect_strategies(&[(CommitStrategy::FromBuffer, 20)]), - ) - .await; + commit_20_accounts_1kb(3, expect_strategies(&[(CommitStrategy::Args, 20)])) + .await; } #[tokio::test] @@ -307,7 +303,7 @@ async fn test_commit_20_accounts_1kb_bundle_size_6() { expect_strategies(&[ (CommitStrategy::FromBufferWithLookupTable, 18), // Two accounts don't make it into the bundles of size 6 - (CommitStrategy::FromBuffer, 2), + (CommitStrategy::Args, 2), ]), ) .await; @@ -466,7 +462,7 @@ async fn commit_multiple_accounts( ix_commit_local(service, intents, expected_strategies).await; } -// TODO(thlorenz): once delegation program supports larger commits add the following +// TODO(thlorenz/snawaz): once delegation program supports larger commits add the following // tests // // ## Scenario 1 diff --git a/test-integration/test-committor-service/tests/test_transaction_preparator.rs b/test-integration/test-committor-service/tests/test_transaction_preparator.rs index de2f4ea57..123f13c38 100644 --- a/test-integration/test-committor-service/tests/test_transaction_preparator.rs +++ b/test-integration/test-committor-service/tests/test_transaction_preparator.rs @@ -7,8 +7,8 @@ use magicblock_committor_service::{ buffer_task::{BufferTask, BufferTaskType}, task_strategist::{TaskStrategist, TransactionStrategy}, utils::TransactionUtils, - BaseActionTask, BaseTask, CommitTask, FinalizeTask, PreparationState, - UndelegateTask, + BaseActionTask, BaseTask, CommitTaskBuilder, FinalizeTask, + PreparationState, UndelegateTask, }, transaction_preparator::TransactionPreparator, }; @@ -35,11 +35,15 @@ async fn test_prepare_commit_tx_with_single_account() { let committed_account = create_committed_account(&account_data); let tasks = vec![ - Box::new(ArgsTask::new(ArgsTaskType::Commit(CommitTask { - commit_id: 1, - committed_account: committed_account.clone(), - allow_undelegation: true, - }))) as Box, + Box::new(ArgsTask::new(ArgsTaskType::Commit( + CommitTaskBuilder::create_commit_task( + 1, + true, + committed_account.clone(), + &fixture.create_task_info_fetcher(), + ) + .await, + ))) as Box, Box::new(ArgsTask::new(ArgsTaskType::Finalize(FinalizeTask { delegated_account: committed_account.pubkey, }))), @@ -89,21 +93,28 @@ async fn test_prepare_commit_tx_with_multiple_accounts() { let account2_data = generate_random_bytes(12); let committed_account2 = create_committed_account(&account2_data); - let buffer_commit_task = BufferTask::new_preparation_required( - BufferTaskType::Commit(CommitTask { - commit_id: 1, - committed_account: committed_account2.clone(), - allow_undelegation: true, - }), - ); + let buffer_commit_task = + BufferTask::new_preparation_required(BufferTaskType::Commit( + CommitTaskBuilder::create_commit_task( + 1, + true, + committed_account2.clone(), + &fixture.create_task_info_fetcher(), + ) + .await, + )); // Create test data let tasks = vec![ // account 1 - Box::new(ArgsTask::new(ArgsTaskType::Commit(CommitTask { - commit_id: 1, - committed_account: committed_account1.clone(), - allow_undelegation: true, - }))) as Box, + Box::new(ArgsTask::new(ArgsTaskType::Commit( + CommitTaskBuilder::create_commit_task( + 1, + true, + committed_account1.clone(), + &fixture.create_task_info_fetcher(), + ) + .await, + ))) as Box, // account 2 Box::new(buffer_commit_task), // finalize account 1 @@ -186,13 +197,16 @@ async fn test_prepare_commit_tx_with_base_actions() { }], }; - let buffer_commit_task = BufferTask::new_preparation_required( - BufferTaskType::Commit(CommitTask { - commit_id: 1, - committed_account: committed_account.clone(), - allow_undelegation: true, - }), - ); + let buffer_commit_task = + BufferTask::new_preparation_required(BufferTaskType::Commit( + CommitTaskBuilder::create_commit_task( + 1, + true, + committed_account.clone(), + &fixture.create_task_info_fetcher(), + ) + .await, + )); let tasks = vec![ // commit account Box::new(buffer_commit_task.clone()) as Box, diff --git a/test-integration/test-committor-service/tests/utils/transactions.rs b/test-integration/test-committor-service/tests/utils/transactions.rs index 82440d93c..b915f49c1 100644 --- a/test-integration/test-committor-service/tests/utils/transactions.rs +++ b/test-integration/test-committor-service/tests/utils/transactions.rs @@ -120,7 +120,17 @@ pub async fn tx_logs_contain( .log_messages .clone() .unwrap_or_else(Vec::new); - logs.iter().any(|log| log.contains(needle)) + logs.iter().any(|log| { + // Lots of existing tests pass "CommitState" as needle argument to this function, but since now CommitTask + // could invoke CommitState or CommitDiff depending on the size of the account, we also look for "CommitDiff" + // in the logs when needle == CommitState. It's easier to make this little adjustment here than computing + // the decision and passing either CommitState or CommitDiff from the tests themselves. + if needle == "CommitState" { + log.contains(needle) || log.contains("CommitDiff") + } else { + log.contains(needle) + } + }) } /// This needs to be run for each test that required a new counter to be delegated diff --git a/test-integration/test-ledger-restore/tests/08_commit_update.rs b/test-integration/test-ledger-restore/tests/08_commit_update.rs index 699a7d03d..da24d2af2 100644 --- a/test-integration/test-ledger-restore/tests/08_commit_update.rs +++ b/test-integration/test-ledger-restore/tests/08_commit_update.rs @@ -52,7 +52,7 @@ fn test_restore_ledger_committed_and_updated_account() { fn write(ledger_path: &Path, payer: &Keypair) -> (Child, u64) { let programs = get_programs_with_flexi_counter(); - let (_, mut validator, ctx) = setup_validator_with_local_remote( + let (_tmpdir, mut validator, ctx) = setup_validator_with_local_remote( ledger_path, Some(programs), true, @@ -167,7 +167,7 @@ fn read(ledger_path: &Path, payer_kp: &Keypair) -> Child { let payer = &payer_kp.pubkey(); let programs = get_programs_with_flexi_counter(); - let (_, mut validator, ctx) = setup_validator_with_local_remote( + let (_tmpdir, mut validator, ctx) = setup_validator_with_local_remote( ledger_path, Some(programs), false, diff --git a/test-integration/test-tools/src/integration_test_context.rs b/test-integration/test-tools/src/integration_test_context.rs index 48ae3a911..1767c85b3 100644 --- a/test-integration/test-tools/src/integration_test_context.rs +++ b/test-integration/test-tools/src/integration_test_context.rs @@ -155,7 +155,9 @@ impl IntegrationTestContext { rpc_client: Option<&RpcClient>, label: &str, ) -> Option> { - let rpc_client = rpc_client.or(self.chain_client.as_ref())?; + let rpc_client = rpc_client.unwrap_or_else(|| { + panic!("rpc_client for [{label}] does not exist") + }); // Try this up to 50 times since devnet here returns the version response instead of // the EncodedConfirmedTransactionWithStatusMeta at times @@ -164,6 +166,13 @@ impl IntegrationTestContext { &sig, RpcTransactionConfig { commitment: Some(self.commitment), + max_supported_transaction_version: if label == "chain" { + // base chain cluster requires explicit v0 support, + // while ephemeral uses default version handling + Some(0) + } else { + None + }, ..Default::default() }, ) { diff --git a/test-integration/test-tools/src/scheduled_commits.rs b/test-integration/test-tools/src/scheduled_commits.rs index 038326a06..0b64cb7c8 100644 --- a/test-integration/test-tools/src/scheduled_commits.rs +++ b/test-integration/test-tools/src/scheduled_commits.rs @@ -179,7 +179,7 @@ impl IntegrationTestContext { { // 1. Find scheduled commit sent signature via // ScheduledCommitSent signature: - let (ephem_logs, scheduled_commmit_sent_sig) = { + let (ephem_logs_l1, scheduled_commmit_sent_sig) = { let logs = self.fetch_ephemeral_logs(sig).with_context(|| { format!( "Scheduled commit sent logs not found for sig {:?}", @@ -196,17 +196,17 @@ impl IntegrationTestContext { }; // 2. Find chain commit signatures - let chain_logs = self + let ephem_logs_l2 = self .fetch_ephemeral_logs(scheduled_commmit_sent_sig) .with_context(|| { format!( "Logs {:#?}\nScheduled commit sent sig {:?}", - ephem_logs, scheduled_commmit_sent_sig + ephem_logs_l1, scheduled_commmit_sent_sig ) })?; let (included, excluded, feepayers, sigs) = - extract_sent_commit_info_from_logs(&chain_logs); + extract_sent_commit_info_from_logs(&ephem_logs_l2); let mut committed_accounts = HashMap::new(); for pubkey in included { @@ -226,6 +226,10 @@ impl IntegrationTestContext { }; } + for sig in sigs.iter() { + self.dump_chain_logs(*sig); + } + Ok(ScheduledCommitResult { included: committed_accounts, excluded,