diff --git a/Cargo.lock b/Cargo.lock index 690bf96b0..069c1a05b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1079,25 +1079,31 @@ dependencies = [ [[package]] name = "avail-core" version = "0.6.2" -source = "git+https://github.com/availproject/avail-core?tag=core-node-3#99e37cf1329d42c118a91b8ccd6d9f8538f89dfe" +source = "git+https://github.com/utsire/avail-core?rev=a4e539dedada70a103a1d364e36465b7bdeb533c#a4e539dedada70a103a1d364e36465b7bdeb533c" dependencies = [ "binary-merkle-tree", + "blake2b_simd", "bounded-collections", "derive_more 0.99.18", "ethabi-decode", "frame-support", + "hash-db 0.16.0", "hash256-std-hasher", "hex", + "impl-serde 0.5.0", "log", "parity-scale-codec", + "primitive-types 0.12.2", "scale-info", "serde", + "sha2 0.10.8", + "sha3", "sp-arithmetic", - "sp-core", - "sp-io", + "sp-debug-derive", "sp-runtime", "sp-runtime-interface", "sp-std", + "sp-storage", "sp-trie", "static_assertions", "thiserror-no-std", @@ -1534,8 +1540,9 @@ dependencies = [ [[package]] name = "blst" -version = "0.3.10" -source = "git+https://github.com/availproject/blst?tag=v0.3.10#556e037926d9c526c2eb6cb1522bea39690416ea" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4378725facc195f1a538864863f6de233b500a8862747e7f165078a419d5e874" dependencies = [ "cc", "glob", @@ -2559,6 +2566,7 @@ dependencies = [ "iai", "iai-callgrind", "kate", + "kate-recovery", "log", "pallet-authority-discovery", "pallet-authorship", @@ -5130,6 +5138,15 @@ dependencies = [ "serde", ] +[[package]] +name = "impl-serde" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a143eada6a1ec4aefa5049037a26a6d597bfd64f8c026d07b77133e02b7dd0b" +dependencies = [ + "serde", +] + [[package]] name = "impl-trait-for-tuples" version = "0.2.3" @@ -5533,7 +5550,7 @@ dependencies = [ [[package]] name = "kate" version = "0.9.2" -source = "git+https://github.com/availproject/avail-core?tag=core-node-3#99e37cf1329d42c118a91b8ccd6d9f8538f89dfe" +source = "git+https://github.com/utsire/avail-core?rev=a4e539dedada70a103a1d364e36465b7bdeb533c#a4e539dedada70a103a1d364e36465b7bdeb533c" dependencies = [ "avail-core", "derive_more 0.99.18", @@ -5553,7 +5570,6 @@ dependencies = [ "serde", "serde_json", "sp-arithmetic", - "sp-core", "static_assertions", "thiserror-no-std", ] @@ -5561,7 +5577,7 @@ dependencies = [ [[package]] name = "kate-recovery" version = "0.10.0" -source = "git+https://github.com/availproject/avail-core?tag=core-node-3#99e37cf1329d42c118a91b8ccd6d9f8538f89dfe" +source = "git+https://github.com/utsire/avail-core?rev=a4e539dedada70a103a1d364e36465b7bdeb533c#a4e539dedada70a103a1d364e36465b7bdeb533c" dependencies = [ "avail-core", "derive_more 0.99.18", @@ -5589,6 +5605,7 @@ dependencies = [ "frame-system", "jsonrpsee", "kate", + "kate-recovery", "log", "sc-client-api", "sp-api", @@ -8418,8 +8435,8 @@ dependencies = [ [[package]] name = "poly-multiproof" -version = "0.0.1" -source = "git+https://github.com/availproject/poly-multiproof?tag=v0.0.1#cd8d31b7eb568dea2fddfc9237e2e31ea7ae7ed3" +version = "0.1.1" +source = "git+https://github.com/aphoh/poly-multiproof#494468b86ac8e16dca21d465d921990c578eaad3" dependencies = [ "ark-bls12-381", "ark-ec 0.4.2", @@ -8429,6 +8446,7 @@ dependencies = [ "ark-std 0.4.0", "blst", "merlin", + "thiserror 2.0.11", ] [[package]] @@ -8543,7 +8561,7 @@ dependencies = [ "fixed-hash", "impl-codec 0.6.0", "impl-rlp", - "impl-serde", + "impl-serde 0.4.0", "scale-info", "uint 0.9.5", ] @@ -11485,7 +11503,7 @@ dependencies = [ "futures", "hash-db 0.16.0", "hash256-std-hasher", - "impl-serde", + "impl-serde 0.4.0", "itertools 0.10.5", "libsecp256k1", "log", @@ -11894,7 +11912,7 @@ name = "sp-storage" version = "19.0.0" source = "git+https://github.com/availproject/polkadot-sdk.git?tag=polkadot-1.7.1-patch-10#f5587b380ee596f90482d402844c49aa140781d8" dependencies = [ - "impl-serde", + "impl-serde 0.4.0", "parity-scale-codec", "ref-cast", "serde", @@ -11980,7 +11998,7 @@ name = "sp-version" version = "29.0.0" source = "git+https://github.com/availproject/polkadot-sdk.git?tag=polkadot-1.7.1-patch-10#f5587b380ee596f90482d402844c49aa140781d8" dependencies = [ - "impl-serde", + "impl-serde 0.4.0", "parity-scale-codec", "parity-wasm", "scale-info", diff --git a/Cargo.toml b/Cargo.toml index bd39ca8c2..28037f976 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,12 +22,12 @@ homepage = "https://www.availproject.org/" [workspace.dependencies] -avail-core = { git = "https://github.com/availproject/avail-core", tag = "core-node-3", default-features = false, features = [ "runtime"] } -kate = { git = "https://github.com/availproject/avail-core", tag = "core-node-3", default-features = false } -kate-recovery = { git = "https://github.com/availproject/avail-core", tag = "core-node-3", default-features = false } -# avail-core = { path = "../avail-core/core", default-features = false, features = [ "runtime"] } -# kate = { path = "../avail-core/kate/", default-features = false } -# kate-recovery = { path = "../avail-core/kate/recovery/", default-features = false} +avail-core = { git = "https://github.com/utsire/avail-core", rev = "a4e539dedada70a103a1d364e36465b7bdeb533c", default-features = false, features = [ "runtime"] } +kate = { git = "https://github.com/utsire/avail-core", rev="a4e539dedada70a103a1d364e36465b7bdeb533c", default-features = false } +kate-recovery = { git = "https://github.com/utsire/avail-core", rev = "a4e539dedada70a103a1d364e36465b7bdeb533c", default-features = false } +#avail-core = { path = "../avail-core/core", default-features = false, features = [ "runtime"] } +#kate = { path = "../avail-core/kate/", default-features = false } +#kate-recovery = { path = "../avail-core/kate/recovery/", default-features = false} avail-base = { path = "base", default-features = false } da-control = { path = "pallets/dactr", default-features = false } diff --git a/pallets/system/src/native/build_extension_v1.rs b/pallets/system/src/native/build_extension_v1.rs index 88b953702..b2da4fc5e 100644 --- a/pallets/system/src/native/build_extension_v1.rs +++ b/pallets/system/src/native/build_extension_v1.rs @@ -17,8 +17,7 @@ use avail_core::{ use kate::{ couscous::multiproof_params, gridgen::{AsBytes, EvaluationGrid}, - pmp::m1_blst::M1NoPrecomp, - Seed, + M1NoPrecomp, Seed, }; use sp_core::H256; use sp_runtime::SaturatedConversion; diff --git a/pallets/system/src/native/build_extension_v2.rs b/pallets/system/src/native/build_extension_v2.rs index 53c4e9f0f..4f46ef500 100644 --- a/pallets/system/src/native/build_extension_v2.rs +++ b/pallets/system/src/native/build_extension_v2.rs @@ -17,8 +17,7 @@ use avail_core::{ use kate::{ couscous::multiproof_params, gridgen::{AsBytes, EvaluationGrid}, - pmp::m1_blst::M1NoPrecomp, - Seed, + M1NoPrecomp, Seed, }; use sp_core::H256; use sp_runtime::SaturatedConversion; diff --git a/rpc/kate-rpc/Cargo.toml b/rpc/kate-rpc/Cargo.toml index 7a7df8211..b3d5436af 100644 --- a/rpc/kate-rpc/Cargo.toml +++ b/rpc/kate-rpc/Cargo.toml @@ -11,6 +11,7 @@ da-runtime.workspace = true frame-system = { workspace = true, default-features = false } avail-core = { workspace = true, default-features = false } kate = { workspace = true, default-features = false } +kate-recovery = { workspace = true, default-features = false } # 3rd party jsonrpsee.workspace = true diff --git a/rpc/kate-rpc/src/lib.rs b/rpc/kate-rpc/src/lib.rs index 4ebad47e7..0d3d8373c 100644 --- a/rpc/kate-rpc/src/lib.rs +++ b/rpc/kate-rpc/src/lib.rs @@ -3,7 +3,7 @@ use avail_core::{ data_proof::ProofResponse, header::HeaderExtension, traits::ExtendedHeader, OpaqueExtrinsic, }; use da_runtime::apis::{DataAvailApi, KateApi as RTKateApi}; -use da_runtime::kate::{GDataProof, GRow}; +use da_runtime::kate::{GDataProof, GMultiProof, GRow}; use kate::com::Cell; use frame_support::BoundedVec; @@ -47,6 +47,13 @@ where at: Option>, ) -> RpcResult>; + #[method(name = "kate_queryMultiProof")] + async fn query_multiproof( + &self, + cells: Cells, + at: Option>, + ) -> RpcResult>; + #[method(name = "kate_blockLength")] async fn query_block_length(&self, at: Option>) -> RpcResult; @@ -240,6 +247,44 @@ where Ok(proof) } + async fn query_multiproof( + &self, + cells: Cells, + at: Option>, + ) -> RpcResult> { + if cells.len() > self.max_cells_size { + return Err( + internal_err!( + "Cannot query ({}) more than {} amount of cells per request. Either increase the max cells size (--kate-max-cells-size) or query less amount of cells per request.", + cells.len(), + self.max_cells_size + ) + ); + } + + let _metric_observer = MetricObserver::new(ObserveKind::KateQueryProof); + + let (api, at, number, block_len, extrinsics, header) = self.scope(at)?; + match header.extension() { + HeaderExtension::V3(ext) => { + if ext.commitment.commitment.is_empty() { + return Err(internal_err!("Requested block {at} has empty commitments")); + } + }, + }; + + let cells = cells + .into_iter() + .map(|cell| (cell.row.0, cell.col.0)) + .collect::>(); + let proof = api + .multiproof(at, number, extrinsics, block_len, cells) + .map_err(|kate_err| internal_err!("KateApi::proof failed: {kate_err:?}"))? + .map_err(|api_err| internal_err!("Failed API: {api_err:?}"))?; + + Ok(proof) + } + async fn query_block_length(&self, at: Option>) -> RpcResult { let _metric_observer = MetricObserver::new(ObserveKind::KateQueryBlockLength); diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 8b98d4408..0a58eabbf 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -17,6 +17,7 @@ targets = ["x86_64-unknown-linux-gnu"] avail-base = { workspace = true, default-features = false } avail-core = { workspace = true, default-features = false } kate = { workspace = true, default-features = false } +kate-recovery = { workspace = true, default-features = false } da-control = { workspace = true, default-features = false } pallet-mandate = { workspace = true, default-features = false } diff --git a/runtime/src/apis.rs b/runtime/src/apis.rs index 9c414c977..c0e092a88 100644 --- a/runtime/src/apis.rs +++ b/runtime/src/apis.rs @@ -1,9 +1,9 @@ use super::kate::{Error as RTKateError, GDataProof, GRow}; use crate::{ - constants, mmr, version::VERSION, AccountId, AuthorityDiscovery, Babe, Block, BlockNumber, - EpochDuration, Executive, Grandpa, Historical, Index, InherentDataExt, Mmr, NominationPools, - OpaqueMetadata, Runtime, RuntimeCall, RuntimeGenesisConfig, SessionKeys, Staking, System, - TransactionPayment, LOG_TARGET, + constants, kate::GMultiProof, mmr, version::VERSION, AccountId, AuthorityDiscovery, Babe, + Block, BlockNumber, EpochDuration, Executive, Grandpa, Historical, Index, InherentDataExt, Mmr, + NominationPools, OpaqueMetadata, Runtime, RuntimeCall, RuntimeGenesisConfig, SessionKeys, + Staking, System, TransactionPayment, LOG_TARGET, }; use avail_base::{HeaderExtensionBuilderData, ProvidePostInherent}; use avail_core::{ @@ -66,6 +66,7 @@ decl_runtime_apis! { fn data_proof(block_number: u32, extrinsics: Vec, tx_idx: u32) -> Option; fn rows(block_number: u32, extrinsics: Vec, block_len: BlockLength, rows: Vec) -> Result, RTKateError >; fn proof(block_number: u32, extrinsics: Vec, block_len: BlockLength, cells: Vec<(u32,u32)> ) -> Result, RTKateError>; + fn multiproof(block_number: u32, extrinsics: Vec, block_len: BlockLength, cells: Vec<(u32,u32)> ) -> Result, RTKateError>; } } @@ -443,6 +444,13 @@ impl_runtime_apis! { log::trace!(target: LOG_TARGET, "KateApi::proof: data_proofs={data_proofs:#?}"); Ok(data_proofs) } + + fn multiproof(block_number: u32, extrinsics: Vec, block_len: BlockLength, cells: Vec<(u32,u32)> ) -> Result, RTKateError> { + let app_extrinsics = HeaderExtensionBuilderData::from_opaque_extrinsics::(block_number, &extrinsics).to_app_extrinsics(); + let data_proofs = super::kate::multiproof::(app_extrinsics, block_len, cells)?; + log::trace!(target: LOG_TARGET, "KateApi::proof: data_proofs={data_proofs:#?}"); + Ok(data_proofs) + } } impl avail_base::PostInherentsProvider for Runtime { diff --git a/runtime/src/kate/mod.rs b/runtime/src/kate/mod.rs index 6a57c7efd..aac6561f9 100644 --- a/runtime/src/kate/mod.rs +++ b/runtime/src/kate/mod.rs @@ -2,7 +2,7 @@ pub mod native; pub mod runtime; // Reexport -pub use runtime::{grid, proof}; +pub use runtime::{grid, multiproof, proof}; use codec::{Decode, Encode}; use core::num::TryFromIntError; @@ -20,6 +20,7 @@ use kate::{com::Error as KateError, gridgen::AppRowError as KateAppRowError}; pub type GRawScalar = U256; pub type GRow = Vec; pub type GDataProof = (GRawScalar, GProof); +pub type GMultiProof = (Vec, GProof); /// # NOTE /// `Serde` requires a custom implementation for `GProof` due to the array size (greater than `[T;32]`). diff --git a/runtime/src/kate/native.rs b/runtime/src/kate/native.rs index 5777a769e..dd7eba33f 100644 --- a/runtime/src/kate/native.rs +++ b/runtime/src/kate/native.rs @@ -1,15 +1,16 @@ -use super::{Error, GDataProof, GProof, GRawScalar, GRow}; +use super::{Error, GDataProof, GMultiProof, GProof, GRawScalar, GRow}; use avail_core::{AppExtrinsic, AppId, BlockLengthColumns, BlockLengthRows}; use core::num::NonZeroU16; use frame_system::{limits::BlockLength, native::hosted_header_builder::MIN_WIDTH}; -use kate::Seed; #[cfg(feature = "std")] use kate::{ com::Cell, couscous::multiproof_params, gridgen::{AsBytes as _, EvaluationGrid as EGrid}, - pmp::m1_blst::M1NoPrecomp, + M1NoPrecomp, }; +use kate::{gridgen::ArkScalar, Seed}; +use kate_recovery::matrix::Dimensions; use sp_runtime::SaturatedConversion as _; use sp_runtime_interface::runtime_interface; use sp_std::vec::Vec; @@ -90,6 +91,54 @@ pub trait HostedKate { Ok(proofs) } + fn multiproof( + extrinsics: Vec, + block_len: BlockLength, + seed: Seed, + cells: Vec<(u32, u32)>, + ) -> Result, Error> { + let srs = SRS.get_or_init(multiproof_params); + let (max_width, max_height) = to_width_height(&block_len); + let grid = EGrid::from_extrinsics(extrinsics, MIN_WIDTH, max_width, max_height, seed)? + .extend_columns(NonZeroU16::new(2).expect("2>0")) + .map_err(|_| Error::ColumnExtension)?; + + let poly = grid.make_polynomial_grid()?; + + let proofs = cells + .into_par_iter() + .map(|(row, col)| -> Result { + let cell = Cell::new(BlockLengthRows(row), BlockLengthColumns(col)); + let target_dims = Dimensions::new(16, 64).expect("16,64>0"); + // TODO: This isn't correct, need to put in the correct mp grid dim + // TODO: safety + if cell.row.0 >= grid.dims().height() as u32 + || cell.col.0 >= grid.dims().width() as u32 + { + return Err(Error::MissingCell { row, col }); + } + let mp = poly.multiproof(srs, &cell, &grid, target_dims)?; + let data = mp + .evals + .into_iter() + .flatten() + .map(|e: ArkScalar| { + e.to_bytes() + .map(GRawScalar::from) + .map_err(|_| Error::InvalidScalarAtRow(row)) + }) + .collect::, _>>()?; + + let proof = mp.proof.to_bytes().map(GProof).map_err(|_| Error::Proof)?; + + // TODO: should we also return the block coords in mp.block? + Ok((data, proof)) + }) + .collect::, _>>()?; + + Ok(proofs) + } + fn app_data( submitted: Vec, block_length: BlockLength, diff --git a/runtime/src/kate/runtime.rs b/runtime/src/kate/runtime.rs index 487ce0c15..ad5761812 100644 --- a/runtime/src/kate/runtime.rs +++ b/runtime/src/kate/runtime.rs @@ -1,4 +1,4 @@ -use super::{native::hosted_kate, Error, GDataProof, GRow}; +use super::{native::hosted_kate, Error, GDataProof, GMultiProof, GRow}; use da_control::LOG_TARGET as DALOG_TARGET; use avail_core::AppExtrinsic; @@ -41,3 +41,12 @@ pub fn proof( let seed = random_seed::(); hosted_kate::proof(app_extrinsics, block_len, seed, cells) } + +pub fn multiproof( + app_extrinsics: Vec, + block_len: BlockLength, + cells: Vec<(u32, u32)>, +) -> Result, Error> { + let seed = random_seed::(); + hosted_kate::multiproof(app_extrinsics, block_len, seed, cells) +}