feat: add merkle tree and encoding persistence

fil-proofs-tooling/Cargo.toml

@@ -11,8 +11,8 @@ readme = "README.md"
 
 [dependencies]
 clap = "2"
-serde = { version = "1", features = ["derive"] }
-serde_json = "1"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
 toml = "0.5"
 failure = "0.1"
 permutate = "0.3"
@@ -40,6 +40,8 @@ blake2s_simd = "0.5.6"
 pretty_env_logger = "0.3.1"
 log = "0.4.8"
 uom = "0.25.0"
+merkletree = "0.12.2"
+bincode = "1.1.2"
 
 [features]
 default = ["gpu"]

fil-proofs-tooling/src/bin/benchy/election_post.rs

@@ -113,7 +113,6 @@ pub fn run(sector_size: usize) -> Result<(), failure::Error> {
 
     let seed = [0u8; 32];
     let comm_r = seal_pre_commit_output.comm_r;
-    let p_aux = seal_pre_commit_output.p_aux.clone();
 
     let _seal_commit_output = seal_commit(
         porep_config,
@@ -134,7 +133,7 @@ pub fn run(sector_size: usize) -> Result<(), failure::Error> {
 
     priv_replica_info.insert(
         sector_id,
-        PrivateReplicaInfo::new(sealed_path_string, comm_r, p_aux, cache_dir.into_path()),
+        PrivateReplicaInfo::new(sealed_path_string, comm_r, cache_dir.into_path())?,
     );
 
     // Measure PoSt generation and verification.

fil-proofs-tooling/src/bin/benchy/stacked.rs

@@ -8,6 +8,7 @@ use failure::bail;
 use log::info;
 use memmap::MmapMut;
 use memmap::MmapOptions;
+use merkletree::store::{StoreConfig, DEFAULT_CACHED_ABOVE_BASE_LAYER};
 use paired::bls12_381::Bls12;
 use rand::Rng;
 
@@ -20,8 +21,10 @@ use storage_proofs::hasher::{Blake2sHasher, Domain, Hasher, PedersenHasher, Sha2
 use storage_proofs::porep::PoRep;
 use storage_proofs::proof::ProofScheme;
 use storage_proofs::stacked::{
-    self, ChallengeRequirements, LayerChallenges, StackedDrg, EXP_DEGREE,
+    self, CacheKey, ChallengeRequirements, LayerChallenges, StackedDrg, TemporaryAuxCache,
+    EXP_DEGREE,
 };
+use tempfile::TempDir;
 
 fn file_backed_mmap_from_zeroes(n: usize, use_tmp: bool) -> Result<MmapMut, failure::Error> {
     let file: File = if use_tmp {
@@ -87,7 +90,10 @@ impl From<Params> for Inputs {
     }
 }
 
-fn generate_report<H: 'static>(params: Params) -> Result<Report, failure::Error>
+fn generate_report<H: 'static>(
+    params: Params,
+    cache_dir: &TempDir,
+) -> Result<Report, failure::Error>
 where
     H: Hasher,
 {
@@ -116,6 +122,14 @@ where
             ..
         } = &params;
 
+        // MT for original data is always named tree-d, and it will be
+        // referenced later in the process as such.
+        let config = StoreConfig::new(
+            cache_dir.path(),
+            CacheKey::CommDTree.to_string(),
+            DEFAULT_CACHED_ABOVE_BASE_LAYER,
+        );
+
         let mut total_proving_wall_time = Duration::new(0, 0);
         let mut total_proving_cpu_time = Duration::new(0, 0);
 
@@ -144,8 +158,13 @@ where
                 wall_time: replication_wall_time,
                 return_value: (pub_inputs, priv_inputs),
             } = measure(|| {
-                let (tau, (p_aux, t_aux)) =
-                    StackedDrg::<H, Sha256Hasher>::replicate(&pp, &replica_id, &mut data, None)?;
+                let (tau, (p_aux, t_aux)) = StackedDrg::<H, Sha256Hasher>::replicate(
+                    &pp,
+                    &replica_id,
+                    &mut data,
+                    None,
+                    Some(config.clone()),
+                )?;
 
                 let pb = stacked::PublicInputs::<H::Domain, <Sha256Hasher as Hasher>::Domain> {
                     replica_id,
@@ -154,6 +173,11 @@ where
                     k: Some(0),
                 };
 
+                // Convert TemporaryAux to TemporaryAuxCache, which instantiates all
+                // elements based on the configs stored in TemporaryAux.
+                let t_aux =
+                    TemporaryAuxCache::new(&t_aux).expect("failed to restore contents of t_aux");
+
                 let pv = stacked::PrivateInputs { p_aux, t_aux };
 
                 Ok((pb, pv))
@@ -264,8 +288,13 @@ where
         if let Some(data) = d {
             if *extract {
                 let m = measure(|| {
-                    StackedDrg::<H, Sha256Hasher>::extract_all(&pp, &replica_id, &data)
-                        .map_err(|err| err.into())
+                    StackedDrg::<H, Sha256Hasher>::extract_all(
+                        &pp,
+                        &replica_id,
+                        &data,
+                        Some(config.clone()),
+                    )
+                    .map_err(|err| err.into())
                 })?;
 
                 assert_ne!(&(*data), m.return_value.as_slice());
@@ -488,10 +517,12 @@ pub fn run(opts: RunOpts) -> Result<(), failure::Error> {
 
     info!("Benchy Stacked: {:?}", &params);
 
+    let cache_dir = tempfile::tempdir().unwrap();
+
     let report = match params.hasher.as_ref() {
-        "pedersen" => generate_report::<PedersenHasher>(params)?,
-        "sha256" => generate_report::<Sha256Hasher>(params)?,
-        "blake2s" => generate_report::<Blake2sHasher>(params)?,
+        "pedersen" => generate_report::<PedersenHasher>(params, &cache_dir)?,
+        "sha256" => generate_report::<Sha256Hasher>(params, &cache_dir)?,
+        "blake2s" => generate_report::<Blake2sHasher>(params, &cache_dir)?,
         _ => bail!("invalid hasher: {}", params.hasher),
     };
 

filecoin-proofs/Cargo.toml

@@ -22,7 +22,7 @@ tempfile = "3"
 byteorder = "1"
 itertools = "0.8"
 serde_cbor = "0.10.2"
-serde = { version = "1", features = ["rc", "derive"] }
+serde = { version = "1.0", features = ["rc", "derive"] }
 serde_json = "1.0"
 regex = "1"
 ff = "0.5.0"
@@ -42,6 +42,8 @@ blake2s_simd = "0.5.8"
 hex = "0.4.0"
 tee = "0.1.0"
 os_pipe = "0.9.1"
+merkletree = "0.12.2"
+bincode = "1.1.2"
 
 [dependencies.reqwest]
 version = "0.9"

filecoin-proofs/examples/stacked.rs

@@ -14,6 +14,7 @@ use gperftools::heap_profiler::HEAP_PROFILER;
 use gperftools::profiler::PROFILER;
 use memmap::MmapMut;
 use memmap::MmapOptions;
+use merkletree::store::{StoreConfig, DEFAULT_CACHED_ABOVE_BASE_LAYER};
 use paired::bls12_381::{Bls12, Fr};
 use rand::Rng;
 use std::fs::{File, OpenOptions};
@@ -33,7 +34,8 @@ use storage_proofs::hasher::{Blake2sHasher, Domain, Hasher, PedersenHasher, Sha2
 use storage_proofs::porep::PoRep;
 use storage_proofs::proof::ProofScheme;
 use storage_proofs::stacked::{
-    self, ChallengeRequirements, LayerChallenges, StackedDrg, EXP_DEGREE,
+    self, CacheKey, ChallengeRequirements, LayerChallenges, StackedDrg, TemporaryAuxCache,
+    EXP_DEGREE,
 };
 
 // We can only one of the profilers at a time, either CPU (`profile`)
@@ -189,6 +191,15 @@ fn do_the_work<H: 'static>(
     let samples: u32 = 5;
     let mut total_proving = Duration::new(0, 0);
 
+    // MT for original data is always named tree-d, and it will be
+    // referenced later in the process as such.
+    let cache_dir = tempfile::tempdir().unwrap();
+    let config = StoreConfig::new(
+        cache_dir.path(),
+        CacheKey::CommDTree.to_string(),
+        DEFAULT_CACHED_ABOVE_BASE_LAYER,
+    );
+
     let (pub_in, priv_in, d) = if bench_only {
         (None, None, None)
     } else {
@@ -201,8 +212,14 @@ fn do_the_work<H: 'static>(
         let seed = rng.gen();
 
         start_profile("replicate");
-        let (tau, (p_aux, t_aux)) =
-            StackedDrg::<H, Blake2sHasher>::replicate(&pp, &replica_id, &mut data, None).unwrap();
+        let (tau, (p_aux, t_aux)) = StackedDrg::<H, Blake2sHasher>::replicate(
+            &pp,
+            &replica_id,
+            &mut data,
+            None,
+            Some(config.clone()),
+        )
+        .unwrap();
         stop_profile();
         let pub_inputs = stacked::PublicInputs::<H::Domain, <Blake2sHasher as Hasher>::Domain> {
             replica_id,
@@ -211,6 +228,11 @@ fn do_the_work<H: 'static>(
             seed,
         };
 
+        // Convert TemporaryAux to TemporaryAuxCache, which instantiates all
+        // elements based on the configs stored in TemporaryAux.
+        let t_aux: TemporaryAuxCache<H, Blake2sHasher> =
+            TemporaryAuxCache::new(&t_aux).expect("failed to restore contents of t_aux");
+
         let priv_inputs = stacked::PrivateInputs { p_aux, t_aux };
 
         replication_duration += start.elapsed();
@@ -376,8 +398,13 @@ fn do_the_work<H: 'static>(
             let start = Instant::now();
             info!("Extracting.");
             start_profile("extract");
-            let decoded_data =
-                StackedDrg::<H, Blake2sHasher>::extract_all(&pp, &replica_id, &data).unwrap();
+            let decoded_data = StackedDrg::<H, Blake2sHasher>::extract_all(
+                &pp,
+                &replica_id,
+                &data,
+                Some(config.clone()),
+            )
+            .unwrap();
             stop_profile();
             let extracting = start.elapsed();
             info!("extracting_time: {:?}", extracting);

filecoin-proofs/src/api/mod.rs

@@ -2,11 +2,12 @@ use std::fs::File;
 use std::io::{BufWriter, Read, Seek, SeekFrom, Write};
 use std::path::{Path, PathBuf};
 
+use merkletree::store::{StoreConfig, DEFAULT_CACHED_ABOVE_BASE_LAYER};
 use storage_proofs::drgraph::DefaultTreeHasher;
 use storage_proofs::hasher::Hasher;
 use storage_proofs::porep::PoRep;
 use storage_proofs::sector::SectorId;
-use storage_proofs::stacked::{generate_replica_id, StackedDrg};
+use storage_proofs::stacked::{generate_replica_id, CacheKey, StackedDrg};
 use tempfile::tempfile;
 
 use crate::api::util::as_safe_commitment;
@@ -39,6 +40,7 @@ use storage_proofs::pieces::generate_piece_commitment_bytes_from_source;
 #[allow(clippy::too_many_arguments)]
 pub fn get_unsealed_range<T: Into<PathBuf> + AsRef<Path>>(
     porep_config: PoRepConfig,
+    cache_path: T,
     sealed_path: T,
     output_path: T,
     prover_id: ProverId,
@@ -62,13 +64,22 @@ pub fn get_unsealed_range<T: Into<PathBuf> + AsRef<Path>>(
     let f_out = File::create(output_path)?;
     let mut buf_writer = BufWriter::new(f_out);
 
+    // MT for original data is always named tree-d, and it will be
+    // referenced later in the process as such.
+    let config = StoreConfig::new(
+        cache_path,
+        CacheKey::CommDTree.to_string(),
+        DEFAULT_CACHED_ABOVE_BASE_LAYER,
+    );
+
     let unsealed = StackedDrg::<DefaultTreeHasher, DefaultPieceHasher>::extract_all(
         &public_params(
             PaddedBytesAmount::from(porep_config),
             usize::from(PoRepProofPartitions::from(porep_config)),
         ),
         &replica_id,
         &data,
+        Some(config),
     )?;
 
     let written = write_unpadded(&unsealed, &mut buf_writer, offset.into(), num_bytes.into())?;

filecoin-proofs/src/api/post.rs

@@ -1,24 +1,28 @@
 use std::collections::BTreeMap;
 use std::fs::File;
-use std::io::Read;
+use std::io::prelude::*;
 
+use bincode::deserialize;
+use merkletree::merkle::{get_merkle_tree_leafs, MerkleTree};
+use merkletree::store::{DiskStore, Store, StoreConfig, DEFAULT_CACHED_ABOVE_BASE_LAYER};
 use paired::bls12_381::Bls12;
 use rayon::prelude::*;
 use storage_proofs::circuit::election_post::ElectionPoStCompound;
 use storage_proofs::circuit::multi_proof::MultiProof;
 use storage_proofs::compound_proof::{self, CompoundProof};
-use storage_proofs::drgraph::{DefaultTreeHasher, Graph};
+use storage_proofs::drgraph::DefaultTreeHasher;
 use storage_proofs::election_post;
 use storage_proofs::error::Error;
 use storage_proofs::fr32::bytes_into_fr;
 use storage_proofs::hasher::Hasher;
 use storage_proofs::proof::NoRequirements;
 use storage_proofs::sector::*;
+use storage_proofs::stacked::CacheKey;
 
 use crate::api::util::as_safe_commitment;
 use crate::caches::{get_post_params, get_post_verifying_key};
 use crate::error;
-use crate::parameters::{post_setup_params, public_params};
+use crate::parameters::post_setup_params;
 use crate::types::{
     ChallengeSeed, Commitment, PaddedBytesAmount, PersistentAux, PoStConfig, ProverId, Tree,
 };
@@ -53,13 +57,25 @@ impl std::cmp::PartialOrd for PrivateReplicaInfo {
 }
 
 impl PrivateReplicaInfo {
-    pub fn new(access: String, comm_r: Commitment, aux: PersistentAux, cache_dir: PathBuf) -> Self {
-        PrivateReplicaInfo {
+    pub fn new(
+        access: String,
+        comm_r: Commitment,
+        cache_dir: PathBuf,
+    ) -> Result<Self, failure::Error> {
+        let aux = {
+            let mut aux_bytes = vec![];
+            let mut f_aux = File::open(cache_dir.join(CacheKey::PAux.to_string()))?;
+            f_aux.read_to_end(&mut aux_bytes)?;
+
+            deserialize(&aux_bytes)
+        }?;
+
+        Ok(PrivateReplicaInfo {
             access,
             comm_r,
             aux,
             cache_dir,
-        }
+        })
     }
 
     pub fn safe_comm_r(&self) -> Result<<DefaultTreeHasher as Hasher>::Domain, failure::Error> {
@@ -78,12 +94,24 @@ impl PrivateReplicaInfo {
 
     /// Generate the merkle tree of this particular replica.
     pub fn merkle_tree(&self, sector_size: u64) -> Result<Tree, Error> {
-        let mut f_in = File::open(&self.access)?;
-        let mut data = Vec::new();
-        f_in.read_to_end(&mut data)?;
+        let tree_size = {
+            let elems =
+                sector_size as usize / std::mem::size_of::<<DefaultTreeHasher as Hasher>::Domain>();
 
-        let bytes = PaddedBytesAmount(sector_size as u64);
-        public_params(bytes, 1).graph.merkle_tree(&data)
+            2 * elems - 1
+        };
+        let mut config = StoreConfig::new(
+            &self.cache_dir,
+            CacheKey::CommRLastTree.to_string(),
+            DEFAULT_CACHED_ABOVE_BASE_LAYER,
+        );
+        config.size = Some(tree_size);
+        let tree_d_store: DiskStore<<DefaultTreeHasher as Hasher>::Domain> =
+            DiskStore::new_from_disk(tree_size, &config)?;
+        let tree_d: Tree =
+            MerkleTree::from_data_store(tree_d_store, get_merkle_tree_leafs(tree_size));
+
+        Ok(tree_d)
     }
 }