Interface and logic to support snap sync for Paprika (#349)

* Multiple changes to support snap sync via RawState:
- handling of boundary node (only hash stored)
- checking root hash without commit
- encoding keccak hash as NibblePath for merkle Leaf type
- IReadOnlyWorldState implementation for BlockState

* Enabled flag SNAP_SYNC_SUPPORT

* Removed duplicated call to EnsureHash

* New IRawState implementation to easier parallelize. Elements to support healing tests.

* Multiple changes to support snap sync via RawState:
- handling of boundary node (only hash stored)
- checking root hash without commit
- encoding keccak hash as NibblePath for merkle Leaf type
- IReadOnlyWorldState implementation for BlockState

* Enabled flag SNAP_SYNC_SUPPORT

* Removed duplicated call to EnsureHash

* New IRawState implementation to easier parallelize. Elements to support healing tests.

* Fix method call after merge

* Dispose CommittedBlockState during commit.

* Added SNAP_SYNC_SUPPORT for release

* Added ReadOnlySyncWorldStateAccessor. Changed visitor to hanlde RlpMemo. Added ForceFlush to aid testing.

* Fixes for leaves at last level.

* Hash calc fixes and changes for arbitrary paths and snap sync specific.

* Hash calc changes. New methods to create nodes from proof data.

* Proof creation changes and sync accessor changes.

* Multiple changes to support snap sync via RawState:
- handling of boundary node (only hash stored)
- checking root hash without commit
- encoding keccak hash as NibblePath for merkle Leaf type
- IReadOnlyWorldState implementation for BlockState

* Enabled flag SNAP_SYNC_SUPPORT

* Removed duplicated call to EnsureHash

* New IRawState implementation to easier parallelize. Elements to support healing tests.

* Multiple changes to support snap sync via RawState:
- handling of boundary node (only hash stored)
- checking root hash without commit
- encoding keccak hash as NibblePath for merkle Leaf type
- IReadOnlyWorldState implementation for BlockState

* New IRawState implementation to easier parallelize. Elements to support healing tests.

* Fix method call after merge

* Dispose CommittedBlockState during commit.

* Added SNAP_SYNC_SUPPORT for release

* Added ReadOnlySyncWorldStateAccessor. Changed visitor to hanlde RlpMemo. Added ForceFlush to aid testing.

* Fixes for leaves at last level.

* Hash calc fixes and changes for arbitrary paths and snap sync specific.

* Hash calc changes. New methods to create nodes from proof data.

* Proof creation changes and sync accessor changes.

* Fixing merge

* Add RegisterDeleteByPrefix to RawStateMT

* Cleanup

* Whitespace format

* Use RlpMemo for parallel branch hash calc. Removed unused code.

* Format fix

* Remove unused proof leaves logic.

* More code cleanup

* Do not override Proof entries as Persistent

* Don't persist Proof entry type. New methods to check persistance and calculate storage root hash.

* Whitespace format

* ApplyRaw doesn't recalculate root hash

* Move Proof entry type handling out of dictionary into a specific sync BlockState implementation.

* Move processing proof nodes into SyncBlockState to avoid 2 commits when processing snap sync response.

* Measure recalculation of storage tree

* Allow parallel store trie recalculation for a single storage trie.

* Do not open new read only transaction after RawState commit - draft change

* Fix test, fix whitespace

* Apply suggestions from code review

Co-authored-by: Szymon Kulec <[email protected]>

* Refactoring and cleanup

* Merkle trie visitor refactoring

* no compiler constant, hint used for Merkle behavior instead

* Changes to RecalculateStorageHash for snap sync RlpMemo dependency. Added tests.

---------

Co-authored-by: Szymon Kulec <[email protected]>

src/Paprika.Tests/Chain/BlockchainTests.cs

@@ -709,7 +709,7 @@ private static Keccak BuildKey(int i)
 
     private class PreCommit : IPreCommitBehavior
     {
-        public Keccak BeforeCommit(ICommitWithStats commit, CacheBudget budget)
+        public Keccak BeforeCommit(ICommitWithStats commit, CacheBudget budget, bool isSnapSync = false)
         {
             var hashCode = RuntimeHelpers.GetHashCode(commit);
             Keccak hash = default;

src/Paprika.Tests/Chain/PreCommitBehaviorTests.cs

@@ -50,7 +50,7 @@ public AssertingKeysPreCommit(HashSet<Keccak> keccaks)
             _found = new HashSet<Keccak>();
         }
 
-        public Keccak BeforeCommit(ICommitWithStats commit, CacheBudget budget)
+        public Keccak BeforeCommit(ICommitWithStats commit, CacheBudget budget, bool isSnapSync = false)
         {
             _found.Clear();
 

src/Paprika.Tests/Chain/RawStateTests.cs

@@ -1,11 +1,11 @@
 using FluentAssertions;
 using Nethermind.Int256;
-using NUnit.Framework;
 using Paprika.Chain;
 using Paprika.Crypto;
 using Paprika.Data;
 using Paprika.Merkle;
 using Paprika.Store;
+using Paprika.RLP;
 
 namespace Paprika.Tests.Chain;
 
@@ -29,39 +29,13 @@ public async Task Raw_access_spin()
         }
 
         raw.DestroyAccount(account);
-        raw.Commit();
+        raw.Commit(keepOpened: true);
 
         raw.Finalize(1);
 
         raw.Hash.Should().Be(Keccak.EmptyTreeHash);
     }
 
-    [Test]
-    public async Task Snap_boundary()
-    {
-        var account = Values.Key1;
-        var keccak = Values.Key2;
-
-        using var db = PagedDb.NativeMemoryDb(256 * 1024, 2);
-        var merkle = new ComputeMerkleBehavior();
-
-        await using var blockchain = new Blockchain(db, merkle);
-        using var raw = blockchain.StartRaw();
-
-        raw.SetBoundary(NibblePath.FromKey(account).SliceTo(1), keccak);
-        raw.Commit();
-
-        var root1 = raw.Hash;
-
-        raw.SetAccount(account, new Account(1, 1));
-        raw.Commit();
-
-        raw.Finalize(1);
-
-        var root2 = raw.Hash;
-
-        root1.Should().NotBe(root2);
-    }
 
     [Test]
     public async Task Metadata_are_preserved()
@@ -81,13 +55,13 @@ public async Task Metadata_are_preserved()
 
         using var raw = blockchain.StartRaw();
         raw.SetAccount(a, new Account(valueA, valueA));
-        raw.Commit();
+        raw.Commit(keepOpened: true);
 
         raw.SetAccount(b, new Account(valueB, valueB));
-        raw.Commit();
+        raw.Commit(keepOpened: true);
 
         raw.SetAccount(c, new Account(valueC, valueC));
-        raw.Commit();
+        raw.Commit(keepOpened: true);
 
         var root = raw.Hash;
 
@@ -116,11 +90,11 @@ public async Task Disposal()
         for (uint i = 0; i < 1_000; i++)
         {
             raw.SetAccount(account, new Account(i, i));
-            raw.Commit();
+            raw.Commit(keepOpened: true);
         }
 
         raw.DestroyAccount(account);
-        raw.Commit();
+        raw.Commit(keepOpened: true);
 
         raw.Finalize(1);
 
@@ -140,10 +114,10 @@ public async Task DeleteByPrefix()
         using var raw = blockchain.StartRaw();
 
         raw.SetAccount(account, new Account(1, 1));
-        raw.Commit();
+        raw.Commit(keepOpened: true);
 
         raw.RegisterDeleteByPrefix(Key.Account(account));
-        raw.Commit();
+        raw.Commit(keepOpened: true);
 
         raw.Finalize(1);
 
@@ -169,10 +143,10 @@ public async Task DeleteByShortPrefix()
 
         raw.SetAccount(account1, new Account(1, 1));
         raw.SetAccount(account2, new Account(2, 2));
-        raw.Commit();
+        raw.Commit(keepOpened: true);
 
         raw.RegisterDeleteByPrefix(Key.Account(NibblePath.FromKey(account2).SliceTo(1)));
-        raw.Commit();
+        raw.Commit(keepOpened: true);
 
         raw.Finalize(1);
 
@@ -185,10 +159,10 @@ public async Task DeleteByShortPrefix()
         using var raw2 = blockchain.StartRaw();
 
         raw2.SetAccount(account2, new Account(2, 2));
-        raw2.Commit();
+        raw2.Commit(keepOpened: true);
 
         raw2.RegisterDeleteByPrefix(Key.Account(NibblePath.Empty));
-        raw2.Commit();
+        raw2.Commit(keepOpened: true);
 
         raw2.Finalize(2);
 
@@ -212,17 +186,91 @@ public void DeleteByPrefixStorage()
 
         raw.SetAccount(account, new Account(1, 1));
         raw.SetStorage(account, Values.Key2, new byte[] { 1, 2, 3, 4, 5 });
-        raw.Commit();
+        raw.Commit(keepOpened: true);
 
         using var read = db.BeginReadOnlyBatch();
         read.TryGet(Key.StorageCell(NibblePath.FromKey(account), Values.Key2), out _).Should().BeTrue();
 
         raw.RegisterDeleteByPrefix(Key.StorageCell(NibblePath.FromKey(account), NibblePath.Empty));
-        raw.Commit();
+        raw.Commit(keepOpened: true);
 
         raw.Finalize(1);
 
         using var read2 = db.BeginReadOnlyBatch();
         read2.TryGet(Key.StorageCell(NibblePath.FromKey(account), Values.Key2), out _).Should().BeFalse();
     }
+
+    [Test]
+    public void CalcRootFromRlpMemoDataState()
+    {
+        using var db = PagedDb.NativeMemoryDb(256 * 1024, 2);
+        var merkle = new ComputeMerkleBehavior();
+
+        var blockchain = new Blockchain(db, merkle);
+
+        using var raw = blockchain.StartRaw();
+
+        var random = GetRandom();
+
+        Span<byte> rlp = stackalloc byte[1024];
+        RlpStream stream = new RlpStream(rlp);
+        stream.StartSequence(529);
+
+        //all children to trigger parallel branch hash calculation
+        byte[] children = new byte[16];
+        Keccak[] childHashes = new Keccak[16];
+
+        for (int i = 0; i < 16; i++)
+        {
+            children[i] = (byte)i;
+            childHashes[i] = random.NextKeccak();
+            stream.Encode(childHashes[i]);
+        }
+        stream.EncodeEmptyArray();
+        KeccakOrRlp.FromSpan(rlp.Slice(0, stream.Position), out var checkKeccakOrRlp);
+
+        raw.CreateMerkleBranch(Keccak.Zero, NibblePath.Empty, children, childHashes);
+        Keccak newRootHash = raw.RefreshRootHash(true);
+
+        newRootHash.Should().Be(checkKeccakOrRlp.Keccak);
+    }
+
+    [Test]
+    public void CalcRootFromRlpMemoDataStorage()
+    {
+        using var db = PagedDb.NativeMemoryDb(256 * 1024, 2);
+        var merkle = new ComputeMerkleBehavior();
+
+        var blockchain = new Blockchain(db, merkle);
+
+        using var raw = blockchain.StartRaw();
+
+        var account = Values.Key1;
+
+        var random = GetRandom();
+
+        Span<byte> rlp = stackalloc byte[1024];
+        RlpStream stream = new RlpStream(rlp);
+        stream.StartSequence(529);
+
+        //all children to trigger parallel branch hash calculation
+        byte[] children = new byte[16];
+        Keccak[] childHashes = new Keccak[16];
+
+        for (int i = 0; i < 16; i++)
+        {
+            children[i] = (byte)i;
+            childHashes[i] = random.NextKeccak();
+            stream.Encode(childHashes[i]);
+        }
+        stream.EncodeEmptyArray();
+        KeccakOrRlp.FromSpan(rlp.Slice(0, stream.Position), out var checkKeccakOrRlp);
+
+        raw.CreateMerkleBranch(account, NibblePath.Empty, children, childHashes);
+        Keccak newRootHash = raw.RecalculateStorageRoot(account, true);
+
+        newRootHash.Should().Be(checkKeccakOrRlp.Keccak);
+    }
+
+    private static Random GetRandom() => new(13);
 }

src/Paprika.Tests/Merkle/AdditionalTests.cs

@@ -41,7 +41,7 @@ public async Task Account_destruction_same_block()
 
         using var read = blockchain.StartReadOnly(hash);
         var account = read.GetAccount(Key0);
-        var recalculatedStorage = merkle.CalculateStorageHash(read, Key0);
+        var recalculatedStorage = merkle.GetStorageHash(read, Key0);
 
         recalculatedStorage.Should().Be(account.StorageRootHash);
     }