Eip4444 - Portal Integration (#3087)

* initial structuring of portal integration

* prelim single point integration

* remove fluffy taskpool + add only rpc layer

* fix ci

* remove unwanted changes

* reorder

* mainnet detection and limit setup

* eip4444 limit

* resolve impl change eth68

* fix initializer

* seperate history expiry from portal

* remove extra type

* better warn message

* simplify return

* hide configureable params

* fix: merge conflict

* fix: remove return

* remove unnecessary copy of the blk.body

* fix with suggested changes

* remove func export

* resolve merge conflicts

* fix

Author: advaita-saha

.gitignore

@@ -15,6 +15,7 @@
 # Ignore dynamic, static libs and libtool archive files
 *.so
 *.dylib
+*.dSYM
 *.a
 *.la
 *.exe

execution_chain/common/common.nim

@@ -362,6 +362,9 @@ func eip150Hash*(com: CommonRef): Hash32 =
 func daoForkBlock*(com: CommonRef): Opt[BlockNumber] =
   com.config.daoForkBlock
 
+func posBlock*(com: CommonRef): Opt[BlockNumber] =
+  com.config.posBlock
+
 func daoForkSupport*(com: CommonRef): bool =
   com.config.daoForkSupport
 

execution_chain/config.nim

@@ -445,6 +445,21 @@ type
         defaultValueDesc: $RpcFlag.Eth
         name: "ws-api" }: seq[string]
 
+      historyExpiry* {.
+        desc: "Enable the data from Portal Network"
+        defaultValue: false
+        name: "history-expiry" }: bool
+
+      historyExpiryLimit* {.
+        hidden
+        desc: "Limit the number of blocks to be kept in history"
+        name: "debug-history-expiry-limit" }: Option[BlockNumber]
+
+      portalUrl* {.
+        desc: "URL of the Portal Network"
+        defaultValue: ""
+        name: "portal-url" }: string
+
       engineApiEnabled* {.
         desc: "Enable the Engine API"
         defaultValue: false

execution_chain/core/chain/forked_chain.nim

@@ -15,8 +15,11 @@ import
   results,
   std/[tables, algorithm],
   ../../common,
-  ../../db/core_db,
-  ../../db/fcu_db,
+  ../../db/[core_db, fcu_db],
+  ../../evm/types,
+  ../../evm/state,
+  ../validate,
+  ../../portal/portal,
   ./forked_chain/[
     chain_desc,
     chain_branch,
@@ -666,6 +669,12 @@ func txRecords*(c: ForkedChainRef, txHash: Hash32): (Hash32, uint64) =
 func isInMemory*(c: ForkedChainRef, blockHash: Hash32): bool =
   c.hashToBlock.hasKey(blockHash)
 
+func isHistoryExpiryActive*(c: ForkedChainRef): bool =
+  not c.portal.isNil
+
+func isPortalActive(c: ForkedChainRef): bool =
+  (not c.portal.isNil) and c.portal.portalEnabled
+
 func memoryBlock*(c: ForkedChainRef, blockHash: Hash32): BlockDesc =
   c.hashToBlock.withValue(blockHash, loc):
     return loc.branch.blocks[loc.index]
@@ -702,15 +711,20 @@ proc headerByNumber*(c: ForkedChainRef, number: BlockNumber): Result[Header, str
     return err("Requested block number not exists: " & $number)
 
   if number < c.baseBranch.tailNumber:
-    return c.baseTxFrame.getBlockHeader(number)
+    let hdr = c.baseTxFrame.getBlockHeader(number).valueOr:
+      if c.isPortalActive:
+        return c.portal.getHeaderByNumber(number)
+      else:
+        return err("Portal inactive, block not found, number = " & $number)
+    return ok(hdr)
 
   var branch = c.activeBranch
   while not branch.isNil:
     if number >= branch.tailNumber:
       return ok(branch.blocks[number - branch.tailNumber].blk.header)
     branch = branch.parent
 
-  err("Header not found, number = " & $number)
+  err("Block not found, number = " & $number)
 
 func finalizedHeader*(c: ForkedChainRef): Header =
   c.hashToBlock.withValue(c.pendingFCU, loc):
@@ -739,7 +753,12 @@ func safeBlock*(c: ForkedChainRef): Block =
 proc headerByHash*(c: ForkedChainRef, blockHash: Hash32): Result[Header, string] =
   c.hashToBlock.withValue(blockHash, loc):
     return ok(loc[].header)
-  c.baseTxFrame.getBlockHeader(blockHash)
+  let hdr = c.baseTxFrame.getBlockHeader(blockHash).valueOr:
+    if c.isPortalActive:
+      return c.portal.getHeaderByHash(blockHash)
+    else:
+      return err("Block header not found")
+  ok(hdr)
 
 proc txDetailsByTxHash*(c: ForkedChainRef, txHash: Hash32): Result[(Hash32, uint64), string] =
   if c.txRecords.hasKey(txHash):
@@ -751,15 +770,9 @@ proc txDetailsByTxHash*(c: ForkedChainRef, txHash: Hash32): Result[(Hash32, uint
     header = ?c.headerByNumber(txDetails.blockNumber)
     blockHash = header.computeBlockHash
   return ok((blockHash, txDetails.index))
-
-proc blockByHash*(c: ForkedChainRef, blockHash: Hash32): Result[Block, string] =
-  # used by getPayloadBodiesByHash
-  # https://github.com/ethereum/execution-apis/blob/v1.0.0-beta.4/src/engine/shanghai.md#specification-3
-  # 4. Client software MAY NOT respond to requests for finalized blocks by hash.
-  c.hashToBlock.withValue(blockHash, loc):
-    return ok(loc[].blk)
-  c.baseTxFrame.getEthBlock(blockHash)
-
+  
+# TODO: Doesn't fetch data from portal
+# Aristo returns empty txs for both non-existent blocks and existing blocks with no txs [ Solve ? ]
 proc blockBodyByHash*(c: ForkedChainRef, blockHash: Hash32): Result[BlockBody, string] =
   c.hashToBlock.withValue(blockHash, loc):
     let blk = loc[].blk
@@ -770,12 +783,32 @@ proc blockBodyByHash*(c: ForkedChainRef, blockHash: Hash32): Result[BlockBody, s
     ))
   c.baseTxFrame.getBlockBody(blockHash)
 
+proc blockByHash*(c: ForkedChainRef, blockHash: Hash32): Result[Block, string] =
+  # used by getPayloadBodiesByHash
+  # https://github.com/ethereum/execution-apis/blob/v1.0.0-beta.4/src/engine/shanghai.md#specification-3
+  # 4. Client software MAY NOT respond to requests for finalized blocks by hash.
+  c.hashToBlock.withValue(blockHash, loc):
+    return ok(loc[].blk)
+  let blk = c.baseTxFrame.getEthBlock(blockHash)
+  # Serves portal data if block not found in db
+  if blk.isErr or (blk.get.transactions.len == 0 and blk.get.header.transactionsRoot != zeroHash32):
+    if c.isPortalActive:
+      return c.portal.getBlockByHash(blockHash)
+  blk
+
 proc blockByNumber*(c: ForkedChainRef, number: BlockNumber): Result[Block, string] =
   if number > c.activeBranch.headNumber:
     return err("Requested block number not exists: " & $number)
 
   if number <= c.baseBranch.tailNumber:
-    return c.baseTxFrame.getEthBlock(number)
+    let blk = c.baseTxFrame.getEthBlock(number)
+    # Txs not there in db - Happens during era1/era import, when we don't store txs and receipts
+    if blk.isErr or (blk.get.transactions.len == 0 and blk.get.header.transactionsRoot != emptyRoot):
+      # Serves portal data if block not found in database
+      if c.isPortalActive:
+        return c.portal.getBlockByNumber(number)
+    else:
+      return blk
 
   var branch = c.activeBranch
   while not branch.isNil:

execution_chain/core/chain/forked_chain/chain_desc.nim

@@ -11,10 +11,10 @@
 {.push raises: [].}
 
 import
-  std/[tables],
+  std/tables,
   ../../../common,
-  ../../../db/core_db,
-  ../../../db/fcu_db,
+  ../../../db/[core_db, fcu_db],
+  ../../../portal/portal,
   ./block_quarantine,
   ./chain_branch
 
@@ -66,6 +66,9 @@ type
       # When move forward, this is the minimum distance
       # to move the base. And the bulk writing can works
       # efficiently.
+      
+    portal*: HistoryExpiryRef
+      # History Expiry tracker and portal access entry point
 
     fcuHead*: FcuHashAndNumber
     fcuSafe*: FcuHashAndNumber

execution_chain/nimbus_execution_client.nim

@@ -28,7 +28,8 @@ import
   ./db/core_db/persistent,
   ./db/storage_types,
   ./sync/wire_protocol,
-  ./common/chain_config_hash
+  ./common/chain_config_hash,
+  ./portal/portal
 
 from beacon_chain/nimbus_binary_common import setupFileLimits
 
@@ -40,11 +41,14 @@ from beacon_chain/nimbus_binary_common import setupFileLimits
 proc basicServices(nimbus: NimbusNode,
                    conf: NimbusConf,
                    com: CommonRef) =
+  # Setup the chain
   let fc = ForkedChainRef.init(com)
   fc.deserialize().isOkOr:
     warn "FC.deserialize", msg=error
 
   nimbus.fc = fc
+  # Setup history expiry and portal
+  nimbus.fc.portal = HistoryExpiryRef.init(conf, com)
   # txPool must be informed of active head
   # so it can know the latest account state
   # e.g. sender nonce, etc