Add engine_getBlobsV3

src/Nethermind/Nethermind.Core/Collections/ArrayPoolList.cs

@@ -243,7 +243,7 @@ public void Dispose()
         ArrayPoolListCore<T>.Dispose(_arrayPool, ref _array, ref _count, ref _capacity, ref _disposed);
 
 #if DEBUG
-    GC.SuppressFinalize(this);
+        GC.SuppressFinalize(this);
 #endif
     }
 

src/Nethermind/Nethermind.Facade/BlockchainBridge.cs

@@ -109,7 +109,7 @@ private bool TryGetCanonicalTransaction(
             return (null, 0, null, 0);
         }
 
-        public (TxReceipt? Receipt, Transaction Transaction, UInt256? baseFee) GetTransaction(Hash256 txHash, bool checkTxnPool = true) =>
+        public (TxReceipt? Receipt, Transaction? Transaction, UInt256? baseFee) GetTransaction(Hash256 txHash, bool checkTxnPool = true) =>
             TryGetCanonicalTransaction(txHash, out Transaction? tx, out TxReceipt? txReceipt, out Block? block, out TxReceipt[]? _)
                 ? (txReceipt, tx, block.BaseFeePerGas)
                 : checkTxnPool && txPool.TryGetPendingTransaction(txHash, out Transaction? transaction)

src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs

@@ -1553,7 +1553,8 @@ public void Should_return_expected_capabilities_for_mainnet()
             nameof(IEngineRpcModule.engine_newPayloadV4),
 
             nameof(IEngineRpcModule.engine_getPayloadV5),
-            nameof(IEngineRpcModule.engine_getBlobsV2)
+            nameof(IEngineRpcModule.engine_getBlobsV2),
+            nameof(IEngineRpcModule.engine_getBlobsV3)
         };
         Assert.That(result, Is.EquivalentTo(expectedMethods));
     }

src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs

@@ -385,7 +385,7 @@ public async Task<string> NewPayloadV3_should_verify_blob_versioned_hashes_again
                  Substitute.For<IHandler<TransitionConfigurationV1, TransitionConfigurationV1>>(),
                  Substitute.For<IHandler<IEnumerable<string>, IEnumerable<string>>>(),
                  Substitute.For<IAsyncHandler<byte[][], IEnumerable<BlobAndProofV1?>>>(),
-                 Substitute.For<IAsyncHandler<byte[][], IEnumerable<BlobAndProofV2>?>>(),
+                 Substitute.For<IAsyncHandler<GetBlobsHandlerV2Request, IEnumerable<BlobAndProofV2?>?>>(),
                  Substitute.For<IEngineRequestsTracker>(),
                  chain.SpecProvider,
                  new GCKeeper(NoGCStrategy.Instance, chain.LogManager),

src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V5.cs

@@ -52,7 +52,7 @@ public async Task GetBlobsV2_should_throw_if_more_than_128_requested_blobs([Valu
             request.Add(Bytes.FromHexString(i.ToString("X64")));
         }
 
-        ResultWrapper<IEnumerable<BlobAndProofV2>?> result = await rpcModule.engine_getBlobsV2(request.ToArray());
+        ResultWrapper<IEnumerable<BlobAndProofV2?>?> result = await rpcModule.engine_getBlobsV2(request.ToArray());
 
         if (requestSize > 128)
         {
@@ -75,7 +75,7 @@ public async Task GetBlobsV2_should_handle_empty_request()
         });
         IEngineRpcModule rpcModule = chain.EngineRpcModule;
 
-        ResultWrapper<IEnumerable<BlobAndProofV2>?> result = await rpcModule.engine_getBlobsV2([]);
+        ResultWrapper<IEnumerable<BlobAndProofV2?>?> result = await rpcModule.engine_getBlobsV2([]);
 
         result.Result.Should().Be(Result.Success);
         result.Data.Should().BeEquivalentTo(ArraySegment<BlobAndProofV2>.Empty);
@@ -99,14 +99,14 @@ public async Task GetBlobsV2_should_return_requested_blobs([Values(1, 2, 3, 4, 5
 
         chain.TxPool.SubmitTx(blobTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted);
 
-        ResultWrapper<IEnumerable<BlobAndProofV2>?> result = await rpcModule.engine_getBlobsV2(blobTx.BlobVersionedHashes!);
+        ResultWrapper<IEnumerable<BlobAndProofV2?>?> result = await rpcModule.engine_getBlobsV2(blobTx.BlobVersionedHashes!);
 
         ShardBlobNetworkWrapper wrapper = (ShardBlobNetworkWrapper)blobTx.NetworkWrapper!;
 
         result.Data.Should().NotBeNull();
-        result.Data!.Select(static b => b.Blob).Should().BeEquivalentTo(wrapper.Blobs);
-        result.Data!.Select(static b => b.Proofs.Length).Should().HaveCount(numberOfBlobs);
-        result.Data!.Select(static b => b.Proofs).Should().BeEquivalentTo(wrapper.Proofs.Chunk(128));
+        result.Data!.Select(static b => b!.Blob).Should().BeEquivalentTo(wrapper.Blobs);
+        result.Data!.Select(static b => b!.Proofs.Length).Should().HaveCount(numberOfBlobs);
+        result.Data!.Select(static b => b!.Proofs).Should().BeEquivalentTo(wrapper.Proofs.Chunk(128));
     }
 
     [Test]
@@ -127,7 +127,7 @@ public async Task GetBlobsV2_should_return_empty_array_when_blobs_not_found([Val
             .SignedAndResolved(chain.EthereumEcdsa, TestItem.PrivateKeyA).TestObject;
 
         // requesting hashes that are not present in TxPool
-        ResultWrapper<IEnumerable<BlobAndProofV2>?> result = await rpcModule.engine_getBlobsV2(blobTx.BlobVersionedHashes!);
+        ResultWrapper<IEnumerable<BlobAndProofV2?>?> result = await rpcModule.engine_getBlobsV2(blobTx.BlobVersionedHashes!);
 
         result.Result.Should().Be(Result.Success);
         result.Data.Should().BeNull();
@@ -162,7 +162,7 @@ public async Task GetBlobsV2_should_return_empty_array_when_only_some_blobs_foun
             blobVersionedHashesRequest.Add(addActualHash ? blobTx.BlobVersionedHashes![actualIndex++]! : Bytes.FromHexString(i.ToString("X64")));
         }
 
-        ResultWrapper<IEnumerable<BlobAndProofV2>?> result = await rpcModule.engine_getBlobsV2(blobVersionedHashesRequest.ToArray());
+        ResultWrapper<IEnumerable<BlobAndProofV2?>?> result = await rpcModule.engine_getBlobsV2(blobVersionedHashesRequest.ToArray());
         if (multiplier > 1)
         {
             result.Result.Should().Be(Result.Success);
@@ -173,9 +173,65 @@ public async Task GetBlobsV2_should_return_empty_array_when_only_some_blobs_foun
             ShardBlobNetworkWrapper wrapper = (ShardBlobNetworkWrapper)blobTx.NetworkWrapper!;
 
             result.Data.Should().NotBeNull();
-            result.Data!.Select(static b => b.Blob).Should().BeEquivalentTo(wrapper.Blobs);
-            result.Data!.Select(static b => b.Proofs.Length).Should().HaveCount(numberOfBlobs);
-            result.Data!.Select(static b => b.Proofs).Should().BeEquivalentTo(wrapper.Proofs.Chunk(128));
+            result.Data!.Select(static b => b!.Blob).Should().BeEquivalentTo(wrapper.Blobs);
+            result.Data!.Select(static b => b!.Proofs.Length).Should().HaveCount(numberOfBlobs);
+            result.Data!.Select(static b => b!.Proofs).Should().BeEquivalentTo(wrapper.Proofs.Chunk(128));
+        }
+    }
+
+    [Test]
+    public async Task GetBlobsV3_should_return_partial_results_with_nulls_for_missing_blobs([Values(1, 2, 3, 4, 5, 6)] int numberOfBlobs, [Values(1, 2)] int multiplier)
+    {
+        int requestSize = multiplier * numberOfBlobs;
+
+        MergeTestBlockchain chain = await CreateBlockchain(releaseSpec: Osaka.Instance, mergeConfig: new MergeConfig()
+        {
+            NewPayloadBlockProcessingTimeout = (int)TimeSpan.FromDays(1).TotalMilliseconds
+        });
+        IEngineRpcModule rpcModule = chain.EngineRpcModule;
+
+        Transaction blobTx = Build.A.Transaction
+            .WithShardBlobTxTypeAndFields(numberOfBlobs, spec: Osaka.Instance)
+            .WithMaxFeePerGas(1.GWei())
+            .WithMaxPriorityFeePerGas(1.GWei())
+            .WithMaxFeePerBlobGas(1000.Wei())
+            .SignedAndResolved(chain.EthereumEcdsa, TestItem.PrivateKeyA).TestObject;
+
+        chain.TxPool.SubmitTx(blobTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted);
+
+        List<byte[]> blobVersionedHashesRequest = new List<byte[]>(requestSize);
+
+        int actualIndex = 0;
+        for (int i = 0; i < requestSize; i++)
+        {
+            bool addActualHash = i % multiplier == 0;
+            blobVersionedHashesRequest.Add(addActualHash ? blobTx.BlobVersionedHashes![actualIndex++]! : Bytes.FromHexString(i.ToString("X64")));
+        }
+
+        ResultWrapper<IEnumerable<BlobAndProofV2?>?> result = await rpcModule.engine_getBlobsV3(blobVersionedHashesRequest.ToArray());
+
+        result.Result.Should().Be(Result.Success);
+        result.Data.Should().NotBeNull();
+        result.Data!.Should().HaveCount(requestSize);
+
+        ShardBlobNetworkWrapper wrapper = (ShardBlobNetworkWrapper)blobTx.NetworkWrapper!;
+
+        // V3 returns partial results with nulls for missing blobs
+        int foundIndex = 0;
+        for (int i = 0; i < requestSize; i++)
+        {
+            bool shouldBeFound = i % multiplier == 0;
+            if (shouldBeFound)
+            {
+                result.Data!.ElementAt(i).Should().NotBeNull();
+                result.Data!.ElementAt(i)!.Blob.Should().BeEquivalentTo(wrapper.Blobs[foundIndex]);
+                result.Data!.ElementAt(i)!.Proofs.Should().BeEquivalentTo(wrapper.Proofs.Skip(foundIndex * 128).Take(128));
+                foundIndex++;
+            }
+            else
+            {
+                result.Data!.ElementAt(i).Should().BeNull();
+            }
         }
     }
 

src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.Osaka.cs

@@ -12,11 +12,14 @@ namespace Nethermind.Merge.Plugin;
 public partial class EngineRpcModule : IEngineRpcModule
 {
     private readonly IAsyncHandler<byte[], GetPayloadV5Result?> _getPayloadHandlerV5;
-    private readonly IAsyncHandler<byte[][], IEnumerable<BlobAndProofV2>?> _getBlobsHandlerV2;
+    private readonly IAsyncHandler<GetBlobsHandlerV2Request, IEnumerable<BlobAndProofV2?>?> _getBlobsHandlerV2;
 
     public Task<ResultWrapper<GetPayloadV5Result?>> engine_getPayloadV5(byte[] payloadId)
         => _getPayloadHandlerV5.HandleAsync(payloadId);
 
-    public Task<ResultWrapper<IEnumerable<BlobAndProofV2>?>> engine_getBlobsV2(byte[][] blobVersionedHashes)
-         => _getBlobsHandlerV2.HandleAsync(blobVersionedHashes);
+    public Task<ResultWrapper<IEnumerable<BlobAndProofV2?>?>> engine_getBlobsV2(byte[][] blobVersionedHashes)
+         => _getBlobsHandlerV2.HandleAsync(new(blobVersionedHashes));
+
+    public Task<ResultWrapper<IEnumerable<BlobAndProofV2?>?>> engine_getBlobsV3(byte[][] blobVersionedHashes)
+         => _getBlobsHandlerV2.HandleAsync(new(blobVersionedHashes, true));
 }

src/Nethermind/Nethermind.Merge.Plugin/EngineRpcModule.cs

@@ -34,7 +34,7 @@ public EngineRpcModule(
         IHandler<TransitionConfigurationV1, TransitionConfigurationV1> transitionConfigurationHandler,
         IHandler<IEnumerable<string>, IEnumerable<string>> capabilitiesHandler,
         IAsyncHandler<byte[][], IEnumerable<BlobAndProofV1?>> getBlobsHandler,
-        IAsyncHandler<byte[][], IEnumerable<BlobAndProofV2>?> getBlobsHandlerV2,
+        IAsyncHandler<GetBlobsHandlerV2Request, IEnumerable<BlobAndProofV2?>?> getBlobsHandlerV2,
         IEngineRequestsTracker engineRequestsTracker,
         ISpecProvider specProvider,
         GCKeeper gcKeeper,

src/Nethermind/Nethermind.Merge.Plugin/Handlers/EngineRpcCapabilitiesProvider.cs

@@ -53,6 +53,7 @@ public EngineRpcCapabilitiesProvider(ISpecProvider specProvider)
             // Osaka
             _capabilities[nameof(IEngineRpcModule.engine_getPayloadV5)] = (spec.IsEip7594Enabled, spec.IsEip7594Enabled);
             _capabilities[nameof(IEngineRpcModule.engine_getBlobsV2)] = (spec.IsEip7594Enabled, false);
+            _capabilities[nameof(IEngineRpcModule.engine_getBlobsV3)] = (spec.IsEip7594Enabled, false);
         }
 
         return _capabilities;

src/Nethermind/Nethermind.Merge.Plugin/Handlers/GetBlobsHandlerV2.cs

@@ -2,7 +2,6 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System.Collections.Generic;
-using System.Linq;
 using System.Threading.Tasks;
 using Nethermind.Core.Collections;
 using Nethermind.JsonRpc;
@@ -11,41 +10,45 @@
 
 namespace Nethermind.Merge.Plugin.Handlers;
 
-public class GetBlobsHandlerV2(ITxPool txPool) : IAsyncHandler<byte[][], IEnumerable<BlobAndProofV2>?>
+public class GetBlobsHandlerV2(ITxPool txPool) : IAsyncHandler<GetBlobsHandlerV2Request, IEnumerable<BlobAndProofV2?>?>
 {
     private const int MaxRequest = 128;
 
-    private static readonly Task<ResultWrapper<IEnumerable<BlobAndProofV2>?>> NotFound = Task.FromResult(ResultWrapper<IEnumerable<BlobAndProofV2>?>.Success(null));
+    private static readonly Task<ResultWrapper<IEnumerable<BlobAndProofV2?>?>> NotFound = Task.FromResult(ResultWrapper<IEnumerable<BlobAndProofV2?>?>.Success(null));
 
-    public Task<ResultWrapper<IEnumerable<BlobAndProofV2>?>> HandleAsync(byte[][] request)
+    public Task<ResultWrapper<IEnumerable<BlobAndProofV2?>?>> HandleAsync(GetBlobsHandlerV2Request request)
     {
-        if (request.Length > MaxRequest)
+        if (request.BlobVersionedHashes.Length > MaxRequest)
         {
             var error = $"The number of requested blobs must not exceed {MaxRequest}";
-            return ResultWrapper<IEnumerable<BlobAndProofV2>?>.Fail(error, MergeErrorCodes.TooLargeRequest);
+            return ResultWrapper<IEnumerable<BlobAndProofV2?>?>.Fail(error, MergeErrorCodes.TooLargeRequest);
         }
 
-        Metrics.GetBlobsRequestsTotal += request.Length;
+        Metrics.GetBlobsRequestsTotal += request.BlobVersionedHashes.Length;
 
-        var count = txPool.GetBlobCounts(request);
+        int count = txPool.GetBlobCounts(request.BlobVersionedHashes);
         Metrics.GetBlobsRequestsInBlobpoolTotal += count;
 
-        // quick fail if we don't have some blob
-        if (count != request.Length)
+        // quick fail if we don't have some blob (unless partial return is allowed)
+        if (!request.AllowPartialReturn && count != request.BlobVersionedHashes.Length)
         {
             return ReturnEmptyArray();
         }
 
-        ArrayPoolList<BlobAndProofV2> response = new(request.Length);
+        ArrayPoolList<BlobAndProofV2?> response = new(request.BlobVersionedHashes.Length);
 
         try
         {
-            foreach (byte[] requestedBlobVersionedHash in request)
+            foreach (byte[] requestedBlobVersionedHash in request.BlobVersionedHashes)
             {
                 if (txPool.TryGetBlobAndProofV1(requestedBlobVersionedHash, out byte[]? blob, out byte[][]? cellProofs))
                 {
                     response.Add(new BlobAndProofV2(blob, cellProofs));
                 }
+                else if (request.AllowPartialReturn)
+                {
+                    response.Add(null);
+                }
                 else
                 {
                     // fail if we were not able to collect full blob data
@@ -55,7 +58,7 @@ public class GetBlobsHandlerV2(ITxPool txPool) : IAsyncHandler<byte[][], IEnumer
             }
 
             Metrics.GetBlobsRequestsSuccessTotal++;
-            return ResultWrapper<IEnumerable<BlobAndProofV2>?>.Success(response);
+            return ResultWrapper<IEnumerable<BlobAndProofV2?>?>.Success(response);
         }
         catch
         {
@@ -64,9 +67,11 @@ public class GetBlobsHandlerV2(ITxPool txPool) : IAsyncHandler<byte[][], IEnumer
         }
     }
 
-    private Task<ResultWrapper<IEnumerable<BlobAndProofV2>?>> ReturnEmptyArray()
+    private Task<ResultWrapper<IEnumerable<BlobAndProofV2?>?>> ReturnEmptyArray()
     {
         Metrics.GetBlobsRequestsFailureTotal++;
         return NotFound;
     }
 }
+
+public readonly record struct GetBlobsHandlerV2Request(byte[][] BlobVersionedHashes, bool AllowPartialReturn = false);

src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Osaka.cs

@@ -21,5 +21,11 @@ public partial interface IEngineRpcModule : IRpcModule
         Description = "Returns requested blobs and proofs.",
         IsSharable = true,
         IsImplemented = true)]
-    public Task<ResultWrapper<IEnumerable<BlobAndProofV2>?>> engine_getBlobsV2(byte[][] blobVersionedHashes);
+    public Task<ResultWrapper<IEnumerable<BlobAndProofV2?>?>> engine_getBlobsV2(byte[][] blobVersionedHashes);
+
+    [JsonRpcMethod(
+        Description = "Returns requested blobs and proofs.",
+        IsSharable = true,
+        IsImplemented = true)]
+    public Task<ResultWrapper<IEnumerable<BlobAndProofV2?>?>> engine_getBlobsV3(byte[][] blobVersionedHashes);
 }

src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs

@@ -330,7 +330,7 @@ protected override void Load(ContainerBuilder builder)
                 .AddSingleton<IHandler<IEnumerable<string>, IEnumerable<string>>, ExchangeCapabilitiesHandler>()
                     .AddSingleton<IRpcCapabilitiesProvider, EngineRpcCapabilitiesProvider>()
                 .AddSingleton<IAsyncHandler<byte[][], IEnumerable<BlobAndProofV1?>>, GetBlobsHandler>()
-                .AddSingleton<IAsyncHandler<byte[][], IEnumerable<BlobAndProofV2>?>, GetBlobsHandlerV2>()
+                .AddSingleton<IAsyncHandler<GetBlobsHandlerV2Request, IEnumerable<BlobAndProofV2?>?>, GetBlobsHandlerV2>()
 
                 .AddSingleton<NoSyncGcRegionStrategy>()
                 .AddSingleton<GCKeeper>((ctx) =>

src/Nethermind/Nethermind.Taiko.Test/TxPoolContentListsTests.cs

@@ -83,7 +83,7 @@ public int[][] Test_TxLists_AreConstructed(
             Substitute.For<IHandler<TransitionConfigurationV1, TransitionConfigurationV1>>(),
             Substitute.For<IHandler<IEnumerable<string>, IEnumerable<string>>>(),
             Substitute.For<IAsyncHandler<byte[][], IEnumerable<BlobAndProofV1?>>>(),
-            Substitute.For<IAsyncHandler<byte[][], IEnumerable<BlobAndProofV2>?>>(),
+            Substitute.For<IAsyncHandler<GetBlobsHandlerV2Request, IEnumerable<BlobAndProofV2?>?>>(),
             Substitute.For<IEngineRequestsTracker>(),
             Substitute.For<ISpecProvider>(),
             null!,

src/Nethermind/Nethermind.Taiko/Rpc/TaikoEngineRpcModule.cs

@@ -46,7 +46,7 @@ public class TaikoEngineRpcModule(IAsyncHandler<byte[], ExecutionPayload?> getPa
         IHandler<TransitionConfigurationV1, TransitionConfigurationV1> transitionConfigurationHandler,
         IHandler<IEnumerable<string>, IEnumerable<string>> capabilitiesHandler,
         IAsyncHandler<byte[][], IEnumerable<BlobAndProofV1?>> getBlobsHandler,
-        IAsyncHandler<byte[][], IEnumerable<BlobAndProofV2>?> getBlobsHandlerV2,
+        IAsyncHandler<GetBlobsHandlerV2Request, IEnumerable<BlobAndProofV2?>?> getBlobsHandlerV2,
         IEngineRequestsTracker engineRequestsTracker,
         ISpecProvider specProvider,
         GCKeeper gcKeeper,