From 1cb4a99ad08601c3face04633fe01214a0e5bedf Mon Sep 17 00:00:00 2001 From: Andres Adjimann Date: Tue, 20 Sep 2022 13:48:03 -0300 Subject: [PATCH] feat: get logs from a fork network directly --- .../src/data-managers/block-manager.ts | 69 ++++-- .../src/data-managers/blocklog-manager.ts | 127 +++++++--- .../tests/forking/contracts/IERC20.sol | 77 ++++++ .../ethereum/tests/forking/logs.test.ts | 219 ++++++++++++++++++ .../src/things/json-rpc/json-rpc-quantity.ts | 8 + 5 files changed, 439 insertions(+), 61 deletions(-) create mode 100644 src/chains/ethereum/ethereum/tests/forking/contracts/IERC20.sol create mode 100644 src/chains/ethereum/ethereum/tests/forking/logs.test.ts diff --git a/src/chains/ethereum/ethereum/src/data-managers/block-manager.ts b/src/chains/ethereum/ethereum/src/data-managers/block-manager.ts index e30eef0b5d..54bb828664 100644 --- a/src/chains/ethereum/ethereum/src/data-managers/block-manager.ts +++ b/src/chains/ethereum/ethereum/src/data-managers/block-manager.ts @@ -177,37 +177,56 @@ export default class BlockManager extends Manager { } async getNumberFromHash(hash: string | Buffer | Tag) { - return this.#blockIndexes.get(Data.toBuffer(hash)).catch(e => { - if (e.status === NOTFOUND) return null; - throw e; - }) as Promise; + const number = await this.#blockIndexes + .get(Data.toBuffer(hash)) + .catch(e => { + if (e.status === NOTFOUND) return null; + throw e; + }); + if (number !== null) { + return Quantity.from(number); + } + const fallback = this.#blockchain.fallback; + if (fallback) { + const json = await fallback.request("eth_getBlockByHash", [ + Data.from(hash), + true + ]); + if (json) { + return Quantity.from(json.number); + } + } + return null; } async getByHash(hash: string | Buffer | Tag) { - const number = await this.getNumberFromHash(hash); - if (number === null) { - const fallback = this.#blockchain.fallback; - if (fallback) { - const json = await fallback.request("eth_getBlockByHash", [ - Data.from(hash), - true - ]); - if (json) { - const blockNumber = BigInt(json.number); - if (blockNumber <= fallback.blockNumber.toBigInt()) { - const common = fallback.getCommonForBlockNumber( - this.#common, - blockNumber - ); - return new Block(BlockManager.rawFromJSON(json, common), common); - } + const number = await this.#blockIndexes + .get(Data.toBuffer(hash)) + .catch(e => { + if (e.status === NOTFOUND) return null; + throw e; + }); + if (number !== null) { + return this.get(number); + } + const fallback = this.#blockchain.fallback; + if (fallback) { + const json = await fallback.request("eth_getBlockByHash", [ + Data.from(hash), + true + ]); + if (json) { + const blockNumber = BigInt(json.number); + if (blockNumber <= fallback.blockNumber.toBigInt()) { + const common = fallback.getCommonForBlockNumber( + this.#common, + blockNumber + ); + return new Block(BlockManager.rawFromJSON(json, common), common); } } - - return null; - } else { - return this.get(number); } + return null; } async getRawByBlockNumber(blockNumber: Quantity): Promise { diff --git a/src/chains/ethereum/ethereum/src/data-managers/blocklog-manager.ts b/src/chains/ethereum/ethereum/src/data-managers/blocklog-manager.ts index dd32185f85..37afcf881f 100644 --- a/src/chains/ethereum/ethereum/src/data-managers/blocklog-manager.ts +++ b/src/chains/ethereum/ethereum/src/data-managers/blocklog-manager.ts @@ -8,6 +8,7 @@ import { GanacheLevelUp } from "../database"; export default class BlockLogManager extends Manager { #blockchain: Blockchain; + constructor(base: GanacheLevelUp, blockchain: Blockchain) { super(base, BlockLogs); this.#blockchain = blockchain; @@ -19,11 +20,13 @@ export default class BlockLogManager extends Manager { log.blockNumber = Quantity.from(key); } else if (this.#blockchain.fallback) { const block = Quantity.from(key); - const res = await this.#blockchain.fallback.request( - "eth_getLogs", - [{ fromBlock: block, toBlock: block }] - ); - return BlockLogs.fromJSON(res); + if (this.#blockchain.fallback.isValidForkBlockNumber(block)) { + const res = await this.#blockchain.fallback.request( + "eth_getLogs", + [{ fromBlock: block, toBlock: block }] + ); + return BlockLogs.fromJSON(res); + } } return log; } @@ -35,42 +38,94 @@ export default class BlockLogManager extends Manager { const blockNumber = await blockchain.blocks.getNumberFromHash( filter.blockHash ); - if (!blockNumber) return []; - - const logs = await this.get(blockNumber); + if (!blockNumber) { + return []; + } + const logs = await this.get(blockNumber.toBuffer()); return logs ? [...logs.filter(addresses, topics)] : []; - } else { - const { addresses, topics, fromBlock, toBlockNumber } = parseFilter( - filter, - blockchain + } + const { fromBlock, toBlock } = parseFilter(filter, blockchain); + if (fromBlock.toBigInt() > toBlock.toBigInt()) { + throw new Error( + "One of the blocks specified in filter (fromBlock, toBlock or blockHash) cannot be found." ); + } - const pendingLogsPromises: Promise[] = [ - this.get(fromBlock.toBuffer()) - ]; - - const fromBlockNumber = fromBlock.toNumber(); - // if we have a range of blocks to search, do that here: - if (fromBlockNumber !== toBlockNumber) { - // fetch all the blockLogs in-between `fromBlock` and `toBlock` (excluding - // from, because we already started fetching that one) - for (let i = fromBlockNumber + 1, l = toBlockNumber + 1; i < l; i++) { - pendingLogsPromises.push(this.get(Quantity.toBuffer(i))); - } - } + const fork = this.#blockchain.fallback; + if (!fork) { + return await this.getLocal( + fromBlock.toNumber(), + toBlock.toNumber(), + filter + ); + } + const from = Quantity.min(fromBlock, toBlock); + const ret: Ethereum.Logs = []; + if (fork.isValidForkBlockNumber(from)) { + ret.push( + ...(await this.getFromFork( + from, + Quantity.min(toBlock, fork.blockNumber), + filter + )) + ); + } + if (!fork.isValidForkBlockNumber(toBlock)) { + ret.push( + ...(await this.getLocal( + Math.max(from.toNumber(), fork.blockNumber.toNumber() + 1), + toBlock.toNumber(), + filter + )) + ); + } + return ret; + } - // now filter and compute all the blocks' blockLogs (in block order) - return Promise.all(pendingLogsPromises).then(blockLogsRange => { - const filteredBlockLogs: Ethereum.Logs = []; - blockLogsRange.forEach(blockLogs => { - // TODO(perf): this loops over all addresses for every block. - // Maybe make it loop only once? - // Issue: https://github.com/trufflesuite/ganache/issues/3482 - if (blockLogs) - filteredBlockLogs.push(...blockLogs.filter(addresses, topics)); - }); - return filteredBlockLogs; + getLocal( + from: number, + to: number, + filter: FilterArgs + ): Promise { + const { addresses, topics } = parseFilterDetails(filter); + const pendingLogsPromises: Promise[] = []; + for (let i = from; i <= to; i++) { + pendingLogsPromises.push(this.get(Quantity.toBuffer(i))); + } + return Promise.all(pendingLogsPromises).then(blockLogsRange => { + const filteredBlockLogs: Ethereum.Logs = []; + blockLogsRange.forEach(blockLogs => { + // TODO(perf): this loops over all addresses for every block. + // Maybe make it loop only once? + // Issue: https://github.com/trufflesuite/ganache/issues/3482 + if (blockLogs) + filteredBlockLogs.push(...blockLogs.filter(addresses, topics)); }); + return filteredBlockLogs; + }); + } + + async getFromFork( + from: Quantity, + to: Quantity, + filter: FilterArgs + ): Promise { + const { topics } = parseFilterDetails(filter); + const f = this.#blockchain.fallback; + if (!f || !f.isValidForkBlockNumber(from)) { + return []; } + return await f.request("eth_getLogs", [ + { + fromBlock: from, + toBlock: f.selectValidForkBlockNumber(to), + address: filter.address + ? Array.isArray(filter.address) + ? filter.address + : [filter.address] + : [], + topics + } + ]); } } diff --git a/src/chains/ethereum/ethereum/tests/forking/contracts/IERC20.sol b/src/chains/ethereum/ethereum/tests/forking/contracts/IERC20.sol new file mode 100644 index 0000000000..a1b17db89f --- /dev/null +++ b/src/chains/ethereum/ethereum/tests/forking/contracts/IERC20.sol @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.11; + + +/** + * @dev Interface of the ERC20 standard as defined in the EIP. + */ +interface IERC20 { + /** + * @dev Emitted when `value` tokens are moved from one account (`from`) to + * another (`to`). + * + * Note that `value` may be zero. + */ + event Transfer(address indexed from, address indexed to, uint256 value); + + /** + * @dev Emitted when the allowance of a `spender` for an `owner` is set by + * a call to {approve}. `value` is the new allowance. + */ + event Approval(address indexed owner, address indexed spender, uint256 value); + + /** + * @dev Returns the amount of tokens in existence. + */ + function totalSupply() external view returns (uint256); + + /** + * @dev Returns the amount of tokens owned by `account`. + */ + function balanceOf(address account) external view returns (uint256); + + /** + * @dev Moves `amount` tokens from the caller's account to `to`. + * + * Returns a boolean value indicating whether the operation succeeded. + * + * Emits a {Transfer} event. + */ + function transfer(address to, uint256 amount) external returns (bool); + + /** + * @dev Returns the remaining number of tokens that `spender` will be + * allowed to spend on behalf of `owner` through {transferFrom}. This is + * zero by default. + * + * This value changes when {approve} or {transferFrom} are called. + */ + function allowance(address owner, address spender) external view returns (uint256); + + /** + * @dev Sets `amount` as the allowance of `spender` over the caller's tokens. + * + * Returns a boolean value indicating whether the operation succeeded. + * + * IMPORTANT: Beware that changing an allowance with this method brings the risk + * that someone may use both the old and the new allowance by unfortunate + * transaction ordering. One possible solution to mitigate this race + * condition is to first reduce the spender's allowance to 0 and set the + * desired value afterwards: + * https://github.com/ethereum/EIPs/issues/20#issuecomment-263524729 + * + * Emits an {Approval} event. + */ + function approve(address spender, uint256 amount) external returns (bool); + + /** + * @dev Moves `amount` tokens from `from` to `to` using the + * allowance mechanism. `amount` is then deducted from the caller's + * allowance. + * + * Returns a boolean value indicating whether the operation succeeded. + * + * Emits a {Transfer} event. + */ + function transferFrom(address from, address to, uint256 amount) external returns (bool); +} diff --git a/src/chains/ethereum/ethereum/tests/forking/logs.test.ts b/src/chains/ethereum/ethereum/tests/forking/logs.test.ts new file mode 100644 index 0000000000..278452e304 --- /dev/null +++ b/src/chains/ethereum/ethereum/tests/forking/logs.test.ts @@ -0,0 +1,219 @@ +import getProvider from "../helpers/getProvider"; +import assert from "assert"; +import { EthereumProvider } from "../../src/provider"; +import compile from "../helpers/compile"; +import { join } from "path"; + +describe("forking", function () { + this.timeout(100000); + + describe("logs", () => { + const blockNumber = 0xb77935; // 12024117 + const URL = "https://mainnet.infura.io/v3/" + process.env.INFURA_KEY; + let provider: EthereumProvider; + let contract: ReturnType; + let contractAddress: string; + let accounts: string[]; + let someUserAddress: string; + let receipts = []; + + before(async function () { + if (!process.env.INFURA_KEY) { + this.skip(); + } + + // USDT, has a lot of transfer events before block 12024117 == 0xb77935 + someUserAddress = "0xcdef4f34e5ceb46c7c55134cda34273349be65b7"; + contractAddress = "0xdAC17F958D2ee523a2206206994597C13D831ec7"; + provider = await getProvider({ + wallet: { unlockedAccounts: [someUserAddress] }, + fork: { + url: URL, + blockNumber, + disableCache: true + } + }); + contract = compile(join(__dirname, "./contracts/IERC20.sol")); + accounts = await provider.send("eth_accounts"); + for (let i = 0; i < accounts.length; i++) { + const data = + "0x" + + contract.contract.evm.methodIdentifiers["transfer(address,uint256)"] + + accounts[0].slice(2).padStart(64, "0") + + i.toString().padStart(64, "0"); + const txHash = await provider.send("eth_sendTransaction", [ + { + from: someUserAddress, + to: contractAddress, + gas: "0x2fefd8", + data: data + } + ]); + const txReceipt = await provider.send("eth_getTransactionReceipt", [ + txHash + ]); + receipts.push(txReceipt); + } + }); + + describe("getLogs", () => { + async function testReject( + fromBlock: number | string, + toBlock: number | string, + expected: string, + address: string = contractAddress + ) { + if (typeof fromBlock == "number") { + fromBlock = `0x${fromBlock.toString(16)}`; + } + if (typeof toBlock === "number") { + toBlock = `0x${toBlock.toString(16)}`; + } + await assert.rejects( + provider.send("eth_getLogs", [ + { + address, + fromBlock, + toBlock + } + ]), + new Error(expected) + ); + } + + async function testGetLogs( + fromBlock: number | string, + toBlock: number | string, + expected: number, + address: string = contractAddress + ) { + if (typeof fromBlock == "number") { + fromBlock = `0x${fromBlock.toString(16)}`; + } + if (typeof toBlock === "number") { + toBlock = `0x${toBlock.toString(16)}`; + } + const logs = await provider.send("eth_getLogs", [ + { + address, + fromBlock, + toBlock + } + ]); + assert.strictEqual( + logs.length, + expected, + `there should be ${logs.length} log(s) between the ${fromBlock} block and the ${toBlock} block` + ); + } + + it("should return the last tx log", async () => { + const logs = await provider.send("eth_getLogs", [ + { address: contractAddress } + ]); + assert.strictEqual(logs.length, 1); + }); + + it("should filter out other blocks when using `latest`", async () => { + const logs = await provider.send("eth_getLogs", [ + { address: contractAddress, toBlock: "latest", fromBlock: "latest" } + ]); + assert.strictEqual(logs.length, 1); + }); + + it("should filter appropriately when using fromBlock and toBlock", async () => { + await testGetLogs(blockNumber - 2, blockNumber - 2, 13); + await testGetLogs(blockNumber - 1, blockNumber - 1, 27); + await testGetLogs(blockNumber, blockNumber, 84); + await testGetLogs(blockNumber + 1, blockNumber + 1, 0); // blockNumber + 1 is an empty block => no logs + await testGetLogs(blockNumber + 2, blockNumber + 2, 1); + + // tests ranges + await testGetLogs(blockNumber - 2, blockNumber - 1, 13 + 27); + await testGetLogs(blockNumber - 2, blockNumber, 13 + 27 + 84); + await testGetLogs(blockNumber - 2, blockNumber + 1, 13 + 27 + 84); + await testGetLogs(blockNumber - 2, blockNumber + 2, 13 + 27 + 84 + 1); + + await testGetLogs(blockNumber, blockNumber + 1, 84); + await testGetLogs(blockNumber, blockNumber + 2, 84 + 1); + + await testGetLogs(blockNumber + 1, blockNumber + 2, 1); + + await testGetLogs(blockNumber + 2, blockNumber + 3, 2); + await testGetLogs( + blockNumber + 1, + blockNumber + 1 + accounts.length, + accounts.length + ); + }); + + it("should revert when using fromBlock and toBlock with strange values", async () => { + const fromBlock = `0x${blockNumber.toString(16)}`; + const toBlock = `0x${(blockNumber - 1).toString(16)}`; + await testReject( + blockNumber, + blockNumber - 1, + "One of the blocks specified in filter (fromBlock, toBlock or blockHash) cannot be found." + ); + await testReject( + "latest", + blockNumber, + "One of the blocks specified in filter (fromBlock, toBlock or blockHash) cannot be found." + ); + await testReject( + "latest", + "earliest", + "One of the blocks specified in filter (fromBlock, toBlock or blockHash) cannot be found." + ); + // This really depends on the forked network + await testReject("earliest", "latest", "socket hang up"); + }); + + it("should filter appropriately when using fromBlock and toBlock with tags", async () => { + // block 0 + await testGetLogs("earliest", "earliest", 0); + + await testGetLogs(blockNumber - 1, "latest", 27 + 84 + accounts.length); + await testGetLogs(blockNumber, "latest", 84 + accounts.length); + await testGetLogs(blockNumber + 1, "latest", accounts.length); + await testGetLogs(blockNumber + 2, "latest", accounts.length); + await testGetLogs(blockNumber + 3, "latest", accounts.length - 1); + }); + + it("should filter appropriately when using blockHash", async () => { + async function testGetLogs( + blockNum: number, + expected: number, + address: string = contractAddress + ) { + const block = await provider.request({ + method: "eth_getBlockByNumber", + params: [`0x${blockNum.toString(16)}`] + }); + const logs = await provider.send("eth_getLogs", [ + { address, blockHash: block.hash } + ]); + assert.strictEqual( + logs.length, + expected, + `there should be ${expected} log(s) at the ${block.hash} block` + ); + } + + await testGetLogs(blockNumber - 1, 27); + await testGetLogs(blockNumber, 84); + await testGetLogs(blockNumber + 1, 0); + await testGetLogs(blockNumber + 2, 1); + + const logs = await provider.send("eth_getLogs", [ + { + address: contractAddress, + blockHash: + "0xdeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddeaddead" + } + ]); + assert.strictEqual(logs.length, 0, `there should be 0 log(s)`); + }); + }); + }); +}); diff --git a/src/packages/utils/src/things/json-rpc/json-rpc-quantity.ts b/src/packages/utils/src/things/json-rpc/json-rpc-quantity.ts index 0051641e32..105b81d5ac 100644 --- a/src/packages/utils/src/things/json-rpc/json-rpc-quantity.ts +++ b/src/packages/utils/src/things/json-rpc/json-rpc-quantity.ts @@ -18,6 +18,14 @@ export class Quantity extends BaseJsonRpcType { return new Quantity(value, nullable); } + public static min(a: Quantity, b: Quantity) { + return a.toBigInt() < b.toBigInt() ? a : b; + } + + public static max(a: Quantity, b: Quantity) { + return a.toBigInt() < b.toBigInt() ? b : a; + } + constructor(value: JsonRpcInputArg, nullable?: boolean) { super(value); if (value === "0x") {