From 03f113732d61925ca3abeb41a098b6235b82d65f Mon Sep 17 00:00:00 2001 From: Shunji Zhan Date: Tue, 16 Apr 2024 15:16:09 +0800 Subject: [PATCH 1/4] get logs from cache --- packages/eth-providers/src/base-provider.ts | 41 +++++++++++++++------ 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/packages/eth-providers/src/base-provider.ts b/packages/eth-providers/src/base-provider.ts index 16d5d4cc..de6f5961 100644 --- a/packages/eth-providers/src/base-provider.ts +++ b/packages/eth-providers/src/base-provider.ts @@ -1795,17 +1795,26 @@ export abstract class BaseProvider extends AbstractProvider { _getSubqlMissedLogs = async (toBlock: number, filter: SanitizedLogFilter): Promise => { const targetBlock = Math.min(toBlock, await this.finalizedBlockNumber); // subql upperbound is finalizedBlockNumber const lastProcessedHeight = await this.subql.getLastProcessedHeight(); - const missedBlockCount = targetBlock - lastProcessedHeight; - if (missedBlockCount <= 0) return []; - const firstMissedHeight = lastProcessedHeight + 1; - const missedHeights = Array.from( - { length: missedBlockCount }, - (_, i) => firstMissedHeight + i, + + return this._getLogsFromCache(firstMissedHeight, targetBlock, filter); + }; + + _getLogsFromCache = async ( + fromBlock: number, + toBlock: number, + filter: SanitizedLogFilter, + ): Promise => { + const blockCount = toBlock - fromBlock + 1; // when to === from, it should return 1 block + if (blockCount <= 0) return []; + + const heights = Array.from( + { length: blockCount }, + (_, i) => fromBlock + i, ); - const missedBlockHashes = await Promise.all(missedHeights.map(this._getBlockHash.bind(this))); + const blockHashes = await Promise.all(heights.map(this._getBlockHash.bind(this))); - return missedBlockHashes + return blockHashes .map(this.blockCache.getLogsAtBlock.bind(this)) .flat() .filter(log => filterLogByBlockNumber(log, filter.fromBlock, filter.toBlock)) @@ -1814,13 +1823,23 @@ export abstract class BaseProvider extends AbstractProvider { // Bloom-filter Queries getLogs = async (rawFilter: LogFilter): Promise => { + const filter = await this._sanitizeRawFilter(rawFilter); + if (!this.subql) { - return logger.throwError( + const _throwErr = () => logger.throwError( 'missing subql url to fetch logs, to initialize base provider with subql, please provide a subqlUrl param.' ); - } - const filter = await this._sanitizeRawFilter(rawFilter); + const earliestCachedBlockHash = this.blockCache.cachedBlockHashes[0]; + if (!earliestCachedBlockHash) return _throwErr(); + + const earliestCachedBlockNumber = await this._getBlockNumber(earliestCachedBlockHash); + const isAllLogsIncache = earliestCachedBlockNumber <= filter.fromBlock; + + return isAllLogsIncache + ? this._getLogsFromCache(filter.fromBlock, filter.toBlock, filter) + : _throwErr(); + } // only filter by blockNumber and address, since topics are filtered at last const [subqlLogs, extraLogs] = await Promise.all([ From 3bb5280cc65d05581e01442b7e55a88178ebb624 Mon Sep 17 00:00:00 2001 From: Shunji Zhan Date: Fri, 19 Apr 2024 15:26:05 +0800 Subject: [PATCH 2/4] add cache --- .../__tests__/e2e/json-rpc-provider.test.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/packages/eth-providers/src/__tests__/e2e/json-rpc-provider.test.ts b/packages/eth-providers/src/__tests__/e2e/json-rpc-provider.test.ts index 69898f02..2517b74f 100644 --- a/packages/eth-providers/src/__tests__/e2e/json-rpc-provider.test.ts +++ b/packages/eth-providers/src/__tests__/e2e/json-rpc-provider.test.ts @@ -174,6 +174,27 @@ describe('JsonRpcProvider', async () => { }); }); + describe('get logs without subql', () => { + it('works', async () => { + const echoFactory = new ContractFactory(echoJson.abi, echoJson.bytecode, wallet); + const echo = await echoFactory.deploy(); + + const { blockNumber: block0 } = await (await echo.scream('hello Gogeta!')).wait(); + let logs = await wallet.provider.getLogs({ + address: echo.address, + }); + expect(logs.length).to.eq(1); + + const { blockNumber: block1 } = await (await echo.scream('hello Vegito!')).wait(); + logs = await wallet.provider.getLogs({ + address: echo.address, + fromBlock: block0, + toBlock: block1, + }); + expect(logs.length).to.eq(2); + }); + }); + describe('subscription', () => { it('subscribe to new block', async () => { const curBlockNumber = await provider.getBlockNumber(); From 309677ef50d8f71ef12f9088d638754e51916870 Mon Sep 17 00:00:00 2001 From: Shunji Zhan Date: Fri, 19 Apr 2024 15:45:57 +0800 Subject: [PATCH 3/4] polish logging --- packages/eth-providers/src/base-provider.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/eth-providers/src/base-provider.ts b/packages/eth-providers/src/base-provider.ts index de6f5961..0014701a 100644 --- a/packages/eth-providers/src/base-provider.ts +++ b/packages/eth-providers/src/base-provider.ts @@ -1826,8 +1826,13 @@ export abstract class BaseProvider extends AbstractProvider { const filter = await this._sanitizeRawFilter(rawFilter); if (!this.subql) { - const _throwErr = () => logger.throwError( - 'missing subql url to fetch logs, to initialize base provider with subql, please provide a subqlUrl param.' + const _throwErr = (earliestCachedBlockNumber?: number) => logger.throwError( + 'cache does not contain enough info to fetch requested logs, please reduce block range or initialize provider with a subql url', + Logger.errors.SERVER_ERROR, + { + requestFromBlock: filter.fromBlock, + earliestCachedBlockNumber, + }, ); const earliestCachedBlockHash = this.blockCache.cachedBlockHashes[0]; @@ -1838,7 +1843,7 @@ export abstract class BaseProvider extends AbstractProvider { return isAllLogsIncache ? this._getLogsFromCache(filter.fromBlock, filter.toBlock, filter) - : _throwErr(); + : _throwErr(earliestCachedBlockNumber); } // only filter by blockNumber and address, since topics are filtered at last From 91467b943bce72697542c05c62ed23a190a5d5ef Mon Sep 17 00:00:00 2001 From: Shunji Zhan Date: Fri, 19 Apr 2024 15:52:33 +0800 Subject: [PATCH 4/4] fix naming --- packages/eth-providers/src/base-provider.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/eth-providers/src/base-provider.ts b/packages/eth-providers/src/base-provider.ts index 0014701a..11784d5e 100644 --- a/packages/eth-providers/src/base-provider.ts +++ b/packages/eth-providers/src/base-provider.ts @@ -1839,9 +1839,9 @@ export abstract class BaseProvider extends AbstractProvider { if (!earliestCachedBlockHash) return _throwErr(); const earliestCachedBlockNumber = await this._getBlockNumber(earliestCachedBlockHash); - const isAllLogsIncache = earliestCachedBlockNumber <= filter.fromBlock; + const isAllTargetLogsInCache = earliestCachedBlockNumber <= filter.fromBlock; - return isAllLogsIncache + return isAllTargetLogsInCache ? this._getLogsFromCache(filter.fromBlock, filter.toBlock, filter) : _throwErr(earliestCachedBlockNumber); }