From 0980d1e0766f1e416883c8e68ebcf04ca8ecd2fc Mon Sep 17 00:00:00 2001 From: Tarik Gul <47201679+TarikGul@users.noreply.github.com> Date: Thu, 21 Dec 2023 13:40:48 -0500 Subject: [PATCH] perf: add concurrency to fee calls (#1368) * perf: add concurrency to fee calls * cleanup nits * fix comment --- src/services/blocks/BlocksService.ts | 295 +++++++++++++++------------ 1 file changed, 160 insertions(+), 135 deletions(-) diff --git a/src/services/blocks/BlocksService.ts b/src/services/blocks/BlocksService.ts index e06b23314..25703acfa 100644 --- a/src/services/blocks/BlocksService.ts +++ b/src/services/blocks/BlocksService.ts @@ -19,7 +19,7 @@ import { ApiPromise } from '@polkadot/api'; import { ApiDecoration } from '@polkadot/api/types'; import { extractAuthor } from '@polkadot/api-derive/type/util'; -import { Compact, GenericCall, Option, Struct, Vec } from '@polkadot/types'; +import { Compact, GenericCall, Option, Struct, Text, u32, Vec } from '@polkadot/types'; import { GenericExtrinsic } from '@polkadot/types/extrinsic'; import { AccountId32, @@ -55,6 +55,7 @@ import { } from '../../types/responses'; import { IOption } from '../../types/util'; import { isPaysFee } from '../../types/util'; +import { PromiseQueue } from '../../util/PromiseQueue'; import { AbstractService } from '../AbstractService'; /** @@ -172,171 +173,195 @@ export class BlocksService extends AbstractService { } const previousBlockHash = await this.fetchPreviousBlockHash(number); - + /** + * Fee calculation logic. This runs the extrinsics concurrently. + */ + const pQueue = new PromiseQueue(4); + const feeTasks = []; for (let idx = 0; idx < block.extrinsics.length; ++idx) { - if (noFees) { - extrinsics[idx].info = {}; - continue; - } + feeTasks.push( + pQueue.run(async () => { + await this.resolveExtFees(extrinsics, block, idx, noFees, previousBlockHash, specVersion, specName); + }), + ); + } - if (!extrinsics[idx].paysFee || !block.extrinsics[idx].isSigned) { - continue; - } + await Promise.all(feeTasks); - if (this.minCalcFeeRuntime === null) { - extrinsics[idx].info = { - error: `Fee calculation not supported for this network`, - }; - continue; - } + const response = { + number, + hash, + parentHash, + stateRoot, + extrinsicsRoot, + authorId, + logs, + onInitialize, + extrinsics, + onFinalize, + finalized, + }; - if (this.minCalcFeeRuntime > specVersion.toNumber()) { - extrinsics[idx].info = { - error: `Fee calculation not supported for ${specVersion.toString()}#${specName.toString()}`, - }; - continue; - } + // Store the block in the cache + this.blockStore.set(hash.toString(), response); - const xtEvents = extrinsics[idx].events; - const completedEvent = xtEvents.find( - ({ method }) => isFrameMethod(method) && (method.method === Event.success || method.method === Event.failure), - ); + return response; + } - if (!completedEvent) { - extrinsics[idx].info = { - error: 'Unable to find success or failure event for extrinsic', - }; + private async resolveExtFees( + extrinsics: IExtrinsic[], + block: Block, + idx: number, + noFees: boolean, + previousBlockHash: BlockHash, + specVersion: u32, + specName: Text, + ) { + const { api } = this; - continue; - } + if (noFees) { + extrinsics[idx].info = {}; + return; + } - const completedData = completedEvent.data; - if (!completedData) { - extrinsics[idx].info = { - error: 'Success or failure event for extrinsic does not contain expected data', - }; + if (!extrinsics[idx].paysFee || !block.extrinsics[idx].isSigned) { + return; + } - continue; - } + if (this.minCalcFeeRuntime === null) { + extrinsics[idx].info = { + error: `Fee calculation not supported for this network`, + }; + return; + } - // Both ExtrinsicSuccess and ExtrinsicFailed events have DispatchInfo - // types as their final arg - const weightInfo = completedData[completedData.length - 1] as DispatchInfo; - if (!weightInfo.weight) { - extrinsics[idx].info = { - error: 'Success or failure event for extrinsic does not specify weight', - }; + if (this.minCalcFeeRuntime > specVersion.toNumber()) { + extrinsics[idx].info = { + error: `Fee calculation not supported for ${specVersion.toString()}#${specName.toString()}`, + }; + return; + } - continue; - } + const xtEvents = extrinsics[idx].events; + const completedEvent = xtEvents.find( + ({ method }) => isFrameMethod(method) && (method.method === Event.success || method.method === Event.failure), + ); - if (!api.rpc.payment?.queryInfo && !api.call.transactionPaymentApi?.queryInfo) { - extrinsics[idx].info = { - error: 'Rpc method payment::queryInfo is not available', - }; + if (!completedEvent) { + extrinsics[idx].info = { + error: 'Unable to find success or failure event for extrinsic', + }; - continue; - } + return; + } - const transactionPaidFeeEvent = xtEvents.find( - ({ method }) => isFrameMethod(method) && method.method === Event.transactionPaidFee, - ); - const extrinsicSuccess = xtEvents.find(({ method }) => isFrameMethod(method) && method.method === Event.success); - const extrinsicFailed = xtEvents.find(({ method }) => isFrameMethod(method) && method.method === Event.failure); - - const eventFailureOrSuccess = extrinsicSuccess || extrinsicFailed; - if (transactionPaidFeeEvent && eventFailureOrSuccess) { - let availableData: ExtrinsicSuccessOrFailedOverride; - if (extrinsicSuccess) { - availableData = eventFailureOrSuccess.data[0] as unknown as ExtrinsicSuccessOrFailedOverride; - } else { - availableData = eventFailureOrSuccess.data[1] as unknown as ExtrinsicSuccessOrFailedOverride; - } + const completedData = completedEvent.data; + if (!completedData) { + extrinsics[idx].info = { + error: 'Success or failure event for extrinsic does not contain expected data', + }; - extrinsics[idx].info = { - weight: availableData.weight, - class: availableData.class, - partialFee: transactionPaidFeeEvent.data[1].toString(), - kind: 'fromEvent', - }; - continue; + return; + } + + // Both ExtrinsicSuccess and ExtrinsicFailed events have DispatchInfo + // types as their final arg + const weightInfo = completedData[completedData.length - 1] as DispatchInfo; + if (!weightInfo.weight) { + extrinsics[idx].info = { + error: 'Success or failure event for extrinsic does not specify weight', + }; + + return; + } + + if (!api.rpc.payment?.queryInfo && !api.call.transactionPaymentApi?.queryInfo) { + extrinsics[idx].info = { + error: 'Rpc method payment::queryInfo is not available', + }; + + return; + } + + const transactionPaidFeeEvent = xtEvents.find( + ({ method }) => isFrameMethod(method) && method.method === Event.transactionPaidFee, + ); + const extrinsicSuccess = xtEvents.find(({ method }) => isFrameMethod(method) && method.method === Event.success); + const extrinsicFailed = xtEvents.find(({ method }) => isFrameMethod(method) && method.method === Event.failure); + + const eventFailureOrSuccess = extrinsicSuccess || extrinsicFailed; + if (transactionPaidFeeEvent && eventFailureOrSuccess) { + let availableData: ExtrinsicSuccessOrFailedOverride; + if (extrinsicSuccess) { + availableData = eventFailureOrSuccess.data[0] as unknown as ExtrinsicSuccessOrFailedOverride; + } else { + availableData = eventFailureOrSuccess.data[1] as unknown as ExtrinsicSuccessOrFailedOverride; } + extrinsics[idx].info = { + weight: availableData.weight, + class: availableData.class, + partialFee: transactionPaidFeeEvent.data[1].toString(), + kind: 'fromEvent', + }; + return; + } + + /** + * Grab the initial partialFee, and information required for calculating a partialFee + * if queryFeeDetails is available in the runtime. + */ + const { + class: dispatchClass, + partialFee, + weight, + } = await this.fetchQueryInfo(block.extrinsics[idx], previousBlockHash); + const versionedWeight = (weight as Weight).refTime ? (weight as Weight).refTime.unwrap() : (weight as WeightV1); + + let finalPartialFee = partialFee.toString(), + dispatchFeeType = 'preDispatch'; + if (transactionPaidFeeEvent) { + finalPartialFee = transactionPaidFeeEvent.data[1].toString(); + dispatchFeeType = 'fromEvent'; + } else { /** - * Grab the initial partialFee, and information required for calculating a partialFee - * if queryFeeDetails is available in the runtime. + * Call queryFeeDetails. It may not be available in the runtime and will + * error automatically when we try to call it. We cache the runtimes it will error so we + * don't try to call it again given a specVersion. */ - const { - class: dispatchClass, - partialFee, - weight, - } = await this.fetchQueryInfo(block.extrinsics[idx], previousBlockHash); - const versionedWeight = (weight as Weight).refTime ? (weight as Weight).refTime.unwrap() : (weight as WeightV1); - - let finalPartialFee = partialFee.toString(), - dispatchFeeType = 'preDispatch'; - if (transactionPaidFeeEvent) { - finalPartialFee = transactionPaidFeeEvent.data[1].toString(); - dispatchFeeType = 'fromEvent'; - } else { - /** - * Call queryFeeDetails. It may not be available in the runtime and will - * error automatically when we try to call it. We cache the runtimes it will error so we - * don't try to call it again given a specVersion. - */ - const doesQueryFeeDetailsExist = this.hasQueryFeeApi.hasQueryFeeDetails(specVersion.toNumber()); - if (doesQueryFeeDetailsExist === 'available') { + const doesQueryFeeDetailsExist = this.hasQueryFeeApi.hasQueryFeeDetails(specVersion.toNumber()); + if (doesQueryFeeDetailsExist === 'available') { + finalPartialFee = await this.fetchQueryFeeDetails( + block.extrinsics[idx], + previousBlockHash, + weightInfo.weight, + versionedWeight.toString(), + ); + + dispatchFeeType = 'postDispatch'; + } else if (doesQueryFeeDetailsExist === 'unknown') { + try { finalPartialFee = await this.fetchQueryFeeDetails( block.extrinsics[idx], previousBlockHash, weightInfo.weight, versionedWeight.toString(), ); - dispatchFeeType = 'postDispatch'; - } else if (doesQueryFeeDetailsExist === 'unknown') { - try { - finalPartialFee = await this.fetchQueryFeeDetails( - block.extrinsics[idx], - previousBlockHash, - weightInfo.weight, - versionedWeight.toString(), - ); - dispatchFeeType = 'postDispatch'; - this.hasQueryFeeApi.setRegisterWithCall(specVersion.toNumber()); - } catch { - this.hasQueryFeeApi.setRegisterWithoutCall(specVersion.toNumber()); - console.warn('The error above is automatically emitted from polkadot-js, and can be ignored.'); - } + this.hasQueryFeeApi.setRegisterWithCall(specVersion.toNumber()); + } catch { + this.hasQueryFeeApi.setRegisterWithoutCall(specVersion.toNumber()); + console.warn('The error above is automatically emitted from polkadot-js, and can be ignored.'); } } - - extrinsics[idx].info = { - weight: weightInfo.weight, - class: dispatchClass, - partialFee: api.registry.createType('Balance', finalPartialFee), - kind: dispatchFeeType, - }; } - const response = { - number, - hash, - parentHash, - stateRoot, - extrinsicsRoot, - authorId, - logs, - onInitialize, - extrinsics, - onFinalize, - finalized, + extrinsics[idx].info = { + weight: weightInfo.weight, + class: dispatchClass, + partialFee: api.registry.createType('Balance', finalPartialFee), + kind: dispatchFeeType, }; - - // Store the block in the cache - this.blockStore.set(hash.toString(), response); - - return response; } /**