refactor: Optimize accounts/{accountId}/staking-payouts blocking query complexity

package.json

@@ -38,7 +38,7 @@
   "dependencies": {
     "@polkadot/api": "^3.0.1",
     "@polkadot/apps-config": "^0.71.2",
-    "@polkadot/util-crypto": "^5.0.1",
+    "@polkadot/util-crypto": "^5.1.1",
     "@substrate/calc": "^0.1.3",
     "confmgr": "^1.0.6",
     "express": "^4.17.1",
@@ -55,8 +55,8 @@
     "@types/triple-beam": "^1.3.2",
     "@typescript-eslint/eslint-plugin": "4.10.0",
     "@typescript-eslint/parser": "4.10.0",
-    "eslint": "^7.15.0",
-    "eslint-config-prettier": "^7.0.0",
+    "eslint": "^7.16.0",
+    "eslint-config-prettier": "^7.1.0",
     "eslint-plugin-prettier": "^3.3.0",
     "eslint-plugin-simple-import-sort": "^7.0.0",
     "jest": "^26.6.3",

src/services/accounts/AccountsStakingPayoutsService.ts

@@ -1,7 +1,13 @@
 import { ApiPromise } from '@polkadot/api';
-import { DeriveEraExposure } from '@polkadot/api-derive/staking/types';
-import { BlockHash } from '@polkadot/types/interfaces';
 import {
+	DeriveEraExposure,
+	DeriveEraExposureNominating,
+} from '@polkadot/api-derive/staking/types';
+import { Option } from '@polkadot/types';
+import {
+	BalanceOf,
+	BlockHash,
+	EraIndex,
 	EraRewardPoints,
 	Perbill,
 	StakingLedger,
@@ -16,6 +22,33 @@ import {
 } from '../../types/responses';
 import { AbstractService } from '../AbstractService';
 
+/**
+ * General information about an era, in tuple form because we initially get it
+ * by destructuring a Promise.all(...)
+ */
+type IErasGeneral = [DeriveEraExposure, EraRewardPoints, Option<BalanceOf>];
+
+/**
+ * Commission and staking ledger of a validator
+ */
+interface ICommissionAndLedger {
+	commission: Perbill;
+	validatorLedger?: StakingLedger;
+}
+
+/**
+ * All the data we need to calculate payouts for an address at a given era.
+ */
+interface IEraData {
+	deriveEraExposure: DeriveEraExposure;
+	eraRewardPoints: EraRewardPoints;
+	erasValidatorRewardOption: Option<BalanceOf>;
+	exposuresWithCommission?: (ICommissionAndLedger & {
+		validatorId: string;
+	})[];
+	eraIndex: EraIndex;
+}
+
 export class AccountsStakingPayoutsService extends AbstractService {
 	/**
 	 * Fetch and derive payouts for `address`.
@@ -55,105 +88,202 @@ export class AccountsStakingPayoutsService extends AbstractService {
 					'than or equal to current_era - history_depth.'
 			);
 		}
+
 		const at = {
 			height: number.unwrap().toString(10),
 			hash,
 		};
 
-		const erasPayouts: (IEraPayouts | { message: string })[] = [];
-
 		// User friendly - we don't error if the user specified era & depth combo <= 0, instead just start at 0
 		const startEra = era - (depth - 1) < 0 ? 0 : era - (depth - 1);
 
-		for (let e = startEra; e <= era; e += 1) {
-			erasPayouts.push(
-				await this.deriveEraPayouts(
-					api,
-					hash,
+		// Fetch general data about the era
+		const allErasGeneral = await this.fetchAllErasGeneral(
+			api,
+			hash,
+			startEra,
+			era
+		);
+
+		// With the general data, we can now fetch the commission of each validator `address` nominates
+		const allErasCommissions = await this.fetchAllErasCommissions(
+			api,
+			hash,
+			address,
+			startEra,
+			allErasGeneral.map((el) => el[0])
+		);
+
+		// Group together data by Era so we can easily associate parts that are used congruently downstream
+		const allEraData = allErasGeneral.map(
+			(
+				[
+					deriveEraExposure,
+					eraRewardPoints,
+					erasValidatorRewardOption,
+				]: IErasGeneral,
+				idx: number
+			): IEraData => {
+				const eraCommissions = allErasCommissions[idx];
+
+				const nominatedExposures = this.deriveNominatedExposures(
 					address,
-					e,
-					unclaimedOnly
-				)
-			);
-		}
+					deriveEraExposure
+				);
+
+				// Zip the `validatorId` with its associated `commission`, making the data easier to reason
+				// about downstream
+				const exposuresWithCommission = nominatedExposures?.map(
+					({ validatorId }, idx) => {
+						return {
+							validatorId,
+							...eraCommissions[idx],
+						};
+					}
+				);
+
+				return {
+					deriveEraExposure,
+					eraRewardPoints,
+					erasValidatorRewardOption,
+					exposuresWithCommission,
+					eraIndex: api.createType('EraIndex', idx + startEra),
+				};
+			}
+		);
 
 		return {
 			at,
-			erasPayouts,
+			erasPayouts: allEraData.map((eraData) =>
+				this.deriveEraPayouts(address, unclaimedOnly, eraData)
+			),
 		};
 	}
 
 	/**
-	 * Derive all the payouts for `address` at `era`.
+	 * Fetch general info about eras in the inclusive range `startEra` .. `era`.
 	 *
-	 * @param api
+	 * @param api `ApiPromise`
+	 * @param hash `BlockHash` to make call at
+	 * @param startEra first era to get data for
+	 * @param era the last era to get data for
+	 */
+	async fetchAllErasGeneral(
+		api: ApiPromise,
+		hash: BlockHash,
+		startEra: number,
+		era: number
+	): Promise<IErasGeneral[]> {
+		const allDeriveQuerys: Promise<IErasGeneral>[] = [];
+		for (let e = startEra; e <= era; e += 1) {
+			const eraIndex = api.createType('EraIndex', e);
+
+			const eraGeneralTuple = Promise.all([
+				api.derive.staking.eraExposure(eraIndex),
+				api.query.staking.erasRewardPoints.at(hash, eraIndex),
+				api.query.staking.erasValidatorReward.at(hash, eraIndex),
+			]);
+
+			allDeriveQuerys.push(eraGeneralTuple);
+		}
+
+		return Promise.all(allDeriveQuerys);
+	}
+
+	/**
+	 * Fetch the commission & staking ledger for each `validatorId` in `deriveErasExposures`.
+	 *
+	 * @param api `ApiPromise`
 	 * @param hash `BlockHash` to make call at
 	 * @param address address of the _Stash_  account to get the payouts of
-	 * @param era the era to query
-	 * @param unclaimedOnly whether or not to only show unclaimed payouts
+	 * @param startEra first era to get data for
+	 * @param deriveErasExposures exposures per era for `address`
 	 */
-	async deriveEraPayouts(
+	fetchAllErasCommissions(
 		api: ApiPromise,
 		hash: BlockHash,
 		address: string,
-		era: number,
-		unclaimedOnly: boolean
-	): Promise<IEraPayouts | { message: string }> {
-		const eraIndex = api.createType('EraIndex', era);
+		startEra: number,
+		deriveErasExposures: DeriveEraExposure[]
+	): Promise<ICommissionAndLedger[][]> {
+		// Cache StakingLedger to reduce redundant queries to node
+		const validatorLedgerCache: { [id: string]: StakingLedger } = {};
 
-		const [
+		const allErasCommissions = deriveErasExposures.map(
+			(deriveEraExposure, idx) => {
+				const currEra = idx + startEra;
+
+				const nominatedExposures = this.deriveNominatedExposures(
+					address,
+					deriveEraExposure
+				);
+
+				if (!nominatedExposures) {
+					return [];
+				}
+
+				const singleEraCommissions = nominatedExposures.map(
+					({ validatorId }) =>
+						this.fetchCommissionAndLedger(
+							api,
+							validatorId,
+							currEra,
+							hash,
+							validatorLedgerCache
+						)
+				);
+
+				return Promise.all(singleEraCommissions);
+			}
+		);
+
+		return Promise.all(allErasCommissions);
+	}
+
+	/**
+	 * Derive all the payouts for `address` at `era`.
+	 *
+	 * @param address address of the _Stash_  account to get the payouts of
+	 * @param era the era to query
+	 * @param eraData data about the address and era we are calculating payouts for
+	 */
+	deriveEraPayouts(
+		address: string,
+		unclaimedOnly: boolean,
+		{
 			deriveEraExposure,
 			eraRewardPoints,
 			erasValidatorRewardOption,
-		] = await Promise.all([
-			api.derive.staking.eraExposure(eraIndex),
-			api.query.staking.erasRewardPoints.at(hash, era),
-			api.query.staking.erasValidatorReward.at(hash, era),
-		]);
-
-		let nominatedExposures = deriveEraExposure.nominators[address];
-		if (deriveEraExposure.validators[address]) {
-			// We treat an `address` that is a validator as nominating itself
-			nominatedExposures = nominatedExposures
-				? nominatedExposures.concat({
-						validatorId: address,
-						validatorIndex: 0,
-				  })
-				: [
-						{
-							validatorId: address,
-							validatorIndex: 0,
-						},
-				  ];
-		}
-
-		if (!nominatedExposures) {
+			exposuresWithCommission,
+			eraIndex,
+		}: IEraData
+	): IEraPayouts | { message: string } {
+		if (!exposuresWithCommission) {
 			return {
-				message: `${address} has no nominations for the era ${era}`,
+				message: `${address} has no nominations for the era ${eraIndex.toString()}`,
 			};
 		}
 
 		if (erasValidatorRewardOption.isNone) {
 			return {
-				message: `No ErasValidatorReward for the era ${era}`,
+				message: `No ErasValidatorReward for the era ${eraIndex.toString()}`,
 			};
 		}
 
-		// Cache StakingLedger to reduce redundant queries to node
-		const validatorLedgerCache: { [id: string]: StakingLedger } = {};
-
-		// Payouts for the era
-		const payouts: IPayout[] = [];
-
 		const totalEraRewardPoints = eraRewardPoints.total;
 		const totalEraPayout = erasValidatorRewardOption.unwrap();
 		const calcPayout = CalcPayout.from_params(
 			totalEraRewardPoints.toNumber(),
 			totalEraPayout.toString(10)
 		);
 
-		// Loop through validators that this nominator backs
-		for (const { validatorId } of nominatedExposures) {
+		// Iterate through validators that this nominator backs and calculate payouts for the era
+		const payouts: IPayout[] = [];
+		for (const {
+			validatorId,
+			commission: validatorCommission,
+			validatorLedger,
+		} of exposuresWithCommission) {
 			const totalValidatorRewardPoints = this.extractTotalValidatorRewardPoints(
 				eraRewardPoints,
 				validatorId
@@ -178,21 +308,9 @@ export class AccountsStakingPayoutsService extends AbstractService {
 				continue;
 			}
 
-			const {
-				commission: validatorCommission,
-				validatorLedger,
-			} = await this.fetchCommissionAndLedger(
-				api,
-				validatorId,
-				era,
-				hash,
-				validatorLedgerCache
-			);
-
 			if (!validatorLedger) {
 				continue;
 			}
-
 			// Check if the reward has already been claimed
 			const claimed = validatorLedger.claimedRewards.includes(eraIndex);
 			if (unclaimedOnly && claimed) {
@@ -241,10 +359,7 @@ export class AccountsStakingPayoutsService extends AbstractService {
 		era: number,
 		hash: BlockHash,
 		validatorLedgerCache: { [id: string]: StakingLedger }
-	): Promise<{
-		commission: Perbill;
-		validatorLedger?: StakingLedger;
-	}> {
+	): Promise<ICommissionAndLedger> {
 		let commission;
 		let validatorLedger;
 		if (validatorId in validatorLedgerCache) {
@@ -339,4 +454,35 @@ export class AccountsStakingPayoutsService extends AbstractService {
 			nominatorExposure,
 		};
 	}
+
+	/**
+	 * Derive the list of validators `address` nominates. Note: we count validators as nominating
+	 * themself.
+	 *
+	 * @param address address of the _Stash_  account to get the payouts of
+	 * @param deriveEraExposure result of query api.derive.staking.eraExposure(eraIndex)
+	 */
+	deriveNominatedExposures(
+		address: string,
+		deriveEraExposure: DeriveEraExposure
+	): DeriveEraExposureNominating[] | undefined {
+		let nominatedExposures: DeriveEraExposureNominating[] | undefined =
+			deriveEraExposure.nominators[address];
+		if (deriveEraExposure.validators[address]) {
+			// We treat an `address` that is a validator as nominating itself
+			nominatedExposures = nominatedExposures
+				? nominatedExposures.concat({
+						validatorId: address,
+						validatorIndex: 0,
+				  })
+				: [
+						{
+							validatorId: address,
+							validatorIndex: 0,
+						},
+				  ];
+		}
+
+		return nominatedExposures;
+	}
 }