feat: add /blocks that enforces range query param. (#954)

* Add range query param type

* correctly parse a range or throw an error

* create new endpoint to query a range of blocks using an async generator

* lint, add `/blocks` route

* fix grumbles

* fix blunder

* sort collection of responses, add comments for range algorithm

* add return types

* cleanup QueueNext type

* port runTasks to AbstractController, and make it generic

* add PromiseQueue

* fix memory allocation, and remove async generator

* add PromiseQueue

* use PromiseQueue

* lint

* remove console

* fix verifyInt

* add /blocks to docs

* change BlockRange to Blocks

* set a max range, extra error handling

* add verifyUInt within util

* remove this.verifyInt and replace it with verifyUInt and verifyNonZeroUInt

* correct error message

* fix small async await grumble

* refactor calling each promise

* allow parseNumberOrThrow to accept zeroes

* fix comment grumble

docs/src/openapi-v1.yaml

@@ -366,6 +366,34 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/AccountValidation'
+  /blocks:
+    get:
+      tags:
+      - blocks
+      summary: Get a range of blocks by their height.
+      description: Given a range query parameter return an array of all the blocks within that range.
+      operationId: getBlock
+      parameters:
+      - name: range
+        in: query
+        description: A range of integers. There is a max limit of 500 blocks per request.
+        required: true
+        example: 0-499
+        schema:
+          type: string
+      responses:
+        "200":
+          description: successful operation
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Blocks'
+        "400":
+          description: invalid Block identifier supplied
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/Error'      
   /blocks/{blockId}:
     get:
       tags:
@@ -1641,6 +1669,10 @@ components:
         Note: Block finalization does not correspond to consensus, i.e. whether
         the block is in the canonical chain. It denotes the finalization of block
         _construction._
+    Blocks:
+      type: array
+      items: 
+        $ref: '#/components/schemas/Block'
     BlockFinalize:
       type: object
       properties:

src/controllers/AbstractController.ts

@@ -27,13 +27,16 @@ import {
 	IAddressParam,
 	INumberParam,
 	IParaIdParam,
+	IRangeQueryParam,
 } from 'src/types/requests';
 
 import { sanitizeNumbers } from '../sanitize';
 import { isBasicLegacyError } from '../types/errors';
 import { ISanitizeOptions } from '../types/sanitize';
+import { verifyNonZeroUInt, verifyUInt } from '../util/integers/verifyInt';
 
 type SidecarRequestHandler =
+	| RequestHandler<unknown, unknown, unknown, IRangeQueryParam>
 	| RequestHandler<IAddressParam>
 	| RequestHandler<IAddressNumberParams>
 	| RequestHandler<INumberParam>
@@ -180,13 +183,52 @@ export default abstract class AbstractController<T extends AbstractService> {
 	protected parseNumberOrThrow(n: string, errorMessage: string): number {
 		const num = Number(n);
 
-		if (!Number.isInteger(num) || num < 0) {
+		if (!verifyUInt(num)) {
 			throw new BadRequest(errorMessage);
 		}
 
 		return num;
 	}
 
+	/**
+	 * Expected format ie: 0-999
+	 */
+	protected parseRangeOfNumbersOrThrow(n: string, maxRange: number): number[] {
+		const splitRange = n.split('-');
+		if (splitRange.length !== 2) {
+			throw new BadRequest('Incorrect range format. Expected example: 0-999');
+		}
+
+		const min = Number(splitRange[0]);
+		const max = Number(splitRange[1]);
+
+		if (!verifyUInt(min)) {
+			throw new BadRequest(
+				'Inputted min value for range must be an unsigned integer.'
+			);
+		}
+
+		if (!verifyNonZeroUInt(max)) {
+			throw new BadRequest(
+				'Inputted max value for range must be an unsigned non zero integer.'
+			);
+		}
+
+		if (min >= max) {
+			throw new BadRequest(
+				'Inputted min value cannot be greater than or equal to the max value.'
+			);
+		}
+
+		if (max - min > maxRange) {
+			throw new BadRequest(
+				`Inputted range is greater than the ${maxRange} range limit.`
+			);
+		}
+
+		return [...Array(max - min + 1).keys()].map((i) => i + min);
+	}
+
 	protected parseQueryParamArrayOrThrow(n: string[]): number[] {
 		return n.map((str) =>
 			this.parseNumberOrThrow(

src/controllers/AbstractControllers.spec.ts

@@ -316,4 +316,48 @@ describe('AbstractController', () => {
 			api.createType = kusamaRegistry.createType.bind(kusamaRegistry);
 		});
 	});
+
+	describe('parseRangeOfNumbersOrThrow', () => {
+		it('Should return the correct array given a range', () => {
+			const res = controller['parseRangeOfNumbersOrThrow']('100-103', 500);
+
+			expect(res).toStrictEqual([100, 101, 102, 103]);
+		});
+
+		it('Should throw an error when the inputted format is wrong', () => {
+			const badFormatRequest = new BadRequest(
+				'Incorrect range format. Expected example: 0-999'
+			);
+			const badMinRequest = new BadRequest(
+				'Inputted min value for range must be an unsigned integer.'
+			);
+			const badMaxRequest = new BadRequest(
+				'Inputted max value for range must be an unsigned non zero integer.'
+			);
+			const badMaxMinRequest = new BadRequest(
+				'Inputted min value cannot be greater than or equal to the max value.'
+			);
+			const badMaxRangeRequest = new BadRequest(
+				'Inputted range is greater than the 500 range limit.'
+			);
+			expect(() =>
+				controller['parseRangeOfNumbersOrThrow']('100', 500)
+			).toThrow(badFormatRequest);
+			expect(() =>
+				controller['parseRangeOfNumbersOrThrow']('h-100', 500)
+			).toThrow(badMinRequest);
+			expect(() =>
+				controller['parseRangeOfNumbersOrThrow']('100-h', 500)
+			).toThrow(badMaxRequest);
+			expect(() =>
+				controller['parseRangeOfNumbersOrThrow']('100-1', 500)
+			).toThrow(badMaxMinRequest);
+			expect(() =>
+				controller['parseRangeOfNumbersOrThrow']('1-1', 500)
+			).toThrow(badMaxMinRequest);
+			expect(() =>
+				controller['parseRangeOfNumbersOrThrow']('2-503', 500)
+			).toThrow(badMaxRangeRequest);
+		});
+	});
 });

src/controllers/blocks/BlocksController.ts

@@ -17,11 +17,13 @@
 import { ApiPromise } from '@polkadot/api';
 import { isHex } from '@polkadot/util';
 import { RequestHandler } from 'express';
+import { BadRequest } from 'http-errors';
 import LRU from 'lru-cache';
 
 import { BlocksService } from '../../services';
-import { INumberParam } from '../../types/requests';
+import { INumberParam, IRangeQueryParam } from '../../types/requests';
 import { IBlock } from '../../types/responses';
+import { PromiseQueue } from '../../util/PromiseQueue';
 import AbstractController from '../AbstractController';
 
 interface ControllerOptions {
@@ -101,6 +103,7 @@ export default class BlocksController extends AbstractController<BlocksService>
 
 	protected initRoutes(): void {
 		this.safeMountAsyncGetHandlers([
+			['/', this.getBlocks],
 			['/head', this.getLatestBlock],
 			['/:number', this.getBlockById],
 			['/head/header', this.getLatestBlockHeader],
@@ -233,4 +236,61 @@ export default class BlocksController extends AbstractController<BlocksService>
 			await this.service.fetchBlockHeader(hash)
 		);
 	};
+
+	/**
+	 * Return a collection of blocks, given a range.
+	 *
+	 * @param req Express Request
+	 * @param res Express Response
+	 */
+	private getBlocks: RequestHandler<
+		unknown,
+		unknown,
+		unknown,
+		IRangeQueryParam
+	> = async (
+		{ query: { range, eventDocs, extrinsicDocs } },
+		res
+	): Promise<void> => {
+		if (!range) throw new BadRequest('range query parameter must be inputted.');
+
+		// We set a max range to 500 blocks.
+		const rangeOfNums = this.parseRangeOfNumbersOrThrow(range, 500);
+
+		const eventDocsArg = eventDocs === 'true';
+		const extrinsicDocsArg = extrinsicDocs === 'true';
+		const queryFinalizedHead = !this.options.finalizes ? false : true;
+		const omitFinalizedTag = !this.options.finalizes ? true : false;
+		const options = {
+			eventDocs: eventDocsArg,
+			extrinsicDocs: extrinsicDocsArg,
+			checkFinalized: false,
+			queryFinalizedHead,
+			omitFinalizedTag,
+		};
+
+		const pQueue = new PromiseQueue(4);
+		const blocksPromise: Promise<unknown>[] = [];
+
+		for (let i = 0; i < rangeOfNums.length; i++) {
+			const result = pQueue.run(async () => {
+				// Get block hash:
+				const hash = await this.getHashForBlock(rangeOfNums[i].toString());
+				// Get API at that hash:
+				const historicApi = await this.api.at(hash);
+				// Get block details using this API/hash:
+				return await this.service.fetchBlock(hash, historicApi, options);
+			});
+			blocksPromise.push(result);
+		}
+
+		const blocks = (await Promise.all(blocksPromise)) as IBlock[];
+
+		/**
+		 * Sort blocks from least to greatest.
+		 */
+		blocks.sort((a, b) => a.number.toNumber() - b.number.toNumber());
+
+		BlocksController.sanitizedSend(res, blocks);
+	};
 }

src/types/requests.ts

@@ -45,3 +45,7 @@ export interface IAddressNumberParams extends IAddressParam {
 export interface IParaIdParam extends ParamsDictionary {
 	paraId: string;
 }
+
+export interface IRangeQueryParam extends Query {
+	range: string;
+}

src/util/PromiseQueue.ts

@@ -0,0 +1,102 @@
+/**
+ * A PromiseQueue, enforcing that no more than `maxTasks` number of tasks
+ * are running at a given time.
+ */
+export class PromiseQueue<T> {
+	// How many tasks are allowed to run concurrently?
+	#maxTasks: number;
+	// How many tasks are currently running concurrently?
+	#runningTasks = 0;
+	// The queued tasks waiting to run
+	#tasks: LinkedList<() => void>;
+
+	constructor(maxTasks: number) {
+		this.#maxTasks = maxTasks;
+		this.#tasks = new LinkedList();
+	}
+
+	// Try to run the next task in the queue.
+	private tryRunNextTask(): void {
+		if (this.#runningTasks >= this.#maxTasks) {
+			return;
+		}
+		const nextTask = this.#tasks.popFront();
+		if (nextTask) {
+			nextTask();
+			this.#runningTasks += 1;
+		}
+	}
+
+	// Take a task and package it up to run, triggering
+	// the next task when it completes (or errors), and returning the
+	// result in the returned promise.
+	private submitTaskToRun(task: () => Promise<T>): Promise<T> {
+		return new Promise((resolve, reject) => {
+			const onFinish = () => {
+				this.#runningTasks -= 1;
+				this.tryRunNextTask();
+			};
+
+			const taskToRun = () => {
+				task()
+					.then((item) => {
+						resolve(item);
+						onFinish();
+					})
+					.catch((err) => {
+						reject(err);
+						onFinish();
+					});
+			};
+
+			this.#tasks.pushBack(taskToRun);
+			this.tryRunNextTask();
+		});
+	}
+
+	/**
+	 * Push a new task onto the queue. It will run when there are fewer
+	 * than `maxTasks` running.
+	 */
+	run(task: () => Promise<T>): Promise<T> {
+		return this.submitTaskToRun(task);
+	}
+}
+
+type LinkedListItem<T> = { item: T; next: null | LinkedListItem<T> };
+
+/**
+ * A quick LinkedList queue implementation; we can add items to the back
+ * or remove them from the front.
+ */
+class LinkedList<T> {
+	#first: LinkedListItem<T> | null = null;
+	#last: LinkedListItem<T> | null = null;
+
+	private init(item: T): void {
+		this.#first = this.#last = { item, next: null };
+	}
+
+	pushBack(item: T) {
+		if (!this.#first) return this.init(item);
+		const entry = { item, next: null };
+		/* eslint-disable-next-line @typescript-eslint/no-non-null-assertion */
+		this.#last!.next = entry;
+		this.#last = entry;
+	}
+
+	popFront(): T | null {
+		if (!this.#first) return null;
+		const entry = this.#first;
+		this.#first = this.#first.next;
+		return entry.item;
+	}
+
+	clear(): void {
+		this.#first = this.#last = null;
+	}
+
+	empty(): boolean {
+		return this.#first === null;
+	}
+}

src/util/integers/verifyInt.spec.ts

@@ -0,0 +1,19 @@
+import { verifyNonZeroUInt, verifyUInt } from './verifyInt';
+
+describe('Verify integers', () => {
+	describe('verifyUInt', () => {
+		it('Should correctly handle unsigned integers correctly', () => {
+			expect(verifyUInt(0)).toBe(true);
+			expect(verifyUInt(1)).toBe(true);
+			expect(verifyUInt(-1)).toBe(false);
+		});
+	});
+
+	describe('verifyNonZeroUInt', () => {
+		it('Should correctly handle unsigned integers correctly', () => {
+			expect(verifyNonZeroUInt(1)).toBe(true);
+			expect(verifyNonZeroUInt(0)).toBe(false);
+			expect(verifyNonZeroUInt(-1)).toBe(false);
+		});
+	});
+});

src/util/integers/verifyInt.ts

@@ -0,0 +1,15 @@
+/**
+ * Verify all integers including zeroes.
+ * @param num
+ */
+export const verifyUInt = (num: Number): boolean => {
+	return Number.isInteger(num) && num >= 0;
+};
+
+/**
+ * Verify all integers except for zero. Will return false when zero is inputted.
+ * @param num
+ */
+export const verifyNonZeroUInt = (num: Number): boolean => {
+	return Number.isInteger(num) && num > 0;
+};