feat: Add route based metrics across API (#1465)

* fix metric middleware

* updated benchmarks' README.md to include new run all script

* update to readmes

* add dynamic metric creation

* lint fix

* updates error metric in error middleware

* change preMiddleware

* init: first try at route based metrics

* lint clean

* metrics by route function

* route specific metrics

* update to env

* revert .gitignore

* Remove yarn.lock

* add yarn.lock

* commit missing files

* restore yarn.lock

* fixes to metrics and docs update

* update readme

* update metrics naming

* docs update

.gitignore

@@ -5,6 +5,7 @@
 /target
 yarn-error.log
 **/*.rs.bk
+
 .env.*
 !.env.local
 !.env.docker

README.md

@@ -256,12 +256,33 @@ You can also define a custom port by running :
 yarn start --prometheus --prometheus-port=<YOUR_CUSTOM_PORT>
 ```
 
+You can also expand the metrics tracking capabilities to include query params by running:
+
+```bash
+yarn start --prometheus --prometheus-queryparams
+```
+
 The metrics endpoint can then be accessed :
 - on the default port : `http://127.0.0.1:9100/metrics` or
 - on your custom port if you defined one : `http://127.0.0.1:<YOUR_CUSTOM_PORT>/metrics`
 
-That way you will have access to the default prometheus metrics and one extra custom metric called `sas_http_errors` (of type counter). This counter is increased by 1 every time an http error has occured in sidecar.
+A JSON format response is available at `http://127.0.0.1:9100/metrics.json`.
+
+That way you will have access to the default prometheus node instance metrics and the following metrics will be emitted for each route:
+
+- `sas_request_errors_total`: type counter and tracks http errors occuring in sidecar
+- `sas_request_success_total`: type counter and tracks successfull http requests
+- `sas_requests_total`: type counter and tracks all http requests
+- `sas_request_duration_seconds`: type histogram and tracks the latency of the requests
+- `sas_response_size_bytes_seconds`: type histogram and tracks the response size of the requests
+- `sas_response_size_latency_ratio_seconds`: type histogram and tracks the response bytes per second of the requests
+
+The blocks controller also includes the following route-specific metrics:
 
+- `sas_extrinsics_in_request_count`: type histogram and tracks the number of extrinsics returned in the request when a range of blocks is queried
+- `sas_extrinsics_per_second_count`: type histogram and tracks the returned extrinics per second
+- `sas_extrinsics_per_block_count`: type histogram and tracks the returned extrinsics per block
+- `sas_seconds_per_block_count`: type histogram and tracks the request time per block
 
 ## Debugging fee and staking payout calculations
 

benchmarks/README.md

@@ -75,10 +75,12 @@ sudo cp wrk /usr/local/bin
 
 ### Running Locally
 
-In order to run each benchmark you should `cd` into the appropriate directory you want to run, set the `WRK_TIME_LENGTH` env var (ex: `export WRK_TIME_LENGTH=30s`) and then `sh init.sh`. You are required to have sidecar running, as well as a fully synced archive node.
+In order to run each benchmark independently you should `cd` into the appropriate directory you want to run, set the `WRK_TIME_LENGTH` env var (ex: `export WRK_TIME_LENGTH=30s`) and then `sh init.sh`. You are required to have sidecar running, as well as a fully synced archive node.
 
 NOTE: Some benchmarks might have multiple `sh` scripts with different names in order to run specific benchmarks.
 
+There is also the option to run all the benchmarks' init.sh files by running `sh init.sh` from `./benchmarks`.
+
 ### Running via Scripts (Root)
 
 Below are flags, and examples on how to run these benchmarks from the root of the repository. See <root>/scripts/README.md for more information. 

benchmarks/init.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+for d in ./*/ ; do (cd "$d" && export WRK_TIME_LENGTH=30s; sh init.sh); done

docs/src/openapi-v1.yaml

@@ -18,6 +18,8 @@ servers:
   description: Polkadot Asset Hub Parity public sidecar
 - url: https://kusama-asset-hub-public-sidecar.parity-chains.parity.io/
   description: Kusama Asset Hub Parity public sidecar
+- url: http://localhost:8080
+  description: Localhost
 tags:
 - name: accounts
 - name: blocks

src/main.ts

@@ -65,9 +65,24 @@ async function main() {
 
 	startUpPrompt(config.SUBSTRATE.URL, chainName.toString(), implName.toString());
 
+	const preMiddlewares = [json(), middleware.httpLoggerCreate(logger)];
+
+	if (args.prometheus) {
+		// Create Metrics App
+		const metricsApp = new Metrics_App({
+			port: 9100,
+			host: config.EXPRESS.HOST,
+		});
+
+		// Generate metrics middleware
+		preMiddlewares.push(metricsApp.preMiddleware());
+		// Start the Metrics server
+		metricsApp.listen();
+	}
+
 	// Create our App
 	const app = new App({
-		preMiddleware: [json(), middleware.httpLoggerCreate(logger)],
+		preMiddleware: preMiddlewares,
 		controllers: getControllersForSpec(api, specName.toString()),
 		postMiddleware: [
 			middleware.txError,
@@ -85,16 +100,6 @@ async function main() {
 
 	server.keepAliveTimeout = config.EXPRESS.KEEP_ALIVE_TIMEOUT;
 	server.headersTimeout = config.EXPRESS.KEEP_ALIVE_TIMEOUT + 5000;
-
-	if (args.prometheus) {
-		// Create Metrics App
-		const metricsApp = new Metrics_App({
-			port: 9100,
-			host: config.EXPRESS.HOST,
-		});
-		// Start the Metrics server
-		metricsApp.listen();
-	}
 }
 
 /**

src/middleware/error/httpErrorMiddleware.ts

@@ -18,8 +18,6 @@ import { ErrorRequestHandler } from 'express';
 import { HttpError } from 'http-errors';
 
 import { Log } from '../../logging/Log';
-import { parseArgs } from '../../parseArgs';
-import { httpErrorCounter } from '../../util/metrics';
 /**
  * Handle HttpError instances.
  *
@@ -35,17 +33,14 @@ export const httpErrorMiddleware: ErrorRequestHandler = (err: unknown, _req, res
 	if (res.headersSent || !(err instanceof HttpError)) {
 		return next(err);
 	}
-	const args = parseArgs();
 	const code = err.status;
 
 	const info = {
 		code,
 		message: err.message,
 		stack: err.stack,
 	};
-	if (args.prometheus) {
-		httpErrorCounter.inc();
-	}
+
 	Log.logger.error(info);
 
 	res.status(code).send(info);

src/parseArgs.ts

@@ -32,6 +32,10 @@ export const parseArgs = (): Namespace => {
 		default: 9100,
 		help: 'specify the port number on which the prometheus metrics are exposed [default: 9100]',
 	});
+	parser.add_argument('-pq', '--prometheus-queryparams', {
+		action: 'store_true',
+		help: 'enambles query parameters in the prometheus metrics',
+	});
 
 	return parser.parse_args() as Namespace;
 };