feat: Add route based metrics across API

README.md

@@ -256,11 +256,24 @@ You can also define a custom port by running :
 yarn start --prometheus --prometheus-port=<YOUR_CUSTOM_PORT>
 ```
 
+You can also expand the metrics tracking capabilities to include query params by running:
+
+```bash
+yarn start --prometheus --prometheus-queryparams
+```
+
 The metrics endpoint can then be accessed :
 - on the default port : `http://127.0.0.1:9100/metrics` or
 - on your custom port if you defined one : `http://127.0.0.1:<YOUR_CUSTOM_PORT>/metrics`
 
-That way you will have access to the default prometheus metrics and one extra custom metric called `sas_http_errors` (of type counter). This counter is increased by 1 every time an http error has occured in sidecar.
+A JSON format response is available at `http://127.0.0.1:9100/metrics.json`.
+
+That way you will have access to the default prometheus metrics and a generic custom metric called `sas_http_errors` (of type counter). This counter is increased by 1 every time an http error has occured in sidecar.
+
+Moreover, the following metrics will be emitted for each route:
+- `sas_https_request_duration_seconds`: type histogram and tracks the latency of the requests
+- `sas_http_response_size_bytes`: type histogram and tracks the response size of the requests
+- `sas_http_response_size_latency_ratio`: type histogram and tracks the response bytes per second of the requests
 
 
 ## Debugging fee and staking payout calculations

benchmarks/README.md

@@ -75,10 +75,12 @@ sudo cp wrk /usr/local/bin
 
 ### Running Locally
 
-In order to run each benchmark you should `cd` into the appropriate directory you want to run, set the `WRK_TIME_LENGTH` env var (ex: `export WRK_TIME_LENGTH=30s`) and then `sh init.sh`. You are required to have sidecar running, as well as a fully synced archive node.
+In order to run each benchmark independently you should `cd` into the appropriate directory you want to run, set the `WRK_TIME_LENGTH` env var (ex: `export WRK_TIME_LENGTH=30s`) and then `sh init.sh`. You are required to have sidecar running, as well as a fully synced archive node.
 
 NOTE: Some benchmarks might have multiple `sh` scripts with different names in order to run specific benchmarks.
 
+There is also the option to run all the benchmarks' init.sh files by running `sh init.sh` from `./benchmarks`.
+
 ### Running via Scripts (Root)
 
 Below are flags, and examples on how to run these benchmarks from the root of the repository. See <root>/scripts/README.md for more information. 

benchmarks/init.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+for d in ./*/ ; do (cd "$d" && export WRK_TIME_LENGTH=30s; sh init.sh); done

src/main.ts

@@ -65,9 +65,24 @@ async function main() {
 
 	startUpPrompt(config.SUBSTRATE.URL, chainName.toString(), implName.toString());
 
+	const middlewares = [json(), middleware.httpLoggerCreate(logger)];
+
+	if (args.prometheus) {
+		// Create Metrics App
+		const metricsApp = new Metrics_App({
+			port: 9100,
+			host: config.EXPRESS.HOST,
+		});
+
+		// Generate metrics middleware
+		middlewares.push(metricsApp.middleware());
+		// Start the Metrics server
+		metricsApp.listen();
+	}
+
 	// Create our App
 	const app = new App({
-		preMiddleware: [json(), middleware.httpLoggerCreate(logger)],
+		preMiddleware: middlewares,
 		controllers: getControllersForSpec(api, specName.toString()),
 		postMiddleware: [
 			middleware.txError,
@@ -85,16 +100,6 @@ async function main() {
 
 	server.keepAliveTimeout = config.EXPRESS.KEEP_ALIVE_TIMEOUT;
 	server.headersTimeout = config.EXPRESS.KEEP_ALIVE_TIMEOUT + 5000;
-
-	if (args.prometheus) {
-		// Create Metrics App
-		const metricsApp = new Metrics_App({
-			port: 9100,
-			host: config.EXPRESS.HOST,
-		});
-		// Start the Metrics server
-		metricsApp.listen();
-	}
 }
 
 /**

src/middleware/error/httpErrorMiddleware.ts

@@ -18,8 +18,6 @@ import { ErrorRequestHandler } from 'express';
 import { HttpError } from 'http-errors';
 
 import { Log } from '../../logging/Log';
-import { parseArgs } from '../../parseArgs';
-import { httpErrorCounter } from '../../util/metrics';
 /**
  * Handle HttpError instances.
  *
@@ -35,17 +33,14 @@ export const httpErrorMiddleware: ErrorRequestHandler = (err: unknown, _req, res
 	if (res.headersSent || !(err instanceof HttpError)) {
 		return next(err);
 	}
-	const args = parseArgs();
 	const code = err.status;
 
 	const info = {
 		code,
 		message: err.message,
 		stack: err.stack,
 	};
-	if (args.prometheus) {
-		httpErrorCounter.inc();
-	}
+
 	Log.logger.error(info);
 
 	res.status(code).send(info);

src/parseArgs.ts

@@ -32,6 +32,10 @@ export const parseArgs = (): Namespace => {
 		default: 9100,
 		help: 'specify the port number on which the prometheus metrics are exposed [default: 9100]',
 	});
+	parser.add_argument('-pq', '--prometheus-queryparams', {
+		action: 'store_true',
+		help: 'enambles query parameters in the prometheus metrics',
+	});
 
 	return parser.parse_args() as Namespace;
 };

src/util/metrics.ts

@@ -5,18 +5,70 @@ import client from 'prom-client';
 import { Log } from '../logging/Log';
 import { parseArgs } from '../parseArgs';
 
-export const httpErrorCounter = new client.Counter({
-	name: 'sas_http_errors',
-	help: 'Number of HTTP Errors',
-});
-
 interface IAppConfiguration {
 	port: number;
 	host: string;
 }
 
+interface IMetric {
+	name: string;
+	help: string;
+	type: MetricType;
+	buckets?: number[];
+	labels?: string[];
+}
+
+export const enum MetricType {
+	Counter = 'counter',
+	Gauge = 'gauge',
+	Histogram = 'histogram',
+	Summary = 'summary',
+}
+
+const metrics: IMetric[] = [
+	{
+		name: 'request_errors',
+		help: 'Number of HTTP Errors',
+		type: MetricType.Counter,
+	},
+	{
+		name: 'request_success',
+		help: 'Number of HTTP Success',
+		type: MetricType.Counter,
+	},
+	{
+		name: 'total_requests',
+		help: 'Total number of HTTP Requests',
+		type: MetricType.Counter,
+	},
+	{
+		name: 'request_duration_seconds',
+		help: 'Duration of HTTP requests in seconds',
+		labels: ['method', 'route', 'status_code'],
+		buckets: [0.1, 0.5, 1, 1.5, 2, 3, 4, 5],
+		type: MetricType.Histogram,
+	},
+	{
+		name: 'response_size_bytes',
+		help: 'Size of HTTP responses in bytes',
+		labels: ['method', 'route', 'status_code'],
+		buckets: [100, 500, 1000, 5000, 10000, 50000, 100000, 500000, 1000000, 5000000],
+		type: MetricType.Histogram,
+	},
+	{
+		name: 'response_size_latency_ratio',
+		help: 'Ratio of response size to latency',
+		labels: ['method', 'route', 'status_code'],
+		buckets: [64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144],
+		type: MetricType.Histogram,
+	},
+];
+
 export default class Metrics_App {
 	private app: Application;
+	private registry: client.Registry;
+	private metrics: Record<string, client.Metric>;
+	private includeQueryParams: boolean;
 	private readonly port: number;
 	private readonly host: string;
 
@@ -26,32 +78,179 @@ export default class Metrics_App {
 	constructor({ host }: IAppConfiguration) {
 		const args = parseArgs();
 
+		this.includeQueryParams = Boolean(args.prometheus_queryparams);
 		this.port = Number(args.prometheus_port);
 		this.app = express();
 		this.host = host;
-
-		this.metricsEndpoint();
+		this.registry = new client.Registry();
+		this.metrics = {};
+		this.init();
 	}
 
 	listen(): void {
 		const { logger } = Log;
 		this.app.listen(this.port, this.host, () => {
-			logger.info(`Metrics Server started at http://${this.host}:${this.port}/`);
+			logger.info(`Metrics Server started at http://${this.host}:${this.port}/metrics`);
 		});
 	}
 
-	/**
-	 * Mount the metrics endpoint.
-	 */
-	private metricsEndpoint() {
-		const register = new client.Registry();
-		register.registerMetric(httpErrorCounter);
-		client.collectDefaultMetrics({ register, prefix: 'sas_' });
+	private createMetricByType(prefix = 'sas', metric: IMetric) {
+		const prefixedName = prefix + '_' + metric.name;
+		if (prefixedName in this.metrics) {
+			return this.metrics[prefixedName];
+		}
+
+		switch (metric.type) {
+			case MetricType.Counter: {
+				const counter = new client.Counter({
+					name: prefixedName,
+					help: metric.help,
+					labelNames: metric.labels || [],
+					registers: [this.registry],
+				});
+
+				this.registry.registerMetric(counter);
+				this.metrics[prefixedName] = counter;
+				return counter;
+			}
+			case MetricType.Histogram: {
+				const histogram = new client.Histogram({
+					name: prefixedName,
+					help: metric.help,
+					labelNames: metric.labels || [],
+					registers: [this.registry],
+					buckets: metric.buckets || [0.1, 0.5, 1, 1.5, 2, 3, 4, 5],
+				});
+
+				this.metrics[prefixedName] = histogram;
+				return histogram;
+			}
+			case MetricType.Gauge:
+				throw new Error('Gauge not implemented');
+			case MetricType.Summary:
+				throw new Error('Summary not implemented');
+			default:
+				throw new Error('Unknown metric type');
+		}
+	}
+
+	private getRoute(req: Request) {
+		let route = req.baseUrl;
+		if (req.route) {
+			// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+			if (req.route?.path !== '/') {
+				// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
+				route = route ? route + req.route?.path : req.route?.path;
+			}
+
+			if (!route || route === '' || typeof route !== 'string') {
+				route = req.originalUrl.split('?')[0];
+			} else {
+				const splittedRoute = route.split('/');
+				const splittedUrl = req.originalUrl.split('?')[0].split('/');
+				const routeIndex = splittedUrl.length - splittedRoute.length + 1;
+
+				const baseUrl = splittedUrl.slice(0, routeIndex).join('/');
+				route = baseUrl + route;
+			}
+
+			if (this.includeQueryParams === true && Object.keys(req.query).length > 0) {
+				route = `${route}?${Object.keys(req.query)
+					.sort()
+					.map((queryParam) => `${queryParam}=<?>`)
+					.join('&')}`;
+			}
+		}
+
+		if (typeof req.params === 'object') {
+			Object.keys(req.params).forEach((paramName) => {
+				route = route.replace(req.params[paramName], ':' + paramName);
+			});
+		}
+
+		if (!route || route === '') {
+			// if (!req.route && res && res.statusCode === 404) {
+			route = 'N/A';
+		}
+
+		return route;
+	}
+
+	middleware() {
+		return (req: Request, res: Response, next: () => void) => {
+			const tot_requests = this.metrics['sas_total_requests'] as client.Counter;
+
+			// request count metrics
+			tot_requests.inc();
+			const request_duration_seconds = this.metrics['sas_request_duration_seconds'] as client.Histogram;
+			const end = request_duration_seconds.startTimer();
+
+			res.once('finish', () => {
+				if (res.statusCode >= 400) {
+					const request_errors = this.metrics['sas_request_errors'] as client.Counter;
+
+					console.log('request_errors', request_errors);
+					request_errors.inc();
+				} else {
+					const request_success = this.metrics['sas_request_success'] as client.Counter;
+					request_success.inc();
+				}
+
+				let resContentLength = '0';
+				if ('_contentLength' in res && res['_contentLength'] != null) {
+					resContentLength = res['_contentLength'] as string;
+				} else {
+					if (res.hasHeader('Content-Length')) {
+						resContentLength = res.getHeader('Content-Length') as string;
+					}
+				}
+				// route specific metrics
+				// ------ blocks -------
+
+				// blocks/:id
+				if (this.getRoute(req).includes('blocks') && req.params?.number) {
+					console.log('blocks/:id', this.getRoute(req), req.params?.number);
+				}
+				// blocks?range
+				if (this.getRoute(req).includes('blocks') && req.query?.range) {
+					console.log('blocks?range', this.getRoute(req), req.query?.range);
+				}
+				// response size metrics
+				const response_size_bytes = this.metrics['sas_response_size_bytes'] as client.Histogram;
+				response_size_bytes
+					.labels({ method: req.method, route: this.getRoute(req), status_code: res.statusCode })
+					.observe(parseFloat(resContentLength));
+				// latency metrics
+				end({ method: req.method, route: this.getRoute(req), status_code: res.statusCode });
+
+				// response size to latency ratio
+				const response_size_latency_ratio = this.metrics['sas_response_size_latency_ratio'] as client.Histogram;
+				response_size_latency_ratio
+					.labels({ method: req.method, route: this.getRoute(req), status_code: res.statusCode })
+					.observe(parseFloat(resContentLength) / end());
+			});
+
+			next();
+		};
+	}
+
+	private init() {
+		// Set up
+		metrics.forEach((metric) => this.createMetricByType('sas', metric));
+
+		client.collectDefaultMetrics({ register: this.registry, prefix: 'sas_' });
+
 		// Set up the metrics endpoint
 		this.app.get('/metrics', (_req: Request, res: Response) => {
 			void (async () => {
-				res.set('Content-Type', register.contentType);
-				res.send(await register.metrics());
+				res.set('Content-Type', this.registry.contentType);
+				res.send(await this.registry.metrics());
+			})();
+		});
+		this.app.get('/metrics.json', (_req: Request, res: Response) => {
+			void (async () => {
+				res.set('Content-Type', this.registry.contentType);
+				res.send(await this.registry.getMetricsAsJSON());
 			})();
 		});
 	}