feat!: add support for http provider using SAS_SUBSTRATE_URL (#1001)

* docs: update README with SAS_SUBSTRATE_HTTP_URL env var

* add HTTP_URL key

* add HTTP_URL spec

* add HTTP_URL to the types

* feat: add http logic to provider and logging

* lint

* fix grumble

* Update README.md

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>

* BREAKING CHANGE: switch SAS_SUBSTRATE_WS_URL to SAS_SUBSTRATE_URL

* fix params for startUpPrompt

* Update src/Specs.ts

Co-authored-by: Sergejs Kostjucenko <85877331+sergejparity@users.noreply.github.com>

* add https to regex

* fix inline comment grumble

Co-authored-by: Alexandru Vasile <60601340+lexnv@users.noreply.github.com>
Co-authored-by: Sergejs Kostjucenko <85877331+sergejparity@users.noreply.github.com>

.env.docker

@@ -10,4 +10,4 @@ SAS_EXPRESS_BIND_HOST=0.0.0.0
 # Port exposed by the container
 SAS_EXPRESS_PORT=8080
 
-SAS_SUBSTRATE_WS_URL=wss://rpc.polkadot.io
+SAS_SUBSTRATE_URL=wss://rpc.polkadot.io

.env.local

@@ -3,4 +3,4 @@
 # For more information on how to use .env files and environment variables
 # consult the Configuration section in the README.
 
-SAS_SUBSTRATE_WS_URL=ws://127.0.0.1:9944
+SAS_SUBSTRATE_URL=ws://127.0.0.1:9944

.gitlab-ci.yml

@@ -183,7 +183,7 @@ deploy-production:
     BENCHMARK_SCRIPT:              "./scripts/ci/benchmarks/lightweight-bench.lua" # https://github.com/wg/wrk/blob/master/SCRIPTING
     CI_IMAGE:                      "paritytech/node-wrk:latest"
   script:
-    - export SAS_SUBSTRATE_WS_URL=$POLKADOT_ADDRESS
+    - export SAS_SUBSTRATE_URL=$POLKADOT_ADDRESS
     - yarn
     - yarn build
     - yarn run main &

README.md

@@ -161,8 +161,8 @@ For more information on our configuration manager visit its readme [here](https:
 
 ### Substrate node
 
-- `SAS_SUBSTRATE_WS_URL`: WebSocket URL to which the RPC proxy will attempt to connect to, defaults to
-    `ws://127.0.0.1:9944`.
+- `SAS_SUBSTRATE_URL`: URL to which the RPC proxy will attempt to connect to, defaults to
+    `ws://127.0.0.1:9944`. Accepts both a websocket, and http URL.
 
 #### Custom substrate types
 

docker-compose.yml

@@ -16,4 +16,4 @@ services:
     ports:
       - "8080:8080"
     environment:
-      SAS_SUBSTRATE_WS_URL: ws://polkadot:9944
+      SAS_SUBSTRATE_URL: ws://polkadot:9944

helm/values.yaml

@@ -24,7 +24,7 @@ serviceAccount:
   name: ""
 
 envVars:
-  SAS_SUBSTRATE_WS_URL: "ws://<polkadot-or-kusama-archive-node>:<port>"
+  SAS_SUBSTRATE_URL: "ws://<polkadot-or-kusama-archive-node>:<port>"
 
 podAnnotations: {}
 

scripts/sidecarScriptApi.ts

@@ -24,7 +24,7 @@ import { IProcOpts, ProcsType, StatusCode } from './types';
  * @param url ws url used in sidecar
  */
 export const setWsUrl = (url: string): void => {
-	process.env.SAS_SUBSTRATE_WS_URL = url;
+	process.env.SAS_SUBSTRATE_URL = url;
 };
 
 /**

src/SidecarConfig.ts

@@ -57,7 +57,7 @@ export class SidecarConfig {
 				PORT: config.Get(MODULES.EXPRESS, CONFIG.PORT) as number,
 			},
 			SUBSTRATE: {
-				WS_URL: config.Get(MODULES.SUBSTRATE, CONFIG.WS_URL) as string,
+				URL: config.Get(MODULES.SUBSTRATE, CONFIG.URL) as string,
 				TYPES_BUNDLE: config.Get(
 					MODULES.SUBSTRATE,
 					CONFIG.TYPES_BUNDLE

src/Specs.ts

@@ -87,13 +87,13 @@ export class Specs {
 			throw APPEND_SPEC_ERROR;
 		}
 
-		// WS
+		// WS OR HTTP
 		this._specs.appendSpec(
 			MODULES.SUBSTRATE,
-			this._specs.getSpec(CONFIG.WS_URL, 'Websocket URL', {
+			this._specs.getSpec(CONFIG.URL, 'Websocket or HTTP URL', {
 				default: 'ws://127.0.0.1:9944',
 				mandatory: true,
-				regexp: /^wss?:\/\/.*(:\d{4,5})?$/,
+				regexp: /^(ws|wss|http|https)?:\/\/.*/,
 			})
 		);
 

src/main.ts

@@ -20,7 +20,7 @@
 import '@polkadot/api-augment';
 
 import { ApiPromise } from '@polkadot/api';
-import { WsProvider } from '@polkadot/rpc-provider';
+import { HttpProvider, WsProvider } from '@polkadot/rpc-provider';
 import { OverrideBundleType, RegistryTypes } from '@polkadot/types/types';
 import { json } from 'express';
 
@@ -44,7 +44,9 @@ async function main() {
 	const { TYPES_BUNDLE, TYPES_SPEC, TYPES_CHAIN, TYPES } = config.SUBSTRATE;
 	// Instantiate a web socket connection to the node and load types
 	const api = await ApiPromise.create({
-		provider: new WsProvider(config.SUBSTRATE.WS_URL),
+		provider: config.SUBSTRATE.URL.startsWith('http')
+			? new HttpProvider(config.SUBSTRATE.URL)
+			: new WsProvider(config.SUBSTRATE.URL),
 		/* eslint-disable @typescript-eslint/no-var-requires */
 		typesBundle: TYPES_BUNDLE
 			? (require(TYPES_BUNDLE) as OverrideBundleType)
@@ -66,7 +68,7 @@ async function main() {
 	]);
 
 	startUpPrompt(
-		config.SUBSTRATE.WS_URL,
+		config.SUBSTRATE.URL,
 		chainName.toString(),
 		implName.toString()
 	);
@@ -94,29 +96,28 @@ async function main() {
  * Prompt the user with some basic info about the node and the network they have
  * connected Sidecar to.
  *
- * @param wsUrl websocket url of the node Sidecar is connected to
+ * @param url Url of the node Sidecar is connected to, can be a websocket or http address
  * @param chainName chain name of the network Sidecar is connected to
  * @param implName implementation name of the node Sidecar is connected to
  */
-function startUpPrompt(wsUrl: string, chainName: string, implName: string) {
+function startUpPrompt(url: string, chainName: string, implName: string) {
 	const { logger } = Log;
-	const { config } = SidecarConfig;
 
 	logger.info(
-		`Connected to chain ${chainName} on the ${implName} client at ${config.SUBSTRATE.WS_URL}`
+		`Connected to chain ${chainName} on the ${implName} client at ${url}`
 	);
 
 	// Split the Url to check for 2 things. Secure connection, and if its a local IP.
-	const splitUrl: string[] = wsUrl.split(':');
+	const splitUrl: string[] = url.split(':');
 	// If its 'ws' its not a secure connection.
-	const isSecure: boolean = splitUrl[0] === 'wss';
+	const isSecure: boolean = splitUrl[0] === 'wss' || splitUrl[0] === 'https';
 	// Check if its a local IP.
 	const isLocal: boolean =
 		splitUrl[1] === '//127.0.0.1' || splitUrl[1] === '//localhost';
 
 	if (!isSecure && !isLocal) {
 		logger.warn(
-			`Using unencrypted connection to a public node (${wsUrl}); All traffic is sent over the internet in cleartext.`
+			`Using unencrypted connection to a public node (${url}); All traffic is sent over the internet in cleartext.`
 		);
 	}
 }

src/types/sidecar-config/CONFIG.ts

@@ -20,7 +20,7 @@
 export enum CONFIG {
 	BIND_HOST = 'BIND_HOST',
 	PORT = 'PORT',
-	WS_URL = 'WS_URL',
+	URL = 'URL',
 	LEVEL = 'LEVEL',
 	JSON = 'JSON',
 	FILTER_RPC = 'FILTER_RPC',

src/types/sidecar-config/SidecarConfig.ts

@@ -24,7 +24,7 @@ export interface ISidecarConfig {
 }
 
 interface ISidecarConfigSubstrate {
-	WS_URL: string;
+	URL: string;
 	TYPES_BUNDLE: string;
 	TYPES_CHAIN: string;
 	TYPES_SPEC: string;