Skip to content

Commit

Permalink
fix: rewrite sidecar e2e script (#618)
Browse files Browse the repository at this point in the history
* fix: rewrite sidecar e2e script
  • Loading branch information
TarikGul committed Jul 27, 2021
1 parent bddc2a2 commit 423574e
Show file tree
Hide file tree
Showing 13 changed files with 292 additions and 105 deletions.
1 change: 1 addition & 0 deletions .eslintignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@
calc
docs
e2e-tests
scripts
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
**/node_modules
/build
/e2e-tests/build
/scripts/build
/target
yarn-error.log
**/*.rs.bk
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -292,9 +292,9 @@ All the commits in this repo follow the [Conventional Commits spec](https://www.

1. Next make sure the resolutions are up to date inside of the `package.json` to match polkadot-js [here](https://github.com/polkadot-js/apps/blob/master/package.json). This is a manual process, and the packages we want to update are only the ones with `@polkadot` appended to it. If any issues may surface, contact the maintainers.

1. After updating the dependencies and resolutions (if applicable), the next step is making sure the release will work against all relevant runtimes for Polkadot, Kusama, and Westend. This can be handled by running `yarn test:init-e2e-tests`. You must have `python3`, and the dependencies inside of `./scripts/requirements.txt` installed to run the script (Read the [README](./scripts/README.md) for more instructions). Before moving forward ensure all tests pass, and if it warns of any missing types feel free to make an issue [here](https://github.com/paritytech/substrate-api-sidecar/issues).
1. After updating the dependencies and resolutions (if applicable), the next step is making sure the release will work against all relevant runtimes for Polkadot, Kusama, and Westend. This can be handled by running `yarn test:init-e2e-tests`. If you would like to test on an individual chain, you may run the same command followed by its chain, ex: `yarn test:init-e2e-tests:polkadot`. Before moving forward ensure all tests pass, and if it warns of any missing types feel free to make an issue [here](https://github.com/paritytech/substrate-api-sidecar/issues).

Note that the e2e tests will connect to running nodes in order to test sidecar against real data, and they may fail owing to those connections taking too long to establish. If you run into any failures, try running tests just for the chain that failed with something like `yarn test:init-e2e-tests:polkadot`.
Note: that the e2e tests will connect to running nodes in order to test sidecar against real data, and they may fail owing to those connections taking too long to establish. If you run into any failures, try running tests just for the chain that failed with something like `yarn test:init-e2e-tests:polkadot`.

1. Update the version in the package.json (this is very important for releasing on NPM).

Expand Down
13 changes: 8 additions & 5 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
"build"
],
"scripts": {
"build": "rimraf build/ && substrate-exec-tsc",
"build": "rimraf build/ && substrate-exec-tsc && echo Build Finished",
"build:calc": "bash ./calc/build.sh",
"build:docker": "docker build -t substrate-api-sidecar .",
"build:docs": "(cd docs && yarn && yarn build)",
Expand All @@ -40,10 +40,13 @@
"lint:e2e-tests": "cd e2e-tests && substrate-dev-run-lint",
"build:e2e-tests": "(cd e2e-tests && substrate-exec-tsc)",
"test:e2e-tests": "yarn build:e2e-tests && node ./e2e-tests/build/index.js --config=./e2e-tests/jest.config.js",
"test:init-e2e-tests": "python3 ./scripts/run_chain_tests.py",
"test:init-e2e-tests:polkadot": "python3 ./scripts/run_chain_tests.py --chain polkadot",
"test:init-e2e-tests:kusama": "python3 ./scripts/run_chain_tests.py --chain kusama",
"test:init-e2e-tests:westend": "python3 ./scripts/run_chain_tests.py --chain westend"
"test:init-e2e-tests": "yarn start:e2e-scripts",
"test:init-e2e-tests:polkadot": "yarn start:e2e-scripts --chain polkadot",
"test:init-e2e-tests:kusama": "yarn start:e2e-scripts --chain kusama",
"test:init-e2e-tests:westend": "yarn start:e2e-scripts --chain westend",
"start:e2e-scripts": "yarn build:scripts && node scripts/build/runChainTests.js",
"build:scripts": "cd scripts && substrate-exec-tsc",
"lint:scripts": "cd scripts && substrate-dev-run-lint"
},
"dependencies": {
"@polkadot/api": "^5.1.1",
Expand Down
1 change: 1 addition & 0 deletions scripts/.eslintignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
**/build/*
10 changes: 1 addition & 9 deletions scripts/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,9 @@
<br></br>


## Script `run_chain_tests.py`
## Script `runChainTests.ts`

### Summary

This script calls the local e2e-tests helper library in order to test the current branch or development environment against
a collection of different blocks, across different runtimes. It does this for Polkadot, Kusama, and Westend.

### Requirements

`python3` - required to run the script

`psutil` - package needed to run the script (installed by the below command)

Run: `pip install -r requirements.txt` from this directory in order to install necessary dependencies.
46 changes: 46 additions & 0 deletions scripts/config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import { IChainConfig } from './types';

const defaultSasStartOpts = {
proc: 'sidecar',
resolver: 'Check the root endpoint',
resolverStartupErr: 'error: uncaughtException: listen EADDRINUSE:',
args: ['start'],
};

const defaultJestOpts = {
proc: 'jest',
resolver: 'PASS',
};

export const defaultSasBuildOpts = {
proc: 'sidecar',
resolver: 'Build Finished',
args: ['build'],
};

export const config: Record<string, IChainConfig> = {
polkadot: {
wsUrl: 'wss://rpc.polkadot.io',
JestProcOpts: {
...defaultJestOpts,
args: ['test:e2e-tests', '--chain', 'polkadot'],
},
SasStartOpts: defaultSasStartOpts,
},
kusama: {
wsUrl: 'wss://kusama-rpc.polkadot.io',
JestProcOpts: {
...defaultJestOpts,
args: ['test:e2e-tests', '--chain', 'kusama'],
},
SasStartOpts: defaultSasStartOpts,
},
westend: {
wsUrl: 'wss://westend-rpc.polkadot.io',
JestProcOpts: {
...defaultJestOpts,
args: ['test:e2e-tests', '--chain', 'westend'],
},
SasStartOpts: defaultSasStartOpts,
},
};
1 change: 0 additions & 1 deletion scripts/requirements.txt

This file was deleted.

204 changes: 204 additions & 0 deletions scripts/runChainTests.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,204 @@
import { ArgumentParser, Namespace } from 'argparse';
import { ChildProcessWithoutNullStreams, spawn } from 'child_process';

import { config, defaultSasBuildOpts } from './config';
import { IProcOpts, StatusCode } from './types';

// Stores all the processes
const procs: { [key: string]: ChildProcessWithoutNullStreams } = {};

// Set the env variable for SAS_SUBSTRATE_WS_URL
const setWsUrl = (url: string): void => {
process.env.SAS_SUBSTRATE_WS_URL = url;
};

/**
* Launch any given process. It accepts an options object.
*
* {
* proc => the name of the process to be saved in our cache
* resolver => If the stdout contains the resolver it will resolve the process
* resolverStartupErr => If the stderr contains the resolver it will resolve the process
* args => an array of args to be attached to the `yarn` command.
* }
*
* @param IProcOpts
*/
const launchProcess = async ({
proc,
resolver,
resolverStartupErr,
args,
}: IProcOpts): Promise<StatusCode> => {
return new Promise<StatusCode>((resolve, reject) => {
const { Success, Failed } = StatusCode;

const command = 'yarn';

procs[proc] = spawn(command, args, { detached: true });

procs[proc].stdout.on('data', (data: Buffer) => {
console.log(data.toString());

if (data.toString().includes(resolver)) {
resolve(Success);
}
});

procs[proc].stderr.on('data', (data: Buffer) => {
console.error(data.toString());

if (resolverStartupErr && data.toString().includes(resolverStartupErr)) {
resolve(Failed);
}
});

procs[proc].on('close', () => {
resolve(Success);
});

procs[proc].on('error', (err) => {
console.log(err);
reject(Failed);
});
});
};

/**
* Launches Sidecar, and if successful it will launch the jest runner. This operation
* handles killing all the processes after the jest runner is done.
*
* @param chain The chain in which to target the e2e tests too.
*/
const launchChainTest = async (chain: string): Promise<boolean> => {
const { wsUrl, SasStartOpts, JestProcOpts } = config[chain];
const { Success } = StatusCode;

// Set the ws url env var
setWsUrl(wsUrl);

console.log('Launching Sidecar...');
const sidecarStart = await launchProcess(SasStartOpts);

if (sidecarStart === Success) {
// Sidecar successfully launched, and jest will now get called
console.log('Launching jest...');
const jest = await launchProcess(JestProcOpts);

if (jest === Success) {
killAll();
return true;
} else {
killAll();
return false;
}
} else {
console.error('Error launching sidecar... exiting...');
killAll();
process.exit(1);
}
};

// Kill all processes spawned and tracked by this file.
const killAll = () => {
console.log('Killing all processes...');
for (const key of Object.keys(procs)) {
if (!procs[key].killed) {
try {
console.log(`Killing ${key}`);
// Kill child and all its descendants.
process.kill(-procs[key].pid, 'SIGTERM');
process.kill(-procs[key].pid, 'SIGKILL');
} catch (e) {
/**
* The error we are catching here silently, is when `-procs[key].pid` takes
* the range of all pid's inside of the subprocess group created with
* `spawn`, and one of the process's is either already closed or doesn't exist anymore.
*
* ex: `Error: kill ESRCH`
*
* This is a very specific use case of an empty catch block and is used
* outside of the scope of the API therefore justifiable, and should be used cautiously
* elsewhere.
*/
}
}
}
};

const checkTests = (...args: boolean[]) => {
const testStatus = args.every((test) => test);

if (testStatus) {
console.log('[PASSED] All Tests Passed!');
process.exit(0);
} else {
console.log('[FAILED] Some Tests Failed!');
process.exit(1);
}
};

const main = async (args: Namespace): Promise<void> => {
const { Failed } = StatusCode;

// Build sidecar
console.log('Building Sidecar...');
const sidecarBuild = await launchProcess(defaultSasBuildOpts);

// When sidecar fails to build, we kill all process's and exit
if (sidecarBuild === Failed) {
console.error('Sidecar failed to build, exiting...');
// Kill all processes
killAll();
// Exit program
process.exit();
}

if (args.chain) {
const selectedChain = await launchChainTest(args.chain);

checkTests(selectedChain);
} else {
// Test the e2e tests against polkadot
const polkadotTest = await launchChainTest('polkadot');

// Test the e2e tests against kusama
const kusamaTest = await launchChainTest('kusama');

// Test the e2e tests against westend
const westendTest = await launchChainTest('westend');

checkTests(polkadotTest, kusamaTest, westendTest);
}
};

/**
* Arg Parser
*/
const parser = new ArgumentParser();

parser.add_argument('--chain', {
choices: ['polkadot', 'kusama', 'westend'],
});

const args = parser.parse_args() as Namespace;

/**
* Signal interrupt
*/
process.on('SIGINT', function () {
console.log('Caught interrupt signal');
killAll();
process.exit();
});

/**
* Signal hangup terminal
*/
process.on('SIGHUP', function () {
console.log('Caught terminal termination');
killAll();
process.exit();
});

main(args).finally(() => process.exit());
Loading

0 comments on commit 423574e

Please sign in to comment.