Skip to content

Commit

Permalink
test: refactor & test exit codes
Browse files Browse the repository at this point in the history
  • Loading branch information
lili2311 committed Sep 4, 2020
1 parent 9151ff0 commit 809a73f
Show file tree
Hide file tree
Showing 5 changed files with 150 additions and 118 deletions.
109 changes: 3 additions & 106 deletions test/acceptance/cli-args.test.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
import { test } from 'tap';
import { exec } from 'child_process';
import { sep, join } from 'path';
import { readFileSync, unlinkSync, rmdirSync, mkdirSync, existsSync } from 'fs';
import { v4 as uuidv4 } from 'uuid';
import { sep } from 'path';

const osName = require('os-name');

Expand Down Expand Up @@ -101,7 +99,7 @@ test('snyk test command should fail when iac file is not supported', (t) => {
}
t.match(
stdout.trim(),
'CustomError: Illegal infrastructure as code target file',
'Illegal infrastructure as code target file',
'correct error output',
);
},
Expand All @@ -118,7 +116,7 @@ test('snyk test command should fail when iac file is not supported', (t) => {
}
t.match(
stdout.trim(),
'CustomError: Not supported infrastructure as code target files in',
'Not supported infrastructure as code target files in',
'correct error output',
);
},
Expand Down Expand Up @@ -346,104 +344,3 @@ test('`test --json-file-output no value produces error message`', (t) => {

optionsToTest.forEach(validate);
});

test('`test --json-file-output can save JSON output to file while sending human readable output to stdout`', (t) => {
t.plan(2);

exec(
`node ${main} test --json-file-output=snyk-direct-json-test-output.json`,
(err, stdout) => {
if (err) {
throw err;
}
t.match(stdout, 'Organization:', 'contains human readable output');
const outputFileContents = readFileSync(
'snyk-direct-json-test-output.json',
'utf-8',
);
unlinkSync('./snyk-direct-json-test-output.json');
const jsonObj = JSON.parse(outputFileContents);
const okValue = jsonObj.ok as boolean;
t.ok(okValue, 'JSON output ok');
},
);
});

test('`test --json-file-output produces same JSON output as normal JSON output to stdout`', (t) => {
t.plan(1);

exec(
`node ${main} test --json --json-file-output=snyk-direct-json-test-output.json`,
(err, stdout) => {
if (err) {
throw err;
}
const stdoutJson = stdout;
const outputFileContents = readFileSync(
'snyk-direct-json-test-output.json',
'utf-8',
);
unlinkSync('./snyk-direct-json-test-output.json');
t.equals(stdoutJson, outputFileContents);
},
);
});

test('`test --json-file-output can handle a relative path`', (t) => {
t.plan(1);

// if 'test-output' doesn't exist, created it
if (!existsSync('test-output')) {
mkdirSync('test-output');
}

const tempFolder = uuidv4();
const outputPath = `test-output/${tempFolder}/snyk-direct-json-test-output.json`;

exec(
`node ${main} test --json --json-file-output=${outputPath}`,
(err, stdout) => {
if (err) {
throw err;
}
const stdoutJson = stdout;
const outputFileContents = readFileSync(outputPath, 'utf-8');
unlinkSync(outputPath);
rmdirSync(`test-output/${tempFolder}`);
t.equals(stdoutJson, outputFileContents);
},
);
});

test(
'`test --json-file-output can handle an absolute path`',
{ skip: iswindows },
(t) => {
t.plan(1);

// if 'test-output' doesn't exist, created it
if (!existsSync('test-output')) {
mkdirSync('test-output');
}

const tempFolder = uuidv4();
const outputPath = join(
process.cwd(),
`test-output/${tempFolder}/snyk-direct-json-test-output.json`,
);

exec(
`node ${main} test --json --json-file-output=${outputPath}`,
(err, stdout) => {
if (err) {
throw err;
}
const stdoutJson = stdout;
const outputFileContents = readFileSync(outputPath, 'utf-8');
unlinkSync(outputPath);
rmdirSync(`test-output/${tempFolder}`);
t.equals(stdoutJson, outputFileContents);
},
);
},
);
Empty file.
24 changes: 13 additions & 11 deletions test/smoke/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ Design goal is to have a single test suite, that can detect if CLI is not workin

CLI is being tested by a series of tests using [Shellspec](https://shellspec.info). See them in a `test/smoke/spec` folder.

Spec in this folder is used as a 1) **"Smoke test" step in CircleCI** to verify that built CLI can run 2) **["Smoke Tests"](https://github.com/snyk/snyk/actions?query=workflow%3A%22Smoke+Tests%22) GitHub Action** to verify that our distribution channels are working.
Spec in this folder is used as a
1) **"Smoke test" step in CircleCI** to verify that built CLI can run
2) **["Smoke Tests"](https://github.com/snyk/snyk/actions?query=workflow%3A%22Smoke+Tests%22) GitHub Action** to verify that our distribution channels are working.

## How to add a new smoke test

Expand All @@ -14,18 +16,18 @@ Before you start adding specs, those files are bash scripts, it's recommended to

It's recommended to have a branch named `feat/smoke-test`, as [this branch will run the GitHub Action](https://github.com/snyk/snyk/blob/f35f39e96ef7aa69b22a846315dda015b12a4564/.github/workflows/smoke-tests.yml#L3-L5).

To run these tests locally, install:
To run these tests locally:

- [Shellspec](https://shellspec.info)
- [jq](https://stedolan.github.io/jq/)
- timeout (if not available on your platform)
1. Install:

cd into `test/smoke` folder and run:

```sh
cd test/smoke
CI=1 SMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN shellspec -f d
```
- [Shellspec](https://shellspec.info)
- [jq](https://stedolan.github.io/jq/)
- timeout (if not available on your platform)
2. Run:
```sh
cd test/smoke
CI=1 SMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN shellspec -f d
```

## TODO

Expand Down
21 changes: 20 additions & 1 deletion test/smoke/spec/snyk_test_spec.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,28 @@ Describe "Snyk test command"
snyk test
}

run_test_in_empty_subfolder() {
cd ../fixtures/empty || return
snyk test
}

It "throws error when file does not exist"
When run snyk test --file=non-existent/package.json
The status should equal 2
The output should include "Could not find the specified file"
The stderr should equal ""
End

It "throws error when no suppored manifests detected"
When run run_test_in_empty_subfolder
The status should equal 3
The output should include "Could not detect supported target files in"
The stderr should equal ""
End

It "finds vulns in a project in the same folder"
When run run_test_in_subfolder
The status should be failure # issues found
The status should equal 1
The output should include "https://snyk.io/vuln/npm:minimatch:20160620"
The stderr should equal ""
End
Expand Down
114 changes: 114 additions & 0 deletions test/system/cli-json-file-output.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
import { test } from 'tap';
import { exec } from 'child_process';
import { sep, join } from 'path';
import { readFileSync, unlinkSync, rmdirSync, mkdirSync, existsSync } from 'fs';
import { v4 as uuidv4 } from 'uuid';

const osName = require('os-name');

const main = './dist/cli/index.js'.replace(/\//g, sep);
const iswindows =
osName()
.toLowerCase()
.indexOf('windows') === 0;

test('`test --json-file-output can save JSON output to file while sending human readable output to stdout`', (t) => {
t.plan(2);

exec(
`node ${main} test --json-file-output=snyk-direct-json-test-output.json`,
(err, stdout) => {
if (err) {
throw err;
}
t.match(stdout, 'Organization:', 'contains human readable output');
const outputFileContents = readFileSync(
'snyk-direct-json-test-output.json',
'utf-8',
);
unlinkSync('./snyk-direct-json-test-output.json');
const jsonObj = JSON.parse(outputFileContents);
const okValue = jsonObj.ok as boolean;
t.ok(okValue, 'JSON output ok');
},
);
});

test('`test --json-file-output produces same JSON output as normal JSON output to stdout`', (t) => {
t.plan(1);

exec(
`node ${main} test --json --json-file-output=snyk-direct-json-test-output.json`,
(err, stdout) => {
if (err) {
throw err;
}
const stdoutJson = stdout;
const outputFileContents = readFileSync(
'snyk-direct-json-test-output.json',
'utf-8',
);
unlinkSync('./snyk-direct-json-test-output.json');
t.equals(stdoutJson, outputFileContents);
},
);
});

test('`test --json-file-output can handle a relative path`', (t) => {
t.plan(1);

// if 'test-output' doesn't exist, created it
if (!existsSync('test-output')) {
mkdirSync('test-output');
}

const tempFolder = uuidv4();
const outputPath = `test-output/${tempFolder}/snyk-direct-json-test-output.json`;

exec(
`node ${main} test --json --json-file-output=${outputPath}`,
(err, stdout) => {
if (err) {
throw err;
}
const stdoutJson = stdout;
const outputFileContents = readFileSync(outputPath, 'utf-8');
unlinkSync(outputPath);
rmdirSync(`test-output/${tempFolder}`);
t.equals(stdoutJson, outputFileContents);
},
);
});

test(
'`test --json-file-output can handle an absolute path`',
{ skip: iswindows },
(t) => {
t.plan(1);

// if 'test-output' doesn't exist, created it
if (!existsSync('test-output')) {
mkdirSync('test-output');
}

const tempFolder = uuidv4();
const outputPath = join(
process.cwd(),
`test-output/${tempFolder}/snyk-direct-json-test-output.json`,
);

exec(
`node ${main} test --json --json-file-output=${outputPath}`,
(err, stdout) => {
if (err) {
throw err;
}
const stdoutJson = stdout;
const outputFileContents = readFileSync(outputPath, 'utf-8');
unlinkSync(outputPath);
rmdirSync(`test-output/${tempFolder}`);
t.equals(stdoutJson, outputFileContents);
},
);
},
);

0 comments on commit 809a73f

Please sign in to comment.