diff --git a/.circleci/config.yml b/.circleci/config.yml index 903122c805..491c517f09 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,7 +78,7 @@ commands: install_sbt_unix: description: Install SBT steps: - - run: + - run: name: Installing sbt command: sdk install sbt 1.3.12 install_node_npm: @@ -109,6 +109,12 @@ commands: - run: name: NPM version command: npm --version + generate_help: + description: Generate CLI help files + steps: + - run: + name: Run CLI help text builder + command: npm run generate-help jobs: regression-test: @@ -117,9 +123,13 @@ jobs: - image: circleci/node:<< parameters.node_version >> steps: - checkout + - setup_remote_docker: + version: 19.03.13 + # docker_layer_caching: true - install_shellspec - install_deps - build_ts + - generate_help - run: name: Run auth command: npm run snyk-auth @@ -213,7 +223,11 @@ jobs: resource_class: small steps: - checkout + - setup_remote_docker: + version: 19.03.13 + # docker_layer_caching: true - install_deps + - generate_help - run: sudo npm i -g semantic-release @semantic-release/exec pkg - run: sudo apt-get install -y osslsigncode - run: diff --git a/.gitignore b/.gitignore index 1f7c828e00..f76c156464 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ snyk_report.html !/docker/snyk_report.css cert.pem key.pem +help/commands-md +help/commands-txt +help/commands-man # Diagnostic reports (https://nodejs.org/api/report.html) report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json diff --git a/help/README.md b/help/README.md new file mode 100644 index 0000000000..41539cc4b3 --- /dev/null +++ b/help/README.md @@ -0,0 +1,57 @@ +# CLI Help files + +Snyk CLI help files are generated from markdown sources in `help/commands-docs` folder. + +There is a simple templating system that pieces markdown sources together. Those are later transformed into a [roff (man-pages) format](). Those are then saved as plaintext to be used by `--help` argument. + +1. Markdown fragments +2. Markdown documents for each command +3. roff man pages +4. plain text version of man page + +Since [package.json supports specifying man files](https://docs.npmjs.com/cli/v6/configuring-npm/package-json#man), they will get exposed under `man snyk`. + +This system improves authoring, as markdown is easier to format. It's keeping the docs consistent and exposes them through `man` command. + +## Updating or adding help documents + +Contact **Team Hammer** or open an issue in this repository when in doubt. + +Keep all changes in `help/commands-docs` folder, as other folders are ignored by `.gitignore` file and are auto-generated in CI pipeline. + +See other documents and help files for hints on how to format arguments. Keep formatting simple, as the transformation to `roff` might have issues with complex structures. + +### CLI options + +```md +- `--severity-threshold`=low|medium|high: + Only report vulnerabilities of provided level or higher. +``` + +CLI flag should be in backticks. Options (filenames, org names…) should use Keyword extension (see below) and literal options (true|false, low|medium|high…) should be typed as above. + +### Keyword extension + +There is one non-standard markdown extension: + +```md + +``` + +Visually, it'll get rendered as underlined text. It's used to mark a "variable". For example this command flag: + +```md +- `--sarif-file-output`=: + (only in `test` command) + Save test output in SARIF format directly to the file, regardless of whether or not you use the `--sarif` option. + This is especially useful if you want to display the human-readable test output via stdout and at the same time save the SARIF format output to a file. +``` + +## Running locally + +- have docker running +- have `npm`/`npx` available + +``` +$ npm run generate-help +``` diff --git a/help/generator/generate-docs.sh b/help/generator/generate-docs.sh new file mode 100755 index 0000000000..faa8ecb194 --- /dev/null +++ b/help/generator/generate-docs.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -e + +IMAGE_NAME=ronn-ng + +if ! docker inspect --type=image $IMAGE_NAME >/dev/null 2>&1; then + echo "Docker image $IMAGE_NAME not found, building..." + docker build -t $IMAGE_NAME -f ./help/generator/ronn-ng.dockerfile ./help +fi + +echo "Running npx command to run help generator" +RONN_COMMAND="docker run -i ronn-ng" npx ts-node ./help/generator/generator.ts diff --git a/help/generator/generator.ts b/help/generator/generator.ts new file mode 100644 index 0000000000..912cb912bb --- /dev/null +++ b/help/generator/generator.ts @@ -0,0 +1,148 @@ +import * as path from 'path'; +import * as fs from 'fs'; +import { exec } from 'child_process'; + +const RONN_COMMAND = process.env.RONN_COMMAND || 'ronn'; +const COMMANDS: Record = { + auth: {}, + test: { + optionsFile: '_SNYK_COMMAND_OPTIONS', + }, + monitor: { + optionsFile: '_SNYK_COMMAND_OPTIONS', + }, + container: {}, + iac: {}, + config: {}, + protect: {}, + policy: {}, + ignore: {}, + wizard: {}, + help: {}, + woof: {}, +}; + +const GENERATED_MARKDOWN_FOLDER = './help/commands-md'; +const GENERATED_MAN_FOLDER = './help/commands-man'; +const GENERATED_TXT_FOLDER = './help/commands-txt'; + +function execShellCommand(cmd): Promise { + return new Promise((resolve) => { + exec(cmd, (error, stdout, stderr) => { + if (error) { + console.warn(error); + } + return resolve(stdout ? stdout : stderr); + }); + }); +} + +async function generateRoff(inputFile): Promise { + return await execShellCommand( + `cat ${inputFile} | ${RONN_COMMAND} --roff --pipe --organization=Snyk.io`, + ); +} + +async function printRoff2Txt(inputFile) { + return await execShellCommand(`cat ${inputFile} | ${RONN_COMMAND} -m`); +} + +async function processMarkdown(markdownDoc, commandName) { + const markdownFilePath = `${GENERATED_MARKDOWN_FOLDER}/${commandName}.md`; + const roffFilePath = `${GENERATED_MAN_FOLDER}/${commandName}.1`; + const txtFilePath = `${GENERATED_TXT_FOLDER}/${commandName}.txt`; + + console.info(`Generating markdown version ${commandName}.md`); + fs.writeFileSync(markdownFilePath, markdownDoc); + + console.info(`Generating roff version ${commandName}.1`); + const roffDoc = await generateRoff(markdownFilePath); + + fs.writeFileSync(roffFilePath, roffDoc); + + console.info(`Generating txt version ${commandName}.txt`); + const txtDoc = (await printRoff2Txt(markdownFilePath)) as string; + + const formattedTxtDoc = txtDoc + .replace(/(.)[\b](.)/gi, (match, firstChar, actualletter) => { + if (firstChar === '_' && actualletter !== '_') { + return `\x1b[4m${actualletter}\x1b[0m`; + } + return `\x1b[1m${actualletter}\x1b[0m`; + }) + .split('\n') + .slice(4, -4) + .join('\n'); + console.log(formattedTxtDoc); + + fs.writeFileSync(txtFilePath, formattedTxtDoc); +} + +async function run() { + // Ensure folders exists + [ + GENERATED_MAN_FOLDER, + GENERATED_MARKDOWN_FOLDER, + GENERATED_TXT_FOLDER, + ].forEach((path) => { + if (!fs.existsSync(path)) { + fs.mkdirSync(path); + } + }); + + const getMdFilePath = (filename: string) => + path.resolve(__dirname, `./../commands-docs/${filename}.md`); + + const readFile = (filename: string) => + fs.readFileSync(getMdFilePath(filename), 'utf8'); + + const readFileIfExists = (filename: string) => + fs.existsSync(getMdFilePath(filename)) ? readFile(filename) : ''; + + const _snykHeader = readFile('_SNYK_COMMAND_HEADER'); + const _snykOptions = readFile('_SNYK_COMMAND_OPTIONS'); + const _snykGlobalOptions = readFile('_SNYK_GLOBAL_OPTIONS'); + const _environment = readFile('_ENVIRONMENT'); + const _examples = readFile('_EXAMPLES'); + const _exitCodes = readFile('_EXIT_CODES'); + const _notices = readFile('_NOTICES'); + + for (const [name, { optionsFile }] of Object.entries(COMMANDS)) { + const commandDoc = readFile(name); + + // Piece together a help file for each command + const doc = `${commandDoc} + +${optionsFile ? readFileIfExists(optionsFile) : ''} + +${_snykGlobalOptions} + +${readFileIfExists(`${name}-examples`)} + +${_exitCodes} + +${_environment} + +${_notices} +`; + + await processMarkdown(doc, 'snyk-' + name); + } + + // This just slaps strings together for the global snyk help doc + const globalDoc = `${_snykHeader} + +${_snykOptions} +${_snykGlobalOptions} + +${_examples} + +${_exitCodes} + +${_environment} + +${_notices} +`; + await processMarkdown(globalDoc, 'snyk'); +} +run(); diff --git a/help/generator/ronn-ng.dockerfile b/help/generator/ronn-ng.dockerfile new file mode 100644 index 0000000000..e6dba65caa --- /dev/null +++ b/help/generator/ronn-ng.dockerfile @@ -0,0 +1,8 @@ +FROM ruby + +RUN gem install ronn-ng +RUN apt-get update && apt-get install -y groff + +ENV MANPAGER=cat + +ENTRYPOINT ["/usr/local/bundle/bin/ronn"] diff --git a/package.json b/package.json index aea69596aa..c794a6e844 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,10 @@ "README.md" ], "directories": { - "test": "test" + "lib": "src", + "test": "test", + "man": "help/commands-man", + "doc": "help/commands-help" }, "bin": { "snyk": "dist/cli/index.js" @@ -25,6 +28,7 @@ "find-circular": "npm run build && madge --circular ./dist", "format": "prettier --write '{src,test,scripts}/**/*.{js,ts}'", "prepare": "npm run build", + "generate-help": "./help/generator/generate-docs.sh", "test:common": "npm run check-tests && npm run lint && node --require ts-node/register src/cli test --org=snyk", "test:acceptance": "tap test/acceptance/**/*.test.* test/acceptance/*.test.* -Rspec --timeout=300 --node-arg=-r --node-arg=ts-node/register", "test:acceptance-windows": "tap test/acceptance/**/*.test.* -Rspec --timeout=300 --node-arg=-r --node-arg=ts-node/register", diff --git a/src/cli/commands/help.ts b/src/cli/commands/help.ts index 62aae08881..2cd48c781d 100644 --- a/src/cli/commands/help.ts +++ b/src/cli/commands/help.ts @@ -1,19 +1,30 @@ import * as fs from 'fs'; import * as path from 'path'; +const DEFAULT_HELP = 'snyk'; + export = async function help(item: string | boolean) { - if (!item || item === true || typeof item !== 'string') { - item = 'help'; + if (!item || item === true || typeof item !== 'string' || item === 'help') { + item = DEFAULT_HELP; } // cleanse the filename to only contain letters // aka: /\W/g but figured this was easier to read item = item.replace(/[^a-z-]/gi, ''); - if (!fs.existsSync(path.resolve(__dirname, '../../../help', item + '.txt'))) { - item = 'help'; + try { + const filename = path.resolve( + __dirname, + '../../../help/commands-txt', + item === DEFAULT_HELP ? DEFAULT_HELP + '.txt' : `snyk-${item}.txt`, + ); + return fs.readFileSync(filename, 'utf8'); + } catch (error) { + const filename = path.resolve( + __dirname, + '../../../help/commands-txt', + DEFAULT_HELP + '.txt', + ); + return fs.readFileSync(filename, 'utf8'); } - - const filename = path.resolve(__dirname, '../../../help', item + '.txt'); - return fs.readFileSync(filename, 'utf8'); };