Merge pull request #929 from snyk/feat/monitor-all-projects

feat: monitor support for --all-projects

src/cli/commands/monitor/index.ts

@@ -1,43 +1,39 @@
 export = monitor;
 
-import * as _ from 'lodash';
-import * as fs from 'then-fs';
-import { apiTokenExists } from '../../../lib/api-token';
-import snyk = require('../../../lib'); // TODO(kyegupov): fix import
-import { monitor as snykMonitor } from '../../../lib/monitor';
-import * as config from '../../../lib/config';
-import * as url from 'url';
 import chalk from 'chalk';
+import * as fs from 'then-fs';
+import * as Debug from 'debug';
 import * as pathUtil from 'path';
-import * as spinner from '../../../lib/spinner';
+import { legacyPlugin as pluginApi } from '@snyk/cli-interface';
 
+import {
+  MonitorOptions,
+  MonitorMeta,
+  MonitorResult,
+  PluginMetadata,
+  Options,
+} from '../../../lib/types';
+import * as config from '../../../lib/config';
 import * as detect from '../../../lib/detect';
-import * as plugins from '../../../lib/plugins';
-import { ModuleInfo } from '../../../lib/module-info'; // TODO(kyegupov): fix import
-import { MonitorOptions, MonitorMeta, Options } from '../../../lib/types';
+import { GoodResult, BadResult } from './types';
+import * as spinner from '../../../lib/spinner';
+import * as analytics from '../../../lib/analytics';
 import { MethodArgs, ArgsOptions } from '../../args';
+import { apiTokenExists } from '../../../lib/api-token';
 import { maybePrintDeps } from '../../../lib/print-deps';
-import * as analytics from '../../../lib/analytics';
-import { MonitorError } from '../../../lib/errors';
-import { legacyPlugin as pluginApi } from '@snyk/cli-interface';
+import { monitor as snykMonitor } from '../../../lib/monitor';
+import { processJsonMonitorResponse } from './process-json-monitor';
+import snyk = require('../../../lib'); // TODO(kyegupov): fix import
 import { formatMonitorOutput } from './formatters/format-monitor-response';
+import { getDepsFromPlugin } from '../../../lib/plugins/get-deps-from-plugin';
 import { getSubProjectCount } from '../../../lib/plugins/get-sub-project-count';
-import { processJsonMonitorResponse } from './process-json-monitor';
+import { extractPackageManager } from '../../../lib/plugins/extract-package-manager';
+import { MultiProjectResultCustom } from '../../../lib/plugins/get-multi-plugin-result';
+import { convertMultiResultToMultiCustom } from '../../../lib/plugins/convert-multi-plugin-res-to-multi-custom';
+import { convertSingleResultToMultiCustom } from '../../../lib/plugins/convert-single-splugin-res-to-multi-custom';
 
 const SEPARATOR = '\n-------------------------------------------------------\n';
-
-interface GoodResult {
-  ok: true;
-  data: string;
-  path: string;
-  projectName?: string;
-}
-
-interface BadResult {
-  ok: false;
-  data: MonitorError;
-  path: string;
-}
+const debug = Debug('snyk');
 
 // This is used instead of `let x; try { x = await ... } catch { cleanup }` to avoid
 // declaring the type of x as possibly undefined.
@@ -86,105 +82,91 @@ async function monitor(...args0: MethodArgs): Promise<any> {
 
   // Part 1: every argument is a scan target; process them sequentially
   for (const path of args as string[]) {
+    debug(`Processing ${path}...`);
     try {
       await validateMonitorPath(path, options.docker);
-
-      let packageManager = detect.detectPackageManager(path, options);
+      const guessedPackageManager = detect.detectPackageManager(path, options);
+      const analysisType =
+        (options.docker ? 'docker' : guessedPackageManager) || 'all';
 
       const targetFile =
         !options.scanAllUnmanaged && options.docker && !options.file // snyk monitor --docker (without --file)
           ? undefined
           : options.file || detect.detectPackageFile(path);
 
-      const modulePlugin = plugins.loadPlugin(packageManager, options);
-
-      const moduleInfo = ModuleInfo(modulePlugin, options.policy);
-
       const displayPath = pathUtil.relative(
         '.',
         pathUtil.join(path, targetFile || ''),
       );
 
-      const analysisType = options.docker ? 'docker' : packageManager;
-
       const analyzingDepsSpinnerLabel =
         'Analyzing ' + analysisType + ' dependencies for ' + displayPath;
 
-      const postingMonitorSpinnerLabel =
-        'Posting monitor snapshot for ' + displayPath + ' ...';
-
       await spinner(analyzingDepsSpinnerLabel);
 
       // Scan the project dependencies via a plugin
 
-      analytics.add('packageManager', packageManager);
       analytics.add('pluginOptions', options);
+      debug('getDepsFromPlugin ...');
 
       // each plugin will be asked to scan once per path
       // some return single InspectResult & newer ones return Multi
-      const inspectResult: pluginApi.InspectResult = await promiseOrCleanup(
-        moduleInfo.inspect(path, targetFile, { ...options }),
+      const inspectResult = await promiseOrCleanup(
+        getDepsFromPlugin(path, {
+          ...options,
+          path,
+          packageManager: guessedPackageManager,
+        }),
         spinner.clear(analyzingDepsSpinnerLabel),
       );
 
       analytics.add('pluginName', inspectResult.plugin.name);
 
-      await spinner.clear(analyzingDepsSpinnerLabel)(inspectResult);
-
+      const postingMonitorSpinnerLabel =
+        'Posting monitor snapshot for ' + displayPath + ' ...';
       await spinner(postingMonitorSpinnerLabel);
-      if (inspectResult.plugin.packageManager) {
-        packageManager = inspectResult.plugin.packageManager;
-      }
-      const monitorMeta: MonitorMeta = {
-        method: 'cli',
-        packageManager,
-        'policy-path': options['policy-path'],
-        'project-name': options['project-name'] || config.PROJECT_NAME,
-        isDocker: !!options.docker,
-        prune: !!options['prune-repeated-subdependencies'],
-        'experimental-dep-graph': !!options['experimental-dep-graph'],
-        'remote-repo-url': options['remote-repo-url'],
-      };
 
       // We send results from "all-sub-projects" scanning as different Monitor objects
       // multi result will become default, so start migrating code to always work with it
-      let perProjectResult: pluginApi.MultiProjectResult;
+      let perProjectResult: MultiProjectResultCustom;
       let foundProjectCount;
 
       if (!pluginApi.isMultiResult(inspectResult)) {
         foundProjectCount = getSubProjectCount(inspectResult);
-        const { plugin, meta, package: depTree } = inspectResult;
-        perProjectResult = {
-          plugin,
-          scannedProjects: [
-            {
-              depTree,
-              meta,
-            },
-          ],
-        };
+        perProjectResult = convertSingleResultToMultiCustom(inspectResult);
       } else {
-        perProjectResult = inspectResult;
+        perProjectResult = convertMultiResultToMultiCustom(inspectResult);
       }
 
       // Post the project dependencies to the Registry
       for (const projectDeps of perProjectResult.scannedProjects) {
+        const packageManager = extractPackageManager(
+          projectDeps,
+          perProjectResult,
+          options as MonitorOptions & Options,
+        );
+        analytics.add('packageManager', packageManager);
+        maybePrintDeps(options, projectDeps.depTree);
+
+        debug(`Processing ${projectDeps.depTree.name}...`);
         maybePrintDeps(options, projectDeps.depTree);
 
-        const res = await promiseOrCleanup(
+        const tFile = projectDeps.targetFile || targetFile;
+        const targetFileRelativePath = tFile
+          ? pathUtil.join(pathUtil.resolve(path), tFile)
+          : '';
+        const res: MonitorResult = await promiseOrCleanup(
           snykMonitor(
             path,
-            monitorMeta,
+            generateMonitorMeta(options, packageManager),
             projectDeps,
             options,
-            perProjectResult.plugin,
-            targetFile,
+            perProjectResult.plugin as PluginMetadata,
+            targetFileRelativePath,
           ),
           spinner.clear(postingMonitorSpinnerLabel),
         );
 
-        await spinner.clear(postingMonitorSpinnerLabel)(res);
-
         res.path = path;
         const projectName = projectDeps.depTree.name;
 
@@ -193,14 +175,16 @@ async function monitor(...args0: MethodArgs): Promise<any> {
           res,
           options,
           projectName,
-          foundProjectCount,
+          getSubProjectCount(inspectResult),
         );
         results.push({ ok: true, data: monOutput, path, projectName });
       }
       // push a good result
     } catch (err) {
       // push this error, the loop continues
       results.push({ ok: false, data: err, path });
+    } finally {
+      spinner.clearAll();
     }
   }
   // Part 2: process the output from the Registry
@@ -234,13 +218,17 @@ async function monitor(...args0: MethodArgs): Promise<any> {
   throw new Error(output);
 }
 
-function convertMultiPluginResultToSingle(
-  result: pluginApi.MultiProjectResult,
-): pluginApi.SinglePackageResult[] {
-  return result.scannedProjects.map((scannedProject) => ({
-    plugin: result.plugin,
-    package: scannedProject.depTree,
-  }));
+function generateMonitorMeta(options, packageManager?): MonitorMeta {
+  return {
+    method: 'cli',
+    packageManager,
+    'policy-path': options['policy-path'],
+    'project-name': options['project-name'] || config.PROJECT_NAME,
+    isDocker: !!options.docker,
+    prune: !!options['prune-repeated-subdependencies'],
+    'experimental-dep-graph': !!options['experimental-dep-graph'],
+    'remote-repo-url': options['remote-repo-url'],
+  };
 }
 
 async function validateMonitorPath(path, isDocker) {

src/cli/commands/test/formatters/legacy-format-issue.ts

@@ -21,7 +21,7 @@ export function formatIssues(
   options: Options & TestOptions,
 ) {
   const vulnID = vuln.list[0].id;
-  const packageManager = options.packageManager;
+  const packageManager = options.packageManager!;
   const localPackageTest = isLocalFolder(options.path);
   const uniquePackages = _.uniq(
     vuln.list.map((i) => {

src/lib/monitor/index.ts

@@ -1,5 +1,6 @@
 import * as Debug from 'debug';
 import * as depGraphLib from '@snyk/dep-graph';
+import * as cliInterface from '@snyk/cli-interface';
 import * as snyk from '..';
 import { apiTokenExists } from '../api-token';
 import request = require('../request');
@@ -8,7 +9,7 @@ import * as os from 'os';
 import * as _ from 'lodash';
 import { isCI } from '../is-ci';
 import * as analytics from '../analytics';
-import { DepTree, MonitorMeta, MonitorResult } from '../types';
+import { DepTree, MonitorMeta, MonitorResult, PluginMetadata } from '../types';
 import * as projectMetadata from '../project-metadata';
 import * as path from 'path';
 import {
@@ -24,8 +25,6 @@ import { filterOutMissingDeps } from './filter-out-missing-deps';
 import { dropEmptyDeps } from './drop-empty-deps';
 import { pruneTree } from './prune-dep-tree';
 import { pluckPolicies } from '../policy';
-import { ScannedProject } from '@snyk/cli-interface/legacy/common';
-import { PluginMetadata } from '@snyk/cli-interface/legacy/plugin';
 
 const debug = Debug('snyk');
 
@@ -62,10 +61,10 @@ interface Meta {
 export async function monitor(
   root: string,
   meta: MonitorMeta,
-  scannedProject: ScannedProject,
+  scannedProject: cliInterface.legacyCommon.ScannedProject,
   options,
   pluginMeta: PluginMetadata,
-  targetFile?: string,
+  targetFileRelativePath?: string,
 ): Promise<MonitorResult> {
   apiTokenExists();
   let treeMissingDeps: string[] = [];
@@ -93,7 +92,7 @@ export async function monitor(
         meta,
         scannedProject,
         pluginMeta,
-        targetFile,
+        targetFileRelativePath,
       );
     }
     if (monitorGraphSupportedRes.userMessage) {
@@ -132,9 +131,6 @@ export async function monitor(
   const policy = await snyk.policy.load(policyLocations, { loose: true });
 
   const target = await projectMetadata.getInfo(pkg, meta);
-  const targetFileRelativePath = targetFile
-    ? path.join(path.resolve(root), targetFile)
-    : '';
 
   if (target && target.branch) {
     analytics.add('targetBranch', target.branch);
@@ -175,6 +171,7 @@ export async function monitor(
           // we take the targetFile from the plugin,
           // because we want to send it only for specific package-managers
           target,
+          // WARNING: be careful changing this as it affects project uniqueness
           targetFile: pluginMeta.targetFile,
           targetFileRelativePath,
         } as MonitorBody,
@@ -212,9 +209,9 @@ export async function monitor(
 export async function monitorGraph(
   root: string,
   meta: MonitorMeta,
-  scannedProject: ScannedProject,
+  scannedProject: cliInterface.legacyCommon.ScannedProject,
   pluginMeta: PluginMetadata,
-  targetFile?: string,
+  targetFileRelativePath?: string,
 ): Promise<MonitorResult> {
   const packageManager = meta.packageManager;
   analytics.add('monitorGraph', true);
@@ -244,9 +241,6 @@ export async function monitorGraph(
   const policy = await snyk.policy.load(policyLocations, { loose: true });
 
   const target = await projectMetadata.getInfo(pkg, meta);
-  const targetFileRelativePath = targetFile
-    ? path.join(path.resolve(root), targetFile)
-    : '';
 
   if (target && target.branch) {
     analytics.add('targetBranch', target.branch);

src/lib/plugins/convert-multi-plugin-res-to-multi-custom.ts

@@ -0,0 +1,22 @@
+import { legacyPlugin as pluginApi } from '@snyk/cli-interface';
+import { MultiProjectResultCustom } from './get-multi-plugin-result';
+import { convertScannedProjectsToCustom } from './convert-scanned-projects-to-custom';
+import { SupportedPackageManagers } from '../package-managers';
+
+export function convertMultiResultToMultiCustom(
+  inspectRes: pluginApi.MultiProjectResult,
+  packageManager?: SupportedPackageManagers,
+  targetFile?: string,
+): MultiProjectResultCustom {
+  // convert all results from the same plugin to MultiProjectResultCustom
+  // and annotate each scannedProject with packageManager
+  return {
+    plugin: inspectRes.plugin,
+    scannedProjects: convertScannedProjectsToCustom(
+      inspectRes.scannedProjects,
+      (inspectRes.plugin.packageManager as SupportedPackageManagers) ||
+        packageManager,
+      targetFile,
+    ),
+  };
+}

src/lib/plugins/convert-scanned-projects-to-custom.ts

@@ -0,0 +1,20 @@
+import * as cliInterface from '@snyk/cli-interface';
+import { ScannedProjectCustom } from './get-multi-plugin-result';
+import { SupportedPackageManagers } from '../package-managers';
+
+export function convertScannedProjectsToCustom(
+  scannedProjects: cliInterface.legacyCommon.ScannedProject[],
+  packageManager?: SupportedPackageManagers,
+  targetFile?: string,
+): ScannedProjectCustom[] {
+  // annotate the package manager & targetFile to be used
+  // for test & monitor
+  return scannedProjects.map((a) => {
+    (a as ScannedProjectCustom).targetFile = a.targetFile || targetFile;
+    (a as ScannedProjectCustom).packageManager = (a as ScannedProjectCustom)
+      .packageManager
+      ? (a as ScannedProjectCustom).packageManager
+      : (packageManager as SupportedPackageManagers);
+    return a as ScannedProjectCustom;
+  });
+}