diff --git a/lib/detect.js b/lib/detect.js index 79d77727fc..5bfd2bd398 100644 --- a/lib/detect.js +++ b/lib/detect.js @@ -10,7 +10,6 @@ var chalk = require('chalk'); var DETECTABLE_FILES = [ 'yarn.lock', - 'package-lock.json', 'package.json', 'package-lock.json', 'Gemfile', @@ -30,17 +29,16 @@ var DETECTABLE_FILES = [ // when file is specified with --file, we look it up here var DETECTABLE_PACKAGE_MANAGERS = { - 'Gemfile': 'rubygems', + Gemfile: 'rubygems', 'Gemfile.lock': 'rubygems', '.gemspec': 'rubygems', - 'package-lock.json': 'npm', 'package.json': 'npm', 'package-lock.json': 'npm', 'pom.xml': 'maven', 'build.gradle': 'gradle', 'build.sbt': 'sbt', 'yarn.lock': 'yarn', - 'Pipfile': 'pip', + Pipfile: 'pip', 'requirements.txt': 'pip', 'Gopkg.lock': 'golangdep', 'vendor.json': 'govendor', diff --git a/lib/plugins/index.js b/lib/plugins/index.js index 3657dbc254..ee5181e8e4 100644 --- a/lib/plugins/index.js +++ b/lib/plugins/index.js @@ -1,7 +1,6 @@ module.exports = { loadPlugin: loadPlugin, }; -var debug = require('debug')('snyk'); function loadPlugin(packageManager, options) { if (options && options.docker) { diff --git a/lib/snyk-test/npm/index.js b/lib/snyk-test/npm/index.js index 8e4caf640f..83cf4fc015 100644 --- a/lib/snyk-test/npm/index.js +++ b/lib/snyk-test/npm/index.js @@ -12,26 +12,10 @@ var _ = require('lodash'); var analytics = require('../../analytics'); var common = require('../common'); var fileSystem = require('fs'); - -// important: this is different from ./config (which is the *user's* config) -var config = require('../../config'); +var lockFileParser = require('snyk-nodejs-lockfile-parser'); module.exports = test; -var debug = require('debug')('snyk'); -var request = require('../../request'); -var path = require('path'); -var fs = require('then-fs'); -var snyk = require('../..'); -var spinner = require('../../spinner'); -var moduleToObject = require('snyk-module'); -var isCI = require('../../is-ci'); -var _ = require('lodash'); -var analytics = require('../../analytics'); -var common = require('../common'); -var fileSystem = require('fs'); -var lockFileParser = require('/Users/lili/www/lockfile-parser'); - // important: this is different from ./config (which is the *user's* config) var config = require('../../config'); @@ -41,40 +25,96 @@ function test(root, options) { if (options.file.endsWith('package-lock.json')) { return generateDependenciesFromLockfile(root, options); } - return getDependenciesFromNodeModules(root, options) + return getDependenciesFromNodeModules(root, options); } function generateDependenciesFromLockfile(root, options) { - debug("Lockfile detected, generating dependency tree from lockfile"); - const targetFileFullPath = path.resolve(root, 'package.json'); - const lockFileFullPath = path.resolve(root, options.file); + debug('Lockfile detected, generating dependency tree from lockfile'); + // TODO: below needs to be tested once we get a tree back from + // the new nodejs-lockfile-parser lib + // logic should be exactly the same as the + // getDependenciesFromNodeModules once the depTree is back + + var targetFileFullPath = path.resolve(root, 'package.json'); + var lockFileFullPath = path.resolve(root, options.file); if (!fileSystem.existsSync(targetFileFullPath)) { throw new Error('LockFile package.json not found at location: ' + targetFileFullPath); } if (!fileSystem.existsSync(lockFileFullPath)) { - throw new Error(`LockFile package-lock.json not found at location: ${lockFileFullPath}`); + throw new Error('LockFile package-lock.json not found at location: ' + + lockFileFullPath); } if (!targetFileFullPath && lockFileFullPath) { throw new Error('Detected a lockfile at location: ' - + lockFileFullPath + '\n However the package.json is missing!') + + lockFileFullPath + '\n However the package.json is missing!'); } - const targetFile = fileSystem.readFileSync(targetFileFullPath); - const lockFile = fileSystem.readFileSync(lockFileFullPath); + var targetFile = fileSystem.readFileSync(targetFileFullPath); + var lockFile = fileSystem.readFileSync(lockFileFullPath); analytics.add('local', true); var resolveModuleSpinnerLabel = 'Analyzing npm dependencies for ' + lockFileFullPath; - debug(resolveModuleSpinnerLabel) - var p = Promise.resolve(); - p = spinner(resolveModuleSpinnerLabel) + debug(resolveModuleSpinnerLabel); + var payload = { + // options.vulnEndpoint is only used for file system tests + url: config.API + (options.vulnEndpoint || '/vuln/npm'), + json: true, + headers: { + 'x-is-ci': isCI, + authorization: 'token ' + snyk.api, + }, + }; + var modules; + options.hasDevDependencies = false; + var policyLocations = [options['policy-path'] || root]; + const p = spinner(resolveModuleSpinnerLabel) .then(function () { - return lockFileParser(targetFile, lockFile, options); + return lockFileParser.buildDepTree(targetFile, lockFile, options); }) + .then(function (pkg) { + modules = pkg; + // if there's no package name, let's get it from the root dir + if (!pkg.name) { + pkg.name = path.basename(path.resolve(root)); + } + policyLocations = policyLocations.concat(pluckPolicies(pkg)); + debug('policies found', policyLocations); + analytics.add('policies', policyLocations.length); + options.hasDevDependencies = pkg.hasDevDependencies; + payload.method = 'POST'; + payload.body = pkg; + payload.qs = common.assembleQueryString(options); + + // load all relevant policies, apply relevant options + return snyk.policy.load(policyLocations, options) + .then(function (policy) { + payload.body.policy = policy.toString(); + return { + package: pkg, + payload: payload, + }; + }, function (error) { // note: inline catch, to handle error from .load + // the .snyk file wasn't found, which is fine, so we'll return + if (error.code === 'ENOENT') { + return { + package: pkg, + payload: payload, + }; + } + throw error; + }); + }) + .then( + spinner.clear(resolveModuleSpinnerLabel) + ); + + return queryForVulns(p, modules, options); + } @@ -85,7 +125,7 @@ function getDependenciesFromNodeModules(root, options) { ]).then(function (res) { var exists = res[0]; var nodeModulesExist = res[1]; - var hasDevDependencies = false; + options.hasDevDependencies = false; if (exists && !nodeModulesExist) { // throw a custom error @@ -111,6 +151,7 @@ function getDependenciesFromNodeModules(root, options) { var policyLocations = [options['policy-path'] || root]; analytics.add('local', exists); var modules = null; + options.root = root; if (exists) { var resolveModuleSpinnerLabel = 'Analyzing npm dependencies for ' + path.relative('.', path.join(root, options.file)); @@ -128,7 +169,7 @@ function getDependenciesFromNodeModules(root, options) { policyLocations = policyLocations.concat(pluckPolicies(pkg)); debug('policies found', policyLocations); analytics.add('policies', policyLocations.length); - hasDevDependencies = pkg.hasDevDependencies; + options.hasDevDependencies = pkg.hasDevDependencies; payload.method = 'POST'; payload.body = pkg; payload.qs = common.assembleQueryString(options); @@ -166,108 +207,111 @@ function getDependenciesFromNodeModules(root, options) { }; }); } + return queryForVulns(p, modules, options); + }); - var lbl = 'Querying vulnerabilities database...'; - return p.then(function (data) { - return spinner(lbl).then(function () { - return data; - }); - }) - .then(function (data) { - var filesystemPolicy = data.payload.body && !!data.payload.body.policy; - analytics.add('packageManager', 'npm'); - analytics.add('packageName', data.package.name); - analytics.add('packageVersion', data.package.version); - analytics.add('package', data.package.name + '@' + data.package.version); - - return new Promise(function (resolve, reject) { - request(data.payload, function (error, res, body) { - if (error) { - return reject(error); - } + return promise; +} - if (res.statusCode !== 200) { - var err = new Error(body && body.error ? - body.error : - res.statusCode); - - err.cliMessage = body && body.cliMessage; - // this is the case where a local module has been tested, but - // doesn't have any production deps, but we've noted that they - // have dep deps, so we'll error with a more useful message - if (res.statusCode === 404 && hasDevDependencies) { - err.code = 'NOT_FOUND_HAS_DEV_DEPS'; - } else { - err.code = res.statusCode; - } +function queryForVulns(p, modules, options) { + var lbl = 'Querying vulnerabilities database...'; + return p.then(function (data) { + return spinner(lbl).then(function () { + return data; + }); + }) + .then(function (data) { + var filesystemPolicy = data.payload.body && !!data.payload.body.policy; + analytics.add('packageManager', 'npm'); + analytics.add('packageName', data.package.name); + analytics.add('packageVersion', data.package.version); + analytics.add('package', data.package.name + '@' + data.package.version); + + return new Promise(function (resolve, reject) { + request(data.payload, function (error, res, body) { + if (error) { + return reject(error); + } - if (res.statusCode === 500) { - debug('Server error', body.stack); - } + if (res.statusCode !== 200) { + var err = new Error(body && body.error ? + body.error : + res.statusCode); + + err.cliMessage = body && body.cliMessage; + // this is the case where a local module has been tested, but + // doesn't have any production deps, but we've noted that they + // have dep deps, so we'll error with a more useful message + if (res.statusCode === 404 && options.hasDevDependencies) { + err.code = 'NOT_FOUND_HAS_DEV_DEPS'; + } else { + err.code = res.statusCode; + } - return reject(err); + if (res.statusCode === 500) { + debug('Server error', body.stack); } - body.filesystemPolicy = filesystemPolicy; + return reject(err); + } + + body.filesystemPolicy = filesystemPolicy; - resolve(body); - }); + resolve(body); }); - }).then(function (res) { - if (modules) { - res.dependencyCount = modules.numDependencies; - if (res.vulnerabilities) { - res.vulnerabilities.forEach(function (vuln) { - var plucked = modules.pluck(vuln.from, vuln.name, vuln.version); - vuln.__filename = plucked.__filename; - vuln.shrinkwrap = plucked.shrinkwrap; - vuln.bundled = plucked.bundled; - - // this is an edgecase when we're testing the directly vuln pkg - if (vuln.from.length === 1) { - return; - } + }); + }).then(function (res) { + if (modules) { + res.dependencyCount = modules.numDependencies; + if (res.vulnerabilities) { + res.vulnerabilities.forEach(function (vuln) { + var plucked = modules.pluck(vuln.from, vuln.name, vuln.version); + vuln.__filename = plucked.__filename; + vuln.shrinkwrap = plucked.shrinkwrap; + vuln.bundled = plucked.bundled; + + // this is an edgecase when we're testing the directly vuln pkg + if (vuln.from.length === 1) { + return; + } - var parentPkg = moduleToObject(vuln.from[1]); - var parent = modules.pluck(vuln.from.slice(0, 2), - parentPkg.name, - parentPkg.version); - vuln.parentDepType = parent.depType; - }); - } + var parentPkg = moduleToObject(vuln.from[1]); + var parent = modules.pluck(vuln.from.slice(0, 2), + parentPkg.name, + parentPkg.version); + vuln.parentDepType = parent.depType; + }); } - return res; + } + return res; + }).then(function (res) { + analytics.add('vulns-pre-policy', res.vulnerabilities.length); + return Promise.resolve().then(function () { + if (options['ignore-policy']) { + return res; + } + + return snyk.policy.loadFromText(res.policy) + .then(function (policy) { + return policy.filter(res, options.root); + }); }).then(function (res) { - analytics.add('vulns-pre-policy', res.vulnerabilities.length); - return Promise.resolve().then(function () { - if (options['ignore-policy']) { - return res; + analytics.add('vulns', res.vulnerabilities.length); + + // add the unique count of vulnerabilities found + res.uniqueCount = 0; + var seen = {}; + res.uniqueCount = res.vulnerabilities.reduce(function (acc, curr) { + if (!seen[curr.id]) { + seen[curr.id] = true; + acc++; } + return acc; + }, 0); - return snyk.policy.loadFromText(res.policy) - .then(function (policy) { - return policy.filter(res, root); - }); - }).then(function (res) { - analytics.add('vulns', res.vulnerabilities.length); - - // add the unique count of vulnerabilities found - res.uniqueCount = 0; - var seen = {}; - res.uniqueCount = res.vulnerabilities.reduce(function (acc, curr) { - if (!seen[curr.id]) { - seen[curr.id] = true; - acc++; - } - return acc; - }, 0); - - return res; - }); - }).then(spinner.clear(lbl)); - }); - - return promise; + return res; + }); + }).then(spinner.clear(lbl)); } function pluckPolicies(pkg) { diff --git a/package.json b/package.json index df20c69bb3..4ae1ceb644 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "snyk-gradle-plugin": "1.3.0", "snyk-module": "1.8.2", "snyk-mvn-plugin": "1.2.0", + "snyk-nodejs-lockfile-parser": "1.1.0", "snyk-nuget-plugin": "1.6.4", "snyk-php-plugin": "1.5.1", "snyk-policy": "1.12.0",