Skip to content

Commit

Permalink
feat: split up some logic for reuse for test
Browse files Browse the repository at this point in the history
  • Loading branch information
@lili2311
lili2311 authored and Mila Votradovec committed Aug 13, 2018
1 parent 4ccfdfe commit d3a575a
Show file tree
Hide file tree
Showing 4 changed files with 167 additions and 125 deletions.
6 changes: 2 additions & 4 deletions lib/detect.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ var chalk = require('chalk');

var DETECTABLE_FILES = [
'yarn.lock',
'package-lock.json',
'package.json',
'package-lock.json',
'Gemfile',
Expand All @@ -30,17 +29,16 @@ var DETECTABLE_FILES = [

// when file is specified with --file, we look it up here
var DETECTABLE_PACKAGE_MANAGERS = {
'Gemfile': 'rubygems',
Gemfile: 'rubygems',
'Gemfile.lock': 'rubygems',
'.gemspec': 'rubygems',
'package-lock.json': 'npm',
'package.json': 'npm',
'package-lock.json': 'npm',
'pom.xml': 'maven',
'build.gradle': 'gradle',
'build.sbt': 'sbt',
'yarn.lock': 'yarn',
'Pipfile': 'pip',
Pipfile: 'pip',
'requirements.txt': 'pip',
'Gopkg.lock': 'golangdep',
'vendor.json': 'govendor',
Expand Down
1 change: 0 additions & 1 deletion lib/plugins/index.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
module.exports = {
loadPlugin: loadPlugin,
};
var debug = require('debug')('snyk');

function loadPlugin(packageManager, options) {
if (options && options.docker) {
Expand Down
284 changes: 164 additions & 120 deletions lib/snyk-test/npm/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,26 +12,10 @@ var _ = require('lodash');
var analytics = require('../../analytics');
var common = require('../common');
var fileSystem = require('fs');

// important: this is different from ./config (which is the *user's* config)
var config = require('../../config');
var lockFileParser = require('snyk-nodejs-lockfile-parser');

module.exports = test;

var debug = require('debug')('snyk');
var request = require('../../request');
var path = require('path');
var fs = require('then-fs');
var snyk = require('../..');
var spinner = require('../../spinner');
var moduleToObject = require('snyk-module');
var isCI = require('../../is-ci');
var _ = require('lodash');
var analytics = require('../../analytics');
var common = require('../common');
var fileSystem = require('fs');
var lockFileParser = require('/Users/lili/www/lockfile-parser');

// important: this is different from ./config (which is the *user's* config)
var config = require('../../config');

Expand All @@ -41,40 +25,96 @@ function test(root, options) {
if (options.file.endsWith('package-lock.json')) {
return generateDependenciesFromLockfile(root, options);
}
return getDependenciesFromNodeModules(root, options)
return getDependenciesFromNodeModules(root, options);
}

function generateDependenciesFromLockfile(root, options) {
debug("Lockfile detected, generating dependency tree from lockfile");
const targetFileFullPath = path.resolve(root, 'package.json');
const lockFileFullPath = path.resolve(root, options.file);
debug('Lockfile detected, generating dependency tree from lockfile');
// TODO: below needs to be tested once we get a tree back from
// the new nodejs-lockfile-parser lib
// logic should be exactly the same as the
// getDependenciesFromNodeModules once the depTree is back

var targetFileFullPath = path.resolve(root, 'package.json');
var lockFileFullPath = path.resolve(root, options.file);

if (!fileSystem.existsSync(targetFileFullPath)) {
throw new Error('LockFile package.json not found at location: ' +
targetFileFullPath);
}
if (!fileSystem.existsSync(lockFileFullPath)) {
throw new Error(`LockFile package-lock.json not found at location: ${lockFileFullPath}`);
throw new Error('LockFile package-lock.json not found at location: ' +
lockFileFullPath);
}

if (!targetFileFullPath && lockFileFullPath) {
throw new Error('Detected a lockfile at location: '
+ lockFileFullPath + '\n However the package.json is missing!')
+ lockFileFullPath + '\n However the package.json is missing!');
}

const targetFile = fileSystem.readFileSync(targetFileFullPath);
const lockFile = fileSystem.readFileSync(lockFileFullPath);
var targetFile = fileSystem.readFileSync(targetFileFullPath);
var lockFile = fileSystem.readFileSync(lockFileFullPath);

analytics.add('local', true);

var resolveModuleSpinnerLabel = 'Analyzing npm dependencies for ' +
lockFileFullPath;
debug(resolveModuleSpinnerLabel)
var p = Promise.resolve();
p = spinner(resolveModuleSpinnerLabel)
debug(resolveModuleSpinnerLabel);
var payload = {
// options.vulnEndpoint is only used for file system tests
url: config.API + (options.vulnEndpoint || '/vuln/npm'),
json: true,
headers: {
'x-is-ci': isCI,
authorization: 'token ' + snyk.api,
},
};
var modules;
options.hasDevDependencies = false;
var policyLocations = [options['policy-path'] || root];
const p = spinner(resolveModuleSpinnerLabel)
.then(function () {
return lockFileParser(targetFile, lockFile, options);
return lockFileParser.buildDepTree(targetFile, lockFile, options);
})
.then(function (pkg) {
modules = pkg;
// if there's no package name, let's get it from the root dir
if (!pkg.name) {
pkg.name = path.basename(path.resolve(root));
}
policyLocations = policyLocations.concat(pluckPolicies(pkg));
debug('policies found', policyLocations);
analytics.add('policies', policyLocations.length);
options.hasDevDependencies = pkg.hasDevDependencies;
payload.method = 'POST';
payload.body = pkg;
payload.qs = common.assembleQueryString(options);

// load all relevant policies, apply relevant options
return snyk.policy.load(policyLocations, options)
.then(function (policy) {
payload.body.policy = policy.toString();
return {
package: pkg,
payload: payload,
};
}, function (error) { // note: inline catch, to handle error from .load
// the .snyk file wasn't found, which is fine, so we'll return
if (error.code === 'ENOENT') {
return {
package: pkg,
payload: payload,
};
}
throw error;
});
})
.then(
spinner.clear(resolveModuleSpinnerLabel)
);

return queryForVulns(p, modules, options);

}


Expand All @@ -85,7 +125,7 @@ function getDependenciesFromNodeModules(root, options) {
]).then(function (res) {
var exists = res[0];
var nodeModulesExist = res[1];
var hasDevDependencies = false;
options.hasDevDependencies = false;

if (exists && !nodeModulesExist) {
// throw a custom error
Expand All @@ -111,6 +151,7 @@ function getDependenciesFromNodeModules(root, options) {
var policyLocations = [options['policy-path'] || root];
analytics.add('local', exists);
var modules = null;
options.root = root;
if (exists) {
var resolveModuleSpinnerLabel = 'Analyzing npm dependencies for ' +
path.relative('.', path.join(root, options.file));
Expand All @@ -128,7 +169,7 @@ function getDependenciesFromNodeModules(root, options) {
policyLocations = policyLocations.concat(pluckPolicies(pkg));
debug('policies found', policyLocations);
analytics.add('policies', policyLocations.length);
hasDevDependencies = pkg.hasDevDependencies;
options.hasDevDependencies = pkg.hasDevDependencies;
payload.method = 'POST';
payload.body = pkg;
payload.qs = common.assembleQueryString(options);
Expand Down Expand Up @@ -166,108 +207,111 @@ function getDependenciesFromNodeModules(root, options) {
};
});
}
return queryForVulns(p, modules, options);
});

var lbl = 'Querying vulnerabilities database...';
return p.then(function (data) {
return spinner(lbl).then(function () {
return data;
});
})
.then(function (data) {
var filesystemPolicy = data.payload.body && !!data.payload.body.policy;
analytics.add('packageManager', 'npm');
analytics.add('packageName', data.package.name);
analytics.add('packageVersion', data.package.version);
analytics.add('package', data.package.name + '@' + data.package.version);

return new Promise(function (resolve, reject) {
request(data.payload, function (error, res, body) {
if (error) {
return reject(error);
}
return promise;
}

if (res.statusCode !== 200) {
var err = new Error(body && body.error ?
body.error :
res.statusCode);

err.cliMessage = body && body.cliMessage;
// this is the case where a local module has been tested, but
// doesn't have any production deps, but we've noted that they
// have dep deps, so we'll error with a more useful message
if (res.statusCode === 404 && hasDevDependencies) {
err.code = 'NOT_FOUND_HAS_DEV_DEPS';
} else {
err.code = res.statusCode;
}
function queryForVulns(p, modules, options) {
var lbl = 'Querying vulnerabilities database...';
return p.then(function (data) {
return spinner(lbl).then(function () {
return data;
});
})
.then(function (data) {
var filesystemPolicy = data.payload.body && !!data.payload.body.policy;
analytics.add('packageManager', 'npm');
analytics.add('packageName', data.package.name);
analytics.add('packageVersion', data.package.version);
analytics.add('package', data.package.name + '@' + data.package.version);

return new Promise(function (resolve, reject) {
request(data.payload, function (error, res, body) {
if (error) {
return reject(error);
}

if (res.statusCode === 500) {
debug('Server error', body.stack);
}
if (res.statusCode !== 200) {
var err = new Error(body && body.error ?
body.error :
res.statusCode);

err.cliMessage = body && body.cliMessage;
// this is the case where a local module has been tested, but
// doesn't have any production deps, but we've noted that they
// have dep deps, so we'll error with a more useful message
if (res.statusCode === 404 && options.hasDevDependencies) {
err.code = 'NOT_FOUND_HAS_DEV_DEPS';
} else {
err.code = res.statusCode;
}

return reject(err);
if (res.statusCode === 500) {
debug('Server error', body.stack);
}

body.filesystemPolicy = filesystemPolicy;
return reject(err);
}

body.filesystemPolicy = filesystemPolicy;

resolve(body);
});
resolve(body);
});
}).then(function (res) {
if (modules) {
res.dependencyCount = modules.numDependencies;
if (res.vulnerabilities) {
res.vulnerabilities.forEach(function (vuln) {
var plucked = modules.pluck(vuln.from, vuln.name, vuln.version);
vuln.__filename = plucked.__filename;
vuln.shrinkwrap = plucked.shrinkwrap;
vuln.bundled = plucked.bundled;

// this is an edgecase when we're testing the directly vuln pkg
if (vuln.from.length === 1) {
return;
}
});
}).then(function (res) {
if (modules) {
res.dependencyCount = modules.numDependencies;
if (res.vulnerabilities) {
res.vulnerabilities.forEach(function (vuln) {
var plucked = modules.pluck(vuln.from, vuln.name, vuln.version);
vuln.__filename = plucked.__filename;
vuln.shrinkwrap = plucked.shrinkwrap;
vuln.bundled = plucked.bundled;

// this is an edgecase when we're testing the directly vuln pkg
if (vuln.from.length === 1) {
return;
}

var parentPkg = moduleToObject(vuln.from[1]);
var parent = modules.pluck(vuln.from.slice(0, 2),
parentPkg.name,
parentPkg.version);
vuln.parentDepType = parent.depType;
});
}
var parentPkg = moduleToObject(vuln.from[1]);
var parent = modules.pluck(vuln.from.slice(0, 2),
parentPkg.name,
parentPkg.version);
vuln.parentDepType = parent.depType;
});
}
return res;
}
return res;
}).then(function (res) {
analytics.add('vulns-pre-policy', res.vulnerabilities.length);
return Promise.resolve().then(function () {
if (options['ignore-policy']) {
return res;
}

return snyk.policy.loadFromText(res.policy)
.then(function (policy) {
return policy.filter(res, options.root);
});
}).then(function (res) {
analytics.add('vulns-pre-policy', res.vulnerabilities.length);
return Promise.resolve().then(function () {
if (options['ignore-policy']) {
return res;
analytics.add('vulns', res.vulnerabilities.length);

// add the unique count of vulnerabilities found
res.uniqueCount = 0;
var seen = {};
res.uniqueCount = res.vulnerabilities.reduce(function (acc, curr) {
if (!seen[curr.id]) {
seen[curr.id] = true;
acc++;
}
return acc;
}, 0);

return snyk.policy.loadFromText(res.policy)
.then(function (policy) {
return policy.filter(res, root);
});
}).then(function (res) {
analytics.add('vulns', res.vulnerabilities.length);

// add the unique count of vulnerabilities found
res.uniqueCount = 0;
var seen = {};
res.uniqueCount = res.vulnerabilities.reduce(function (acc, curr) {
if (!seen[curr.id]) {
seen[curr.id] = true;
acc++;
}
return acc;
}, 0);

return res;
});
}).then(spinner.clear(lbl));
});

return promise;
return res;
});
}).then(spinner.clear(lbl));
}

function pluckPolicies(pkg) {
Expand Down
Loading

0 comments on commit d3a575a

Please sign in to comment.