From b559aa5e2d394f43c2a49f4af2fa707a7e39a809 Mon Sep 17 00:00:00 2001 From: Arjun Gadhia Date: Wed, 25 Mar 2015 18:11:47 +0000 Subject: [PATCH 1/6] WIP: push assets to s3 --- bin/next-build-tools.js | 2 +- package.json | 1 + tasks/deploy-hashed-assets-s3.js | 99 ++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 tasks/deploy-hashed-assets-s3.js diff --git a/bin/next-build-tools.js b/bin/next-build-tools.js index f050d719..a2b4d91d 100755 --- a/bin/next-build-tools.js +++ b/bin/next-build-tools.js @@ -13,7 +13,7 @@ var verify = require('../tasks/verify'); var destroy = require('../tasks/destroy'); var nightwatch = require('../tasks/nightwatch'); var downloadConfiguration = require('../tasks/download-configuration'); -var deployHashedAssets = require('../tasks/deploy-hashed-assets'); +var deployHashedAssets = require('../tasks/deploy-hashed-assets-s3'); function list(val) { return val.split(','); diff --git a/package.json b/package.json index c578c878..9abf0f47 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "test": "./bin/next-build-tools.js verify" }, "dependencies": { + "aws-sdk": "^2.1.19", "commander": "^2.6.0", "debug": "^2.1.1", "denodeify": "^1.2.0", diff --git a/tasks/deploy-hashed-assets-s3.js b/tasks/deploy-hashed-assets-s3.js new file mode 100644 index 00000000..08aac305 --- /dev/null +++ b/tasks/deploy-hashed-assets-s3.js @@ -0,0 +1,99 @@ +"use strict"; + +var packageJson = require(process.cwd() + '/package.json'); +var denodeify = require('denodeify'); +var normalizeName = require('../lib/normalize-name'); +var readFile = denodeify(require('fs').readFile); +var writeFile = denodeify(require('fs').writeFile); +var glob = denodeify(require('glob')); +var crypto = require('crypto'); +var basename = require('path').basename; +var aws = require('aws-sdk'); + +AWS.config.update({ + accessKeyId: process.env.aws_access, + secretAccessKey: process.env.aws_secret, + region: 'eu-west-1' +}); + +function hashAndUpload(opts) { + var file = opts.file; + var app = opts.app; + + console.log(file, app); + + return new Promise(resolve, reject) { + resolve(); + } + + // var s3bucket = new AWS.S3({params: {Bucket: 'ft-next-hashed-assets', Key: ''}}); + // var fileStream = fs.createReadStream(file); + // var params = {Key: 'myKey', Body: 'Hello!'}; + // s3bucket.upload(params, function(err, data) { + // if (err) { + // console.log("Error uploading data: ", err); + // } else { + // console.log("Successfully uploaded data to myBucket/myKey"); + // } + // }); + + // var api = 'https://api.github.com/repos/Financial-Times/next-hashed-assets/contents/'; + + // aws + +} + +module.exports = function(app) { + + app = app || normalizeName(packageJson.name, { version: false }); + + return glob(process.cwd() + '/public/*.@(css|js|map)') + .then(function(files) { + return Promise.all(files.map(function(file) { + return readFile(file) + .then(function(content) { + return { + name: basename(file), + content: content + }; + }); + })); + }) + .then(function(files) { + var mapHashName = ''; + return files + .map(function(file) { + var hash = crypto.createHash('sha1').update(file.content.toString('utf8')).digest('hex'); + file.hashedName = file.name.replace(/(.*)(\.[a-z0-9])/i, '$1-' + hash.substring(0, 8) + '$2'); + if (file.name === 'main.js.map') { + mapHashName = file.hashedName; + } + return file; + }) + .map(function(file) { + var content; + if (file.name === 'main.js') { + content = file.content.toString('utf8'); + content = content.replace('/# sourceMappingURL=/' + app + '/' + file.name + '.map', '/# sourceMappingURL=/next-hashed-assets/' + app + '/' + mapHashName); + file.content = new Buffer(content, 'utf8'); + } + return file; + }); + }) + .then(function(files) { + var promise = files.reduce(function(promise, file) { + return promise.then(function(obj) { + return hashAndUpload({ app: app, file: file }) + .then(function(file) { + obj[file.name] = file.hashedName; + return obj; + }); + }); + }, Promise.resolve({})); + return promise; + }) + .then(function(hashes) { + console.log("Writing public/asset-hashes.json"); + return writeFile(process.cwd() + '/public/asset-hashes.json', JSON.stringify(hashes, undefined, 2)); + }); +}; From 34406e8358736b1d518226f65f7ff9cce6b12534 Mon Sep 17 00:00:00 2001 From: Arjun Gadhia Date: Mon, 30 Mar 2015 14:49:21 +0100 Subject: [PATCH 2/6] Deploy hashed assets to Amazon S3 --- bin/next-build-tools.js | 10 ++++++- tasks/deploy-hashed-assets-s3.js | 48 ++++++++++++++++---------------- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/bin/next-build-tools.js b/bin/next-build-tools.js index a2b4d91d..d76614a5 100755 --- a/bin/next-build-tools.js +++ b/bin/next-build-tools.js @@ -13,7 +13,8 @@ var verify = require('../tasks/verify'); var destroy = require('../tasks/destroy'); var nightwatch = require('../tasks/nightwatch'); var downloadConfiguration = require('../tasks/download-configuration'); -var deployHashedAssets = require('../tasks/deploy-hashed-assets-s3'); +var deployHashedAssets = require('../tasks/deploy-hashed-assets'); +var deployHashedAssetsToS3 = require('../tasks/deploy-hashed-assets-s3'); function list(val) { return val.split(','); @@ -99,6 +100,13 @@ program deployHashedAssets().catch(exit); }); + program + .command('deploy-hashed-assets-s3') + .description('deploys ./hashed-assets/ to Amazon S3') + .action(function() { + deployHashedAssetsToS3().catch(exit); + }); + program .command('destroy [app]') .description('deletes the app from heroku') diff --git a/tasks/deploy-hashed-assets-s3.js b/tasks/deploy-hashed-assets-s3.js index 08aac305..c056be53 100644 --- a/tasks/deploy-hashed-assets-s3.js +++ b/tasks/deploy-hashed-assets-s3.js @@ -10,7 +10,7 @@ var crypto = require('crypto'); var basename = require('path').basename; var aws = require('aws-sdk'); -AWS.config.update({ +aws.config.update({ accessKeyId: process.env.aws_access, secretAccessKey: process.env.aws_secret, region: 'eu-west-1' @@ -19,34 +19,34 @@ AWS.config.update({ function hashAndUpload(opts) { var file = opts.file; var app = opts.app; + var bucket = 'ft-next-hashed-assets'; + var key = app + '/' + file.hashedName; - console.log(file, app); - - return new Promise(resolve, reject) { - resolve(); - } - - // var s3bucket = new AWS.S3({params: {Bucket: 'ft-next-hashed-assets', Key: ''}}); - // var fileStream = fs.createReadStream(file); - // var params = {Key: 'myKey', Body: 'Hello!'}; - // s3bucket.upload(params, function(err, data) { - // if (err) { - // console.log("Error uploading data: ", err); - // } else { - // console.log("Successfully uploaded data to myBucket/myKey"); - // } - // }); - - // var api = 'https://api.github.com/repos/Financial-Times/next-hashed-assets/contents/'; - - // aws - + return new Promise(function(resolve, reject) { + var s3bucket = new aws.S3({params: {Bucket: bucket}}); + var params = { + Key: key, + Body: file.content, + ACL: 'public-read', + CacheControl: 'public, max-age=604800000' + }; + s3bucket.upload(params, function(err, data) { + if (err) { + console.log("Error uploading data: ", err); + reject(err); + } else { + resolve({ + name: file.name, + hashedName: file.hashedName + }); + } + }); + }); } module.exports = function(app) { - app = app || normalizeName(packageJson.name, { version: false }); - + console.log('Deploying hashed assets to S3...') return glob(process.cwd() + '/public/*.@(css|js|map)') .then(function(files) { return Promise.all(files.map(function(file) { From e8e1ef042016b97885dcf15a35cbd6797e43a49e Mon Sep 17 00:00:00 2001 From: Arjun Gadhia Date: Mon, 30 Mar 2015 15:40:14 +0100 Subject: [PATCH 3/6] Deploy to both places under single API command --- README.md | 1 + bin/next-build-tools.js | 6 ------ tasks/deploy-hashed-assets-s3.js | 11 +++++++++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index d1333133..d741ed34 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ clean runs git clean -fxd deploy [app] runs haikro deployment scripts with sensible defaults for Next projects + deploy-hashed-assets deploys hashed asset files to github.io and S3 (if AWS keys set correctly) configure [options] [source] [target] downloads environment variables from next-config-vars and uploads them to the current app download-configuration downloads environment variables from app from Heroku to make adding them to the next-config-vars service easier provision [app] provisions a new instance of an application server diff --git a/bin/next-build-tools.js b/bin/next-build-tools.js index d76614a5..93d81018 100755 --- a/bin/next-build-tools.js +++ b/bin/next-build-tools.js @@ -98,12 +98,6 @@ program .description('deploys ./hashed-assets/ to on GitHub') .action(function() { deployHashedAssets().catch(exit); - }); - - program - .command('deploy-hashed-assets-s3') - .description('deploys ./hashed-assets/ to Amazon S3') - .action(function() { deployHashedAssetsToS3().catch(exit); }); diff --git a/tasks/deploy-hashed-assets-s3.js b/tasks/deploy-hashed-assets-s3.js index c056be53..10535374 100644 --- a/tasks/deploy-hashed-assets-s3.js +++ b/tasks/deploy-hashed-assets-s3.js @@ -17,6 +17,7 @@ aws.config.update({ }); function hashAndUpload(opts) { + var file = opts.file; var app = opts.app; var bucket = 'ft-next-hashed-assets'; @@ -25,7 +26,7 @@ function hashAndUpload(opts) { return new Promise(function(resolve, reject) { var s3bucket = new aws.S3({params: {Bucket: bucket}}); var params = { - Key: key, + Key: key, Body: file.content, ACL: 'public-read', CacheControl: 'public, max-age=604800000' @@ -45,8 +46,14 @@ function hashAndUpload(opts) { } module.exports = function(app) { + if(!(process.env.aws_access && process.env.aws_secret)) { + console.warn("Missing AWS keys. Hashed assets will not deploy to S3"); + return Promise.resolve(); + } + app = app || normalizeName(packageJson.name, { version: false }); - console.log('Deploying hashed assets to S3...') + + console.log('Deploying hashed assets to S3...'); return glob(process.cwd() + '/public/*.@(css|js|map)') .then(function(files) { return Promise.all(files.map(function(file) { From 91ec0e30e539bd22b121c94277c1c385862fbc45 Mon Sep 17 00:00:00 2001 From: Arjun Gadhia Date: Tue, 31 Mar 2015 11:07:20 +0100 Subject: [PATCH 4/6] Use promise.all --- bin/next-build-tools.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/next-build-tools.js b/bin/next-build-tools.js index 93d81018..aa749364 100755 --- a/bin/next-build-tools.js +++ b/bin/next-build-tools.js @@ -97,8 +97,10 @@ program .command('deploy-hashed-assets') .description('deploys ./hashed-assets/ to on GitHub') .action(function() { - deployHashedAssets().catch(exit); - deployHashedAssetsToS3().catch(exit); + Promise.all([ + deployHashedAssets(), + deployHashedAssetsToS3() + ]).catch(exit); }); program From ca3f35e9ab1ec087415962e7f124c90c5ec399cd Mon Sep 17 00:00:00 2001 From: Arjun Gadhia Date: Tue, 31 Mar 2015 11:12:06 +0100 Subject: [PATCH 5/6] Lint spaces --- bin/next-build-tools.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/next-build-tools.js b/bin/next-build-tools.js index aa749364..91f999ca 100755 --- a/bin/next-build-tools.js +++ b/bin/next-build-tools.js @@ -98,8 +98,8 @@ program .description('deploys ./hashed-assets/ to on GitHub') .action(function() { Promise.all([ - deployHashedAssets(), - deployHashedAssetsToS3() + deployHashedAssets(), + deployHashedAssetsToS3() ]).catch(exit); }); From f05715c116b0b111b3d600ea1277deed94ba47ca Mon Sep 17 00:00:00 2001 From: Arjun Gadhia Date: Tue, 31 Mar 2015 12:15:10 +0100 Subject: [PATCH 6/6] Update with prod bucket/keys --- tasks/deploy-hashed-assets-s3.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tasks/deploy-hashed-assets-s3.js b/tasks/deploy-hashed-assets-s3.js index 10535374..3417dfe6 100644 --- a/tasks/deploy-hashed-assets-s3.js +++ b/tasks/deploy-hashed-assets-s3.js @@ -11,8 +11,8 @@ var basename = require('path').basename; var aws = require('aws-sdk'); aws.config.update({ - accessKeyId: process.env.aws_access, - secretAccessKey: process.env.aws_secret, + accessKeyId: process.env.aws_access_hashed_assets, + secretAccessKey: process.env.aws_secret_hashed_assets, region: 'eu-west-1' }); @@ -20,7 +20,7 @@ function hashAndUpload(opts) { var file = opts.file; var app = opts.app; - var bucket = 'ft-next-hashed-assets'; + var bucket = 'ft-next-hashed-assets-prod'; var key = app + '/' + file.hashedName; return new Promise(function(resolve, reject) {