Add support for a text manifest for differential deploys.

Add support for a text manifest that can be consulted to avoid re-uploading files that were part of the last upload. Text manifest can be generated with ember-cli-deploy-manifest.

Author: lukemelia

index.js

@@ -17,14 +17,14 @@ module.exports = {
   createDeployPlugin: function(options) {
     function _beginMessage(ui, filesToUpload, bucket) {
       ui.write(blue('|    '));
-      ui.writeLine(blue('- uploading ' + filesToUpload.length + ' files to `' + bucket + '`'));
+      ui.writeLine(blue('- preparing to upload to S3 bucket `' + bucket + '`'));
 
       return Promise.resolve();
     }
 
-    function _successMessage(ui, filesToUpload) {
+    function _successMessage(ui, filesUploaded) {
       ui.write(blue('|    '));
-      ui.writeLine(blue('- uploaded ' + filesToUpload.length + ' files ok'));
+      ui.writeLine(blue('- uploaded ' + filesUploaded.length + ' files ok'));
 
       return Promise.resolve();
     }
@@ -73,12 +73,13 @@ module.exports = {
           filePaths: filesToUpload,
           gzippedFilePaths: gzippedFiles,
           prefix: config.prefix,
-          bucket: config.bucket
+          bucket: config.bucket,
+          manifestPath: context.manifestPath
         };
 
         return _beginMessage(ui, filesToUpload, config.bucket)
           .then(s3.upload.bind(s3, options))
-          .then(_successMessage.bind(this, ui, filesToUpload))
+          .then(_successMessage.bind(this, ui))
           .catch(_errorMessage.bind(this, ui));
       }
     };

lib/s3.js

@@ -1,15 +1,14 @@
+/* global require */
 var CoreObject = require('core-object');
 var fs         = require('fs');
 var path       = require('path');
 var mime       = require('mime');
 
 var Promise    = require('ember-cli/lib/ext/promise');
-var denodeify  = Promise.denodeify;
-var readFile   = denodeify(fs.readFile);
 
+var _          = require('lodash');
 var chalk      = require('chalk');
 var blue       = chalk.blue;
-
 var EXPIRE_IN_2030               = new Date('2030');
 var TWO_YEAR_CACHE_PERIOD_IN_SEC = 60 * 60 * 24 * 365 * 2;
 
@@ -29,22 +28,64 @@ module.exports = CoreObject.extend({
 
   upload: function(options) {
     options = options || {};
+    return this._determineFilePaths(options).then(function(filePaths){
+      return Promise.all(this._putObjects(filePaths, options));
+    }.bind(this));
+  },
 
+  _determineFilePaths: function(options) {
+    var ui = this._ui;
+    var filePaths = options.filePaths || [];
+    if (typeof filePaths === 'string') {
+      filePaths = [filePaths];
+    }
+    var prefix       = options.prefix;
+    var manifestPath = options.manifestPath;
+    if (manifestPath) {
+      var key = path.join(prefix, manifestPath);
+      ui.write(blue('|    '));
+      ui.writeLine(blue("- Downloading manifest for differential deploy from `" + key + "`..."));
+      return new Promise(function(resolve, reject){
+        var params = { Bucket: options.bucket, Key: key};
+        this._client.getObject(params, function(error, data) {
+          if (error) {
+            reject(error);
+          } else {
+            resolve(data.Body.toString().split('\n'));
+          }
+        }.bind(this));
+      }.bind(this)).then(function(manifestEntries){
+      ui.write(blue('|    '));
+        ui.writeLine(blue("- Manifest found. Differential deploy will be applied."));
+        return _.difference(filePaths, manifestEntries);
+      }).catch(function(reason){
+        console.log(reason);
+        ui.write(blue('|    '));
+        ui.writeLine(blue("- Manifest not found. Disabling differential deploy."));
+        return Promise.resolve(filePaths);
+      });
+    } else {
+      return Promise.resolve(filePaths);
+    }
+  },
+
+  _putObjects: function(filePaths, options) {
     var ui               = this._ui;
     var cwd              = options.cwd;
     var bucket           = options.bucket;
     var prefix           = options.prefix;
     var acl              = options.acl || 'public-read';
-    var filePaths        = options.filePaths || [];
     var gzippedFilePaths = options.gzippedFilePaths || [];
     var cacheControl     = 'max-age='+TWO_YEAR_CACHE_PERIOD_IN_SEC+', public';
     var expires          = EXPIRE_IN_2030;
 
-    if (typeof filePaths === 'string') {
-      filePaths = [filePaths];
+    var manifestPath = options.manifestPath;
+    var pathsToUpload = filePaths;
+    if (manifestPath) {
+      pathsToUpload.push(manifestPath);
     }
 
-    var promises = filePaths.map(function(filePath) {
+    return pathsToUpload.map(function(filePath) {
       var basePath    = path.join(cwd, filePath);
       var data        = fs.readFileSync(basePath);
       var contentType = mime.lookup(basePath);
@@ -71,13 +112,10 @@ module.exports = CoreObject.extend({
           } else {
             ui.write(blue('|    '));
             ui.writeLine(blue('- ✔  ' + key));
-
-            resolve();
+            resolve(filePath);
           }
         });
       }.bind(this));
     }.bind(this));
-
-    return Promise.all(promises);
   }
 });

package.json

@@ -46,6 +46,7 @@
     "chalk": "^1.0.0",
     "core-object": "^1.1.0",
     "ember-cli-babel": "^5.0.0",
+    "lodash": "^3.9.3",
     "mime": "^1.3.4",
     "minimatch": "^2.0.4"
   },

tests/fixtures/dist/manifest.txt

@@ -0,0 +1,2 @@
+app.css
+app.js

tests/unit/index-nodetest.js

@@ -92,16 +92,7 @@ describe('s3 plugin', function() {
       return assert.isFulfilled(plugin.upload.call(plugin, context))
         .then(function() {
           assert.equal(mockUi.messages.length, 4);
-
-          var messages = mockUi.messages.reduce(function(previous, current) {
-            if (/- uploading 2 files to `cccc`/.test(current)) {
-              previous.push(current);
-            }
-
-            return previous;
-          }, []);
-
-          assert.equal(messages.length, 1);
+          assert.match(mockUi.messages[0], /preparing to upload to S3 bucket `cccc`/);
         });
     });
 

tests/unit/lib/s3-nodetest.js

@@ -111,5 +111,79 @@ describe('s3', function() {
           });
       });
     });
+
+    describe('with a manifestPath specified', function () {
+      it('uploads all files when manifest is missing from server', function (done) {
+        var subject = new S3({
+          ui: mockUi,
+          client: {
+            putObject: function(params, cb) {
+              cb();
+            },
+            getObject: function(params, cb){
+              cb(new Error("File not found"));
+            }
+          }
+        });
+        var options = {
+          filePaths: ['app.js', 'app.css'],
+          cwd: process.cwd() + '/tests/fixtures/dist',
+          prefix: 'js-app',
+          manifestPath: 'manifest.txt'
+        };
+
+        var promise = subject.upload(options);
+
+        return assert.isFulfilled(promise)
+          .then(function() {
+            assert.equal(mockUi.messages.length, 5);
+            assert.match(mockUi.messages[0], /- Downloading manifest for differential deploy.../);
+            assert.match(mockUi.messages[1], /- Manifest not found. Disabling differential deploy\./);
+            assert.match(mockUi.messages[2], /- ✔  js-app\/app\.js/);
+            assert.match(mockUi.messages[3], /- ✔  js-app\/app\.css/);
+            assert.match(mockUi.messages[4], /- ✔  js-app\/manifest\.txt/);
+            done();
+          }).catch(function(reason){
+            done(reason);
+          });
+      });
+
+      it('only uploads missing files when manifest is present on server', function (done) {
+        var subject = new S3({
+          ui: mockUi,
+          client: {
+            putObject: function(params, cb) {
+              cb();
+            },
+            getObject: function(params, cb){
+              cb(undefined, {
+                Body: "app.js"
+              });
+            }
+          }
+        });
+
+        var options = {
+          filePaths: ['app.js', 'app.css'],
+          cwd: process.cwd() + '/tests/fixtures/dist',
+          prefix: 'js-app',
+          manifestPath: 'manifest.txt'
+        };
+
+        var promise = subject.upload(options);
+
+        return assert.isFulfilled(promise)
+          .then(function() {
+            assert.equal(mockUi.messages.length, 4);
+            assert.match(mockUi.messages[0], /- Downloading manifest for differential deploy.../);
+            assert.match(mockUi.messages[1], /- Manifest found. Differential deploy will be applied\./);
+            assert.match(mockUi.messages[2], /- ✔  js-app\/app\.css/);
+            assert.match(mockUi.messages[3], /- ✔  js-app\/manifest\.txt/);
+            done();
+          }).catch(function(reason){
+            done(reason);
+          });
+      });
+    });
   });
 });