Refactor

.gitignore

@@ -1,3 +1,4 @@
 csv-jsons/
 node_modules/
-.env
+.env
+launch.json

index.js

@@ -1,27 +1,30 @@
+const Sharinpix = require('sharinpix');
 const express = require('express');
 const app = express();
+
 if (app.settings.env === 'development') {
     require('dotenv').config();
 }
+Sharinpix.configure(process.env.SHARINPIX_URL);
+const sharinpixInstance = Sharinpix.get_instance();
 const bodyParser = require('body-parser');
-const parseCsvImages = require('./lib/parse-csv-images');
+const parseCsv = require('./lib/csv-upload');
 const fileUpload = require('express-fileupload');
 const Stream = require('stream');
 const unirest = require('unirest');
 const jwt = require('jsonwebtoken');
-const _ = require('underscore');
 const uuidV4 = require('uuid/v4');
 const fs = require('fs');
 const jsonfile = require('jsonfile');
 const morgan = require('morgan');
+const date = new Date();
 
-app.set('json spaces', 3);
+app.set('json spaces', 2);
 
-// For Content-Type: application/x-www-form-urlencoded
 app.use(bodyParser.urlencoded({
     extended: false
 }));
-// For Content-Type: application/json
+
 app.use(bodyParser.json());
 app.use(fileUpload());
 app.use(morgan(':remote-addr | :method :url :status :res[content-length] bytes - :response-time ms'));
@@ -30,155 +33,29 @@ app.get('/', function(req, res) {
     res.sendFile('views/index.html', {root: __dirname});
 });
 
-app.post('/webhook', function (req, res) {
-    let payload = JSON.parse(req.body.p);
-    if (!payload.metadatas || !payload.metadatas.filepath || !payload.metadatas.externalId) {
-        return res.send({
-            success: false,
-            message: 'Appropriate metadatas not found, request ignored.'
-        });
-    }
-    let filepath = payload.metadatas.filepath;
-    let externalId = payload.metadatas.externalId;
-    let imageId = payload.public_id;
-    let imageWidth = payload.width;
-    let imageHeight = payload.height;
-    let albumId = payload.album.public_id;
-    let abilities = {};
-    abilities[albumId] = {
-        Access: {
-            einstein_box: true
-        }
-    };
-    let token = jwt.sign({
-        abilities: abilities,
-        iss: process.env.SHARINPIX_SECRET_ID
-    }, process.env.SHARINPIX_SECRET);
-    jsonfile.readFile(filepath, function(err, images) {
-        let image = images[externalId];
-        let imageAttributesPair = _.pairs(image.otherAttributes);
-        for (let attribute of imageAttributesPair) {
-            let enteredOnce = false;
-            (function(attribute){
-                if (/^box\d+$/.test(attribute[0]) && typeof attribute[1] === 'object') {
-                    enteredOnce = true;
-                    let box = attribute[1];
-                    let label = box.label;
-                    let percentageWidth = (box.width / imageWidth) * 100;
-                    let percentageHeight = (box.height / imageHeight) * 100;
-                    let percentageX = (box.x / imageWidth) * 100;
-                    let percentageY = (box.y / imageHeight) * 100;
-                    unirest.post(`https://${process.env.ENDPOINT_DOMAIN}/api/v1/images/${imageId}/einstein_box`).headers({
-                        'Content-Type': 'application/json',
-                        'Authorization': `Token token="${token}"`
-                    }).send({
-                        label: label,
-                        width: percentageWidth,
-                        height: percentageHeight,
-                        top: percentageY,
-                        left: percentageX
-                    }).end(function(response){
-                        if (imageAttributesPair[imageAttributesPair.length - 1] === attribute) {
-                            let csvJson = jsonfile.readFileSync(filepath);
-                            csvJson[externalId].webhookCompleted = true;
-                            jsonfile.writeFileSync(filepath, csvJson);
-                        }
-                        console.log('done');
-                    });
-                }
-            })(attribute)
-            if (!enteredOnce && imageAttributesPair[imageAttributesPair.length - 1] === attribute) {
-                let csvJson = jsonfile.readFileSync(filepath);
-                csvJson[externalId].webhookCompleted = true;
-                jsonfile.writeFileSync(filepath, csvJson);
-            }
-        }
-    });
-    res.send('Ok');
-});
-
 app.post('/upload-csv', function(req, res) {
     if (!req.files)
         return res.status(400).send('No files were uploaded.');
-
-    // The name of the input field (i.e. "sampleFile") is used to retrieve the uploaded file 
     let csvFile = req.files.csvFile;
     let albumId = req.body.albumId;
-
     let bufferStream = new Stream.PassThrough();
     bufferStream.end(csvFile.data);
-    let csvJsonId = uuidV4();
-    let outputFilePath = `${__dirname}/csv-jsons/${csvJsonId}.csv`;
+    let utcdate = ("0" + date.getUTCDate()).slice(-2);
+    let month =   ("0" + date.getMonth()).slice(-2);
+    let csvJsonId = date.getFullYear()+'-'+month+'-'+utcdate+'-'+date.getHours()+date.getMinutes()+date.getSeconds();
+    let outputFilePath = `${__dirname}/csv-jsons/${csvJsonId}.json`;
     if (!fs.existsSync(`${__dirname}/csv-jsons`)) {
         fs.mkdirSync(`${__dirname}/csv-jsons`);
     }
-    parseCsvImages(bufferStream, outputFilePath, function(imagesHash) {
-        let images = _.map(imagesHash, function(image, externalId) {
-            image['externalId'] = externalId;
-            return image;
-        });
-        process.exit;
-        let abilities = {};
-        abilities[albumId] = {
-            Access: {
-                see: true,
-                image_upload: true
-            }
-        };
-        let token = jwt.sign({
-            abilities: abilities,
-            iss: process.env.SHARINPIX_SECRET_ID
-        }, process.env.SHARINPIX_SECRET);
-        images.forEach(function(image) {
-            unirest.post('https://' + process.env.ENDPOINT_DOMAIN + '/api/v1/imports').headers({
-                'Content-Type': 'application/json',
-                'Authorization': 'Token token="' + token + '"'
-            }).send({
-                album_id: albumId,
-                filename: image.name,
-                url: image.url,
-                import_type: 'url',
-                metadatas: {
-                    externalId: image.externalId,
-                    filepath: outputFilePath
-                }
-            }).end(function (response) {
-                let csvJson = jsonfile.readFileSync(outputFilePath);
-                csvJson[image.externalId].sentForUpload = true;
-                jsonfile.writeFileSync(outputFilePath, csvJson);
-                console.log('Image `' + image.name + '` sent for import.');
-            });
-        })
-    })
-    res.redirect(`/status/${csvJsonId}`);
-});
-
-app.get('/status/:csvJsonId', function(req, res) {
-    let csvJsonId = req.params.csvJsonId;
-    if (fs.existsSync(`${__dirname}/csv-jsons/${csvJsonId}.csv`)) {
-        let csvJson = jsonfile.readFileSync(`${__dirname}/csv-jsons/${csvJsonId}.csv`);
-        let csvJsonPairs = _.pairs(csvJson);
-        let response = {};
-        for (let csvJsonPair of csvJsonPairs) {
-            response[csvJsonPair[1].name] = {
-                sentForUpload: csvJsonPair[1].sentForUpload ? true : false,
-                webhookCompleted: csvJsonPair[1].webhookCompleted ? true : false
-            }
-        }
-        res.json(response);
-    } else {
-        res.json({
-            success: false,
-            message: 'File does not exists'
-        });
-    }
+    parseCsv(albumId, bufferStream, outputFilePath, sharinpixInstance);
+    res.redirect(`/`);
 });
 
 app.use(function(req, res) {
     res.status(404);
     res.send('404 - Not found');
 });
 
-app.listen(process.env.PORT, function () {
-    console.log(`Example app listening on port ${process.env.PORT}`);
-});
+app.listen(process.env.PORT || 3000, function () {
+    console.log('listening on port '+ (process.env.PORT || 3000));    
+});

lib/csv-upload.js

@@ -0,0 +1,149 @@
+const parseCsvImages = require('./parse-csv-images');
+const _ = require('lodash');
+const async = require('async');
+
+module.exports = function(albumId, bufferStream, outputFilePath, sharinpixInstance){
+    this.sharinpixInstance = sharinpixInstance;
+    parseCsvImages(bufferStream, outputFilePath, function(imagesHash) {
+        let imagesBoxHash = imagesHash;
+        let images = _.map(imagesHash, function(image, externalId) {
+            image['externalId'] = externalId;
+            image = (({ url, name, externalId }) => ({ url, name, externalId }))(image);
+            return image;
+        });
+        var imagesBox = _.map(imagesBoxHash, function(image, externalId){
+            externalId = externalId.toString();
+            image = (({ width, height, otherAttributes }) => ({ width, height, externalId, otherAttributes }))(image);    
+            return image;
+        });
+        var imagesBoxObject =  _.keyBy(imagesBox, 'externalId');
+        let abilities = {};
+        abilities[albumId] = { Access: { see: true, image_upload: true, einstein_box: true }  };
+        let claims = { abilities: abilities };
+        parallelTasks = [];
+
+        _.each(images, function(image){
+            let body = { album_id: albumId, filename: image.name, 
+                url: image.url, import_type: 'url',
+                metadatas: { externalId: image.externalId, filepath: outputFilePath }
+            };    
+
+            parallelTasks.push(
+                function(callback) {
+                    sharinpixInstance.post('/imports', body, claims).then (
+                    function(res) {
+                        callback(null, { id: res.id, external_id: image.externalId });
+                    },
+                    function(err) {
+                        callback(body, null);
+                    });
+                }
+            );
+        })
+
+        parallelRequests(parallelTasks, 5, function(errors, results){
+            if(results !== null && results.length > 0){
+                checkImports(results);
+            }
+            if(errors !== null && errors.length > 0){
+            }
+        })
+
+        let importTasks = [];
+        function checkImports(importResults) {
+            _.each(importResults, function(imp){
+                importTasks.push(
+                    function(callback){
+                        async.retry(
+                            {
+                                errorFilter: function(err) {
+                                    return err.image_id == null;
+                                },
+                                interval: 3000
+                            }, 
+                            function(done){
+                                sharinpixInstance.get(`/imports/${imp.id}`, {admin: true}).then(
+                                    function(impResult){
+                                        if (impResult.image_id == null){
+                                            done(impResult);
+                                        }
+                                        else {
+                                            done(null, impResult);
+                                        }
+                                    },
+                                    function(impError){
+                                        done(null, {})
+                                    }
+                                )
+                            }, 
+                            function(err, result) {
+                                callback(err, result);
+                            }
+                        );
+                    }
+                )
+            })
+            parallelRequests(importTasks, 5, function(errors, results){
+                if(results !== null && results.length > 0){
+                    createBox(results)
+                }
+                if(errors !== null && errors.length > 0){
+                }
+            })
+        }
+
+        let boxTasks = [];
+        function createBox(importRes){
+            _.each(importRes, function(imp){
+                let einsteinBoxes = refineStruct(imagesBoxObject[imp.params.metadatas.externalId]);                
+                _.each(einsteinBoxes, function(box){
+                    box.image_id = imp.image_id;
+                    boxTasks.push(
+                        function(callback){
+                            sharinpixInstance.post(`/images/${box.image_id}/einstein_box`, box, claims).then(
+                                function(res){
+                                    callback(null, res);
+                                },
+                                function(err){
+                                    callback(err, null);
+                                }
+                            )
+                        }
+                    )
+                })
+                parallelRequests(boxTasks, 5, function(err, result){
+                })
+            })
+
+        }
+
+        function refineStruct(element){
+            let imageWidth = element.width;
+            let imageHeight = element.height;
+            element2 = (({otherAttributes}) => ({otherAttributes}))(element);
+            let imagesBoxFlat = _.values(_.values(element2));
+            let einsteinBoxes = _.map(imagesBoxFlat, (item) => {
+                item = _.values(item);
+                return item;
+            });
+            let x = _.flatten(einsteinBoxes);
+            let onlyboxes = _.filter(x, function(box){
+                return box != '';
+            })
+            let boxes = _.map(onlyboxes, function(box){
+                let percentageWidth = (box.width / imageWidth) * 100;
+                let percentageHeight = (box.height / imageHeight) * 100;
+                let percentageX = (box.x / imageWidth) * 100;
+                let percentageY = (box.y / imageHeight) * 100;
+                return {label: box.label, width: percentageWidth, height: percentageHeight, left: percentageX, top: percentageY };
+            })
+            return boxes;
+        }
+
+        function parallelRequests(tasks, limit, callback){
+            async.parallelLimit(tasks, limit, function(err, results){
+                callback(err, results);
+            });
+        }
+    });
+}