Halfway through in refactoring the report generation to a separate process running asynchronously of the Express server.

app.js

@@ -1,49 +1,50 @@
 'use strict';
 
-let config = require('config');
+const config = require('config');
-let log = require('npmlog');
+const log = require('npmlog');
 
-let _ = require('./lib/translate')._;
+const _ = require('./lib/translate')._;
-let util = require('util');
+const util = require('util');
 
-let express = require('express');
+const express = require('express');
-let bodyParser = require('body-parser');
+const bodyParser = require('body-parser');
-let path = require('path');
+const path = require('path');
-let favicon = require('serve-favicon');
+const favicon = require('serve-favicon');
-let logger = require('morgan');
+const logger = require('morgan');
-let cookieParser = require('cookie-parser');
+const cookieParser = require('cookie-parser');
-let session = require('express-session');
+const session = require('express-session');
-let RedisStore = require('connect-redis')(session);
+const RedisStore = require('connect-redis')(session);
-let flash = require('connect-flash');
+const flash = require('connect-flash');
-let hbs = require('hbs');
+const hbs = require('hbs');
-let compression = require('compression');
+const handlebarsHelpers = require('./lib/handlebars-helpers');
-let passport = require('./lib/passport');
+const compression = require('compression');
-let tools = require('./lib/tools');
+const passport = require('./lib/passport');
+const tools = require('./lib/tools');
 
-let routes = require('./routes/index');
+const routes = require('./routes/index');
-let users = require('./routes/users');
+const users = require('./routes/users');
-let lists = require('./routes/lists');
+const lists = require('./routes/lists');
-let settings = require('./routes/settings');
+const settings = require('./routes/settings');
-let settingsModel = require('./lib/models/settings');
+const settingsModel = require('./lib/models/settings');
-let templates = require('./routes/templates');
+const templates = require('./routes/templates');
-let campaigns = require('./routes/campaigns');
+const campaigns = require('./routes/campaigns');
-let links = require('./routes/links');
+const links = require('./routes/links');
-let fields = require('./routes/fields');
+const fields = require('./routes/fields');
-let forms = require('./routes/forms');
+const forms = require('./routes/forms');
-let segments = require('./routes/segments');
+const segments = require('./routes/segments');
-let triggers = require('./routes/triggers');
+const triggers = require('./routes/triggers');
-let webhooks = require('./routes/webhooks');
+const webhooks = require('./routes/webhooks');
-let subscription = require('./routes/subscription');
+const subscription = require('./routes/subscription');
-let archive = require('./routes/archive');
+const archive = require('./routes/archive');
-let api = require('./routes/api');
+const api = require('./routes/api');
-let blacklist = require('./routes/blacklist');
+const blacklist = require('./routes/blacklist');
-let editorapi = require('./routes/editorapi');
+const editorapi = require('./routes/editorapi');
-let grapejs = require('./routes/grapejs');
+const grapejs = require('./routes/grapejs');
-let mosaico = require('./routes/mosaico');
+const mosaico = require('./routes/mosaico');
-let reports = require('./routes/reports');
+const reports = require('./routes/reports');
-let reportsTemplates = require('./routes/report-templates');
+const reportsTemplates = require('./routes/report-templates');
 
-let app = express();
+const app = express();
 
 // view engine setup
 app.set('views', path.join(__dirname, 'views'));
@@ -108,43 +109,8 @@ hbs.registerHelper('flash_messages', function () { // eslint-disable-line prefer
     );
 });
 
-// {{#translate}}abc{{/translate}}
+handlebarsHelpers.registerHelpers(hbs.handlebars);
-hbs.registerHelper('translate', function (context, options) { // eslint-disable-line prefer-arrow-callback
-    if (typeof options === 'undefined' && context) {
-        options = context;
-        context = false;
-    }
 
-    let result = _(options.fn(this)); // eslint-disable-line no-invalid-this
-
-    if (Array.isArray(context)) {
-        result = util.format(result, ...context);
-    }
-    return new hbs.handlebars.SafeString(result);
-});
-
-/* Credits to http://chrismontrois.net/2016/01/30/handlebars-switch/
-
- {{#switch letter}}
- {{#case "a"}}
- A is for alpaca
- {{/case}}
- {{#case "b"}}
- B is for bluebird
- {{/case}}
- {{/switch}}
- */
-hbs.registerHelper("switch", function(value, options) {
-    this._switch_value_ = value;
-    var html = options.fn(this); // Process the body of the switch block
-    delete this._switch_value_;
-    return html;
-});
-hbs.registerHelper("case", function(value, options) {
-    if (value == this._switch_value_) {
-        return options.fn(this);
-    }
-});
 
 app.use(compression());
 app.use(favicon(path.join(__dirname, 'public', 'favicon.ico')));

lib/fs-tools.js

@@ -0,0 +1,14 @@
+'use strict';
+
+module.exports = {
+    nameToFileName,
+};
+
+function nameToFileName(name) {
+    return name.
+    trim().
+    toLowerCase().
+    replace(/[ .+/]/g, '-').
+    replace(/[^a-z0-9\-_]/gi, '').
+    replace(/--*/g, '-');
+}

lib/handlebars-helpers.js

@@ -0,0 +1,46 @@
+'use strict';
+
+const _ = require('../lib/translate')._;
+
+module.exports.registerHelpers = (handlebars) => {
+    // {{#translate}}abc{{/translate}}
+    handlebars.registerHelper('translate', function (context, options) { // eslint-disable-line prefer-arrow-callback
+        if (typeof options === 'undefined' && context) {
+            options = context;
+            context = false;
+        }
+
+        let result = _(options.fn(this)); // eslint-disable-line no-invalid-this
+
+        if (Array.isArray(context)) {
+            result = util.format(result, ...context);
+        }
+        return new handlebars.SafeString(result);
+    });
+
+
+    /* Credits to http://chrismontrois.net/2016/01/30/handlebars-switch/
+
+     {{#switch letter}}
+     {{#case "a"}}
+     A is for alpaca
+     {{/case}}
+     {{#case "b"}}
+     B is for bluebird
+     {{/case}}
+     {{/switch}}
+     */
+    handlebars.registerHelper("switch", function(value, options) {
+        this._switch_value_ = value;
+        var html = options.fn(this); // Process the body of the switch block
+        delete this._switch_value_;
+        return html;
+    });
+
+    handlebars.registerHelper("case", function(value, options) {
+        if (value == this._switch_value_) {
+            return options.fn(this);
+        }
+    });
+
+};

lib/models/reports.js

@@ -8,7 +8,7 @@ const tools = require('../tools');
 const _ = require('../translate')._;
 const log = require('npmlog');
 
-const allowedKeys = ['name', 'description', 'report_template', 'params'];
+const allowedKeys = ['name', 'description', 'report_template', 'params', 'filename'];
 
 module.exports.list = (start, limit, callback) => {
     tableHelpers.list('reports', ['*'], 'name', start, limit, callback);
@@ -16,7 +16,7 @@ module.exports.list = (start, limit, callback) => {
 
 module.exports.filter = (request, callback) => {
     tableHelpers.filter('reports JOIN report_templates ON reports.report_template = report_templates.id',
-        ['reports.id AS id', 'reports.name AS name', 'reports.description AS description', 'reports.report_template AS report_template', 'reports.params AS params', 'reports.created AS created', 'report_templates.name AS report_template_name', 'report_templates.mime_type AS mime_type' ],
+        ['reports.id AS id', 'reports.name AS name', 'reports.description AS description', 'reports.state AS state', 'reports.filename AS filename', 'reports.report_template AS report_template', 'reports.params AS params', 'reports.created AS created', 'report_templates.name AS report_template_name', 'report_templates.mime_type AS mime_type' ],
         request, ['#', 'name', 'report_templates.name', 'description', 'created'], ['name'], 'created DESC', null, callback);
 };
 

protected/reports/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!README.md

protected/reports/README.md

@@ -0,0 +1 @@
+This directory serves for generated reports.

routes/report-templates.js

@@ -55,10 +55,7 @@ router.post('/ajax', (req, res) => {
 });
 
 router.get('/create', passport.csrfProtection, (req, res) => {
-    const data = tools.convertKeys(req.query, {
+    const data = req.query;
-        skip: ['layout']
-    });
-
     const wizard = req.query['type'] || '';
 
     if (wizard == 'subscribers-all') {
@@ -86,7 +83,6 @@ router.get('/create', passport.csrfProtection, (req, res) => {
             '    }\n' +
             '\n' +
             '    const data = {\n' +
-            '        title: "Sample Report on " + inputs.campaign.name,\n' +
             '        results: results\n' +
             '    };\n' +
             '\n' +
@@ -152,7 +148,6 @@ router.get('/create', passport.csrfProtection, (req, res) => {
             '    }\n' +
             '\n' +
             '    let data = {\n' +
-            '        title: "Sample Report on " + inputs.campaign.name,\n' +
             '        results: results\n' +
             '    };\n' +
             '\n' +
@@ -226,7 +221,6 @@ router.get('/create', passport.csrfProtection, (req, res) => {
             '    }\n' +
             '\n' +
             '    let data = {\n' +
-            '        title: "Sample Export of " + inputs.list.name,\n' +
             '        results: results\n' +
             '    };\n' +
             '\n' +

routes/reports.js

@@ -12,8 +12,8 @@ const tools = require('../lib/tools');
 const util = require('util');
 const htmlescape = require('escape-html');
 const striptags = require('striptags');
-const hbs = require('hbs');
+const fs = require('fs');
-const vm = require('vm');
+const fsTools = require('../lib/fs-tools');
 
 router.all('/*', (req, res, next) => {
     if (!req.user) {
@@ -31,10 +31,23 @@ router.get('/', (req, res) => {
 });
 
 router.post('/ajax', (req, res) => {
-    function getViewIcon(mimeType) {
+    function getViewLink(row) {
-        let icon = 'search';
+        if (row.state == 0) {
-        if (mimeType == 'text/csv') icon = 'download-alt';
+            // TODO: Render waiting
-        return icon;
+            // TODO: Add error output
+            return '<span class="glyphicon glyphicon-hourglass" aria-hidden="true"></span> ';
+        } else if (row.state == 1) {
+            let icon = 'eye-open';
+            if (row.mimeType == 'text/csv') icon = 'download-alt';
+
+            // TODO: Add error output
+            return '<a href="/reports/view/' + row.id + '"><span class="glyphicon glyphicon-' + icon + '" aria-hidden="true"></span></a> ';
+        } else if (row.state == 2) {
+            // TODO: Add error output
+            return '<span class="glyphicon glyphicon-thumbs-down" aria-hidden="true"></span> ';
+        }
+
+        return '';
     }
 
     reports.filter(req.body, (err, data, total, filteredTotal) => {
@@ -55,7 +68,7 @@ router.post('/ajax', (req, res) => {
                 htmlescape(row.reportTemplateName || ''),
                 htmlescape(striptags(row.description) || ''),
                 '<span class="datestring" data-date="' + row.created.toISOString() + '" title="' + row.created.toISOString() + '">' + row.created.toISOString() + '</span>',
-                '<a href="/reports/view/' + row.id + '"><span class="glyphicon glyphicon-' + getViewIcon(row.mimeType) + '" aria-hidden="true"></span></a> ' +
+                getViewLink(row) +
                 '<a href="/reports/edit/' + row.id + '"><span class="glyphicon glyphicon-wrench" aria-hidden="true"></span></a>']
             )
         });
@@ -63,10 +76,7 @@ router.post('/ajax', (req, res) => {
 });
 
 router.get('/create', passport.csrfProtection, (req, res) => {
-    const reqData = tools.convertKeys(req.query, {
+    const reqData = req.query;
-        skip: ['layout']
-    });
-
     reqData.csrfToken = req.csrfToken();
     reqData.title = _('Create Report');
     reqData.useEditor = true;
@@ -106,6 +116,8 @@ router.get('/create', passport.csrfProtection, (req, res) => {
 
 router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) => {
     const reqData = req.body;
+    delete reqData.filename; // This is to make sure no one inserts a fake filename when editing the report.
+
     const reportTemplateId = Number(reqData.reportTemplate);
 
     addParamsObject(reportTemplateId, reqData, (err, data) => {
@@ -126,10 +138,7 @@ router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) =
 });
 
 router.get('/edit/:id', passport.csrfProtection, (req, res) => {
-    const reqData = tools.convertKeys(req.query, {
+    const reqData = req.query;
-        skip: ['layout']
-    });
-
     reports.get(req.params.id, (err, report) => {
         if (err || !report) {
             req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
@@ -170,6 +179,8 @@ router.get('/edit/:id', passport.csrfProtection, (req, res) => {
 
 router.post('/edit', passport.parseForm, passport.csrfProtection, (req, res) => {
     const reqData = req.body;
+    delete reqData.filename; // This is to make sure no one inserts a fake filename when editing the report.
+
     const reportTemplateId = Number(reqData.reportTemplate);
 
     addParamsObject(reportTemplateId, reqData, (err, data) => {
@@ -216,120 +227,49 @@ router.get('/view/:id', passport.csrfProtection, (req, res) => {
 
         reportTemplates.get(report.reportTemplate, (err, reportTemplate) => {
             if (err) {
-                return callback(err);
+                req.flash('danger', err && err.message || err || _('Could not find report template'));
+                return res.redirect('/reports');
             }
 
-            resolveUserFields(reportTemplate.userFieldsObject, report.paramsObject, (err, inputs) => {
+            if (report.state == 1) {
-                if (err) {
+                if (reportTemplate.mimeType == 'text/html') {
-                    req.flash('danger', err.message || err);
+
-                    return res.redirect('/reports');
+                    fs.readFile(path.join(__dirname, '../protected/reports', report.filename + '.report'), (err, reportContent) => {
+                        if (err) {
+                            req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
+                            return res.redirect('/reports');
+                        }
+
+                        const data = {
+                            csrfToken: req.csrfToken(),
+                            report: new hbs.handlebars.SafeString(reportContent),
+                            title: report.name
+                        };
+
+                        res.render('reports/view', data);
+                    });
+
+                } else if (reportTemplate.mimeType == 'text/csv') {
+                    const headers = {
+                        'Content-Disposition': 'attachment;filename=' + fsTools.nameToFileName(report.name) + '.csv',
+                        'Content-Type': 'text/csv'
+                    };
+
+                    res.sendFile(path.join(__dirname, '../protected/reports', report.filename + '.report'), {headers: headers});
+
+                } else {
+                    req.flash('danger', _('Unknown type of template'));
+                    res.redirect('/reports');
                 }
 
-                const sandbox = {
+            } else {
-                    require: require,
+                req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
-                    inputs: inputs,
+                return res.redirect('/reports');
-                    callback: (err, outputs) => {
+            }
-                        if (err) {
-                            req.flash('danger', err.message || err);
-                            return res.redirect('/reports');
-                        }
-
-                        const hbsTmpl = hbs.handlebars.compile(reportTemplate.hbs);
-                        const reportText = hbsTmpl(outputs);
-
-                        if (reportTemplate.mimeType == 'text/html') {
-                            const data = {
-                                csrfToken: req.csrfToken(),
-                                report: new hbs.handlebars.SafeString(reportText),
-                                title: outputs.title
-                            };
-
-                            res.render('reports/view', data);
-
-                        } else if (reportTemplate.mimeType == 'text/csv') {
-                            res.set('Content-Disposition', 'attachment;filename=' + toFileName(report.name) + '.csv');
-                            res.set('Content-Type', 'text/csv');
-                            res.send(new Buffer(reportText));
-
-                        } else {
-                            req.flash('danger', _('Unknown type of template'));
-                            return res.redirect('/reports');
-                        }
-                    }
-                };
-
-                const script = new vm.Script(reportTemplate.js);
-                script.runInNewContext(sandbox, { displayErrors: true, timeout: 10000 });
-            });
         });
     });
 });
 
-function toFileName(name) {
-    return name.
-        trim().
-        toLowerCase().
-        replace(/[ .+/]/g, '-').
-        replace(/[^a-z0-9\-_]/gi, '');
-}
-
-function resolveEntities(getter, ids, callback) {
-    const idsRemaining = ids.slice();
-    const resolved = [];
-
-    function doWork() {
-        if (idsRemaining.length == 0) {
-            return callback(null, resolved);
-        }
-
-        getter(idsRemaining.shift(), (err, entity) => {
-            if (err) {
-                return callback(err);
-            }
-
-            resolved.push(entity);
-            return doWork();
-        });
-    }
-
-    setImmediate(doWork);
-}
-
-const userFieldTypeToGetter = {
-    'campaign': (id, callback) => campaigns.get(id, false, callback),
-    'list': lists.get
-};
-
-function resolveUserFields(userFields, params, callback) {
-    const userFieldsRemaining = userFields.slice();
-    const resolved = {};
-
-    function doWork() {
-        if (userFieldsRemaining.length == 0) {
-            return callback(null, resolved);
-        }
-
-        const spec = userFieldsRemaining.shift();
-        const getter = userFieldTypeToGetter[spec.type];
-
-        if (getter) {
-            return resolveEntities(getter, params[spec.id], (err, entities) => {
-                if (spec.minOccurences == 1 && spec.maxOccurences == 1) {
-                    resolved[spec.id] = entities[0];
-                } else {
-                    resolved[spec.id] = entities;
-                }
-
-                doWork();
-            });
-        } else {
-            return callback(new Error(_('Unknown user field type "' + spec.type + '".')));
-        }
-    }
-
-    setImmediate(doWork);
-}
-
 function addUserFields(reportTemplateId, reqData, report, callback) {
     reportTemplates.get(reportTemplateId, (err, reportTemplate) => {
         if (err) {

services/reports.js

@@ -0,0 +1,146 @@
+'use strict';
+
+const reports = require('../lib/models/reports');
+const reportTemplates = require('../lib/models/report-templates');
+const lists = require('../lib/models/lists');
+const campaigns = require('../lib/models/campaigns');
+const handlebars = require('handlebars');
+const handlebarsHelpers = require('../lib/handlebars-helpers');
+const _ = require('../lib/translate')._;
+const hbs = require('hbs');
+const vm = require('vm');
+const log = require('npmlog');
+const fs = require('fs');
+const path = require('path');
+const fsTools = require('../lib/fs-tools');
+
+handlebarsHelpers.registerHelpers(handlebars);
+
+function resolveEntities(getter, ids, callback) {
+    const idsRemaining = ids.slice();
+    const resolved = [];
+
+    function doWork() {
+        if (idsRemaining.length == 0) {
+            return callback(null, resolved);
+        }
+
+        getter(idsRemaining.shift(), (err, entity) => {
+            if (err) {
+                return callback(err);
+            }
+
+            resolved.push(entity);
+            return doWork();
+        });
+    }
+
+    setImmediate(doWork);
+}
+
+const userFieldTypeToGetter = {
+    'campaign': (id, callback) => campaigns.get(id, false, callback),
+    'list': lists.get
+};
+
+function resolveUserFields(userFields, params, callback) {
+    const userFieldsRemaining = userFields.slice();
+    const resolved = {};
+
+    function doWork() {
+        if (userFieldsRemaining.length == 0) {
+            return callback(null, resolved);
+        }
+
+        const spec = userFieldsRemaining.shift();
+        const getter = userFieldTypeToGetter[spec.type];
+
+        if (getter) {
+            return resolveEntities(getter, params[spec.id], (err, entities) => {
+                if (spec.minOccurences == 1 && spec.maxOccurences == 1) {
+                    resolved[spec.id] = entities[0];
+                } else {
+                    resolved[spec.id] = entities;
+                }
+
+                doWork();
+            });
+        } else {
+            return callback(new Error(_('Unknown user field type "' + spec.type + '".')));
+        }
+    }
+
+    setImmediate(doWork);
+}
+
+function doneSuccess(id) {
+    // TODO: Mark in the DB as completed + update the date/time
+    // TODO update the report filename in the DB
+
+    process.exit(0);
+}
+
+function doneFail(id) {
+    // TODO: Mark in the DB as failed
+    process.exit(1);
+}
+
+function start(id) {
+    // TODO: Mark in the DB as running
+}
+
+// TODO: Retrieve report task from the DB and run it
+
+function processReport(reportId) {
+    reports.get(reportId, (err, report) => {
+        if (err || !report) {
+            log.error('reports', err && err.message || err || _('Could not find report with specified ID'));
+            doneFail(reportId);
+        }
+
+        reportTemplates.get(report.reportTemplate, (err, reportTemplate) => {
+            if (err) {
+                log.error('reports', err && err.message || err || _('Could not find report template'));
+                doneFail(reportId);
+            }
+
+            resolveUserFields(reportTemplate.userFieldsObject, report.paramsObject, (err, inputs) => {
+                if (err) {
+                    log.error('reports', err.message || err);
+                    doneFail(reportId);
+                }
+
+                const filename = fsTools.nameToFileName(report.name);
+
+                const sandbox = {
+                    require: require,
+                    inputs: inputs,
+                    callback: (err, outputs) => {
+                        if (err) {
+                            log.error('reports', err.message || err);
+                            doneFail(reportId);
+                        }
+
+                        const hbsTmpl = handlebars.compile(reportTemplate.hbs);
+                        const reportText = hbsTmpl(outputs);
+
+                        fs.writeFile(path.join(__dirname, '../protected/reports', filename + '.report'), reportText, (err, reportContent) => {
+                            if (err) {
+                                log.error('reports', err && err.message || err || _('Could not find report with specified ID'));
+                                doneFail(reportId);
+                            }
+
+                            doneSuccess(reportId, filename);
+                            process
+                        });
+                    }
+                };
+
+                const script = new vm.Script(reportTemplate.js);
+                script.runInNewContext(sandbox, {displayErrors: true, timeout: 120000});
+            });
+        });
+    });
+}
+
+processReport(1);

setup/sql/upgrade-00027.sql

@@ -22,6 +22,9 @@ CREATE TABLE `reports` (
   `description` text,
   `report_template` int(11) unsigned NOT NULL,
   `params` longtext,
+  `filename`  varchar(255) DEFAULT NULL,
+  `state`  int(11) unsigned NOT NULL DEFAULT 0,
+  `last_run` DATETIME DEFAULT NULL,
   `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   PRIMARY KEY (`id`),
   KEY `report_template` (`report_template`),