public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Halfway through in refactoring the report generation to a separate process running asynchronously of the Express server.

Browse source
This commit is contained in:
Tomas Bures 2017-04-17 18:31:01 -04:00
parent 2056645023
commit e7d12f1dbc
10 changed files with 319 additions and 206 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

118
app.js
View file

@ -1,49 +1,50 @@
'use strict';
let config = require('config');
let log = require('npmlog');
const config = require('config');
const log = require('npmlog');
let _ = require('./lib/translate')._;
let util = require('util');
const _ = require('./lib/translate')._;
const util = require('util');
let express = require('express');
let bodyParser = require('body-parser');
let path = require('path');
let favicon = require('serve-favicon');
let logger = require('morgan');
let cookieParser = require('cookie-parser');
let session = require('express-session');
let RedisStore = require('connect-redis')(session);
let flash = require('connect-flash');
let hbs = require('hbs');
let compression = require('compression');
let passport = require('./lib/passport');
let tools = require('./lib/tools');
const express = require('express');
const bodyParser = require('body-parser');
const path = require('path');
const favicon = require('serve-favicon');
const logger = require('morgan');
const cookieParser = require('cookie-parser');
const session = require('express-session');
const RedisStore = require('connect-redis')(session);
const flash = require('connect-flash');
const hbs = require('hbs');
const handlebarsHelpers = require('./lib/handlebars-helpers');
const compression = require('compression');
const passport = require('./lib/passport');
const tools = require('./lib/tools');
let routes = require('./routes/index');
let users = require('./routes/users');
let lists = require('./routes/lists');
let settings = require('./routes/settings');
let settingsModel = require('./lib/models/settings');
let templates = require('./routes/templates');
let campaigns = require('./routes/campaigns');
let links = require('./routes/links');
let fields = require('./routes/fields');
let forms = require('./routes/forms');
let segments = require('./routes/segments');
let triggers = require('./routes/triggers');
let webhooks = require('./routes/webhooks');
let subscription = require('./routes/subscription');
let archive = require('./routes/archive');
let api = require('./routes/api');
let blacklist = require('./routes/blacklist');
let editorapi = require('./routes/editorapi');
let grapejs = require('./routes/grapejs');
let mosaico = require('./routes/mosaico');
let reports = require('./routes/reports');
let reportsTemplates = require('./routes/report-templates');
const routes = require('./routes/index');
const users = require('./routes/users');
const lists = require('./routes/lists');
const settings = require('./routes/settings');
const settingsModel = require('./lib/models/settings');
const templates = require('./routes/templates');
const campaigns = require('./routes/campaigns');
const links = require('./routes/links');
const fields = require('./routes/fields');
const forms = require('./routes/forms');
const segments = require('./routes/segments');
const triggers = require('./routes/triggers');
const webhooks = require('./routes/webhooks');
const subscription = require('./routes/subscription');
const archive = require('./routes/archive');
const api = require('./routes/api');
const blacklist = require('./routes/blacklist');
const editorapi = require('./routes/editorapi');
const grapejs = require('./routes/grapejs');
const mosaico = require('./routes/mosaico');
const reports = require('./routes/reports');
const reportsTemplates = require('./routes/report-templates');
let app = express();
const app = express();
// view engine setup
app.set('views', path.join(__dirname, 'views'));
@ -108,43 +109,8 @@ hbs.registerHelper('flash_messages', function () { // eslint-disable-line prefer
);
});
// {{#translate}}abc{{/translate}}
hbs.registerHelper('translate', function (context, options) { // eslint-disable-line prefer-arrow-callback
if (typeof options === 'undefined' && context) {
options = context;
context = false;
}
handlebarsHelpers.registerHelpers(hbs.handlebars);
let result = _(options.fn(this)); // eslint-disable-line no-invalid-this
if (Array.isArray(context)) {
result = util.format(result, ...context);
}
return new hbs.handlebars.SafeString(result);
});
/* Credits to http://chrismontrois.net/2016/01/30/handlebars-switch/
{{#switch letter}}
{{#case "a"}}
A is for alpaca
{{/case}}
{{#case "b"}}
B is for bluebird
{{/case}}
{{/switch}}
*/
hbs.registerHelper("switch", function(value, options) {
this._switch_value_ = value;
var html = options.fn(this); // Process the body of the switch block
delete this._switch_value_;
return html;
});
hbs.registerHelper("case", function(value, options) {
if (value == this._switch_value_) {
return options.fn(this);
}
});
app.use(compression());
app.use(favicon(path.join(__dirname, 'public', 'favicon.ico')));

14
lib/fs-tools.js Normal file
View file

@ -0,0 +1,14 @@
'use strict';
module.exports = {
nameToFileName,
};
function nameToFileName(name) {
return name.
trim().
toLowerCase().
replace(/[ .+/]/g, '-').
replace(/[^a-z0-9\-_]/gi, '').
replace(/--*/g, '-');
}

46
lib/handlebars-helpers.js Normal file
View file

@ -0,0 +1,46 @@
'use strict';
const _ = require('../lib/translate')._;
module.exports.registerHelpers = (handlebars) => {
// {{#translate}}abc{{/translate}}
handlebars.registerHelper('translate', function (context, options) { // eslint-disable-line prefer-arrow-callback
if (typeof options === 'undefined' && context) {
options = context;
context = false;
}
let result = _(options.fn(this)); // eslint-disable-line no-invalid-this
if (Array.isArray(context)) {
result = util.format(result, ...context);
}
return new handlebars.SafeString(result);
});
/* Credits to http://chrismontrois.net/2016/01/30/handlebars-switch/
{{#switch letter}}
{{#case "a"}}
A is for alpaca
{{/case}}
{{#case "b"}}
B is for bluebird
{{/case}}
{{/switch}}
*/
handlebars.registerHelper("switch", function(value, options) {
this._switch_value_ = value;
var html = options.fn(this); // Process the body of the switch block
delete this._switch_value_;
return html;
});
handlebars.registerHelper("case", function(value, options) {
if (value == this._switch_value_) {
return options.fn(this);
}
});
};

4
lib/models/reports.js
View file

@ -8,7 +8,7 @@ const tools = require('../tools');
const _ = require('../translate')._;
const log = require('npmlog');
const allowedKeys = ['name', 'description', 'report_template', 'params'];
const allowedKeys = ['name', 'description', 'report_template', 'params', 'filename'];
module.exports.list = (start, limit, callback) => {
tableHelpers.list('reports', ['*'], 'name', start, limit, callback);
@ -16,7 +16,7 @@ module.exports.list = (start, limit, callback) => {
module.exports.filter = (request, callback) => {
tableHelpers.filter('reports JOIN report_templates ON reports.report_template = report_templates.id',
['reports.id AS id', 'reports.name AS name', 'reports.description AS description', 'reports.report_template AS report_template', 'reports.params AS params', 'reports.created AS created', 'report_templates.name AS report_template_name', 'report_templates.mime_type AS mime_type' ],
['reports.id AS id', 'reports.name AS name', 'reports.description AS description', 'reports.state AS state', 'reports.filename AS filename', 'reports.report_template AS report_template', 'reports.params AS params', 'reports.created AS created', 'report_templates.name AS report_template_name', 'report_templates.mime_type AS mime_type' ],
request, ['#', 'name', 'report_templates.name', 'description', 'created'], ['name'], 'created DESC', null, callback);
};

3
protected/reports/.gitignore vendored Normal file
View file

@ -0,0 +1,3 @@
*
!.gitignore
!README.md

1
protected/reports/README.md Normal file
View file

@ -0,0 +1 @@
This directory serves for generated reports.

8
routes/report-templates.js
View file

@ -55,10 +55,7 @@ router.post('/ajax', (req, res) => {
});
router.get('/create', passport.csrfProtection, (req, res) => {
const data = tools.convertKeys(req.query, {
skip: ['layout']
});
const data = req.query;
const wizard = req.query['type'] || '';
if (wizard == 'subscribers-all') {
@ -86,7 +83,6 @@ router.get('/create', passport.csrfProtection, (req, res) => {
' }\n' +
'\n' +
' const data = {\n' +
' title: "Sample Report on " + inputs.campaign.name,\n' +
' results: results\n' +
' };\n' +
'\n' +
@ -152,7 +148,6 @@ router.get('/create', passport.csrfProtection, (req, res) => {
' }\n' +
'\n' +
' let data = {\n' +
' title: "Sample Report on " + inputs.campaign.name,\n' +
' results: results\n' +
' };\n' +
'\n' +
@ -226,7 +221,6 @@ router.get('/create', passport.csrfProtection, (req, res) => {
' }\n' +
'\n' +
' let data = {\n' +
' title: "Sample Export of " + inputs.list.name,\n' +
' results: results\n' +
' };\n' +
'\n' +

182
routes/reports.js
View file

@ -12,8 +12,8 @@ const tools = require('../lib/tools');
const util = require('util');
const htmlescape = require('escape-html');
const striptags = require('striptags');
const hbs = require('hbs');
const vm = require('vm');
const fs = require('fs');
const fsTools = require('../lib/fs-tools');
router.all('/*', (req, res, next) => {
if (!req.user) {
@ -31,10 +31,23 @@ router.get('/', (req, res) => {
});
router.post('/ajax', (req, res) => {
function getViewIcon(mimeType) {
let icon = 'search';
if (mimeType == 'text/csv') icon = 'download-alt';
return icon;
function getViewLink(row) {
if (row.state == 0) {
// TODO: Render waiting
// TODO: Add error output
return '<span class="glyphicon glyphicon-hourglass" aria-hidden="true"></span> ';
} else if (row.state == 1) {
let icon = 'eye-open';
if (row.mimeType == 'text/csv') icon = 'download-alt';
// TODO: Add error output
return '<a href="/reports/view/' + row.id + '"><span class="glyphicon glyphicon-' + icon + '" aria-hidden="true"></span></a> ';
} else if (row.state == 2) {
// TODO: Add error output
return '<span class="glyphicon glyphicon-thumbs-down" aria-hidden="true"></span> ';
}
return '';
}
reports.filter(req.body, (err, data, total, filteredTotal) => {
@ -55,7 +68,7 @@ router.post('/ajax', (req, res) => {
htmlescape(row.reportTemplateName || ''),
htmlescape(striptags(row.description) || ''),
'<span class="datestring" data-date="' + row.created.toISOString() + '" title="' + row.created.toISOString() + '">' + row.created.toISOString() + '</span>',
'<a href="/reports/view/' + row.id + '"><span class="glyphicon glyphicon-' + getViewIcon(row.mimeType) + '" aria-hidden="true"></span></a> ' +
getViewLink(row) +
'<a href="/reports/edit/' + row.id + '"><span class="glyphicon glyphicon-wrench" aria-hidden="true"></span></a>']
)
});
@ -63,10 +76,7 @@ router.post('/ajax', (req, res) => {
});
router.get('/create', passport.csrfProtection, (req, res) => {
const reqData = tools.convertKeys(req.query, {
skip: ['layout']
});
const reqData = req.query;
reqData.csrfToken = req.csrfToken();
reqData.title = _('Create Report');
reqData.useEditor = true;
@ -106,6 +116,8 @@ router.get('/create', passport.csrfProtection, (req, res) => {
router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) => {
const reqData = req.body;
delete reqData.filename; // This is to make sure no one inserts a fake filename when editing the report.
const reportTemplateId = Number(reqData.reportTemplate);
addParamsObject(reportTemplateId, reqData, (err, data) => {
@ -126,10 +138,7 @@ router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) =
});
router.get('/edit/:id', passport.csrfProtection, (req, res) => {
const reqData = tools.convertKeys(req.query, {
skip: ['layout']
});
const reqData = req.query;
reports.get(req.params.id, (err, report) => {
if (err || !report) {
req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
@ -170,6 +179,8 @@ router.get('/edit/:id', passport.csrfProtection, (req, res) => {
router.post('/edit', passport.parseForm, passport.csrfProtection, (req, res) => {
const reqData = req.body;
delete reqData.filename; // This is to make sure no one inserts a fake filename when editing the report.
const reportTemplateId = Number(reqData.reportTemplate);
addParamsObject(reportTemplateId, reqData, (err, data) => {
@ -216,120 +227,49 @@ router.get('/view/:id', passport.csrfProtection, (req, res) => {
reportTemplates.get(report.reportTemplate, (err, reportTemplate) => {
if (err) {
return callback(err);
req.flash('danger', err && err.message || err || _('Could not find report template'));
return res.redirect('/reports');
}
resolveUserFields(reportTemplate.userFieldsObject, report.paramsObject, (err, inputs) => {
if (err) {
req.flash('danger', err.message || err);
return res.redirect('/reports');
if (report.state == 1) {
if (reportTemplate.mimeType == 'text/html') {
fs.readFile(path.join(__dirname, '../protected/reports', report.filename + '.report'), (err, reportContent) => {
if (err) {
req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
return res.redirect('/reports');
}
const data = {
csrfToken: req.csrfToken(),
report: new hbs.handlebars.SafeString(reportContent),
title: report.name
};
res.render('reports/view', data);
});
} else if (reportTemplate.mimeType == 'text/csv') {
const headers = {
'Content-Disposition': 'attachment;filename=' + fsTools.nameToFileName(report.name) + '.csv',
'Content-Type': 'text/csv'
};
res.sendFile(path.join(__dirname, '../protected/reports', report.filename + '.report'), {headers: headers});
} else {
req.flash('danger', _('Unknown type of template'));
res.redirect('/reports');
}
const sandbox = {
require: require,
inputs: inputs,
callback: (err, outputs) => {
if (err) {
req.flash('danger', err.message || err);
return res.redirect('/reports');
}
const hbsTmpl = hbs.handlebars.compile(reportTemplate.hbs);
const reportText = hbsTmpl(outputs);
if (reportTemplate.mimeType == 'text/html') {
const data = {
csrfToken: req.csrfToken(),
report: new hbs.handlebars.SafeString(reportText),
title: outputs.title
};
res.render('reports/view', data);
} else if (reportTemplate.mimeType == 'text/csv') {
res.set('Content-Disposition', 'attachment;filename=' + toFileName(report.name) + '.csv');
res.set('Content-Type', 'text/csv');
res.send(new Buffer(reportText));
} else {
req.flash('danger', _('Unknown type of template'));
return res.redirect('/reports');
}
}
};
const script = new vm.Script(reportTemplate.js);
script.runInNewContext(sandbox, { displayErrors: true, timeout: 10000 });
});
} else {
req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
return res.redirect('/reports');
}
});
});
});
function toFileName(name) {
return name.
trim().
toLowerCase().
replace(/[ .+/]/g, '-').
replace(/[^a-z0-9\-_]/gi, '');
}
function resolveEntities(getter, ids, callback) {
const idsRemaining = ids.slice();
const resolved = [];
function doWork() {
if (idsRemaining.length == 0) {
return callback(null, resolved);
}
getter(idsRemaining.shift(), (err, entity) => {
if (err) {
return callback(err);
}
resolved.push(entity);
return doWork();
});
}
setImmediate(doWork);
}
const userFieldTypeToGetter = {
'campaign': (id, callback) => campaigns.get(id, false, callback),
'list': lists.get
};
function resolveUserFields(userFields, params, callback) {
const userFieldsRemaining = userFields.slice();
const resolved = {};
function doWork() {
if (userFieldsRemaining.length == 0) {
return callback(null, resolved);
}
const spec = userFieldsRemaining.shift();
const getter = userFieldTypeToGetter[spec.type];
if (getter) {
return resolveEntities(getter, params[spec.id], (err, entities) => {
if (spec.minOccurences == 1 && spec.maxOccurences == 1) {
resolved[spec.id] = entities[0];
} else {
resolved[spec.id] = entities;
}
doWork();
});
} else {
return callback(new Error(_('Unknown user field type "' + spec.type + '".')));
}
}
setImmediate(doWork);
}
function addUserFields(reportTemplateId, reqData, report, callback) {
reportTemplates.get(reportTemplateId, (err, reportTemplate) => {
if (err) {

146
services/reports.js Normal file
View file

@ -0,0 +1,146 @@
'use strict';
const reports = require('../lib/models/reports');
const reportTemplates = require('../lib/models/report-templates');
const lists = require('../lib/models/lists');
const campaigns = require('../lib/models/campaigns');
const handlebars = require('handlebars');
const handlebarsHelpers = require('../lib/handlebars-helpers');
const _ = require('../lib/translate')._;
const hbs = require('hbs');
const vm = require('vm');
const log = require('npmlog');
const fs = require('fs');
const path = require('path');
const fsTools = require('../lib/fs-tools');
handlebarsHelpers.registerHelpers(handlebars);
function resolveEntities(getter, ids, callback) {
const idsRemaining = ids.slice();
const resolved = [];
function doWork() {
if (idsRemaining.length == 0) {
return callback(null, resolved);
}
getter(idsRemaining.shift(), (err, entity) => {
if (err) {
return callback(err);
}
resolved.push(entity);
return doWork();
});
}
setImmediate(doWork);
}
const userFieldTypeToGetter = {
'campaign': (id, callback) => campaigns.get(id, false, callback),
'list': lists.get
};
function resolveUserFields(userFields, params, callback) {
const userFieldsRemaining = userFields.slice();
const resolved = {};
function doWork() {
if (userFieldsRemaining.length == 0) {
return callback(null, resolved);
}
const spec = userFieldsRemaining.shift();
const getter = userFieldTypeToGetter[spec.type];
if (getter) {
return resolveEntities(getter, params[spec.id], (err, entities) => {
if (spec.minOccurences == 1 && spec.maxOccurences == 1) {
resolved[spec.id] = entities[0];
} else {
resolved[spec.id] = entities;
}
doWork();
});
} else {
return callback(new Error(_('Unknown user field type "' + spec.type + '".')));
}
}
setImmediate(doWork);
}
function doneSuccess(id) {
// TODO: Mark in the DB as completed + update the date/time
// TODO update the report filename in the DB
process.exit(0);
}
function doneFail(id) {
// TODO: Mark in the DB as failed
process.exit(1);
}
function start(id) {
// TODO: Mark in the DB as running
}
// TODO: Retrieve report task from the DB and run it
function processReport(reportId) {
reports.get(reportId, (err, report) => {
if (err || !report) {
log.error('reports', err && err.message || err || _('Could not find report with specified ID'));
doneFail(reportId);
}
reportTemplates.get(report.reportTemplate, (err, reportTemplate) => {
if (err) {
log.error('reports', err && err.message || err || _('Could not find report template'));
doneFail(reportId);
}
resolveUserFields(reportTemplate.userFieldsObject, report.paramsObject, (err, inputs) => {
if (err) {
log.error('reports', err.message || err);
doneFail(reportId);
}
const filename = fsTools.nameToFileName(report.name);
const sandbox = {
require: require,
inputs: inputs,
callback: (err, outputs) => {
if (err) {
log.error('reports', err.message || err);
doneFail(reportId);
}
const hbsTmpl = handlebars.compile(reportTemplate.hbs);
const reportText = hbsTmpl(outputs);
fs.writeFile(path.join(__dirname, '../protected/reports', filename + '.report'), reportText, (err, reportContent) => {
if (err) {
log.error('reports', err && err.message || err || _('Could not find report with specified ID'));
doneFail(reportId);
}
doneSuccess(reportId, filename);
process
});
}
};
const script = new vm.Script(reportTemplate.js);
script.runInNewContext(sandbox, {displayErrors: true, timeout: 120000});
});
});
});
}
processReport(1);

3
setup/sql/upgrade-00027.sql
View file

@ -22,6 +22,9 @@ CREATE TABLE `reports` (
`description` text,
`report_template` int(11) unsigned NOT NULL,
`params` longtext,
`filename` varchar(255) DEFAULT NULL,
`state` int(11) unsigned NOT NULL DEFAULT 0,
`last_run` DATETIME DEFAULT NULL,
`created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
KEY `report_template` (`report_template`),