public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

The "Reports" feature seems functional.

Browse source 
Some small refactoring (column widths) of rendering tables in Lists, Templates, and Campaigns so that it is the same as Reports.
This commit is contained in:
Tomas Bures 2017-04-20 19:42:01 -04:00
parent e7d12f1dbc
commit 8237dd5d77
22 changed files with 584 additions and 236 deletions
Download patch file Download diff file Expand all files Collapse all files

33
index.js
View file

@ -18,6 +18,7 @@ let tzupdate = require('./services/tzupdate');
let feedcheck = require('./services/feedcheck');
let dbcheck = require('./lib/dbcheck');
let tools = require('./lib/tools');
let reportProcessor = require('./services/report-processor');
let port = config.www.port;
let host = config.www.host;
@ -120,23 +121,25 @@ server.on('listening', () => {
spawnSenders(() => {
feedcheck(() => {
postfixBounceServer(() => {
log.info('Service', 'All services started');
if (config.group) {
try {
process.setgid(config.group);
log.info('Service', 'Changed group to "%s" (%s)', config.group, process.getgid());
} catch (E) {
log.info('Service', 'Failed to change group to "%s" (%s)', config.group, E.message);
reportProcessor.init(() => {
log.info('Service', 'All services started');
if (config.group) {
try {
process.setgid(config.group);
log.info('Service', 'Changed group to "%s" (%s)', config.group, process.getgid());
} catch (E) {
log.info('Service', 'Failed to change group to "%s" (%s)', config.group, E.message);
}
}
}
if (config.user) {
try {
process.setuid(config.user);
log.info('Service', 'Changed user to "%s" (%s)', config.user, process.getuid());
} catch (E) {
log.info('Service', 'Failed to change user to "%s" (%s)', config.user, E.message);
if (config.user) {
try {
process.setuid(config.user);
log.info('Service', 'Changed user to "%s" (%s)', config.user, process.getuid());
} catch (E) {
log.info('Service', 'Failed to change user to "%s" (%s)', config.user, E.message);
}
}
}
});
});
});
});

14
lib/fs-tools.js
View file

@ -1,14 +0,0 @@
'use strict';
module.exports = {
nameToFileName,
};
function nameToFileName(name) {
return name.
trim().
toLowerCase().
replace(/[ .+/]/g, '-').
replace(/[^a-z0-9\-_]/gi, '').
replace(/--*/g, '-');
}

2
lib/models/campaigns.js
View file

@ -19,7 +19,7 @@ let tableHelpers = require('../table-helpers');
let allowedKeys = ['description', 'from', 'address', 'reply_to', 'subject', 'editor_name', 'editor_data', 'template', 'source_url', 'list', 'segment', 'html', 'text', 'tracking_disabled'];
module.exports.list = (start, limit, callback) => {
tableHelpers.list('campaigns', ['*'], 'scheduled', start, limit, callback);
tableHelpers.list('campaigns', ['*'], 'scheduled', null, start, limit, callback);
};
module.exports.filter = (request, parent, callback) => {

2
lib/models/lists.js
View file

@ -10,7 +10,7 @@ let tableHelpers = require('../table-helpers');
let allowedKeys = ['description', 'default_form', 'public_subscribe'];
module.exports.list = (start, limit, callback) => {
tableHelpers.list('lists', ['*'], 'name', start, limit, callback);
tableHelpers.list('lists', ['*'], 'name', null, start, limit, callback);
};
module.exports.filter = (request, parent, callback) => {

2
lib/models/report-templates.js
View file

@ -8,7 +8,7 @@ const _ = require('../translate')._;
const allowedKeys = ['name', 'description', 'mime_type', 'user_fields', 'js', 'hbs'];
module.exports.list = (start, limit, callback) => {
tableHelpers.list('report_templates', ['*'], 'name', start, limit, callback);
tableHelpers.list('report_templates', ['*'], 'name', null, start, limit, callback);
};
module.exports.quicklist = callback => {

66
lib/models/reports.js
View file

@ -8,16 +8,29 @@ const tools = require('../tools');
const _ = require('../translate')._;
const log = require('npmlog');
const allowedKeys = ['name', 'description', 'report_template', 'params', 'filename'];
const allowedKeys = ['name', 'description', 'report_template', 'params'];
const ReportState = {
SCHEDULED: 0,
PROCESSING: 1,
FINISHED: 2,
FAILED: 3
};
module.exports.ReportState = ReportState;
module.exports.list = (start, limit, callback) => {
tableHelpers.list('reports', ['*'], 'name', start, limit, callback);
tableHelpers.list('reports', ['*'], 'name', null, start, limit, callback);
};
module.exports.listWithState = (state, start, limit, callback) => {
tableHelpers.list('reports', ['*'], 'name', { where: 'state=?', values: [state] }, start, limit, callback);
}
module.exports.filter = (request, callback) => {
tableHelpers.filter('reports JOIN report_templates ON reports.report_template = report_templates.id',
['reports.id AS id', 'reports.name AS name', 'reports.description AS description', 'reports.state AS state', 'reports.filename AS filename', 'reports.report_template AS report_template', 'reports.params AS params', 'reports.created AS created', 'report_templates.name AS report_template_name', 'report_templates.mime_type AS mime_type' ],
request, ['#', 'name', 'report_templates.name', 'description', 'created'], ['name'], 'created DESC', null, callback);
['reports.id AS id', 'reports.name AS name', 'reports.description AS description', 'reports.state AS state', 'reports.report_template AS report_template', 'reports.params AS params', 'reports.last_run AS last_run', 'report_templates.name AS report_template_name', 'report_templates.mime_type AS mime_type' ],
request, ['#', 'name', 'report_templates.name', 'description', 'last_run'], ['name'], 'name ASC', null, callback);
};
module.exports.get = (id, callback) => {
@ -60,29 +73,56 @@ module.exports.get = (id, callback) => {
});
};
module.exports.createOrUpdate = (createMode, data, callback) => {
data = data || {};
// This method is not supposed to be used for unsanitized inputs. It does not do any checks.
module.exports.updateFields = (id, fieldValueMap, callback) => {
db.getConnection((err, connection) => {
if (err) {
return next(err);
}
const id = 'id' in data ? Number(data.id) : 0;
const clauses = [];
const values = [];
for (let key of Object.keys(fieldValueMap)) {
clauses.push(tools.toDbKey(key) + '=?');
values.push(fieldValueMap[key]);
}
values.push(id);
const query = 'UPDATE reports SET ' + clauses.join(', ') + ' WHERE id=? LIMIT 1';
connection.query(query, values, (err, result) => {
connection.release();
if (err) {
return callback(err);
}
return callback(null, result && result.affectedRows || false)
});
});
};
module.exports.createOrUpdate = (createMode, report, callback) => {
report = report || {};
const id = 'id' in report ? Number(report.id) : 0;
if (!createMode && id < 1) {
return callback(new Error(_('Missing report ID')));
}
const template = tools.convertKeys(data);
const name = (template.name || '').toString().trim();
const name = (report.name || '').toString().trim();
if (!name) {
return callback(new Error(_('Report name must be set')));
}
const reportTemplateId = Number(template.reportTemplate);
const reportTemplateId = Number(report.reportTemplate);
reportTemplates.get(reportTemplateId, (err, reportTemplate) => {
if (err) {
callback(err);
}
const params = data.paramsObject;
const params = report.paramsObject;
for (const spec of reportTemplate.userFieldsObject) {
if (params[spec.id].length < spec.minOccurences) {
return callback(new Error(_('At least ' + spec.minOccurences + ' rows in "' + spec.name + '" have to be selected.')));
@ -97,8 +137,8 @@ module.exports.createOrUpdate = (createMode, data, callback) => {
const values = [name, JSON.stringify(params)];
Object.keys(template).forEach(key => {
let value = typeof template[key] === 'number' ? template[key] : (template[key] || '').toString().trim();
Object.keys(report).forEach(key => {
let value = typeof report[key] === 'number' ? report[key] : (report[key] || '').toString().trim();
key = tools.toDbKey(key);
if (key === 'description') {

2
lib/models/subscriptions.js
View file

@ -21,7 +21,7 @@ module.exports.list = (listId, start, limit, callback) => {
return callback(new Error('Missing List ID'));
}
tableHelpers.list('subscription__' + listId, ['*'], 'email', start, limit, (err, rows, total) => {
tableHelpers.list('subscription__' + listId, ['*'], 'email', null, start, limit, (err, rows, total) => {
if (!err) {
rows = rows.map(row => tools.convertKeys(row));
}

2
lib/models/templates.js
View file

@ -8,7 +8,7 @@ let tableHelpers = require('../table-helpers');
let allowedKeys = ['description', 'editor_name', 'editor_data', 'html', 'text'];
module.exports.list = (start, limit, callback) => {
tableHelpers.list('templates', ['*'], 'name', start, limit, callback);
tableHelpers.list('templates', ['*'], 'name', null, start, limit, callback);
};
module.exports.filter = (request, parent, callback) => {

12
lib/table-helpers.js
View file

@ -4,7 +4,7 @@ let db = require('./db');
let tools = require('./tools');
let log = require('npmlog');
module.exports.list = (source, fields, orderBy, start, limit, callback) => {
module.exports.list = (source, fields, orderBy, queryData, start, limit, callback) => {
db.getConnection((err, connection) => {
if (err) {
return callback(err);
@ -22,7 +22,15 @@ module.exports.list = (source, fields, orderBy, start, limit, callback) => {
}
}
connection.query('SELECT SQL_CALC_FOUND_ROWS ' + fields.join(', ') + ' FROM ' + source + ' ORDER BY ' + orderBy + ' DESC' + limitQuery, limitValues, (err, rows) => {
let whereClause = '';
let whereValues = [];
if (queryData) {
whereClause = ' WHERE ' + queryData.where;
whereValues = queryData.values;
}
connection.query('SELECT SQL_CALC_FOUND_ROWS ' + fields.join(', ') + ' FROM ' + source + whereClause + ' ORDER BY ' + orderBy + ' DESC' + limitQuery, whereValues.concat(limitValues), (err, rows) => {
if (err) {
connection.release();
return callback(err);

10
lib/tools.js
View file

@ -28,6 +28,7 @@ module.exports = {
prepareHtml,
purifyHTML,
mergeTemplateIntoLayout,
nameToFileName,
workers: new Set()
};
@ -300,3 +301,12 @@ function mergeTemplateIntoLayout(template, layout, callback) {
return done(template, layout);
}
}
function nameToFileName(name) {
return name.
trim().
toLowerCase().
replace(/[ .+/]/g, '-').
replace(/[^a-z0-9\-_]/gi, '').
replace(/--*/g, '-');
}

8
public/css/mailtrain.css
View file

@ -38,4 +38,12 @@ tbody>tr.selected {
.table-hover>tbody>tr.selected:hover {
background-color: rgb(205, 212, 226);
}
.row-actions .row-action {
padding-right: 15px;
}
.row-actions .row-action:last-child {
padding-right: 0px;
}

314
public/javascript/tables.js
View file

@ -4,7 +4,62 @@
'use strict';
(function(){
(function() {
function refreshTargets(data) {
for (var target in data) {
var newContent = $(data[target]);
$(target).replaceWith(newContent);
installHandlers(newContent.parent());
}
}
function getAjaxUrl(self) {
var topicId = self.data('topicId');
var topicUrl = self.data('topicUrl');
return topicUrl + '/ajax/' + topicId;
}
function setupAjaxRefresh() {
var self = $(this);
var ajaxUrl = getAjaxUrl(self);
var interval = Number(self.data('interval')) || 60;
setTimeout(function () {
$.get(ajaxUrl, function(data) {
refreshTargets(data);
});
}, interval * 1000);
}
function setupAjaxAction() {
var self = $(this);
var ajaxUrl = getAjaxUrl(self);
var processing = false;
self.click(function () {
if (!processing) {
$.get(ajaxUrl, function (data) {
refreshTargets(data);
});
processing = true;
}
return false;
});
}
function setupDatestring() {
var self = $(this);
self.html(moment(self.data('date')).fromNow());
}
function getDataTableOptions(elem) {
var rowSort = $(elem).data('rowSort') || false;
@ -92,6 +147,15 @@
return opts;
}
function installHandlers(elem) {
$('.ajax-refresh', elem).each(setupAjaxRefresh);
$('.ajax-action', elem).each(setupAjaxAction);
$('.datestring', elem).each(setupDatestring);
}
installHandlers($(document));
$('.data-table').each(function () {
var opts = getDataTableOptions(this);
$(this).DataTable(opts);
@ -112,131 +176,135 @@
opts.serverSide = true;
opts.processing = true;
opts.createdRow = function( row, data, dataIndex ) {
installHandlers($(row));
}
$(this).DataTable(opts).on('draw', function () {
$('.datestring').each(function () {
$(this).html(moment($(this).data('date')).fromNow());
});
$('.datestring').each(setupDatestring);
});
});
$('.data-stats-pie-chart').each(function () {
var column = $(this).data('column') || 'country';
var limit = $(this).data('limit') || 20;
var topicId = $(this).data('topicId');
var topicUrl = $(this).data('topicUrl') || '/campaigns/clicked';
var ajaxUrl = topicUrl + '/ajax/' + topicId + '/stats';
var self = $(this);
$.post(ajaxUrl, {column: column, limit: limit}, function(data) {
google.charts.load('current', {'packages':['corechart']});
google.charts.setOnLoadCallback(drawChart);
function drawChart() {
var gTable = new google.visualization.DataTable();
gTable.addColumn('string', 'Column');
gTable.addColumn('number', 'Value');
gTable.addRows(data.data);
var options = {'width':500, 'height':400};
var chart = new google.visualization.PieChart(self[0]);
chart.draw(gTable, options);
}
});
});
$('.datestring').each(function () {
$(this).html(moment($(this).data('date')).fromNow());
});
$('.delete-form,.confirm-submit').on('submit', function (e) {
if (!confirm($(this).data('confirmMessage') || 'Are you sure? This action can not be undone')) {
e.preventDefault();
}
});
$('.fm-date-us.date').datepicker({
format: 'mm/dd/yyyy',
weekStart: 0,
autoclose: true
});
$('.fm-date-eur.date').datepicker({
format: 'dd/mm/yyyy',
weekStart: 1,
autoclose: true
});
$('.fm-date-generic.date').datepicker({
format: 'yyyy-mm-dd',
weekStart: 1,
autoclose: true
});
$('.fm-birthday-us.date').datepicker({
format: 'mm/dd',
weekStart: 0,
autoclose: true
});
$('.fm-birthday-eur.date').datepicker({
format: 'dd/mm',
weekStart: 1,
autoclose: true
});
$('.fm-birthday-generic.date').datepicker({
format: 'mm-dd',
weekStart: 1,
autoclose: true
});
$('.page-refresh').each(function () {
var interval = Number($(this).data('interval')) || 60;
setTimeout(function () {
window.location.reload();
}, interval * 1000);
});
$('.click-select').on('click', function () {
$(this).select();
});
if (typeof moment.tz !== 'undefined') {
(function () {
var tz = moment.tz.guess();
if (tz) {
$('.tz-detect').val(tz);
}
})();
}
// setup SMTP check
var smtpForm = document.querySelector('form#smtp-verify');
if (smtpForm) {
smtpForm.addEventListener('submit', function (e) {
e.preventDefault();
var form = document.getElementById('settings-form');
var formData = new FormData(form);
var result = fetch('/settings/smtp-verify', {
method: 'POST',
body: formData,
credentials: 'same-origin'
});
var $btn = $('#verify-button').button('loading');
result.then(function (res) {
return res.json();
}).then(function (data) {
alert(data.error ? 'Invalid Mailer settings\n' + data.error : data.message);
$btn.button('reset');
}).catch(function (err) {
alert(err.message);
$btn.button('reset');
});
});
}
})();
$('.data-stats-pie-chart').each(function () {
var column = $(this).data('column') || 'country';
var limit = $(this).data('limit') || 20;
var topicId = $(this).data('topicId');
var topicUrl = $(this).data('topicUrl') || '/campaigns/clicked';
var ajaxUrl = topicUrl + '/ajax/' + topicId + '/stats';
var self = $(this);
$.post(ajaxUrl, {column: column, limit: limit}, function(data) {
google.charts.load('current', {'packages':['corechart']});
google.charts.setOnLoadCallback(drawChart);
function drawChart() {
var gTable = new google.visualization.DataTable();
gTable.addColumn('string', 'Column');
gTable.addColumn('number', 'Value');
gTable.addRows(data.data);
var options = {'width':500, 'height':400};
var chart = new google.visualization.PieChart(self[0]);
chart.draw(gTable, options);
}
});
});
$('.datestring').each(function () {
$(this).html(moment($(this).data('date')).fromNow());
});
$('.delete-form,.confirm-submit').on('submit', function (e) {
if (!confirm($(this).data('confirmMessage') || 'Are you sure? This action can not be undone')) {
e.preventDefault();
}
});
$('.fm-date-us.date').datepicker({
format: 'mm/dd/yyyy',
weekStart: 0,
autoclose: true
});
$('.fm-date-eur.date').datepicker({
format: 'dd/mm/yyyy',
weekStart: 1,
autoclose: true
});
$('.fm-date-generic.date').datepicker({
format: 'yyyy-mm-dd',
weekStart: 1,
autoclose: true
});
$('.fm-birthday-us.date').datepicker({
format: 'mm/dd',
weekStart: 0,
autoclose: true
});
$('.fm-birthday-eur.date').datepicker({
format: 'dd/mm',
weekStart: 1,
autoclose: true
});
$('.fm-birthday-generic.date').datepicker({
format: 'mm-dd',
weekStart: 1,
autoclose: true
});
$('.page-refresh').each(function () {
var interval = Number($(this).data('interval')) || 60;
setTimeout(function () {
window.location.reload();
}, interval * 1000);
});
$('.click-select').on('click', function () {
$(this).select();
});
if (typeof moment.tz !== 'undefined') {
(function () {
var tz = moment.tz.guess();
if (tz) {
$('.tz-detect').val(tz);
}
})();
}
// setup SMTP check
var smtpForm = document.querySelector('form#smtp-verify');
if (smtpForm) {
smtpForm.addEventListener('submit', function (e) {
e.preventDefault();
var form = document.getElementById('settings-form');
var formData = new FormData(form);
var result = fetch('/settings/smtp-verify', {
method: 'POST',
body: formData,
credentials: 'same-origin'
});
var $btn = $('#verify-button').button('loading');
result.then(function (res) {
return res.json();
}).then(function (data) {
alert(data.error ? 'Invalid Mailer settings\n' + data.error : data.message);
$btn.button('reset');
}).catch(function (err) {
alert(err.message);
$btn.button('reset');
});
});
}

2
routes/report-templates.js
View file

@ -93,7 +93,7 @@ router.get('/create', passport.csrfProtection, (req, res) => {
'<h2>{{title}}</h2>\n' +
'\n' +
'<div class="table-responsive">\n' +
' <table class="table table-bordered table-hover data-table display nowrap" width="100%" data-row-sort="1,1,1" data-paging="false">\n' +
' <table class="table table-bordered table-hover data-table display nowrap" width="100%" data-row-sort="1,1" data-paging="false">\n' +
' <thead>\n' +
' <th>\n' +
' {{#translate}}Email{{/translate}}\n' +

138
routes/reports.js
View file

@ -6,6 +6,7 @@ const router = new express.Router();
const _ = require('../lib/translate')._;
const reportTemplates = require('../lib/models/report-templates');
const reports = require('../lib/models/reports');
const reportProcessor = require('../services/report-processor');
const campaigns = require('../lib/models/campaigns');
const lists = require('../lib/models/lists');
const tools = require('../lib/tools');
@ -13,7 +14,7 @@ const util = require('util');
const htmlescape = require('escape-html');
const striptags = require('striptags');
const fs = require('fs');
const fsTools = require('../lib/fs-tools');
const hbs = require('hbs');
router.all('/*', (req, res, next) => {
if (!req.user) {
@ -30,26 +31,9 @@ router.get('/', (req, res) => {
});
});
router.post('/ajax', (req, res) => {
function getViewLink(row) {
if (row.state == 0) {
// TODO: Render waiting
// TODO: Add error output
return '<span class="glyphicon glyphicon-hourglass" aria-hidden="true"></span> ';
} else if (row.state == 1) {
let icon = 'eye-open';
if (row.mimeType == 'text/csv') icon = 'download-alt';
// TODO: Add error output
return '<a href="/reports/view/' + row.id + '"><span class="glyphicon glyphicon-' + icon + '" aria-hidden="true"></span></a> ';
} else if (row.state == 2) {
// TODO: Add error output
return '<span class="glyphicon glyphicon-thumbs-down" aria-hidden="true"></span> ';
}
return '';
}
reports.filter(req.body, (err, data, total, filteredTotal) => {
if (err) {
return res.json({
@ -67,14 +51,29 @@ router.post('/ajax', (req, res) => {
htmlescape(row.name || ''),
htmlescape(row.reportTemplateName || ''),
htmlescape(striptags(row.description) || ''),
'<span class="datestring" data-date="' + row.created.toISOString() + '" title="' + row.created.toISOString() + '">' + row.created.toISOString() + '</span>',
getViewLink(row) +
'<a href="/reports/edit/' + row.id + '"><span class="glyphicon glyphicon-wrench" aria-hidden="true"></span></a>']
)
getRowLastRun(row),
getRowActions(row)
])
});
});
});
router.get('/row/ajax/:id', (req, res) => {
respondRowActions(req.params.id, res);
});
router.get('/start/ajax/:id', (req, res) => {
reportProcessor.start(req.params.id, () => {
respondRowActions(req.params.id, res);
});
});
router.get('/stop/ajax/:id', (req, res) => {
reportProcessor.stop(req.params.id, () => {
respondRowActions(req.params.id, res);
});
});
router.get('/create', passport.csrfProtection, (req, res) => {
const reqData = req.query;
reqData.csrfToken = req.csrfToken();
@ -116,7 +115,6 @@ router.get('/create', passport.csrfProtection, (req, res) => {
router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) => {
const reqData = req.body;
delete reqData.filename; // This is to make sure no one inserts a fake filename when editing the report.
const reportTemplateId = Number(reqData.reportTemplate);
@ -131,6 +129,9 @@ router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) =
req.flash('danger', err && err.message || err || _('Could not create report'));
return res.redirect('/reports/create?' + tools.queryParams(data));
}
reportProcessor.start(id);
req.flash('success', util.format(_('Report “%s” created'), data.name));
res.redirect('/reports');
});
@ -179,8 +180,6 @@ router.get('/edit/:id', passport.csrfProtection, (req, res) => {
router.post('/edit', passport.parseForm, passport.csrfProtection, (req, res) => {
const reqData = req.body;
delete reqData.filename; // This is to make sure no one inserts a fake filename when editing the report.
const reportTemplateId = Number(reqData.reportTemplate);
addParamsObject(reportTemplateId, reqData, (err, data) => {
@ -231,10 +230,10 @@ router.get('/view/:id', passport.csrfProtection, (req, res) => {
return res.redirect('/reports');
}
if (report.state == 1) {
if (report.state == reports.ReportState.FINISHED) {
if (reportTemplate.mimeType == 'text/html') {
fs.readFile(path.join(__dirname, '../protected/reports', report.filename + '.report'), (err, reportContent) => {
fs.readFile(reportProcessor.getFileName(report, 'report'), (err, reportContent) => {
if (err) {
req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
return res.redirect('/reports');
@ -251,11 +250,11 @@ router.get('/view/:id', passport.csrfProtection, (req, res) => {
} else if (reportTemplate.mimeType == 'text/csv') {
const headers = {
'Content-Disposition': 'attachment;filename=' + fsTools.nameToFileName(report.name) + '.csv',
'Content-Disposition': 'attachment;filename=' + tools.nameToFileName(report.name) + '.csv',
'Content-Type': 'text/csv'
};
res.sendFile(path.join(__dirname, '../protected/reports', report.filename + '.report'), {headers: headers});
res.sendFile(reportProcessor.getFileName(report, 'report'), {headers: headers});
} else {
req.flash('danger', _('Unknown type of template'));
@ -270,6 +269,83 @@ router.get('/view/:id', passport.csrfProtection, (req, res) => {
});
});
router.get('/output/:id', passport.csrfProtection, (req, res) => {
reports.get(req.params.id, (err, report) => {
if (err || !report) {
req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
return res.redirect('/reports');
}
fs.readFile(reportProcessor.getFileName(report, 'output'), (err, output) => {
let data = {
csrfToken: req.csrfToken(),
title: 'Output for report ' + report.name
};
if (err) {
data.error = 'No output.';
} else {
data.output = output;
}
res.render('reports/output', data);
});
});
});
function getRowLastRun(row) {
return '<span id="row-last-run-' + row.id + '">' + (row.lastRun ? '<span class="datestring" data-date="' + row.lastRun.toISOString() + '" title="' + row.lastRun.toISOString() + '">' + row.lastRun.toISOString() + '</span>' : '') + '</span>';
}
function getRowActions(row) {
let requestRefresh = false;
let view, startStop;
let topic = 'data-topic-id="' + row.id + '"';
if (row.state == reports.ReportState.PROCESSING || row.state == reports.ReportState.SCHEDULED) {
view = '<span class="row-action glyphicon glyphicon-hourglass" aria-hidden="true" title="Processing"></span>';
startStop = '<a class="row-action ajax-action" href="" data-topic-url="/reports/stop" ' + topic + ' title="Stop"><span class="glyphicon glyphicon-stop" aria-hidden="true"></span></a>';
requestRefresh = true;
} else if (row.state == reports.ReportState.FINISHED) {
let icon = 'eye-open';
if (row.mimeType == 'text/csv') icon = 'download-alt';
view = '<a class="row-action" href="/reports/view/' + row.id + '" title="View report"><span class="glyphicon glyphicon-' + icon + '" aria-hidden="true"></span></a>';
startStop = '<a class="row-action ajax-action" href="" data-topic-url="/reports/start" ' + topic + ' title="Refresh report"><span class="glyphicon glyphicon-repeat" aria-hidden="true"></span></a>';
} else if (row.state == reports.ReportState.FAILED) {
view = '<span class="row-action glyphicon glyphicon-thumbs-down" aria-hidden="true" title="Report generation failed"></span>';
startStop = '<a class="row-action ajax-action" href="" data-topic-url="/reports/start" ' + topic + ' title="Refresh report"><span class="glyphicon glyphicon-repeat" aria-hidden="true"></span></a>';
}
let actions = view;
actions += '<a class="row-action" href="/reports/output/' + row.id + '" title="View console output"><span class="glyphicon glyphicon-modal-window" aria-hidden="true"></span></a>';
actions += startStop;
actions += '<a class="row-action" href="/reports/edit/' + row.id + '"><span class="glyphicon glyphicon-wrench" aria-hidden="true" title="Edit"></span></a>';
return '<span id="row-actions-' + row.id + '"' + (requestRefresh ? ' class="row-actions ajax-refresh" data-interval="5" data-topic-url="/reports/row" ' + topic : ' class="row-actions"') + '>' +
actions +
'</span>';
}
function respondRowActions(id, res) {
reports.get(id, (err, report) => {
if (err) {
return res.json({
error: err,
});
}
const data = {};
data['#row-last-run-' + id] = getRowLastRun(report);
data['#row-actions-' + id] = getRowActions(report);
res.json(data);
});
}
function addUserFields(reportTemplateId, reqData, report, callback) {
reportTemplates.get(reportTemplateId, (err, reportTemplate) => {
if (err) {

33
services/reports.js → services/report-processor-worker.js
View file

@ -11,8 +11,7 @@ const hbs = require('hbs');
const vm = require('vm');
const log = require('npmlog');
const fs = require('fs');
const path = require('path');
const fsTools = require('../lib/fs-tools');
const reportProcessor = require('./report-processor');
handlebarsHelpers.registerHelpers(handlebars);
@ -74,64 +73,52 @@ function resolveUserFields(userFields, params, callback) {
}
function doneSuccess(id) {
// TODO: Mark in the DB as completed + update the date/time
// TODO update the report filename in the DB
process.exit(0);
}
function doneFail(id) {
// TODO: Mark in the DB as failed
process.exit(1);
}
function start(id) {
// TODO: Mark in the DB as running
}
// TODO: Retrieve report task from the DB and run it
function processReport(reportId) {
reports.get(reportId, (err, report) => {
if (err || !report) {
log.error('reports', err && err.message || err || _('Could not find report with specified ID'));
doneFail(reportId);
doneFail();
}
reportTemplates.get(report.reportTemplate, (err, reportTemplate) => {
if (err) {
log.error('reports', err && err.message || err || _('Could not find report template'));
doneFail(reportId);
doneFail();
}
resolveUserFields(reportTemplate.userFieldsObject, report.paramsObject, (err, inputs) => {
if (err) {
log.error('reports', err.message || err);
doneFail(reportId);
doneFail();
}
const filename = fsTools.nameToFileName(report.name);
const sandbox = {
require: require,
inputs: inputs,
console: console,
callback: (err, outputs) => {
if (err) {
log.error('reports', err.message || err);
doneFail(reportId);
doneFail();
}
const hbsTmpl = handlebars.compile(reportTemplate.hbs);
const reportText = hbsTmpl(outputs);
fs.writeFile(path.join(__dirname, '../protected/reports', filename + '.report'), reportText, (err, reportContent) => {
fs.writeFile(reportProcessor.getFileName(report, 'report'), reportText, (err, reportContent) => {
if (err) {
log.error('reports', err && err.message || err || _('Could not find report with specified ID'));
doneFail(reportId);
doneFail();
}
doneSuccess(reportId, filename);
process
doneSuccess();
});
}
};
@ -143,4 +130,4 @@ function processReport(reportId) {
});
}
processReport(1);
processReport(Number(process.argv[2]));

154
services/report-processor.js Normal file
View file

@ -0,0 +1,154 @@
'use strict';
const log = require('npmlog');
const db = require('../lib/db');
const reports = require('../lib/models/reports');
const _ = require('../lib/translate')._;
const path = require('path');
const tools = require('../lib/tools');
const fs = require('fs');
const fork = require('child_process').fork;
let runningWorkersCount = 0;
let maxWorkersCount = 1;
let workers = {};
function getFileName(report, suffix) {
return path.join(__dirname, '..', 'protected', 'reports', report.id + '-' + tools.nameToFileName(report.name) + '.' + suffix);
}
module.exports.getFileName = getFileName;
function spawnWorker(report) {
fs.open(getFileName(report, 'output'), 'w', (err, outFd) => {
if (err) {
log.error('ReportProcessor', err);
return;
}
runningWorkersCount++;
const options = {
stdio: ['ignore', outFd, outFd, 'ipc'],
cwd: path.join(__dirname, '..')
};
let child = fork(path.join(__dirname, 'report-processor-worker.js'), [report.id], options);
let pid = child.pid;
workers[report.id] = child;
log.info('ReportProcessor', 'Worker process for "%s" started with pid %s. Current worker count is %s.', report.name, pid, runningWorkersCount);
child.on('close', (code, signal) => {
runningWorkersCount--;
delete workers[report.id];
log.info('ReportProcessor', 'Worker process for "%s" (pid %s) exited with code %s signal %s. Current worker count is %s.', report.name, pid, code, signal, runningWorkersCount);
fs.close(outFd, (err) => {
if (err) {
log.error('ReportProcessor', err);
}
const fields = {};
if (code ===0 ) {
fields.state = reports.ReportState.FINISHED;
fields.lastRun = new Date();
} else {
fields.state = reports.ReportState.FAILED;
}
reports.updateFields(report.id, fields, (err) => {
if (err) {
log.error('ReportProcessor', err);
}
setImmediate(worker);
});
});
});
});
};
function worker() {
reports.listWithState(reports.ReportState.SCHEDULED, 0, maxWorkersCount - runningWorkersCount, (err, reportList) => {
if (err) {
log.error('ReportProcessor', err);
return;
}
for (let report of reportList) {
reports.updateFields(report.id, { state: reports.ReportState.PROCESSING }, (err) => {
if (err) {
log.error('ReportProcessor', err);
return;
}
spawnWorker(report);
});
}
});
}
module.exports.start = (reportId, callback) => {
if (!workers[reportId]) {
log.info('ReportProcessor', 'Scheduling report id: %s', reportId);
reports.updateFields(reportId, { state: reports.ReportState.SCHEDULED, lastRun: null}, (err) => {
if (err) {
return callback(err);
}
if (runningWorkersCount < maxWorkersCount) {
log.info('ReportProcessor', 'Starting worker because runningWorkersCount=%s maxWorkersCount=%s', runningWorkersCount, maxWorkersCount);
worker();
} else {
log.info('ReportProcessor', 'Not starting worker because runningWorkersCount=%s maxWorkersCount=%s', runningWorkersCount, maxWorkersCount);
}
callback(null);
});
} else {
log.info('ReportProcessor', 'Worker for report id: %s is already running.', reportId);
}
};
module.exports.stop = (reportId, callback) => {
const child = workers[reportId];
if (child) {
log.info('ReportProcessor', 'Killing worker for report id: %s', reportId);
child.kill();
reports.updateFields(reportId, { state: reports.ReportState.FAILED}, callback);
} else {
log.info('ReportProcessor', 'No running worker found for report id: %s', reportId);
}
};
module.exports.init = (callback) => {
reports.listWithState(reports.ReportState.PROCESSING, 0, 0, (err, reportList) => {
if (err) {
log.error('ReportProcessor', err);
}
function scheduleReport() {
if (reportList.length > 0) {
const report = reportList.shift();
reports.updateFields(report.id, { state: reports.ReportState.SCHEDULED}, (err) => {
if (err) {
log.error('ReportProcessor', err);
}
scheduleReport();
});
}
worker();
callback();
}
scheduleReport();
});
};

1
setup/sql/upgrade-00027.sql
View file

@ -22,7 +22,6 @@ CREATE TABLE `reports` (
`description` text,
`report_template` int(11) unsigned NOT NULL,
`params` longtext,
`filename` varchar(255) DEFAULT NULL,
`state` int(11) unsigned NOT NULL DEFAULT 0,
`last_run` DATETIME DEFAULT NULL,
`created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,

4
views/campaigns/campaigns.hbs
View file

@ -23,7 +23,7 @@
<div class="table-responsive">
<table data-topic-url="/campaigns" data-sort-column="4" data-sort-order="desc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,0,1,1,0">
<thead>
<th class="col-md-1">
<th style="width: 1%">
#
</th>
<th>
@ -38,7 +38,7 @@
<th>
{{#translate}}Created{{/translate}}
</th>
<th class="col-md-1">
<th style="width: 1%">
&nbsp;
</th>
</thead>

4
views/lists/lists.hbs
View file

@ -14,7 +14,7 @@
<div class="table-responsive">
<table data-topic-url="/lists" data-sort-column="1" data-sort-order="asc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,1,1,0,0">
<thead>
<th class="col-md-1">
<th style="width: 1%">
#
</th>
<th>
@ -29,7 +29,7 @@
<th>
{{#translate}}Description{{/translate}}
</th>
<th class="col-md-1">
<th style="width: 1%">
&nbsp;
</th>
</thead>

8
views/reports/output.hbs Normal file
View file

@ -0,0 +1,8 @@
<ol class="breadcrumb">
<li><a href="/">{{#translate}}Home{{/translate}}</a></li>
<li><a href="/reports/">{{#translate}}Reports{{/translate}}</a></li>
<li class="active">{{title}}</li>
</ol>
<i>{{error}}</i>
<pre>{{output}}</pre>

5
views/reports/reports.hbs
View file

@ -16,7 +16,7 @@
<div class="table-responsive">
<table data-topic-url="/reports" data-sort-column="2" data-sort-order="desc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,1,0,1,0">
<thead>
<th class="col-md-1">
<th style="width: 1%">
#
</th>
<th>
@ -31,9 +31,10 @@
<th>
{{#translate}}Created{{/translate}}
</th>
<th class="col-md-1">
<th style="width: 1%">
&nbsp;
</th>
</thead>
</table>
</div>

4
views/templates/templates.hbs
View file

@ -14,7 +14,7 @@
<div class="table-responsive">
<table data-topic-url="/templates" data-sort-column="1" data-sort-order="asc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,0,0">
<thead>
<th class="col-md-1">
<th style="width: 1%">
#
</th>
<th>
@ -23,7 +23,7 @@
<th>
{{#translate}}Description{{/translate}}
</th>
<th class="col-md-1">
<th style="width: 1%">
&nbsp;
</th>
</thead>