The "Reports" feature seems functional.

Some small refactoring (column widths) of rendering tables in Lists, Templates, and Campaigns so that it is the same as Reports.

index.js

@@ -18,6 +18,7 @@ let tzupdate = require('./services/tzupdate');
 let feedcheck = require('./services/feedcheck');
 let dbcheck = require('./lib/dbcheck');
 let tools = require('./lib/tools');
+let reportProcessor = require('./services/report-processor');
 
 let port = config.www.port;
 let host = config.www.host;
@@ -120,23 +121,25 @@ server.on('listening', () => {
                         spawnSenders(() => {
                             feedcheck(() => {
                                 postfixBounceServer(() => {
-                                    log.info('Service', 'All services started');
+                                    reportProcessor.init(() => {
-                                    if (config.group) {
+                                        log.info('Service', 'All services started');
-                                        try {
+                                        if (config.group) {
-                                            process.setgid(config.group);
+                                            try {
-                                            log.info('Service', 'Changed group to "%s" (%s)', config.group, process.getgid());
+                                                process.setgid(config.group);
-                                        } catch (E) {
+                                                log.info('Service', 'Changed group to "%s" (%s)', config.group, process.getgid());
-                                            log.info('Service', 'Failed to change group to "%s" (%s)', config.group, E.message);
+                                            } catch (E) {
+                                                log.info('Service', 'Failed to change group to "%s" (%s)', config.group, E.message);
+                                            }
                                         }
-                                    }
+                                        if (config.user) {
-                                    if (config.user) {
+                                            try {
-                                        try {
+                                                process.setuid(config.user);
-                                            process.setuid(config.user);
+                                                log.info('Service', 'Changed user to "%s" (%s)', config.user, process.getuid());
-                                            log.info('Service', 'Changed user to "%s" (%s)', config.user, process.getuid());
+                                            } catch (E) {
-                                        } catch (E) {
+                                                log.info('Service', 'Failed to change user to "%s" (%s)', config.user, E.message);
-                                            log.info('Service', 'Failed to change user to "%s" (%s)', config.user, E.message);
+                                            }
                                         }
-                                    }
+                                    });
                                 });
                             });
                         });

lib/fs-tools.js

@@ -1,14 +0,0 @@
-'use strict';
-
-module.exports = {
-    nameToFileName,
-};
-
-function nameToFileName(name) {
-    return name.
-    trim().
-    toLowerCase().
-    replace(/[ .+/]/g, '-').
-    replace(/[^a-z0-9\-_]/gi, '').
-    replace(/--*/g, '-');
-}

lib/models/campaigns.js

@@ -19,7 +19,7 @@ let tableHelpers = require('../table-helpers');
 let allowedKeys = ['description', 'from', 'address', 'reply_to', 'subject', 'editor_name', 'editor_data', 'template', 'source_url', 'list', 'segment', 'html', 'text', 'tracking_disabled'];
 
 module.exports.list = (start, limit, callback) => {
-    tableHelpers.list('campaigns', ['*'], 'scheduled', start, limit, callback);
+    tableHelpers.list('campaigns', ['*'], 'scheduled', null, start, limit, callback);
 };
 
 module.exports.filter = (request, parent, callback) => {

lib/models/lists.js

@@ -10,7 +10,7 @@ let tableHelpers = require('../table-helpers');
 let allowedKeys = ['description', 'default_form', 'public_subscribe'];
 
 module.exports.list = (start, limit, callback) => {
-    tableHelpers.list('lists', ['*'], 'name', start, limit, callback);
+    tableHelpers.list('lists', ['*'], 'name', null, start, limit, callback);
 };
 
 module.exports.filter = (request, parent, callback) => {

lib/models/report-templates.js

@@ -8,7 +8,7 @@ const _ = require('../translate')._;
 const allowedKeys = ['name', 'description', 'mime_type', 'user_fields', 'js', 'hbs'];
 
 module.exports.list = (start, limit, callback) => {
-    tableHelpers.list('report_templates', ['*'], 'name', start, limit, callback);
+    tableHelpers.list('report_templates', ['*'], 'name', null, start, limit, callback);
 };
 
 module.exports.quicklist = callback => {

lib/models/reports.js

@@ -8,16 +8,29 @@ const tools = require('../tools');
 const _ = require('../translate')._;
 const log = require('npmlog');
 
-const allowedKeys = ['name', 'description', 'report_template', 'params', 'filename'];
+const allowedKeys = ['name', 'description', 'report_template', 'params'];
+
+const ReportState = {
+    SCHEDULED: 0,
+    PROCESSING: 1,
+    FINISHED: 2,
+    FAILED: 3
+};
+
+module.exports.ReportState = ReportState;
 
 module.exports.list = (start, limit, callback) => {
-    tableHelpers.list('reports', ['*'], 'name', start, limit, callback);
+    tableHelpers.list('reports', ['*'], 'name', null, start, limit, callback);
 };
 
+module.exports.listWithState = (state, start, limit, callback) => {
+    tableHelpers.list('reports', ['*'], 'name', { where: 'state=?', values: [state] }, start, limit, callback);
+}
+
 module.exports.filter = (request, callback) => {
     tableHelpers.filter('reports JOIN report_templates ON reports.report_template = report_templates.id',
-        ['reports.id AS id', 'reports.name AS name', 'reports.description AS description', 'reports.state AS state', 'reports.filename AS filename', 'reports.report_template AS report_template', 'reports.params AS params', 'reports.created AS created', 'report_templates.name AS report_template_name', 'report_templates.mime_type AS mime_type' ],
+        ['reports.id AS id', 'reports.name AS name', 'reports.description AS description', 'reports.state AS state', 'reports.report_template AS report_template', 'reports.params AS params', 'reports.last_run AS last_run', 'report_templates.name AS report_template_name', 'report_templates.mime_type AS mime_type' ],
-        request, ['#', 'name', 'report_templates.name', 'description', 'created'], ['name'], 'created DESC', null, callback);
+        request, ['#', 'name', 'report_templates.name', 'description', 'last_run'], ['name'], 'name ASC', null, callback);
 };
 
 module.exports.get = (id, callback) => {
@@ -60,29 +73,56 @@ module.exports.get = (id, callback) => {
     });
 };
 
-module.exports.createOrUpdate = (createMode, data, callback) => {
+// This method is not supposed to be used for unsanitized inputs. It does not do any checks.
-    data = data || {};
+module.exports.updateFields = (id, fieldValueMap, callback) => {
+    db.getConnection((err, connection) => {
+        if (err) {
+            return next(err);
+        }
 
-    const id = 'id' in data ? Number(data.id) : 0;
+        const clauses = [];
+        const values = [];
+        for (let key of Object.keys(fieldValueMap)) {
+            clauses.push(tools.toDbKey(key) + '=?');
+            values.push(fieldValueMap[key]);
+        }
+
+        values.push(id);
+
+        const query = 'UPDATE reports SET ' + clauses.join(', ') + ' WHERE id=? LIMIT 1';
+        connection.query(query, values, (err, result) => {
+            connection.release();
+            if (err) {
+                return callback(err);
+            }
+
+            return callback(null, result && result.affectedRows || false)
+        });
+    });
+};
+
+module.exports.createOrUpdate = (createMode, report, callback) => {
+    report = report || {};
+
+    const id = 'id' in report ? Number(report.id) : 0;
 
     if (!createMode && id < 1) {
         return callback(new Error(_('Missing report ID')));
     }
 
-    const template = tools.convertKeys(data);
+    const name = (report.name || '').toString().trim();
-    const name = (template.name || '').toString().trim();
 
     if (!name) {
         return callback(new Error(_('Report name must be set')));
     }
 
-    const reportTemplateId = Number(template.reportTemplate);
+    const reportTemplateId = Number(report.reportTemplate);
     reportTemplates.get(reportTemplateId, (err, reportTemplate) => {
         if (err) {
             callback(err);
         }
 
-        const params = data.paramsObject;
+        const params = report.paramsObject;
         for (const spec of reportTemplate.userFieldsObject) {
             if (params[spec.id].length < spec.minOccurences) {
                 return callback(new Error(_('At least ' + spec.minOccurences + ' rows in "' + spec.name + '" have to be selected.')));
@@ -97,8 +137,8 @@ module.exports.createOrUpdate = (createMode, data, callback) => {
         const values = [name, JSON.stringify(params)];
 
 
-        Object.keys(template).forEach(key => {
+        Object.keys(report).forEach(key => {
-            let value = typeof template[key] === 'number' ? template[key] : (template[key] || '').toString().trim();
+            let value = typeof report[key] === 'number' ? report[key] : (report[key] || '').toString().trim();
             key = tools.toDbKey(key);
 
             if (key === 'description') {

lib/models/subscriptions.js

@@ -21,7 +21,7 @@ module.exports.list = (listId, start, limit, callback) => {
         return callback(new Error('Missing List ID'));
     }
 
-    tableHelpers.list('subscription__' + listId, ['*'], 'email', start, limit, (err, rows, total) => {
+    tableHelpers.list('subscription__' + listId, ['*'], 'email', null, start, limit, (err, rows, total) => {
         if (!err) {
             rows = rows.map(row => tools.convertKeys(row));
         }

lib/models/templates.js

@@ -8,7 +8,7 @@ let tableHelpers = require('../table-helpers');
 let allowedKeys = ['description', 'editor_name', 'editor_data', 'html', 'text'];
 
 module.exports.list = (start, limit, callback) => {
-    tableHelpers.list('templates', ['*'], 'name', start, limit, callback);
+    tableHelpers.list('templates', ['*'], 'name', null, start, limit, callback);
 };
 
 module.exports.filter = (request, parent, callback) => {

lib/table-helpers.js

@@ -4,7 +4,7 @@ let db = require('./db');
 let tools = require('./tools');
 let log = require('npmlog');
 
-module.exports.list = (source, fields, orderBy, start, limit, callback) => {
+module.exports.list = (source, fields, orderBy, queryData, start, limit, callback) => {
     db.getConnection((err, connection) => {
         if (err) {
             return callback(err);
@@ -22,7 +22,15 @@ module.exports.list = (source, fields, orderBy, start, limit, callback) => {
             }
         }
 
-        connection.query('SELECT SQL_CALC_FOUND_ROWS ' + fields.join(', ') + ' FROM ' + source + ' ORDER BY ' + orderBy + ' DESC' + limitQuery, limitValues, (err, rows) => {
+        let whereClause = '';
+        let whereValues = [];
+
+        if (queryData) {
+            whereClause = ' WHERE ' + queryData.where;
+            whereValues = queryData.values;
+        }
+
+        connection.query('SELECT SQL_CALC_FOUND_ROWS ' + fields.join(', ') + ' FROM ' + source + whereClause + ' ORDER BY ' + orderBy + ' DESC' + limitQuery, whereValues.concat(limitValues), (err, rows) => {
             if (err) {
                 connection.release();
                 return callback(err);

lib/tools.js

@@ -28,6 +28,7 @@ module.exports = {
     prepareHtml,
     purifyHTML,
     mergeTemplateIntoLayout,
+    nameToFileName,
     workers: new Set()
 };
 
@@ -300,3 +301,12 @@ function mergeTemplateIntoLayout(template, layout, callback) {
         return done(template, layout);
     }
 }
+
+function nameToFileName(name) {
+    return name.
+    trim().
+    toLowerCase().
+    replace(/[ .+/]/g, '-').
+    replace(/[^a-z0-9\-_]/gi, '').
+    replace(/--*/g, '-');
+}

public/css/mailtrain.css

@@ -38,4 +38,12 @@ tbody>tr.selected {
 
 .table-hover>tbody>tr.selected:hover {
     background-color: rgb(205, 212, 226);
+}
+
+.row-actions .row-action {
+    padding-right: 15px;
+}
+
+.row-actions .row-action:last-child {
+    padding-right: 0px;
 }

public/javascript/tables.js

@@ -4,7 +4,62 @@
 
 'use strict';
 
-(function(){
+(function() {
+    function refreshTargets(data) {
+        for (var target in data) {
+            var newContent = $(data[target]);
+
+            $(target).replaceWith(newContent);
+            installHandlers(newContent.parent());
+        }
+    }
+
+    function getAjaxUrl(self) {
+        var topicId = self.data('topicId');
+        var topicUrl = self.data('topicUrl');
+
+        return topicUrl + '/ajax/' + topicId;
+    }
+
+    function setupAjaxRefresh() {
+        var self = $(this);
+        var ajaxUrl = getAjaxUrl(self);
+
+        var interval = Number(self.data('interval')) || 60;
+
+        setTimeout(function () {
+            $.get(ajaxUrl, function(data) {
+                refreshTargets(data);
+            });
+
+        }, interval * 1000);
+    }
+
+    function setupAjaxAction() {
+        var self = $(this);
+        var ajaxUrl = getAjaxUrl(self);
+
+        var processing = false;
+
+        self.click(function () {
+            if (!processing) {
+                $.get(ajaxUrl, function (data) {
+                    refreshTargets(data);
+                });
+
+                processing = true;
+            }
+
+            return false;
+        });
+    }
+
+    function setupDatestring() {
+        var self = $(this);
+        self.html(moment(self.data('date')).fromNow());
+    }
+
+
     function getDataTableOptions(elem) {
         var rowSort = $(elem).data('rowSort') || false;
 
@@ -92,6 +147,15 @@
         return opts;
     }
 
+
+    function installHandlers(elem) {
+        $('.ajax-refresh', elem).each(setupAjaxRefresh);
+        $('.ajax-action', elem).each(setupAjaxAction);
+        $('.datestring', elem).each(setupDatestring);
+    }
+
+    installHandlers($(document));
+
     $('.data-table').each(function () {
         var opts = getDataTableOptions(this);
         $(this).DataTable(opts);
@@ -112,131 +176,135 @@
         opts.serverSide = true;
         opts.processing = true;
 
+        opts.createdRow = function( row, data, dataIndex ) {
+            installHandlers($(row));
+        }
+
         $(this).DataTable(opts).on('draw', function () {
-            $('.datestring').each(function () {
+            $('.datestring').each(setupDatestring);
-                $(this).html(moment($(this).data('date')).fromNow());
-            });
         });
     });
 
+    $('.data-stats-pie-chart').each(function () {
+        var column = $(this).data('column') || 'country';
+        var limit = $(this).data('limit') || 20;
+        var topicId = $(this).data('topicId');
+        var topicUrl = $(this).data('topicUrl') || '/campaigns/clicked';
+        var ajaxUrl = topicUrl + '/ajax/' + topicId + '/stats';
+        var self = $(this);
+
+        $.post(ajaxUrl, {column: column, limit: limit}, function(data) {
+          google.charts.load('current', {'packages':['corechart']});
+          google.charts.setOnLoadCallback(drawChart);
+
+          function drawChart() {
+            var gTable = new google.visualization.DataTable();
+            gTable.addColumn('string', 'Column');
+            gTable.addColumn('number', 'Value');
+            gTable.addRows(data.data);
+
+            var options = {'width':500, 'height':400};
+            var chart = new google.visualization.PieChart(self[0]);
+            chart.draw(gTable, options);
+          }
+        });
+    });
+
+    $('.datestring').each(function () {
+        $(this).html(moment($(this).data('date')).fromNow());
+    });
+
+    $('.delete-form,.confirm-submit').on('submit', function (e) {
+        if (!confirm($(this).data('confirmMessage') || 'Are you sure? This action can not be undone')) {
+            e.preventDefault();
+        }
+    });
+
+    $('.fm-date-us.date').datepicker({
+        format: 'mm/dd/yyyy',
+        weekStart: 0,
+        autoclose: true
+    });
+
+    $('.fm-date-eur.date').datepicker({
+        format: 'dd/mm/yyyy',
+        weekStart: 1,
+        autoclose: true
+    });
+
+    $('.fm-date-generic.date').datepicker({
+        format: 'yyyy-mm-dd',
+        weekStart: 1,
+        autoclose: true
+    });
+
+    $('.fm-birthday-us.date').datepicker({
+        format: 'mm/dd',
+        weekStart: 0,
+        autoclose: true
+    });
+
+    $('.fm-birthday-eur.date').datepicker({
+        format: 'dd/mm',
+        weekStart: 1,
+        autoclose: true
+    });
+
+    $('.fm-birthday-generic.date').datepicker({
+        format: 'mm-dd',
+        weekStart: 1,
+        autoclose: true
+    });
+
+    $('.page-refresh').each(function () {
+        var interval = Number($(this).data('interval')) || 60;
+        setTimeout(function () {
+            window.location.reload();
+        }, interval * 1000);
+    });
+
+
+    $('.click-select').on('click', function () {
+        $(this).select();
+    });
+
+    if (typeof moment.tz !== 'undefined') {
+        (function () {
+            var tz = moment.tz.guess();
+            if (tz) {
+                $('.tz-detect').val(tz);
+            }
+        })();
+    }
+
+    // setup SMTP check
+    var smtpForm = document.querySelector('form#smtp-verify');
+    if (smtpForm) {
+        smtpForm.addEventListener('submit', function (e) {
+            e.preventDefault();
+
+            var form = document.getElementById('settings-form');
+            var formData = new FormData(form);
+            var result = fetch('/settings/smtp-verify', {
+                method: 'POST',
+                body: formData,
+                credentials: 'same-origin'
+            });
+
+            var $btn = $('#verify-button').button('loading');
+
+            result.then(function (res) {
+                return res.json();
+            }).then(function (data) {
+                alert(data.error ? 'Invalid Mailer settings\n' + data.error : data.message);
+                $btn.button('reset');
+            }).catch(function (err) {
+                alert(err.message);
+                $btn.button('reset');
+            });
+
+        });
+    }
+
 })();
 
-$('.data-stats-pie-chart').each(function () {
-    var column = $(this).data('column') || 'country';
-    var limit = $(this).data('limit') || 20;
-    var topicId = $(this).data('topicId');
-    var topicUrl = $(this).data('topicUrl') || '/campaigns/clicked';
-    var ajaxUrl = topicUrl + '/ajax/' + topicId + '/stats';
-    var self = $(this);
-
-    $.post(ajaxUrl, {column: column, limit: limit}, function(data) {
-      google.charts.load('current', {'packages':['corechart']});
-      google.charts.setOnLoadCallback(drawChart);
-
-      function drawChart() {
-        var gTable = new google.visualization.DataTable();
-        gTable.addColumn('string', 'Column');
-        gTable.addColumn('number', 'Value');
-        gTable.addRows(data.data);
-
-        var options = {'width':500, 'height':400};
-        var chart = new google.visualization.PieChart(self[0]);
-        chart.draw(gTable, options);
-      }
-    });
-});
-
-$('.datestring').each(function () {
-    $(this).html(moment($(this).data('date')).fromNow());
-});
-
-$('.delete-form,.confirm-submit').on('submit', function (e) {
-    if (!confirm($(this).data('confirmMessage') || 'Are you sure? This action can not be undone')) {
-        e.preventDefault();
-    }
-});
-
-$('.fm-date-us.date').datepicker({
-    format: 'mm/dd/yyyy',
-    weekStart: 0,
-    autoclose: true
-});
-
-$('.fm-date-eur.date').datepicker({
-    format: 'dd/mm/yyyy',
-    weekStart: 1,
-    autoclose: true
-});
-
-$('.fm-date-generic.date').datepicker({
-    format: 'yyyy-mm-dd',
-    weekStart: 1,
-    autoclose: true
-});
-
-$('.fm-birthday-us.date').datepicker({
-    format: 'mm/dd',
-    weekStart: 0,
-    autoclose: true
-});
-
-$('.fm-birthday-eur.date').datepicker({
-    format: 'dd/mm',
-    weekStart: 1,
-    autoclose: true
-});
-
-$('.fm-birthday-generic.date').datepicker({
-    format: 'mm-dd',
-    weekStart: 1,
-    autoclose: true
-});
-
-$('.page-refresh').each(function () {
-    var interval = Number($(this).data('interval')) || 60;
-    setTimeout(function () {
-        window.location.reload();
-    }, interval * 1000);
-});
-
-$('.click-select').on('click', function () {
-    $(this).select();
-});
-
-if (typeof moment.tz !== 'undefined') {
-    (function () {
-        var tz = moment.tz.guess();
-        if (tz) {
-            $('.tz-detect').val(tz);
-        }
-    })();
-}
-
-// setup SMTP check
-var smtpForm = document.querySelector('form#smtp-verify');
-if (smtpForm) {
-    smtpForm.addEventListener('submit', function (e) {
-        e.preventDefault();
-
-        var form = document.getElementById('settings-form');
-        var formData = new FormData(form);
-        var result = fetch('/settings/smtp-verify', {
-            method: 'POST',
-            body: formData,
-            credentials: 'same-origin'
-        });
-
-        var $btn = $('#verify-button').button('loading');
-
-        result.then(function (res) {
-            return res.json();
-        }).then(function (data) {
-            alert(data.error ? 'Invalid Mailer settings\n' + data.error : data.message);
-            $btn.button('reset');
-        }).catch(function (err) {
-            alert(err.message);
-            $btn.button('reset');
-        });
-
-    });
-}

routes/report-templates.js

@@ -93,7 +93,7 @@ router.get('/create', passport.csrfProtection, (req, res) => {
             '<h2>{{title}}</h2>\n' +
             '\n' +
             '<div class="table-responsive">\n' +
-            '    <table class="table table-bordered table-hover data-table display nowrap" width="100%" data-row-sort="1,1,1" data-paging="false">\n' +
+            '    <table class="table table-bordered table-hover data-table display nowrap" width="100%" data-row-sort="1,1" data-paging="false">\n' +
             '        <thead>\n' +
             '        <th>\n' +
             '            {{#translate}}Email{{/translate}}\n' +

routes/reports.js

@@ -6,6 +6,7 @@ const router = new express.Router();
 const _ = require('../lib/translate')._;
 const reportTemplates = require('../lib/models/report-templates');
 const reports = require('../lib/models/reports');
+const reportProcessor = require('../services/report-processor');
 const campaigns = require('../lib/models/campaigns');
 const lists = require('../lib/models/lists');
 const tools = require('../lib/tools');
@@ -13,7 +14,7 @@ const util = require('util');
 const htmlescape = require('escape-html');
 const striptags = require('striptags');
 const fs = require('fs');
-const fsTools = require('../lib/fs-tools');
+const hbs = require('hbs');
 
 router.all('/*', (req, res, next) => {
     if (!req.user) {
@@ -30,26 +31,9 @@ router.get('/', (req, res) => {
     });
 });
 
+
+
 router.post('/ajax', (req, res) => {
-    function getViewLink(row) {
-        if (row.state == 0) {
-            // TODO: Render waiting
-            // TODO: Add error output
-            return '<span class="glyphicon glyphicon-hourglass" aria-hidden="true"></span> ';
-        } else if (row.state == 1) {
-            let icon = 'eye-open';
-            if (row.mimeType == 'text/csv') icon = 'download-alt';
-
-            // TODO: Add error output
-            return '<a href="/reports/view/' + row.id + '"><span class="glyphicon glyphicon-' + icon + '" aria-hidden="true"></span></a> ';
-        } else if (row.state == 2) {
-            // TODO: Add error output
-            return '<span class="glyphicon glyphicon-thumbs-down" aria-hidden="true"></span> ';
-        }
-
-        return '';
-    }
-
     reports.filter(req.body, (err, data, total, filteredTotal) => {
         if (err) {
             return res.json({
@@ -67,14 +51,29 @@ router.post('/ajax', (req, res) => {
                 htmlescape(row.name || ''),
                 htmlescape(row.reportTemplateName || ''),
                 htmlescape(striptags(row.description) || ''),
-                '<span class="datestring" data-date="' + row.created.toISOString() + '" title="' + row.created.toISOString() + '">' + row.created.toISOString() + '</span>',
+                getRowLastRun(row),
-                getViewLink(row) +
+                getRowActions(row)
-                '<a href="/reports/edit/' + row.id + '"><span class="glyphicon glyphicon-wrench" aria-hidden="true"></span></a>']
+            ])
-            )
         });
     });
 });
 
+router.get('/row/ajax/:id', (req, res) => {
+    respondRowActions(req.params.id, res);
+});
+
+router.get('/start/ajax/:id', (req, res) => {
+    reportProcessor.start(req.params.id, () => {
+        respondRowActions(req.params.id, res);
+    });
+});
+
+router.get('/stop/ajax/:id', (req, res) => {
+    reportProcessor.stop(req.params.id, () => {
+        respondRowActions(req.params.id, res);
+    });
+});
+
 router.get('/create', passport.csrfProtection, (req, res) => {
     const reqData = req.query;
     reqData.csrfToken = req.csrfToken();
@@ -116,7 +115,6 @@ router.get('/create', passport.csrfProtection, (req, res) => {
 
 router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) => {
     const reqData = req.body;
-    delete reqData.filename; // This is to make sure no one inserts a fake filename when editing the report.
 
     const reportTemplateId = Number(reqData.reportTemplate);
 
@@ -131,6 +129,9 @@ router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) =
                 req.flash('danger', err && err.message || err || _('Could not create report'));
                 return res.redirect('/reports/create?' + tools.queryParams(data));
             }
+
+            reportProcessor.start(id);
+
             req.flash('success', util.format(_('Report “%s” created'), data.name));
             res.redirect('/reports');
         });
@@ -179,8 +180,6 @@ router.get('/edit/:id', passport.csrfProtection, (req, res) => {
 
 router.post('/edit', passport.parseForm, passport.csrfProtection, (req, res) => {
     const reqData = req.body;
-    delete reqData.filename; // This is to make sure no one inserts a fake filename when editing the report.
-
     const reportTemplateId = Number(reqData.reportTemplate);
 
     addParamsObject(reportTemplateId, reqData, (err, data) => {
@@ -231,10 +230,10 @@ router.get('/view/:id', passport.csrfProtection, (req, res) => {
                 return res.redirect('/reports');
             }
 
-            if (report.state == 1) {
+            if (report.state == reports.ReportState.FINISHED) {
                 if (reportTemplate.mimeType == 'text/html') {
 
-                    fs.readFile(path.join(__dirname, '../protected/reports', report.filename + '.report'), (err, reportContent) => {
+                    fs.readFile(reportProcessor.getFileName(report, 'report'), (err, reportContent) => {
                         if (err) {
                             req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
                             return res.redirect('/reports');
@@ -251,11 +250,11 @@ router.get('/view/:id', passport.csrfProtection, (req, res) => {
 
                 } else if (reportTemplate.mimeType == 'text/csv') {
                     const headers = {
-                        'Content-Disposition': 'attachment;filename=' + fsTools.nameToFileName(report.name) + '.csv',
+                        'Content-Disposition': 'attachment;filename=' + tools.nameToFileName(report.name) + '.csv',
                         'Content-Type': 'text/csv'
                     };
 
-                    res.sendFile(path.join(__dirname, '../protected/reports', report.filename + '.report'), {headers: headers});
+                    res.sendFile(reportProcessor.getFileName(report, 'report'), {headers: headers});
 
                 } else {
                     req.flash('danger', _('Unknown type of template'));
@@ -270,6 +269,83 @@ router.get('/view/:id', passport.csrfProtection, (req, res) => {
     });
 });
 
+router.get('/output/:id', passport.csrfProtection, (req, res) => {
+    reports.get(req.params.id, (err, report) => {
+        if (err || !report) {
+            req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
+            return res.redirect('/reports');
+        }
+
+        fs.readFile(reportProcessor.getFileName(report, 'output'), (err, output) => {
+            let data = {
+                csrfToken: req.csrfToken(),
+                title: 'Output for report ' + report.name
+            };
+
+            if (err) {
+                data.error = 'No output.';
+            } else {
+                data.output = output;
+            }
+
+            res.render('reports/output', data);
+        });
+    });
+});
+
+function getRowLastRun(row) {
+    return '<span id="row-last-run-' + row.id + '">' + (row.lastRun ? '<span class="datestring" data-date="' + row.lastRun.toISOString() + '" title="' + row.lastRun.toISOString() + '">' + row.lastRun.toISOString() + '</span>' : '') + '</span>';
+}
+
+function getRowActions(row) {
+    let requestRefresh = false;
+    let view, startStop;
+    let topic = 'data-topic-id="' + row.id + '"';
+
+    if (row.state == reports.ReportState.PROCESSING || row.state == reports.ReportState.SCHEDULED) {
+        view = '<span class="row-action glyphicon glyphicon-hourglass" aria-hidden="true" title="Processing"></span>';
+        startStop = '<a class="row-action ajax-action" href="" data-topic-url="/reports/stop" ' + topic + ' title="Stop"><span class="glyphicon glyphicon-stop" aria-hidden="true"></span></a>';
+        requestRefresh = true;
+
+    } else if (row.state == reports.ReportState.FINISHED) {
+        let icon = 'eye-open';
+        if (row.mimeType == 'text/csv') icon = 'download-alt';
+
+        view = '<a class="row-action" href="/reports/view/' + row.id + '" title="View report"><span class="glyphicon glyphicon-' + icon + '" aria-hidden="true"></span></a>';
+        startStop = '<a class="row-action ajax-action" href="" data-topic-url="/reports/start" ' + topic + ' title="Refresh report"><span class="glyphicon glyphicon-repeat" aria-hidden="true"></span></a>';
+
+    } else if (row.state == reports.ReportState.FAILED) {
+        view = '<span class="row-action glyphicon glyphicon-thumbs-down" aria-hidden="true" title="Report generation failed"></span>';
+        startStop = '<a class="row-action ajax-action" href="" data-topic-url="/reports/start" ' + topic + ' title="Refresh report"><span class="glyphicon glyphicon-repeat" aria-hidden="true"></span></a>';
+    }
+
+    let actions = view;
+    actions += '<a class="row-action" href="/reports/output/' + row.id + '" title="View console output"><span class="glyphicon glyphicon-modal-window" aria-hidden="true"></span></a>';
+    actions += startStop;
+    actions += '<a class="row-action" href="/reports/edit/' + row.id + '"><span class="glyphicon glyphicon-wrench" aria-hidden="true" title="Edit"></span></a>';
+
+    return '<span id="row-actions-' + row.id + '"' + (requestRefresh ? ' class="row-actions ajax-refresh" data-interval="5" data-topic-url="/reports/row" ' + topic : ' class="row-actions"') + '>' +
+        actions +
+        '</span>';
+}
+
+function respondRowActions(id, res) {
+    reports.get(id, (err, report) => {
+        if (err) {
+            return res.json({
+                error: err,
+            });
+        }
+
+        const data = {};
+        data['#row-last-run-' + id] = getRowLastRun(report);
+        data['#row-actions-' + id] = getRowActions(report);
+
+        res.json(data);
+    });
+}
+
+
 function addUserFields(reportTemplateId, reqData, report, callback) {
     reportTemplates.get(reportTemplateId, (err, reportTemplate) => {
         if (err) {

services/report-processor-worker.js

@@ -11,8 +11,7 @@ const hbs = require('hbs');
 const vm = require('vm');
 const log = require('npmlog');
 const fs = require('fs');
-const path = require('path');
+const reportProcessor = require('./report-processor');
-const fsTools = require('../lib/fs-tools');
 
 handlebarsHelpers.registerHelpers(handlebars);
 
@@ -74,64 +73,52 @@ function resolveUserFields(userFields, params, callback) {
 }
 
 function doneSuccess(id) {
-    // TODO: Mark in the DB as completed + update the date/time
-    // TODO update the report filename in the DB
-
     process.exit(0);
 }
 
 function doneFail(id) {
-    // TODO: Mark in the DB as failed
     process.exit(1);
 }
 
-function start(id) {
-    // TODO: Mark in the DB as running
-}
-
-// TODO: Retrieve report task from the DB and run it
-
 function processReport(reportId) {
     reports.get(reportId, (err, report) => {
         if (err || !report) {
             log.error('reports', err && err.message || err || _('Could not find report with specified ID'));
-            doneFail(reportId);
+            doneFail();
         }
 
         reportTemplates.get(report.reportTemplate, (err, reportTemplate) => {
             if (err) {
                 log.error('reports', err && err.message || err || _('Could not find report template'));
-                doneFail(reportId);
+                doneFail();
             }
 
             resolveUserFields(reportTemplate.userFieldsObject, report.paramsObject, (err, inputs) => {
                 if (err) {
                     log.error('reports', err.message || err);
-                    doneFail(reportId);
+                    doneFail();
                 }
 
-                const filename = fsTools.nameToFileName(report.name);
-
                 const sandbox = {
                     require: require,
                     inputs: inputs,
+                    console: console,
                     callback: (err, outputs) => {
                         if (err) {
                             log.error('reports', err.message || err);
-                            doneFail(reportId);
+                            doneFail();
                         }
 
                         const hbsTmpl = handlebars.compile(reportTemplate.hbs);
                         const reportText = hbsTmpl(outputs);
 
-                        fs.writeFile(path.join(__dirname, '../protected/reports', filename + '.report'), reportText, (err, reportContent) => {
+                        fs.writeFile(reportProcessor.getFileName(report, 'report'), reportText, (err, reportContent) => {
                             if (err) {
                                 log.error('reports', err && err.message || err || _('Could not find report with specified ID'));
-                                doneFail(reportId);
+                                doneFail();
                             }
 
-                            doneSuccess(reportId, filename);
+                            doneSuccess();
-                            process
                         });
                     }
                 };
@@ -143,4 +130,4 @@ function processReport(reportId) {
     });
 }
 
-processReport(1);
+processReport(Number(process.argv[2]));

services/report-processor.js

@@ -0,0 +1,154 @@
+'use strict';
+
+const log = require('npmlog');
+const db = require('../lib/db');
+const reports = require('../lib/models/reports');
+const _ = require('../lib/translate')._;
+const path = require('path');
+const tools = require('../lib/tools');
+const fs = require('fs');
+const fork = require('child_process').fork;
+
+let runningWorkersCount = 0;
+let maxWorkersCount = 1;
+
+let workers = {};
+
+function getFileName(report, suffix) {
+    return path.join(__dirname, '..', 'protected', 'reports', report.id + '-' + tools.nameToFileName(report.name) + '.' + suffix);
+}
+
+module.exports.getFileName = getFileName;
+
+function spawnWorker(report) {
+
+    fs.open(getFileName(report, 'output'), 'w', (err, outFd) => {
+        if (err) {
+            log.error('ReportProcessor', err);
+            return;
+        }
+
+        runningWorkersCount++;
+
+        const options = {
+            stdio: ['ignore', outFd, outFd, 'ipc'],
+            cwd: path.join(__dirname, '..')
+        };
+
+        let child = fork(path.join(__dirname, 'report-processor-worker.js'), [report.id], options);
+        let pid = child.pid;
+        workers[report.id] = child;
+
+        log.info('ReportProcessor', 'Worker process for "%s" started with pid %s. Current worker count is %s.', report.name, pid, runningWorkersCount);
+
+        child.on('close', (code, signal) => {
+            runningWorkersCount--;
+
+            delete workers[report.id];
+            log.info('ReportProcessor', 'Worker process for "%s" (pid %s) exited with code %s signal %s. Current worker count is %s.', report.name, pid, code, signal, runningWorkersCount);
+
+            fs.close(outFd, (err) => {
+                if (err) {
+                    log.error('ReportProcessor', err);
+                }
+
+                const fields = {};
+                if (code ===0 ) {
+                    fields.state = reports.ReportState.FINISHED;
+                    fields.lastRun = new Date();
+                } else {
+                    fields.state = reports.ReportState.FAILED;
+                }
+
+                reports.updateFields(report.id, fields, (err) => {
+                    if (err) {
+                        log.error('ReportProcessor', err);
+                    }
+
+                    setImmediate(worker);
+                });
+            });
+        });
+    });
+};
+
+function worker() {
+    reports.listWithState(reports.ReportState.SCHEDULED, 0, maxWorkersCount - runningWorkersCount, (err, reportList) => {
+        if (err) {
+            log.error('ReportProcessor', err);
+            return;
+        }
+
+        for (let report of reportList) {
+            reports.updateFields(report.id, { state: reports.ReportState.PROCESSING }, (err) => {
+                if (err) {
+                    log.error('ReportProcessor', err);
+                    return;
+                }
+
+                spawnWorker(report);
+            });
+        }
+    });
+}
+
+module.exports.start = (reportId, callback) => {
+    if (!workers[reportId]) {
+        log.info('ReportProcessor', 'Scheduling report id: %s', reportId);
+        reports.updateFields(reportId, { state: reports.ReportState.SCHEDULED, lastRun: null}, (err) => {
+            if (err) {
+                return callback(err);
+            }
+
+            if (runningWorkersCount < maxWorkersCount) {
+                log.info('ReportProcessor', 'Starting worker because runningWorkersCount=%s maxWorkersCount=%s', runningWorkersCount, maxWorkersCount);
+
+                worker();
+            } else {
+                log.info('ReportProcessor', 'Not starting worker because runningWorkersCount=%s maxWorkersCount=%s', runningWorkersCount, maxWorkersCount);
+            }
+
+            callback(null);
+        });
+    } else {
+        log.info('ReportProcessor', 'Worker for report id: %s is already running.', reportId);
+    }
+};
+
+module.exports.stop = (reportId, callback) => {
+    const child = workers[reportId];
+    if (child) {
+        log.info('ReportProcessor', 'Killing worker for report id: %s', reportId);
+        child.kill();
+        reports.updateFields(reportId, { state: reports.ReportState.FAILED}, callback);
+    } else {
+        log.info('ReportProcessor', 'No running worker found for report id: %s', reportId);
+    }
+};
+
+module.exports.init = (callback) => {
+    reports.listWithState(reports.ReportState.PROCESSING, 0, 0, (err, reportList) => {
+        if (err) {
+            log.error('ReportProcessor', err);
+        }
+
+        function scheduleReport() {
+            if (reportList.length > 0) {
+                const report = reportList.shift();
+
+                reports.updateFields(report.id, { state: reports.ReportState.SCHEDULED}, (err) => {
+                    if (err) {
+                        log.error('ReportProcessor', err);
+                    }
+
+                    scheduleReport();
+                });
+            }
+
+            worker();
+            callback();
+        }
+
+        scheduleReport();
+    });
+};

setup/sql/upgrade-00027.sql

@@ -22,7 +22,6 @@ CREATE TABLE `reports` (
   `description` text,
   `report_template` int(11) unsigned NOT NULL,
   `params` longtext,
-  `filename`  varchar(255) DEFAULT NULL,
   `state`  int(11) unsigned NOT NULL DEFAULT 0,
   `last_run` DATETIME DEFAULT NULL,
   `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,

views/campaigns/campaigns.hbs

@@ -23,7 +23,7 @@
 <div class="table-responsive">
     <table data-topic-url="/campaigns" data-sort-column="4" data-sort-order="desc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,0,1,1,0">
         <thead>
-            <th class="col-md-1">
+            <th style="width: 1%">
                 #
             </th>
             <th>
@@ -38,7 +38,7 @@
             <th>
                 {{#translate}}Created{{/translate}}
             </th>
-            <th class="col-md-1">
+            <th style="width: 1%">
                 &nbsp;
             </th>
         </thead>

views/lists/lists.hbs

@@ -14,7 +14,7 @@
 <div class="table-responsive">
     <table data-topic-url="/lists" data-sort-column="1" data-sort-order="asc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,1,1,0,0">
         <thead>
-        <th class="col-md-1">
+        <th style="width: 1%">
             #
         </th>
         <th>
@@ -29,7 +29,7 @@
         <th>
             {{#translate}}Description{{/translate}}
         </th>
-        <th class="col-md-1">
+        <th style="width: 1%">
             &nbsp;
         </th>
         </thead>

views/reports/output.hbs

@@ -0,0 +1,8 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li><a href="/reports/">{{#translate}}Reports{{/translate}}</a></li>
+    <li class="active">{{title}}</li>
+</ol>
+
+<i>{{error}}</i>
+<pre>{{output}}</pre>

views/reports/reports.hbs

@@ -16,7 +16,7 @@
 <div class="table-responsive">
     <table data-topic-url="/reports" data-sort-column="2" data-sort-order="desc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,1,0,1,0">
         <thead>
-        <th class="col-md-1">
+        <th style="width: 1%">
             #
         </th>
         <th>
@@ -31,9 +31,10 @@
         <th>
             {{#translate}}Created{{/translate}}
         </th>
-        <th class="col-md-1">
+        <th style="width: 1%">
             &nbsp;
         </th>
         </thead>
     </table>
 </div>
+

views/templates/templates.hbs

@@ -14,7 +14,7 @@
 <div class="table-responsive">
     <table data-topic-url="/templates" data-sort-column="1" data-sort-order="asc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,0,0">
         <thead>
-            <th class="col-md-1">
+            <th style="width: 1%">
                 #
             </th>
             <th>
@@ -23,7 +23,7 @@
             <th>
                 {{#translate}}Description{{/translate}}
             </th>
-            <th class="col-md-1">
+            <th style="width: 1%">
                 &nbsp;
             </th>
         </thead>