Merge pull request #207 from bures/master

Protected lists

.gitignore

@@ -3,6 +3,8 @@ npm-debug.log
 .DS_Store
 config/development.*
 config/production.*
+workers/reports/config/development.*
+workers/reports/config/production.*
 dump.rdb
 
 # generate POT file every time you want to update your PO file

app.js

@@ -1,47 +1,49 @@
 'use strict';
 
-let config = require('config');
+const config = require('config');
-let log = require('npmlog');
+const log = require('npmlog');
 
-let _ = require('./lib/translate')._;
+const _ = require('./lib/translate')._;
-let util = require('util');
 
-let express = require('express');
+const express = require('express');
-let bodyParser = require('body-parser');
+const bodyParser = require('body-parser');
-let path = require('path');
+const path = require('path');
-let favicon = require('serve-favicon');
+const favicon = require('serve-favicon');
-let logger = require('morgan');
+const logger = require('morgan');
-let cookieParser = require('cookie-parser');
+const cookieParser = require('cookie-parser');
-let session = require('express-session');
+const session = require('express-session');
-let RedisStore = require('connect-redis')(session);
+const RedisStore = require('connect-redis')(session);
-let flash = require('connect-flash');
+const flash = require('connect-flash');
-let hbs = require('hbs');
+const hbs = require('hbs');
-let compression = require('compression');
+const handlebarsHelpers = require('./lib/handlebars-helpers');
-let passport = require('./lib/passport');
+const compression = require('compression');
-let tools = require('./lib/tools');
+const passport = require('./lib/passport');
+const tools = require('./lib/tools');
 
-let routes = require('./routes/index');
+const routes = require('./routes/index');
-let users = require('./routes/users');
+const users = require('./routes/users');
-let lists = require('./routes/lists');
+const lists = require('./routes/lists');
-let settings = require('./routes/settings');
+const settings = require('./routes/settings');
-let settingsModel = require('./lib/models/settings');
+const settingsModel = require('./lib/models/settings');
-let templates = require('./routes/templates');
+const templates = require('./routes/templates');
-let campaigns = require('./routes/campaigns');
+const campaigns = require('./routes/campaigns');
-let links = require('./routes/links');
+const links = require('./routes/links');
-let fields = require('./routes/fields');
+const fields = require('./routes/fields');
-let forms = require('./routes/forms');
+const forms = require('./routes/forms');
-let segments = require('./routes/segments');
+const segments = require('./routes/segments');
-let triggers = require('./routes/triggers');
+const triggers = require('./routes/triggers');
-let webhooks = require('./routes/webhooks');
+const webhooks = require('./routes/webhooks');
-let subscription = require('./routes/subscription');
+const subscription = require('./routes/subscription');
-let archive = require('./routes/archive');
+const archive = require('./routes/archive');
-let api = require('./routes/api');
+const api = require('./routes/api');
-let blacklist = require('./routes/blacklist');
+const blacklist = require('./routes/blacklist');
-let editorapi = require('./routes/editorapi');
+const editorapi = require('./routes/editorapi');
-let grapejs = require('./routes/grapejs');
+const grapejs = require('./routes/grapejs');
-let mosaico = require('./routes/mosaico');
+const mosaico = require('./routes/mosaico');
+const reports = require('./routes/reports');
+const reportsTemplates = require('./routes/report-templates');
 
-let app = express();
+const app = express();
 
 // view engine setup
 app.set('views', path.join(__dirname, 'views'));
@@ -57,6 +59,8 @@ app.disable('x-powered-by');
 
 hbs.registerPartials(__dirname + '/views/partials');
 hbs.registerPartials(__dirname + '/views/subscription/partials/');
+hbs.registerPartials(__dirname + '/views/report-templates/partials/');
+hbs.registerPartials(__dirname + '/views/reports/partials/');
 
 /**
  * We need this helper to make sure that we consume flash messages only
@@ -104,20 +108,8 @@ hbs.registerHelper('flash_messages', function () { // eslint-disable-line prefer
     );
 });
 
-// {{#translate}}abc{{/translate}}
+handlebarsHelpers.registerHelpers(hbs.handlebars);
-hbs.registerHelper('translate', function (context, options) { // eslint-disable-line prefer-arrow-callback
-    if (typeof options === 'undefined' && context) {
-        options = context;
-        context = false;
-    }
 
-    let result = _(options.fn(this)); // eslint-disable-line no-invalid-this
-
-    if (Array.isArray(context)) {
-        result = util.format(result, ...context);
-    }
-    return new hbs.handlebars.SafeString(result);
-});
 
 app.use(compression());
 app.use(favicon(path.join(__dirname, 'public', 'favicon.ico')));
@@ -222,6 +214,11 @@ app.use('/editorapi', editorapi);
 app.use('/grapejs', grapejs);
 app.use('/mosaico', mosaico);
 
+if (config.reports && config.reports.enabled === true) {
+    app.use('/reports', reports);
+    app.use('/report-templates', reportsTemplates);
+}
+
 // catch 404 and forward to error handler
 app.use((req, res, next) => {
     let err = new Error(_('Not Found'));

config/default.toml

@@ -43,8 +43,14 @@ language="en"
 
 # If you start out as a root user (eg. if you want to use ports lower than 1000)
 # then you can downgrade the user once all services are up and running
-#user="nobody"
+#user="mailtrain"
-#group="nogroup"
+#group="mailtrain"
+
+# If Mailtrain is started as root, "Reports" feature drops the privileges of script generating the report to disallow
+# any modifications of Mailtrain code and even prohibits reading the production configuration (which contains the MySQL
+# password for read/write operations). The rouser/rogroup determines the user to be used
+#rouser="nobody"
+#rogroup="nogroup"
 
 [log]
 # silly|verbose|info|http|warn|error|silent
@@ -150,3 +156,18 @@ templates=[["versafix-1", "Versafix One"]]
 [grapejs]
 # Installed templates
 templates=[["demo", "Demo Template"]]
+
+[reports]
+# The whole reporting functionality can be disabled below if the they are not needed and the DB cannot be
+# properly protected.
+# Reports rely on custom user defined Javascript snippets defined in the report template. The snippets are run on the
+# server when generating a report. As these snippets are stored in the DB, they pose a security risk because they can
+# help gaining access to the server if the DB cannot
+# be properly protected (e.g. if it is shared with another application with security weaknesses).
+# Mailtrain mitigates this problem by running the custom Javascript snippets in a chrooted environment and under a
+# DB user that cannot modify the database (see userRO in [mysql] above). However the chrooted environment is available
+# only if Mailtrain is started as root. The chrooted environment still does not prevent the custom JS script in
+# performing network operations and in generating XSS attacks as part of the report.
+# The bottom line is that if people who are creating report templates or have write access to the DB cannot be trusted,
+# then it's safer to switch off the reporting functionality below.
+enabled=false

index.js

@@ -4,20 +4,23 @@
  * Module dependencies.
  */
 
-let config = require('config');
+const config = require('config');
-let log = require('npmlog');
+const log = require('npmlog');
-let app = require('./app');
+const app = require('./app');
-let http = require('http');
+const http = require('http');
-let fork = require('child_process').fork;
+const fork = require('child_process').fork;
-let triggers = require('./services/triggers');
+const triggers = require('./services/triggers');
-let importer = require('./services/importer');
+const importer = require('./services/importer');
-let verpServer = require('./services/verp-server');
+const verpServer = require('./services/verp-server');
-let testServer = require('./services/test-server');
+const testServer = require('./services/test-server');
-let postfixBounceServer = require('./services/postfix-bounce-server');
+const postfixBounceServer = require('./services/postfix-bounce-server');
-let tzupdate = require('./services/tzupdate');
+const tzupdate = require('./services/tzupdate');
-let feedcheck = require('./services/feedcheck');
+const feedcheck = require('./services/feedcheck');
-let dbcheck = require('./lib/dbcheck');
+const dbcheck = require('./lib/dbcheck');
-let tools = require('./lib/tools');
+const tools = require('./lib/tools');
+const reportProcessor = require('./lib/report-processor');
+const executor = require('./lib/executor');
+const privilegeHelpers = require('./lib/privilege-helpers');
 
 let port = config.www.port;
 let host = config.www.host;
@@ -112,31 +115,22 @@ server.on('listening', () => {
     log.info('Express', 'WWW server listening on %s', bind);
 
     // start additional services
-    testServer(() => {
+    function startNextServices() {
-        verpServer(() => {
+        testServer(() => {
-            tzupdate(() => {
+            verpServer(() => {
-                importer(() => {
+
-                    triggers(() => {
+                privilegeHelpers.dropRootPrivileges();
-                        spawnSenders(() => {
+
-                            feedcheck(() => {
+                tzupdate(() => {
-                                postfixBounceServer(() => {
+                    importer(() => {
-                                    log.info('Service', 'All services started');
+                        triggers(() => {
-                                    if (config.group) {
+                            spawnSenders(() => {
-                                        try {
+                                feedcheck(() => {
-                                            process.setgid(config.group);
+                                    postfixBounceServer(() => {
-                                            log.info('Service', 'Changed group to "%s" (%s)', config.group, process.getgid());
+                                        reportProcessor.init(() => {
-                                        } catch (E) {
+                                            log.info('Service', 'All services started');
-                                            log.info('Service', 'Failed to change group to "%s" (%s)', config.group, E.message);
+                                        });
-                                        }
+                                    });
-                                    }
-                                    if (config.user) {
-                                        try {
-                                            process.setuid(config.user);
-                                            log.info('Service', 'Changed user to "%s" (%s)', config.user, process.getuid());
-                                        } catch (E) {
-                                            log.info('Service', 'Failed to change user to "%s" (%s)', config.user, E.message);
-                                        }
-                                    }
                                 });
                             });
                         });
@@ -144,5 +138,11 @@ server.on('listening', () => {
                 });
             });
         });
-    });
+    }
+
+    if (config.reports && config.reports.enabled === true) {
+        executor.spawn(startNextServices);
+    } else {
+        startNextServices();
+    }
 });

lib/db.js

@@ -6,7 +6,7 @@ let redis = require('redis');
 let Lock = require('redfour');
 
 module.exports = mysql.createPool(config.mysql);
-if (config.redis.enabled) {
+if (config.redis && config.redis.enabled) {
 
     module.exports.redis = redis.createClient(config.redis);
 

lib/executor.js

@@ -0,0 +1,83 @@
+'use strict';
+
+const fork = require('child_process').fork;
+const log = require('npmlog');
+const path = require('path');
+
+const requestCallbacks = {};
+let messageTid = 0;
+let executorProcess;
+
+module.exports = {
+    spawn,
+    start,
+    stop
+};
+
+function spawn(callback) {
+    log.info('Executor', 'Spawning executor process.');
+
+    executorProcess = fork(path.join(__dirname, '..', 'services', 'executor.js'), [], {
+        cwd: path.join(__dirname, '..'),
+        env: {NODE_ENV: process.env.NODE_ENV}
+    });
+
+    executorProcess.on('message', msg => {
+        if (msg) {
+            if (msg.type === 'process-started') {
+                let requestCallback = requestCallbacks[msg.tid];
+                if (requestCallback && requestCallback.startedCallback) {
+                    requestCallback.startedCallback(msg.tid);
+                }
+
+            } else if (msg.type === 'process-failed') {
+                let requestCallback = requestCallbacks[msg.tid];
+                if (requestCallback && requestCallback.failedCallback) {
+                    requestCallback.failedCallback(msg.msg);
+                }
+
+                delete requestCallbacks[msg.tid];
+
+            } else if (msg.type === 'process-finished') {
+                let requestCallback = requestCallbacks[msg.tid];
+                if (requestCallback && requestCallback.startedCallback) {
+                    requestCallback.finishedCallback(msg.code, msg.signal);
+                }
+
+                delete requestCallbacks[msg.tid];
+
+            } else if (msg.type === 'executor-started') {
+                log.info('Executor', 'Executor process started.');
+                return callback();
+            }
+        }
+    });
+
+    executorProcess.on('close', (code, signal) => {
+        log.info('Executor', 'Executor process exited with code %s signal %s.', code, signal);
+    });
+}
+
+function start(type, data, startedCallback, finishedCallback, failedCallback) {
+    requestCallbacks[messageTid] = {
+        startedCallback,
+        finishedCallback,
+        failedCallback
+    };
+
+    executorProcess.send({
+        type: 'start-' + type,
+        data,
+        tid: messageTid
+    });
+
+    messageTid++;
+}
+
+function stop(tid) {
+    executorProcess.send({
+        type: 'stop-process',
+        tid
+    });
+}
+

lib/file-helpers.js

@@ -0,0 +1,32 @@
+'use strict';
+
+const path = require('path');
+
+function nameToFileName(name) {
+    return name.
+    trim().
+    toLowerCase().
+    replace(/[ .+/]/g, '-').
+    replace(/[^a-z0-9\-_]/gi, '').
+    replace(/--*/g, '-');
+}
+
+
+function getReportFileBase(report) {
+    return path.join(__dirname, '..', 'protected', 'reports', report.id + '-' + nameToFileName(report.name));
+}
+
+function getReportContentFile(report) {
+    return getReportFileBase(report) + '.out';
+}
+
+function getReportOutputFile(report) {
+    return getReportFileBase(report) + '.err';
+}
+
+
+module.exports = {
+    getReportContentFile,
+    getReportOutputFile,
+    nameToFileName
+};

lib/handlebars-helpers.js

@@ -0,0 +1,49 @@
+'use strict';
+
+const util = require('util');
+
+const _ = require('../lib/translate')._;
+
+module.exports.registerHelpers = handlebars => {
+    // {{#translate}}abc{{/translate}}
+    handlebars.registerHelper('translate', function (context, options) { // eslint-disable-line prefer-arrow-callback
+        if (typeof options === 'undefined' && context) {
+            options = context;
+            context = false;
+        }
+
+        let result = _(options.fn(this)); // eslint-disable-line no-invalid-this
+
+        if (Array.isArray(context)) {
+            result = util.format(result, ...context);
+        }
+        return new handlebars.SafeString(result);
+    });
+
+
+    /* Credits to http://chrismontrois.net/2016/01/30/handlebars-switch/
+
+     {{#switch letter}}
+     {{#case "a"}}
+     A is for alpaca
+     {{/case}}
+     {{#case "b"}}
+     B is for bluebird
+     {{/case}}
+     {{/switch}}
+     */
+    /* eslint no-invalid-this: "off" */
+    handlebars.registerHelper('switch', function(value, options) {
+        this._switch_value_ = value;
+        const html = options.fn(this); // Process the body of the switch block
+        delete this._switch_value_;
+        return html;
+    });
+
+    handlebars.registerHelper('case', function(value, options) {
+        if (value === this._switch_value_) {
+            return options.fn(this);
+        }
+    });
+
+};

lib/models/campaigns.js

@@ -14,186 +14,44 @@ let mailer = require('../mailer');
 let humanize = require('humanize');
 let _ = require('../translate')._;
 let util = require('util');
+let tableHelpers = require('../table-helpers');
 
 let allowedKeys = ['description', 'from', 'address', 'reply_to', 'subject', 'editor_name', 'editor_data', 'template', 'source_url', 'list', 'segment', 'html', 'text', 'tracking_disabled'];
 
 module.exports.list = (start, limit, callback) => {
-    db.getConnection((err, connection) => {
+    tableHelpers.list('campaigns', ['*'], 'scheduled', null, start, limit, callback);
-        if (err) {
-            return callback(err);
-        }
-
-        connection.query('SELECT SQL_CALC_FOUND_ROWS * FROM campaigns ORDER BY scheduled DESC LIMIT ? OFFSET ?', [limit, start], (err, rows) => {
-            if (err) {
-                connection.release();
-                return callback(err);
-            }
-            connection.query('SELECT FOUND_ROWS() AS total', (err, total) => {
-                connection.release();
-                if (err) {
-                    return callback(err);
-                }
-                return callback(null, rows, total && total[0] && total[0].total);
-            });
-        });
-    });
 };
 
 module.exports.filter = (request, parent, callback) => {
-    let columns = ['#', 'name', 'description', 'status', 'created'];
+    let queryData;
-    let processQuery = queryData => {
-
-        db.getConnection((err, connection) => {
-            if (err) {
-                return callback(err);
-            }
-
-            let query = 'SELECT COUNT(id) AS total FROM `campaigns`';
-            let values = [];
-
-            if (queryData.where) {
-                query += ' WHERE ' + queryData.where;
-                values = values.concat(queryData.values || []);
-            }
-
-            connection.query(query, values, (err, total) => {
-                if (err) {
-                    connection.release();
-                    return callback(err);
-                }
-                total = total && total[0] && total[0].total || 0;
-
-                let ordering = [];
-
-                if (request.order && request.order.length) {
-
-                    request.order.forEach(order => {
-                        let orderField = columns[Number(order.column)];
-                        let orderDirection = (order.dir || '').toString().toLowerCase() === 'desc' ? 'DESC' : 'ASC';
-                        if (orderField) {
-                            ordering.push('`' + orderField + '` ' + orderDirection);
-                        }
-                    });
-                }
-
-                if (!ordering.length) {
-                    ordering.push('`created` DESC');
-                }
-
-                let args = [Number(request.length) || 50, Number(request.start) || 0];
-                let query;
-
-                if (request.search && request.search.value) {
-                    query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `campaigns` WHERE name LIKE ? ' + (queryData.where ? ' AND (' + queryData.where + ')' : '') + ' ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-
-                    let searchVal = '%' + request.search.value.replace(/\\/g, '\\\\').replace(/([%_])/g, '\\$1') + '%';
-                    args = [searchVal].concat(queryData.values || []).concat(args);
-                } else {
-                    query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `campaigns` WHERE 1 ' + (queryData.where ? ' AND (' + queryData.where + ')' : '') + ' ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-                    args = [].concat(queryData.values || []).concat(args);
-                }
-
-                connection.query(query, args, (err, rows) => {
-                    if (err) {
-                        connection.release();
-                        return callback(err);
-                    }
-                    connection.query('SELECT FOUND_ROWS() AS total', (err, filteredTotal) => {
-                        connection.release();
-                        if (err) {
-                            return callback(err);
-                        }
-
-                        let subscriptions = rows.map(row => tools.convertKeys(row));
-
-                        filteredTotal = filteredTotal && filteredTotal[0] && filteredTotal[0].total || 0;
-                        return callback(null, subscriptions, total, filteredTotal);
-                    });
-                });
-            });
-        });
-    };
-
     if (parent) {
-        processQuery({
+        queryData = {
             // only find normal and RSS parent campaigns at this point
             where: '`parent`=?',
             values: [parent]
-        });
+        };
     } else {
-
+        queryData = {
-        processQuery({
             // only find normal and RSS parent campaigns at this point
             where: '`type` IN (?,?,?)',
             values: [1, 2, 4]
-        });
+        };
     }
+
+    tableHelpers.filter('campaigns', ['*'], request, ['#', 'name', 'description', 'status', 'created'], ['name'], 'created DESC', queryData, callback);
+};
+
+module.exports.filterQuicklist = (request, callback) => {
+    tableHelpers.filter('campaigns', ['id', 'name', 'description', 'created'], request, ['#', 'name', 'description', 'created'], ['name'], 'name ASC', null, callback);
 };
 
 module.exports.filterClickedSubscribers = (campaign, linkId, request, columns, callback) => {
-    db.getConnection((err, connection) => {
+    let queryData = {
-        if (err) {
+        where: 'campaign_tracker__' + campaign.id + '.list=? AND campaign_tracker__' + campaign.id + '.link=?',
-            return callback(err);
+        values: [campaign.list, linkId]
-        }
+    };
 
-        let query = 'SELECT COUNT(`subscription__' + campaign.list + '`.`id`) AS total FROM `subscription__' + campaign.list + '` JOIN `campaign_tracker__' + campaign.id + '` ON `campaign_tracker__' + campaign.id + '`.`list`=? AND `campaign_tracker__' + campaign.id + '`.`subscriber`=`subscription__' + campaign.list + '`.`id` AND `campaign_tracker__' + campaign.id + '`.`link`=?';
+    tableHelpers.filter('subscription__' + campaign.list + ' JOIN campaign_tracker__' + campaign.id + ' ON campaign_tracker__' + campaign.id + '.subscriber=subscription__' + campaign.list + '.id', ['*'], request, columns, ['email', 'first_name', 'last_name'], 'email ASC', queryData, callback);
-        let values = [campaign.list, linkId];
-
-        connection.query(query, values, (err, total) => {
-            if (err) {
-                connection.release();
-                return callback(err);
-            }
-            total = total && total[0] && total[0].total || 0;
-
-            let ordering = [];
-
-            if (request.order && request.order.length) {
-
-                request.order.forEach(order => {
-                    let orderField = columns[Number(order.column)];
-                    let orderDirection = (order.dir || '').toString().toLowerCase() === 'desc' ? 'DESC' : 'ASC';
-                    if (orderField) {
-                        ordering.push('`' + orderField + '` ' + orderDirection);
-                    }
-                });
-            }
-
-            if (!ordering.length) {
-                ordering.push('`email` ASC');
-            }
-
-            let args = [Number(request.length) || 50, Number(request.start) || 0];
-
-            if (request.search && request.search.value) {
-                query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + campaign.list + '`  JOIN `campaign_tracker__' + campaign.id + '` ON `campaign_tracker__' + campaign.id + '`.`list`=? AND `campaign_tracker__' + campaign.id + '`.`subscriber`=`subscription__' + campaign.list + '`.`id` AND `campaign_tracker__' + campaign.id + '`.`link`=? WHERE email LIKE ? OR first_name LIKE ? OR last_name LIKE ? ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-
-                let searchVal = '%' + request.search.value.replace(/\\/g, '\\\\').replace(/([%_])/g, '\\$1') + '%';
-                args = values.concat([searchVal, searchVal, searchVal]).concat(args);
-            } else {
-                query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + campaign.list + '` JOIN `campaign_tracker__' + campaign.id + '` ON `campaign_tracker__' + campaign.id + '`.`list`=? AND `campaign_tracker__' + campaign.id + '`.`subscriber`=`subscription__' + campaign.list + '`.`id` AND `campaign_tracker__' + campaign.id + '`.`link`=? ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-                args = values.concat(args);
-            }
-
-            connection.query(query, args, (err, rows) => {
-                if (err) {
-                    connection.release();
-                    return callback(err);
-                }
-                connection.query('SELECT FOUND_ROWS() AS total', (err, filteredTotal) => {
-                    connection.release();
-                    if (err) {
-                        return callback(err);
-                    }
-
-                    let subscriptions = rows.map(row => tools.convertKeys(row));
-
-                    filteredTotal = filteredTotal && filteredTotal[0] && filteredTotal[0].total || 0;
-                    return callback(null, subscriptions, total, filteredTotal);
-                });
-            });
-        });
-    });
 };
 
 module.exports.statsClickedSubscribersByColumn = (campaign, linkId, request, column, limit, callback) => {
@@ -233,72 +91,12 @@ module.exports.statsClickedSubscribersByColumn = (campaign, linkId, request, col
 };
 
 module.exports.filterStatusSubscribers = (campaign, status, request, columns, callback) => {
-    db.getConnection((err, connection) => {
+    let queryData = {
-        if (err) {
+        where: 'campaign__' + campaign.id + '.list=? AND campaign__' + campaign.id + '.segment=? AND campaign__' + campaign.id + '.status=?',
-            return callback(err);
+        values: [campaign.list, campaign.segment && campaign.segment.id || 0, status]
-        }
+    };
-
-        status = Number(status) || 0;
-        let query = 'SELECT COUNT(`subscription__' + campaign.list + '`.`id`) AS total FROM `subscription__' + campaign.list + '` JOIN `campaign__' + campaign.id + '` ON `campaign__' + campaign.id + '`.`list`=? AND `campaign__' + campaign.id + '`.`segment`=? AND `campaign__' + campaign.id + '`.`subscription`=`subscription__' + campaign.list + '`.`id` WHERE `campaign__' + campaign.id + '`.`status`=?';
-        let values = [campaign.list, campaign.segment && campaign.segment.id || 0, status];
-
-        connection.query(query, values, (err, total) => {
-            if (err) {
-
-                connection.release();
-                return callback(err);
-            }
-            total = total && total[0] && total[0].total || 0;
-
-            let ordering = [];
-
-            if (request.order && request.order.length) {
-
-                request.order.forEach(order => {
-                    let orderField = columns[Number(order.column)];
-                    let orderDirection = (order.dir || '').toString().toLowerCase() === 'desc' ? 'DESC' : 'ASC';
-                    if (orderField) {
-                        ordering.push('`' + orderField + '` ' + orderDirection);
-                    }
-                });
-            }
-
-            if (!ordering.length) {
-                ordering.push('`email` ASC');
-            }
-
-            let args = [Number(request.length) || 50, Number(request.start) || 0];
-
-            if (request.search && request.search.value) {
-                query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + campaign.list + '`  JOIN `campaign__' + campaign.id + '` ON `campaign__' + campaign.id + '`.`list`=? AND `campaign__' + campaign.id + '`.`segment`=? AND `campaign__' + campaign.id + '`.`subscription`=`subscription__' + campaign.list + '`.`id` WHERE `campaign__' + campaign.id + '`.`status`=? AND (email LIKE ? OR first_name LIKE ? OR last_name LIKE ?) ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-
-                let searchVal = '%' + request.search.value.replace(/\\/g, '\\\\').replace(/([%_])/g, '\\$1') + '%';
-                args = values.concat([searchVal, searchVal, searchVal]).concat(args);
-            } else {
-                query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + campaign.list + '` JOIN `campaign__' + campaign.id + '` ON `campaign__' + campaign.id + '`.`list`=? AND `campaign__' + campaign.id + '`.`segment`=? AND `campaign__' + campaign.id + '`.`subscription`=`subscription__' + campaign.list + '`.`id` WHERE `campaign__' + campaign.id + '`.`status`=? ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-                args = values.concat(args);
-            }
-
-            connection.query(query, args, (err, rows) => {
-                if (err) {
-                    connection.release();
-                    return callback(err);
-                }
-                connection.query('SELECT FOUND_ROWS() AS total', (err, filteredTotal) => {
-                    connection.release();
-                    if (err) {
-                        return callback(err);
-                    }
-
-                    let subscriptions = rows.map(row => tools.convertKeys(row));
-
-                    filteredTotal = filteredTotal && filteredTotal[0] && filteredTotal[0].total || 0;
-                    return callback(null, subscriptions, total, filteredTotal);
-                });
-            });
-        });
-    });
 
+    tableHelpers.filter('subscription__' + campaign.list + ' JOIN campaign__' + campaign.id + ' ON campaign__' + campaign.id + '.subscription=subscription__' + campaign.list + '.id', ['*'], request, columns, ['email', 'first_name', 'last_name'], 'email ASC', queryData, callback);
 };
 
 module.exports.getByCid = (cid, callback) => {

lib/models/lists.js

@@ -5,29 +5,20 @@ let tools = require('../tools');
 let shortid = require('shortid');
 let segments = require('./segments');
 let _ = require('../translate')._;
+let tableHelpers = require('../table-helpers');
 
-let allowedKeys = ['description', 'default_form'];
+let allowedKeys = ['description', 'default_form', 'public_subscribe'];
 
 module.exports.list = (start, limit, callback) => {
-    db.getConnection((err, connection) => {
+    tableHelpers.list('lists', ['*'], 'name', null, start, limit, callback);
-        if (err) {
+};
-            return callback(err);
-        }
 
-        connection.query('SELECT SQL_CALC_FOUND_ROWS * FROM lists ORDER BY name LIMIT ? OFFSET ?', [limit, start], (err, rows) => {
+module.exports.filter = (request, parent, callback) => {
-            if (err) {
+    tableHelpers.filter('lists', ['*'], request, ['#', 'name', 'cid', 'subscribers', 'description'], ['name'], 'name ASC', null, callback);
-                connection.release();
+};
-                return callback(err);
+
-            }
+module.exports.filterQuicklist = (request, callback) => {
-            connection.query('SELECT FOUND_ROWS() AS total', (err, total) => {
+    tableHelpers.filter('lists', ['id', 'name', 'subscribers'], request, ['#', 'name', 'subscribers'], ['name'], 'name ASC', null, callback);
-                connection.release();
-                if (err) {
-                    return callback(err);
-                }
-                return callback(null, rows, total && total[0] && total[0].total);
-            });
-        });
-    });
 };
 
 module.exports.quicklist = callback => {
@@ -111,6 +102,8 @@ module.exports.get = (id, callback) => {
 module.exports.create = (list, callback) => {
 
     let data = tools.convertKeys(list);
+    data.publicSubscribe = data.publicSubscribe ? 1 : 0;
+
     let name = (data.name || '').toString().trim();
 
     if (!data) {
@@ -120,8 +113,8 @@ module.exports.create = (list, callback) => {
     let keys = ['name'];
     let values = [name];
 
-    Object.keys(list).forEach(key => {
+    Object.keys(data).forEach(key => {
-        let value = list[key].trim();
+        let value = data[key].toString().trim();
         key = tools.toDbKey(key);
         if (key === 'description') {
             value = tools.purifyHTML(value);
@@ -169,6 +162,7 @@ module.exports.update = (id, updates, callback) => {
     id = Number(id) || 0;
 
     let data = tools.convertKeys(updates);
+    data.publicSubscribe = data.publicSubscribe ? 1 : 0;
 
     let name = (data.name || '').toString().trim();
     let keys = ['name'];
@@ -182,8 +176,8 @@ module.exports.update = (id, updates, callback) => {
         return callback(new Error(_('List Name must be set')));
     }
 
-    Object.keys(updates).forEach(key => {
+    Object.keys(data).forEach(key => {
-        let value = updates[key].trim();
+        let value = data[key].toString().trim();
         key = tools.toDbKey(key);
         if (key === 'description') {
             value = tools.purifyHTML(value);

lib/models/report-templates.js

@@ -0,0 +1,161 @@
+'use strict';
+
+const db = require('../db');
+const tableHelpers = require('../table-helpers');
+const tools = require('../tools');
+const _ = require('../translate')._;
+
+const allowedKeys = ['name', 'description', 'mime_type', 'user_fields', 'js', 'hbs'];
+
+module.exports.list = (start, limit, callback) => {
+    tableHelpers.list('report_templates', ['*'], 'name', null, start, limit, callback);
+};
+
+module.exports.quicklist = callback => {
+    tableHelpers.quicklist('report_templates', ['id', 'name'], 'name', callback);
+};
+
+module.exports.filter = (request, callback) => {
+    tableHelpers.filter('report_templates', ['*'], request, ['#', 'name', 'description', 'created'], ['name'], 'created DESC', null, callback);
+};
+
+module.exports.get = (id, callback) => {
+    id = Number(id) || 0;
+
+    if (id < 1) {
+        return callback(new Error(_('Missing report template ID')));
+    }
+
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        connection.query('SELECT * FROM report_templates WHERE id=?', [id], (err, rows) => {
+            connection.release();
+            if (err) {
+                return callback(err);
+            }
+
+            if (!rows || !rows.length) {
+                return callback(null, false);
+            }
+
+            const template = tools.convertKeys(rows[0]);
+
+            const userFields = template.userFields.trim();
+            if (userFields !== '') {
+                try {
+                    template.userFieldsObject = JSON.parse(userFields);
+                } catch (err) {
+                    // This is to handle situation when for some reason we get corrupted JSON in the DB.
+                    template.userFieldsObject = {};
+                    template.userFields = '{}';
+                }
+            } else {
+                template.userFieldsObject = {};
+            }
+
+            return callback(null, template);
+        });
+    });
+};
+
+module.exports.createOrUpdate = (createMode, data, callback) => {
+    data = data || {};
+
+    const id = 'id' in data ? Number(data.id) : 0;
+
+    if (!createMode && id < 1) {
+        return callback(new Error(_('Missing report template ID')));
+    }
+
+    const template = tools.convertKeys(data);
+    const name = (template.name || '').toString().trim();
+
+    if (!name) {
+        return callback(new Error(_('Report template name must be set')));
+    }
+
+    const keys = ['name'];
+    const values = [name];
+
+    Object.keys(template).forEach(key => {
+        let value = typeof template[key] === 'number' ? template[key] : (template[key] || '').toString().trim();
+        key = tools.toDbKey(key);
+
+        if (key === 'description') {
+            value = tools.purifyHTML(value);
+        }
+
+        if (key === 'user_fields') {
+            value = value.trim();
+
+            if (value !== '') {
+                try {
+                    JSON.parse(value);
+                } catch (err) {
+                    return callback(err);
+                }
+            }
+        }
+
+        if (allowedKeys.indexOf(key) >= 0 && keys.indexOf(key) < 0) {
+            keys.push(key);
+            values.push(value);
+        }
+    });
+
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        let query;
+
+        if (createMode) {
+            query = 'INSERT INTO report_templates (`' + keys.join('`, `') + '`) VALUES (' + values.map(() => '?').join(',') + ')';
+        } else {
+            query = 'UPDATE report_templates SET ' + keys.map(key => '`' + key + '`=?').join(', ') + ' WHERE id=? LIMIT 1';
+            values.push(id);
+        }
+
+        connection.query(query, values, (err, result) => {
+            connection.release();
+            if (err) {
+                return callback(err);
+            }
+
+            if (createMode) {
+                return callback(null, result && result.insertId || false);
+            } else {
+                return callback(null, result && result.affectedRows || false);
+            }
+        });
+    });
+};
+
+module.exports.delete = (id, callback) => {
+    id = Number(id) || 0;
+
+    if (id < 1) {
+        return callback(new Error(_('Missing report template ID')));
+    }
+
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        connection.query('DELETE FROM report_templates WHERE id=? LIMIT 1', [id], (err, result) => {
+            connection.release();
+            if (err) {
+                return callback(err);
+            }
+
+            const affected = result && result.affectedRows || 0;
+            return callback(err, affected);
+        });
+    });
+};
+

lib/models/reports.js

@@ -0,0 +1,261 @@
+'use strict';
+
+const db = require('../db');
+const tableHelpers = require('../table-helpers');
+const fields = require('./fields');
+const reportTemplates = require('./report-templates');
+const tools = require('../tools');
+const _ = require('../translate')._;
+
+const allowedKeys = ['name', 'description', 'report_template', 'params'];
+
+const ReportState = {
+    SCHEDULED: 0,
+    PROCESSING: 1,
+    FINISHED: 2,
+    FAILED: 3
+};
+
+module.exports.ReportState = ReportState;
+
+module.exports.list = (start, limit, callback) => {
+    tableHelpers.list('reports', ['*'], 'name', null, start, limit, callback);
+};
+
+module.exports.listWithState = (state, start, limit, callback) => {
+    tableHelpers.list('reports', ['*'], 'name', { where: 'state=?', values: [state] }, start, limit, callback);
+};
+
+module.exports.filter = (request, callback) => {
+    tableHelpers.filter('reports JOIN report_templates ON reports.report_template = report_templates.id',
+        ['reports.id AS id', 'reports.name AS name', 'reports.description AS description', 'reports.state AS state', 'reports.report_template AS report_template', 'reports.params AS params', 'reports.last_run AS last_run', 'report_templates.name AS report_template_name', 'report_templates.mime_type AS mime_type' ],
+        request, ['#', 'name', 'report_templates.name', 'description', 'last_run'], ['name'], 'name ASC', null, callback);
+};
+
+module.exports.get = (id, callback) => {
+    id = Number(id) || 0;
+
+    if (id < 1) {
+        return callback(new Error(_('Missing report ID')));
+    }
+
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        connection.query('SELECT * FROM reports WHERE id=?', [id], (err, rows) => {
+            connection.release();
+            if (err) {
+                return callback(err);
+            }
+
+            if (!rows || !rows.length) {
+                return callback(null, false);
+            }
+
+            const template = tools.convertKeys(rows[0]);
+
+            const params = template.params.trim();
+            if (params !== '') {
+                try {
+                    template.paramsObject = JSON.parse(params);
+                } catch (err) {
+                    return callback(err);
+                }
+            } else {
+                template.params = {};
+            }
+
+            return callback(null, template);
+        });
+    });
+};
+
+// This method is not supposed to be used for unsanitized inputs. It does not do any checks.
+module.exports.updateFields = (id, fieldValueMap, callback) => {
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        const clauses = [];
+        const values = [];
+        for (let key of Object.keys(fieldValueMap)) {
+            clauses.push(tools.toDbKey(key) + '=?');
+            values.push(fieldValueMap[key]);
+        }
+
+        values.push(id);
+
+        const query = 'UPDATE reports SET ' + clauses.join(', ') + ' WHERE id=? LIMIT 1';
+        connection.query(query, values, (err, result) => {
+            connection.release();
+            if (err) {
+                return callback(err);
+            }
+
+            return callback(null, result && result.affectedRows || false);
+        });
+    });
+};
+
+module.exports.createOrUpdate = (createMode, report, callback) => {
+    report = report || {};
+
+    const id = 'id' in report ? Number(report.id) : 0;
+
+    if (!createMode && id < 1) {
+        return callback(new Error(_('Missing report ID')));
+    }
+
+    const name = (report.name || '').toString().trim();
+
+    if (!name) {
+        return callback(new Error(_('Report name must be set')));
+    }
+
+    const reportTemplateId = Number(report.reportTemplate);
+    reportTemplates.get(reportTemplateId, (err, reportTemplate) => {
+        if (err) {
+            return callback(err);
+        }
+
+        const params = report.paramsObject;
+        for (const spec of reportTemplate.userFieldsObject) {
+            if (params[spec.id].length < spec.minOccurences) {
+                return callback(new Error(_('At least ' + spec.minOccurences + ' rows in "' + spec.name + '" have to be selected.')));
+            }
+
+            if (params[spec.id].length > spec.maxOccurences) {
+                return callback(new Error(_('At most ' + spec.minOccurences + ' rows in "' + spec.name + '" can be selected.')));
+            }
+        }
+
+        const keys = ['name', 'params'];
+        const values = [name, JSON.stringify(params)];
+
+
+        Object.keys(report).forEach(key => {
+            let value = typeof report[key] === 'number' ? report[key] : (report[key] || '').toString().trim();
+            key = tools.toDbKey(key);
+
+            if (key === 'description') {
+                value = tools.purifyHTML(value);
+            }
+
+            if (allowedKeys.indexOf(key) >= 0 && keys.indexOf(key) < 0) {
+                keys.push(key);
+                values.push(value);
+            }
+        });
+
+        db.getConnection((err, connection) => {
+            if (err) {
+                return callback(err);
+            }
+
+            let query;
+
+            if (createMode) {
+                query = 'INSERT INTO reports (`' + keys.join('`, `') + '`) VALUES (' + values.map(() => '?').join(',') + ')';
+            } else {
+                query = 'UPDATE reports SET ' + keys.map(key => '`' + key + '`=?').join(', ') + ' WHERE id=? LIMIT 1';
+                values.push(id);
+            }
+
+            connection.query(query, values, (err, result) => {
+                connection.release();
+                if (err) {
+                    return callback(err);
+                }
+
+                if (createMode) {
+                    return callback(null, result && result.insertId || false);
+                } else {
+                    return callback(null, result && result.affectedRows || false);
+                }
+            });
+        });
+    });
+};
+
+module.exports.delete = (id, callback) => {
+    id = Number(id) || 0;
+
+    if (id < 1) {
+        return callback(new Error(_('Missing report ID')));
+    }
+
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        connection.query('DELETE FROM reports WHERE id=? LIMIT 1', [id], (err, result) => {
+            connection.release();
+            if (err) {
+                return callback(err);
+            }
+
+            const affected = result && result.affectedRows || 0;
+            return callback(err, affected);
+        });
+    });
+};
+
+const campaignFieldsMapping = {
+    tracker_count: 'tracker.count',
+    country: 'tracker.country',
+    device_type: 'tracker.device_type',
+    status: 'campaign.status',
+    first_name: 'subscribers.first_name',
+    last_name: 'subscribers.last_name',
+    email: 'subscribers.email'
+};
+
+module.exports.getCampaignResults = (campaign, select, clause, callback) => {
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        fields.list(campaign.list, (err, fieldList) => {
+            if (err) {
+                return callback(err);
+            }
+
+            const fieldsMapping = fieldList.reduce((map, field) => {
+                map[customFieldName(field.key)] = 'subscribers.' + field.column;
+                return map;
+            }, Object.assign({}, campaignFieldsMapping));
+
+            let selFields = [];
+            for (let idx = 0; idx < select.length; idx++) {
+                const item = select[idx];
+                if (item in fieldsMapping) {
+                    selFields.push(fieldsMapping[item] + ' AS ' + item);
+                } else if (item === '*') {
+                    selFields = selFields.concat(Object.keys(fieldsMapping).map(item => fieldsMapping[item] + ' AS ' + item));
+                } else {
+                    selFields.push(item);
+                }
+            }
+
+            const query = 'SELECT ' + selFields.join(', ') + ' FROM `subscription__' + campaign.list + '` subscribers INNER JOIN `campaign__' + campaign.id + '` campaign on subscribers.id=campaign.subscription LEFT JOIN `campaign_tracker__' + campaign.id + '` tracker on subscribers.id=tracker.subscriber ' + clause;
+
+            connection.query(query, (err, results) => {
+                if (err) {
+                    connection.release();
+                    return callback(err);
+                }
+
+                return callback(null, results);
+            });
+        });
+    });
+};
+
+function customFieldName(id) {
+    return id.replace(/MERGE_/, 'CUSTOM_').toLowerCase();
+}

lib/models/subscriptions.js

@@ -13,6 +13,7 @@ let urllib = require('url');
 let log = require('npmlog');
 let _ = require('../translate')._;
 let util = require('util');
+let tableHelpers = require('../table-helpers');
 
 module.exports.list = (listId, start, limit, callback) => {
     listId = Number(listId) || 0;
@@ -20,26 +21,11 @@ module.exports.list = (listId, start, limit, callback) => {
         return callback(new Error('Missing List ID'));
     }
 
-    db.getConnection((err, connection) => {
+    tableHelpers.list('subscription__' + listId, ['*'], 'email', null, start, limit, (err, rows, total) => {
-        if (err) {
+        if (!err) {
-            return callback(err);
+            rows = rows.map(row => tools.convertKeys(row));
         }
-
+        return callback(err, rows, total);
-        connection.query('SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + listId + '` ORDER BY email LIMIT ? OFFSET ?', [limit, start], (err, rows) => {
-            if (err) {
-                connection.release();
-                return callback(err);
-            }
-            connection.query('SELECT FOUND_ROWS() AS total', (err, total) => {
-                connection.release();
-                if (err) {
-                    return callback(err);
-                }
-
-                let subscriptions = rows.map(row => tools.convertKeys(row));
-                return callback(null, subscriptions, total && total[0] && total[0].total);
-            });
-        });
     });
 };
 
@@ -80,7 +66,6 @@ module.exports.listTestUsers = (listId, callback) => {
     });
 };
 
-
 module.exports.filter = (listId, request, columns, segmentId, callback) => {
     listId = Number(listId) || 0;
     segmentId = Number(segmentId) || 0;
@@ -89,88 +74,16 @@ module.exports.filter = (listId, request, columns, segmentId, callback) => {
         return callback(new Error(_('Missing List ID')));
     }
 
-    let processQuery = queryData => {
-
-        db.getConnection((err, connection) => {
-            if (err) {
-                return callback(err);
-            }
-
-            let query = 'SELECT COUNT(id) AS total FROM `subscription__' + listId + '`';
-            let values = [];
-
-            if (queryData.where) {
-                query += ' WHERE ' + queryData.where;
-                values = values.concat(queryData.values || []);
-            }
-
-            connection.query(query, values, (err, total) => {
-                if (err) {
-                    connection.release();
-                    return callback(err);
-                }
-                total = total && total[0] && total[0].total || 0;
-
-                let ordering = [];
-
-                if (request.order && request.order.length) {
-
-                    request.order.forEach(order => {
-                        let orderField = columns[Number(order.column)];
-                        let orderDirection = (order.dir || '').toString().toLowerCase() === 'desc' ? 'DESC' : 'ASC';
-                        if (orderField) {
-                            ordering.push('`' + orderField + '` ' + orderDirection);
-                        }
-                    });
-                }
-
-                if (!ordering.length) {
-                    ordering.push('`email` ASC');
-                }
-
-                let args = [Number(request.length) || 50, Number(request.start) || 0];
-                let query;
-
-                if (request.search && request.search.value) {
-                    query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + listId + '` WHERE email LIKE ? OR first_name LIKE ? OR last_name LIKE ? ' + (queryData.where ? ' AND (' + queryData.where + ')' : '') + ' ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-
-                    let searchVal = '%' + request.search.value.replace(/\\/g, '\\\\').replace(/([%_])/g, '\\$1') + '%';
-                    args = [searchVal, searchVal, searchVal].concat(queryData.values || []).concat(args);
-                } else {
-                    query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + listId + '` WHERE 1 ' + (queryData.where ? ' AND (' + queryData.where + ')' : '') + ' ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-                    args = [].concat(queryData.values || []).concat(args);
-                }
-
-                connection.query(query, args, (err, rows) => {
-                    if (err) {
-                        connection.release();
-                        return callback(err);
-                    }
-                    connection.query('SELECT FOUND_ROWS() AS total', (err, filteredTotal) => {
-                        connection.release();
-                        if (err) {
-                            return callback(err);
-                        }
-
-                        let subscriptions = rows.map(row => tools.convertKeys(row));
-
-                        filteredTotal = filteredTotal && filteredTotal[0] && filteredTotal[0].total || 0;
-                        return callback(null, subscriptions, total, filteredTotal);
-                    });
-                });
-            });
-        });
-    };
-
     if (segmentId) {
         segments.getQuery(segmentId, false, (err, queryData) => {
             if (err) {
                 return callback(err);
             }
-            processQuery(queryData);
+
+            tableHelpers.filter('subscription__' + listId, ['*'], request, columns, ['email', 'first_name', 'last_name'], 'email ASC', queryData, callback);
         });
     } else {
-        processQuery(false);
+        tableHelpers.filter('subscription__' + listId, ['*'], request, columns, ['email', 'first_name', 'last_name'], 'email ASC', null, callback);
     }
 
 };

lib/models/templates.js

@@ -3,45 +3,20 @@
 let db = require('../db');
 let tools = require('../tools');
 let _ = require('../translate')._;
+let tableHelpers = require('../table-helpers');
 
 let allowedKeys = ['description', 'editor_name', 'editor_data', 'html', 'text'];
 
 module.exports.list = (start, limit, callback) => {
-    db.getConnection((err, connection) => {
+    tableHelpers.list('templates', ['*'], 'name', null, start, limit, callback);
-        if (err) {
+};
-            return callback(err);
-        }
 
-        connection.query('SELECT SQL_CALC_FOUND_ROWS * FROM templates ORDER BY name LIMIT ? OFFSET ?', [limit, start], (err, rows) => {
+module.exports.filter = (request, parent, callback) => {
-            if (err) {
+    tableHelpers.filter('templates', ['*'], request, ['#', 'name', 'description'], ['name'], 'name ASC', null, callback);
-                connection.release();
-                return callback(err);
-            }
-            connection.query('SELECT FOUND_ROWS() AS total', (err, total) => {
-                connection.release();
-                if (err) {
-                    return callback(err);
-                }
-                return callback(null, rows, total && total[0] && total[0].total);
-            });
-        });
-    });
 };
 
 module.exports.quicklist = callback => {
-    db.getConnection((err, connection) => {
+    tableHelpers.quicklist('templates', ['id', 'name'], 'name', callback);
-        if (err) {
-            return callback(err);
-        }
-
-        connection.query('SELECT id, name FROM templates ORDER BY name LIMIT 1000', (err, rows) => {
-            connection.release();
-            if (err) {
-                return callback(err);
-            }
-            return callback(null, (rows || []).map(tools.convertKeys));
-        });
-    });
 };
 
 module.exports.get = (id, callback) => {

lib/models/triggers.js

@@ -5,6 +5,7 @@ let db = require('../db');
 let lists = require('./lists');
 let util = require('util');
 let _ = require('../translate')._;
+let tableHelpers = require('../table-helpers');
 
 module.exports.defaultColumns = [{
     column: 'created',
@@ -339,70 +340,12 @@ module.exports.delete = (id, callback) => {
 };
 
 module.exports.filterSubscribers = (trigger, request, columns, callback) => {
-    db.getConnection((err, connection) => {
+    let queryData = {
-        if (err) {
+        where: 'trigger__' + trigger.id + '.list=?',
-            return callback(err);
+        values: [trigger.list]
-        }
+    };
-
-        let query = 'SELECT COUNT(`subscription__' + trigger.list + '`.`id`) AS total FROM `subscription__' + trigger.list + '` JOIN `trigger__' + trigger.id + '` ON `trigger__' + trigger.id + '`.`list`=? AND `trigger__' + trigger.id + '`.`subscription`=`subscription__' + trigger.list + '`.`id`';
-        let values = [trigger.list];
-
-        connection.query(query, values, (err, total) => {
-            if (err) {
-                connection.release();
-                return callback(err);
-            }
-            total = total && total[0] && total[0].total || 0;
-
-            let ordering = [];
-
-            if (request.order && request.order.length) {
-
-                request.order.forEach(order => {
-                    let orderField = columns[Number(order.column)];
-                    let orderDirection = (order.dir || '').toString().toLowerCase() === 'desc' ? 'DESC' : 'ASC';
-                    if (orderField) {
-                        ordering.push('`' + orderField + '` ' + orderDirection);
-                    }
-                });
-            }
-
-            if (!ordering.length) {
-                ordering.push('`email` ASC');
-            }
-
-            let args = [Number(request.length) || 50, Number(request.start) || 0];
-
-            if (request.search && request.search.value) {
-                query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + trigger.list + '` JOIN `trigger__' + trigger.id + '` ON `trigger__' + trigger.id + '`.`list`=? AND `trigger__' + trigger.id + '`.`subscription`=`subscription__' + trigger.list + '`.`id AND (email LIKE ? OR first_name LIKE ? OR last_name LIKE ?) ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-
-                let searchVal = '%' + request.search.value.replace(/\\/g, '\\\\').replace(/([%_])/g, '\\$1') + '%';
-                args = values.concat([searchVal, searchVal, searchVal]).concat(args);
-            } else {
-                query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + trigger.list + '` JOIN `trigger__' + trigger.id + '` ON `trigger__' + trigger.id + '`.`list`=? AND `trigger__' + trigger.id + '`.`subscription`=`subscription__' + trigger.list + '`.`id` ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-                args = values.concat(args);
-            }
-
-            connection.query(query, args, (err, rows) => {
-                if (err) {
-                    connection.release();
-                    return callback(err);
-                }
-                connection.query('SELECT FOUND_ROWS() AS total', (err, filteredTotal) => {
-                    connection.release();
-                    if (err) {
-                        return callback(err);
-                    }
-
-                    let subscriptions = rows.map(row => tools.convertKeys(row));
-
-                    filteredTotal = filteredTotal && filteredTotal[0] && filteredTotal[0].total || 0;
-                    return callback(null, subscriptions, total, filteredTotal);
-                });
-            });
-        });
-    });
 
+    tableHelpers.filter('subscription__' + trigger.list + ' JOIN trigger__' + trigger.id + ' ON trigger__' + trigger.id + '.subscription=subscription__' + trigger.list + '.id', ['*'], request, columns, ['email', 'first_name', 'last_name'], 'email ASC', queryData, callback);
 };
 
 function createTriggerTable(id, callback) {

lib/privilege-helpers.js

@@ -0,0 +1,77 @@
+'use strict';
+
+const log = require('npmlog');
+const config = require('config');
+
+const fs = require('fs');
+
+const tryRequire = require('try-require');
+const posix = tryRequire('posix');
+
+function _getConfigUidGid(prefix, defaultUid, defaultGid) {
+    let uid = defaultUid;
+    let gid = defaultGid;
+
+    if (posix) {
+        try {
+            if (config.user) {
+                uid = posix.getpwnam(config[prefix + 'user']).uid;
+            }
+        } catch (err) {
+            log.info('PrivilegeHelpers', 'Failed to resolve user id "%s"', config[prefix + 'user']);
+        }
+
+        try {
+            if (config.user) {
+                gid = posix.getpwnam(config[prefix + 'group']).gid;
+            }
+        } catch (err) {
+            log.info('PrivilegeHelpers', 'Failed to resolve group id "%s"', config[prefix + 'group']);
+        }
+    } else {
+        log.info('PrivilegeHelpers', 'Posix module not installed. Cannot resolve uid/gid');
+    }
+
+    return { uid, gid };
+}
+
+function getConfigUidGid() {
+    return _getConfigUidGid('', process.getuid(), process.getgid());
+}
+
+function getConfigROUidGid() {
+    let rwIds = getConfigUidGid();
+    return _getConfigUidGid('ro', rwIds.uid, rwIds.gid);
+}
+
+function ensureMailtrainOwner(file, callback) {
+    const ids = getConfigUidGid();
+    fs.chown(file, ids.uid, ids.gid, callback);
+}
+
+function dropRootPrivileges() {
+    if (config.group) {
+        try {
+            process.setgid(config.group);
+            log.info('PrivilegeHelpers', 'Changed group to "%s" (%s)', config.group, process.getgid());
+        } catch (E) {
+            log.info('PrivilegeHelpers', 'Failed to change group to "%s" (%s)', config.group, E.message);
+        }
+    }
+
+    if (config.user) {
+        try {
+            process.setuid(config.user);
+            log.info('PrivilegeHelpers', 'Changed user to "%s" (%s)', config.user, process.getuid());
+        } catch (E) {
+            log.info('PrivilegeHelpers', 'Failed to change user to "%s" (%s)', config.user, E.message);
+        }
+    }
+}
+
+module.exports = {
+    dropRootPrivileges,
+    ensureMailtrainOwner,
+    getConfigUidGid,
+    getConfigROUidGid
+};

lib/report-processor.js

@@ -0,0 +1,147 @@
+'use strict';
+
+const log = require('npmlog');
+const reports = require('./models/reports');
+const executor = require('./executor');
+
+let runningWorkersCount = 0;
+let maxWorkersCount = 1;
+
+let workers = {};
+
+function startWorker(report) {
+
+    function onStarted(tid) {
+        log.info('ReportProcessor', 'Worker process for "%s" started with tid %s. Current worker count is %s.', report.name, tid, runningWorkersCount);
+        workers[report.id] = tid;
+    }
+
+    function onFinished(code, signal) {
+        runningWorkersCount--;
+        log.info('ReportProcessor', 'Worker process for "%s" (tid %s) exited with code %s signal %s. Current worker count is %s.', report.name, workers[report.id], code, signal, runningWorkersCount);
+        delete workers[report.id];
+
+        const fields = {};
+        if (code === 0) {
+            fields.state = reports.ReportState.FINISHED;
+            fields.lastRun = new Date();
+        } else {
+            fields.state = reports.ReportState.FAILED;
+        }
+
+        reports.updateFields(report.id, fields, err => {
+            if (err) {
+                log.error('ReportProcessor', err);
+            }
+
+            setImmediate(startWorkers);
+        });
+    }
+
+    function onFailed(msg) {
+        runningWorkersCount--;
+        log.error('ReportProcessor', 'Executing worker process for "%s" (tid %s) failed with message "%s". Current worker count is %s.', report.name, workers[report.id], msg, runningWorkersCount);
+        delete workers[report.id];
+
+        const fields = {
+            state: reports.ReportState.FAILED
+        };
+
+        reports.updateFields(report.id, fields, err => {
+            if (err) {
+                log.error('ReportProcessor', err);
+            }
+
+            setImmediate(startWorkers);
+        });
+    }
+
+    const reportData = {
+        id: report.id,
+        name: report.name
+    };
+
+    runningWorkersCount++;
+    executor.start('report-processor-worker', reportData, onStarted, onFinished, onFailed);
+}
+
+function startWorkers() {
+    reports.listWithState(reports.ReportState.SCHEDULED, 0, maxWorkersCount - runningWorkersCount, (err, reportList) => {
+        if (err) {
+            log.error('ReportProcessor', err);
+            return;
+        }
+
+        for (let report of reportList) {
+            reports.updateFields(report.id, { state: reports.ReportState.PROCESSING }, err => {
+                if (err) {
+                    log.error('ReportProcessor', err);
+                    return;
+                }
+
+                startWorker(report);
+            });
+        }
+    });
+}
+
+module.exports.start = (reportId, callback) => {
+    if (!workers[reportId]) {
+        log.info('ReportProcessor', 'Scheduling report id: %s', reportId);
+        reports.updateFields(reportId, { state: reports.ReportState.SCHEDULED, lastRun: null}, err => {
+            if (err) {
+                return callback(err);
+            }
+
+            if (runningWorkersCount < maxWorkersCount) {
+                log.info('ReportProcessor', 'Starting worker because runningWorkersCount=%s maxWorkersCount=%s', runningWorkersCount, maxWorkersCount);
+
+                startWorkers();
+            } else {
+                log.info('ReportProcessor', 'Not starting worker because runningWorkersCount=%s maxWorkersCount=%s', runningWorkersCount, maxWorkersCount);
+            }
+
+            callback(null);
+        });
+    } else {
+        log.info('ReportProcessor', 'Worker for report id: %s is already running.', reportId);
+    }
+};
+
+module.exports.stop = (reportId, callback) => {
+    const tid = workers[reportId];
+    if (tid) {
+        log.info('ReportProcessor', 'Killing worker for report id: %s', reportId);
+        executor.stop(tid);
+        reports.updateFields(reportId, { state: reports.ReportState.FAILED}, callback);
+    } else {
+        log.info('ReportProcessor', 'No running worker found for report id: %s', reportId);
+    }
+};
+
+module.exports.init = callback => {
+    reports.listWithState(reports.ReportState.PROCESSING, 0, 0, (err, reportList) => {
+        if (err) {
+            log.error('ReportProcessor', err);
+        }
+
+        function scheduleReport() {
+            if (reportList.length > 0) {
+                const report = reportList.shift();
+
+                reports.updateFields(report.id, { state: reports.ReportState.SCHEDULED}, err => {
+                    if (err) {
+                        log.error('ReportProcessor', err);
+                    }
+
+                    scheduleReport();
+                });
+            }
+
+            startWorkers();
+            return callback();
+        }
+
+        scheduleReport();
+    });
+};

lib/table-helpers.js

@@ -0,0 +1,134 @@
+'use strict';
+
+const db = require('./db');
+const tools = require('./tools');
+
+module.exports.list = (source, fields, orderBy, queryData, start, limit, callback) => {
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        let limitQuery = '';
+        let limitValues = [];
+        if (limit) {
+            limitQuery = ' LIMIT ?';
+            limitValues.push(limit);
+
+            if (start) {
+                limitQuery += ' OFFSET ?';
+                limitValues.push(start);
+            }
+        }
+
+        let whereClause = '';
+        let whereValues = [];
+
+        if (queryData) {
+            whereClause = ' WHERE ' + queryData.where;
+            whereValues = queryData.values;
+        }
+
+        connection.query('SELECT SQL_CALC_FOUND_ROWS ' + fields.join(', ') + ' FROM ' + source + whereClause + ' ORDER BY ' + orderBy + ' DESC' + limitQuery, whereValues.concat(limitValues), (err, rows) => {
+            if (err) {
+                connection.release();
+                return callback(err);
+            }
+            connection.query('SELECT FOUND_ROWS() AS total', (err, total) => {
+                connection.release();
+                if (err) {
+                    return callback(err);
+                }
+                return callback(null, rows, total && total[0] && total[0].total);
+            });
+        });
+    });
+};
+
+module.exports.quicklist = (source, fields, orderBy, callback) => {
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        connection.query('SELECT ' + fields.join(', ') + ' FROM ' + source + ' ORDER BY ' + orderBy + ' LIMIT 1000', (err, rows) => {
+            connection.release();
+            if (err) {
+                return callback(err);
+            }
+            return callback(null, (rows || []).map(tools.convertKeys));
+        });
+    });
+};
+
+module.exports.filter = (source, fields, request, columns, searchFields, defaultOrdering, queryData, callback) => {
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        let query = 'SELECT COUNT(*) AS total FROM ' + source;
+        let values = [];
+
+        if (queryData) {
+            query += ' WHERE ' + queryData.where;
+            values = values.concat(queryData.values || []);
+        }
+
+        connection.query(query, values, (err, total) => {
+            if (err) {
+                connection.release();
+                return callback(err);
+            }
+            total = total && total[0] && total[0].total || 0;
+
+            let ordering = [];
+
+            if (request.order && request.order.length) {
+
+                request.order.forEach(order => {
+                    let orderField = columns[Number(order.column)];
+                    let orderDirection = (order.dir || '').toString().toLowerCase() === 'desc' ? 'DESC' : 'ASC';
+                    if (orderField) {
+                        ordering.push(orderField + ' ' + orderDirection);
+                    }
+                });
+            }
+
+            if (!ordering.length) {
+                ordering.push(defaultOrdering);
+            }
+
+            let searchWhere = '';
+            let searchArgs = [];
+
+            if (request.search && request.search.value) {
+                let searchVal = '%' + request.search.value.replace(/\\/g, '\\\\').replace(/([%_])/g, '\\$1') + '%';
+
+                searchWhere = searchFields.map(field => field + ' LIKE ?').join(' OR ');
+                searchArgs = searchFields.map(() => searchVal);
+            }
+
+            let query = 'SELECT SQL_CALC_FOUND_ROWS ' + fields.join(', ') + ' FROM ' + source +' WHERE ' + (searchWhere ? '(' + searchWhere + ')': '1') + (queryData ? ' AND (' + queryData.where + ')' : '') + ' ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
+            let args = searchArgs.concat(queryData ? queryData.values : []).concat([Number(request.length) || 50, Number(request.start) || 0]);
+
+            connection.query(query, args, (err, rows) => {
+                if (err) {
+                    connection.release();
+                    return callback(err);
+                }
+                connection.query('SELECT FOUND_ROWS() AS total', (err, filteredTotal) => {
+                    connection.release();
+                    if (err) {
+                        return callback(err);
+                    }
+
+                    rows = rows.map(row => tools.convertKeys(row));
+
+                    filteredTotal = filteredTotal && filteredTotal[0] && filteredTotal[0].total || 0;
+                    return callback(null, rows, total, filteredTotal);
+                });
+            });
+        });
+    });
+};

lib/tools.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const config = require('config');
 let fs = require('fs');
 let path = require('path');
 let db = require('./db');
@@ -130,6 +131,14 @@ function updateMenu(res) {
         url: '/triggers',
         key: 'triggers'
     });
+
+    if (config.reports && config.reports.enabled === true) {
+        res.locals.menu.push({
+            title: _('Reports'),
+            url: '/reports',
+            key: 'reports'
+        });
+    }
 }
 
 function validateEmail(address, checkBlocked, callback) {
@@ -296,3 +305,4 @@ function mergeTemplateIntoLayout(template, layout, callback) {
         return done(template, layout);
     }
 }
+

meta.json

@@ -1,3 +1,3 @@
 {
-    "schemaVersion": 25
+    "schemaVersion": 27
 }

package.json

@@ -33,6 +33,9 @@
     "grunt-eslint": "^19.0.0",
     "jsxgettext-andris": "^0.9.0-patch.1"
   },
+  "optionalDependencies": {
+    "posix": "^4.1.1"
+  },
   "dependencies": {
     "async": "^2.3.0",
     "aws-sdk": "^2.37.0",
@@ -97,6 +100,7 @@
     "slugify": "^1.1.0",
     "smtp-server": "^2.0.3",
     "striptags": "^3.0.1",
-    "toml": "^2.3.2"
+    "toml": "^2.3.2",
+    "try-require": "^1.2.1"
   }
 }

protected/reports/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!README.md

protected/reports/README.md

@@ -0,0 +1 @@
+This directory serves for generated reports.

public/css/mailtrain.css

@@ -31,3 +31,19 @@ h2 .glyphicon {
 h3 .glyphicon {
     font-size: .8em;
 }
+
+tbody>tr.selected {
+    background-color: rgb(218, 231, 255);
+}
+
+.table-hover>tbody>tr.selected:hover {
+    background-color: rgb(205, 212, 226);
+}
+
+.row-actions .row-action {
+    padding-right: 15px;
+}
+
+.row-actions .row-action:last-child {
+    padding-right: 0px;
+}

public/javascript/editor.js

@@ -39,6 +39,12 @@ $('div[class*="code-editor-"]').each(function () {
         editor.getSession().setUseWorker(false);
     } else if ($(this).hasClass('code-editor-css')) {
         mode = 'css';
+    } else if ($(this).hasClass('code-editor-javascript')) {
+        mode = 'javascript';
+    } else if ($(this).hasClass('code-editor-json')) {
+        mode = 'json';
+    } else if ($(this).hasClass('code-editor-handlebars')) {
+        mode = 'handlebars';
     }
 
     editor.setTheme('ace/theme/chrome');

public/javascript/tables.js

@@ -4,190 +4,307 @@
 
 'use strict';
 
-$('.data-table').each(function () {
+(function() {
-    var rowSort = $(this).data('rowSort') || false;
+    function refreshTargets(data) {
-    var columns = false;
+        for (var target in data) {
+            var newContent = $(data[target]);
 
-    if (rowSort) {
+            $(target).replaceWith(newContent);
-        columns = rowSort.split(',').map(function (sort) {
+            installHandlers(newContent.parent());
-            return {
+        }
-                orderable: sort === '1'
+    }
-            };
+
+    function getAjaxUrl(self) {
+        var topicId = self.data('topicId');
+        var topicUrl = self.data('topicUrl');
+
+        return topicUrl + '/ajax/' + topicId;
+    }
+
+    function setupAjaxRefresh() {
+        var self = $(this);
+        var ajaxUrl = getAjaxUrl(self);
+
+        var interval = Number(self.data('interval')) || 60;
+
+        setTimeout(function () {
+            $.get(ajaxUrl, function(data) {
+                refreshTargets(data);
+            });
+
+        }, interval * 1000);
+    }
+
+    function setupAjaxAction() {
+        var self = $(this);
+        var ajaxUrl = getAjaxUrl(self);
+
+        var processing = false;
+
+        self.click(function () {
+            if (!processing) {
+                $.get(ajaxUrl, function (data) {
+                    refreshTargets(data);
+                });
+
+                processing = true;
+            }
+
+            return false;
         });
     }
 
-    $(this).DataTable({
+    function setupDatestring() {
-        scrollX: true,
+        var self = $(this);
-        order: [
+        self.html(moment(self.data('date')).fromNow());
-            [1, 'asc']
+    }
-        ],
+
-        columns: columns,
+
-        pageLength: 50
+    function getDataTableOptions(elem) {
+        var rowSort = $(elem).data('rowSort') || false;
+
+        var columns = false;
+
+        var sortColumn = $(elem).data('sortColumn') === undefined ? 1 : Number($(elem).data('sortColumn'));
+        var sortOrder = ($(elem).data('sortOrder') || 'asc').toString().trim().toLowerCase();
+
+        var paging = $(elem).data('paging') === false ? false : true;
+
+        // allow only asc and desc
+        if (sortOrder !== 'desc') {
+            sortOrder = 'asc';
+        }
+
+        var columnsCount = 0;
+        var columnsSort = []
+
+        if (rowSort) {
+            columns = rowSort.split(',').map(function (sort) {
+                return {
+                    orderable: sort === '1'
+                };
+            });
+        }
+
+        var opts = {
+            scrollX: true,
+            order: [
+                [sortColumn, sortOrder]
+            ],
+            columns: columns,
+            paging: paging,
+            info: paging, /* This controls the "Showing 1 to 16 of 16 entries" */
+            pageLength: 50
+        };
+
+        if ($(elem).hasClass('data-table-selectable') || $(elem).hasClass('data-table-multiselectable')) {
+            var isMulti = $(elem).hasClass('data-table-multiselectable');
+
+            var dataElem = $(elem).siblings("input").first();
+
+            opts.rowCallback = function( row, data ) {
+                var selected = dataElem.val() == '' ? [] : dataElem.val().split(',').map(function(item) { return Number(item); });
+
+                if (!isMulti && selected.length > 0) {
+                    selected = [selected[0]];
+                }
+
+                if ($.inArray(data.DT_RowId, selected) !== -1) {
+                    $(row).addClass('selected');
+                }
+            }
+
+            $(elem).on('click', 'tbody tr', function () {
+                var id = this.id;
+                var selected = dataElem.val() == '' ? [] : dataElem.val().split(',');
+
+                var index = $.inArray(id, selected);
+
+                if (isMulti) {
+                    if ( index === -1 ) {
+                        selected.push(id);
+                    } else {
+                        selected.splice(index, 1);
+                    }
+
+                    $(this).toggleClass('selected');
+                } else {
+                    for (var selIdx=0; selIdx < selected.length; selIdx++) {
+                        if (selected[selIdx] != id) {
+                            $('#' + selected[selIdx], elem).removeClass('selected');
+                        }
+                    }
+
+                    $('#' + id, elem).addClass('selected');
+
+                    selected = [id];
+                }
+
+                dataElem.val(selected.join(','));
+            } );
+        }
+
+        return opts;
+    }
+
+
+    function installHandlers(elem) {
+        $('.ajax-refresh', elem).each(setupAjaxRefresh);
+        $('.ajax-action', elem).each(setupAjaxAction);
+        $('.datestring', elem).each(setupDatestring);
+    }
+
+    installHandlers($(document));
+
+    $('.data-table').each(function () {
+        var opts = getDataTableOptions(this);
+        $(this).DataTable(opts);
     });
-});
 
-$('.data-table-ajax').each(function () {
+    $('.data-table-ajax').each(function () {
-    var rowSort = $(this).data('rowSort') || false;
+        var topicUrl = $(this).data('topicUrl') || '/lists';
-    var columns = false;
+        var topicArgs = $(this).data('topicArgs') || false;
+        var topicId = $(this).data('topicId') || '';
 
-    var topicUrl = $(this).data('topicUrl') || '/lists';
+        var ajaxUrl = topicUrl + '/ajax/' + topicId + (topicArgs ? '?' + topicArgs : '');
-    var topicArgs = $(this).data('topicArgs') || false;
-    var topicId = $(this).data('topicId') || '';
 
-    var sortColumn = Number($(this).data('sortColumn')) || 1;
+        var opts = getDataTableOptions(this);
-    var sortOrder = ($(this).data('sortOrder') || 'asc').toString().trim().toLowerCase();
+        opts.ajax = {
-
-    // allow only asc and desc
-    if (sortOrder !== 'desc') {
-        sortOrder = 'asc';
-    }
-
-    var ajaxUrl = topicUrl + '/ajax/' + topicId + (topicArgs ? '?' + topicArgs : '');
-
-    if (rowSort) {
-        columns = rowSort.split(',').map(function (sort) {
-            return {
-                orderable: sort === '1'
-            };
-        });
-    }
-
-    $(this).DataTable({
-        scrollX: true,
-        serverSide: true,
-        ajax: {
             url: ajaxUrl,
             type: 'POST'
-        },
+        };
-        order: [
+        opts.serverSide = true;
-            [sortColumn, sortOrder]
+        opts.processing = true;
-        ],
-        columns: columns,
-        pageLength: 50,
-        processing: true
-    }).on('draw', function () {
-        $('.datestring').each(function () {
-            $(this).html(moment($(this).data('date')).fromNow());
-        });
-    });
-});
 
-$('.data-stats-pie-chart').each(function () {
+        opts.createdRow = function( row, data, dataIndex ) {
-    var column = $(this).data('column') || 'country';
+            installHandlers($(row));
-    var limit = $(this).data('limit') || 20;
-    var topicId = $(this).data('topicId');
-    var topicUrl = $(this).data('topicUrl') || '/campaigns/clicked';
-    var ajaxUrl = topicUrl + '/ajax/' + topicId + '/stats';
-    var self = $(this);
-
-    $.post(ajaxUrl, {column: column, limit: limit}, function(data) {
-      google.charts.load('current', {'packages':['corechart']});
-      google.charts.setOnLoadCallback(drawChart);
-
-      function drawChart() {
-        var gTable = new google.visualization.DataTable();
-        gTable.addColumn('string', 'Column');
-        gTable.addColumn('number', 'Value');
-        gTable.addRows(data.data);
-
-        var options = {'width':500, 'height':400};
-        var chart = new google.visualization.PieChart(self[0]);
-        chart.draw(gTable, options);
-      }
-    });
-});
-
-$('.datestring').each(function () {
-    $(this).html(moment($(this).data('date')).fromNow());
-});
-
-$('.delete-form,.confirm-submit').on('submit', function (e) {
-    if (!confirm($(this).data('confirmMessage') || 'Are you sure? This action can not be undone')) {
-        e.preventDefault();
-    }
-});
-
-$('.fm-date-us.date').datepicker({
-    format: 'mm/dd/yyyy',
-    weekStart: 0,
-    autoclose: true
-});
-
-$('.fm-date-eur.date').datepicker({
-    format: 'dd/mm/yyyy',
-    weekStart: 1,
-    autoclose: true
-});
-
-$('.fm-date-generic.date').datepicker({
-    format: 'yyyy-mm-dd',
-    weekStart: 1,
-    autoclose: true
-});
-
-$('.fm-birthday-us.date').datepicker({
-    format: 'mm/dd',
-    weekStart: 0,
-    autoclose: true
-});
-
-$('.fm-birthday-eur.date').datepicker({
-    format: 'dd/mm',
-    weekStart: 1,
-    autoclose: true
-});
-
-$('.fm-birthday-generic.date').datepicker({
-    format: 'mm-dd',
-    weekStart: 1,
-    autoclose: true
-});
-
-$('.page-refresh').each(function () {
-    var interval = Number($(this).data('interval')) || 60;
-    setTimeout(function () {
-        window.location.reload();
-    }, interval * 1000);
-});
-
-$('.click-select').on('click', function () {
-    $(this).select();
-});
-
-if (typeof moment.tz !== 'undefined') {
-    (function () {
-        var tz = moment.tz.guess();
-        if (tz) {
-            $('.tz-detect').val(tz);
         }
-    })();
-}
 
-// setup SMTP check
+        $(this).DataTable(opts).on('draw', function () {
-var smtpForm = document.querySelector('form#smtp-verify');
+            $('.datestring').each(setupDatestring);
-if (smtpForm) {
-    smtpForm.addEventListener('submit', function (e) {
-        e.preventDefault();
-
-        var form = document.getElementById('settings-form');
-        var formData = new FormData(form);
-        var result = fetch('/settings/smtp-verify', {
-            method: 'POST',
-            body: formData,
-            credentials: 'same-origin'
         });
-
-        var $btn = $('#verify-button').button('loading');
-
-        result.then(function (res) {
-            return res.json();
-        }).then(function (data) {
-            alert(data.error ? 'Invalid Mailer settings\n' + data.error : data.message);
-            $btn.button('reset');
-        }).catch(function (err) {
-            alert(err.message);
-            $btn.button('reset');
-        });
-
     });
-}
+
+    $('.data-stats-pie-chart').each(function () {
+        var column = $(this).data('column') || 'country';
+        var limit = $(this).data('limit') || 20;
+        var topicId = $(this).data('topicId');
+        var topicUrl = $(this).data('topicUrl') || '/campaigns/clicked';
+        var ajaxUrl = topicUrl + '/ajax/' + topicId + '/stats';
+        var self = $(this);
+
+        $.post(ajaxUrl, {column: column, limit: limit}, function(data) {
+          google.charts.load('current', {'packages':['corechart']});
+          google.charts.setOnLoadCallback(drawChart);
+
+          function drawChart() {
+            var gTable = new google.visualization.DataTable();
+            gTable.addColumn('string', 'Column');
+            gTable.addColumn('number', 'Value');
+            gTable.addRows(data.data);
+
+            var options = {'width':500, 'height':400};
+            var chart = new google.visualization.PieChart(self[0]);
+            chart.draw(gTable, options);
+          }
+        });
+    });
+
+    $('.datestring').each(function () {
+        $(this).html(moment($(this).data('date')).fromNow());
+    });
+
+    $('.delete-form,.confirm-submit').on('submit', function (e) {
+        if (!confirm($(this).data('confirmMessage') || 'Are you sure? This action can not be undone')) {
+            e.preventDefault();
+        }
+    });
+
+    $('.fm-date-us.date').datepicker({
+        format: 'mm/dd/yyyy',
+        weekStart: 0,
+        autoclose: true
+    });
+
+    $('.fm-date-eur.date').datepicker({
+        format: 'dd/mm/yyyy',
+        weekStart: 1,
+        autoclose: true
+    });
+
+    $('.fm-date-generic.date').datepicker({
+        format: 'yyyy-mm-dd',
+        weekStart: 1,
+        autoclose: true
+    });
+
+    $('.fm-birthday-us.date').datepicker({
+        format: 'mm/dd',
+        weekStart: 0,
+        autoclose: true
+    });
+
+    $('.fm-birthday-eur.date').datepicker({
+        format: 'dd/mm',
+        weekStart: 1,
+        autoclose: true
+    });
+
+    $('.fm-birthday-generic.date').datepicker({
+        format: 'mm-dd',
+        weekStart: 1,
+        autoclose: true
+    });
+
+    $('.page-refresh').each(function () {
+        var interval = Number($(this).data('interval')) || 60;
+        setTimeout(function () {
+            window.location.reload();
+        }, interval * 1000);
+    });
+
+
+    $('.click-select').on('click', function () {
+        $(this).select();
+    });
+
+    if (typeof moment.tz !== 'undefined') {
+        (function () {
+            var tz = moment.tz.guess();
+            if (tz) {
+                $('.tz-detect').val(tz);
+            }
+        })();
+    }
+
+    // setup SMTP check
+    var smtpForm = document.querySelector('form#smtp-verify');
+    if (smtpForm) {
+        smtpForm.addEventListener('submit', function (e) {
+            e.preventDefault();
+
+            var form = document.getElementById('settings-form');
+            var formData = new FormData(form);
+            var result = fetch('/settings/smtp-verify', {
+                method: 'POST',
+                body: formData,
+                credentials: 'same-origin'
+            });
+
+            var $btn = $('#verify-button').button('loading');
+
+            result.then(function (res) {
+                return res.json();
+            }).then(function (data) {
+                alert(data.error ? 'Invalid Mailer settings\n' + data.error : data.message);
+                $btn.button('reset');
+            }).catch(function (err) {
+                alert(err.message);
+                $btn.button('reset');
+            });
+
+        });
+    }
+
+})();
+

routes/campaigns.js

@@ -673,6 +673,80 @@ router.post('/status/ajax/:id/:status', (req, res) => {
     });
 });
 
+router.post('/clicked/ajax/:id/:linkId', (req, res) => {
+    let linkId = Number(req.params.linkId) || 0;
+
+    campaigns.get(req.params.id, true, (err, campaign) => {
+        if (err || !campaign) {
+            return res.json({
+                error: err && err.message || err || _('Campaign not found'),
+                data: []
+            });
+        }
+        lists.get(campaign.list, (err, list) => {
+            if (err) {
+                return res.json({
+                    error: err && err.message || err,
+                    data: []
+                });
+            }
+
+            let campaignCid = campaign.cid;
+            let listCid = list.cid;
+
+            let columns = ['#', 'email', 'first_name', 'last_name', 'campaign_tracker__' + campaign.id + '`.`created', 'count'];
+            campaigns.filterClickedSubscribers(campaign, linkId, req.body, columns, (err, data, total, filteredTotal) => {
+                if (err) {
+                    return res.json({
+                        error: err.message || err,
+                        data: []
+                    });
+                }
+
+                res.json({
+                    draw: req.body.draw,
+                    recordsTotal: total,
+                    recordsFiltered: filteredTotal,
+                    data: data.map((row, i) => [
+                        '<a href="/archive/' + encodeURIComponent(campaignCid) + '/' + encodeURIComponent(listCid) + '/' + encodeURIComponent(row.cid) + '?track=no">' + ((Number(req.body.start) || 0) + 1 + i) + '</a>',
+                        htmlescape(row.email || ''),
+                        htmlescape(row.firstName || ''),
+                        htmlescape(row.lastName || ''),
+                        row.created && row.created.toISOString ? '<span class="datestring" data-date="' + row.created.toISOString() + '" title="' + row.created.toISOString() + '">' + row.created.toISOString() + '</span>' : 'N/A',
+                        row.count,
+                        '<span class="glyphicon glyphicon-wrench" aria-hidden="true"></span><a href="/lists/subscription/' + campaign.list + '/edit/' + row.cid + '">' + _('Edit') + '</a>'
+                    ])
+                });
+            });
+        });
+    });
+});
+
+router.post('/quicklist/ajax', (req, res) => {
+    campaigns.filterQuicklist(req.body, (err, data, total, filteredTotal) => {
+        if (err) {
+            return res.json({
+                error: err.message || err,
+                data: []
+            });
+        }
+
+        res.json({
+            draw: req.body.draw,
+            recordsTotal: total,
+            recordsFiltered: filteredTotal,
+            data: data.map((row, i) => ({
+                "0": (Number(req.body.start) || 0) + 1 + i,
+                "1": '<span class="glyphicon glyphicon-inbox" aria-hidden="true"></span> <a href="/campaigns/view/' + row.id + '">' + htmlescape(row.name || '') + '</a>',
+                "2": htmlescape(striptags(row.description) || ''),
+                "3": '<span class="datestring" data-date="' + row.created.toISOString() + '" title="' + row.created.toISOString() + '">' + row.created.toISOString() + '</span>',
+                "DT_RowId": row.id
+            }))
+        });
+    });
+});
+
+
 router.post('/delete', passport.parseForm, passport.csrfProtection, (req, res) => {
     campaigns.delete(req.body.id, (err, deleted) => {
         if (err) {

routes/lists.js

@@ -55,23 +55,8 @@ router.all('/*', (req, res, next) => {
 });
 
 router.get('/', (req, res) => {
-    let limit = 999999999;
+    res.render('lists/lists', {
-    let start = 0;
+        title: _('Lists')
-
-    lists.list(start, limit, (err, rows, total) => {
-        if (err) {
-            req.flash('danger', err.message || err);
-            return res.redirect('/');
-        }
-
-        res.render('lists/lists', {
-            rows: rows.map((row, i) => {
-                row.index = start + i + 1;
-                row.description = striptags(row.description);
-                return row;
-            }),
-            total
-        });
     });
 });
 
@@ -82,6 +67,10 @@ router.get('/create', passport.csrfProtection, (req, res) => {
 
     data.csrfToken = req.csrfToken();
 
+    if (!('publicSubscribe' in data)) {
+        data.publicSubscribe = true;
+    }
+
     res.render('lists/create', data);
 });
 
@@ -155,6 +144,32 @@ router.post('/delete', passport.parseForm, passport.csrfProtection, (req, res) =
     });
 });
 
+router.post('/ajax', (req, res) => {
+    lists.filter(req.body, Number(req.query.parent) || false, (err, data, total, filteredTotal) => {
+        if (err) {
+            return res.json({
+                error: err.message || err,
+                data: []
+            });
+        }
+
+        res.json({
+            draw: req.body.draw,
+            recordsTotal: total,
+            recordsFiltered: filteredTotal,
+            data: data.map((row, i) => [
+                (Number(req.body.start) || 0) + 1 + i,
+                '<span class="glyphicon glyphicon-list-alt" aria-hidden="true"></span> <a href="/lists/view/' + row.id + '">' + htmlescape(row.name || '') + '</a>',
+                '<code>' + row.cid + '</code>',
+                row.subscribers,
+                htmlescape(striptags(row.description) || ''),
+                '<span class="glyphicon glyphicon-wrench" aria-hidden="true"></span><a href="/lists/edit/' + row.id + '">' + _('Edit') + '</a>' ]
+            )
+        });
+    });
+});
+
+
 router.post('/ajax/:id', (req, res) => {
     lists.get(req.params.id, (err, list) => {
         if (err || !list) {
@@ -733,4 +748,27 @@ router.get('/subscription/:id/import/:importId/failed', (req, res) => {
     });
 });
 
+router.post('/quicklist/ajax', (req, res) => {
+    lists.filterQuicklist(req.body, (err, data, total, filteredTotal) => {
+        if (err) {
+            return res.json({
+                error: err.message || err,
+                data: []
+            });
+        }
+
+        res.json({
+            draw: req.body.draw,
+            recordsTotal: total,
+            recordsFiltered: filteredTotal,
+            data: data.map((row, i) => ({
+                "0": (Number(req.body.start) || 0) + 1 + i,
+                "1": '<span class="glyphicon glyphicon-inbox" aria-hidden="true"></span> <a href="/lists/view/' + row.id + '">' + htmlescape(row.name || '') + '</a>',
+                "2": row.subscribers,
+                "DT_RowId": row.id
+            }))
+        });
+    });
+});
+
 module.exports = router;

routes/report-templates.js

@@ -0,0 +1,307 @@
+'use strict';
+
+const express = require('express');
+const passport = require('../lib/passport');
+const router = new express.Router();
+const _ = require('../lib/translate')._;
+const reportTemplates = require('../lib/models/report-templates');
+const tools = require('../lib/tools');
+const util = require('util');
+const htmlescape = require('escape-html');
+const striptags = require('striptags');
+
+const allowedMimeTypes = {
+    'text/html': 'HTML',
+    'text/csv': 'CSV'
+};
+
+router.all('/*', (req, res, next) => {
+    if (!req.user) {
+        req.flash('danger', _('Need to be logged in to access restricted content'));
+        return res.redirect('/users/login?next=' + encodeURIComponent(req.originalUrl));
+    }
+    res.setSelectedMenu('reports');
+    next();
+});
+
+router.get('/', (req, res) => {
+    res.render('report-templates/report-templates', {
+        title: _('Report Templates')
+    });
+});
+
+router.post('/ajax', (req, res) => {
+    reportTemplates.filter(req.body, (err, data, total, filteredTotal) => {
+        if (err) {
+            return res.json({
+                error: err.message || err,
+                data: []
+            });
+        }
+
+        res.json({
+            draw: req.body.draw,
+            recordsTotal: total,
+            recordsFiltered: filteredTotal,
+            data: data.map((row, i) => [
+                (Number(req.body.start) || 0) + 1 + i,
+                htmlescape(row.name || ''),
+                htmlescape(striptags(row.description) || ''),
+                '<span class="datestring" data-date="' + row.created.toISOString() + '" title="' + row.created.toISOString() + '">' + row.created.toISOString() + '</span>',
+                '<span class="glyphicon glyphicon-wrench" aria-hidden="true"></span><a href="/report-templates/edit/' + row.id + '"> ' + _('Edit') + '</a>']
+            )
+        });
+    });
+});
+
+router.get('/create', passport.csrfProtection, (req, res) => {
+    const data = req.query;
+    const wizard = req.query['type'] || '';
+
+    if (wizard == 'subscribers-all') {
+        if (!('description' in data)) data.description = 'Generates a campaign report listing all subscribers along with their statistics.';
+
+        if (!('mimeType' in data)) data.mimeType = 'text/html';
+
+        if (!('userFields' in data)) data.userFields =
+            '[\n' +
+            '    {\n' +
+            '        "id": "campaign",\n' +
+            '        "name": "Campaign",\n' +
+            '        "type": "campaign",\n' +
+            '        "minOccurences": 1,\n' +
+            '        "maxOccurences": 1\n' +
+            '    }\n' +
+            ']';
+
+        if (!('js' in data)) data.js =
+            'campaigns.results(inputs.campaign, ["*"], "", (err, results) => {\n' +
+            '    if (err) {\n' +
+            '        return callback(err);\n' +
+            '    }\n' +
+            '\n' +
+            '    const data = {\n' +
+            '        results: results\n' +
+            '    };\n' +
+            '\n' +
+            '    return callback(null, data);\n' +
+            '});';
+
+        if (!('hbs' in data)) data.hbs =
+            '<h2>{{title}}</h2>\n' +
+            '\n' +
+            '<div class="table-responsive">\n' +
+            '    <table class="table table-bordered table-hover data-table display nowrap" width="100%" data-row-sort="1,1" data-paging="false">\n' +
+            '        <thead>\n' +
+            '        <th>\n' +
+            '            {{#translate}}Email{{/translate}}\n' +
+            '        </th>\n' +
+            '        <th>\n' +
+            '            {{#translate}}Tracker Count{{/translate}}\n' +
+            '        </th>\n' +
+            '        </thead>\n' +
+            '        {{#if results}}\n' +
+            '            <tbody>\n' +
+            '            {{#each results}}\n' +
+            '                <tr>\n' +
+            '                    <th scope="row">\n' +
+            '                        {{email}}\n' +
+            '                    </th>\n' +
+            '                    <td style="width: 20%;">\n' +
+            '                        {{tracker_count}}\n' +
+            '                    </td>\n' +
+            '                </tr>\n' +
+            '            {{/each}}\n' +
+            '            </tbody>\n' +
+            '        {{/if}}\n' +
+            '    </table>\n' +
+            '</div>';
+
+    } else if (wizard == 'subscribers-grouped') {
+        if (!('description' in data)) data.description = 'Generates a campaign report with results are aggregated by some "Country" custom field.';
+
+        if (!('mimeType' in data)) data.mimeType = 'text/html';
+
+        if (!('userFields' in data)) data.userFields =
+            '[\n' +
+            '    {\n' +
+            '        "id": "campaign",\n' +
+            '        "name": "Campaign",\n' +
+            '        "type": "campaign",\n' +
+            '        "minOccurences": 1,\n' +
+            '        "maxOccurences": 1\n' +
+            '    }\n' +
+            ']';
+
+        if (!('js' in data)) data.js =
+            'campaigns.results(inputs.campaign, ["custom_country", "count(*) AS count_all", "SUM(IF(tracker.count IS NULL, 0, 1)) AS count_opened"], "GROUP BY custom_country", (err, results) => {\n' +
+            '    if (err) {\n' +
+            '        return callback(err);\n' +
+            '    }\n' +
+            '\n' +
+            '    for (let row of results) {\n' +
+            '        row["percentage"] = Math.round((row.count_opened / row.count_all) * 100);\n' +
+            '    }\n' +
+            '\n' +
+            '    let data = {\n' +
+            '        results: results\n' +
+            '    };\n' +
+            '\n' +
+            '    return callback(null, data);\n' +
+            '});';
+
+        if (!('hbs' in data)) data.hbs =
+            '<h2>{{title}}</h2>\n' +
+            '\n' +
+            '<div class="table-responsive">\n' +
+            '    <table class="table table-bordered table-hover data-table display nowrap" width="100%" data-row-sort="1,1,1,1" data-paging="false">\n' +
+            '        <thead>\n' +
+            '        <th>\n' +
+            '            {{#translate}}Country{{/translate}}\n' +
+            '        </th>\n' +
+            '        <th>\n' +
+            '            {{#translate}}Opened{{/translate}}\n' +
+            '        </th>\n' +
+            '        <th>\n' +
+            '            {{#translate}}All{{/translate}}\n' +
+            '        </th>\n' +
+            '        <th>\n' +
+            '            {{#translate}}Percentage{{/translate}}\n' +
+            '        </th>\n' +
+            '        </thead>\n' +
+            '        {{#if results}}\n' +
+            '            <tbody>\n' +
+            '            {{#each results}}\n' +
+            '                <tr>\n' +
+            '                    <th scope="row">\n' +
+            '                        {{custom_country}}\n' +
+            '                    </th>\n' +
+            '                    <td style="width: 20%;">\n' +
+            '                        {{count_opened}}\n' +
+            '                    </td>\n' +
+            '                    <td style="width: 20%;">\n' +
+            '                        {{count_all}}\n' +
+            '                    </td>\n' +
+            '                    <td style="width: 20%;">\n' +
+            '                        {{percentage}}%\n' +
+            '                    </td>\n' +
+            '                </tr>\n' +
+            '            {{/each}}\n' +
+            '            </tbody>\n' +
+            '        {{/if}}\n' +
+            '    </table>\n' +
+            '</div>';
+
+    } else if (wizard == 'export-list-csv') {
+        if (!('description' in data)) data.description = 'Exports a list as a CSV file.';
+
+        if (!('mimeType' in data)) data.mimeType = 'text/csv';
+
+        if (!('userFields' in data)) data.userFields =
+            '[\n' +
+            '    {\n' +
+            '        "id": "list",\n' +
+            '        "name": "List",\n' +
+            '        "type": "list",\n' +
+            '        "minOccurences": 1,\n' +
+            '        "maxOccurences": 1\n' +
+            '    }\n' +
+            ']';
+
+        if (!('js' in data)) data.js =
+            'subscriptions.list(inputs.list.id,0,0, (err, results) => {\n' +
+            '    if (err) {\n' +
+            '        return callback(err);\n' +
+            '    }\n' +
+            '\n' +
+            '    let data = {\n' +
+            '        results: results\n' +
+            '    };\n' +
+            '\n' +
+            '    return callback(null, data);\n' +
+            '});';
+
+        if (!('hbs' in data)) data.hbs =
+            '{{#each results}}\n' +
+            '{{firstName}},{{lastName}},{{email}}\n' +
+            '{{/each}}';
+    }
+
+    data.csrfToken = req.csrfToken();
+    data.title = _('Create Report Template');
+    data.useEditor = true;
+
+    data.mimeTypes = Object.keys(allowedMimeTypes).map(key => ({
+        key: key,
+        value: allowedMimeTypes[key],
+        selected: data.mimeType == key
+    }));
+
+    res.render('report-templates/create', data);
+});
+
+router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) => {
+    reportTemplates.createOrUpdate(true, req.body, (err, id) => {
+        if (err || !id) {
+            req.flash('danger', err && err.message || err || _('Could not create report template'));
+            return res.redirect('/report-templates/create?' + tools.queryParams(req.body));
+        }
+        req.flash('success', util.format(_('Report template “%s” created'), req.body.name));
+        res.redirect('/report-templates');
+    });
+});
+
+router.get('/edit/:id', passport.csrfProtection, (req, res) => {
+    reportTemplates.get(req.params.id, (err, template) => {
+        if (err || !template) {
+            req.flash('danger', err && err.message || err || _('Could not find report template with specified ID'));
+            return res.redirect('/report-templates');
+        }
+
+        template.csrfToken = req.csrfToken();
+        template.title = _('Edit Report Template');
+        template.useEditor = true;
+
+        template.mimeTypes = Object.keys(allowedMimeTypes).map(key => ({
+            key: key,
+            value: allowedMimeTypes[key],
+            selected: template.mimeType == key
+        }));
+
+        res.render('report-templates/edit', template);
+    });
+});
+
+router.post('/edit', passport.parseForm, passport.csrfProtection, (req, res) => {
+    reportTemplates.createOrUpdate(false, req.body, (err, updated) => {
+        if (err) {
+            req.flash('danger', err.message || err);
+        } else if (updated) {
+            req.flash('success', _('Report template updated'));
+        } else {
+            req.flash('info', _('Report template not updated'));
+        }
+
+        if (req.body['submit'] == 'update-and-stay') {
+            return res.redirect('/report-templates/edit/' + req.body.id);
+        } else {
+            return res.redirect('/report-templates');
+        }
+    });
+});
+
+router.post('/delete', passport.parseForm, passport.csrfProtection, (req, res) => {
+    reportTemplates.delete(req.body.id, (err, deleted) => {
+        if (err) {
+            req.flash('danger', err && err.message || err);
+        } else if (deleted) {
+            req.flash('success', _('Report template deleted'));
+        } else {
+            req.flash('info', _('Could not delete specified report template'));
+        }
+
+        return res.redirect('/report-templates');
+    });
+});
+
+module.exports = router;

routes/reports.js

@@ -0,0 +1,406 @@
+'use strict';
+
+const express = require('express');
+const passport = require('../lib/passport');
+const router = new express.Router();
+const _ = require('../lib/translate')._;
+const reportTemplates = require('../lib/models/report-templates');
+const reports = require('../lib/models/reports');
+const reportProcessor = require('../lib/report-processor');
+const campaigns = require('../lib/models/campaigns');
+const lists = require('../lib/models/lists');
+const tools = require('../lib/tools');
+const fileHelpers = require('../lib/file-helpers');
+const util = require('util');
+const htmlescape = require('escape-html');
+const striptags = require('striptags');
+const fs = require('fs');
+const hbs = require('hbs');
+
+router.all('/*', (req, res, next) => {
+    if (!req.user) {
+        req.flash('danger', _('Need to be logged in to access restricted content'));
+        return res.redirect('/users/login?next=' + encodeURIComponent(req.originalUrl));
+    }
+    res.setSelectedMenu('reports');
+    next();
+});
+
+router.get('/', (req, res) => {
+    res.render('reports/reports', {
+        title: _('Reports')
+    });
+});
+
+
+
+router.post('/ajax', (req, res) => {
+    reports.filter(req.body, (err, data, total, filteredTotal) => {
+        if (err) {
+            return res.json({
+                error: err.message || err,
+                data: []
+            });
+        }
+
+        res.json({
+            draw: req.body.draw,
+            recordsTotal: total,
+            recordsFiltered: filteredTotal,
+            data: data.map((row, i) => [
+                (Number(req.body.start) || 0) + 1 + i,
+                htmlescape(row.name || ''),
+                htmlescape(row.reportTemplateName || ''),
+                htmlescape(striptags(row.description) || ''),
+                getRowLastRun(row),
+                getRowActions(row)
+            ])
+        });
+    });
+});
+
+router.get('/row/ajax/:id', (req, res) => {
+    respondRowActions(req.params.id, res);
+});
+
+router.get('/start/ajax/:id', (req, res) => {
+    reportProcessor.start(req.params.id, () => {
+        respondRowActions(req.params.id, res);
+    });
+});
+
+router.get('/stop/ajax/:id', (req, res) => {
+    reportProcessor.stop(req.params.id, () => {
+        respondRowActions(req.params.id, res);
+    });
+});
+
+router.get('/create', passport.csrfProtection, (req, res) => {
+    const reqData = req.query;
+    reqData.csrfToken = req.csrfToken();
+    reqData.title = _('Create Report');
+    reqData.useEditor = true;
+
+    reportTemplates.quicklist((err, items) => {
+        if (err) {
+            req.flash('danger', err.message || err);
+            return res.redirect('/reports');
+        }
+
+        const reportTemplateId = Number(reqData.reportTemplate);
+
+        if (reportTemplateId) {
+            items.forEach(item => {
+                if (item.id === reportTemplateId) {
+                    item.selected = true;
+                }
+            });
+        }
+
+        reqData.reportTemplates = items;
+
+        if (!reportTemplateId) {
+            res.render('reports/create-select-template', reqData);
+        } else {
+            addUserFields(reportTemplateId, reqData, null, (err, data) => {
+                if (err) {
+                    req.flash('danger', err.message || err);
+                    return res.redirect('/reports');
+                }
+
+                res.render('reports/create', data);
+            });
+        }
+    });
+});
+
+router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) => {
+    const reqData = req.body;
+
+    const reportTemplateId = Number(reqData.reportTemplate);
+
+    addParamsObject(reportTemplateId, reqData, (err, data) => {
+        if (err) {
+            req.flash('danger', err && err.message || err || _('Could not create report'));
+            return res.redirect('/reports/create?' + tools.queryParams(data));
+        }
+
+        reports.createOrUpdate(true, data, (err, id) => {
+            if (err || !id) {
+                req.flash('danger', err && err.message || err || _('Could not create report'));
+                return res.redirect('/reports/create?' + tools.queryParams(data));
+            }
+
+            reportProcessor.start(id, () => {
+                req.flash('success', util.format(_('Report “%s” created'), data.name));
+                res.redirect('/reports');
+            });
+        });
+    });
+});
+
+router.get('/edit/:id', passport.csrfProtection, (req, res) => {
+    const reqData = req.query;
+    reports.get(req.params.id, (err, report) => {
+        if (err || !report) {
+            req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
+            return res.redirect('/reports');
+        }
+
+        report.csrfToken = req.csrfToken();
+        report.title = _('Edit Report');
+        report.useEditor = true;
+
+        reportTemplates.quicklist((err, items) => {
+            if (err) {
+                req.flash('danger', err.message || err);
+                return res.redirect('/');
+            }
+
+            const reportTemplateId = report.reportTemplate;
+
+            items.forEach(item => {
+                if (item.id === reportTemplateId) {
+                    item.selected = true;
+                }
+            });
+
+            report.reportTemplates = items;
+
+            addUserFields(reportTemplateId, reqData, report, (err, data) => {
+                if (err) {
+                    req.flash('danger', err.message || err);
+                    return res.redirect('/reports');
+                }
+
+                res.render('reports/edit', data);
+            });
+        });
+    });
+});
+
+router.post('/edit', passport.parseForm, passport.csrfProtection, (req, res) => {
+    const reqData = req.body;
+    const reportTemplateId = Number(reqData.reportTemplate);
+
+    addParamsObject(reportTemplateId, reqData, (err, data) => {
+        if (err) {
+            req.flash('danger', err && err.message || err || _('Could not update report'));
+            return res.redirect('/reports/create?' + tools.queryParams(data));
+        }
+
+        reports.createOrUpdate(false, data, (err, updated) => {
+            if (err) {
+                req.flash('danger', err && err.message || err || _('Could not update report'));
+                return res.redirect('/reports/edit/' + data.id + '?' + tools.queryParams(data));
+            } else if (updated) {
+                req.flash('success', _('Report updated'));
+            } else {
+                req.flash('info', _('Report not updated'));
+            }
+
+            return res.redirect('/reports');
+        });
+    });
+});
+
+router.post('/delete', passport.parseForm, passport.csrfProtection, (req, res) => {
+    reports.delete(req.body.id, (err, deleted) => {
+        if (err) {
+            req.flash('danger', err && err.message || err);
+        } else if (deleted) {
+            req.flash('success', _('Report deleted'));
+        } else {
+            req.flash('info', _('Could not delete specified report'));
+        }
+
+        return res.redirect('/reports');
+    });
+});
+
+router.get('/view/:id', (req, res) => {
+    reports.get(req.params.id, (err, report) => {
+        if (err || !report) {
+            req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
+            return res.redirect('/reports');
+        }
+
+        reportTemplates.get(report.reportTemplate, (err, reportTemplate) => {
+            if (err) {
+                req.flash('danger', err && err.message || err || _('Could not find report template'));
+                return res.redirect('/reports');
+            }
+
+            if (report.state == reports.ReportState.FINISHED) {
+                if (reportTemplate.mimeType == 'text/html') {
+
+                    fs.readFile(fileHelpers.getReportContentFile(report), (err, reportContent) => {
+                        if (err) {
+                            req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
+                            return res.redirect('/reports');
+                        }
+
+                        const data = {
+                            report: new hbs.handlebars.SafeString(reportContent),
+                            title: report.name
+                        };
+
+                        res.render('reports/view', data);
+                    });
+
+                } else if (reportTemplate.mimeType == 'text/csv') {
+                    const headers = {
+                        'Content-Disposition': 'attachment;filename=' + fileHelpers.nameToFileName(report.name) + '.csv',
+                        'Content-Type': 'text/csv'
+                    };
+
+                    res.sendFile(fileHelpers.getReportContentFile(report), {headers: headers});
+
+                } else {
+                    req.flash('danger', _('Unknown type of template'));
+                    res.redirect('/reports');
+                }
+
+            } else {
+                req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
+                return res.redirect('/reports');
+            }
+        });
+    });
+});
+
+router.get('/output/:id', (req, res) => {
+    reports.get(req.params.id, (err, report) => {
+        if (err || !report) {
+            req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
+            return res.redirect('/reports');
+        }
+
+        fs.readFile(fileHelpers.getReportOutputFile(report), (err, output) => {
+            let data = {
+                title: 'Output for report ' + report.name
+            };
+
+            if (err) {
+                data.error = 'No output.';
+            } else {
+                data.output = output;
+            }
+
+            res.render('reports/output', data);
+        });
+    });
+});
+
+function getRowLastRun(row) {
+    return '<span id="row-last-run-' + row.id + '">' + (row.lastRun ? '<span class="datestring" data-date="' + row.lastRun.toISOString() + '" title="' + row.lastRun.toISOString() + '">' + row.lastRun.toISOString() + '</span>' : '') + '</span>';
+}
+
+function getRowActions(row) {
+    /* FIXME: add csrf protection to stop and refresh actions */
+
+    let requestRefresh = false;
+    let view, startStop;
+    let topic = 'data-topic-id="' + row.id + '"';
+
+    if (row.state == reports.ReportState.PROCESSING || row.state == reports.ReportState.SCHEDULED) {
+        view = '<span class="row-action glyphicon glyphicon-hourglass" aria-hidden="true" title="Processing"></span>';
+        startStop = '<a class="row-action ajax-action" href="" data-topic-url="/reports/stop" ' + topic + ' title="Stop"><span class="glyphicon glyphicon-stop" aria-hidden="true"></span></a>';
+        requestRefresh = true;
+
+    } else if (row.state == reports.ReportState.FINISHED) {
+        let icon = 'eye-open';
+        if (row.mimeType == 'text/csv') icon = 'download-alt';
+
+        view = '<a class="row-action" href="/reports/view/' + row.id + '" title="View report"><span class="glyphicon glyphicon-' + icon + '" aria-hidden="true"></span></a>';
+        startStop = '<a class="row-action ajax-action" href="" data-topic-url="/reports/start" ' + topic + ' title="Refresh report"><span class="glyphicon glyphicon-repeat" aria-hidden="true"></span></a>';
+
+    } else if (row.state == reports.ReportState.FAILED) {
+        view = '<span class="row-action glyphicon glyphicon-thumbs-down" aria-hidden="true" title="Report generation failed"></span>';
+        startStop = '<a class="row-action ajax-action" href="" data-topic-url="/reports/start" ' + topic + ' title="Refresh report"><span class="glyphicon glyphicon-repeat" aria-hidden="true"></span></a>';
+    }
+
+    let actions = view;
+    actions += '<a class="row-action" href="/reports/output/' + row.id + '" title="View console output"><span class="glyphicon glyphicon-modal-window" aria-hidden="true"></span></a>';
+    actions += startStop;
+    actions += '<a class="row-action" href="/reports/edit/' + row.id + '"><span class="glyphicon glyphicon-wrench" aria-hidden="true" title="Edit"></span></a>';
+
+    return '<span id="row-actions-' + row.id + '"' + (requestRefresh ? ' class="row-actions ajax-refresh" data-interval="5" data-topic-url="/reports/row" ' + topic : ' class="row-actions"') + '>' +
+        actions +
+        '</span>';
+}
+
+function respondRowActions(id, res) {
+    reports.get(id, (err, report) => {
+        if (err) {
+            return res.json({
+                error: err,
+            });
+        }
+
+        const data = {};
+        data['#row-last-run-' + id] = getRowLastRun(report);
+        data['#row-actions-' + id] = getRowActions(report);
+
+        res.json(data);
+    });
+}
+
+
+function addUserFields(reportTemplateId, reqData, report, callback) {
+    reportTemplates.get(reportTemplateId, (err, reportTemplate) => {
+        if (err) {
+            return callback(err);
+        }
+
+        const userFields = [];
+
+        for (const spec of reportTemplate.userFieldsObject) {
+            let value = '';
+            if ((spec.id + 'Selection') in reqData) {
+                value = reqData[spec.id + 'Selection'];
+            } else if (report && report.paramsObject && spec.id in report.paramsObject) {
+                value = report.paramsObject[spec.id].join(',');
+            }
+
+            userFields.push({
+                'id': spec.id,
+                'name': spec.name,
+                'type': spec.type,
+                'value': value,
+                'isMulti': !(spec.minOccurences == 1 && spec.maxOccurences == 1)
+            });
+        }
+
+        const data = report ? report : reqData;
+        data.userFields = userFields;
+
+        callback(null, data);
+    });
+}
+
+function addParamsObject(reportTemplateId, data, callback) {
+    reportTemplates.get(reportTemplateId, (err, reportTemplate) => {
+        if (err) {
+            return callback(err);
+        }
+
+        const paramsObject = {};
+
+        for (const spec of reportTemplate.userFieldsObject) {
+            const sel = data[spec.id + 'Selection'];
+
+            if (!sel) {
+                paramsObject[spec.id] = [];
+            } else {
+                paramsObject[spec.id] = sel.split(',').map(item => Number(item));
+            }
+        }
+
+        data.paramsObject = paramsObject;
+
+        callback(null, data);
+    });
+}
+
+module.exports = router;

routes/subscription.js

@@ -176,9 +176,14 @@ router.get('/subscribe/:cid', (req, res, next) => {
 
 router.get('/:cid', passport.csrfProtection, (req, res, next) => {
     lists.getByCid(req.params.cid, (err, list) => {
-        if (!err && !list) {
+        if (!err) {
-            err = new Error(_('Selected list not found'));
+            if (!list) {
-            err.status = 404;
+                err = new Error(_('Selected list not found'));
+                err.status = 404;
+            } else if (!list.publicSubscribe) {
+                err = new Error(_('The list does not allow public subscriptions.'));
+                err.status = 403;
+            }
         }
 
         if (err) {
@@ -501,9 +506,14 @@ router.post('/:cid/subscribe', passport.parseForm, corsOrCsrfProtection, (req, r
     let testsPass = subTimeTest && addressTest;
 
     lists.getByCid(req.params.cid, (err, list) => {
-        if (!err && !list) {
+        if (!err) {
-            err = new Error(_('Selected list not found'));
+            if (!list) {
-            err.status = 404;
+                err = new Error(_('Selected list not found'));
+                err.status = 404;
+            } else if (!list.publicSubscribe) {
+                err = new Error(_('The list does not allow public subscriptions.'));
+                err.status = 403;
+            }
         }
 
         if (err) {

routes/templates.js

@@ -8,6 +8,7 @@ let settings = require('../lib/models/settings');
 let tools = require('../lib/tools');
 let helpers = require('../lib/helpers');
 let striptags = require('striptags');
+let htmlescape = require('escape-html');
 let passport = require('../lib/passport');
 let mailer = require('../lib/mailer');
 let _ = require('../lib/translate')._;
@@ -22,23 +23,8 @@ router.all('/*', (req, res, next) => {
 });
 
 router.get('/', (req, res) => {
-    let limit = 999999999;
+    res.render('templates/templates', {
-    let start = 0;
+        title: _('Templates')
-
-    templates.list(start, limit, (err, rows, total) => {
-        if (err) {
-            req.flash('danger', err.message || err);
-            return res.redirect('/');
-        }
-
-        res.render('templates/templates', {
-            rows: rows.map((row, i) => {
-                row.index = start + i + 1;
-                row.description = striptags(row.description);
-                return row;
-            }),
-            total
-        });
     });
 });
 
@@ -164,4 +150,27 @@ router.post('/delete', passport.parseForm, passport.csrfProtection, (req, res) =
     });
 });
 
+router.post('/ajax', (req, res) => {
+    templates.filter(req.body, Number(req.query.parent) || false, (err, data, total, filteredTotal) => {
+        if (err) {
+            return res.json({
+                error: err.message || err,
+                data: []
+            });
+        }
+
+        res.json({
+            draw: req.body.draw,
+            recordsTotal: total,
+            recordsFiltered: filteredTotal,
+            data: data.map((row, i) => [
+                (Number(req.body.start) || 0) + 1 + i,
+                '<span class="glyphicon glyphicon-file" aria-hidden="true"></span> ' + htmlescape(row.name || ''),
+                htmlescape(striptags(row.description) || ''),
+                '<span class="glyphicon glyphicon-wrench" aria-hidden="true"></span><a href="/templates/edit/' + row.id + '">' + _('Edit') + '</a>' ]
+            )
+        });
+    });
+});
+
 module.exports = router;

services/executor.js

@@ -0,0 +1,131 @@
+'use strict';
+
+/* Privileged executor. If Mailtrain is started as root, this process keeps the root privilege to be able to spawn workers
+   that can chroot.
+  */
+
+const fileHelpers = require('../lib/file-helpers');
+const fork = require('child_process').fork;
+const path = require('path');
+const log = require('npmlog');
+const fs = require('fs');
+const privilegeHelpers = require('../lib/privilege-helpers');
+
+let processes = {};
+
+function spawnProcess(tid, executable, args, outFile, errFile, cwd, uid, gid) {
+
+    function reportFail(msg) {
+        process.send({
+            type: 'process-failed',
+            msg,
+            tid
+        });
+    }
+
+    fs.open(outFile, 'w', (err, outFd) => {
+        if (err) {
+            log.error('Executor', err);
+            reportFail('Cannot create standard output file.');
+            return;
+        }
+
+        fs.open(errFile, 'w', (err, errFd) => {
+            if (err) {
+                log.error('Executor', err);
+                reportFail('Cannot create standard error file.');
+                return;
+            }
+
+            privilegeHelpers.ensureMailtrainOwner(outFile, (err) => {
+                if (err) {
+                    log.warn('Executor', 'Cannot change owner of output file of process tid:%s.', tid)
+                }
+
+                privilegeHelpers.ensureMailtrainOwner(errFile, (err) => {
+                    if (err) {
+                        log.warn('Executor', 'Cannot change owner of error output file of process tid:%s.', tid)
+                    }
+
+                    const options = {
+                        stdio: ['ignore', outFd, errFd, 'ipc'],
+                        cwd,
+                        env: {NODE_ENV: process.env.NODE_ENV},
+                        uid,
+                        gid
+                    };
+
+                    let child;
+
+                    try {
+                        child = fork(executable, args, options);
+                    } catch (err) {
+                        log.error('Executor', 'Cannot start process with tid:%s.', tid);
+                        reportFail('Cannot start process.');
+                        return;
+                    }
+
+                    const pid = child.pid;
+                    processes[tid] = child;
+
+                    log.info('Executor', 'Process started with tid:%s pid:%s.', tid, pid);
+                    process.send({
+                        type: 'process-started',
+                        tid
+                    });
+
+                    child.on('close', (code, signal) => {
+
+                        delete processes[tid];
+                        log.info('Executor', 'Process tid:%s pid:%s exited with code %s signal %s.', tid, pid, code, signal);
+
+                        fs.close(outFd, (err) => {
+                            if (err) {
+                                log.error('Executor', err);
+                            }
+
+                            fs.close(errFd, (err) => {
+                                if (err) {
+                                    log.error('Executor', err);
+                                }
+
+                                process.send({
+                                    type: 'process-finished',
+                                    tid,
+                                    code,
+                                    signal
+                                });
+                            });
+                        });
+                    });
+                });
+            });
+        });
+    });
+}
+
+process.on('message', msg => {
+    if (msg) {
+        const type = msg.type;
+
+        if (type === 'start-report-processor-worker') {
+
+            const ids = privilegeHelpers.getConfigROUidGid();
+            spawnProcess(msg.tid, path.join(__dirname, '..', 'workers', 'reports', 'report-processor.js'), [msg.data.id], fileHelpers.getReportContentFile(msg.data), fileHelpers.getReportOutputFile(msg.data), path.join(__dirname, '..', 'workers', 'reports'), ids.uid, ids.gid);
+
+        } else if (type === 'stop-process') {
+            const child = processes[msg.tid];
+
+            if (child) {
+                log.info('Executor', 'Killing process tid:%s pid:%s', msg.tid, child.pid);
+                child.kill();
+            } else {
+                log.info('Executor', 'No running process found with tid:%s pid:%s', msg.tid, child.pid);
+            }
+        }
+    }
+});
+
+process.send({
+    type: 'executor-started'
+});

setup/install-centos7.sh

@@ -0,0 +1,228 @@
+#!/bin/bash
+
+# This installation script works on CentOS 7
+# Run as root!
+
+if [[ $EUID -ne 0 ]]; then
+   echo "This script must be run as root" 1>&2
+   exit 1
+fi
+
+set -e
+
+yum -y install epel-release
+
+curl --silent --location https://rpm.nodesource.com/setup_6.x | bash -
+yum -y install mariadb-server nodejs ImageMagick git python redis pwgen bind-utils gcc-c++ make
+
+systemctl start mariadb
+systemctl enable mariadb
+
+systemctl start redis
+systemctl enable redis
+
+
+PUBLIC_IP=`curl -s https://api.ipify.org`
+if [ ! -z "$PUBLIC_IP" ]; then
+    HOSTNAME=`dig +short -x $PUBLIC_IP | sed 's/\.$//'`
+    HOSTNAME="${HOSTNAME:-$PUBLIC_IP}"
+fi
+HOSTNAME="${HOSTNAME:-`hostname`}"
+
+MYSQL_PASSWORD=`pwgen 12 -1`
+MYSQL_RO_PASSWORD=`pwgen 12 -1`
+DKIM_API_KEY=`pwgen 12 -1`
+SMTP_PASS=`pwgen 12 -1`
+
+# Setup MySQL user for Mailtrain
+mysql -u root -e "CREATE USER 'mailtrain'@'localhost' IDENTIFIED BY '$MYSQL_PASSWORD';"
+mysql -u root -e "GRANT ALL PRIVILEGES ON mailtrain.* TO 'mailtrain'@'localhost';"
+mysql -u root -e "CREATE USER 'mailtrain_ro'@'localhost' IDENTIFIED BY '$MYSQL_RO_PASSWORD';"
+mysql -u root -e "GRANT SELECT ON mailtrain.* TO 'mailtrain_ro'@'localhost';"
+mysql -u mailtrain --password="$MYSQL_PASSWORD" -e "CREATE database mailtrain;"
+
+# Enable firewall, allow connections to SSH, HTTP, HTTPS and SMTP
+for port in 80/tcp 443/tcp 25/tcp; do firewall-cmd --add-port=$port --permanent; done
+firewall-cmd --reload
+
+# Fetch Mailtrain files
+mkdir -p /opt/mailtrain
+cd /opt/mailtrain
+git clone git://github.com/Mailtrain-org/mailtrain.git .
+
+# Normally we would let Mailtrain itself to import the initial SQL data but in this case
+# we need to modify it, before we start Mailtrain
+mysql -u mailtrain -p"$MYSQL_PASSWORD" mailtrain < setup/sql/mailtrain.sql
+
+mysql -u mailtrain -p"$MYSQL_PASSWORD" mailtrain <<EOT
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('admin_email','admin@$HOSTNAME') ON DUPLICATE KEY UPDATE \`value\`='admin@$HOSTNAME';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('default_address','admin@$HOSTNAME') ON DUPLICATE KEY UPDATE \`value\`='admin@$HOSTNAME';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('smtp_hostname','localhost') ON DUPLICATE KEY UPDATE \`value\`='localhost';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('smtp_disable_auth','') ON DUPLICATE KEY UPDATE \`value\`='';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('smtp_user','mailtrain') ON DUPLICATE KEY UPDATE \`value\`='mailtrain';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('smtp_pass','$SMTP_PASS') ON DUPLICATE KEY UPDATE \`value\`='$SMTP_PASS';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('smtp_encryption','NONE') ON DUPLICATE KEY UPDATE \`value\`='NONE';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('smtp_port','2525') ON DUPLICATE KEY UPDATE \`value\`='2525';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('default_homepage','http://$HOSTNAME/') ON DUPLICATE KEY UPDATE \`value\`='http://$HOSTNAME/';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('service_url','http://$HOSTNAME/') ON DUPLICATE KEY UPDATE \`value\`='http://$HOSTNAME/';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('dkim_api_key','$DKIM_API_KEY') ON DUPLICATE KEY UPDATE \`value\`='$DKIM_API_KEY';
+EOT
+
+# Add new user for the mailtrain daemon to run as
+useradd mailtrain || true
+useradd zone-mta || true
+
+# Setup installation configuration
+cat >> config/production.toml <<EOT
+user="mailtrain"
+group="mailtrain"
+rouser="nobody"
+rogroup="nobody"
+[log]
+level="error"
+[www]
+port=80
+secret="`pwgen -1`"
+[mysql]
+password="$MYSQL_PASSWORD"
+[redis]
+enabled=true
+[queue]
+processes=5
+[reports]
+enabled=true
+EOT
+
+cat >> workers/reports/config/production.toml <<EOT
+[log]
+level="error"
+[mysql]
+user="mailtrain_ro"
+password="$MYSQL_RO_PASSWORD"
+EOT
+
+# Install required node packages
+npm install --no-progress --production
+chown -R mailtrain:mailtrain .
+chmod o-rwx config
+
+# Setup log rotation to not spend up entire storage on logs
+cat <<EOM > /etc/logrotate.d/mailtrain
+/var/log/mailtrain.log {
+    daily
+    rotate 12
+    compress
+    delaycompress
+    missingok
+    notifempty
+    copytruncate
+    nomail
+}
+EOM
+
+# Set up systemd service script
+cp setup/mailtrain-centos7.service /etc/systemd/system/mailtrain.service
+systemctl enable mailtrain.service
+
+# Fetch ZoneMTA files
+mkdir -p /opt/zone-mta
+cd /opt/zone-mta
+git clone git://github.com/zone-eu/zone-mta.git .
+git checkout 6964091273
+
+# Ensure queue folder
+mkdir -p /var/data/zone-mta/mailtrain
+
+# Setup installation configuration
+cat >> config/production.json <<EOT
+{
+    "name": "Mailtrain",
+    "user": "zone-mta",
+    "group": "zone-mta",
+    "queue": {
+        "db": "/var/data/zone-mta/mailtrain"
+    },
+    "smtpInterfaces": {
+        "feeder": {
+            "enabled": true,
+            "port": 2525,
+            "processes": 2,
+            "authentication": true
+        }
+    },
+    "api": {
+        "maildrop": false,
+        "user": "mailtrain",
+        "pass": "$SMTP_PASS"
+    },
+    "log": {
+        "level": "info",
+        "syslog": true
+    },
+    "plugins": {
+        "core/email-bounce": false,
+        "core/http-bounce": {
+            "enabled": "main",
+            "url": "http://localhost/webhooks/zone-mta"
+        },
+        "core/http-auth": {
+            "enabled": ["receiver", "main"],
+            "url": "http://localhost:8080/test-auth"
+        },
+        "core/default-headers": {
+            "enabled": ["receiver", "main", "sender"],
+            "futureDate": false,
+            "xOriginatingIP": false
+        },
+        "core/http-config": {
+            "enabled": ["main", "receiver"],
+            "url": "http://localhost/webhooks/zone-mta/sender-config?api_token=$DKIM_API_KEY"
+        },
+        "core/rcpt-mx": false
+    },
+    "pools": {
+        "default": [{
+            "address": "0.0.0.0",
+            "name": "$HOSTNAME"
+        }]
+    },
+    "zones": {
+        "default": {
+            "processes": 3,
+            "connections": 5,
+            "throttling": false,
+            "pool": "default"
+        },
+        "transactional": {
+            "processes": 1,
+            "connections": 1,
+            "pool": "default"
+        }
+    },
+    "domainConfig": {
+        "default": {
+            "maxConnections": 4
+        }
+    }
+}
+EOT
+
+# Install required node packages
+npm install --no-progress --production
+npm install leveldown
+
+# Ensure queue folder is owned by MTA user
+chown -R zone-mta:zone-mta /var/data/zone-mta/mailtrain
+
+# Set up systemd service script
+cp setup/zone-mta.service /etc/systemd/system/
+systemctl enable zone-mta.service
+
+# Start the service
+systemctl daemon-reload
+
+systemctl start zone-mta.service
+systemctl start mailtrain.service
+
+echo "Success! Open http://$HOSTNAME/ and log in as admin:test";
+

setup/install.sh

@@ -28,12 +28,15 @@ fi
 HOSTNAME="${HOSTNAME:-`hostname`}"
 
 MYSQL_PASSWORD=`pwgen 12 -1`
+MYSQL_RO_PASSWORD=`pwgen 12 -1`
 DKIM_API_KEY=`pwgen 12 -1`
 SMTP_PASS=`pwgen 12 -1`
 
 # Setup MySQL user for Mailtrain
 mysql -u root -e "CREATE USER 'mailtrain'@'localhost' IDENTIFIED BY '$MYSQL_PASSWORD';"
 mysql -u root -e "GRANT ALL PRIVILEGES ON mailtrain.* TO 'mailtrain'@'localhost';"
+mysql -u root -e "CREATE USER 'mailtrain_ro'@'localhost' IDENTIFIED BY '$MYSQL_RO_PASSWORD';"
+mysql -u root -e "GRANT SELECT ON mailtrain.* TO 'mailtrain_ro'@'localhost';"
 mysql -u mailtrain --password="$MYSQL_PASSWORD" -e "CREATE database mailtrain;"
 
 # Enable firewall, allow connections to SSH, HTTP, HTTPS and SMTP
@@ -87,9 +90,18 @@ enabled=true
 processes=5
 EOT
 
+cat >> workers/reports/config/production.toml <<EOT
+[log]
+level="error"
+[mysql]
+user="mailtrain_ro"
+password="$MYSQL_RO_PASSWORD"
+EOT
+
 # Install required node packages
 npm install --no-progress --production
 chown -R mailtrain:mailtrain .
+chmod o-rwx config
 
 # Setup log rotation to not spend up entire storage on logs
 cat <<EOM > /etc/logrotate.d/mailtrain

setup/mailtrain-centos7.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Mailtrain server
+Requires=mariadb.service
+After=syslog.target network.target
+
+[Service]
+Environment="NODE_ENV=production"
+WorkingDirectory=/opt/mailtrain
+ExecStart=/usr/bin/node index.js
+Type=simple
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
+# Alias=mailtrain.service

setup/sql/upgrade-00026.sql

@@ -0,0 +1,11 @@
+# Header section
+# Define incrementing schema version number
+SET @schema_version = '26';
+
+# Add field
+ALTER TABLE `lists` ADD COLUMN `public_subscribe` tinyint(1) unsigned DEFAULT 1 NOT NULL AFTER `created`;
+
+# Footer section
+LOCK TABLES `settings` WRITE;
+INSERT INTO `settings` (`key`, `value`) VALUES('db_schema_version', @schema_version) ON DUPLICATE KEY UPDATE `value`=@schema_version;
+UNLOCK TABLES;

setup/sql/upgrade-00027.sql

@@ -0,0 +1,37 @@
+# Header section
+# Define incrementing schema version number
+SET @schema_version = '27';
+
+# Create table to report templates
+CREATE TABLE `report_templates` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+  `name` varchar(255) DEFAULT '',
+  `mime_type` varchar(255) DEFAULT 'text/html' NOT NULL,
+  `description` text,
+  `user_fields` longtext,
+  `js` longtext,
+  `hbs` longtext,
+  `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+# Create table to store reports
+CREATE TABLE `reports` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+  `name` varchar(255) DEFAULT '',
+  `description` text,
+  `report_template` int(11) unsigned NOT NULL,
+  `params` longtext,
+  `state`  int(11) unsigned NOT NULL DEFAULT 0,
+  `last_run` DATETIME DEFAULT NULL,
+  `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `report_template` (`report_template`),
+  CONSTRAINT `report_template_ibfk_1` FOREIGN KEY (`report_template`) REFERENCES `report_templates` (`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+
+# Footer section
+LOCK TABLES `settings` WRITE;
+INSERT INTO `settings` (`key`, `value`) VALUES('db_schema_version', @schema_version) ON DUPLICATE KEY UPDATE `value`=@schema_version;
+UNLOCK TABLES;

views/campaigns/campaigns.hbs

@@ -23,7 +23,7 @@
 <div class="table-responsive">
     <table data-topic-url="/campaigns" data-sort-column="4" data-sort-order="desc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,0,1,1,0">
         <thead>
-            <th class="col-md-1">
+            <th style="width: 1%">
                 #
             </th>
             <th>
@@ -38,7 +38,7 @@
             <th>
                 {{#translate}}Created{{/translate}}
             </th>
-            <th class="col-md-1">
+            <th style="width: 1%">
                 &nbsp;
             </th>
         </thead>

views/lists/create.hbs

@@ -26,6 +26,16 @@
 
     <hr />
 
+    <div class="col-sm-offset-2">
+        <div class="checkbox">
+            <label>
+                <input type="checkbox" name="public_subscribe" value="1" {{#if publicSubscribe}} checked {{/if}}> {{#translate}}Allow public users to subscribe themselves{{/translate}}
+            </label>
+        </div>
+    </div>
+
+    <hr />
+
     <div class="form-group">
         <div class="col-sm-offset-2 col-sm-10">
             <button type="submit" class="btn btn-primary"><i class="glyphicon glyphicon-plus"></i> {{#translate}}Create List{{/translate}}</button>

views/lists/edit.hbs

@@ -56,6 +56,16 @@
 
     <hr />
 
+    <div class="col-sm-offset-2">
+        <div class="checkbox">
+            <label>
+                <input type="checkbox" name="public_subscribe" value="1" {{#if publicSubscribe}} checked {{/if}}> {{#translate}}Allow public users to subscribe themselves{{/translate}}
+            </label>
+        </div>
+    </div>
+
+    <hr />
+
     <div class="form-group">
         <div class="col-sm-offset-2 col-sm-10">
             <div class="pull-right">

views/lists/lists.hbs

@@ -12,57 +12,26 @@
 <hr>
 
 <div class="table-responsive">
-    <table class="table table-bordered table-hover data-table display nowrap" width="100%" data-row-sort="0,1,1,1,0,0">
+    <table data-topic-url="/lists" data-sort-column="1" data-sort-order="asc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,1,1,0,0">
         <thead>
-            <th>
+        <th style="width: 1%">
-                #
+            #
-            </th>
+        </th>
-            <th>
+        <th>
-                {{#translate}}Name{{/translate}}
+            {{#translate}}Name{{/translate}}
-            </th>
+        </th>
-            <th>
+        <th>
-                {{#translate}}ID{{/translate}}
+            {{#translate}}ID{{/translate}}
-            </th>
+        </th>
-            <th>
+        <th>
-                {{#translate}}Subscribers{{/translate}}
+            {{#translate}}Subscribers{{/translate}}
-            </th>
+        </th>
-            <th>
+        <th>
-                {{#translate}}Description{{/translate}}
+            {{#translate}}Description{{/translate}}
-            </th>
+        </th>
-            <th>
+        <th style="width: 1%">
-                &nbsp;
+            &nbsp;
-            </th>
+        </th>
         </thead>
-        {{#if rows}}
-            <tbody>
-                {{#each rows}}
-                    <tr>
-                        <th scope="row">
-                            {{index}}
-                        </th>
-                        <td style="width: 30%;">
-                            <span class="glyphicon glyphicon-list-alt" aria-hidden="true"></span>
-                            <a href="/lists/view/{{id}}">
-                            {{name}}
-                        </a>
-                        </td>
-                        <td>
-                            <code>{{cid}}</code>
-                        </td>
-                        <td class="text-center">
-                            {{subscribers}}
-                        </td>
-                        <td class="text-muted" style="width: 70%;">
-                            {{description}}
-                        </td>
-                        <td>
-                            <a href="/lists/edit/{{id}}">
-                            {{#translate}}Edit{{/translate}}
-                        </a>
-                        </td>
-                    </tr>
-                {{/each}}
-            </tbody>
-        {{/if}}
     </table>
 </div>

views/report-templates/create.hbs

@@ -0,0 +1,24 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li><a href="/reports">{{#translate}}Reports{{/translate}}</a></li>
+    <li><a href="/report-templates">{{#translate}}Templates{{/translate}}</a></li>
+    <li class="active">{{#translate}}Create Template{{/translate}}</li>
+</ol>
+
+<h2>{{#translate}}Create Report Template{{/translate}}</h2>
+
+<hr>
+
+<form class="form-horizontal" method="post" action="/report-templates/create">
+    <input type="hidden" name="_csrf" value="{{csrfToken}}">
+
+{{> report_template_fields }}
+
+    <hr />
+
+    <div class="form-group">
+        <div class="col-sm-offset-2 col-sm-10">
+            <button type="submit" class="btn btn-primary"><i class="glyphicon glyphicon-plus"></i> {{#translate}}Create Template{{/translate}}</button>
+        </div>
+    </div>
+</form>

views/report-templates/edit.hbs

@@ -0,0 +1,36 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li><a href="/reports">{{#translate}}Reports{{/translate}}</a></li>
+    <li><a href="/report-templates">{{#translate}}Templates{{/translate}}</a></li>
+    <li class="active">{{#translate}}Edit Template{{/translate}}</li>
+</ol>
+
+<h2>{{#translate}}Edit Report Template{{/translate}}</h2>
+
+<hr>
+
+<form method="post" class="delete-form" id="report-templates-delete" action="/report-templates/delete">
+    <input type="hidden" name="_csrf" value="{{csrfToken}}">
+    <input type="hidden" name="id" value="{{id}}" />
+</form>
+
+
+<form class="form-horizontal" method="post" action="/report-templates/edit">
+    <input type="hidden" name="_csrf" value="{{csrfToken}}">
+    <input type="hidden" name="id" value="{{id}}" />
+
+{{> report_template_fields }}
+
+    <hr />
+
+    <div class="form-group">
+        <div class="col-sm-offset-2 col-sm-10">
+            <div class="pull-right">
+                <button type="submit" form="report-templates-delete" class="btn btn-danger"><i class="glyphicon glyphicon-remove"></i> {{#translate}}Delete Template{{/translate}}</button>
+            </div>
+            <button type="submit" name="submit" value="update-and-stay"  class="btn btn-primary"><i class="glyphicon glyphicon-ok"></i> {{#translate}}Update and Stay{{/translate}}</button>
+            <button type="submit" name="submit" value="update-and-leave" class="btn btn-primary"><i class="glyphicon glyphicon-ok"></i> {{#translate}}Update and Leave{{/translate}}</button>
+        </div>
+    </div>
+
+</form>

views/report-templates/partials/report-template-fields.hbs

@@ -0,0 +1,59 @@
+    <div class="form-group">
+        <label for="name" class="col-sm-2 control-label">{{#translate}}Name{{/translate}}</label>
+        <div class="col-sm-10">
+            <input type="text" class="form-control input-lg" name="name" id="name" value="{{name}}" placeholder="{{#translate}}Template Name{{/translate}}" autofocus required>
+        </div>
+    </div>
+
+    <div class="form-group">
+        <label for="description" class="col-sm-2 control-label">{{#translate}}Description{{/translate}}</label>
+        <div class="col-sm-10">
+            <textarea class="form-control" rows="3" name="description" id="description">{{description}}</textarea>
+            <span class="help-block">{{#translate}}HTML is allowed{{/translate}}</span>
+        </div>
+    </div>
+
+    <div class="form-group">
+        <label for="mimeType" class="col-sm-2 control-label">{{#translate}}Type{{/translate}}</label>
+        <div class="col-sm-10">
+            <select name="mimeType" class="form-control">
+                {{#each mimeTypes}}
+                    <option value="{{key}}" {{#if selected}} selected {{/if}}>{{value}}</option>
+                {{/each}}
+            </select>
+        </div>
+    </div>
+
+    <div class="form-group">
+        <label class="col-sm-2 control-label">{{#translate}}User selectable fields{{/translate}}</label>
+        <div class="col-sm-offset-2 col-sm-10">
+            <div class="help-block" style="margin-top: -8px;">
+                <small>JSON specification of user selectable fields.</small>
+            </div>
+            <div class="code-editor-json" style="height: 250px; border: 1px solid #ccc;"></div>
+            <input type="hidden" name="userFields" value="{{userFields}}">
+        </div>
+    </div>
+
+    <div class="form-group">
+        <label class="col-sm-2 control-label">{{#translate}}Data processing code{{/translate}}</label>
+        <div class="col-sm-offset-2 col-sm-10">
+            <div class="help-block" style="margin-top: -8px;">
+                <small>Write the body of the JavaScript function with signature <code>function(inputs, callback)</code> that returns an object to be rendered by the Handlebars template below.</small>
+            </div>
+            <div class="code-editor-javascript" style="height: 700px; border: 1px solid #ccc;"></div>
+            <input type="hidden" name="js" value="{{js}}">
+        </div>
+    </div>
+
+    <div class="form-group">
+        <label class="col-sm-2 control-label">{{#translate}}Rendering template{{/translate}}</label>
+        <div class="col-sm-offset-2 col-sm-10">
+            <div class="help-block" style="margin-top: -8px;">
+                <small>Use HTML with Handlebars syntax. See documentation <a href="http://handlebarsjs.com/">here</a>.</small>
+            </div>
+            <div class="code-editor-handlebars" style="height: 700px; border: 1px solid #ccc;"></div>
+            <input type="hidden" name="hbs" value="{{hbs}}">
+        </div>
+    </div>
+

views/report-templates/report-templates.hbs

@@ -0,0 +1,45 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li><a href="/reports">{{#translate}}Reports{{/translate}}</a></li>
+    <li class="active">{{#translate}}Templates{{/translate}}</li>
+</ol>
+
+<div class="pull-right">
+    <div class="btn-group">
+        <button type="button" class="btn btn-primary dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+            {{#translate}}Create Template{{/translate}} <span class="caret"></span>
+        </button>
+        <ul class="dropdown-menu">
+            <li><a href="/report-templates/create">{{#translate}}Blank{{/translate}}</a></li>
+            <li><a href="/report-templates/create?type=subscribers-all">{{#translate}}All Subscribers{{/translate}}</a></li>
+            <li><a href="/report-templates/create?type=subscribers-grouped">{{#translate}}Grouped Subscribers{{/translate}}</a></li>
+            <li><a href="/report-templates/create?type=export-list-csv">{{#translate}}Export List as CSV{{/translate}}</a></li>
+        </ul>
+    </div>
+</div>
+
+<h2>{{#translate}}Report Templates{{/translate}}</h2>
+
+<hr>
+
+<div class="table-responsive">
+    <table data-topic-url="/report-templates" data-sort-column="2" data-sort-order="desc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,0,1,0">
+        <thead>
+            <th class="col-md-1">
+                #
+            </th>
+            <th>
+                {{#translate}}Name{{/translate}}
+            </th>
+            <th>
+                {{#translate}}Description{{/translate}}
+            </th>
+            <th>
+                {{#translate}}Created{{/translate}}
+            </th>
+            <th class="col-md-1">
+                &nbsp;
+            </th>
+        </thead>
+    </table>
+</div>

views/reports/create-select-template.hbs

@@ -0,0 +1,22 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li><a href="/reports">{{#translate}}Reports{{/translate}}</a></li>
+    <li class="active">{{#translate}}Create Report{{/translate}}</li>
+</ol>
+
+<h2>{{#translate}}Create Report{{/translate}}</h2>
+
+<hr>
+
+<form class="form-horizontal" method="get" action="/reports/create">
+
+{{> report_select_template }}
+
+    <hr />
+
+    <div class="form-group">
+        <div class="col-sm-offset-2 col-sm-10">
+            <button type="submit" class="btn btn-primary"><i class="glyphicon glyphicon-chevron-right"></i> {{#translate}}Next{{/translate}}</button>
+        </div>
+    </div>
+</form>

views/reports/create.hbs

@@ -0,0 +1,23 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li><a href="/reports">{{#translate}}Reports{{/translate}}</a></li>
+    <li class="active">{{#translate}}Create Report{{/translate}}</li>
+</ol>
+
+<h2>{{#translate}}Create Report{{/translate}}</h2>
+
+<hr>
+
+<form class="form-horizontal" method="post" action="/reports/create">
+    <input type="hidden" name="_csrf" value="{{csrfToken}}">
+
+{{> report_fields }}
+
+    <hr />
+
+    <div class="form-group">
+        <div class="col-sm-offset-2 col-sm-10">
+            <button type="submit" class="btn btn-primary"><i class="glyphicon glyphicon-plus"></i> {{#translate}}Create Report{{/translate}}</button>
+        </div>
+    </div>
+</form>

views/reports/edit.hbs

@@ -0,0 +1,34 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li><a href="/reports">{{#translate}}Reports{{/translate}}</a></li>
+    <li class="active">{{#translate}}Edit Report{{/translate}}</li>
+</ol>
+
+<h2>{{#translate}}Edit Report{{/translate}}</h2>
+
+<hr>
+
+<form method="post" class="delete-form" id="reports-delete" action="/reports/delete">
+    <input type="hidden" name="_csrf" value="{{csrfToken}}">
+    <input type="hidden" name="id" value="{{id}}" />
+</form>
+
+
+<form class="form-horizontal" method="post" action="/reports/edit">
+    <input type="hidden" name="_csrf" value="{{csrfToken}}">
+    <input type="hidden" name="id" value="{{id}}" />
+
+{{> report_fields }}
+
+    <hr />
+
+    <div class="form-group">
+        <div class="col-sm-offset-2 col-sm-10">
+            <div class="pull-right">
+                <button type="submit" form="reports-delete" class="btn btn-danger"><i class="glyphicon glyphicon-remove"></i> {{#translate}}Delete Report{{/translate}}</button>
+            </div>
+            <button type="submit" class="btn btn-primary"><i class="glyphicon glyphicon-ok"></i> {{#translate}}Update{{/translate}}</button>
+        </div>
+    </div>
+
+</form>

views/reports/output.hbs

@@ -0,0 +1,8 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li><a href="/reports/">{{#translate}}Reports{{/translate}}</a></li>
+    <li class="active">{{title}}</li>
+</ol>
+
+<i>{{error}}</i>
+<pre>{{output}}</pre>

views/reports/partials/report-fields.hbs

@@ -0,0 +1,73 @@
+    {{> report_select_template options="readonly" }}
+
+    <div class="form-group">
+        <label for="name" class="col-sm-2 control-label">{{#translate}}Name{{/translate}}</label>
+        <div class="col-sm-10">
+            <input type="text" class="form-control input-lg" name="name" id="name" value="{{name}}" placeholder="{{#translate}}Report Name{{/translate}}" autofocus required>
+        </div>
+    </div>
+
+    <div class="form-group">
+        <label for="description" class="col-sm-2 control-label">{{#translate}}Description{{/translate}}</label>
+        <div class="col-sm-10">
+            <textarea class="form-control" rows="3" name="description" id="description">{{description}}</textarea>
+            <span class="help-block">{{#translate}}HTML is allowed{{/translate}}</span>
+        </div>
+    </div>
+
+    {{#each userFields}}
+        {{#switch type}}
+            {{#case "campaign"}}
+                <div class="form-group">
+                    <label for="description" class="col-sm-2 control-label">{{name}}</label>
+                    <div class="col-sm-10">
+                        <div class="table-responsive">
+                            <table data-topic-url="/campaigns/quicklist" data-sort-column="2" data-sort-order="desc" class="table table-bordered table-hover data-table-ajax data-table-{{#if isMulti}}multi{{/if}}selectable display nowrap" width="100%" data-row-sort="0,1,0,1">
+                                <thead>
+                                <th class="col-md-1">
+                                    #
+                                </th>
+                                <th>
+                                    {{#translate}}Name{{/translate}}
+                                </th>
+                                <th>
+                                    {{#translate}}Description{{/translate}}
+                                </th>
+                                <th>
+                                    {{#translate}}Created{{/translate}}
+                                </th>
+                                </thead>
+                            </table>
+                            <input type="hidden" name="{{id}}Selection" value="{{value}}" />
+                        </div>
+                        <span class="help-block">{{#translate}}Select a campaign in the table above by clicking on the respective row number.{{/translate}}</span>
+                    </div>
+                </div>
+            {{/case}}
+            {{#case "list"}}
+                <div class="form-group">
+                    <label for="description" class="col-sm-2 control-label">{{name}}</label>
+                    <div class="col-sm-10">
+                        <div class="table-responsive">
+                            <table data-topic-url="/lists/quicklist" data-sort-column="2" data-sort-order="asc" class="table table-bordered table-hover data-table-ajax data-table-{{#if isMulti}}multi{{/if}}selectable display nowrap" width="100%" data-row-sort="0,1,1">
+                                <thead>
+                                <th class="col-md-1">
+                                    #
+                                </th>
+                                <th>
+                                    {{#translate}}Name{{/translate}}
+                                </th>
+                                <th>
+                                    {{#translate}}Subscribers{{/translate}}
+                                </th>
+                                </thead>
+                            </table>
+                            <input type="hidden" name="{{id}}Selection" value="{{value}}" />
+                        </div>
+                        <span class="help-block">{{#translate}}Select a campaign in the table above by clicking on the respective row number.{{/translate}}</span>
+                    </div>
+                </div>
+            {{/case}}
+        {{/switch}}
+    {{/each}}
+

views/reports/partials/report-select-template.hbs

@@ -0,0 +1,11 @@
+    <div class="form-group">
+        <label for="name" class="col-sm-2 control-label">{{#translate}}Report Template{{/translate}}</label>
+        <div class="col-sm-10">
+            <select class="form-control" id="reportTemplate" name="reportTemplate" required {{options}}>
+                <option value=""> –– {{#translate}}Select{{/translate}} –– </option>
+                {{#each reportTemplates}}
+                    <option value="{{id}}" {{#if selected}} selected {{/if}}>{{name}}</option>
+                {{/each}}
+            </select>
+        </div>
+    </div>

views/reports/reports.hbs

@@ -0,0 +1,40 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li class="active">{{#translate}}Reports{{/translate}}</li>
+</ol>
+
+<div class="pull-right">
+    <a class="btn btn-primary" href="/reports/create" role="button"><i class="glyphicon glyphicon-plus"></i> {{#translate}}Create Report{{/translate}}</a>
+
+    <a class="btn btn-primary" href="/report-templates" role="button">{{#translate}}Report Templates{{/translate}}</a>
+</div>
+
+<h2>{{#translate}}Reports{{/translate}}</h2>
+
+<hr>
+
+<div class="table-responsive">
+    <table data-topic-url="/reports" data-sort-column="2" data-sort-order="desc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,1,0,1,0">
+        <thead>
+        <th style="width: 1%">
+            #
+        </th>
+        <th>
+            {{#translate}}Name{{/translate}}
+        </th>
+        <th>
+            {{#translate}}Template{{/translate}}
+        </th>
+        <th>
+            {{#translate}}Description{{/translate}}
+        </th>
+        <th>
+            {{#translate}}Created{{/translate}}
+        </th>
+        <th style="width: 1%">
+            &nbsp;
+        </th>
+        </thead>
+    </table>
+</div>
+

views/reports/view.hbs

@@ -0,0 +1,7 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li><a href="/reports/">{{#translate}}Reports{{/translate}}</a></li>
+    <li class="active">{{title}}</li>
+</ol>
+
+{{report}}

views/templates/templates.hbs

@@ -12,9 +12,9 @@
 <hr>
 
 <div class="table-responsive">
-    <table class="table table-bordered table-hover data-table display nowrap" width="100%" data-row-sort="0,1,0,0">
+    <table data-topic-url="/templates" data-sort-column="1" data-sort-order="asc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,0,0">
         <thead>
-            <th class="col-md-1">
+            <th style="width: 1%">
                 #
             </th>
             <th>
@@ -23,32 +23,9 @@
             <th>
                 {{#translate}}Description{{/translate}}
             </th>
-            <th class="col-md-1">
+            <th style="width: 1%">
                 &nbsp;
             </th>
         </thead>
-        {{#if rows}}
-            <tbody>
-                {{#each rows}}
-                    <tr>
-                        <th scope="row">
-                            {{index}}
-                        </th>
-                        <td>
-                            <span class="glyphicon glyphicon-file" aria-hidden="true"></span> {{name}}
-                        </td>
-                        <td>
-                            <p class="text-muted">{{description}}</p>
-                        </td>
-                        <td>
-                            <span class="glyphicon glyphicon-wrench" aria-hidden="true"></span>
-                            <a href="/templates/edit/{{id}}">
-                            {{#translate}}Edit{{/translate}}
-                        </a>
-                        </td>
-                    </tr>
-                {{/each}}
-            </tbody>
-        {{/if}}
     </table>
 </div>

workers/reports/config/default.toml

@@ -0,0 +1,18 @@
+# Process title visible in monitoring logs and process listing
+title="mailtrain"
+
+# Default language to use
+language="en"
+
+[log]
+# silly|verbose|info|http|warn|error|silent
+level="verbose"
+
+[mysql]
+host="localhost"
+user="mailtrain"
+password="mailtrain"
+database="mailtrain"
+port=3306
+charset="utf8mb4"
+timezone="local"

workers/reports/report-processor.js

@@ -0,0 +1,147 @@
+'use strict';
+
+const reports = require('../../lib/models/reports');
+const reportTemplates = require('../../lib/models/report-templates');
+const lists = require('../../lib/models/lists');
+const subscriptions = require('../../lib/models/subscriptions');
+const campaigns = require('../../lib/models/campaigns');
+const handlebars = require('handlebars');
+const handlebarsHelpers = require('../../lib/handlebars-helpers');
+const _ = require('../../lib/translate')._;
+const hbs = require('hbs');
+const vm = require('vm');
+const log = require('npmlog');
+const fs = require('fs');
+
+handlebarsHelpers.registerHelpers(handlebars);
+
+let reportId = Number(process.argv[2]);
+let reportDir;
+
+function resolveEntities(getter, ids, callback) {
+    const idsRemaining = ids.slice();
+    const resolved = [];
+
+    function doWork() {
+        if (idsRemaining.length == 0) {
+            return callback(null, resolved);
+        }
+
+        getter(idsRemaining.shift(), (err, entity) => {
+            if (err) {
+                return callback(err);
+            }
+
+            resolved.push(entity);
+            return doWork();
+        });
+    }
+
+    setImmediate(doWork);
+}
+
+const userFieldTypeToGetter = {
+    'campaign': (id, callback) => campaigns.get(id, false, callback),
+    'list': lists.get
+};
+
+function resolveUserFields(userFields, params, callback) {
+    const userFieldsRemaining = userFields.slice();
+    const resolved = {};
+
+    function doWork() {
+        if (userFieldsRemaining.length == 0) {
+            return callback(null, resolved);
+        }
+
+        const spec = userFieldsRemaining.shift();
+        const getter = userFieldTypeToGetter[spec.type];
+
+        if (getter) {
+            return resolveEntities(getter, params[spec.id], (err, entities) => {
+                if (spec.minOccurences == 1 && spec.maxOccurences == 1) {
+                    resolved[spec.id] = entities[0];
+                } else {
+                    resolved[spec.id] = entities;
+                }
+
+                doWork();
+            });
+        } else {
+            return callback(new Error(_('Unknown user field type "' + spec.type + '".')));
+        }
+    }
+
+    setImmediate(doWork);
+}
+
+function doneSuccess() {
+    process.exit(0);
+}
+
+function doneFail() {
+    process.exit(1)
+}
+
+
+
+reports.get(reportId, (err, report) => {
+    if (err || !report) {
+        log.error('reports', err && err.message || err || _('Could not find report with specified ID'));
+        doneFail();
+    }
+
+    reportTemplates.get(report.reportTemplate, (err, reportTemplate) => {
+        if (err) {
+            log.error('reports', err && err.message || err || _('Could not find report template'));
+            doneFail();
+        }
+
+        resolveUserFields(reportTemplate.userFieldsObject, report.paramsObject, (err, inputs) => {
+            if (err) {
+                log.error('reports', err.message || err);
+                doneFail();
+            }
+
+            const campaignsProxy = {
+                results: reports.getCampaignResults,
+                list: campaigns.list,
+                get: campaigns.get
+            };
+
+            const subscriptionsProxy = {
+                list: subscriptions.list
+            };
+
+            const sandbox = {
+                console,
+                campaigns: campaignsProxy,
+                subscriptions: subscriptionsProxy,
+                inputs,
+
+                callback: (err, outputs) => {
+                    if (err) {
+                        log.error('reports', err.message || err);
+                        doneFail();
+                    }
+
+                    const hbsTmpl = handlebars.compile(reportTemplate.hbs);
+                    const reportText = hbsTmpl(outputs);
+
+                    process.stdout.write(reportText);
+                    doneSuccess();
+                }
+            };
+
+            const script = new vm.Script(reportTemplate.js);
+
+            try {
+                script.runInNewContext(sandbox, {displayErrors: true, timeout: 120000});
+            } catch (err) {
+                console.error(err);
+                doneFail();
+            }
+        });
+    });
+});
+