Merge remote-tracking branch 'upstream/master'

.gitignore

@@ -3,6 +3,10 @@ npm-debug.log
 .DS_Store
 config/development.*
 config/production.*
+config/test.*
+workers/reports/config/development.*
+workers/reports/config/production.*
+workers/reports/config/test.*
 dump.rdb
 
 # generate POT file every time you want to update your PO file

Gruntfile.js

@@ -9,7 +9,7 @@ module.exports = function (grunt) {
         },
 
         nodeunit: {
-            all: ['test/**/*-test.js']
+            all: ['test/nodeunit/**/*-test.js']
         },
 
         jsxgettext: {

README.md

@@ -1,9 +1,11 @@
 # Mailtrain
 
-[Mailtrain](http://mailtrain.org) is a self hosted newsletter application built on Node.js (v5+) and MySQL (v5.5+ or MariaDB).
+[Mailtrain](http://mailtrain.org) is a self hosted newsletter application built on Node.js (v7+) and MySQL (v5.5+ or MariaDB).
 
 ![](http://mailtrain.org/mailtrain.png)
 
+> Mailtrain requires at least **Node.js v7**. If you want to use an older version of Node.js then you should use version v1.24 of Mailtrain. You can either download it [here](https://github.com/Mailtrain-org/mailtrain/archive/v1.24.0.zip) or if using git then run `git checkout v1.24.0` before starting it
+
 ## Features
 
 Mailtrain supports subscriber list management, list segmentation, custom fields, email templates, large CSV list import files, etc.
@@ -45,7 +47,7 @@ Check out [ZoneMTA](https://github.com/zone-eu/zone-mta) as an alternative self
 
 ## Requirements
 
-  * Nodejs v6+
+  * Nodejs v7+
   * MySQL v5.5 or MariaDB
   * Redis. Optional, disabled by default. Used for session storage and for caching state between multiple processes. If you do not have Redis enabled then you can only use a single sender process
 

app.js

@@ -1,47 +1,49 @@
 'use strict';
 
-let config = require('config');
+const config = require('config');
-let log = require('npmlog');
+const log = require('npmlog');
 
-let _ = require('./lib/translate')._;
+const _ = require('./lib/translate')._;
-let util = require('util');
 
-let express = require('express');
+const express = require('express');
-let bodyParser = require('body-parser');
+const bodyParser = require('body-parser');
-let path = require('path');
+const path = require('path');
-let favicon = require('serve-favicon');
+const favicon = require('serve-favicon');
-let logger = require('morgan');
+const logger = require('morgan');
-let cookieParser = require('cookie-parser');
+const cookieParser = require('cookie-parser');
-let session = require('express-session');
+const session = require('express-session');
-let RedisStore = require('connect-redis')(session);
+const RedisStore = require('connect-redis')(session);
-let flash = require('connect-flash');
+const flash = require('connect-flash');
-let hbs = require('hbs');
+const hbs = require('hbs');
-let compression = require('compression');
+const handlebarsHelpers = require('./lib/handlebars-helpers');
-let passport = require('./lib/passport');
+const compression = require('compression');
-let tools = require('./lib/tools');
+const passport = require('./lib/passport');
+const tools = require('./lib/tools');
 
-let routes = require('./routes/index');
+const routes = require('./routes/index');
-let users = require('./routes/users');
+const users = require('./routes/users');
-let lists = require('./routes/lists');
+const lists = require('./routes/lists');
-let settings = require('./routes/settings');
+const settings = require('./routes/settings');
-let settingsModel = require('./lib/models/settings');
+const settingsModel = require('./lib/models/settings');
-let templates = require('./routes/templates');
+const templates = require('./routes/templates');
-let campaigns = require('./routes/campaigns');
+const campaigns = require('./routes/campaigns');
-let links = require('./routes/links');
+const links = require('./routes/links');
-let fields = require('./routes/fields');
+const fields = require('./routes/fields');
-let forms = require('./routes/forms');
+const forms = require('./routes/forms');
-let segments = require('./routes/segments');
+const segments = require('./routes/segments');
-let triggers = require('./routes/triggers');
+const triggers = require('./routes/triggers');
-let webhooks = require('./routes/webhooks');
+const webhooks = require('./routes/webhooks');
-let subscription = require('./routes/subscription');
+const subscription = require('./routes/subscription');
-let archive = require('./routes/archive');
+const archive = require('./routes/archive');
-let api = require('./routes/api');
+const api = require('./routes/api');
-let blacklist = require('./routes/blacklist');
+const blacklist = require('./routes/blacklist');
-let editorapi = require('./routes/editorapi');
+const editorapi = require('./routes/editorapi');
-let grapejs = require('./routes/grapejs');
+const grapejs = require('./routes/grapejs');
-let mosaico = require('./routes/mosaico');
+const mosaico = require('./routes/mosaico');
+const reports = require('./routes/reports');
+const reportsTemplates = require('./routes/report-templates');
 
-let app = express();
+const app = express();
 
 // view engine setup
 app.set('views', path.join(__dirname, 'views'));
@@ -57,6 +59,8 @@ app.disable('x-powered-by');
 
 hbs.registerPartials(__dirname + '/views/partials');
 hbs.registerPartials(__dirname + '/views/subscription/partials/');
+hbs.registerPartials(__dirname + '/views/report-templates/partials/');
+hbs.registerPartials(__dirname + '/views/reports/partials/');
 
 /**
  * We need this helper to make sure that we consume flash messages only
@@ -104,20 +108,8 @@ hbs.registerHelper('flash_messages', function () { // eslint-disable-line prefer
     );
 });
 
-// {{#translate}}abc{{/translate}}
+handlebarsHelpers.registerHelpers(hbs.handlebars);
-hbs.registerHelper('translate', function (context, options) { // eslint-disable-line prefer-arrow-callback
-    if (typeof options === 'undefined' && context) {
-        options = context;
-        context = false;
-    }
 
-    let result = _(options.fn(this)); // eslint-disable-line no-invalid-this
-
-    if (Array.isArray(context)) {
-        result = util.format(result, ...context);
-    }
-    return new hbs.handlebars.SafeString(result);
-});
 
 app.use(compression());
 app.use(favicon(path.join(__dirname, 'public', 'favicon.ico')));
@@ -191,6 +183,11 @@ app.use((req, res, next) => {
     res.locals.customStyles = config.customstyles || [];
     res.locals.customScripts = config.customscripts || [];
 
+    let bodyClasses = [];
+    app.get('env') === 'test' && bodyClasses.push('page--' + (req.path.substring(1).replace(/\//g, '--') || 'home'));
+    req.user && bodyClasses.push('logged-in user-' + req.user.username);
+    res.locals.bodyClass = bodyClasses.join(' ');
+
     settingsModel.list(['ua_code', 'shoutout'], (err, configItems) => {
         if (err) {
             return next(err);
@@ -222,6 +219,11 @@ app.use('/editorapi', editorapi);
 app.use('/grapejs', grapejs);
 app.use('/mosaico', mosaico);
 
+if (config.reports && config.reports.enabled === true) {
+    app.use('/reports', reports);
+    app.use('/report-templates', reportsTemplates);
+}
+
 // catch 404 and forward to error handler
 app.use((req, res, next) => {
     let err = new Error(_('Not Found'));

config/default.toml

@@ -43,8 +43,14 @@ language="en"
 
 # If you start out as a root user (eg. if you want to use ports lower than 1000)
 # then you can downgrade the user once all services are up and running
-#user="nobody"
+#user="mailtrain"
-#group="nogroup"
+#group="mailtrain"
+
+# If Mailtrain is started as root, "Reports" feature drops the privileges of script generating the report to disallow
+# any modifications of Mailtrain code and even prohibits reading the production configuration (which contains the MySQL
+# password for read/write operations). The rouser/rogroup determines the user to be used
+#rouser="nobody"
+#rogroup="nogroup"
 
 [log]
 # silly|verbose|info|http|warn|error|silent
@@ -97,16 +103,12 @@ db=5
 enabled=false
 port=2525
 host="0.0.0.0"
-
+# With DMARC, the Return-Path and From address must match the same domain.
-[testserver]
+# By default we get around this by using the VERP address in the Sender header,
-# Starts a vanity server that redirects all mail to /dev/null
+# with the side effect that some email clients diplay an ugly "on behalf of" message.
-# Mostly needed for local development
+# You can safely disable this Sender header if you're not using DMARC or your
-enabled=false
+# VERP hostname is in the same domain as the From address.
-port=5587
+# disablesenderheader=true
-host="0.0.0.0"
-username="testuser"
-password="testpass"
-logger=false
 
 [ldap]
 # enable to use ldap user backend
@@ -150,3 +152,32 @@ templates=[["versafix-1", "Versafix One"]]
 [grapejs]
 # Installed templates
 templates=[["demo", "Demo Template"]]
+
+[reports]
+# The whole reporting functionality can be disabled below if the they are not needed and the DB cannot be
+# properly protected.
+# Reports rely on custom user defined Javascript snippets defined in the report template. The snippets are run on the
+# server when generating a report. As these snippets are stored in the DB, they pose a security risk because they can
+# help gaining access to the server if the DB cannot
+# be properly protected (e.g. if it is shared with another application with security weaknesses).
+# Mailtrain mitigates this problem by running the custom Javascript snippets in a chrooted environment and under a
+# DB user that cannot modify the database (see userRO in [mysql] above). However the chrooted environment is available
+# only if Mailtrain is started as root. The chrooted environment still does not prevent the custom JS script in
+# performing network operations and in generating XSS attacks as part of the report.
+# The bottom line is that if people who are creating report templates or have write access to the DB cannot be trusted,
+# then it's safer to switch off the reporting functionality below.
+enabled=false
+
+[testserver]
+# Starts a vanity server that redirects all mail to /dev/null
+# Mostly needed for local development
+enabled=false
+port=5587
+mailboxserverport=3001
+host="0.0.0.0"
+username="testuser"
+password="testpass"
+logger=false
+
+[seleniumwebdriver]
+browser="phantomjs"

index.js

@@ -4,20 +4,23 @@
  * Module dependencies.
  */
 
-let config = require('config');
+const config = require('config');
-let log = require('npmlog');
+const log = require('npmlog');
-let app = require('./app');
+const app = require('./app');
-let http = require('http');
+const http = require('http');
-let fork = require('child_process').fork;
+const fork = require('child_process').fork;
-let triggers = require('./services/triggers');
+const triggers = require('./services/triggers');
-let importer = require('./services/importer');
+const importer = require('./services/importer');
-let verpServer = require('./services/verp-server');
+const verpServer = require('./services/verp-server');
-let testServer = require('./services/test-server');
+const testServer = require('./services/test-server');
-let postfixBounceServer = require('./services/postfix-bounce-server');
+const postfixBounceServer = require('./services/postfix-bounce-server');
-let tzupdate = require('./services/tzupdate');
+const tzupdate = require('./services/tzupdate');
-let feedcheck = require('./services/feedcheck');
+const feedcheck = require('./services/feedcheck');
-let dbcheck = require('./lib/dbcheck');
+const dbcheck = require('./lib/dbcheck');
-let tools = require('./lib/tools');
+const tools = require('./lib/tools');
+const reportProcessor = require('./lib/report-processor');
+const executor = require('./lib/executor');
+const privilegeHelpers = require('./lib/privilege-helpers');
 
 let port = config.www.port;
 let host = config.www.host;
@@ -112,31 +115,22 @@ server.on('listening', () => {
     log.info('Express', 'WWW server listening on %s', bind);
 
     // start additional services
-    testServer(() => {
+    function startNextServices() {
-        verpServer(() => {
+        testServer(() => {
-            tzupdate(() => {
+            verpServer(() => {
-                importer(() => {
+
-                    triggers(() => {
+                privilegeHelpers.dropRootPrivileges();
-                        spawnSenders(() => {
+
-                            feedcheck(() => {
+                tzupdate(() => {
-                                postfixBounceServer(() => {
+                    importer(() => {
-                                    log.info('Service', 'All services started');
+                        triggers(() => {
-                                    if (config.group) {
+                            spawnSenders(() => {
-                                        try {
+                                feedcheck(() => {
-                                            process.setgid(config.group);
+                                    postfixBounceServer(() => {
-                                            log.info('Service', 'Changed group to "%s" (%s)', config.group, process.getgid());
+                                        reportProcessor.init(() => {
-                                        } catch (E) {
+                                            log.info('Service', 'All services started');
-                                            log.info('Service', 'Failed to change group to "%s" (%s)', config.group, E.message);
+                                        });
-                                        }
+                                    });
-                                    }
-                                    if (config.user) {
-                                        try {
-                                            process.setuid(config.user);
-                                            log.info('Service', 'Changed user to "%s" (%s)', config.user, process.getuid());
-                                        } catch (E) {
-                                            log.info('Service', 'Failed to change user to "%s" (%s)', config.user, E.message);
-                                        }
-                                    }
                                 });
                             });
                         });
@@ -144,5 +138,11 @@ server.on('listening', () => {
                 });
             });
         });
-    });
+    }
+
+    if (config.reports && config.reports.enabled === true) {
+        executor.spawn(startNextServices);
+    } else {
+        startNextServices();
+    }
 });

lib/db.js

@@ -6,7 +6,7 @@ let redis = require('redis');
 let Lock = require('redfour');
 
 module.exports = mysql.createPool(config.mysql);
-if (config.redis.enabled) {
+if (config.redis && config.redis.enabled) {
 
     module.exports.redis = redis.createClient(config.redis);
 

lib/dbcheck.js

@@ -107,13 +107,14 @@ function getSql(path, data, callback) {
         if (err) {
             return callback(err);
         }
-        let renderer = Handlebars.compile(source);
+        const rendered = data ? Handlebars.compile(source)(data) : source;
-        return callback(null, renderer(data || {}));
+        return callback(null, rendered);
     });
 }
 
 function runInitial(callback) {
-    let fname = process.env.DB_FROM_START ? 'base.sql' : 'mailtrain.sql';
+    let dump = process.env.NODE_ENV === 'test' ? 'mailtrain-test.sql' : 'mailtrain.sql';
+    let fname = process.env.DB_FROM_START ? 'base.sql' : dump;
     let path = pathlib.join(__dirname, '..', 'setup', 'sql', fname);
     log.info('sql', 'Loading tables from %s', fname);
     applyUpdate({

lib/editor-helpers.js

@@ -6,7 +6,8 @@ let templates = require('../lib/models/templates');
 let campaigns = require('../lib/models/campaigns');
 
 module.exports = {
-    getResource
+    getResource,
+    getMergeTagsForResource
 };
 
 function getResource(type, id, callback) {
@@ -53,7 +54,7 @@ function getMergeTagsForResource(resource, callback) {
             return callback(err.message || err);
         }
 
-        if (!resource.list) {
+        if (!Number(resource.list)) {
             return callback(null, defaultMergeTags);
         }
 
@@ -62,7 +63,17 @@ function getMergeTagsForResource(resource, callback) {
                 return callback(err.message || err);
             }
 
-            callback(null, defaultMergeTags.concat(listMergeTags));
+            if (resource.type !== 2) {
+                return callback(null, defaultMergeTags.concat(listMergeTags));
+            }
+
+            helpers.getRSSMergeTags((err, rssMergeTags) => {
+                if (err) {
+                    return callback(err.message || err);
+                }
+
+                callback(null, defaultMergeTags.concat(listMergeTags, rssMergeTags));
+            });
         });
     });
 }

lib/executor.js

@@ -0,0 +1,83 @@
+'use strict';
+
+const fork = require('child_process').fork;
+const log = require('npmlog');
+const path = require('path');
+
+const requestCallbacks = {};
+let messageTid = 0;
+let executorProcess;
+
+module.exports = {
+    spawn,
+    start,
+    stop
+};
+
+function spawn(callback) {
+    log.info('Executor', 'Spawning executor process.');
+
+    executorProcess = fork(path.join(__dirname, '..', 'services', 'executor.js'), [], {
+        cwd: path.join(__dirname, '..'),
+        env: {NODE_ENV: process.env.NODE_ENV}
+    });
+
+    executorProcess.on('message', msg => {
+        if (msg) {
+            if (msg.type === 'process-started') {
+                let requestCallback = requestCallbacks[msg.tid];
+                if (requestCallback && requestCallback.startedCallback) {
+                    requestCallback.startedCallback(msg.tid);
+                }
+
+            } else if (msg.type === 'process-failed') {
+                let requestCallback = requestCallbacks[msg.tid];
+                if (requestCallback && requestCallback.failedCallback) {
+                    requestCallback.failedCallback(msg.msg);
+                }
+
+                delete requestCallbacks[msg.tid];
+
+            } else if (msg.type === 'process-finished') {
+                let requestCallback = requestCallbacks[msg.tid];
+                if (requestCallback && requestCallback.startedCallback) {
+                    requestCallback.finishedCallback(msg.code, msg.signal);
+                }
+
+                delete requestCallbacks[msg.tid];
+
+            } else if (msg.type === 'executor-started') {
+                log.info('Executor', 'Executor process started.');
+                return callback();
+            }
+        }
+    });
+
+    executorProcess.on('close', (code, signal) => {
+        log.info('Executor', 'Executor process exited with code %s signal %s.', code, signal);
+    });
+}
+
+function start(type, data, startedCallback, finishedCallback, failedCallback) {
+    requestCallbacks[messageTid] = {
+        startedCallback,
+        finishedCallback,
+        failedCallback
+    };
+
+    executorProcess.send({
+        type: 'start-' + type,
+        data,
+        tid: messageTid
+    });
+
+    messageTid++;
+}
+
+function stop(tid) {
+    executorProcess.send({
+        type: 'stop-process',
+        tid
+    });
+}
+

lib/feed.js

@@ -50,7 +50,9 @@ module.exports.fetch = (url, callback) => {
                 date: item.date || item.pubdate || item.pubDate || new Date(),
                 guid: item.guid || item.link,
                 link: item.link,
-                content: item.description || item.summary
+                content: item.description || item.summary,
+                summary: item.summary || item.description,
+                image_url: item.image.url
             };
             entries.push(entry);
         }

lib/file-helpers.js

@@ -0,0 +1,32 @@
+'use strict';
+
+const path = require('path');
+
+function nameToFileName(name) {
+    return name.
+    trim().
+    toLowerCase().
+    replace(/[ .+/]/g, '-').
+    replace(/[^a-z0-9\-_]/gi, '').
+    replace(/--*/g, '-');
+}
+
+
+function getReportFileBase(report) {
+    return path.join(__dirname, '..', 'protected', 'reports', report.id + '-' + nameToFileName(report.name));
+}
+
+function getReportContentFile(report) {
+    return getReportFileBase(report) + '.out';
+}
+
+function getReportOutputFile(report) {
+    return getReportFileBase(report) + '.err';
+}
+
+
+module.exports = {
+    getReportContentFile,
+    getReportOutputFile,
+    nameToFileName
+};

lib/handlebars-helpers.js

@@ -0,0 +1,49 @@
+'use strict';
+
+const util = require('util');
+
+const _ = require('../lib/translate')._;
+
+module.exports.registerHelpers = handlebars => {
+    // {{#translate}}abc{{/translate}}
+    handlebars.registerHelper('translate', function (context, options) { // eslint-disable-line prefer-arrow-callback
+        if (typeof options === 'undefined' && context) {
+            options = context;
+            context = false;
+        }
+
+        let result = _(options.fn(this)); // eslint-disable-line no-invalid-this
+
+        if (Array.isArray(context)) {
+            result = util.format(result, ...context);
+        }
+        return new handlebars.SafeString(result);
+    });
+
+
+    /* Credits to http://chrismontrois.net/2016/01/30/handlebars-switch/
+
+     {{#switch letter}}
+     {{#case "a"}}
+     A is for alpaca
+     {{/case}}
+     {{#case "b"}}
+     B is for bluebird
+     {{/case}}
+     {{/switch}}
+     */
+    /* eslint no-invalid-this: "off" */
+    handlebars.registerHelper('switch', function(value, options) {
+        this._switch_value_ = value;
+        const html = options.fn(this); // Process the body of the switch block
+        delete this._switch_value_;
+        return html;
+    });
+
+    handlebars.registerHelper('case', function(value, options) {
+        if (value === this._switch_value_) {
+            return options.fn(this);
+        }
+    });
+
+};

lib/helpers.js

@@ -16,6 +16,7 @@ let hbs = require('hbs');
 
 module.exports = {
     getDefaultMergeTags,
+    getRSSMergeTags,
     getListMergeTags,
     captureFlashMessages,
     injectCustomFormData,
@@ -59,6 +60,32 @@ function getDefaultMergeTags(callback) {
     }]);
 }
 
+function getRSSMergeTags(callback) {
+    // Using a callback for the sake of future-proofness
+    callback(null, [{
+        key: 'RSS_ENTRY',
+        value: _('content from an RSS entry')
+    }, {
+        key: 'RSS_ENTRY_TITLE',
+        value: _('RSS entry title')
+    }, {
+        key: 'RSS_ENTRY_DATE',
+        value: _('RSS entry date')
+    }, {
+        key: 'RSS_ENTRY_LINK',
+        value: _('RSS entry link')
+    }, {
+        key: 'RSS_ENTRY_CONTENT',
+        value: _('content from an RSS entry')
+    }, {
+        key: 'RSS_ENTRY_SUMMARY',
+        value: _('RSS entry summary')
+    }, {
+        key: 'RSS_ENTRY_IMAGE_URL',
+        value: _('RSS entry image URL')
+    }]);
+}
+
 function getListMergeTags(listId, callback) {
     lists.get(listId, (err, list) => {
         if (err) {

lib/models/campaigns.js

@@ -14,186 +14,44 @@ let mailer = require('../mailer');
 let humanize = require('humanize');
 let _ = require('../translate')._;
 let util = require('util');
+let tableHelpers = require('../table-helpers');
 
 let allowedKeys = ['description', 'from', 'address', 'reply_to', 'subject', 'editor_name', 'editor_data', 'template', 'source_url', 'list', 'segment', 'html', 'text', 'tracking_disabled'];
 
 module.exports.list = (start, limit, callback) => {
-    db.getConnection((err, connection) => {
+    tableHelpers.list('campaigns', ['*'], 'scheduled', null, start, limit, callback);
-        if (err) {
-            return callback(err);
-        }
-
-        connection.query('SELECT SQL_CALC_FOUND_ROWS * FROM campaigns ORDER BY scheduled DESC LIMIT ? OFFSET ?', [limit, start], (err, rows) => {
-            if (err) {
-                connection.release();
-                return callback(err);
-            }
-            connection.query('SELECT FOUND_ROWS() AS total', (err, total) => {
-                connection.release();
-                if (err) {
-                    return callback(err);
-                }
-                return callback(null, rows, total && total[0] && total[0].total);
-            });
-        });
-    });
 };
 
 module.exports.filter = (request, parent, callback) => {
-    let columns = ['#', 'name', 'description', 'status', 'created'];
+    let queryData;
-    let processQuery = queryData => {
-
-        db.getConnection((err, connection) => {
-            if (err) {
-                return callback(err);
-            }
-
-            let query = 'SELECT COUNT(id) AS total FROM `campaigns`';
-            let values = [];
-
-            if (queryData.where) {
-                query += ' WHERE ' + queryData.where;
-                values = values.concat(queryData.values || []);
-            }
-
-            connection.query(query, values, (err, total) => {
-                if (err) {
-                    connection.release();
-                    return callback(err);
-                }
-                total = total && total[0] && total[0].total || 0;
-
-                let ordering = [];
-
-                if (request.order && request.order.length) {
-
-                    request.order.forEach(order => {
-                        let orderField = columns[Number(order.column)];
-                        let orderDirection = (order.dir || '').toString().toLowerCase() === 'desc' ? 'DESC' : 'ASC';
-                        if (orderField) {
-                            ordering.push('`' + orderField + '` ' + orderDirection);
-                        }
-                    });
-                }
-
-                if (!ordering.length) {
-                    ordering.push('`created` DESC');
-                }
-
-                let args = [Number(request.length) || 50, Number(request.start) || 0];
-                let query;
-
-                if (request.search && request.search.value) {
-                    query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `campaigns` WHERE name LIKE ? ' + (queryData.where ? ' AND (' + queryData.where + ')' : '') + ' ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-
-                    let searchVal = '%' + request.search.value.replace(/\\/g, '\\\\').replace(/([%_])/g, '\\$1') + '%';
-                    args = [searchVal].concat(queryData.values || []).concat(args);
-                } else {
-                    query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `campaigns` WHERE 1 ' + (queryData.where ? ' AND (' + queryData.where + ')' : '') + ' ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-                    args = [].concat(queryData.values || []).concat(args);
-                }
-
-                connection.query(query, args, (err, rows) => {
-                    if (err) {
-                        connection.release();
-                        return callback(err);
-                    }
-                    connection.query('SELECT FOUND_ROWS() AS total', (err, filteredTotal) => {
-                        connection.release();
-                        if (err) {
-                            return callback(err);
-                        }
-
-                        let subscriptions = rows.map(row => tools.convertKeys(row));
-
-                        filteredTotal = filteredTotal && filteredTotal[0] && filteredTotal[0].total || 0;
-                        return callback(null, subscriptions, total, filteredTotal);
-                    });
-                });
-            });
-        });
-    };
-
     if (parent) {
-        processQuery({
+        queryData = {
             // only find normal and RSS parent campaigns at this point
             where: '`parent`=?',
             values: [parent]
-        });
+        };
     } else {
-
+        queryData = {
-        processQuery({
             // only find normal and RSS parent campaigns at this point
             where: '`type` IN (?,?,?)',
             values: [1, 2, 4]
-        });
+        };
     }
+
+    tableHelpers.filter('campaigns', ['*'], request, ['#', 'name', 'description', 'status', 'created'], ['name'], 'created DESC', queryData, callback);
+};
+
+module.exports.filterQuicklist = (request, callback) => {
+    tableHelpers.filter('campaigns', ['id', 'name', 'description', 'created'], request, ['#', 'name', 'description', 'created'], ['name'], 'name ASC', null, callback);
 };
 
 module.exports.filterClickedSubscribers = (campaign, linkId, request, columns, callback) => {
-    db.getConnection((err, connection) => {
+    let queryData = {
-        if (err) {
+        where: 'campaign_tracker__' + campaign.id + '.list=? AND campaign_tracker__' + campaign.id + '.link=?',
-            return callback(err);
+        values: [campaign.list, linkId]
-        }
+    };
 
-        let query = 'SELECT COUNT(`subscription__' + campaign.list + '`.`id`) AS total FROM `subscription__' + campaign.list + '` JOIN `campaign_tracker__' + campaign.id + '` ON `campaign_tracker__' + campaign.id + '`.`list`=? AND `campaign_tracker__' + campaign.id + '`.`subscriber`=`subscription__' + campaign.list + '`.`id` AND `campaign_tracker__' + campaign.id + '`.`link`=?';
+    tableHelpers.filter('subscription__' + campaign.list + ' JOIN campaign_tracker__' + campaign.id + ' ON campaign_tracker__' + campaign.id + '.subscriber=subscription__' + campaign.list + '.id', ['*'], request, columns, ['email', 'first_name', 'last_name'], 'email ASC', queryData, callback);
-        let values = [campaign.list, linkId];
-
-        connection.query(query, values, (err, total) => {
-            if (err) {
-                connection.release();
-                return callback(err);
-            }
-            total = total && total[0] && total[0].total || 0;
-
-            let ordering = [];
-
-            if (request.order && request.order.length) {
-
-                request.order.forEach(order => {
-                    let orderField = columns[Number(order.column)];
-                    let orderDirection = (order.dir || '').toString().toLowerCase() === 'desc' ? 'DESC' : 'ASC';
-                    if (orderField) {
-                        ordering.push('`' + orderField + '` ' + orderDirection);
-                    }
-                });
-            }
-
-            if (!ordering.length) {
-                ordering.push('`email` ASC');
-            }
-
-            let args = [Number(request.length) || 50, Number(request.start) || 0];
-
-            if (request.search && request.search.value) {
-                query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + campaign.list + '`  JOIN `campaign_tracker__' + campaign.id + '` ON `campaign_tracker__' + campaign.id + '`.`list`=? AND `campaign_tracker__' + campaign.id + '`.`subscriber`=`subscription__' + campaign.list + '`.`id` AND `campaign_tracker__' + campaign.id + '`.`link`=? WHERE email LIKE ? OR first_name LIKE ? OR last_name LIKE ? ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-
-                let searchVal = '%' + request.search.value.replace(/\\/g, '\\\\').replace(/([%_])/g, '\\$1') + '%';
-                args = values.concat([searchVal, searchVal, searchVal]).concat(args);
-            } else {
-                query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + campaign.list + '` JOIN `campaign_tracker__' + campaign.id + '` ON `campaign_tracker__' + campaign.id + '`.`list`=? AND `campaign_tracker__' + campaign.id + '`.`subscriber`=`subscription__' + campaign.list + '`.`id` AND `campaign_tracker__' + campaign.id + '`.`link`=? ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-                args = values.concat(args);
-            }
-
-            connection.query(query, args, (err, rows) => {
-                if (err) {
-                    connection.release();
-                    return callback(err);
-                }
-                connection.query('SELECT FOUND_ROWS() AS total', (err, filteredTotal) => {
-                    connection.release();
-                    if (err) {
-                        return callback(err);
-                    }
-
-                    let subscriptions = rows.map(row => tools.convertKeys(row));
-
-                    filteredTotal = filteredTotal && filteredTotal[0] && filteredTotal[0].total || 0;
-                    return callback(null, subscriptions, total, filteredTotal);
-                });
-            });
-        });
-    });
 };
 
 module.exports.statsClickedSubscribersByColumn = (campaign, linkId, request, column, limit, callback) => {
@@ -233,72 +91,12 @@ module.exports.statsClickedSubscribersByColumn = (campaign, linkId, request, col
 };
 
 module.exports.filterStatusSubscribers = (campaign, status, request, columns, callback) => {
-    db.getConnection((err, connection) => {
+    let queryData = {
-        if (err) {
+        where: 'campaign__' + campaign.id + '.list=? AND campaign__' + campaign.id + '.segment=? AND campaign__' + campaign.id + '.status=?',
-            return callback(err);
+        values: [campaign.list, campaign.segment && campaign.segment.id || 0, status]
-        }
+    };
-
-        status = Number(status) || 0;
-        let query = 'SELECT COUNT(`subscription__' + campaign.list + '`.`id`) AS total FROM `subscription__' + campaign.list + '` JOIN `campaign__' + campaign.id + '` ON `campaign__' + campaign.id + '`.`list`=? AND `campaign__' + campaign.id + '`.`segment`=? AND `campaign__' + campaign.id + '`.`subscription`=`subscription__' + campaign.list + '`.`id` WHERE `campaign__' + campaign.id + '`.`status`=?';
-        let values = [campaign.list, campaign.segment && campaign.segment.id || 0, status];
-
-        connection.query(query, values, (err, total) => {
-            if (err) {
-
-                connection.release();
-                return callback(err);
-            }
-            total = total && total[0] && total[0].total || 0;
-
-            let ordering = [];
-
-            if (request.order && request.order.length) {
-
-                request.order.forEach(order => {
-                    let orderField = columns[Number(order.column)];
-                    let orderDirection = (order.dir || '').toString().toLowerCase() === 'desc' ? 'DESC' : 'ASC';
-                    if (orderField) {
-                        ordering.push('`' + orderField + '` ' + orderDirection);
-                    }
-                });
-            }
-
-            if (!ordering.length) {
-                ordering.push('`email` ASC');
-            }
-
-            let args = [Number(request.length) || 50, Number(request.start) || 0];
-
-            if (request.search && request.search.value) {
-                query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + campaign.list + '`  JOIN `campaign__' + campaign.id + '` ON `campaign__' + campaign.id + '`.`list`=? AND `campaign__' + campaign.id + '`.`segment`=? AND `campaign__' + campaign.id + '`.`subscription`=`subscription__' + campaign.list + '`.`id` WHERE `campaign__' + campaign.id + '`.`status`=? AND (email LIKE ? OR first_name LIKE ? OR last_name LIKE ?) ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-
-                let searchVal = '%' + request.search.value.replace(/\\/g, '\\\\').replace(/([%_])/g, '\\$1') + '%';
-                args = values.concat([searchVal, searchVal, searchVal]).concat(args);
-            } else {
-                query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + campaign.list + '` JOIN `campaign__' + campaign.id + '` ON `campaign__' + campaign.id + '`.`list`=? AND `campaign__' + campaign.id + '`.`segment`=? AND `campaign__' + campaign.id + '`.`subscription`=`subscription__' + campaign.list + '`.`id` WHERE `campaign__' + campaign.id + '`.`status`=? ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-                args = values.concat(args);
-            }
-
-            connection.query(query, args, (err, rows) => {
-                if (err) {
-                    connection.release();
-                    return callback(err);
-                }
-                connection.query('SELECT FOUND_ROWS() AS total', (err, filteredTotal) => {
-                    connection.release();
-                    if (err) {
-                        return callback(err);
-                    }
-
-                    let subscriptions = rows.map(row => tools.convertKeys(row));
-
-                    filteredTotal = filteredTotal && filteredTotal[0] && filteredTotal[0].total || 0;
-                    return callback(null, subscriptions, total, filteredTotal);
-                });
-            });
-        });
-    });
 
+    tableHelpers.filter('subscription__' + campaign.list + ' JOIN campaign__' + campaign.id + ' ON campaign__' + campaign.id + '.subscription=subscription__' + campaign.list + '.id', ['*'], request, columns, ['email', 'first_name', 'last_name'], 'email ASC', queryData, callback);
 };
 
 module.exports.getByCid = (cid, callback) => {

lib/models/lists.js

@@ -5,29 +5,20 @@ let tools = require('../tools');
 let shortid = require('shortid');
 let segments = require('./segments');
 let _ = require('../translate')._;
+let tableHelpers = require('../table-helpers');
 
-let allowedKeys = ['description', 'default_form'];
+let allowedKeys = ['description', 'default_form', 'public_subscribe'];
 
 module.exports.list = (start, limit, callback) => {
-    db.getConnection((err, connection) => {
+    tableHelpers.list('lists', ['*'], 'name', null, start, limit, callback);
-        if (err) {
+};
-            return callback(err);
-        }
 
-        connection.query('SELECT SQL_CALC_FOUND_ROWS * FROM lists ORDER BY name LIMIT ? OFFSET ?', [limit, start], (err, rows) => {
+module.exports.filter = (request, parent, callback) => {
-            if (err) {
+    tableHelpers.filter('lists', ['*'], request, ['#', 'name', 'cid', 'subscribers', 'description'], ['name'], 'name ASC', null, callback);
-                connection.release();
+};
-                return callback(err);
+
-            }
+module.exports.filterQuicklist = (request, callback) => {
-            connection.query('SELECT FOUND_ROWS() AS total', (err, total) => {
+    tableHelpers.filter('lists', ['id', 'name', 'subscribers'], request, ['#', 'name', 'subscribers'], ['name'], 'name ASC', null, callback);
-                connection.release();
-                if (err) {
-                    return callback(err);
-                }
-                return callback(null, rows, total && total[0] && total[0].total);
-            });
-        });
-    });
 };
 
 module.exports.quicklist = callback => {
@@ -111,6 +102,8 @@ module.exports.get = (id, callback) => {
 module.exports.create = (list, callback) => {
 
     let data = tools.convertKeys(list);
+    data.publicSubscribe = data.publicSubscribe ? 1 : 0;
+
     let name = (data.name || '').toString().trim();
 
     if (!data) {
@@ -120,8 +113,8 @@ module.exports.create = (list, callback) => {
     let keys = ['name'];
     let values = [name];
 
-    Object.keys(list).forEach(key => {
+    Object.keys(data).forEach(key => {
-        let value = list[key].trim();
+        let value = data[key].toString().trim();
         key = tools.toDbKey(key);
         if (key === 'description') {
             value = tools.purifyHTML(value);
@@ -169,6 +162,7 @@ module.exports.update = (id, updates, callback) => {
     id = Number(id) || 0;
 
     let data = tools.convertKeys(updates);
+    data.publicSubscribe = data.publicSubscribe ? 1 : 0;
 
     let name = (data.name || '').toString().trim();
     let keys = ['name'];
@@ -182,8 +176,8 @@ module.exports.update = (id, updates, callback) => {
         return callback(new Error(_('List Name must be set')));
     }
 
-    Object.keys(updates).forEach(key => {
+    Object.keys(data).forEach(key => {
-        let value = updates[key].trim();
+        let value = data[key].toString().trim();
         key = tools.toDbKey(key);
         if (key === 'description') {
             value = tools.purifyHTML(value);

lib/models/report-templates.js

@@ -0,0 +1,161 @@
+'use strict';
+
+const db = require('../db');
+const tableHelpers = require('../table-helpers');
+const tools = require('../tools');
+const _ = require('../translate')._;
+
+const allowedKeys = ['name', 'description', 'mime_type', 'user_fields', 'js', 'hbs'];
+
+module.exports.list = (start, limit, callback) => {
+    tableHelpers.list('report_templates', ['*'], 'name', null, start, limit, callback);
+};
+
+module.exports.quicklist = callback => {
+    tableHelpers.quicklist('report_templates', ['id', 'name'], 'name', callback);
+};
+
+module.exports.filter = (request, callback) => {
+    tableHelpers.filter('report_templates', ['*'], request, ['#', 'name', 'description', 'created'], ['name'], 'created DESC', null, callback);
+};
+
+module.exports.get = (id, callback) => {
+    id = Number(id) || 0;
+
+    if (id < 1) {
+        return callback(new Error(_('Missing report template ID')));
+    }
+
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        connection.query('SELECT * FROM report_templates WHERE id=?', [id], (err, rows) => {
+            connection.release();
+            if (err) {
+                return callback(err);
+            }
+
+            if (!rows || !rows.length) {
+                return callback(null, false);
+            }
+
+            const template = tools.convertKeys(rows[0]);
+
+            const userFields = template.userFields.trim();
+            if (userFields !== '') {
+                try {
+                    template.userFieldsObject = JSON.parse(userFields);
+                } catch (err) {
+                    // This is to handle situation when for some reason we get corrupted JSON in the DB.
+                    template.userFieldsObject = {};
+                    template.userFields = '{}';
+                }
+            } else {
+                template.userFieldsObject = {};
+            }
+
+            return callback(null, template);
+        });
+    });
+};
+
+module.exports.createOrUpdate = (createMode, data, callback) => {
+    data = data || {};
+
+    const id = 'id' in data ? Number(data.id) : 0;
+
+    if (!createMode && id < 1) {
+        return callback(new Error(_('Missing report template ID')));
+    }
+
+    const template = tools.convertKeys(data);
+    const name = (template.name || '').toString().trim();
+
+    if (!name) {
+        return callback(new Error(_('Report template name must be set')));
+    }
+
+    const keys = ['name'];
+    const values = [name];
+
+    Object.keys(template).forEach(key => {
+        let value = typeof template[key] === 'number' ? template[key] : (template[key] || '').toString().trim();
+        key = tools.toDbKey(key);
+
+        if (key === 'description') {
+            value = tools.purifyHTML(value);
+        }
+
+        if (key === 'user_fields') {
+            value = value.trim();
+
+            if (value !== '') {
+                try {
+                    JSON.parse(value);
+                } catch (err) {
+                    return callback(err);
+                }
+            }
+        }
+
+        if (allowedKeys.indexOf(key) >= 0 && keys.indexOf(key) < 0) {
+            keys.push(key);
+            values.push(value);
+        }
+    });
+
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        let query;
+
+        if (createMode) {
+            query = 'INSERT INTO report_templates (`' + keys.join('`, `') + '`) VALUES (' + values.map(() => '?').join(',') + ')';
+        } else {
+            query = 'UPDATE report_templates SET ' + keys.map(key => '`' + key + '`=?').join(', ') + ' WHERE id=? LIMIT 1';
+            values.push(id);
+        }
+
+        connection.query(query, values, (err, result) => {
+            connection.release();
+            if (err) {
+                return callback(err);
+            }
+
+            if (createMode) {
+                return callback(null, result && result.insertId || false);
+            } else {
+                return callback(null, result && result.affectedRows || false);
+            }
+        });
+    });
+};
+
+module.exports.delete = (id, callback) => {
+    id = Number(id) || 0;
+
+    if (id < 1) {
+        return callback(new Error(_('Missing report template ID')));
+    }
+
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        connection.query('DELETE FROM report_templates WHERE id=? LIMIT 1', [id], (err, result) => {
+            connection.release();
+            if (err) {
+                return callback(err);
+            }
+
+            const affected = result && result.affectedRows || 0;
+            return callback(err, affected);
+        });
+    });
+};
+

lib/models/reports.js

@@ -0,0 +1,261 @@
+'use strict';
+
+const db = require('../db');
+const tableHelpers = require('../table-helpers');
+const fields = require('./fields');
+const reportTemplates = require('./report-templates');
+const tools = require('../tools');
+const _ = require('../translate')._;
+
+const allowedKeys = ['name', 'description', 'report_template', 'params'];
+
+const ReportState = {
+    SCHEDULED: 0,
+    PROCESSING: 1,
+    FINISHED: 2,
+    FAILED: 3
+};
+
+module.exports.ReportState = ReportState;
+
+module.exports.list = (start, limit, callback) => {
+    tableHelpers.list('reports', ['*'], 'name', null, start, limit, callback);
+};
+
+module.exports.listWithState = (state, start, limit, callback) => {
+    tableHelpers.list('reports', ['*'], 'name', { where: 'state=?', values: [state] }, start, limit, callback);
+};
+
+module.exports.filter = (request, callback) => {
+    tableHelpers.filter('reports JOIN report_templates ON reports.report_template = report_templates.id',
+        ['reports.id AS id', 'reports.name AS name', 'reports.description AS description', 'reports.state AS state', 'reports.report_template AS report_template', 'reports.params AS params', 'reports.last_run AS last_run', 'report_templates.name AS report_template_name', 'report_templates.mime_type AS mime_type' ],
+        request, ['#', 'name', 'report_templates.name', 'description', 'last_run'], ['name'], 'name ASC', null, callback);
+};
+
+module.exports.get = (id, callback) => {
+    id = Number(id) || 0;
+
+    if (id < 1) {
+        return callback(new Error(_('Missing report ID')));
+    }
+
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        connection.query('SELECT * FROM reports WHERE id=?', [id], (err, rows) => {
+            connection.release();
+            if (err) {
+                return callback(err);
+            }
+
+            if (!rows || !rows.length) {
+                return callback(null, false);
+            }
+
+            const template = tools.convertKeys(rows[0]);
+
+            const params = template.params.trim();
+            if (params !== '') {
+                try {
+                    template.paramsObject = JSON.parse(params);
+                } catch (err) {
+                    return callback(err);
+                }
+            } else {
+                template.params = {};
+            }
+
+            return callback(null, template);
+        });
+    });
+};
+
+// This method is not supposed to be used for unsanitized inputs. It does not do any checks.
+module.exports.updateFields = (id, fieldValueMap, callback) => {
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        const clauses = [];
+        const values = [];
+        for (let key of Object.keys(fieldValueMap)) {
+            clauses.push(tools.toDbKey(key) + '=?');
+            values.push(fieldValueMap[key]);
+        }
+
+        values.push(id);
+
+        const query = 'UPDATE reports SET ' + clauses.join(', ') + ' WHERE id=? LIMIT 1';
+        connection.query(query, values, (err, result) => {
+            connection.release();
+            if (err) {
+                return callback(err);
+            }
+
+            return callback(null, result && result.affectedRows || false);
+        });
+    });
+};
+
+module.exports.createOrUpdate = (createMode, report, callback) => {
+    report = report || {};
+
+    const id = 'id' in report ? Number(report.id) : 0;
+
+    if (!createMode && id < 1) {
+        return callback(new Error(_('Missing report ID')));
+    }
+
+    const name = (report.name || '').toString().trim();
+
+    if (!name) {
+        return callback(new Error(_('Report name must be set')));
+    }
+
+    const reportTemplateId = Number(report.reportTemplate);
+    reportTemplates.get(reportTemplateId, (err, reportTemplate) => {
+        if (err) {
+            return callback(err);
+        }
+
+        const params = report.paramsObject;
+        for (const spec of reportTemplate.userFieldsObject) {
+            if (params[spec.id].length < spec.minOccurences) {
+                return callback(new Error(_('At least ' + spec.minOccurences + ' rows in "' + spec.name + '" have to be selected.')));
+            }
+
+            if (params[spec.id].length > spec.maxOccurences) {
+                return callback(new Error(_('At most ' + spec.minOccurences + ' rows in "' + spec.name + '" can be selected.')));
+            }
+        }
+
+        const keys = ['name', 'params'];
+        const values = [name, JSON.stringify(params)];
+
+
+        Object.keys(report).forEach(key => {
+            let value = typeof report[key] === 'number' ? report[key] : (report[key] || '').toString().trim();
+            key = tools.toDbKey(key);
+
+            if (key === 'description') {
+                value = tools.purifyHTML(value);
+            }
+
+            if (allowedKeys.indexOf(key) >= 0 && keys.indexOf(key) < 0) {
+                keys.push(key);
+                values.push(value);
+            }
+        });
+
+        db.getConnection((err, connection) => {
+            if (err) {
+                return callback(err);
+            }
+
+            let query;
+
+            if (createMode) {
+                query = 'INSERT INTO reports (`' + keys.join('`, `') + '`) VALUES (' + values.map(() => '?').join(',') + ')';
+            } else {
+                query = 'UPDATE reports SET ' + keys.map(key => '`' + key + '`=?').join(', ') + ' WHERE id=? LIMIT 1';
+                values.push(id);
+            }
+
+            connection.query(query, values, (err, result) => {
+                connection.release();
+                if (err) {
+                    return callback(err);
+                }
+
+                if (createMode) {
+                    return callback(null, result && result.insertId || false);
+                } else {
+                    return callback(null, result && result.affectedRows || false);
+                }
+            });
+        });
+    });
+};
+
+module.exports.delete = (id, callback) => {
+    id = Number(id) || 0;
+
+    if (id < 1) {
+        return callback(new Error(_('Missing report ID')));
+    }
+
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        connection.query('DELETE FROM reports WHERE id=? LIMIT 1', [id], (err, result) => {
+            connection.release();
+            if (err) {
+                return callback(err);
+            }
+
+            const affected = result && result.affectedRows || 0;
+            return callback(err, affected);
+        });
+    });
+};
+
+const campaignFieldsMapping = {
+    tracker_count: 'tracker.count',
+    country: 'tracker.country',
+    device_type: 'tracker.device_type',
+    status: 'campaign.status',
+    first_name: 'subscribers.first_name',
+    last_name: 'subscribers.last_name',
+    email: 'subscribers.email'
+};
+
+module.exports.getCampaignResults = (campaign, select, clause, callback) => {
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        fields.list(campaign.list, (err, fieldList) => {
+            if (err) {
+                return callback(err);
+            }
+
+            const fieldsMapping = fieldList.reduce((map, field) => {
+                map[customFieldName(field.key)] = 'subscribers.' + field.column;
+                return map;
+            }, Object.assign({}, campaignFieldsMapping));
+
+            let selFields = [];
+            for (let idx = 0; idx < select.length; idx++) {
+                const item = select[idx];
+                if (item in fieldsMapping) {
+                    selFields.push(fieldsMapping[item] + ' AS ' + item);
+                } else if (item === '*') {
+                    selFields = selFields.concat(Object.keys(fieldsMapping).map(item => fieldsMapping[item] + ' AS ' + item));
+                } else {
+                    selFields.push(item);
+                }
+            }
+
+            const query = 'SELECT ' + selFields.join(', ') + ' FROM `subscription__' + campaign.list + '` subscribers INNER JOIN `campaign__' + campaign.id + '` campaign on subscribers.id=campaign.subscription LEFT JOIN `campaign_tracker__' + campaign.id + '` tracker on subscribers.id=tracker.subscriber ' + clause;
+
+            connection.query(query, (err, results) => {
+                if (err) {
+                    connection.release();
+                    return callback(err);
+                }
+
+                return callback(null, results);
+            });
+        });
+    });
+};
+
+function customFieldName(id) {
+    return id.replace(/MERGE_/, 'CUSTOM_').toLowerCase();
+}

lib/models/subscriptions.js

@@ -13,6 +13,7 @@ let urllib = require('url');
 let log = require('npmlog');
 let _ = require('../translate')._;
 let util = require('util');
+let tableHelpers = require('../table-helpers');
 
 module.exports.list = (listId, start, limit, callback) => {
     listId = Number(listId) || 0;
@@ -20,26 +21,11 @@ module.exports.list = (listId, start, limit, callback) => {
         return callback(new Error('Missing List ID'));
     }
 
-    db.getConnection((err, connection) => {
+    tableHelpers.list('subscription__' + listId, ['*'], 'email', null, start, limit, (err, rows, total) => {
-        if (err) {
+        if (!err) {
-            return callback(err);
+            rows = rows.map(row => tools.convertKeys(row));
         }
-
+        return callback(err, rows, total);
-        connection.query('SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + listId + '` ORDER BY email LIMIT ? OFFSET ?', [limit, start], (err, rows) => {
-            if (err) {
-                connection.release();
-                return callback(err);
-            }
-            connection.query('SELECT FOUND_ROWS() AS total', (err, total) => {
-                connection.release();
-                if (err) {
-                    return callback(err);
-                }
-
-                let subscriptions = rows.map(row => tools.convertKeys(row));
-                return callback(null, subscriptions, total && total[0] && total[0].total);
-            });
-        });
     });
 };
 
@@ -80,7 +66,6 @@ module.exports.listTestUsers = (listId, callback) => {
     });
 };
 
-
 module.exports.filter = (listId, request, columns, segmentId, callback) => {
     listId = Number(listId) || 0;
     segmentId = Number(segmentId) || 0;
@@ -89,88 +74,16 @@ module.exports.filter = (listId, request, columns, segmentId, callback) => {
         return callback(new Error(_('Missing List ID')));
     }
 
-    let processQuery = queryData => {
-
-        db.getConnection((err, connection) => {
-            if (err) {
-                return callback(err);
-            }
-
-            let query = 'SELECT COUNT(id) AS total FROM `subscription__' + listId + '`';
-            let values = [];
-
-            if (queryData.where) {
-                query += ' WHERE ' + queryData.where;
-                values = values.concat(queryData.values || []);
-            }
-
-            connection.query(query, values, (err, total) => {
-                if (err) {
-                    connection.release();
-                    return callback(err);
-                }
-                total = total && total[0] && total[0].total || 0;
-
-                let ordering = [];
-
-                if (request.order && request.order.length) {
-
-                    request.order.forEach(order => {
-                        let orderField = columns[Number(order.column)];
-                        let orderDirection = (order.dir || '').toString().toLowerCase() === 'desc' ? 'DESC' : 'ASC';
-                        if (orderField) {
-                            ordering.push('`' + orderField + '` ' + orderDirection);
-                        }
-                    });
-                }
-
-                if (!ordering.length) {
-                    ordering.push('`email` ASC');
-                }
-
-                let args = [Number(request.length) || 50, Number(request.start) || 0];
-                let query;
-
-                if (request.search && request.search.value) {
-                    query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + listId + '` WHERE email LIKE ? OR first_name LIKE ? OR last_name LIKE ? ' + (queryData.where ? ' AND (' + queryData.where + ')' : '') + ' ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-
-                    let searchVal = '%' + request.search.value.replace(/\\/g, '\\\\').replace(/([%_])/g, '\\$1') + '%';
-                    args = [searchVal, searchVal, searchVal].concat(queryData.values || []).concat(args);
-                } else {
-                    query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + listId + '` WHERE 1 ' + (queryData.where ? ' AND (' + queryData.where + ')' : '') + ' ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-                    args = [].concat(queryData.values || []).concat(args);
-                }
-
-                connection.query(query, args, (err, rows) => {
-                    if (err) {
-                        connection.release();
-                        return callback(err);
-                    }
-                    connection.query('SELECT FOUND_ROWS() AS total', (err, filteredTotal) => {
-                        connection.release();
-                        if (err) {
-                            return callback(err);
-                        }
-
-                        let subscriptions = rows.map(row => tools.convertKeys(row));
-
-                        filteredTotal = filteredTotal && filteredTotal[0] && filteredTotal[0].total || 0;
-                        return callback(null, subscriptions, total, filteredTotal);
-                    });
-                });
-            });
-        });
-    };
-
     if (segmentId) {
         segments.getQuery(segmentId, false, (err, queryData) => {
             if (err) {
                 return callback(err);
             }
-            processQuery(queryData);
+
+            tableHelpers.filter('subscription__' + listId, ['*'], request, columns, ['email', 'first_name', 'last_name'], 'email ASC', queryData, callback);
         });
     } else {
-        processQuery(false);
+        tableHelpers.filter('subscription__' + listId, ['*'], request, columns, ['email', 'first_name', 'last_name'], 'email ASC', null, callback);
     }
 
 };

lib/models/templates.js

@@ -3,45 +3,20 @@
 let db = require('../db');
 let tools = require('../tools');
 let _ = require('../translate')._;
+let tableHelpers = require('../table-helpers');
 
 let allowedKeys = ['description', 'editor_name', 'editor_data', 'html', 'text'];
 
 module.exports.list = (start, limit, callback) => {
-    db.getConnection((err, connection) => {
+    tableHelpers.list('templates', ['*'], 'name', null, start, limit, callback);
-        if (err) {
+};
-            return callback(err);
-        }
 
-        connection.query('SELECT SQL_CALC_FOUND_ROWS * FROM templates ORDER BY name LIMIT ? OFFSET ?', [limit, start], (err, rows) => {
+module.exports.filter = (request, parent, callback) => {
-            if (err) {
+    tableHelpers.filter('templates', ['*'], request, ['#', 'name', 'description'], ['name'], 'name ASC', null, callback);
-                connection.release();
-                return callback(err);
-            }
-            connection.query('SELECT FOUND_ROWS() AS total', (err, total) => {
-                connection.release();
-                if (err) {
-                    return callback(err);
-                }
-                return callback(null, rows, total && total[0] && total[0].total);
-            });
-        });
-    });
 };
 
 module.exports.quicklist = callback => {
-    db.getConnection((err, connection) => {
+    tableHelpers.quicklist('templates', ['id', 'name'], 'name', callback);
-        if (err) {
-            return callback(err);
-        }
-
-        connection.query('SELECT id, name FROM templates ORDER BY name LIMIT 1000', (err, rows) => {
-            connection.release();
-            if (err) {
-                return callback(err);
-            }
-            return callback(null, (rows || []).map(tools.convertKeys));
-        });
-    });
 };
 
 module.exports.get = (id, callback) => {

lib/models/triggers.js

@@ -5,6 +5,7 @@ let db = require('../db');
 let lists = require('./lists');
 let util = require('util');
 let _ = require('../translate')._;
+let tableHelpers = require('../table-helpers');
 
 module.exports.defaultColumns = [{
     column: 'created',
@@ -339,70 +340,12 @@ module.exports.delete = (id, callback) => {
 };
 
 module.exports.filterSubscribers = (trigger, request, columns, callback) => {
-    db.getConnection((err, connection) => {
+    let queryData = {
-        if (err) {
+        where: 'trigger__' + trigger.id + '.list=?',
-            return callback(err);
+        values: [trigger.list]
-        }
+    };
-
-        let query = 'SELECT COUNT(`subscription__' + trigger.list + '`.`id`) AS total FROM `subscription__' + trigger.list + '` JOIN `trigger__' + trigger.id + '` ON `trigger__' + trigger.id + '`.`list`=? AND `trigger__' + trigger.id + '`.`subscription`=`subscription__' + trigger.list + '`.`id`';
-        let values = [trigger.list];
-
-        connection.query(query, values, (err, total) => {
-            if (err) {
-                connection.release();
-                return callback(err);
-            }
-            total = total && total[0] && total[0].total || 0;
-
-            let ordering = [];
-
-            if (request.order && request.order.length) {
-
-                request.order.forEach(order => {
-                    let orderField = columns[Number(order.column)];
-                    let orderDirection = (order.dir || '').toString().toLowerCase() === 'desc' ? 'DESC' : 'ASC';
-                    if (orderField) {
-                        ordering.push('`' + orderField + '` ' + orderDirection);
-                    }
-                });
-            }
-
-            if (!ordering.length) {
-                ordering.push('`email` ASC');
-            }
-
-            let args = [Number(request.length) || 50, Number(request.start) || 0];
-
-            if (request.search && request.search.value) {
-                query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + trigger.list + '` JOIN `trigger__' + trigger.id + '` ON `trigger__' + trigger.id + '`.`list`=? AND `trigger__' + trigger.id + '`.`subscription`=`subscription__' + trigger.list + '`.`id AND (email LIKE ? OR first_name LIKE ? OR last_name LIKE ?) ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-
-                let searchVal = '%' + request.search.value.replace(/\\/g, '\\\\').replace(/([%_])/g, '\\$1') + '%';
-                args = values.concat([searchVal, searchVal, searchVal]).concat(args);
-            } else {
-                query = 'SELECT SQL_CALC_FOUND_ROWS * FROM `subscription__' + trigger.list + '` JOIN `trigger__' + trigger.id + '` ON `trigger__' + trigger.id + '`.`list`=? AND `trigger__' + trigger.id + '`.`subscription`=`subscription__' + trigger.list + '`.`id` ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
-                args = values.concat(args);
-            }
-
-            connection.query(query, args, (err, rows) => {
-                if (err) {
-                    connection.release();
-                    return callback(err);
-                }
-                connection.query('SELECT FOUND_ROWS() AS total', (err, filteredTotal) => {
-                    connection.release();
-                    if (err) {
-                        return callback(err);
-                    }
-
-                    let subscriptions = rows.map(row => tools.convertKeys(row));
-
-                    filteredTotal = filteredTotal && filteredTotal[0] && filteredTotal[0].total || 0;
-                    return callback(null, subscriptions, total, filteredTotal);
-                });
-            });
-        });
-    });
 
+    tableHelpers.filter('subscription__' + trigger.list + ' JOIN trigger__' + trigger.id + ' ON trigger__' + trigger.id + '.subscription=subscription__' + trigger.list + '.id', ['*'], request, columns, ['email', 'first_name', 'last_name'], 'email ASC', queryData, callback);
 };
 
 function createTriggerTable(id, callback) {

lib/privilege-helpers.js

@@ -0,0 +1,77 @@
+'use strict';
+
+const log = require('npmlog');
+const config = require('config');
+
+const fs = require('fs');
+
+const tryRequire = require('try-require');
+const posix = tryRequire('posix');
+
+function _getConfigUidGid(prefix, defaultUid, defaultGid) {
+    let uid = defaultUid;
+    let gid = defaultGid;
+
+    if (posix) {
+        try {
+            if (config.user) {
+                uid = posix.getpwnam(config[prefix + 'user']).uid;
+            }
+        } catch (err) {
+            log.info('PrivilegeHelpers', 'Failed to resolve user id "%s"', config[prefix + 'user']);
+        }
+
+        try {
+            if (config.user) {
+                gid = posix.getpwnam(config[prefix + 'group']).gid;
+            }
+        } catch (err) {
+            log.info('PrivilegeHelpers', 'Failed to resolve group id "%s"', config[prefix + 'group']);
+        }
+    } else {
+        log.info('PrivilegeHelpers', 'Posix module not installed. Cannot resolve uid/gid');
+    }
+
+    return { uid, gid };
+}
+
+function getConfigUidGid() {
+    return _getConfigUidGid('', process.getuid(), process.getgid());
+}
+
+function getConfigROUidGid() {
+    let rwIds = getConfigUidGid();
+    return _getConfigUidGid('ro', rwIds.uid, rwIds.gid);
+}
+
+function ensureMailtrainOwner(file, callback) {
+    const ids = getConfigUidGid();
+    fs.chown(file, ids.uid, ids.gid, callback);
+}
+
+function dropRootPrivileges() {
+    if (config.group) {
+        try {
+            process.setgid(config.group);
+            log.info('PrivilegeHelpers', 'Changed group to "%s" (%s)', config.group, process.getgid());
+        } catch (E) {
+            log.info('PrivilegeHelpers', 'Failed to change group to "%s" (%s)', config.group, E.message);
+        }
+    }
+
+    if (config.user) {
+        try {
+            process.setuid(config.user);
+            log.info('PrivilegeHelpers', 'Changed user to "%s" (%s)', config.user, process.getuid());
+        } catch (E) {
+            log.info('PrivilegeHelpers', 'Failed to change user to "%s" (%s)', config.user, E.message);
+        }
+    }
+}
+
+module.exports = {
+    dropRootPrivileges,
+    ensureMailtrainOwner,
+    getConfigUidGid,
+    getConfigROUidGid
+};

lib/report-processor.js

@@ -0,0 +1,147 @@
+'use strict';
+
+const log = require('npmlog');
+const reports = require('./models/reports');
+const executor = require('./executor');
+
+let runningWorkersCount = 0;
+let maxWorkersCount = 1;
+
+let workers = {};
+
+function startWorker(report) {
+
+    function onStarted(tid) {
+        log.info('ReportProcessor', 'Worker process for "%s" started with tid %s. Current worker count is %s.', report.name, tid, runningWorkersCount);
+        workers[report.id] = tid;
+    }
+
+    function onFinished(code, signal) {
+        runningWorkersCount--;
+        log.info('ReportProcessor', 'Worker process for "%s" (tid %s) exited with code %s signal %s. Current worker count is %s.', report.name, workers[report.id], code, signal, runningWorkersCount);
+        delete workers[report.id];
+
+        const fields = {};
+        if (code === 0) {
+            fields.state = reports.ReportState.FINISHED;
+            fields.lastRun = new Date();
+        } else {
+            fields.state = reports.ReportState.FAILED;
+        }
+
+        reports.updateFields(report.id, fields, err => {
+            if (err) {
+                log.error('ReportProcessor', err);
+            }
+
+            setImmediate(startWorkers);
+        });
+    }
+
+    function onFailed(msg) {
+        runningWorkersCount--;
+        log.error('ReportProcessor', 'Executing worker process for "%s" (tid %s) failed with message "%s". Current worker count is %s.', report.name, workers[report.id], msg, runningWorkersCount);
+        delete workers[report.id];
+
+        const fields = {
+            state: reports.ReportState.FAILED
+        };
+
+        reports.updateFields(report.id, fields, err => {
+            if (err) {
+                log.error('ReportProcessor', err);
+            }
+
+            setImmediate(startWorkers);
+        });
+    }
+
+    const reportData = {
+        id: report.id,
+        name: report.name
+    };
+
+    runningWorkersCount++;
+    executor.start('report-processor-worker', reportData, onStarted, onFinished, onFailed);
+}
+
+function startWorkers() {
+    reports.listWithState(reports.ReportState.SCHEDULED, 0, maxWorkersCount - runningWorkersCount, (err, reportList) => {
+        if (err) {
+            log.error('ReportProcessor', err);
+            return;
+        }
+
+        for (let report of reportList) {
+            reports.updateFields(report.id, { state: reports.ReportState.PROCESSING }, err => {
+                if (err) {
+                    log.error('ReportProcessor', err);
+                    return;
+                }
+
+                startWorker(report);
+            });
+        }
+    });
+}
+
+module.exports.start = (reportId, callback) => {
+    if (!workers[reportId]) {
+        log.info('ReportProcessor', 'Scheduling report id: %s', reportId);
+        reports.updateFields(reportId, { state: reports.ReportState.SCHEDULED, lastRun: null}, err => {
+            if (err) {
+                return callback(err);
+            }
+
+            if (runningWorkersCount < maxWorkersCount) {
+                log.info('ReportProcessor', 'Starting worker because runningWorkersCount=%s maxWorkersCount=%s', runningWorkersCount, maxWorkersCount);
+
+                startWorkers();
+            } else {
+                log.info('ReportProcessor', 'Not starting worker because runningWorkersCount=%s maxWorkersCount=%s', runningWorkersCount, maxWorkersCount);
+            }
+
+            callback(null);
+        });
+    } else {
+        log.info('ReportProcessor', 'Worker for report id: %s is already running.', reportId);
+    }
+};
+
+module.exports.stop = (reportId, callback) => {
+    const tid = workers[reportId];
+    if (tid) {
+        log.info('ReportProcessor', 'Killing worker for report id: %s', reportId);
+        executor.stop(tid);
+        reports.updateFields(reportId, { state: reports.ReportState.FAILED}, callback);
+    } else {
+        log.info('ReportProcessor', 'No running worker found for report id: %s', reportId);
+    }
+};
+
+module.exports.init = callback => {
+    reports.listWithState(reports.ReportState.PROCESSING, 0, 0, (err, reportList) => {
+        if (err) {
+            log.error('ReportProcessor', err);
+        }
+
+        function scheduleReport() {
+            if (reportList.length > 0) {
+                const report = reportList.shift();
+
+                reports.updateFields(report.id, { state: reports.ReportState.SCHEDULED}, err => {
+                    if (err) {
+                        log.error('ReportProcessor', err);
+                    }
+
+                    scheduleReport();
+                });
+            }
+
+            startWorkers();
+            return callback();
+        }
+
+        scheduleReport();
+    });
+};

lib/table-helpers.js

@@ -0,0 +1,134 @@
+'use strict';
+
+const db = require('./db');
+const tools = require('./tools');
+
+module.exports.list = (source, fields, orderBy, queryData, start, limit, callback) => {
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        let limitQuery = '';
+        let limitValues = [];
+        if (limit) {
+            limitQuery = ' LIMIT ?';
+            limitValues.push(limit);
+
+            if (start) {
+                limitQuery += ' OFFSET ?';
+                limitValues.push(start);
+            }
+        }
+
+        let whereClause = '';
+        let whereValues = [];
+
+        if (queryData) {
+            whereClause = ' WHERE ' + queryData.where;
+            whereValues = queryData.values;
+        }
+
+        connection.query('SELECT SQL_CALC_FOUND_ROWS ' + fields.join(', ') + ' FROM ' + source + whereClause + ' ORDER BY ' + orderBy + ' DESC' + limitQuery, whereValues.concat(limitValues), (err, rows) => {
+            if (err) {
+                connection.release();
+                return callback(err);
+            }
+            connection.query('SELECT FOUND_ROWS() AS total', (err, total) => {
+                connection.release();
+                if (err) {
+                    return callback(err);
+                }
+                return callback(null, rows, total && total[0] && total[0].total);
+            });
+        });
+    });
+};
+
+module.exports.quicklist = (source, fields, orderBy, callback) => {
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        connection.query('SELECT ' + fields.join(', ') + ' FROM ' + source + ' ORDER BY ' + orderBy + ' LIMIT 1000', (err, rows) => {
+            connection.release();
+            if (err) {
+                return callback(err);
+            }
+            return callback(null, (rows || []).map(tools.convertKeys));
+        });
+    });
+};
+
+module.exports.filter = (source, fields, request, columns, searchFields, defaultOrdering, queryData, callback) => {
+    db.getConnection((err, connection) => {
+        if (err) {
+            return callback(err);
+        }
+
+        let query = 'SELECT COUNT(*) AS total FROM ' + source;
+        let values = [];
+
+        if (queryData) {
+            query += ' WHERE ' + queryData.where;
+            values = values.concat(queryData.values || []);
+        }
+
+        connection.query(query, values, (err, total) => {
+            if (err) {
+                connection.release();
+                return callback(err);
+            }
+            total = total && total[0] && total[0].total || 0;
+
+            let ordering = [];
+
+            if (request.order && request.order.length) {
+
+                request.order.forEach(order => {
+                    let orderField = columns[Number(order.column)];
+                    let orderDirection = (order.dir || '').toString().toLowerCase() === 'desc' ? 'DESC' : 'ASC';
+                    if (orderField) {
+                        ordering.push(orderField + ' ' + orderDirection);
+                    }
+                });
+            }
+
+            if (!ordering.length) {
+                ordering.push(defaultOrdering);
+            }
+
+            let searchWhere = '';
+            let searchArgs = [];
+
+            if (request.search && request.search.value) {
+                let searchVal = '%' + request.search.value.replace(/\\/g, '\\\\').replace(/([%_])/g, '\\$1') + '%';
+
+                searchWhere = searchFields.map(field => field + ' LIKE ?').join(' OR ');
+                searchArgs = searchFields.map(() => searchVal);
+            }
+
+            let query = 'SELECT SQL_CALC_FOUND_ROWS ' + fields.join(', ') + ' FROM ' + source +' WHERE ' + (searchWhere ? '(' + searchWhere + ')': '1') + (queryData ? ' AND (' + queryData.where + ')' : '') + ' ORDER BY ' + ordering.join(', ') + ' LIMIT ? OFFSET ?';
+            let args = searchArgs.concat(queryData ? queryData.values : []).concat([Number(request.length) || 50, Number(request.start) || 0]);
+
+            connection.query(query, args, (err, rows) => {
+                if (err) {
+                    connection.release();
+                    return callback(err);
+                }
+                connection.query('SELECT FOUND_ROWS() AS total', (err, filteredTotal) => {
+                    connection.release();
+                    if (err) {
+                        return callback(err);
+                    }
+
+                    rows = rows.map(row => tools.convertKeys(row));
+
+                    filteredTotal = filteredTotal && filteredTotal[0] && filteredTotal[0].total || 0;
+                    return callback(null, rows, total, filteredTotal);
+                });
+            });
+        });
+    });
+};

lib/tools.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const config = require('config');
 let fs = require('fs');
 let path = require('path');
 let db = require('./db');
@@ -130,6 +131,14 @@ function updateMenu(res) {
         url: '/triggers',
         key: 'triggers'
     });
+
+    if (config.reports && config.reports.enabled === true) {
+        res.locals.menu.push({
+            title: _('Reports'),
+            url: '/reports',
+            key: 'reports'
+        });
+    }
 }
 
 function validateEmail(address, checkBlocked, callback) {
@@ -296,3 +305,4 @@ function mergeTemplateIntoLayout(template, layout, callback) {
         return done(template, layout);
     }
 }
+

meta.json

@@ -1,3 +1,3 @@
 {
-    "schemaVersion": 25
+    "schemaVersion": 27
 }

package.json

@@ -8,12 +8,17 @@
     "test": "grunt",
     "start": "node index.js",
     "sqlinit": "node setup/sql/init.js",
-    "sqldump": "node setup/sql/dump.js | sed -e '/^\\/\\*.*\\*\\/;$/d' -e 's/.[0-9]\\{4\\}-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]./NOW()/g' > setup/sql/mailtrain.sql",
+    "sqldump": "node setup/sql/dump.js | sed -e '/^\\/\\*.*\\*\\/;$/d' -e 's/.[0-9]\\{4\\}-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]./NOW()/g' > setup/sql/mailtrain${DUMP_NAME_SUFFIX}.sql",
     "sqldrop": "node setup/sql/drop.js",
     "sqlgen": "npm run sqldrop && DB_FROM_START=Y npm run sqlinit && npm run sqldump",
     "langs:hbs": "jsxgettext -L handlebars -k translate -o langs/hbs.pot views/layout.hbs views/index.hbs",
     "langs:js": "jsxgettext -o languages/js.pot routes/index.js",
-    "langs": "npm run langs:hbs && npm run langs:js"
+    "langs": "npm run langs:hbs && npm run langs:js",
+    "sqldumptest": "NODE_ENV=test DUMP_NAME_SUFFIX=-test npm run sqldump",
+    "sqlresettest": "NODE_ENV=test npm run sqldrop && NODE_ENV=test npm run sqlinit",
+    "starttest": "NODE_ENV=test node index.js",
+    "_e2e": "NODE_ENV=test mocha test/e2e/index.js",
+    "e2e": "npm run sqlresettest && npm run _e2e"
   },
   "repository": {
     "type": "git",
@@ -26,12 +31,20 @@
     "node": ">=5.0.0"
   },
   "devDependencies": {
+    "babel-eslint": "^7.2.3",
+    "chai": "^3.5.0",
     "eslint-config-nodemailer": "^1.0.0",
     "grunt": "^1.0.1",
     "grunt-cli": "^1.2.0",
     "grunt-contrib-nodeunit": "^1.0.0",
     "grunt-eslint": "^19.0.0",
-    "jsxgettext-andris": "^0.9.0-patch.1"
+    "jsxgettext-andris": "^0.9.0-patch.1",
+    "mocha": "^3.3.0",
+    "phantomjs": "^2.1.7",
+    "selenium-webdriver": "^3.4.0"
+  },
+  "optionalDependencies": {
+    "posix": "^4.1.1"
   },
   "dependencies": {
     "async": "^2.3.0",
@@ -70,6 +83,7 @@
     "jsdom": "^9.12.0",
     "juice": "^4.0.2",
     "libmime": "^3.1.0",
+    "mailparser": "^2.0.5",
     "marked": "^0.3.6",
     "memory-cache": "^0.1.6",
     "mjml": "3.3.0",
@@ -97,6 +111,7 @@
     "slugify": "^1.1.0",
     "smtp-server": "^2.0.3",
     "striptags": "^3.0.1",
-    "toml": "^2.3.2"
+    "toml": "^2.3.2",
+    "try-require": "^1.2.1"
   }
 }

protected/reports/.gitignore

@@ -0,0 +1,3 @@
+*
+!.gitignore
+!README.md

protected/reports/README.md

@@ -0,0 +1 @@
+This directory serves for generated reports.

public/css/mailtrain.css

@@ -31,3 +31,19 @@ h2 .glyphicon {
 h3 .glyphicon {
     font-size: .8em;
 }
+
+tbody>tr.selected {
+    background-color: rgb(218, 231, 255);
+}
+
+.table-hover>tbody>tr.selected:hover {
+    background-color: rgb(205, 212, 226);
+}
+
+.row-actions .row-action {
+    padding-right: 15px;
+}
+
+.row-actions .row-action:last-child {
+    padding-right: 0px;
+}

public/javascript/editor.js

@@ -39,6 +39,12 @@ $('div[class*="code-editor-"]').each(function () {
         editor.getSession().setUseWorker(false);
     } else if ($(this).hasClass('code-editor-css')) {
         mode = 'css';
+    } else if ($(this).hasClass('code-editor-javascript')) {
+        mode = 'javascript';
+    } else if ($(this).hasClass('code-editor-json')) {
+        mode = 'json';
+    } else if ($(this).hasClass('code-editor-handlebars')) {
+        mode = 'handlebars';
     }
 
     editor.setTheme('ace/theme/chrome');

public/javascript/tables.js

@@ -4,190 +4,307 @@
 
 'use strict';
 
-$('.data-table').each(function () {
+(function() {
-    var rowSort = $(this).data('rowSort') || false;
+    function refreshTargets(data) {
-    var columns = false;
+        for (var target in data) {
+            var newContent = $(data[target]);
 
-    if (rowSort) {
+            $(target).replaceWith(newContent);
-        columns = rowSort.split(',').map(function (sort) {
+            installHandlers(newContent.parent());
-            return {
+        }
-                orderable: sort === '1'
+    }
-            };
+
+    function getAjaxUrl(self) {
+        var topicId = self.data('topicId');
+        var topicUrl = self.data('topicUrl');
+
+        return topicUrl + '/ajax/' + topicId;
+    }
+
+    function setupAjaxRefresh() {
+        var self = $(this);
+        var ajaxUrl = getAjaxUrl(self);
+
+        var interval = Number(self.data('interval')) || 60;
+
+        setTimeout(function () {
+            $.get(ajaxUrl, function(data) {
+                refreshTargets(data);
+            });
+
+        }, interval * 1000);
+    }
+
+    function setupAjaxAction() {
+        var self = $(this);
+        var ajaxUrl = getAjaxUrl(self);
+
+        var processing = false;
+
+        self.click(function () {
+            if (!processing) {
+                $.get(ajaxUrl, function (data) {
+                    refreshTargets(data);
+                });
+
+                processing = true;
+            }
+
+            return false;
         });
     }
 
-    $(this).DataTable({
+    function setupDatestring() {
-        scrollX: true,
+        var self = $(this);
-        order: [
+        self.html(moment(self.data('date')).fromNow());
-            [1, 'asc']
+    }
-        ],
+
-        columns: columns,
+
-        pageLength: 50
+    function getDataTableOptions(elem) {
+        var rowSort = $(elem).data('rowSort') || false;
+
+        var columns = false;
+
+        var sortColumn = $(elem).data('sortColumn') === undefined ? 1 : Number($(elem).data('sortColumn'));
+        var sortOrder = ($(elem).data('sortOrder') || 'asc').toString().trim().toLowerCase();
+
+        var paging = $(elem).data('paging') === false ? false : true;
+
+        // allow only asc and desc
+        if (sortOrder !== 'desc') {
+            sortOrder = 'asc';
+        }
+
+        var columnsCount = 0;
+        var columnsSort = []
+
+        if (rowSort) {
+            columns = rowSort.split(',').map(function (sort) {
+                return {
+                    orderable: sort === '1'
+                };
+            });
+        }
+
+        var opts = {
+            scrollX: true,
+            order: [
+                [sortColumn, sortOrder]
+            ],
+            columns: columns,
+            paging: paging,
+            info: paging, /* This controls the "Showing 1 to 16 of 16 entries" */
+            pageLength: 50
+        };
+
+        if ($(elem).hasClass('data-table-selectable') || $(elem).hasClass('data-table-multiselectable')) {
+            var isMulti = $(elem).hasClass('data-table-multiselectable');
+
+            var dataElem = $(elem).siblings("input").first();
+
+            opts.rowCallback = function( row, data ) {
+                var selected = dataElem.val() == '' ? [] : dataElem.val().split(',').map(function(item) { return Number(item); });
+
+                if (!isMulti && selected.length > 0) {
+                    selected = [selected[0]];
+                }
+
+                if ($.inArray(data.DT_RowId, selected) !== -1) {
+                    $(row).addClass('selected');
+                }
+            }
+
+            $(elem).on('click', 'tbody tr', function () {
+                var id = this.id;
+                var selected = dataElem.val() == '' ? [] : dataElem.val().split(',');
+
+                var index = $.inArray(id, selected);
+
+                if (isMulti) {
+                    if ( index === -1 ) {
+                        selected.push(id);
+                    } else {
+                        selected.splice(index, 1);
+                    }
+
+                    $(this).toggleClass('selected');
+                } else {
+                    for (var selIdx=0; selIdx < selected.length; selIdx++) {
+                        if (selected[selIdx] != id) {
+                            $('#' + selected[selIdx], elem).removeClass('selected');
+                        }
+                    }
+
+                    $('#' + id, elem).addClass('selected');
+
+                    selected = [id];
+                }
+
+                dataElem.val(selected.join(','));
+            } );
+        }
+
+        return opts;
+    }
+
+
+    function installHandlers(elem) {
+        $('.ajax-refresh', elem).each(setupAjaxRefresh);
+        $('.ajax-action', elem).each(setupAjaxAction);
+        $('.datestring', elem).each(setupDatestring);
+    }
+
+    installHandlers($(document));
+
+    $('.data-table').each(function () {
+        var opts = getDataTableOptions(this);
+        $(this).DataTable(opts);
     });
-});
 
-$('.data-table-ajax').each(function () {
+    $('.data-table-ajax').each(function () {
-    var rowSort = $(this).data('rowSort') || false;
+        var topicUrl = $(this).data('topicUrl') || '/lists';
-    var columns = false;
+        var topicArgs = $(this).data('topicArgs') || false;
+        var topicId = $(this).data('topicId') || '';
 
-    var topicUrl = $(this).data('topicUrl') || '/lists';
+        var ajaxUrl = topicUrl + '/ajax/' + topicId + (topicArgs ? '?' + topicArgs : '');
-    var topicArgs = $(this).data('topicArgs') || false;
-    var topicId = $(this).data('topicId') || '';
 
-    var sortColumn = Number($(this).data('sortColumn')) || 1;
+        var opts = getDataTableOptions(this);
-    var sortOrder = ($(this).data('sortOrder') || 'asc').toString().trim().toLowerCase();
+        opts.ajax = {
-
-    // allow only asc and desc
-    if (sortOrder !== 'desc') {
-        sortOrder = 'asc';
-    }
-
-    var ajaxUrl = topicUrl + '/ajax/' + topicId + (topicArgs ? '?' + topicArgs : '');
-
-    if (rowSort) {
-        columns = rowSort.split(',').map(function (sort) {
-            return {
-                orderable: sort === '1'
-            };
-        });
-    }
-
-    $(this).DataTable({
-        scrollX: true,
-        serverSide: true,
-        ajax: {
             url: ajaxUrl,
             type: 'POST'
-        },
+        };
-        order: [
+        opts.serverSide = true;
-            [sortColumn, sortOrder]
+        opts.processing = true;
-        ],
-        columns: columns,
-        pageLength: 50,
-        processing: true
-    }).on('draw', function () {
-        $('.datestring').each(function () {
-            $(this).html(moment($(this).data('date')).fromNow());
-        });
-    });
-});
 
-$('.data-stats-pie-chart').each(function () {
+        opts.createdRow = function( row, data, dataIndex ) {
-    var column = $(this).data('column') || 'country';
+            installHandlers($(row));
-    var limit = $(this).data('limit') || 20;
-    var topicId = $(this).data('topicId');
-    var topicUrl = $(this).data('topicUrl') || '/campaigns/clicked';
-    var ajaxUrl = topicUrl + '/ajax/' + topicId + '/stats';
-    var self = $(this);
-
-    $.post(ajaxUrl, {column: column, limit: limit}, function(data) {
-      google.charts.load('current', {'packages':['corechart']});
-      google.charts.setOnLoadCallback(drawChart);
-
-      function drawChart() {
-        var gTable = new google.visualization.DataTable();
-        gTable.addColumn('string', 'Column');
-        gTable.addColumn('number', 'Value');
-        gTable.addRows(data.data);
-
-        var options = {'width':500, 'height':400};
-        var chart = new google.visualization.PieChart(self[0]);
-        chart.draw(gTable, options);
-      }
-    });
-});
-
-$('.datestring').each(function () {
-    $(this).html(moment($(this).data('date')).fromNow());
-});
-
-$('.delete-form,.confirm-submit').on('submit', function (e) {
-    if (!confirm($(this).data('confirmMessage') || 'Are you sure? This action can not be undone')) {
-        e.preventDefault();
-    }
-});
-
-$('.fm-date-us.date').datepicker({
-    format: 'mm/dd/yyyy',
-    weekStart: 0,
-    autoclose: true
-});
-
-$('.fm-date-eur.date').datepicker({
-    format: 'dd/mm/yyyy',
-    weekStart: 1,
-    autoclose: true
-});
-
-$('.fm-date-generic.date').datepicker({
-    format: 'yyyy-mm-dd',
-    weekStart: 1,
-    autoclose: true
-});
-
-$('.fm-birthday-us.date').datepicker({
-    format: 'mm/dd',
-    weekStart: 0,
-    autoclose: true
-});
-
-$('.fm-birthday-eur.date').datepicker({
-    format: 'dd/mm',
-    weekStart: 1,
-    autoclose: true
-});
-
-$('.fm-birthday-generic.date').datepicker({
-    format: 'mm-dd',
-    weekStart: 1,
-    autoclose: true
-});
-
-$('.page-refresh').each(function () {
-    var interval = Number($(this).data('interval')) || 60;
-    setTimeout(function () {
-        window.location.reload();
-    }, interval * 1000);
-});
-
-$('.click-select').on('click', function () {
-    $(this).select();
-});
-
-if (typeof moment.tz !== 'undefined') {
-    (function () {
-        var tz = moment.tz.guess();
-        if (tz) {
-            $('.tz-detect').val(tz);
         }
-    })();
-}
 
-// setup SMTP check
+        $(this).DataTable(opts).on('draw', function () {
-var smtpForm = document.querySelector('form#smtp-verify');
+            $('.datestring').each(setupDatestring);
-if (smtpForm) {
-    smtpForm.addEventListener('submit', function (e) {
-        e.preventDefault();
-
-        var form = document.getElementById('settings-form');
-        var formData = new FormData(form);
-        var result = fetch('/settings/smtp-verify', {
-            method: 'POST',
-            body: formData,
-            credentials: 'same-origin'
         });
-
-        var $btn = $('#verify-button').button('loading');
-
-        result.then(function (res) {
-            return res.json();
-        }).then(function (data) {
-            alert(data.error ? 'Invalid Mailer settings\n' + data.error : data.message);
-            $btn.button('reset');
-        }).catch(function (err) {
-            alert(err.message);
-            $btn.button('reset');
-        });
-
     });
-}
+
+    $('.data-stats-pie-chart').each(function () {
+        var column = $(this).data('column') || 'country';
+        var limit = $(this).data('limit') || 20;
+        var topicId = $(this).data('topicId');
+        var topicUrl = $(this).data('topicUrl') || '/campaigns/clicked';
+        var ajaxUrl = topicUrl + '/ajax/' + topicId + '/stats';
+        var self = $(this);
+
+        $.post(ajaxUrl, {column: column, limit: limit}, function(data) {
+          google.charts.load('current', {'packages':['corechart']});
+          google.charts.setOnLoadCallback(drawChart);
+
+          function drawChart() {
+            var gTable = new google.visualization.DataTable();
+            gTable.addColumn('string', 'Column');
+            gTable.addColumn('number', 'Value');
+            gTable.addRows(data.data);
+
+            var options = {'width':500, 'height':400};
+            var chart = new google.visualization.PieChart(self[0]);
+            chart.draw(gTable, options);
+          }
+        });
+    });
+
+    $('.datestring').each(function () {
+        $(this).html(moment($(this).data('date')).fromNow());
+    });
+
+    $('.delete-form,.confirm-submit').on('submit', function (e) {
+        if (!confirm($(this).data('confirmMessage') || 'Are you sure? This action can not be undone')) {
+            e.preventDefault();
+        }
+    });
+
+    $('.fm-date-us.date').datepicker({
+        format: 'mm/dd/yyyy',
+        weekStart: 0,
+        autoclose: true
+    });
+
+    $('.fm-date-eur.date').datepicker({
+        format: 'dd/mm/yyyy',
+        weekStart: 1,
+        autoclose: true
+    });
+
+    $('.fm-date-generic.date').datepicker({
+        format: 'yyyy-mm-dd',
+        weekStart: 1,
+        autoclose: true
+    });
+
+    $('.fm-birthday-us.date').datepicker({
+        format: 'mm/dd',
+        weekStart: 0,
+        autoclose: true
+    });
+
+    $('.fm-birthday-eur.date').datepicker({
+        format: 'dd/mm',
+        weekStart: 1,
+        autoclose: true
+    });
+
+    $('.fm-birthday-generic.date').datepicker({
+        format: 'mm-dd',
+        weekStart: 1,
+        autoclose: true
+    });
+
+    $('.page-refresh').each(function () {
+        var interval = Number($(this).data('interval')) || 60;
+        setTimeout(function () {
+            window.location.reload();
+        }, interval * 1000);
+    });
+
+
+    $('.click-select').on('click', function () {
+        $(this).select();
+    });
+
+    if (typeof moment.tz !== 'undefined') {
+        (function () {
+            var tz = moment.tz.guess();
+            if (tz) {
+                $('.tz-detect').val(tz);
+            }
+        })();
+    }
+
+    // setup SMTP check
+    var smtpForm = document.querySelector('form#smtp-verify');
+    if (smtpForm) {
+        smtpForm.addEventListener('submit', function (e) {
+            e.preventDefault();
+
+            var form = document.getElementById('settings-form');
+            var formData = new FormData(form);
+            var result = fetch('/settings/smtp-verify', {
+                method: 'POST',
+                body: formData,
+                credentials: 'same-origin'
+            });
+
+            var $btn = $('#verify-button').button('loading');
+
+            result.then(function (res) {
+                return res.json();
+            }).then(function (data) {
+                alert(data.error ? 'Invalid Mailer settings\n' + data.error : data.message);
+                $btn.button('reset');
+            }).catch(function (err) {
+                alert(err.message);
+                $btn.button('reset');
+            });
+
+        });
+    }
+
+})();
+

routes/campaigns.js

@@ -9,7 +9,7 @@ let campaigns = require('../lib/models/campaigns');
 let subscriptions = require('../lib/models/subscriptions');
 let settings = require('../lib/models/settings');
 let tools = require('../lib/tools');
-let helpers = require('../lib/helpers');
+let editorHelpers = require('../lib/editor-helpers.js');
 let striptags = require('striptags');
 let passport = require('../lib/passport');
 let htmlescape = require('escape-html');
@@ -186,25 +186,14 @@ router.get('/edit/:id', passport.csrfProtection, (req, res, next) => {
                             view = 'campaigns/edit';
                     }
 
-                    helpers.getDefaultMergeTags((err, defaultMergeTags) => {
+                    editorHelpers.getMergeTagsForResource(campaign, (err, mergeTags) => {
                         if (err) {
                             req.flash('danger', err.message || err);
                             return res.redirect('/');
                         }
 
-                        helpers.getListMergeTags(campaign.list, (err, listMergeTags) => {
+                        campaign.mergeTags = mergeTags;
-                            if (err) {
+                        res.render(view, campaign);
-                                req.flash('danger', err.message || err);
-                                return res.redirect('/');
-                            }
-
-                            campaign.mergeTags = defaultMergeTags.concat(listMergeTags);
-                            campaign.type === 2 && campaign.mergeTags.push({
-                                key: 'RSS_ENTRY',
-                                value: _('content from an RSS entry')
-                            });
-                            res.render(view, campaign);
-                        });
                     });
                 });
             });
@@ -673,6 +662,80 @@ router.post('/status/ajax/:id/:status', (req, res) => {
     });
 });
 
+router.post('/clicked/ajax/:id/:linkId', (req, res) => {
+    let linkId = Number(req.params.linkId) || 0;
+
+    campaigns.get(req.params.id, true, (err, campaign) => {
+        if (err || !campaign) {
+            return res.json({
+                error: err && err.message || err || _('Campaign not found'),
+                data: []
+            });
+        }
+        lists.get(campaign.list, (err, list) => {
+            if (err) {
+                return res.json({
+                    error: err && err.message || err,
+                    data: []
+                });
+            }
+
+            let campaignCid = campaign.cid;
+            let listCid = list.cid;
+
+            let columns = ['#', 'email', 'first_name', 'last_name', 'campaign_tracker__' + campaign.id + '`.`created', 'count'];
+            campaigns.filterClickedSubscribers(campaign, linkId, req.body, columns, (err, data, total, filteredTotal) => {
+                if (err) {
+                    return res.json({
+                        error: err.message || err,
+                        data: []
+                    });
+                }
+
+                res.json({
+                    draw: req.body.draw,
+                    recordsTotal: total,
+                    recordsFiltered: filteredTotal,
+                    data: data.map((row, i) => [
+                        '<a href="/archive/' + encodeURIComponent(campaignCid) + '/' + encodeURIComponent(listCid) + '/' + encodeURIComponent(row.cid) + '?track=no">' + ((Number(req.body.start) || 0) + 1 + i) + '</a>',
+                        htmlescape(row.email || ''),
+                        htmlescape(row.firstName || ''),
+                        htmlescape(row.lastName || ''),
+                        row.created && row.created.toISOString ? '<span class="datestring" data-date="' + row.created.toISOString() + '" title="' + row.created.toISOString() + '">' + row.created.toISOString() + '</span>' : 'N/A',
+                        row.count,
+                        '<span class="glyphicon glyphicon-wrench" aria-hidden="true"></span><a href="/lists/subscription/' + campaign.list + '/edit/' + row.cid + '">' + _('Edit') + '</a>'
+                    ])
+                });
+            });
+        });
+    });
+});
+
+router.post('/quicklist/ajax', (req, res) => {
+    campaigns.filterQuicklist(req.body, (err, data, total, filteredTotal) => {
+        if (err) {
+            return res.json({
+                error: err.message || err,
+                data: []
+            });
+        }
+
+        res.json({
+            draw: req.body.draw,
+            recordsTotal: total,
+            recordsFiltered: filteredTotal,
+            data: data.map((row, i) => ({
+                "0": (Number(req.body.start) || 0) + 1 + i,
+                "1": '<span class="glyphicon glyphicon-inbox" aria-hidden="true"></span> <a href="/campaigns/view/' + row.id + '">' + htmlescape(row.name || '') + '</a>',
+                "2": htmlescape(striptags(row.description) || ''),
+                "3": '<span class="datestring" data-date="' + row.created.toISOString() + '" title="' + row.created.toISOString() + '">' + row.created.toISOString() + '</span>',
+                "DT_RowId": row.id
+            }))
+        });
+    });
+});
+
+
 router.post('/delete', passport.parseForm, passport.csrfProtection, (req, res) => {
     campaigns.delete(req.body.id, (err, deleted) => {
         if (err) {

routes/lists.js

@@ -55,23 +55,8 @@ router.all('/*', (req, res, next) => {
 });
 
 router.get('/', (req, res) => {
-    let limit = 999999999;
+    res.render('lists/lists', {
-    let start = 0;
+        title: _('Lists')
-
-    lists.list(start, limit, (err, rows, total) => {
-        if (err) {
-            req.flash('danger', err.message || err);
-            return res.redirect('/');
-        }
-
-        res.render('lists/lists', {
-            rows: rows.map((row, i) => {
-                row.index = start + i + 1;
-                row.description = striptags(row.description);
-                return row;
-            }),
-            total
-        });
     });
 });
 
@@ -82,6 +67,10 @@ router.get('/create', passport.csrfProtection, (req, res) => {
 
     data.csrfToken = req.csrfToken();
 
+    if (!('publicSubscribe' in data)) {
+        data.publicSubscribe = true;
+    }
+
     res.render('lists/create', data);
 });
 
@@ -155,6 +144,32 @@ router.post('/delete', passport.parseForm, passport.csrfProtection, (req, res) =
     });
 });
 
+router.post('/ajax', (req, res) => {
+    lists.filter(req.body, Number(req.query.parent) || false, (err, data, total, filteredTotal) => {
+        if (err) {
+            return res.json({
+                error: err.message || err,
+                data: []
+            });
+        }
+
+        res.json({
+            draw: req.body.draw,
+            recordsTotal: total,
+            recordsFiltered: filteredTotal,
+            data: data.map((row, i) => [
+                (Number(req.body.start) || 0) + 1 + i,
+                '<span class="glyphicon glyphicon-list-alt" aria-hidden="true"></span> <a href="/lists/view/' + row.id + '">' + htmlescape(row.name || '') + '</a>',
+                '<code>' + row.cid + '</code>',
+                row.subscribers,
+                htmlescape(striptags(row.description) || ''),
+                '<span class="glyphicon glyphicon-wrench" aria-hidden="true"></span><a href="/lists/edit/' + row.id + '">' + _('Edit') + '</a>' ]
+            )
+        });
+    });
+});
+
+
 router.post('/ajax/:id', (req, res) => {
     lists.get(req.params.id, (err, list) => {
         if (err || !list) {
@@ -733,4 +748,27 @@ router.get('/subscription/:id/import/:importId/failed', (req, res) => {
     });
 });
 
+router.post('/quicklist/ajax', (req, res) => {
+    lists.filterQuicklist(req.body, (err, data, total, filteredTotal) => {
+        if (err) {
+            return res.json({
+                error: err.message || err,
+                data: []
+            });
+        }
+
+        res.json({
+            draw: req.body.draw,
+            recordsTotal: total,
+            recordsFiltered: filteredTotal,
+            data: data.map((row, i) => ({
+                "0": (Number(req.body.start) || 0) + 1 + i,
+                "1": '<span class="glyphicon glyphicon-inbox" aria-hidden="true"></span> <a href="/lists/view/' + row.id + '">' + htmlescape(row.name || '') + '</a>',
+                "2": row.subscribers,
+                "DT_RowId": row.id
+            }))
+        });
+    });
+});
+
 module.exports = router;

routes/report-templates.js

@@ -0,0 +1,307 @@
+'use strict';
+
+const express = require('express');
+const passport = require('../lib/passport');
+const router = new express.Router();
+const _ = require('../lib/translate')._;
+const reportTemplates = require('../lib/models/report-templates');
+const tools = require('../lib/tools');
+const util = require('util');
+const htmlescape = require('escape-html');
+const striptags = require('striptags');
+
+const allowedMimeTypes = {
+    'text/html': 'HTML',
+    'text/csv': 'CSV'
+};
+
+router.all('/*', (req, res, next) => {
+    if (!req.user) {
+        req.flash('danger', _('Need to be logged in to access restricted content'));
+        return res.redirect('/users/login?next=' + encodeURIComponent(req.originalUrl));
+    }
+    res.setSelectedMenu('reports');
+    next();
+});
+
+router.get('/', (req, res) => {
+    res.render('report-templates/report-templates', {
+        title: _('Report Templates')
+    });
+});
+
+router.post('/ajax', (req, res) => {
+    reportTemplates.filter(req.body, (err, data, total, filteredTotal) => {
+        if (err) {
+            return res.json({
+                error: err.message || err,
+                data: []
+            });
+        }
+
+        res.json({
+            draw: req.body.draw,
+            recordsTotal: total,
+            recordsFiltered: filteredTotal,
+            data: data.map((row, i) => [
+                (Number(req.body.start) || 0) + 1 + i,
+                htmlescape(row.name || ''),
+                htmlescape(striptags(row.description) || ''),
+                '<span class="datestring" data-date="' + row.created.toISOString() + '" title="' + row.created.toISOString() + '">' + row.created.toISOString() + '</span>',
+                '<span class="glyphicon glyphicon-wrench" aria-hidden="true"></span><a href="/report-templates/edit/' + row.id + '"> ' + _('Edit') + '</a>']
+            )
+        });
+    });
+});
+
+router.get('/create', passport.csrfProtection, (req, res) => {
+    const data = req.query;
+    const wizard = req.query['type'] || '';
+
+    if (wizard == 'subscribers-all') {
+        if (!('description' in data)) data.description = 'Generates a campaign report listing all subscribers along with their statistics.';
+
+        if (!('mimeType' in data)) data.mimeType = 'text/html';
+
+        if (!('userFields' in data)) data.userFields =
+            '[\n' +
+            '    {\n' +
+            '        "id": "campaign",\n' +
+            '        "name": "Campaign",\n' +
+            '        "type": "campaign",\n' +
+            '        "minOccurences": 1,\n' +
+            '        "maxOccurences": 1\n' +
+            '    }\n' +
+            ']';
+
+        if (!('js' in data)) data.js =
+            'campaigns.results(inputs.campaign, ["*"], "", (err, results) => {\n' +
+            '    if (err) {\n' +
+            '        return callback(err);\n' +
+            '    }\n' +
+            '\n' +
+            '    const data = {\n' +
+            '        results: results\n' +
+            '    };\n' +
+            '\n' +
+            '    return callback(null, data);\n' +
+            '});';
+
+        if (!('hbs' in data)) data.hbs =
+            '<h2>{{title}}</h2>\n' +
+            '\n' +
+            '<div class="table-responsive">\n' +
+            '    <table class="table table-bordered table-hover data-table display nowrap" width="100%" data-row-sort="1,1" data-paging="false">\n' +
+            '        <thead>\n' +
+            '        <th>\n' +
+            '            {{#translate}}Email{{/translate}}\n' +
+            '        </th>\n' +
+            '        <th>\n' +
+            '            {{#translate}}Tracker Count{{/translate}}\n' +
+            '        </th>\n' +
+            '        </thead>\n' +
+            '        {{#if results}}\n' +
+            '            <tbody>\n' +
+            '            {{#each results}}\n' +
+            '                <tr>\n' +
+            '                    <th scope="row">\n' +
+            '                        {{email}}\n' +
+            '                    </th>\n' +
+            '                    <td style="width: 20%;">\n' +
+            '                        {{tracker_count}}\n' +
+            '                    </td>\n' +
+            '                </tr>\n' +
+            '            {{/each}}\n' +
+            '            </tbody>\n' +
+            '        {{/if}}\n' +
+            '    </table>\n' +
+            '</div>';
+
+    } else if (wizard == 'subscribers-grouped') {
+        if (!('description' in data)) data.description = 'Generates a campaign report with results are aggregated by some "Country" custom field.';
+
+        if (!('mimeType' in data)) data.mimeType = 'text/html';
+
+        if (!('userFields' in data)) data.userFields =
+            '[\n' +
+            '    {\n' +
+            '        "id": "campaign",\n' +
+            '        "name": "Campaign",\n' +
+            '        "type": "campaign",\n' +
+            '        "minOccurences": 1,\n' +
+            '        "maxOccurences": 1\n' +
+            '    }\n' +
+            ']';
+
+        if (!('js' in data)) data.js =
+            'campaigns.results(inputs.campaign, ["custom_country", "count(*) AS count_all", "SUM(IF(tracker.count IS NULL, 0, 1)) AS count_opened"], "GROUP BY custom_country", (err, results) => {\n' +
+            '    if (err) {\n' +
+            '        return callback(err);\n' +
+            '    }\n' +
+            '\n' +
+            '    for (let row of results) {\n' +
+            '        row["percentage"] = Math.round((row.count_opened / row.count_all) * 100);\n' +
+            '    }\n' +
+            '\n' +
+            '    let data = {\n' +
+            '        results: results\n' +
+            '    };\n' +
+            '\n' +
+            '    return callback(null, data);\n' +
+            '});';
+
+        if (!('hbs' in data)) data.hbs =
+            '<h2>{{title}}</h2>\n' +
+            '\n' +
+            '<div class="table-responsive">\n' +
+            '    <table class="table table-bordered table-hover data-table display nowrap" width="100%" data-row-sort="1,1,1,1" data-paging="false">\n' +
+            '        <thead>\n' +
+            '        <th>\n' +
+            '            {{#translate}}Country{{/translate}}\n' +
+            '        </th>\n' +
+            '        <th>\n' +
+            '            {{#translate}}Opened{{/translate}}\n' +
+            '        </th>\n' +
+            '        <th>\n' +
+            '            {{#translate}}All{{/translate}}\n' +
+            '        </th>\n' +
+            '        <th>\n' +
+            '            {{#translate}}Percentage{{/translate}}\n' +
+            '        </th>\n' +
+            '        </thead>\n' +
+            '        {{#if results}}\n' +
+            '            <tbody>\n' +
+            '            {{#each results}}\n' +
+            '                <tr>\n' +
+            '                    <th scope="row">\n' +
+            '                        {{custom_country}}\n' +
+            '                    </th>\n' +
+            '                    <td style="width: 20%;">\n' +
+            '                        {{count_opened}}\n' +
+            '                    </td>\n' +
+            '                    <td style="width: 20%;">\n' +
+            '                        {{count_all}}\n' +
+            '                    </td>\n' +
+            '                    <td style="width: 20%;">\n' +
+            '                        {{percentage}}%\n' +
+            '                    </td>\n' +
+            '                </tr>\n' +
+            '            {{/each}}\n' +
+            '            </tbody>\n' +
+            '        {{/if}}\n' +
+            '    </table>\n' +
+            '</div>';
+
+    } else if (wizard == 'export-list-csv') {
+        if (!('description' in data)) data.description = 'Exports a list as a CSV file.';
+
+        if (!('mimeType' in data)) data.mimeType = 'text/csv';
+
+        if (!('userFields' in data)) data.userFields =
+            '[\n' +
+            '    {\n' +
+            '        "id": "list",\n' +
+            '        "name": "List",\n' +
+            '        "type": "list",\n' +
+            '        "minOccurences": 1,\n' +
+            '        "maxOccurences": 1\n' +
+            '    }\n' +
+            ']';
+
+        if (!('js' in data)) data.js =
+            'subscriptions.list(inputs.list.id,0,0, (err, results) => {\n' +
+            '    if (err) {\n' +
+            '        return callback(err);\n' +
+            '    }\n' +
+            '\n' +
+            '    let data = {\n' +
+            '        results: results\n' +
+            '    };\n' +
+            '\n' +
+            '    return callback(null, data);\n' +
+            '});';
+
+        if (!('hbs' in data)) data.hbs =
+            '{{#each results}}\n' +
+            '{{firstName}},{{lastName}},{{email}}\n' +
+            '{{/each}}';
+    }
+
+    data.csrfToken = req.csrfToken();
+    data.title = _('Create Report Template');
+    data.useEditor = true;
+
+    data.mimeTypes = Object.keys(allowedMimeTypes).map(key => ({
+        key: key,
+        value: allowedMimeTypes[key],
+        selected: data.mimeType == key
+    }));
+
+    res.render('report-templates/create', data);
+});
+
+router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) => {
+    reportTemplates.createOrUpdate(true, req.body, (err, id) => {
+        if (err || !id) {
+            req.flash('danger', err && err.message || err || _('Could not create report template'));
+            return res.redirect('/report-templates/create?' + tools.queryParams(req.body));
+        }
+        req.flash('success', util.format(_('Report template “%s” created'), req.body.name));
+        res.redirect('/report-templates');
+    });
+});
+
+router.get('/edit/:id', passport.csrfProtection, (req, res) => {
+    reportTemplates.get(req.params.id, (err, template) => {
+        if (err || !template) {
+            req.flash('danger', err && err.message || err || _('Could not find report template with specified ID'));
+            return res.redirect('/report-templates');
+        }
+
+        template.csrfToken = req.csrfToken();
+        template.title = _('Edit Report Template');
+        template.useEditor = true;
+
+        template.mimeTypes = Object.keys(allowedMimeTypes).map(key => ({
+            key: key,
+            value: allowedMimeTypes[key],
+            selected: template.mimeType == key
+        }));
+
+        res.render('report-templates/edit', template);
+    });
+});
+
+router.post('/edit', passport.parseForm, passport.csrfProtection, (req, res) => {
+    reportTemplates.createOrUpdate(false, req.body, (err, updated) => {
+        if (err) {
+            req.flash('danger', err.message || err);
+        } else if (updated) {
+            req.flash('success', _('Report template updated'));
+        } else {
+            req.flash('info', _('Report template not updated'));
+        }
+
+        if (req.body['submit'] == 'update-and-stay') {
+            return res.redirect('/report-templates/edit/' + req.body.id);
+        } else {
+            return res.redirect('/report-templates');
+        }
+    });
+});
+
+router.post('/delete', passport.parseForm, passport.csrfProtection, (req, res) => {
+    reportTemplates.delete(req.body.id, (err, deleted) => {
+        if (err) {
+            req.flash('danger', err && err.message || err);
+        } else if (deleted) {
+            req.flash('success', _('Report template deleted'));
+        } else {
+            req.flash('info', _('Could not delete specified report template'));
+        }
+
+        return res.redirect('/report-templates');
+    });
+});
+
+module.exports = router;

routes/reports.js

@@ -0,0 +1,406 @@
+'use strict';
+
+const express = require('express');
+const passport = require('../lib/passport');
+const router = new express.Router();
+const _ = require('../lib/translate')._;
+const reportTemplates = require('../lib/models/report-templates');
+const reports = require('../lib/models/reports');
+const reportProcessor = require('../lib/report-processor');
+const campaigns = require('../lib/models/campaigns');
+const lists = require('../lib/models/lists');
+const tools = require('../lib/tools');
+const fileHelpers = require('../lib/file-helpers');
+const util = require('util');
+const htmlescape = require('escape-html');
+const striptags = require('striptags');
+const fs = require('fs');
+const hbs = require('hbs');
+
+router.all('/*', (req, res, next) => {
+    if (!req.user) {
+        req.flash('danger', _('Need to be logged in to access restricted content'));
+        return res.redirect('/users/login?next=' + encodeURIComponent(req.originalUrl));
+    }
+    res.setSelectedMenu('reports');
+    next();
+});
+
+router.get('/', (req, res) => {
+    res.render('reports/reports', {
+        title: _('Reports')
+    });
+});
+
+
+
+router.post('/ajax', (req, res) => {
+    reports.filter(req.body, (err, data, total, filteredTotal) => {
+        if (err) {
+            return res.json({
+                error: err.message || err,
+                data: []
+            });
+        }
+
+        res.json({
+            draw: req.body.draw,
+            recordsTotal: total,
+            recordsFiltered: filteredTotal,
+            data: data.map((row, i) => [
+                (Number(req.body.start) || 0) + 1 + i,
+                htmlescape(row.name || ''),
+                htmlescape(row.reportTemplateName || ''),
+                htmlescape(striptags(row.description) || ''),
+                getRowLastRun(row),
+                getRowActions(row)
+            ])
+        });
+    });
+});
+
+router.get('/row/ajax/:id', (req, res) => {
+    respondRowActions(req.params.id, res);
+});
+
+router.get('/start/ajax/:id', (req, res) => {
+    reportProcessor.start(req.params.id, () => {
+        respondRowActions(req.params.id, res);
+    });
+});
+
+router.get('/stop/ajax/:id', (req, res) => {
+    reportProcessor.stop(req.params.id, () => {
+        respondRowActions(req.params.id, res);
+    });
+});
+
+router.get('/create', passport.csrfProtection, (req, res) => {
+    const reqData = req.query;
+    reqData.csrfToken = req.csrfToken();
+    reqData.title = _('Create Report');
+    reqData.useEditor = true;
+
+    reportTemplates.quicklist((err, items) => {
+        if (err) {
+            req.flash('danger', err.message || err);
+            return res.redirect('/reports');
+        }
+
+        const reportTemplateId = Number(reqData.reportTemplate);
+
+        if (reportTemplateId) {
+            items.forEach(item => {
+                if (item.id === reportTemplateId) {
+                    item.selected = true;
+                }
+            });
+        }
+
+        reqData.reportTemplates = items;
+
+        if (!reportTemplateId) {
+            res.render('reports/create-select-template', reqData);
+        } else {
+            addUserFields(reportTemplateId, reqData, null, (err, data) => {
+                if (err) {
+                    req.flash('danger', err.message || err);
+                    return res.redirect('/reports');
+                }
+
+                res.render('reports/create', data);
+            });
+        }
+    });
+});
+
+router.post('/create', passport.parseForm, passport.csrfProtection, (req, res) => {
+    const reqData = req.body;
+
+    const reportTemplateId = Number(reqData.reportTemplate);
+
+    addParamsObject(reportTemplateId, reqData, (err, data) => {
+        if (err) {
+            req.flash('danger', err && err.message || err || _('Could not create report'));
+            return res.redirect('/reports/create?' + tools.queryParams(data));
+        }
+
+        reports.createOrUpdate(true, data, (err, id) => {
+            if (err || !id) {
+                req.flash('danger', err && err.message || err || _('Could not create report'));
+                return res.redirect('/reports/create?' + tools.queryParams(data));
+            }
+
+            reportProcessor.start(id, () => {
+                req.flash('success', util.format(_('Report “%s” created'), data.name));
+                res.redirect('/reports');
+            });
+        });
+    });
+});
+
+router.get('/edit/:id', passport.csrfProtection, (req, res) => {
+    const reqData = req.query;
+    reports.get(req.params.id, (err, report) => {
+        if (err || !report) {
+            req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
+            return res.redirect('/reports');
+        }
+
+        report.csrfToken = req.csrfToken();
+        report.title = _('Edit Report');
+        report.useEditor = true;
+
+        reportTemplates.quicklist((err, items) => {
+            if (err) {
+                req.flash('danger', err.message || err);
+                return res.redirect('/');
+            }
+
+            const reportTemplateId = report.reportTemplate;
+
+            items.forEach(item => {
+                if (item.id === reportTemplateId) {
+                    item.selected = true;
+                }
+            });
+
+            report.reportTemplates = items;
+
+            addUserFields(reportTemplateId, reqData, report, (err, data) => {
+                if (err) {
+                    req.flash('danger', err.message || err);
+                    return res.redirect('/reports');
+                }
+
+                res.render('reports/edit', data);
+            });
+        });
+    });
+});
+
+router.post('/edit', passport.parseForm, passport.csrfProtection, (req, res) => {
+    const reqData = req.body;
+    const reportTemplateId = Number(reqData.reportTemplate);
+
+    addParamsObject(reportTemplateId, reqData, (err, data) => {
+        if (err) {
+            req.flash('danger', err && err.message || err || _('Could not update report'));
+            return res.redirect('/reports/create?' + tools.queryParams(data));
+        }
+
+        reports.createOrUpdate(false, data, (err, updated) => {
+            if (err) {
+                req.flash('danger', err && err.message || err || _('Could not update report'));
+                return res.redirect('/reports/edit/' + data.id + '?' + tools.queryParams(data));
+            } else if (updated) {
+                req.flash('success', _('Report updated'));
+            } else {
+                req.flash('info', _('Report not updated'));
+            }
+
+            return res.redirect('/reports');
+        });
+    });
+});
+
+router.post('/delete', passport.parseForm, passport.csrfProtection, (req, res) => {
+    reports.delete(req.body.id, (err, deleted) => {
+        if (err) {
+            req.flash('danger', err && err.message || err);
+        } else if (deleted) {
+            req.flash('success', _('Report deleted'));
+        } else {
+            req.flash('info', _('Could not delete specified report'));
+        }
+
+        return res.redirect('/reports');
+    });
+});
+
+router.get('/view/:id', (req, res) => {
+    reports.get(req.params.id, (err, report) => {
+        if (err || !report) {
+            req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
+            return res.redirect('/reports');
+        }
+
+        reportTemplates.get(report.reportTemplate, (err, reportTemplate) => {
+            if (err) {
+                req.flash('danger', err && err.message || err || _('Could not find report template'));
+                return res.redirect('/reports');
+            }
+
+            if (report.state == reports.ReportState.FINISHED) {
+                if (reportTemplate.mimeType == 'text/html') {
+
+                    fs.readFile(fileHelpers.getReportContentFile(report), (err, reportContent) => {
+                        if (err) {
+                            req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
+                            return res.redirect('/reports');
+                        }
+
+                        const data = {
+                            report: new hbs.handlebars.SafeString(reportContent),
+                            title: report.name
+                        };
+
+                        res.render('reports/view', data);
+                    });
+
+                } else if (reportTemplate.mimeType == 'text/csv') {
+                    const headers = {
+                        'Content-Disposition': 'attachment;filename=' + fileHelpers.nameToFileName(report.name) + '.csv',
+                        'Content-Type': 'text/csv'
+                    };
+
+                    res.sendFile(fileHelpers.getReportContentFile(report), {headers: headers});
+
+                } else {
+                    req.flash('danger', _('Unknown type of template'));
+                    res.redirect('/reports');
+                }
+
+            } else {
+                req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
+                return res.redirect('/reports');
+            }
+        });
+    });
+});
+
+router.get('/output/:id', (req, res) => {
+    reports.get(req.params.id, (err, report) => {
+        if (err || !report) {
+            req.flash('danger', err && err.message || err || _('Could not find report with specified ID'));
+            return res.redirect('/reports');
+        }
+
+        fs.readFile(fileHelpers.getReportOutputFile(report), (err, output) => {
+            let data = {
+                title: 'Output for report ' + report.name
+            };
+
+            if (err) {
+                data.error = 'No output.';
+            } else {
+                data.output = output;
+            }
+
+            res.render('reports/output', data);
+        });
+    });
+});
+
+function getRowLastRun(row) {
+    return '<span id="row-last-run-' + row.id + '">' + (row.lastRun ? '<span class="datestring" data-date="' + row.lastRun.toISOString() + '" title="' + row.lastRun.toISOString() + '">' + row.lastRun.toISOString() + '</span>' : '') + '</span>';
+}
+
+function getRowActions(row) {
+    /* FIXME: add csrf protection to stop and refresh actions */
+
+    let requestRefresh = false;
+    let view, startStop;
+    let topic = 'data-topic-id="' + row.id + '"';
+
+    if (row.state == reports.ReportState.PROCESSING || row.state == reports.ReportState.SCHEDULED) {
+        view = '<span class="row-action glyphicon glyphicon-hourglass" aria-hidden="true" title="Processing"></span>';
+        startStop = '<a class="row-action ajax-action" href="" data-topic-url="/reports/stop" ' + topic + ' title="Stop"><span class="glyphicon glyphicon-stop" aria-hidden="true"></span></a>';
+        requestRefresh = true;
+
+    } else if (row.state == reports.ReportState.FINISHED) {
+        let icon = 'eye-open';
+        if (row.mimeType == 'text/csv') icon = 'download-alt';
+
+        view = '<a class="row-action" href="/reports/view/' + row.id + '" title="View report"><span class="glyphicon glyphicon-' + icon + '" aria-hidden="true"></span></a>';
+        startStop = '<a class="row-action ajax-action" href="" data-topic-url="/reports/start" ' + topic + ' title="Refresh report"><span class="glyphicon glyphicon-repeat" aria-hidden="true"></span></a>';
+
+    } else if (row.state == reports.ReportState.FAILED) {
+        view = '<span class="row-action glyphicon glyphicon-thumbs-down" aria-hidden="true" title="Report generation failed"></span>';
+        startStop = '<a class="row-action ajax-action" href="" data-topic-url="/reports/start" ' + topic + ' title="Refresh report"><span class="glyphicon glyphicon-repeat" aria-hidden="true"></span></a>';
+    }
+
+    let actions = view;
+    actions += '<a class="row-action" href="/reports/output/' + row.id + '" title="View console output"><span class="glyphicon glyphicon-modal-window" aria-hidden="true"></span></a>';
+    actions += startStop;
+    actions += '<a class="row-action" href="/reports/edit/' + row.id + '"><span class="glyphicon glyphicon-wrench" aria-hidden="true" title="Edit"></span></a>';
+
+    return '<span id="row-actions-' + row.id + '"' + (requestRefresh ? ' class="row-actions ajax-refresh" data-interval="5" data-topic-url="/reports/row" ' + topic : ' class="row-actions"') + '>' +
+        actions +
+        '</span>';
+}
+
+function respondRowActions(id, res) {
+    reports.get(id, (err, report) => {
+        if (err) {
+            return res.json({
+                error: err,
+            });
+        }
+
+        const data = {};
+        data['#row-last-run-' + id] = getRowLastRun(report);
+        data['#row-actions-' + id] = getRowActions(report);
+
+        res.json(data);
+    });
+}
+
+
+function addUserFields(reportTemplateId, reqData, report, callback) {
+    reportTemplates.get(reportTemplateId, (err, reportTemplate) => {
+        if (err) {
+            return callback(err);
+        }
+
+        const userFields = [];
+
+        for (const spec of reportTemplate.userFieldsObject) {
+            let value = '';
+            if ((spec.id + 'Selection') in reqData) {
+                value = reqData[spec.id + 'Selection'];
+            } else if (report && report.paramsObject && spec.id in report.paramsObject) {
+                value = report.paramsObject[spec.id].join(',');
+            }
+
+            userFields.push({
+                'id': spec.id,
+                'name': spec.name,
+                'type': spec.type,
+                'value': value,
+                'isMulti': !(spec.minOccurences == 1 && spec.maxOccurences == 1)
+            });
+        }
+
+        const data = report ? report : reqData;
+        data.userFields = userFields;
+
+        callback(null, data);
+    });
+}
+
+function addParamsObject(reportTemplateId, data, callback) {
+    reportTemplates.get(reportTemplateId, (err, reportTemplate) => {
+        if (err) {
+            return callback(err);
+        }
+
+        const paramsObject = {};
+
+        for (const spec of reportTemplate.userFieldsObject) {
+            const sel = data[spec.id + 'Selection'];
+
+            if (!sel) {
+                paramsObject[spec.id] = [];
+            } else {
+                paramsObject[spec.id] = sel.split(',').map(item => Number(item));
+            }
+        }
+
+        data.paramsObject = paramsObject;
+
+        callback(null, data);
+    });
+}
+
+module.exports = router;

routes/subscription.js

@@ -176,9 +176,14 @@ router.get('/subscribe/:cid', (req, res, next) => {
 
 router.get('/:cid', passport.csrfProtection, (req, res, next) => {
     lists.getByCid(req.params.cid, (err, list) => {
-        if (!err && !list) {
+        if (!err) {
-            err = new Error(_('Selected list not found'));
+            if (!list) {
-            err.status = 404;
+                err = new Error(_('Selected list not found'));
+                err.status = 404;
+            } else if (!list.publicSubscribe) {
+                err = new Error(_('The list does not allow public subscriptions.'));
+                err.status = 403;
+            }
         }
 
         if (err) {
@@ -501,9 +506,14 @@ router.post('/:cid/subscribe', passport.parseForm, corsOrCsrfProtection, (req, r
     let testsPass = subTimeTest && addressTest;
 
     lists.getByCid(req.params.cid, (err, list) => {
-        if (!err && !list) {
+        if (!err) {
-            err = new Error(_('Selected list not found'));
+            if (!list) {
-            err.status = 404;
+                err = new Error(_('Selected list not found'));
+                err.status = 404;
+            } else if (!list.publicSubscribe) {
+                err = new Error(_('The list does not allow public subscriptions.'));
+                err.status = 403;
+            }
         }
 
         if (err) {

routes/templates.js

@@ -8,6 +8,7 @@ let settings = require('../lib/models/settings');
 let tools = require('../lib/tools');
 let helpers = require('../lib/helpers');
 let striptags = require('striptags');
+let htmlescape = require('escape-html');
 let passport = require('../lib/passport');
 let mailer = require('../lib/mailer');
 let _ = require('../lib/translate')._;
@@ -22,23 +23,8 @@ router.all('/*', (req, res, next) => {
 });
 
 router.get('/', (req, res) => {
-    let limit = 999999999;
+    res.render('templates/templates', {
-    let start = 0;
+        title: _('Templates')
-
-    templates.list(start, limit, (err, rows, total) => {
-        if (err) {
-            req.flash('danger', err.message || err);
-            return res.redirect('/');
-        }
-
-        res.render('templates/templates', {
-            rows: rows.map((row, i) => {
-                row.index = start + i + 1;
-                row.description = striptags(row.description);
-                return row;
-            }),
-            total
-        });
     });
 });
 
@@ -164,4 +150,27 @@ router.post('/delete', passport.parseForm, passport.csrfProtection, (req, res) =
     });
 });
 
+router.post('/ajax', (req, res) => {
+    templates.filter(req.body, Number(req.query.parent) || false, (err, data, total, filteredTotal) => {
+        if (err) {
+            return res.json({
+                error: err.message || err,
+                data: []
+            });
+        }
+
+        res.json({
+            draw: req.body.draw,
+            recordsTotal: total,
+            recordsFiltered: filteredTotal,
+            data: data.map((row, i) => [
+                (Number(req.body.start) || 0) + 1 + i,
+                '<span class="glyphicon glyphicon-file" aria-hidden="true"></span> ' + htmlescape(row.name || ''),
+                htmlescape(striptags(row.description) || ''),
+                '<span class="glyphicon glyphicon-wrench" aria-hidden="true"></span><a href="/templates/edit/' + row.id + '">' + _('Edit') + '</a>' ]
+            )
+        });
+    });
+});
+
 module.exports = router;

services/executor.js

@@ -0,0 +1,131 @@
+'use strict';
+
+/* Privileged executor. If Mailtrain is started as root, this process keeps the root privilege to be able to spawn workers
+   that can chroot.
+  */
+
+const fileHelpers = require('../lib/file-helpers');
+const fork = require('child_process').fork;
+const path = require('path');
+const log = require('npmlog');
+const fs = require('fs');
+const privilegeHelpers = require('../lib/privilege-helpers');
+
+let processes = {};
+
+function spawnProcess(tid, executable, args, outFile, errFile, cwd, uid, gid) {
+
+    function reportFail(msg) {
+        process.send({
+            type: 'process-failed',
+            msg,
+            tid
+        });
+    }
+
+    fs.open(outFile, 'w', (err, outFd) => {
+        if (err) {
+            log.error('Executor', err);
+            reportFail('Cannot create standard output file.');
+            return;
+        }
+
+        fs.open(errFile, 'w', (err, errFd) => {
+            if (err) {
+                log.error('Executor', err);
+                reportFail('Cannot create standard error file.');
+                return;
+            }
+
+            privilegeHelpers.ensureMailtrainOwner(outFile, (err) => {
+                if (err) {
+                    log.warn('Executor', 'Cannot change owner of output file of process tid:%s.', tid)
+                }
+
+                privilegeHelpers.ensureMailtrainOwner(errFile, (err) => {
+                    if (err) {
+                        log.warn('Executor', 'Cannot change owner of error output file of process tid:%s.', tid)
+                    }
+
+                    const options = {
+                        stdio: ['ignore', outFd, errFd, 'ipc'],
+                        cwd,
+                        env: {NODE_ENV: process.env.NODE_ENV},
+                        uid,
+                        gid
+                    };
+
+                    let child;
+
+                    try {
+                        child = fork(executable, args, options);
+                    } catch (err) {
+                        log.error('Executor', 'Cannot start process with tid:%s.', tid);
+                        reportFail('Cannot start process.');
+                        return;
+                    }
+
+                    const pid = child.pid;
+                    processes[tid] = child;
+
+                    log.info('Executor', 'Process started with tid:%s pid:%s.', tid, pid);
+                    process.send({
+                        type: 'process-started',
+                        tid
+                    });
+
+                    child.on('close', (code, signal) => {
+
+                        delete processes[tid];
+                        log.info('Executor', 'Process tid:%s pid:%s exited with code %s signal %s.', tid, pid, code, signal);
+
+                        fs.close(outFd, (err) => {
+                            if (err) {
+                                log.error('Executor', err);
+                            }
+
+                            fs.close(errFd, (err) => {
+                                if (err) {
+                                    log.error('Executor', err);
+                                }
+
+                                process.send({
+                                    type: 'process-finished',
+                                    tid,
+                                    code,
+                                    signal
+                                });
+                            });
+                        });
+                    });
+                });
+            });
+        });
+    });
+}
+
+process.on('message', msg => {
+    if (msg) {
+        const type = msg.type;
+
+        if (type === 'start-report-processor-worker') {
+
+            const ids = privilegeHelpers.getConfigROUidGid();
+            spawnProcess(msg.tid, path.join(__dirname, '..', 'workers', 'reports', 'report-processor.js'), [msg.data.id], fileHelpers.getReportContentFile(msg.data), fileHelpers.getReportOutputFile(msg.data), path.join(__dirname, '..', 'workers', 'reports'), ids.uid, ids.gid);
+
+        } else if (type === 'stop-process') {
+            const child = processes[msg.tid];
+
+            if (child) {
+                log.info('Executor', 'Killing process tid:%s pid:%s', msg.tid, child.pid);
+                child.kill();
+            } else {
+                log.info('Executor', 'No running process found with tid:%s pid:%s', msg.tid, child.pid);
+            }
+        }
+    }
+});
+
+process.send({
+    type: 'executor-started'
+});

services/feedcheck.js

@@ -132,8 +132,11 @@ function checkEntries(parent, entries, callback) {
                 let entryId = result.insertId;
                 let html = (parent.html || '').toString().trim();
 
-                if (/\[RSS_ENTRY\]/i.test(html)) {
+                if (/\[RSS_ENTRY[\w]*\]/i.test(html)) {
-                    html = html.replace(/\[RSS_ENTRY\]/, entry.content);
+                    html = html.replace(/\[RSS_ENTRY\]/, entry.content); //for backward compatibility
+                    Object.keys(entry).forEach(key => {
+                        html = html.replace('\[RSS_ENTRY_'+key.toUpperCase()+'\]', entry[key])
+                    });
                 } else {
                     html = entry.content + html;
                 }

services/sender.js

@@ -318,6 +318,7 @@ function formatMessage(message, callback) {
                 }
 
                 let useVerp = config.verp.enabled && configItems.verpUse && configItems.verpHostname;
+                let useVerpSenderHeader = useVerp && config.verp.disablesenderheader !== true;
                 fields.list(list.id, (err, fieldList) => {
                     if (err) {
                         return callback(err);
@@ -389,7 +390,7 @@ function formatMessage(message, callback) {
                                         name: [].concat(message.subscription.firstName || []).concat(message.subscription.lastName || []).join(' '),
                                         address: message.subscription.email
                                     },
-                                    sender: useVerp ? campaignAddress + '@' + configItems.verpHostname : false,
+                                    sender: useVerpSenderHeader ? campaignAddress + '@' + configItems.verpHostname : false,
 
                                     envelope: useVerp ? {
                                         from: campaignAddress + '@' + configItems.verpHostname,

services/test-server.js

@@ -4,12 +4,37 @@ let log = require('npmlog');
 let config = require('config');
 let crypto = require('crypto');
 let humanize = require('humanize');
+let http = require('http');
 
 let SMTPServer = require('smtp-server').SMTPServer;
+let simpleParser = require('mailparser').simpleParser;
 
 let totalMessages = 0;
 let received = 0;
 
+let mailstore = {
+    accounts: {},
+    saveMessage(address, message) {
+        if (!this.accounts[address]) {
+            this.accounts[address] = [];
+        }
+        this.accounts[address].push(message);
+    },
+    getMail(address, callback) {
+        if (!this.accounts[address] || this.accounts[address].length === 0) {
+            let err = new Error('No mail for ' + address);
+            err.status = 404;
+            return callback(err);
+        }
+        simpleParser(this.accounts[address].shift(), (err, mail) => {
+            if (err) {
+                return callback(err.message || err);
+            }
+            callback(null, mail);
+        });
+    }
+};
+
 // Setup server
 let server = new SMTPServer({
 
@@ -74,8 +99,12 @@ let server = new SMTPServer({
     // Handle message stream
     onData: (stream, session, callback) => {
         let hash = crypto.createHash('md5');
+        let message = '';
         stream.on('data', chunk => {
             hash.update(chunk);
+            if (/^keep/i.test(session.envelope.rcptTo[0].address)) {
+                message += chunk;
+            }
         });
         stream.on('end', () => {
             let err;
@@ -84,6 +113,12 @@ let server = new SMTPServer({
                 err.responseCode = 552;
                 return callback(err);
             }
+
+            // Store message for e2e tests
+            if (/^keep/i.test(session.envelope.rcptTo[0].address)) {
+                mailstore.saveMessage(session.envelope.rcptTo[0].address, message);
+            }
+
             received++;
             callback(null, 'Message queued as ' + hash.digest('hex')); // accept the message once the stream is ended
         });
@@ -94,6 +129,41 @@ server.on('error', err => {
     log.error('Test SMTP', err.stack);
 });
 
+let mailBoxServer = http.createServer((req, res) => {
+    let renderer = data => (
+        '<!doctype html><html><head><title>' + data.title + '</title></head><body>' + data.body + '</body></html>'
+    );
+
+    let address = req.url.substring(1);
+    mailstore.getMail(address, (err, mail) => {
+        if (err) {
+            let html = renderer({
+                title: 'error',
+                body: err.message || err
+            });
+            res.writeHead(err.status || 500, { 'Content-Type': 'text/html' });
+            return res.end(html);
+        }
+
+        let html = mail.html || renderer({
+            title: 'error',
+            body: 'This mail has no HTML part'
+        });
+
+        // https://nodemailer.com/extras/mailparser/#mail-object
+        delete mail.html;
+        delete mail.textAsHtml;
+        delete mail.attachments;
+
+        let script = '<script> var mailObject = ' + JSON.stringify(mail) + '; console.log(mailObject); </script>';
+        html = html.replace(/<\/body\b/i, match => script + match);
+        html = html.replace(/target="_blank"/g, 'target="_self"');
+
+        res.writeHead(200, { 'Content-Type': 'text/html' });
+        res.end(html);
+    });
+});
+
 module.exports = callback => {
     if (config.testserver.enabled) {
         server.listen(config.testserver.port, config.testserver.host, () => {
@@ -112,7 +182,10 @@ module.exports = callback => {
                 }
             }, 60 * 1000);
 
-            setImmediate(callback);
+            mailBoxServer.listen(config.testserver.mailboxserverport, config.testserver.host, () => {
+                log.info('Test SMTP', 'Mail Box Server listening on port %s', config.testserver.mailboxserverport);
+                setImmediate(callback);
+            });
         });
     } else {
         setImmediate(callback);

setup/install-centos7.sh

@@ -0,0 +1,227 @@
+#!/bin/bash
+
+# This installation script works on CentOS 7
+# Run as root!
+
+if [[ $EUID -ne 0 ]]; then
+   echo "This script must be run as root" 1>&2
+   exit 1
+fi
+
+set -e
+
+yum -y install epel-release
+
+curl --silent --location https://rpm.nodesource.com/setup_7.x | bash -
+yum -y install mariadb-server nodejs ImageMagick git python redis pwgen bind-utils gcc-c++ make
+
+systemctl start mariadb
+systemctl enable mariadb
+
+systemctl start redis
+systemctl enable redis
+
+
+PUBLIC_IP=`curl -s https://api.ipify.org`
+if [ ! -z "$PUBLIC_IP" ]; then
+    HOSTNAME=`dig +short -x $PUBLIC_IP | sed 's/\.$//'`
+    HOSTNAME="${HOSTNAME:-$PUBLIC_IP}"
+fi
+HOSTNAME="${HOSTNAME:-`hostname`}"
+
+MYSQL_PASSWORD=`pwgen 12 -1`
+MYSQL_RO_PASSWORD=`pwgen 12 -1`
+DKIM_API_KEY=`pwgen 12 -1`
+SMTP_PASS=`pwgen 12 -1`
+
+# Setup MySQL user for Mailtrain
+mysql -u root -e "CREATE USER 'mailtrain'@'localhost' IDENTIFIED BY '$MYSQL_PASSWORD';"
+mysql -u root -e "GRANT ALL PRIVILEGES ON mailtrain.* TO 'mailtrain'@'localhost';"
+mysql -u root -e "CREATE USER 'mailtrain_ro'@'localhost' IDENTIFIED BY '$MYSQL_RO_PASSWORD';"
+mysql -u root -e "GRANT SELECT ON mailtrain.* TO 'mailtrain_ro'@'localhost';"
+mysql -u mailtrain --password="$MYSQL_PASSWORD" -e "CREATE database mailtrain;"
+
+# Enable firewall, allow connections to SSH, HTTP, HTTPS and SMTP
+for port in 80/tcp 443/tcp 25/tcp; do firewall-cmd --add-port=$port --permanent; done
+firewall-cmd --reload
+
+# Fetch Mailtrain files
+mkdir -p /opt/mailtrain
+cd /opt/mailtrain
+git clone git://github.com/Mailtrain-org/mailtrain.git .
+
+# Normally we would let Mailtrain itself to import the initial SQL data but in this case
+# we need to modify it, before we start Mailtrain
+mysql -u mailtrain -p"$MYSQL_PASSWORD" mailtrain < setup/sql/mailtrain.sql
+
+mysql -u mailtrain -p"$MYSQL_PASSWORD" mailtrain <<EOT
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('admin_email','admin@$HOSTNAME') ON DUPLICATE KEY UPDATE \`value\`='admin@$HOSTNAME';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('default_address','admin@$HOSTNAME') ON DUPLICATE KEY UPDATE \`value\`='admin@$HOSTNAME';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('smtp_hostname','localhost') ON DUPLICATE KEY UPDATE \`value\`='localhost';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('smtp_disable_auth','') ON DUPLICATE KEY UPDATE \`value\`='';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('smtp_user','mailtrain') ON DUPLICATE KEY UPDATE \`value\`='mailtrain';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('smtp_pass','$SMTP_PASS') ON DUPLICATE KEY UPDATE \`value\`='$SMTP_PASS';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('smtp_encryption','NONE') ON DUPLICATE KEY UPDATE \`value\`='NONE';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('smtp_port','2525') ON DUPLICATE KEY UPDATE \`value\`='2525';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('default_homepage','http://$HOSTNAME/') ON DUPLICATE KEY UPDATE \`value\`='http://$HOSTNAME/';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('service_url','http://$HOSTNAME/') ON DUPLICATE KEY UPDATE \`value\`='http://$HOSTNAME/';
+INSERT INTO \`settings\` (\`key\`, \`value\`) VALUES ('dkim_api_key','$DKIM_API_KEY') ON DUPLICATE KEY UPDATE \`value\`='$DKIM_API_KEY';
+EOT
+
+# Add new user for the mailtrain daemon to run as
+useradd mailtrain || true
+useradd zone-mta || true
+
+# Setup installation configuration
+cat >> config/production.toml <<EOT
+user="mailtrain"
+group="mailtrain"
+rouser="nobody"
+rogroup="nobody"
+[log]
+level="error"
+[www]
+port=80
+secret="`pwgen -1`"
+[mysql]
+password="$MYSQL_PASSWORD"
+[redis]
+enabled=true
+[queue]
+processes=5
+[reports]
+enabled=true
+EOT
+
+cat >> workers/reports/config/production.toml <<EOT
+[log]
+level="error"
+[mysql]
+user="mailtrain_ro"
+password="$MYSQL_RO_PASSWORD"
+EOT
+
+# Install required node packages
+npm install --no-progress --production
+chown -R mailtrain:mailtrain .
+chmod o-rwx config
+
+# Setup log rotation to not spend up entire storage on logs
+cat <<EOM > /etc/logrotate.d/mailtrain
+/var/log/mailtrain.log {
+    daily
+    rotate 12
+    compress
+    delaycompress
+    missingok
+    notifempty
+    copytruncate
+    nomail
+}
+EOM
+
+# Set up systemd service script
+cp setup/mailtrain-centos7.service /etc/systemd/system/mailtrain.service
+systemctl enable mailtrain.service
+
+# Fetch ZoneMTA files
+mkdir -p /opt/zone-mta
+cd /opt/zone-mta
+git clone git://github.com/zone-eu/zone-mta.git .
+git checkout 6964091273
+
+# Ensure queue folder
+mkdir -p /var/data/zone-mta/mailtrain
+
+# Setup installation configuration
+cat >> config/production.json <<EOT
+{
+    "name": "Mailtrain",
+    "user": "zone-mta",
+    "group": "zone-mta",
+    "queue": {
+        "db": "/var/data/zone-mta/mailtrain"
+    },
+    "smtpInterfaces": {
+        "feeder": {
+            "enabled": true,
+            "port": 2525,
+            "processes": 2,
+            "authentication": true
+        }
+    },
+    "api": {
+        "maildrop": false,
+        "user": "mailtrain",
+        "pass": "$SMTP_PASS"
+    },
+    "log": {
+        "level": "info",
+        "syslog": true
+    },
+    "plugins": {
+        "core/email-bounce": false,
+        "core/http-bounce": {
+            "enabled": "main",
+            "url": "http://localhost/webhooks/zone-mta"
+        },
+        "core/http-auth": {
+            "enabled": ["receiver", "main"],
+            "url": "http://localhost:8080/test-auth"
+        },
+        "core/default-headers": {
+            "enabled": ["receiver", "main", "sender"],
+            "futureDate": false,
+            "xOriginatingIP": false
+        },
+        "core/http-config": {
+            "enabled": ["main", "receiver"],
+            "url": "http://localhost/webhooks/zone-mta/sender-config?api_token=$DKIM_API_KEY"
+        },
+        "core/rcpt-mx": false
+    },
+    "pools": {
+        "default": [{
+            "address": "0.0.0.0",
+            "name": "$HOSTNAME"
+        }]
+    },
+    "zones": {
+        "default": {
+            "processes": 3,
+            "connections": 5,
+            "throttling": false,
+            "pool": "default"
+        },
+        "transactional": {
+            "processes": 1,
+            "connections": 1,
+            "pool": "default"
+        }
+    },
+    "domainConfig": {
+        "default": {
+            "maxConnections": 4
+        }
+    }
+}
+EOT
+
+# Install required node packages
+npm install --no-progress --production
+npm install leveldown
+
+# Ensure queue folder is owned by MTA user
+chown -R zone-mta:zone-mta /var/data/zone-mta/mailtrain
+
+# Set up systemd service script
+cp setup/zone-mta.service /etc/systemd/system/
+systemctl enable zone-mta.service
+
+# Start the service
+systemctl daemon-reload
+
+systemctl start zone-mta.service
+systemctl start mailtrain.service
+
+echo "Success! Open http://$HOSTNAME/ and log in as admin:test";

setup/install.sh

@@ -12,7 +12,7 @@ set -e
 
 export DEBIAN_FRONTEND=noninteractive
 
-curl -sL https://deb.nodesource.com/setup_6.x | bash -
+curl -sL https://deb.nodesource.com/setup_7.x | bash -
 apt-get -q -y install mariadb-server pwgen nodejs imagemagick git ufw build-essential dnsutils python software-properties-common
 
 apt-add-repository -y ppa:chris-lea/redis-server
@@ -28,12 +28,15 @@ fi
 HOSTNAME="${HOSTNAME:-`hostname`}"
 
 MYSQL_PASSWORD=`pwgen 12 -1`
+MYSQL_RO_PASSWORD=`pwgen 12 -1`
 DKIM_API_KEY=`pwgen 12 -1`
 SMTP_PASS=`pwgen 12 -1`
 
 # Setup MySQL user for Mailtrain
 mysql -u root -e "CREATE USER 'mailtrain'@'localhost' IDENTIFIED BY '$MYSQL_PASSWORD';"
 mysql -u root -e "GRANT ALL PRIVILEGES ON mailtrain.* TO 'mailtrain'@'localhost';"
+mysql -u root -e "CREATE USER 'mailtrain_ro'@'localhost' IDENTIFIED BY '$MYSQL_RO_PASSWORD';"
+mysql -u root -e "GRANT SELECT ON mailtrain.* TO 'mailtrain_ro'@'localhost';"
 mysql -u mailtrain --password="$MYSQL_PASSWORD" -e "CREATE database mailtrain;"
 
 # Enable firewall, allow connections to SSH, HTTP, HTTPS and SMTP
@@ -87,9 +90,18 @@ enabled=true
 processes=5
 EOT
 
+cat >> workers/reports/config/production.toml <<EOT
+[log]
+level="error"
+[mysql]
+user="mailtrain_ro"
+password="$MYSQL_RO_PASSWORD"
+EOT
+
 # Install required node packages
 npm install --no-progress --production
 chown -R mailtrain:mailtrain .
+chmod o-rwx config
 
 # Setup log rotation to not spend up entire storage on logs
 cat <<EOM > /etc/logrotate.d/mailtrain

setup/mailtrain-centos7.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Mailtrain server
+Requires=mariadb.service
+After=syslog.target network.target
+
+[Service]
+Environment="NODE_ENV=production"
+WorkingDirectory=/opt/mailtrain
+ExecStart=/usr/bin/node index.js
+Type=simple
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target
+# Alias=mailtrain.service

setup/sql/drop.js

@@ -1,17 +1,23 @@
 'use strict';
 
-if (process.env.NODE_ENV === 'production') {
-    console.log('This script does not run in production'); // eslint-disable-line no-console
-    process.exit(1);
-}
-
 let config = require('config');
 let spawn = require('child_process').spawn;
 let log = require('npmlog');
 let path = require('path');
+let fs = require('fs');
 
 log.level = 'verbose';
 
+if (process.env.NODE_ENV === 'production') {
+    log.error('sqldrop', 'This script does not run in production');
+    process.exit(1);
+}
+
+if (process.env.NODE_ENV === 'test' && !fs.existsSync(path.join(__dirname, '..', '..', 'config', 'test.toml'))) {
+    log.error('sqldrop', 'This script only runs in test if config/test.toml (i.e. a dedicated test database) is present');
+    process.exit(1);
+}
+
 function createDump(callback) {
     let cmd = spawn(path.join(__dirname, 'drop.sh'), [], {
         env: {

setup/sql/init.js

@@ -1,15 +1,22 @@
 'use strict';
 
-if (process.env.NODE_ENV === 'production') {
-    console.log('This script does not run in production'); // eslint-disable-line no-console
-    process.exit(1);
-}
-
 let dbcheck = require('../../lib/dbcheck');
 let log = require('npmlog');
+let path = require('path');
+let fs = require('fs');
 
 log.level = 'verbose';
 
+if (process.env.NODE_ENV === 'production') {
+    log.error('sqlinit', 'This script does not run in production');
+    process.exit(1);
+}
+
+if (process.env.NODE_ENV === 'test' && !fs.existsSync(path.join(__dirname, '..', '..', 'config', 'test.toml'))) {
+    log.error('sqlinit', 'This script only runs in test if config/test.toml (i.e. a dedicated test database) is present');
+    process.exit(1);
+}
+
 dbcheck(err => {
     if (err) {
         log.error('DB', err);

setup/sql/upgrade-00026.sql

@@ -0,0 +1,11 @@
+# Header section
+# Define incrementing schema version number
+SET @schema_version = '26';
+
+# Add field
+ALTER TABLE `lists` ADD COLUMN `public_subscribe` tinyint(1) unsigned DEFAULT 1 NOT NULL AFTER `created`;
+
+# Footer section
+LOCK TABLES `settings` WRITE;
+INSERT INTO `settings` (`key`, `value`) VALUES('db_schema_version', @schema_version) ON DUPLICATE KEY UPDATE `value`=@schema_version;
+UNLOCK TABLES;

setup/sql/upgrade-00027.sql

@@ -0,0 +1,37 @@
+# Header section
+# Define incrementing schema version number
+SET @schema_version = '27';
+
+# Create table to report templates
+CREATE TABLE `report_templates` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+  `name` varchar(255) DEFAULT '',
+  `mime_type` varchar(255) DEFAULT 'text/html' NOT NULL,
+  `description` text,
+  `user_fields` longtext,
+  `js` longtext,
+  `hbs` longtext,
+  `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+# Create table to store reports
+CREATE TABLE `reports` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+  `name` varchar(255) DEFAULT '',
+  `description` text,
+  `report_template` int(11) unsigned NOT NULL,
+  `params` longtext,
+  `state`  int(11) unsigned NOT NULL DEFAULT 0,
+  `last_run` DATETIME DEFAULT NULL,
+  `created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `report_template` (`report_template`),
+  CONSTRAINT `report_template_ibfk_1` FOREIGN KEY (`report_template`) REFERENCES `report_templates` (`id`) ON DELETE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+
+# Footer section
+LOCK TABLES `settings` WRITE;
+INSERT INTO `settings` (`key`, `value`) VALUES('db_schema_version', @schema_version) ON DUPLICATE KEY UPDATE `value`=@schema_version;
+UNLOCK TABLES;

test/e2e/.eslintrc

@@ -0,0 +1,11 @@
+{
+  "parser": "babel-eslint",
+  "rules": {
+      "strict": 0,
+      "no-invalid-this": 0,
+      "no-unused-expressions": 0
+  },
+  "env": {
+      "mocha": true
+  }
+}

test/e2e/README.md

@@ -0,0 +1,44 @@
+# e2e Tests
+
+Running e2e tests requires Node 7.6 or later and a dedicated test database. It uses mocha, selenium-webdriver and phantomjs.
+
+## Installation
+
+These e2e tests have to be performed against predefined resources (e.g. lists, users, etc.) and therefore a dedicated test database and test config is required.
+
+Both can be created by running `sudo sh test/e2e/install.sh` from within your mailtrain directory. This creates a MYSQL user and database called `mailtrain_test`, and generates the required `config/test.toml`.
+
+## Running e2e Tests
+
+For tests to succeed Mailtrian must be started in `test` mode on port 3000 (as http://localhost:3000/ is the predefined service url). The tests itself have to be started in a second Terminal window.
+
+1. Start Mailtrain with `npm run starttest`
+2. Start e2e tests with `npm run e2e`
+
+## Using Different Browsers
+
+By default e2e tests use `phantomjs`. If you want to use a different browser you need to install its driver and adjust your `config/test.toml`.
+
+* Install the `firefox` driver with `npm install geckodriver`
+* Install the `chrome` driver with `npm install chromedriver`
+* Other drivers can be found [here](https://seleniumhq.github.io/selenium/docs/api/javascript/)
+
+Then adjust your config:
+
+```
+[seleniumwebdriver]
+browser="firefox"
+```
+
+Current Firefox issue (and patch): https://github.com/mozilla/geckodriver/issues/683
+
+## Writing e2e Tests
+
+You should spend your time on features rather than writing tests, yet in some cases, like for example the subscription process, manual testing is just silly. You best get started by reading the current test suites, or just open an issue describing the scenario you want to get tested.
+
+Available commands:
+
+* `npm run sqldumptest` - exports the test DB to `setup/sql/mailtrain-test.sql`
+* `npm run sqlresettest` - drops all tables then loads `setup/sql/mailtrain-test.sql`
+* `npm run _e2e` - just runs e2e tests
+* `npm run e2e` - runs `sqlresettest` then `_e2e`

test/e2e/helpers/config.js

@@ -0,0 +1,31 @@
+'use strict';
+
+const config = require('config');
+
+module.exports = {
+    app: config,
+    baseUrl: 'http://localhost:' + config.www.port,
+    users: {
+        admin: {
+            username: 'admin',
+            password: 'test'
+        }
+    },
+    lists: {
+        one: {
+            id: 1,
+            cid: 'Hkj1vCoJb',
+            publicSubscribe: 1,
+            unsubscriptionMode: 0
+        }
+    },
+    settings: {
+        'service-url' : 'http://localhost:' + config.www.port + '/',
+        'default-homepage': 'https://mailtrain.org',
+        'smtp-hostname': config.testserver.host,
+        'smtp-port': config.testserver.port,
+        'smtp-encryption': 'NONE',
+        'smtp-user': config.testserver.username,
+        'smtp-pass': config.testserver.password
+    }
+};

test/e2e/helpers/driver.js

@@ -0,0 +1,15 @@
+'use strict';
+
+const config = require('./config');
+const webdriver = require('selenium-webdriver');
+
+const driver = new webdriver.Builder()
+    .forBrowser(config.app.seleniumwebdriver.browser || 'phantomjs')
+    .build();
+
+if (global.USE_SHARED_DRIVER === true) {
+    driver.originalQuit = driver.quit;
+    driver.quit = () => {};
+}
+
+module.exports = driver;

test/e2e/helpers/exit-unless-test.js

@@ -0,0 +1,21 @@
+'use strict';
+
+const config = require('./config');
+const log = require('npmlog');
+const path = require('path');
+const fs = require('fs');
+
+if (process.env.NODE_ENV !== 'test' || !fs.existsSync(path.join(__dirname, '..', '..', '..', 'config', 'test.toml'))) {
+    log.error('e2e', 'This script only runs in test and config/test.toml (i.e. a dedicated test database) is present');
+    process.exit(1);
+}
+
+if (config.app.testserver.enabled !== true) {
+    log.error('e2e', 'This script only runs if the testserver is enabled. Check config/test.toml');
+    process.exit(1);
+}
+
+if (config.app.www.port !== 3000) {
+    log.error('e2e', 'This script requires Mailtrain to be running on port 3000. Check config/test.toml');
+    process.exit(1);
+}

test/e2e/index.js

@@ -0,0 +1,36 @@
+'use strict';
+
+require('./helpers/exit-unless-test');
+
+global.USE_SHARED_DRIVER = true;
+
+const driver = require('./helpers/driver');
+const only = 'only';
+const skip = 'skip';
+
+
+
+let tests = [
+    ['tests/login'],
+    ['tests/subscription']
+];
+
+
+
+tests = tests.filter(t => t[1] !== skip);
+
+if (tests.some(t => t[1] === only)) {
+    tests = tests.filter(t => t[1] === only);
+}
+
+describe('e2e', function() {
+    this.timeout(10000);
+
+    tests.forEach(t => {
+        describe(t[0], () => {
+            require('./' + t[0]); // eslint-disable-line global-require
+        });
+    });
+
+    after(() => driver.originalQuit());
+});

test/e2e/install.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# This installation script works on Ubuntu 14.04 and 16.04
+# Run as root!
+
+if [[ $EUID -ne 0 ]]; then
+   echo "This script must be run as root" 1>&2
+   exit 1
+fi
+
+set -e
+
+export DEBIAN_FRONTEND=noninteractive
+
+MYSQL_PASSWORD=`pwgen 12 -1`
+
+# Setup MySQL user for Mailtrain Tests
+mysql -u root -e "CREATE USER 'mailtrain_test'@'localhost' IDENTIFIED BY '$MYSQL_PASSWORD';"
+mysql -u root -e "GRANT ALL PRIVILEGES ON mailtrain_test.* TO 'mailtrain_test'@'localhost';"
+mysql -u mailtrain_test --password="$MYSQL_PASSWORD" -e "CREATE database mailtrain_test;"
+
+# Setup installation configuration
+cat >> config/test.toml <<EOT
+[www]
+port=3000
+[mysql]
+user="mailtrain_test"
+password="$MYSQL_PASSWORD"
+database="mailtrain_test"
+[testserver]
+enabled=true
+[seleniumwebdriver]
+browser="phantomjs"
+EOT
+
+echo "Success! The test database has been created.";

test/e2e/page-objects/flash.js

@@ -0,0 +1,21 @@
+'use strict';
+
+const page = require('./page');
+
+module.exports = driver => Object.assign(page(driver), {
+    elementToWaitFor: 'alert',
+    elements: {
+        alert: 'div.alert:not(.js-warning)'
+    },
+    getText() {
+        return this.element('alert').getText();
+    },
+    clear() {
+        return this.driver.executeScript(`
+            var elements = document.getElementsByClassName('alert');
+            while(elements.length > 0){
+                elements[0].parentNode.removeChild(elements[0]);
+            }
+        `);
+    }
+});

test/e2e/page-objects/home.js

@@ -0,0 +1,11 @@
+'use strict';
+
+const page = require('./page');
+
+module.exports = driver => Object.assign(page(driver), {
+    url: '/',
+    elementToWaitFor: 'body',
+    elements: {
+        body: 'body.page--home'
+    }
+});

test/e2e/page-objects/page.js

@@ -0,0 +1,55 @@
+'use strict';
+
+const config = require('../helpers/config');
+const webdriver = require('selenium-webdriver');
+const By = webdriver.By;
+const until = webdriver.until;
+
+module.exports = driver => ({
+    driver,
+    elements: {},
+
+    element(key) {
+        return this.driver.findElement(By.css(this.elements[key] || key));
+    },
+
+    navigate(path) {
+        this.driver.navigate().to(config.baseUrl + (path || this.url));
+        return this.waitUntilVisible();
+    },
+
+    waitUntilVisible() {
+        let selector = this.elements[this.elementToWaitFor];
+        if (!selector && this.url) {
+            selector = 'body.page--' + (this.url.substring(1).replace(/\//g, '--') || 'home');
+        }
+        return selector ? this.driver.wait(until.elementLocated(By.css(selector))) : this.driver.sleep(1000);
+    },
+
+    submit() {
+        return this.element('submitButton').click();
+    },
+
+    click(key) {
+        return this.element(key).click();
+    },
+
+    getText(key) {
+        return this.element(key).getText();
+    },
+
+    getValue(key) {
+        return this.element(key).getAttribute('value');
+    },
+
+    setValue(key, value) {
+        return this.element(key).sendKeys(value);
+    },
+
+    containsText(str) {
+        // let text = await driver.findElement({ css: 'body' }).getText();
+        return this.driver.executeScript(`
+            return (document.documentElement.textContent || document.documentElement.innerText).indexOf('${str}') > -1;
+        `);
+    }
+});

test/e2e/page-objects/subscription.js

@@ -0,0 +1,84 @@
+'use strict';
+
+const config = require('../helpers/config');
+const page = require('./page');
+
+const web = {
+    enterEmail(value) {
+        this.element('emailInput').clear();
+        return this.element('emailInput').sendKeys(value);
+    }
+};
+
+const mail = {
+    navigate(address) {
+        this.driver.sleep(100);
+        this.driver.navigate().to(`http://localhost:${config.app.testserver.mailboxserverport}/${address}`);
+        return this.waitUntilVisible();
+    }
+};
+
+module.exports = (driver, list) => ({
+
+    webSubscribe: Object.assign(page(driver), web, {
+        url: `/subscription/${list.cid}`,
+        elementToWaitFor: 'form',
+        elements: {
+            form: `form[action="/subscription/${list.cid}/subscribe"]`,
+            emailInput: '#main-form input[name="email"]',
+            submitButton: 'a[href="#submit"]'
+        }
+    }),
+
+    webConfirmSubscriptionNotice: Object.assign(page(driver), web, {
+        url: `/subscription/${list.cid}/confirm-notice`,
+        elementToWaitFor: 'homepageButton',
+        elements: {
+            homepageButton: `a[href="${config.settings['default-homepage']}"]`
+        }
+    }),
+
+    mailConfirmSubscription: Object.assign(page(driver), mail, {
+        elementToWaitFor: 'confirmLink',
+        elements: {
+            confirmLink: `a[href^="${config.settings['service-url']}subscription/subscribe/"]`
+        }
+    }),
+
+    webSubscribedNotice: Object.assign(page(driver), web, {
+        elementToWaitFor: 'homepageButton',
+        elements: {
+            homepageButton: 'a[href^="https://mailtrain.org"]'
+        }
+    }),
+
+    mailSubscriptionConfirmed: Object.assign(page(driver), mail, {
+        elementToWaitFor: 'unsubscribeLink',
+        elements: {
+            unsubscribeLink: 'a[href*="/unsubscribe/"]',
+            manageLink: 'a[href*="/manage/"]'
+        }
+    }),
+
+    webUnsubscribe: Object.assign(page(driver), web, {
+        elementToWaitFor: 'submitButton',
+        elements: {
+            submitButton: 'a[href="#submit"]'
+        }
+    }),
+
+    webUnsubscribedNotice: Object.assign(page(driver), web, {
+        elementToWaitFor: 'homepageButton',
+        elements: {
+            homepageButton: 'a[href^="https://mailtrain.org"]'
+        }
+    }),
+
+    mailUnsubscriptionConfirmed: Object.assign(page(driver), mail, {
+        elementToWaitFor: 'resubscribeLink',
+        elements: {
+            resubscribeLink: `a[href^="${config.settings['service-url']}subscription/${list.cid}"]`
+        }
+    })
+
+});

test/e2e/page-objects/users.js

@@ -0,0 +1,32 @@
+'use strict';
+
+const page = require('./page');
+
+module.exports = driver => ({
+
+    login: Object.assign(page(driver), {
+        url: '/users/login',
+        elementToWaitFor: 'submitButton',
+        elements: {
+            usernameInput: 'form[action="/users/login"] input[name="username"]',
+            passwordInput: 'form[action="/users/login"] input[name="password"]',
+            submitButton: 'form[action="/users/login"] [type=submit]'
+        },
+        enterUsername(value) {
+            // this.element('usernameInput').clear();
+            return this.element('usernameInput').sendKeys(value);
+        },
+        enterPassword(value) {
+            return this.element('passwordInput').sendKeys(value);
+        }
+    }),
+
+    account: Object.assign(page(driver), {
+        url: '/users/account',
+        elementToWaitFor: 'emailInput',
+        elements: {
+            emailInput: 'form[action="/users/account"] input[name="email"]'
+        }
+    })
+
+});

test/e2e/tests/login.js

@@ -0,0 +1,57 @@
+'use strict';
+
+const config = require('../helpers/config');
+const expect = require('chai').expect;
+const driver = require('../helpers/driver');
+const home = require('../page-objects/home')(driver);
+const flash = require('../page-objects/flash')(driver);
+const {
+    login,
+    account
+} = require('../page-objects/users')(driver);
+
+describe('login', function() {
+    this.timeout(10000);
+
+    before(() => driver.manage().deleteAllCookies());
+
+    it('can access home page', async () => {
+        await home.navigate();
+    });
+
+    it('can not access restricted content', async () => {
+        driver.navigate().to(config.baseUrl + '/settings');
+        flash.waitUntilVisible();
+        expect(await flash.getText()).to.contain('Need to be logged in to access restricted content');
+        await flash.clear();
+    });
+
+    it('can not login with false credentials', async () => {
+        login.enterUsername(config.users.admin.username);
+        login.enterPassword('invalid');
+        login.submit();
+        flash.waitUntilVisible();
+        expect(await flash.getText()).to.contain('Incorrect username or password');
+        await flash.clear();
+    });
+
+    it('can login as admin', async () => {
+        login.enterUsername(config.users.admin.username);
+        login.enterPassword(config.users.admin.password);
+        login.submit();
+        flash.waitUntilVisible();
+        expect(await flash.getText()).to.contain('Logged in as admin');
+    });
+
+    it('can access account page as admin', async () => {
+        await account.navigate();
+    });
+
+    it('can logout', async () => {
+        driver.navigate().to(config.baseUrl + '/users/logout');
+        flash.waitUntilVisible();
+        expect(await flash.getText()).to.contain('logged out');
+    });
+
+    after(() => driver.quit());
+});

test/e2e/tests/subscription.js

@@ -0,0 +1,101 @@
+'use strict';
+
+const config = require('../helpers/config');
+const shortid = require('shortid');
+const expect = require('chai').expect;
+const driver = require('../helpers/driver');
+
+const page = require('../page-objects/page')(driver);
+const flash = require('../page-objects/flash')(driver);
+
+const {
+    webSubscribe,
+    webConfirmSubscriptionNotice,
+    mailConfirmSubscription,
+    webSubscribedNotice,
+    mailSubscriptionConfirmed,
+    webUnsubscribe,
+    webUnsubscribedNotice,
+    mailUnsubscriptionConfirmed
+} = require('../page-objects/subscription')(driver, config.lists.one);
+
+const testuser = {
+    email: 'keep.' + shortid.generate() + '@mailtrain.org'
+};
+
+// console.log(testuser.email);
+
+describe('subscribe (list one)', function() {
+    this.timeout(10000);
+
+    before(() => driver.manage().deleteAllCookies());
+
+    it('visits web-subscribe', async () => {
+        await webSubscribe.navigate();
+    });
+
+    it('submits invalid email (error)', async () => {
+        webSubscribe.enterEmail('foo@bar.nope');
+        webSubscribe.submit();
+        flash.waitUntilVisible();
+        expect(await flash.getText()).to.contain('Invalid email address');
+    });
+
+    it('submits valid email', async () => {
+        webSubscribe.enterEmail(testuser.email);
+        await webSubscribe.submit();
+    });
+
+    it('sees web-confirm-subscription-notice', async () => {
+        webConfirmSubscriptionNotice.waitUntilVisible();
+        expect(await page.containsText('Almost Finished')).to.be.true;
+    });
+
+    it('receives mail-confirm-subscription', async () => {
+        mailConfirmSubscription.navigate(testuser.email);
+        expect(await page.containsText('Please Confirm Subscription')).to.be.true;
+    });
+
+    it('clicks confirm subscription', async () => {
+        await mailConfirmSubscription.click('confirmLink');
+    });
+
+    it('sees web-subscribed-notice', async () => {
+        webSubscribedNotice.waitUntilVisible();
+        expect(await page.containsText('Subscription Confirmed')).to.be.true;
+    });
+
+    it('receives mail-subscription-confirmed', async () => {
+        mailSubscriptionConfirmed.navigate(testuser.email);
+        expect(await page.containsText('Subscription Confirmed')).to.be.true;
+    });
+});
+
+describe('unsubscribe (list one)', function() {
+    this.timeout(10000);
+
+    it('clicks unsubscribe', async () => {
+        await mailSubscriptionConfirmed.click('unsubscribeLink');
+    });
+
+    it('sees web-unsubscribe', async () => {
+        webUnsubscribe.waitUntilVisible();
+        expect(await page.containsText('Unsubscribe')).to.be.true;
+    });
+
+    it('clicks confirm unsubscription', async () => {
+        await webUnsubscribe.submit();
+    });
+
+    it('sees web-unsubscribed-notice', async () => {
+        webUnsubscribedNotice.waitUntilVisible();
+        expect(await page.containsText('Unsubscribe Successful')).to.be.true;
+    });
+
+    it('receives mail-unsubscription-confirmed', async () => {
+        mailUnsubscriptionConfirmed.navigate(testuser.email);
+        expect(await page.containsText('You Are Now Unsubscribed')).to.be.true;
+    });
+
+    after(() => driver.quit());
+});

views/campaigns/campaigns.hbs

@@ -23,7 +23,7 @@
 <div class="table-responsive">
     <table data-topic-url="/campaigns" data-sort-column="4" data-sort-order="desc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,0,1,1,0">
         <thead>
-            <th class="col-md-1">
+            <th style="width: 1%">
                 #
             </th>
             <th>
@@ -38,7 +38,7 @@
             <th>
                 {{#translate}}Created{{/translate}}
             </th>
-            <th class="col-md-1">
+            <th style="width: 1%">
                 &nbsp;
             </th>
         </thead>

views/campaigns/edit-rss.hbs

@@ -71,8 +71,6 @@
             </div>
         </div>
 
-        {{> merge_tag_reference}}
-
         <div class="form-group">
             <label for="template" class="col-sm-2 control-label">{{#translate}}RSS Feed Url{{/translate}}</label>
             <div class="col-sm-10">
@@ -81,6 +79,8 @@
             </div>
         </div>
 
+        {{> merge_tag_reference}}
+
         {{#if disableWysiwyg}}
 {{> codeeditor}}
         {{else}}

views/layout.hbs

@@ -35,7 +35,7 @@
 
 </head>
 
-<body class="{{#if user}}logged-in user-{{user.username}}{{/if}}">
+<body class="{{bodyClass}}">
 
     <nav class="navbar navbar-default navbar-static-top">
         <div class="container">

views/lists/create.hbs

@@ -26,6 +26,16 @@
 
     <hr />
 
+    <div class="col-sm-offset-2">
+        <div class="checkbox">
+            <label>
+                <input type="checkbox" name="public_subscribe" value="1" {{#if publicSubscribe}} checked {{/if}}> {{#translate}}Allow public users to subscribe themselves{{/translate}}
+            </label>
+        </div>
+    </div>
+
+    <hr />
+
     <div class="form-group">
         <div class="col-sm-offset-2 col-sm-10">
             <button type="submit" class="btn btn-primary"><i class="glyphicon glyphicon-plus"></i> {{#translate}}Create List{{/translate}}</button>

views/lists/edit.hbs

@@ -56,6 +56,16 @@
 
     <hr />
 
+    <div class="col-sm-offset-2">
+        <div class="checkbox">
+            <label>
+                <input type="checkbox" name="public_subscribe" value="1" {{#if publicSubscribe}} checked {{/if}}> {{#translate}}Allow public users to subscribe themselves{{/translate}}
+            </label>
+        </div>
+    </div>
+
+    <hr />
+
     <div class="form-group">
         <div class="col-sm-offset-2 col-sm-10">
             <div class="pull-right">

views/lists/lists.hbs

@@ -12,57 +12,26 @@
 <hr>
 
 <div class="table-responsive">
-    <table class="table table-bordered table-hover data-table display nowrap" width="100%" data-row-sort="0,1,1,1,0,0">
+    <table data-topic-url="/lists" data-sort-column="1" data-sort-order="asc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,1,1,0,0">
         <thead>
-            <th>
+        <th style="width: 1%">
-                #
+            #
-            </th>
+        </th>
-            <th>
+        <th>
-                {{#translate}}Name{{/translate}}
+            {{#translate}}Name{{/translate}}
-            </th>
+        </th>
-            <th>
+        <th>
-                {{#translate}}ID{{/translate}}
+            {{#translate}}ID{{/translate}}
-            </th>
+        </th>
-            <th>
+        <th>
-                {{#translate}}Subscribers{{/translate}}
+            {{#translate}}Subscribers{{/translate}}
-            </th>
+        </th>
-            <th>
+        <th>
-                {{#translate}}Description{{/translate}}
+            {{#translate}}Description{{/translate}}
-            </th>
+        </th>
-            <th>
+        <th style="width: 1%">
-                &nbsp;
+            &nbsp;
-            </th>
+        </th>
         </thead>
-        {{#if rows}}
-            <tbody>
-                {{#each rows}}
-                    <tr>
-                        <th scope="row">
-                            {{index}}
-                        </th>
-                        <td style="width: 30%;">
-                            <span class="glyphicon glyphicon-list-alt" aria-hidden="true"></span>
-                            <a href="/lists/view/{{id}}">
-                            {{name}}
-                        </a>
-                        </td>
-                        <td>
-                            <code>{{cid}}</code>
-                        </td>
-                        <td class="text-center">
-                            {{subscribers}}
-                        </td>
-                        <td class="text-muted" style="width: 70%;">
-                            {{description}}
-                        </td>
-                        <td>
-                            <a href="/lists/edit/{{id}}">
-                            {{#translate}}Edit{{/translate}}
-                        </a>
-                        </td>
-                    </tr>
-                {{/each}}
-            </tbody>
-        {{/if}}
     </table>
 </div>

views/report-templates/create.hbs

@@ -0,0 +1,24 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li><a href="/reports">{{#translate}}Reports{{/translate}}</a></li>
+    <li><a href="/report-templates">{{#translate}}Templates{{/translate}}</a></li>
+    <li class="active">{{#translate}}Create Template{{/translate}}</li>
+</ol>
+
+<h2>{{#translate}}Create Report Template{{/translate}}</h2>
+
+<hr>
+
+<form class="form-horizontal" method="post" action="/report-templates/create">
+    <input type="hidden" name="_csrf" value="{{csrfToken}}">
+
+{{> report_template_fields }}
+
+    <hr />
+
+    <div class="form-group">
+        <div class="col-sm-offset-2 col-sm-10">
+            <button type="submit" class="btn btn-primary"><i class="glyphicon glyphicon-plus"></i> {{#translate}}Create Template{{/translate}}</button>
+        </div>
+    </div>
+</form>

views/report-templates/edit.hbs

@@ -0,0 +1,36 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li><a href="/reports">{{#translate}}Reports{{/translate}}</a></li>
+    <li><a href="/report-templates">{{#translate}}Templates{{/translate}}</a></li>
+    <li class="active">{{#translate}}Edit Template{{/translate}}</li>
+</ol>
+
+<h2>{{#translate}}Edit Report Template{{/translate}}</h2>
+
+<hr>
+
+<form method="post" class="delete-form" id="report-templates-delete" action="/report-templates/delete">
+    <input type="hidden" name="_csrf" value="{{csrfToken}}">
+    <input type="hidden" name="id" value="{{id}}" />
+</form>
+
+
+<form class="form-horizontal" method="post" action="/report-templates/edit">
+    <input type="hidden" name="_csrf" value="{{csrfToken}}">
+    <input type="hidden" name="id" value="{{id}}" />
+
+{{> report_template_fields }}
+
+    <hr />
+
+    <div class="form-group">
+        <div class="col-sm-offset-2 col-sm-10">
+            <div class="pull-right">
+                <button type="submit" form="report-templates-delete" class="btn btn-danger"><i class="glyphicon glyphicon-remove"></i> {{#translate}}Delete Template{{/translate}}</button>
+            </div>
+            <button type="submit" name="submit" value="update-and-stay"  class="btn btn-primary"><i class="glyphicon glyphicon-ok"></i> {{#translate}}Update and Stay{{/translate}}</button>
+            <button type="submit" name="submit" value="update-and-leave" class="btn btn-primary"><i class="glyphicon glyphicon-ok"></i> {{#translate}}Update and Leave{{/translate}}</button>
+        </div>
+    </div>
+
+</form>

views/report-templates/partials/report-template-fields.hbs

@@ -0,0 +1,59 @@
+    <div class="form-group">
+        <label for="name" class="col-sm-2 control-label">{{#translate}}Name{{/translate}}</label>
+        <div class="col-sm-10">
+            <input type="text" class="form-control input-lg" name="name" id="name" value="{{name}}" placeholder="{{#translate}}Template Name{{/translate}}" autofocus required>
+        </div>
+    </div>
+
+    <div class="form-group">
+        <label for="description" class="col-sm-2 control-label">{{#translate}}Description{{/translate}}</label>
+        <div class="col-sm-10">
+            <textarea class="form-control" rows="3" name="description" id="description">{{description}}</textarea>
+            <span class="help-block">{{#translate}}HTML is allowed{{/translate}}</span>
+        </div>
+    </div>
+
+    <div class="form-group">
+        <label for="mimeType" class="col-sm-2 control-label">{{#translate}}Type{{/translate}}</label>
+        <div class="col-sm-10">
+            <select name="mimeType" class="form-control">
+                {{#each mimeTypes}}
+                    <option value="{{key}}" {{#if selected}} selected {{/if}}>{{value}}</option>
+                {{/each}}
+            </select>
+        </div>
+    </div>
+
+    <div class="form-group">
+        <label class="col-sm-2 control-label">{{#translate}}User selectable fields{{/translate}}</label>
+        <div class="col-sm-offset-2 col-sm-10">
+            <div class="help-block" style="margin-top: -8px;">
+                <small>JSON specification of user selectable fields.</small>
+            </div>
+            <div class="code-editor-json" style="height: 250px; border: 1px solid #ccc;"></div>
+            <input type="hidden" name="userFields" value="{{userFields}}">
+        </div>
+    </div>
+
+    <div class="form-group">
+        <label class="col-sm-2 control-label">{{#translate}}Data processing code{{/translate}}</label>
+        <div class="col-sm-offset-2 col-sm-10">
+            <div class="help-block" style="margin-top: -8px;">
+                <small>Write the body of the JavaScript function with signature <code>function(inputs, callback)</code> that returns an object to be rendered by the Handlebars template below.</small>
+            </div>
+            <div class="code-editor-javascript" style="height: 700px; border: 1px solid #ccc;"></div>
+            <input type="hidden" name="js" value="{{js}}">
+        </div>
+    </div>
+
+    <div class="form-group">
+        <label class="col-sm-2 control-label">{{#translate}}Rendering template{{/translate}}</label>
+        <div class="col-sm-offset-2 col-sm-10">
+            <div class="help-block" style="margin-top: -8px;">
+                <small>Use HTML with Handlebars syntax. See documentation <a href="http://handlebarsjs.com/">here</a>.</small>
+            </div>
+            <div class="code-editor-handlebars" style="height: 700px; border: 1px solid #ccc;"></div>
+            <input type="hidden" name="hbs" value="{{hbs}}">
+        </div>
+    </div>
+

views/report-templates/report-templates.hbs

@@ -0,0 +1,45 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li><a href="/reports">{{#translate}}Reports{{/translate}}</a></li>
+    <li class="active">{{#translate}}Templates{{/translate}}</li>
+</ol>
+
+<div class="pull-right">
+    <div class="btn-group">
+        <button type="button" class="btn btn-primary dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+            {{#translate}}Create Template{{/translate}} <span class="caret"></span>
+        </button>
+        <ul class="dropdown-menu">
+            <li><a href="/report-templates/create">{{#translate}}Blank{{/translate}}</a></li>
+            <li><a href="/report-templates/create?type=subscribers-all">{{#translate}}All Subscribers{{/translate}}</a></li>
+            <li><a href="/report-templates/create?type=subscribers-grouped">{{#translate}}Grouped Subscribers{{/translate}}</a></li>
+            <li><a href="/report-templates/create?type=export-list-csv">{{#translate}}Export List as CSV{{/translate}}</a></li>
+        </ul>
+    </div>
+</div>
+
+<h2>{{#translate}}Report Templates{{/translate}}</h2>
+
+<hr>
+
+<div class="table-responsive">
+    <table data-topic-url="/report-templates" data-sort-column="2" data-sort-order="desc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,0,1,0">
+        <thead>
+            <th class="col-md-1">
+                #
+            </th>
+            <th>
+                {{#translate}}Name{{/translate}}
+            </th>
+            <th>
+                {{#translate}}Description{{/translate}}
+            </th>
+            <th>
+                {{#translate}}Created{{/translate}}
+            </th>
+            <th class="col-md-1">
+                &nbsp;
+            </th>
+        </thead>
+    </table>
+</div>

views/reports/create-select-template.hbs

@@ -0,0 +1,22 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li><a href="/reports">{{#translate}}Reports{{/translate}}</a></li>
+    <li class="active">{{#translate}}Create Report{{/translate}}</li>
+</ol>
+
+<h2>{{#translate}}Create Report{{/translate}}</h2>
+
+<hr>
+
+<form class="form-horizontal" method="get" action="/reports/create">
+
+{{> report_select_template }}
+
+    <hr />
+
+    <div class="form-group">
+        <div class="col-sm-offset-2 col-sm-10">
+            <button type="submit" class="btn btn-primary"><i class="glyphicon glyphicon-chevron-right"></i> {{#translate}}Next{{/translate}}</button>
+        </div>
+    </div>
+</form>

views/reports/create.hbs

@@ -0,0 +1,23 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li><a href="/reports">{{#translate}}Reports{{/translate}}</a></li>
+    <li class="active">{{#translate}}Create Report{{/translate}}</li>
+</ol>
+
+<h2>{{#translate}}Create Report{{/translate}}</h2>
+
+<hr>
+
+<form class="form-horizontal" method="post" action="/reports/create">
+    <input type="hidden" name="_csrf" value="{{csrfToken}}">
+
+{{> report_fields }}
+
+    <hr />
+
+    <div class="form-group">
+        <div class="col-sm-offset-2 col-sm-10">
+            <button type="submit" class="btn btn-primary"><i class="glyphicon glyphicon-plus"></i> {{#translate}}Create Report{{/translate}}</button>
+        </div>
+    </div>
+</form>

views/reports/edit.hbs

@@ -0,0 +1,34 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li><a href="/reports">{{#translate}}Reports{{/translate}}</a></li>
+    <li class="active">{{#translate}}Edit Report{{/translate}}</li>
+</ol>
+
+<h2>{{#translate}}Edit Report{{/translate}}</h2>
+
+<hr>
+
+<form method="post" class="delete-form" id="reports-delete" action="/reports/delete">
+    <input type="hidden" name="_csrf" value="{{csrfToken}}">
+    <input type="hidden" name="id" value="{{id}}" />
+</form>
+
+
+<form class="form-horizontal" method="post" action="/reports/edit">
+    <input type="hidden" name="_csrf" value="{{csrfToken}}">
+    <input type="hidden" name="id" value="{{id}}" />
+
+{{> report_fields }}
+
+    <hr />
+
+    <div class="form-group">
+        <div class="col-sm-offset-2 col-sm-10">
+            <div class="pull-right">
+                <button type="submit" form="reports-delete" class="btn btn-danger"><i class="glyphicon glyphicon-remove"></i> {{#translate}}Delete Report{{/translate}}</button>
+            </div>
+            <button type="submit" class="btn btn-primary"><i class="glyphicon glyphicon-ok"></i> {{#translate}}Update{{/translate}}</button>
+        </div>
+    </div>
+
+</form>

views/reports/output.hbs

@@ -0,0 +1,8 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li><a href="/reports/">{{#translate}}Reports{{/translate}}</a></li>
+    <li class="active">{{title}}</li>
+</ol>
+
+<i>{{error}}</i>
+<pre>{{output}}</pre>

views/reports/partials/report-fields.hbs

@@ -0,0 +1,73 @@
+    {{> report_select_template options="readonly" }}
+
+    <div class="form-group">
+        <label for="name" class="col-sm-2 control-label">{{#translate}}Name{{/translate}}</label>
+        <div class="col-sm-10">
+            <input type="text" class="form-control input-lg" name="name" id="name" value="{{name}}" placeholder="{{#translate}}Report Name{{/translate}}" autofocus required>
+        </div>
+    </div>
+
+    <div class="form-group">
+        <label for="description" class="col-sm-2 control-label">{{#translate}}Description{{/translate}}</label>
+        <div class="col-sm-10">
+            <textarea class="form-control" rows="3" name="description" id="description">{{description}}</textarea>
+            <span class="help-block">{{#translate}}HTML is allowed{{/translate}}</span>
+        </div>
+    </div>
+
+    {{#each userFields}}
+        {{#switch type}}
+            {{#case "campaign"}}
+                <div class="form-group">
+                    <label for="description" class="col-sm-2 control-label">{{name}}</label>
+                    <div class="col-sm-10">
+                        <div class="table-responsive">
+                            <table data-topic-url="/campaigns/quicklist" data-sort-column="2" data-sort-order="desc" class="table table-bordered table-hover data-table-ajax data-table-{{#if isMulti}}multi{{/if}}selectable display nowrap" width="100%" data-row-sort="0,1,0,1">
+                                <thead>
+                                <th class="col-md-1">
+                                    #
+                                </th>
+                                <th>
+                                    {{#translate}}Name{{/translate}}
+                                </th>
+                                <th>
+                                    {{#translate}}Description{{/translate}}
+                                </th>
+                                <th>
+                                    {{#translate}}Created{{/translate}}
+                                </th>
+                                </thead>
+                            </table>
+                            <input type="hidden" name="{{id}}Selection" value="{{value}}" />
+                        </div>
+                        <span class="help-block">{{#translate}}Select a campaign in the table above by clicking on the respective row number.{{/translate}}</span>
+                    </div>
+                </div>
+            {{/case}}
+            {{#case "list"}}
+                <div class="form-group">
+                    <label for="description" class="col-sm-2 control-label">{{name}}</label>
+                    <div class="col-sm-10">
+                        <div class="table-responsive">
+                            <table data-topic-url="/lists/quicklist" data-sort-column="2" data-sort-order="asc" class="table table-bordered table-hover data-table-ajax data-table-{{#if isMulti}}multi{{/if}}selectable display nowrap" width="100%" data-row-sort="0,1,1">
+                                <thead>
+                                <th class="col-md-1">
+                                    #
+                                </th>
+                                <th>
+                                    {{#translate}}Name{{/translate}}
+                                </th>
+                                <th>
+                                    {{#translate}}Subscribers{{/translate}}
+                                </th>
+                                </thead>
+                            </table>
+                            <input type="hidden" name="{{id}}Selection" value="{{value}}" />
+                        </div>
+                        <span class="help-block">{{#translate}}Select a campaign in the table above by clicking on the respective row number.{{/translate}}</span>
+                    </div>
+                </div>
+            {{/case}}
+        {{/switch}}
+    {{/each}}
+

views/reports/partials/report-select-template.hbs

@@ -0,0 +1,11 @@
+    <div class="form-group">
+        <label for="name" class="col-sm-2 control-label">{{#translate}}Report Template{{/translate}}</label>
+        <div class="col-sm-10">
+            <select class="form-control" id="reportTemplate" name="reportTemplate" required {{options}}>
+                <option value=""> –– {{#translate}}Select{{/translate}} –– </option>
+                {{#each reportTemplates}}
+                    <option value="{{id}}" {{#if selected}} selected {{/if}}>{{name}}</option>
+                {{/each}}
+            </select>
+        </div>
+    </div>

views/reports/reports.hbs

@@ -0,0 +1,40 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li class="active">{{#translate}}Reports{{/translate}}</li>
+</ol>
+
+<div class="pull-right">
+    <a class="btn btn-primary" href="/reports/create" role="button"><i class="glyphicon glyphicon-plus"></i> {{#translate}}Create Report{{/translate}}</a>
+
+    <a class="btn btn-primary" href="/report-templates" role="button">{{#translate}}Report Templates{{/translate}}</a>
+</div>
+
+<h2>{{#translate}}Reports{{/translate}}</h2>
+
+<hr>
+
+<div class="table-responsive">
+    <table data-topic-url="/reports" data-sort-column="2" data-sort-order="desc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,1,0,1,0">
+        <thead>
+        <th style="width: 1%">
+            #
+        </th>
+        <th>
+            {{#translate}}Name{{/translate}}
+        </th>
+        <th>
+            {{#translate}}Template{{/translate}}
+        </th>
+        <th>
+            {{#translate}}Description{{/translate}}
+        </th>
+        <th>
+            {{#translate}}Created{{/translate}}
+        </th>
+        <th style="width: 1%">
+            &nbsp;
+        </th>
+        </thead>
+    </table>
+</div>
+

views/reports/view.hbs

@@ -0,0 +1,7 @@
+<ol class="breadcrumb">
+    <li><a href="/">{{#translate}}Home{{/translate}}</a></li>
+    <li><a href="/reports/">{{#translate}}Reports{{/translate}}</a></li>
+    <li class="active">{{title}}</li>
+</ol>
+
+{{report}}

views/templates/templates.hbs

@@ -12,9 +12,9 @@
 <hr>
 
 <div class="table-responsive">
-    <table class="table table-bordered table-hover data-table display nowrap" width="100%" data-row-sort="0,1,0,0">
+    <table data-topic-url="/templates" data-sort-column="1" data-sort-order="asc" class="table table-bordered table-hover data-table-ajax display nowrap" width="100%" data-row-sort="0,1,0,0">
         <thead>
-            <th class="col-md-1">
+            <th style="width: 1%">
                 #
             </th>
             <th>
@@ -23,32 +23,9 @@
             <th>
                 {{#translate}}Description{{/translate}}
             </th>
-            <th class="col-md-1">
+            <th style="width: 1%">
                 &nbsp;
             </th>
         </thead>
-        {{#if rows}}
-            <tbody>
-                {{#each rows}}
-                    <tr>
-                        <th scope="row">
-                            {{index}}
-                        </th>
-                        <td>
-                            <span class="glyphicon glyphicon-file" aria-hidden="true"></span> {{name}}
-                        </td>
-                        <td>
-                            <p class="text-muted">{{description}}</p>
-                        </td>
-                        <td>
-                            <span class="glyphicon glyphicon-wrench" aria-hidden="true"></span>
-                            <a href="/templates/edit/{{id}}">
-                            {{#translate}}Edit{{/translate}}
-                        </a>
-                        </td>
-                    </tr>
-                {{/each}}
-            </tbody>
-        {{/if}}
     </table>
 </div>

workers/reports/config/default.toml

@@ -0,0 +1,18 @@
+# Process title visible in monitoring logs and process listing
+title="mailtrain"
+
+# Default language to use
+language="en"
+
+[log]
+# silly|verbose|info|http|warn|error|silent
+level="verbose"
+
+[mysql]
+host="localhost"
+user="mailtrain"
+password="mailtrain"
+database="mailtrain"
+port=3306
+charset="utf8mb4"
+timezone="local"

workers/reports/report-processor.js

@@ -0,0 +1,147 @@
+'use strict';
+
+const reports = require('../../lib/models/reports');
+const reportTemplates = require('../../lib/models/report-templates');
+const lists = require('../../lib/models/lists');
+const subscriptions = require('../../lib/models/subscriptions');
+const campaigns = require('../../lib/models/campaigns');
+const handlebars = require('handlebars');
+const handlebarsHelpers = require('../../lib/handlebars-helpers');
+const _ = require('../../lib/translate')._;
+const hbs = require('hbs');
+const vm = require('vm');
+const log = require('npmlog');
+const fs = require('fs');
+
+handlebarsHelpers.registerHelpers(handlebars);
+
+let reportId = Number(process.argv[2]);
+let reportDir;
+
+function resolveEntities(getter, ids, callback) {
+    const idsRemaining = ids.slice();
+    const resolved = [];
+
+    function doWork() {
+        if (idsRemaining.length == 0) {
+            return callback(null, resolved);
+        }
+
+        getter(idsRemaining.shift(), (err, entity) => {
+            if (err) {
+                return callback(err);
+            }
+
+            resolved.push(entity);
+            return doWork();
+        });
+    }
+
+    setImmediate(doWork);
+}
+
+const userFieldTypeToGetter = {
+    'campaign': (id, callback) => campaigns.get(id, false, callback),
+    'list': lists.get
+};
+
+function resolveUserFields(userFields, params, callback) {
+    const userFieldsRemaining = userFields.slice();
+    const resolved = {};
+
+    function doWork() {
+        if (userFieldsRemaining.length == 0) {
+            return callback(null, resolved);
+        }
+
+        const spec = userFieldsRemaining.shift();
+        const getter = userFieldTypeToGetter[spec.type];
+
+        if (getter) {
+            return resolveEntities(getter, params[spec.id], (err, entities) => {
+                if (spec.minOccurences == 1 && spec.maxOccurences == 1) {
+                    resolved[spec.id] = entities[0];
+                } else {
+                    resolved[spec.id] = entities;
+                }
+
+                doWork();
+            });
+        } else {
+            return callback(new Error(_('Unknown user field type "' + spec.type + '".')));
+        }
+    }
+
+    setImmediate(doWork);
+}
+
+function doneSuccess() {
+    process.exit(0);
+}
+
+function doneFail() {
+    process.exit(1)
+}
+
+
+
+reports.get(reportId, (err, report) => {
+    if (err || !report) {
+        log.error('reports', err && err.message || err || _('Could not find report with specified ID'));
+        doneFail();
+    }
+
+    reportTemplates.get(report.reportTemplate, (err, reportTemplate) => {
+        if (err) {
+            log.error('reports', err && err.message || err || _('Could not find report template'));
+            doneFail();
+        }
+
+        resolveUserFields(reportTemplate.userFieldsObject, report.paramsObject, (err, inputs) => {
+            if (err) {
+                log.error('reports', err.message || err);
+                doneFail();
+            }
+
+            const campaignsProxy = {
+                results: reports.getCampaignResults,
+                list: campaigns.list,
+                get: campaigns.get
+            };
+
+            const subscriptionsProxy = {
+                list: subscriptions.list
+            };
+
+            const sandbox = {
+                console,
+                campaigns: campaignsProxy,
+                subscriptions: subscriptionsProxy,
+                inputs,
+
+                callback: (err, outputs) => {
+                    if (err) {
+                        log.error('reports', err.message || err);
+                        doneFail();
+                    }
+
+                    const hbsTmpl = handlebars.compile(reportTemplate.hbs);
+                    const reportText = hbsTmpl(outputs);
+
+                    process.stdout.write(reportText);
+                    doneSuccess();
+                }
+            };
+
+            const script = new vm.Script(reportTemplate.js);
+
+            try {
+                script.runInNewContext(sandbox, {displayErrors: true, timeout: 120000});
+            } catch (err) {
+                console.error(err);
+                doneFail();
+            }
+        });
+    });
+});
+